diff --git a/README.md b/README.md index cf1c8a7..c8ae9c6 100644 --- a/README.md +++ b/README.md @@ -50,11 +50,27 @@ Every namespace and every object we manage carries two labels, so "mine" and "th | Label | Values | |---|---| | `bulbashenko.com/layer` | `app`: our services (site, auth, data). `platform`: what we install for them (tunnel, certs, Postgres operator, ingress, GitOps UI, access). `system`: Kubernetes/k3s internals, not touched. | -| `bulbashenko.com/app` | `site`, `auth`, `data`, `tunnel`, `certs`, `postgres-operator`, `ingress`, `gitops`, `access`, `controllers`, `configs`, `kubernetes` | +| `bulbashenko.com/app` | `site`, `auth`, `data`, `tunnel`, `certs`, `postgres-operator`, `ingress`, `gitops`, `access`, `monitoring`, `controllers`, `configs`, `kubernetes` | Where they come from: Namespace manifests (explicit), `labels:` in each directory's `kustomization.yaml` (`includeSelectors: false`; pod templates included for apps), Flux `commonMetadata` for the auth repo, and `infrastructure/configs/system-namespaces.yaml` for namespaces Kubernetes or Flux create themselves (prune disabled). Examples: `kubectl get pods -A -l bulbashenko.com/layer=app`, in k9s `:pods` then `/-l bulbashenko.com/layer=app`. +kube-state-metrics exports both labels (`label_bulbashenko_com_layer`, `label_bulbashenko_com_app` on `kube_namespace_labels` +and `kube_pod_labels`); Grafana's dashboard "Cluster by layer" groups CPU, memory, restarts and readiness by them. + +## Monitoring + +victoria-metrics-k8s-stack in `monitoring`: the operator (`infrastructure/controllers/victoria-metrics-operator.yaml`, brings +the CRDs) and the stack (`infrastructure/configs/monitoring/`: VMSingle with 30 days retention, vmagent, vmalert, +Alertmanager, node-exporter, kube-state-metrics, Grafana). + +- Grafana: https://grafana.int.bulbashenko.com (mesh only, sign-in through Ory client `grafana`; only `infra-admins` get in, + as Grafana server admins). No PVC: dashboards and datasources come from git, upstream dashboards and alert rules from the + chart's sync Job. Break-glass: `/login?disableAutoLogin` with the local admin from the Secret `grafana`. +- Our dashboards: JSON files in `infrastructure/configs/monitoring/dashboards/` (folder "bulbashenko"). +- Scrape targets: platform components in `infrastructure/configs/monitoring/scrapes.yaml`; app targets next to the app + (`apps/production/data/scrapes.yaml` for CNPG and Garage, `apps/mail/scrape.yaml` for Stalwart). +- Alerts: Alertmanager has no receiver yet (Telegram: infra#75). ## Deploy PRs diff --git a/apps/edge/headscale/config.yaml b/apps/edge/headscale/config.yaml index f684f56..2c9c63d 100644 --- a/apps/edge/headscale/config.yaml +++ b/apps/edge/headscale/config.yaml @@ -74,6 +74,7 @@ dns: extra_records: - {name: "users.int.bulbashenko.com", type: "A", value: "100.64.0.5"} - {name: "flux.int.bulbashenko.com", type: "A", value: "100.64.0.5"} + - {name: "grafana.int.bulbashenko.com", type: "A", value: "100.64.0.5"} - {name: "mail.int.bulbashenko.com", type: "A", value: "100.64.0.5"} oidc: diff --git a/apps/mail/kustomization.yaml b/apps/mail/kustomization.yaml index b28dcb0..8e6db4f 100644 --- a/apps/mail/kustomization.yaml +++ b/apps/mail/kustomization.yaml @@ -10,3 +10,4 @@ labels: resources: - namespace.yaml - secret.sops.yaml + - scrape.yaml diff --git a/apps/mail/scrape.yaml b/apps/mail/scrape.yaml new file mode 100644 index 0000000..7d11fca --- /dev/null +++ b/apps/mail/scrape.yaml @@ -0,0 +1,16 @@ +# Stalwart's Prometheus endpoint. It is switched on in Stalwart's own settings (stored in its database, not in git): +# Settings > Telemetry > Metrics > Prometheus = Enabled, authUsername "vmagent", +# authSecret = EnvironmentVariable STALWART_METRICS_PASSWORD (from the Secret stalwart, loaded through envFrom). +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: stalwart +spec: + selector: + matchLabels: { app: stalwart } + podMetricsEndpoints: + - port: http + path: /metrics/prometheus + basicAuth: + username: { name: stalwart, key: STALWART_METRICS_USERNAME } + password: { name: stalwart, key: STALWART_METRICS_PASSWORD } diff --git a/apps/mail/secret.sops.yaml b/apps/mail/secret.sops.yaml index 9553a14..84ea5b1 100644 --- a/apps/mail/secret.sops.yaml +++ b/apps/mail/secret.sops.yaml @@ -11,6 +11,8 @@ stringData: #ENC[AES256_GCM,data:ok5FhWOKWGGbKruLAOQf4FvAshUgbAK+yXvcLlpGatbFfoAtbojbD7td+XnJP7vIW4aaDV2R6ESPiQB+e8O6Dee+SPk=,iv:cjCVE2pFflZzvTQ61B8JvEVrgEWcSKnQ4Nsr297Q9Ho=,tag:OFJdwKnS0I9HWgwNfMPSNw==,type:comment] RESTORE_S3_ACCESS_KEY_ID: ENC[AES256_GCM,data:fhI433WBw2wppqO6SyzmY281L+xN/18AwKw=,iv:2SJkqwT7kxkV1v8p0NH+RlfN1QSJN3SONM4ICyXeSYU=,tag:GUtZ6jFKEzQw0cAV3d6bwg==,type:str] RESTORE_S3_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:6pebhU4E0KDfXdXO7h70C31zSrlJLLdvVS9FVddtErCN/jz2r7gLF68GOX9vSIUIAKgUDyaydPzDKatnNHFvpw==,iv:k/hQWClBg2kmcakkwMu4Q3EP7Zn0aPpryyiksX4CecE=,tag:C6AImusZkzSSPBJo3xiY7w==,type:str] + STALWART_METRICS_USERNAME: ENC[AES256_GCM,data:zxHMkjSXDg==,iv:Cp80siBy7UBJyRv836ygul8ai+yjlu2CCYk01vAkL1k=,tag:fPz/SxinxL5r5kSYwezKzw==,type:str] + STALWART_METRICS_PASSWORD: ENC[AES256_GCM,data:zE9rgaeqCMARLY0WJvHFVWXKWhh6HDJxJzumaILZmKnf9pjfdETfVzQ/x9uZlV0E,iv:0tqMNYm5sf3k9SRBY6k78Su3nbR4T1QKNOcAxF6eL84=,tag:D4OQygDzdLmf55VNgSXZ4w==,type:str] sops: age: - enc: | @@ -32,6 +34,6 @@ sops: -----END AGE ENCRYPTED FILE----- recipient: age1c9jlhvq47psz20sl2wu7d2jwstjjylfx769tuumree8hdf5klf2sjluulk encrypted_regex: ^(data|stringData)$ - lastmodified: "2026-10-12T00:11:31Z" - mac: ENC[AES256_GCM,data:XIlZzPy3qRUDPcTbErOiKaxhTTatcKko8LCJuSe4MzbcqWP5yS+WrsM4KftC5GIffHybb8iCVH0J7WqNsS+7VNSvGtCHs5Pcyu5IcuDngHtjBqML9+YhqUEm8/tpMj9c3VjLR+TebV00XGK7gdZRbV5AO9iuEUpBAPqgHAZLol8=,iv:L2E9bYl+jD+RwaCcED5tcGnSFqNXmqHal3vITK3uBZo=,tag:uoIYkxDpetK70qd2iX8LEA==,type:str] + lastmodified: "2026-10-12T18:41:13Z" + mac: ENC[AES256_GCM,data:cMzWoxVzhZ+qRa9XWJpts7OnDOhUuWA2cIzHtRs4tHw7yX5ai/nJfnuJagxJXYYhvPEzNpGwL7EAWdP5WxFnjd5JF3006oQF8Bku+lO7rWSm2r65DZ/Kv1JZJUYcr3KlWWvsFfdn3snXLmwq0lW5O3SxSr2fTzuNlN85NApirtg=,iv:7e+VXzNTPAUM0bysrJshdaXzKXtTTcvJZs0tV7DiiUM=,tag:Gfwzl7an54VUddYCds1w3Q==,type:str] version: 3.13.3 diff --git a/apps/production/auth/secret.sops.yaml b/apps/production/auth/secret.sops.yaml index d4a5897..adc0782 100644 --- a/apps/production/auth/secret.sops.yaml +++ b/apps/production/auth/secret.sops.yaml @@ -21,6 +21,7 @@ stringData: OIDC_COOLIFY_CLIENT_SECRET: ENC[AES256_GCM,data:aNdiVYvGGRtWdnF7DORSB8Ll2n4hPXS+icQy25Tw6wl4XRf2wWAOry4apEquN/NcizI2bCtgTUVcbdlWQaVsD01LSzbu+HW/,iv:Hgg9xbdmJDWsB9jJwRM3ctWto/AnNStDA7ogRl09uB0=,tag:bM0WRuNBM6I2vzq/pd0oow==,type:str] OIDC_HEADSCALE_CLIENT_SECRET: ENC[AES256_GCM,data:1xzbyQuxaTW//H8KYpTw64CNnRaFCAInxtG8zBCXTViTJ00AOPaPutAo6Q+2i7Xn82HMLpfEZaZVCF1yKyIkC5Ncp/6/ZIWm,iv:gC9lvY+LpZyXF509PRLX+mae13IK/MHQC0tmPu/pcFA=,tag:jXgw7tfkJjKZHGhjilZXhA==,type:str] OIDC_FLUXWEB_CLIENT_SECRET: ENC[AES256_GCM,data:yvPF2uT+yaSrZ6e5W3OACb6tNUJDgXEIK10/48OaXvlqxorWg21atb9HX5m05JdOaLQa5za0GTj4lujLyoJG75YSjQO9zppr,iv:9imtlEWm2RatCkAdMVA5t2WWMDQ1zRC3gmLQvE3AFQg=,tag:OE/6eOFgNqjBgA7oAtCOCA==,type:str] + OIDC_GRAFANA_CLIENT_SECRET: ENC[AES256_GCM,data:9b9GcMGaDMQp6FPZgTV9zx6RcGyUfyHPBsonjHYQvEoSeSUkTXh8YPxdCjXyP1oodNj/TFxwxVFJ8VDJENuuSA==,iv:VhfZovpw+uC3CsG8gfK2EiGK3Nf3a/O+4Bfcm02BwxQ=,tag:9WBY8RyEdCSSw1w+QKwbWg==,type:str] sops: age: - enc: | @@ -42,6 +43,6 @@ sops: -----END AGE ENCRYPTED FILE----- recipient: age1c9jlhvq47psz20sl2wu7d2jwstjjylfx769tuumree8hdf5klf2sjluulk encrypted_regex: ^(data|stringData)$ - lastmodified: "2026-10-11T22:24:19Z" - mac: ENC[AES256_GCM,data:0Z+2jRxHAr5ZURLxDnyRgVzN0B1Fb6hrXR+ZQZ3G1fiitXbo9EfnNbYRM6bLxih3M9NXVcHnd6k/hreKTdNb+1CUaUnr6VDtyobrwn4lGfXvS5EI7qaGn6/aljq9P+j5vg0md05Qs85WMlVgwwNn5N7qXqX0hmHeJywD4J1uQC8=,iv:Z50bvORyjDhSaxLmSebmYcujugFddKLQFRZKlr7uGHA=,tag:UC+66+gbA6b4/zquNKjtnQ==,type:str] + lastmodified: "2026-10-12T18:41:13Z" + mac: ENC[AES256_GCM,data:LubQ4j7oGbk2s0EjPRxdgp8YL+EhZqyCJWHQIVKRiFpxqcm0Ihy9e39+rZMFNMeMoo0abz2VKrz9ncJNMnl8rN8fYOxpvmNb4MvccgyhraB2UjQuc2obBcJ9kTVUjiAJP+5TyaWqPpEHM8vGXVPzgIvAvz2UwzK6er2cXfjJJbs=,iv:cjZY5NHFP3UckXGSwC0TfqkjGVh1Pe7F85VHmxrruI8=,tag:FmHS7lgVeLq83LT86GrjsQ==,type:str] version: 3.13.3 diff --git a/apps/production/data/kustomization.yaml b/apps/production/data/kustomization.yaml index 39d923b..c9ed8bc 100644 --- a/apps/production/data/kustomization.yaml +++ b/apps/production/data/kustomization.yaml @@ -24,6 +24,7 @@ resources: - routes.yaml - backup.yaml - backup-initial.yaml + - scrapes.yaml configMapGenerator: - name: garage-ha-config files: diff --git a/apps/production/data/scrapes.yaml b/apps/production/data/scrapes.yaml new file mode 100644 index 0000000..5e48a52 --- /dev/null +++ b/apps/production/data/scrapes.yaml @@ -0,0 +1,23 @@ +# Metrics for the monitoring stack (infrastructure/configs/monitoring); vmagent picks up VM*Scrape objects in every namespace. +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: pg +spec: + selector: + matchLabels: { cnpg.io/cluster: pg, cnpg.io/podRole: instance } + podMetricsEndpoints: + - port: metrics +--- +# Garage serves /metrics on the admin port and wants the metrics token (metrics_token_file in garage-ha.toml). +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: garage +spec: + selector: + matchLabels: { app: garage-ha } + podMetricsEndpoints: + - port: admin + path: /metrics + bearerTokenSecret: { name: garage, key: GARAGE_METRICS_TOKEN } diff --git a/infrastructure/configs/kustomization.yaml b/infrastructure/configs/kustomization.yaml index ef6dbf0..ed19e88 100644 --- a/infrastructure/configs/kustomization.yaml +++ b/infrastructure/configs/kustomization.yaml @@ -6,4 +6,5 @@ resources: - traefik - cloudflared - flux-web + - monitoring - system-namespaces.yaml diff --git a/infrastructure/configs/monitoring/dashboards/cluster-by-layer.json b/infrastructure/configs/monitoring/dashboards/cluster-by-layer.json new file mode 100644 index 0000000..d0152d3 --- /dev/null +++ b/infrastructure/configs/monitoring/dashboards/cluster-by-layer.json @@ -0,0 +1,631 @@ +{ + "uid": "bulbashenko-layers", + "title": "Cluster by layer", + "tags": [ + "bulbashenko" + ], + "timezone": "browser", + "schemaVersion": 39, + "refresh": "1m", + "time": { + "from": "now-6h", + "to": "now" + }, + "editable": true, + "description": "Resource use and health grouped by the ownership labels bulbashenko.com/layer and bulbashenko.com/app (infra README: Labels).", + "templating": { + "list": [ + { + "name": "layer", + "label": "Layer", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "query": { + "query": "label_values(kube_namespace_labels, label_bulbashenko_com_layer)", + "refId": "q" + }, + "includeAll": true, + "multi": true, + "current": { + "text": "All", + "value": "$__all" + }, + "allValue": ".*", + "refresh": 2 + } + ] + }, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "Pods not ready", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "gridPos": { + "x": 0, + "y": 0, + "w": 6, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "expr": "sum(kube_pod_status_ready{condition=\"false\"} * on (namespace) group_left (label_bulbashenko_com_layer, label_bulbashenko_com_app) max by (namespace, label_bulbashenko_com_layer, label_bulbashenko_com_app) (kube_namespace_labels{label_bulbashenko_com_layer=~\"$layer\"})) or vector(0)", + "instant": true + } + ] + }, + { + "id": 2, + "type": "stat", + "title": "Container restarts (1h)", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "gridPos": { + "x": 6, + "y": 0, + "w": 6, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "expr": "sum(increase(kube_pod_container_status_restarts_total[1h]) * on (namespace) group_left (label_bulbashenko_com_layer, label_bulbashenko_com_app) max by (namespace, label_bulbashenko_com_layer, label_bulbashenko_com_app) (kube_namespace_labels{label_bulbashenko_com_layer=~\"$layer\"})) or vector(0)", + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Targets down", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "gridPos": { + "x": 12, + "y": 0, + "w": 6, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "expr": "count(up == 0) or vector(0)", + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Firing alerts", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "gridPos": { + "x": 18, + "y": 0, + "w": 6, + "h": 4 + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ] + }, + "colorMode": "background" + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "expr": "count(ALERTS{alertstate=\"firing\",alertname!=\"Watchdog\",alertname!=\"InfoInhibitor\"}) or vector(0)", + "instant": true + } + ] + }, + { + "id": 5, + "type": "timeseries", + "title": "CPU by layer", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "cores", + "custom": { + "stacking": { + "mode": "normal" + }, + "fillOpacity": 20 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "expr": "sum by (label_bulbashenko_com_layer) (rate(container_cpu_usage_seconds_total{container!=\"\",pod!=\"\"}[5m]) * on (namespace) group_left (label_bulbashenko_com_layer, label_bulbashenko_com_app) max by (namespace, label_bulbashenko_com_layer, label_bulbashenko_com_app) (kube_namespace_labels{label_bulbashenko_com_layer=~\"$layer\"}))", + "legendFormat": "{{label_bulbashenko_com_layer}}" + } + ] + }, + { + "id": 6, + "type": "timeseries", + "title": "Memory by layer", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "stacking": { + "mode": "normal" + }, + "fillOpacity": 20 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "expr": "sum by (label_bulbashenko_com_layer) (container_memory_working_set_bytes{container!=\"\",pod!=\"\"} * on (namespace) group_left (label_bulbashenko_com_layer, label_bulbashenko_com_app) max by (namespace, label_bulbashenko_com_layer, label_bulbashenko_com_app) (kube_namespace_labels{label_bulbashenko_com_layer=~\"$layer\"}))", + "legendFormat": "{{label_bulbashenko_com_layer}}" + } + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "CPU by app", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "gridPos": { + "x": 0, + "y": 12, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "cores", + "custom": { + "stacking": { + "mode": "normal" + }, + "fillOpacity": 20 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "expr": "sum by (label_bulbashenko_com_app) (rate(container_cpu_usage_seconds_total{container!=\"\",pod!=\"\"}[5m]) * on (namespace) group_left (label_bulbashenko_com_layer, label_bulbashenko_com_app) max by (namespace, label_bulbashenko_com_layer, label_bulbashenko_com_app) (kube_namespace_labels{label_bulbashenko_com_layer=~\"$layer\"}))", + "legendFormat": "{{label_bulbashenko_com_app}}" + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "Memory by app", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "gridPos": { + "x": 12, + "y": 12, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "stacking": { + "mode": "normal" + }, + "fillOpacity": 20 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "expr": "sum by (label_bulbashenko_com_app) (container_memory_working_set_bytes{container!=\"\",pod!=\"\"} * on (namespace) group_left (label_bulbashenko_com_layer, label_bulbashenko_com_app) max by (namespace, label_bulbashenko_com_layer, label_bulbashenko_com_app) (kube_namespace_labels{label_bulbashenko_com_layer=~\"$layer\"}))", + "legendFormat": "{{label_bulbashenko_com_app}}" + } + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "Restarts by app (1h)", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "gridPos": { + "x": 0, + "y": 20, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "custom": { + "stacking": { + "mode": "none" + }, + "fillOpacity": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "expr": "sum by (label_bulbashenko_com_app) (increase(kube_pod_container_status_restarts_total[1h]) * on (namespace) group_left (label_bulbashenko_com_layer, label_bulbashenko_com_app) max by (namespace, label_bulbashenko_com_layer, label_bulbashenko_com_app) (kube_namespace_labels{label_bulbashenko_com_layer=~\"$layer\"})) > 0", + "legendFormat": "{{label_bulbashenko_com_app}}" + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Node memory used", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "gridPos": { + "x": 12, + "y": 20, + "w": 12, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "custom": { + "stacking": { + "mode": "none" + }, + "fillOpacity": 0 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "expr": "(1 - node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * on (instance) group_left (nodename) node_uname_info", + "legendFormat": "{{nodename}}" + } + ] + }, + { + "id": 11, + "type": "table", + "title": "Scrape targets", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "gridPos": { + "x": 0, + "y": 28, + "w": 24, + "h": 8 + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "up", + "color": "green" + } + } + } + ], + "custom": { + "cellOptions": { + "type": "color-text" + } + } + }, + "overrides": [] + }, + "options": { + "sortBy": [ + { + "displayName": "Value", + "desc": false + } + ] + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "VictoriaMetrics" + }, + "expr": "min by (job, namespace) (up)", + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + } + } + } + ] + } + ] +} \ No newline at end of file diff --git a/infrastructure/configs/monitoring/kustomization.yaml b/infrastructure/configs/monitoring/kustomization.yaml new file mode 100644 index 0000000..5e378e3 --- /dev/null +++ b/infrastructure/configs/monitoring/kustomization.yaml @@ -0,0 +1,25 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# Ownership labels (see README: Labels). Selectors are left alone. +labels: + - pairs: + bulbashenko.com/layer: platform + bulbashenko.com/app: monitoring + includeSelectors: false +resources: + - secret.sops.yaml + - stack.yaml + - scrapes.yaml + - route.yaml +# Our own dashboards; the Grafana sidecar picks up ConfigMaps labelled grafana_dashboard=1 in this namespace. +configMapGenerator: + - name: grafana-dashboards-bulbashenko + namespace: monitoring + files: + - dashboards/cluster-by-layer.json + options: + disableNameSuffixHash: true + labels: + grafana_dashboard: "1" + annotations: + grafana_folder: bulbashenko diff --git a/infrastructure/configs/monitoring/route.yaml b/infrastructure/configs/monitoring/route.yaml new file mode 100644 index 0000000..eb5aba1 --- /dev/null +++ b/infrastructure/configs/monitoring/route.yaml @@ -0,0 +1,15 @@ +# Grafana on the mesh only; it does its own OIDC sign-in (client grafana in the auth repo). +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: grafana + namespace: monitoring +spec: + entryPoints: [websecure] + routes: + - match: Host(`grafana.int.bulbashenko.com`) + middlewares: + - { name: mesh-only, namespace: kube-system } + services: + - { name: grafana, port: 80 } + tls: {} diff --git a/infrastructure/configs/monitoring/scrapes.yaml b/infrastructure/configs/monitoring/scrapes.yaml new file mode 100644 index 0000000..5901cd1 --- /dev/null +++ b/infrastructure/configs/monitoring/scrapes.yaml @@ -0,0 +1,74 @@ +# Platform components the k8s-stack chart does not scrape on its own. App targets (CNPG, Garage, Stalwart) are +# scraped from objects next to the apps (apps/production/data, apps/mail). +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: traefik + namespace: monitoring +spec: + namespaceSelector: { matchNames: [kube-system] } + selector: + matchLabels: { app.kubernetes.io/name: traefik } + podMetricsEndpoints: + - port: metrics +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: cloudflared + namespace: monitoring +spec: + namespaceSelector: { matchNames: [cloudflared] } + selector: + matchLabels: { app: cloudflared } + podMetricsEndpoints: + - port: metrics +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: cert-manager + namespace: monitoring +spec: + namespaceSelector: { matchNames: [cert-manager] } + selector: + matchLabels: { app.kubernetes.io/instance: cert-manager } + podMetricsEndpoints: + - port: http-metrics +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: cnpg-operator + namespace: monitoring +spec: + namespaceSelector: { matchNames: [cnpg-system] } + selector: + matchLabels: { app.kubernetes.io/name: cloudnative-pg } + podMetricsEndpoints: + - port: metrics +--- +# Flux controllers (http-prom) and Flux Operator (http-metrics); flux-system's allow-scraping NetworkPolicy +# admits scrapers from any namespace. +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMPodScrape +metadata: + name: flux + namespace: monitoring +spec: + namespaceSelector: { matchNames: [flux-system] } + selector: {} + podMetricsEndpoints: + - port: http-prom + - port: http-metrics +--- +apiVersion: operator.victoriametrics.com/v1beta1 +kind: VMServiceScrape +metadata: + name: vm-operator + namespace: monitoring +spec: + selector: + matchLabels: { app.kubernetes.io/name: victoria-metrics-operator } + endpoints: + - port: http diff --git a/infrastructure/configs/monitoring/secret.sops.yaml b/infrastructure/configs/monitoring/secret.sops.yaml new file mode 100644 index 0000000..13a7c53 --- /dev/null +++ b/infrastructure/configs/monitoring/secret.sops.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Secret +metadata: + name: grafana + namespace: monitoring +type: Opaque +stringData: + admin-user: ENC[AES256_GCM,data:fMJ90lg=,iv:vOqLwxnxaBr5grmjgVfIkHOA76r5ECCmn+dxwxJ8v3Q=,tag:csiDqkONejbpyn+c54WC8g==,type:str] + admin-password: ENC[AES256_GCM,data:029sxX8MPFsDCrkHHCYd2e7vZv6beBmXRTDxhtKKa0M=,iv:SJd8Vz1hrMqzxcZ9/kH9q5xNOtUDqmTLltIaHtYpAYw=,tag:gH7g7ra9E2A7XQNTROmSrw==,type:str] + oidc-client-secret: ENC[AES256_GCM,data:vVz8nQRKEdSeVKUvPZjbH3Bgi0TSetfDtoe8LZVNZ2+MN1yIVA2cGjDaLphrODJNozmJOg0Xpux6vUDC5PvKnQ==,iv:ezkCA3ePrCGPhNh9CNCXoageI8X3QIFV5+QXFWvhwOg=,tag:0uP7gekPgeYsvvbiyF6sSA==,type:str] +sops: + age: + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6Y3NTeFl0OFlnMzZBZ090 + MVQ3L2dkQkFsQlhyaGthMGIwbTB5S0RQblRnClZsREVNQlgzSk1OaW1MT0YzYUc0 + ZWNNQ0c0cDFuZWw5Z0FyK0FWSkhLY2sKLS0tIGtpc2Z3TXhnSzFKT3N2WXFCcFlM + RjBDL0llVjVqL1U4d2hRbmNaMXM3b1EKysHhszi9yMrIo1WtGuy63jr7y9xV8qqG + RRi6RYKr//cal83VUAWNARfI6C/aHadj6zj4Urli/SyqEYfao+lCWg== + -----END AGE ENCRYPTED FILE----- + recipient: age12qgx04vs6thq39ym6jl5wlgcymjzrdln7w6wyck7px28k4p8fedq09dq5g + - enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1SGxxVFlyQUEwQUZYVW5Z + WG40N1JNbHZqS25WMWs1d29XL3ZNUWkyQXd3CmtNemc0KzdXZU0vWWFUaC9peXhs + M0lZenVhaURSQ2piWWhPWEpRWEpWdm8KLS0tIGNvdGM2S2N2YURsT3hCMmtreWlO + cEYzVjBDWnQ0dkM1TWZWV2Z6OXJ3VTgK+KLM8y7HHgK5yKn4AyeN/EFxgsTJ080N + W7zSuAici5nXYACC89xByEJlLqAOC1mloKSdV0CqhmyQMqH4ZQ1S2g== + -----END AGE ENCRYPTED FILE----- + recipient: age1c9jlhvq47psz20sl2wu7d2jwstjjylfx769tuumree8hdf5klf2sjluulk + encrypted_regex: ^(data|stringData)$ + lastmodified: "2026-10-12T18:41:13Z" + mac: ENC[AES256_GCM,data:s3vubPYyrpMSE3/PgS2K378i8/HEABBfmDQyw87yTeLDC1GFcXmNB2RiOXKgWk2+t11P0KctgmOA8L9V3j+HnkZ1wj9IUy3gFBs51Tdhszq6dNR7AGzidCTm1KHFqCzGj+UFzqfbUTSaa/GREWqsHYuinebp0QP1blZ/GcawaMY=,iv:eI/JTVaQV6OdyFSETuKLbs02vgQBLCOeuD9jwHCnhv8=,tag:CdQ2diMLipsEYTT+opltxA==,type:str] + version: 3.13.3 diff --git a/infrastructure/configs/monitoring/stack.yaml b/infrastructure/configs/monitoring/stack.yaml new file mode 100644 index 0000000..f613c8e --- /dev/null +++ b/infrastructure/configs/monitoring/stack.yaml @@ -0,0 +1,182 @@ +# victoria-metrics-k8s-stack: VMSingle (storage), vmagent (scraping), vmalert + Alertmanager (rules; no receiver yet, +# Telegram is infra#75), node-exporter, kube-state-metrics and Grafana. The operator itself is a separate release in +# infrastructure/controllers. Dashboards and rules are fetched from upstream by the chart's sync Job at deploy time. +apiVersion: source.toolkit.fluxcd.io/v1 +kind: OCIRepository +metadata: + name: victoria-metrics-k8s-stack + namespace: monitoring +spec: + interval: 12h + url: oci://ghcr.io/victoriametrics/helm-charts/victoria-metrics-k8s-stack + ref: + tag: 0.96.0 + layerSelector: + mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip + operation: copy +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: vm + namespace: monitoring +spec: + interval: 1h + timeout: 10m + chartRef: + kind: OCIRepository + name: victoria-metrics-k8s-stack + values: + fullnameOverride: vm + global: + clusterLabel: cluster + # Links in alerts (vmalert external.url) point at Grafana. + external: + grafana: + host: https://grafana.int.bulbashenko.com + + # Installed by infrastructure/controllers/victoria-metrics-operator.yaml. + victoria-metrics-operator: + enabled: false + + defaultDashboards: + sources: + # Enabled by the operator subchart's switch by default; the operator runs, just not from this chart. + victoriametrics-operator: + enabled: true + defaultRules: + groups: + # k3s runs no etcd (sqlite datastore). + etcd: + enabled: false + sources: + vmoperator: + enabled: true + + vmsingle: + spec: + # Disk on every node is 40 GB; ~3 GB expected for 30 days at a 30 s scrape interval. + retentionPeriod: 30d + storage: + resources: + requests: + storage: 10Gi + resources: + requests: { cpu: 50m, memory: 256Mi } + limits: { memory: 768Mi } + + vmagent: + spec: + scrapeInterval: 30s + externalLabels: + cluster: hel1 + resources: + requests: { cpu: 30m, memory: 128Mi } + limits: { memory: 384Mi } + + vmalert: + spec: + evaluationInterval: 30s + resources: + requests: { cpu: 10m, memory: 48Mi } + + alertmanager: + spec: + externalURL: "" + resources: + requests: { cpu: 10m, memory: 32Mi } + + # k3s runs the controller manager, scheduler and kube-proxy inside the k3s process, bound to localhost: + # nothing to scrape from the pod network. + kubeControllerManager: + enabled: false + kubeScheduler: + enabled: false + kubeProxy: + enabled: false + kubeEtcd: + enabled: false + + prometheus-node-exporter: + resources: + requests: { cpu: 10m, memory: 24Mi } + + kube-state-metrics: + # Ownership labels (README: Labels) as label_bulbashenko_com_* on kube_namespace_labels / kube_pod_labels, + # for the "Cluster by layer" dashboard. + metricLabelsAllowlist: + - namespaces=[bulbashenko.com/layer,bulbashenko.com/app] + - pods=[bulbashenko.com/layer,bulbashenko.com/app] + resources: + requests: { cpu: 10m, memory: 48Mi } + + defaultDatasources: + victoriametrics: + # Plain Prometheus datasource only: the VictoriaMetrics plugin would be downloaded at every pod start. + datasources: + - name: VictoriaMetrics + type: prometheus + access: proxy + uid: VictoriaMetrics + isDefault: true + + grafana: + fullnameOverride: grafana + testFramework: + enabled: false + # No PVC: dashboards and datasources are provisioned from git, people sign in through Ory. + persistence: + enabled: false + resources: + requests: { cpu: 20m, memory: 128Mi } + limits: { memory: 384Mi } + # Local admin: used by the dashboard/datasource sidecars (basic auth against the reload API) and as break-glass + # sign-in at /login?disableAutoLogin if Ory is down; the login form is hidden otherwise. + admin: + existingSecret: grafana + userKey: admin-user + passwordKey: admin-password + envValueFrom: + GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: + secretKeyRef: { name: grafana, key: oidc-client-secret } + sidecar: + dashboards: + # Dashboards from this directory land in their own folder (see grafana-dashboards in kustomization.yaml). + folderAnnotation: grafana_folder + provider: + foldersFromFilesStructure: true + grafana.ini: + server: + root_url: https://grafana.int.bulbashenko.com + analytics: + reporting_enabled: false + check_for_updates: false + check_for_plugin_updates: false + news: + news_feed_enabled: false + users: + allow_sign_up: false + auth: + disable_login_form: true + signout_redirect_url: https://auth.bulbashenko.com/oauth2/sessions/logout + # Ory Hydra (client `grafana` in bulbashenko-com/auth). Only infra-admins pass consent, and they get + # Grafana server admin. + auth.generic_oauth: + enabled: true + name: bulbashenko SSO + auto_login: true + allow_sign_up: true + client_id: grafana + scopes: openid profile email groups + auth_url: https://auth.bulbashenko.com/oauth2/auth + token_url: https://auth.bulbashenko.com/oauth2/token + api_url: https://auth.bulbashenko.com/userinfo + auth_style: InHeader + use_pkce: true + login_attribute_path: preferred_username + email_attribute_path: email + name_attribute_path: name + groups_attribute_path: groups + role_attribute_path: contains(groups[*], 'infra-admins') && 'GrafanaAdmin' || 'Viewer' + role_attribute_strict: true + allow_assign_grafana_admin: true diff --git a/infrastructure/controllers/kustomization.yaml b/infrastructure/controllers/kustomization.yaml index ce8900c..44f895c 100644 --- a/infrastructure/controllers/kustomization.yaml +++ b/infrastructure/controllers/kustomization.yaml @@ -8,3 +8,4 @@ resources: - cert-manager.yaml - cloudnative-pg.yaml - barman-cloud.yaml + - victoria-metrics-operator.yaml diff --git a/infrastructure/controllers/victoria-metrics-operator.yaml b/infrastructure/controllers/victoria-metrics-operator.yaml new file mode 100644 index 0000000..8eed992 --- /dev/null +++ b/infrastructure/controllers/victoria-metrics-operator.yaml @@ -0,0 +1,52 @@ +# VictoriaMetrics Operator: brings the VM CRDs (VMSingle, VMAgent, VM*Scrape, VMRule, ...). Lives here so the CRDs +# exist before infrastructure/configs/monitoring (the k8s-stack release and the scrape objects) is applied. +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring + labels: + bulbashenko.com/layer: platform + bulbashenko.com/app: monitoring +--- +apiVersion: source.toolkit.fluxcd.io/v1 +kind: OCIRepository +metadata: + name: victoria-metrics-operator + namespace: monitoring +spec: + interval: 12h + url: oci://ghcr.io/victoriametrics/helm-charts/victoria-metrics-operator + ref: + tag: 0.68.1 + layerSelector: + mediaType: application/vnd.cncf.helm.chart.content.v1.tar+gzip + operation: copy +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: victoria-metrics-operator + namespace: monitoring +spec: + interval: 1h + # The validating webhook's certificate comes from cert-manager. + dependsOn: + - name: cert-manager + namespace: cert-manager + chartRef: + kind: OCIRepository + name: victoria-metrics-operator + install: + crds: CreateReplace + upgrade: + crds: CreateReplace + values: + fullnameOverride: vm-operator + # CRDs from the chart's crds/ directory, so Helm installs them before anything else and Flux replaces them on upgrade. + crds: + plain: true + admissionWebhooks: + certManager: + enabled: true + resources: + requests: { cpu: 20m, memory: 64Mi }