bulbashenko · Oct 12, 2026 · Oct 12, 2026
diff --git a/README.md b/README.md
@@ -50,11 +50,27 @@ Every namespace and every object we manage carries two labels, so "mine" and "th
 | Label | Values |
 |---|---|
 | `bulbashenko.com/layer` | `app`: our services (site, auth, data). `platform`: what we install for them (tunnel, certs, Postgres operator, ingress, GitOps UI, access). `system`: Kubernetes/k3s internals, not touched. |
-| `bulbashenko.com/app` | `site`, `auth`, `data`, `tunnel`, `certs`, `postgres-operator`, `ingress`, `gitops`, `access`, `controllers`, `configs`, `kubernetes` |
+| `bulbashenko.com/app` | `site`, `auth`, `data`, `tunnel`, `certs`, `postgres-operator`, `ingress`, `gitops`, `access`, `monitoring`, `controllers`, `configs`, `kubernetes` |
 
 Where they come from: Namespace manifests (explicit), `labels:` in each directory's `kustomization.yaml` (`includeSelectors: false`; pod templates included for apps), Flux `commonMetadata` for the auth repo, and `infrastructure/configs/system-namespaces.yaml` for namespaces Kubernetes or Flux create themselves (prune disabled).
 
 Examples: `kubectl get pods -A -l bulbashenko.com/layer=app`, in k9s `:pods` then `/-l bulbashenko.com/layer=app`.
+kube-state-metrics exports both labels (`label_bulbashenko_com_layer`, `label_bulbashenko_com_app` on `kube_namespace_labels`
+and `kube_pod_labels`); Grafana's dashboard "Cluster by layer" groups CPU, memory, restarts and readiness by them.
+
+## Monitoring
+
+victoria-metrics-k8s-stack in `monitoring`: the operator (`infrastructure/controllers/victoria-metrics-operator.yaml`, brings
+the CRDs) and the stack (`infrastructure/configs/monitoring/`: VMSingle with 30 days retention, vmagent, vmalert,
+Alertmanager, node-exporter, kube-state-metrics, Grafana).
+
+- Grafana: https://grafana.int.bulbashenko.com (mesh only, sign-in through Ory client `grafana`; only `infra-admins` get in,
+  as Grafana server admins). No PVC: dashboards and datasources come from git, upstream dashboards and alert rules from the
+  chart's sync Job. Break-glass: `/login?disableAutoLogin` with the local admin from the Secret `grafana`.
+- Our dashboards: JSON files in `infrastructure/configs/monitoring/dashboards/` (folder "bulbashenko").
+- Scrape targets: platform components in `infrastructure/configs/monitoring/scrapes.yaml`; app targets next to the app
+  (`apps/production/data/scrapes.yaml` for CNPG and Garage, `apps/mail/scrape.yaml` for Stalwart).
+- Alerts: Alertmanager has no receiver yet (Telegram: infra#75).
 
 ## Deploy PRs
 

diff --git a/apps/edge/headscale/config.yaml b/apps/edge/headscale/config.yaml
@@ -74,6 +74,7 @@ dns:
   extra_records:
     - {name: "users.int.bulbashenko.com", type: "A", value: "100.64.0.5"}
     - {name: "flux.int.bulbashenko.com", type: "A", value: "100.64.0.5"}
+    - {name: "grafana.int.bulbashenko.com", type: "A", value: "100.64.0.5"}
     - {name: "mail.int.bulbashenko.com", type: "A", value: "100.64.0.5"}
 
 oidc:

diff --git a/apps/mail/kustomization.yaml b/apps/mail/kustomization.yaml
@@ -10,3 +10,4 @@ labels:
 resources:
   - namespace.yaml
   - secret.sops.yaml
+  - scrape.yaml
diff --git a/apps/mail/scrape.yaml b/apps/mail/scrape.yaml
@@ -0,0 +1,16 @@
+# Stalwart's Prometheus endpoint. It is switched on in Stalwart's own settings (stored in its database, not in git):
+# Settings > Telemetry > Metrics > Prometheus = Enabled, authUsername "vmagent",
+# authSecret = EnvironmentVariable STALWART_METRICS_PASSWORD (from the Secret stalwart, loaded through envFrom).
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: stalwart
+spec:
+  selector:
+    matchLabels: { app: stalwart }
+  podMetricsEndpoints:
+    - port: http
+      path: /metrics/prometheus
+      basicAuth:
+        username: { name: stalwart, key: STALWART_METRICS_USERNAME }
+        password: { name: stalwart, key: STALWART_METRICS_PASSWORD }
diff --git a/apps/mail/secret.sops.yaml b/apps/mail/secret.sops.yaml
@@ -11,6 +11,8 @@ stringData:
     #ENC[AES256_GCM,data:ok5FhWOKWGGbKruLAOQf4FvAshUgbAK+yXvcLlpGatbFfoAtbojbD7td+XnJP7vIW4aaDV2R6ESPiQB+e8O6Dee+SPk=,iv:cjCVE2pFflZzvTQ61B8JvEVrgEWcSKnQ4Nsr297Q9Ho=,tag:OFJdwKnS0I9HWgwNfMPSNw==,type:comment]
     RESTORE_S3_ACCESS_KEY_ID: ENC[AES256_GCM,data:fhI433WBw2wppqO6SyzmY281L+xN/18AwKw=,iv:2SJkqwT7kxkV1v8p0NH+RlfN1QSJN3SONM4ICyXeSYU=,tag:GUtZ6jFKEzQw0cAV3d6bwg==,type:str]
     RESTORE_S3_SECRET_ACCESS_KEY: ENC[AES256_GCM,data:6pebhU4E0KDfXdXO7h70C31zSrlJLLdvVS9FVddtErCN/jz2r7gLF68GOX9vSIUIAKgUDyaydPzDKatnNHFvpw==,iv:k/hQWClBg2kmcakkwMu4Q3EP7Zn0aPpryyiksX4CecE=,tag:C6AImusZkzSSPBJo3xiY7w==,type:str]
+    STALWART_METRICS_USERNAME: ENC[AES256_GCM,data:zxHMkjSXDg==,iv:Cp80siBy7UBJyRv836ygul8ai+yjlu2CCYk01vAkL1k=,tag:fPz/SxinxL5r5kSYwezKzw==,type:str]
+    STALWART_METRICS_PASSWORD: ENC[AES256_GCM,data:zE9rgaeqCMARLY0WJvHFVWXKWhh6HDJxJzumaILZmKnf9pjfdETfVzQ/x9uZlV0E,iv:0tqMNYm5sf3k9SRBY6k78Su3nbR4T1QKNOcAxF6eL84=,tag:D4OQygDzdLmf55VNgSXZ4w==,type:str]
 sops:
     age:
         - enc: |
@@ -32,6 +34,6 @@ sops:
             -----END AGE ENCRYPTED FILE-----
           recipient: age1c9jlhvq47psz20sl2wu7d2jwstjjylfx769tuumree8hdf5klf2sjluulk
     encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-12T00:11:31Z"
-    mac: ENC[AES256_GCM,data:XIlZzPy3qRUDPcTbErOiKaxhTTatcKko8LCJuSe4MzbcqWP5yS+WrsM4KftC5GIffHybb8iCVH0J7WqNsS+7VNSvGtCHs5Pcyu5IcuDngHtjBqML9+YhqUEm8/tpMj9c3VjLR+TebV00XGK7gdZRbV5AO9iuEUpBAPqgHAZLol8=,iv:L2E9bYl+jD+RwaCcED5tcGnSFqNXmqHal3vITK3uBZo=,tag:uoIYkxDpetK70qd2iX8LEA==,type:str]
+    lastmodified: "2026-10-12T18:41:13Z"
+    mac: ENC[AES256_GCM,data:cMzWoxVzhZ+qRa9XWJpts7OnDOhUuWA2cIzHtRs4tHw7yX5ai/nJfnuJagxJXYYhvPEzNpGwL7EAWdP5WxFnjd5JF3006oQF8Bku+lO7rWSm2r65DZ/Kv1JZJUYcr3KlWWvsFfdn3snXLmwq0lW5O3SxSr2fTzuNlN85NApirtg=,iv:7e+VXzNTPAUM0bysrJshdaXzKXtTTcvJZs0tV7DiiUM=,tag:Gfwzl7an54VUddYCds1w3Q==,type:str]
     version: 3.13.3
diff --git a/apps/production/auth/secret.sops.yaml b/apps/production/auth/secret.sops.yaml
@@ -21,6 +21,7 @@ stringData:
     OIDC_COOLIFY_CLIENT_SECRET: ENC[AES256_GCM,data:aNdiVYvGGRtWdnF7DORSB8Ll2n4hPXS+icQy25Tw6wl4XRf2wWAOry4apEquN/NcizI2bCtgTUVcbdlWQaVsD01LSzbu+HW/,iv:Hgg9xbdmJDWsB9jJwRM3ctWto/AnNStDA7ogRl09uB0=,tag:bM0WRuNBM6I2vzq/pd0oow==,type:str]
     OIDC_HEADSCALE_CLIENT_SECRET: ENC[AES256_GCM,data:1xzbyQuxaTW//H8KYpTw64CNnRaFCAInxtG8zBCXTViTJ00AOPaPutAo6Q+2i7Xn82HMLpfEZaZVCF1yKyIkC5Ncp/6/ZIWm,iv:gC9lvY+LpZyXF509PRLX+mae13IK/MHQC0tmPu/pcFA=,tag:jXgw7tfkJjKZHGhjilZXhA==,type:str]
     OIDC_FLUXWEB_CLIENT_SECRET: ENC[AES256_GCM,data:yvPF2uT+yaSrZ6e5W3OACb6tNUJDgXEIK10/48OaXvlqxorWg21atb9HX5m05JdOaLQa5za0GTj4lujLyoJG75YSjQO9zppr,iv:9imtlEWm2RatCkAdMVA5t2WWMDQ1zRC3gmLQvE3AFQg=,tag:OE/6eOFgNqjBgA7oAtCOCA==,type:str]
+    OIDC_GRAFANA_CLIENT_SECRET: ENC[AES256_GCM,data:9b9GcMGaDMQp6FPZgTV9zx6RcGyUfyHPBsonjHYQvEoSeSUkTXh8YPxdCjXyP1oodNj/TFxwxVFJ8VDJENuuSA==,iv:VhfZovpw+uC3CsG8gfK2EiGK3Nf3a/O+4Bfcm02BwxQ=,tag:9WBY8RyEdCSSw1w+QKwbWg==,type:str]
 sops:
     age:
         - enc: |
@@ -42,6 +43,6 @@ sops:
             -----END AGE ENCRYPTED FILE-----
           recipient: age1c9jlhvq47psz20sl2wu7d2jwstjjylfx769tuumree8hdf5klf2sjluulk
     encrypted_regex: ^(data|stringData)$
-    lastmodified: "2026-10-11T22:24:19Z"
-    mac: ENC[AES256_GCM,data:0Z+2jRxHAr5ZURLxDnyRgVzN0B1Fb6hrXR+ZQZ3G1fiitXbo9EfnNbYRM6bLxih3M9NXVcHnd6k/hreKTdNb+1CUaUnr6VDtyobrwn4lGfXvS5EI7qaGn6/aljq9P+j5vg0md05Qs85WMlVgwwNn5N7qXqX0hmHeJywD4J1uQC8=,iv:Z50bvORyjDhSaxLmSebmYcujugFddKLQFRZKlr7uGHA=,tag:UC+66+gbA6b4/zquNKjtnQ==,type:str]
+    lastmodified: "2026-10-12T18:41:13Z"
+    mac: ENC[AES256_GCM,data:LubQ4j7oGbk2s0EjPRxdgp8YL+EhZqyCJWHQIVKRiFpxqcm0Ihy9e39+rZMFNMeMoo0abz2VKrz9ncJNMnl8rN8fYOxpvmNb4MvccgyhraB2UjQuc2obBcJ9kTVUjiAJP+5TyaWqPpEHM8vGXVPzgIvAvz2UwzK6er2cXfjJJbs=,iv:cjZY5NHFP3UckXGSwC0TfqkjGVh1Pe7F85VHmxrruI8=,tag:FmHS7lgVeLq83LT86GrjsQ==,type:str]
     version: 3.13.3
diff --git a/apps/production/data/kustomization.yaml b/apps/production/data/kustomization.yaml
@@ -24,6 +24,7 @@ resources:
   - routes.yaml
   - backup.yaml
   - backup-initial.yaml
+  - scrapes.yaml
 configMapGenerator:
   - name: garage-ha-config
     files:

diff --git a/apps/production/data/scrapes.yaml b/apps/production/data/scrapes.yaml
@@ -0,0 +1,23 @@
+# Metrics for the monitoring stack (infrastructure/configs/monitoring); vmagent picks up VM*Scrape objects in every namespace.
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: pg
+spec:
+  selector:
+    matchLabels: { cnpg.io/cluster: pg, cnpg.io/podRole: instance }
+  podMetricsEndpoints:
+    - port: metrics
+---
+# Garage serves /metrics on the admin port and wants the metrics token (metrics_token_file in garage-ha.toml).
+apiVersion: operator.victoriametrics.com/v1beta1
+kind: VMPodScrape
+metadata:
+  name: garage
+spec:
+  selector:
+    matchLabels: { app: garage-ha }
+  podMetricsEndpoints:
+    - port: admin
+      path: /metrics
+      bearerTokenSecret: { name: garage, key: GARAGE_METRICS_TOKEN }
diff --git a/infrastructure/configs/kustomization.yaml b/infrastructure/configs/kustomization.yaml
@@ -6,4 +6,5 @@ resources:
   - traefik
   - cloudflared
   - flux-web
+  - monitoring
   - system-namespaces.yaml