From 3734c39471fb0815a58e8e440c7df8046f863f6d Mon Sep 17 00:00:00 2001 From: Aleksandr Albekov Date: Mon, 12 Oct 2026 21:48:09 +0200 Subject: [PATCH 1/2] Add the GitHub App credentials for Grafana's GitHub datasource App bulbashenko-com-grafana (read-only: Actions, Pull requests), installed on all org repos. App ID, installation ID and private key go into the Secret grafana; the secret revision bump restarts Grafana so the env vars are picked up. Refs #74 Co-Authored-By: Claude Opus 5.5 --- infrastructure/configs/monitoring/secret.sops.yaml | 7 +++++-- infrastructure/configs/monitoring/stack.yaml | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/infrastructure/configs/monitoring/secret.sops.yaml b/infrastructure/configs/monitoring/secret.sops.yaml index fe64d6b..9331511 100644 --- a/infrastructure/configs/monitoring/secret.sops.yaml +++ b/infrastructure/configs/monitoring/secret.sops.yaml @@ -8,6 +8,9 @@ stringData: admin-user: ENC[AES256_GCM,data:DIZEGbpSfxHB8Qs=,iv:JmBr2VvPEgudEK5xXMCZVbo5QKdnHFAStI1mtxxv/mQ=,tag:Vt3IwD1BgJuQq7jPkB+CIQ==,type:str] admin-password: ENC[AES256_GCM,data:029sxX8MPFsDCrkHHCYd2e7vZv6beBmXRTDxhtKKa0M=,iv:SJd8Vz1hrMqzxcZ9/kH9q5xNOtUDqmTLltIaHtYpAYw=,tag:gH7g7ra9E2A7XQNTROmSrw==,type:str] oidc-client-secret: ENC[AES256_GCM,data:vVz8nQRKEdSeVKUvPZjbH3Bgi0TSetfDtoe8LZVNZ2+MN1yIVA2cGjDaLphrODJNozmJOg0Xpux6vUDC5PvKnQ==,iv:ezkCA3ePrCGPhNh9CNCXoageI8X3QIFV5+QXFWvhwOg=,tag:0uP7gekPgeYsvvbiyF6sSA==,type:str] + github-app-id: ENC[AES256_GCM,data:dV0+LIJEuA==,iv:5KJh7oNWYXU9WC4s2stWRZ3v5547WYlNCF3uNI39kqc=,tag:/8DfgCqXZxdDnwJwfm5rhg==,type:str] + github-app-installation-id: ENC[AES256_GCM,data:7FDXDfUH8MW3,iv:kvJ35LolM5MgoDgJqsyw1+N2oTKZGn9AU2G2cYS+rsY=,tag:GQS//aMGWo3tjdtwfsBbww==,type:str] + github-app-private-key: ENC[AES256_GCM,data:/cqK4krS1//pfDdSYzu0rLz/qDrVkcdKfyFXQ/EE1HJLElIhOaUUC9zpD4Dkb+F1U9chuxj+m1n3BdBJdTKUACtPM/6Sz9IZpsGQJIyTFODbIj2ezDb80mO70/jlYTIz8oc+0QAGdwRMoH3xVO5v/7InkNytpbTSEH9qjMkTjVLMRPbMQ4huvquRr1eD20z2JLX4W0zvxzihMOSbCIiV9P5Hf5oWYxEgHC1vLgMg7GBXKXp6taQDAfgrWIS5nG2AD5Pie6jZRWhIkIgYXBvDbWq10Kp/g5u12dIHFviHiruSqJVdkuIpWPWI1xF7sBoIIwsIb//wG4TkRfCHNMqj65+c7W2woK71xTdhQylb9tybzWqqyoXE53b1JFPXmy1+Qn0AAOZIt4/iz8xsqy9pi9Uo6mdtW5cApaj+LpCuEShg5/dfHhhZlEbqGWVmD8SK2ZPWqjrn1SoWU1izRn84Ns7uIHRAmFhRjoj6eFxJvgX2pBWZkoTwvQFQeYrHLiOoAwBNla1G45AsGk8kOXD3zyXSUz9QTVnTBccq362jeykKh4PcOVKr3aglfM7zNHHNjoqd8CM6IOsfaBmXzSirtanNj/dtDtrbpoVRT7z3vM+S0PSdBHeciEZF/hPWSrdDaLDV86PJZhkMvYu0kxl6ZaRZ9didt/E91KgljZM6NbmCptq1dfgcxR44aSiDt8nRkJ3GKT+hNXX1G2dSMN48WYNgHrue3IGz2mTyTs4lLQ8HDv5SkcUjRFhJb2tN67ucZ6DC7n/42RXaUn8XYeQMjeZAlULKAj50BwR5CKWWEoMAv2EDEVm2UmYPcXGMAuzd7BHWktMsI51kKrAeS5LNARvLs6TSe8IM0t5G7JlOgUbBZfnfBE6Hb6WqD23iPAX1/GzKGK077mHOY/oMwUq1VYjrwnSZxBUClXqTYquhaO4rF1i9ibIrqliIdsVpXYInX8lE7oghHGdei5cw5tEyCnva9nxGCRUZafZxOjtspqSd2CyzFDcF8y1ljJzYBb+1fIAEi+5zjOg15OG/V21Hl1LfnRmepLgj5KbRAYWG1DqBs/F4+UawNLMUq3uB87cUaUaLo7WLmvQNZhlnewqF3gAFb0L5WcTlawbqnFvcxF20PsrsFqJlRaDK3BAGfQqQ8FPrEaOI+iZa/DGh139wU4cy7msHpHij6yIgyakTeUwyknGvibIknAKlsciSWx5xI6QKHTQcfUM8yCv9utDmxDEKUv9AbTPk4Dko5enTI2wZ3sjrn4+zDCIt58m8uut7FqVCSrvedPruxqE8bAjeeTNATt/Ty0foQtik0gAL4dSCSmnLszpzT6IQvlp6EjQie2k4eXvaaom8CJiH9AshrhCqvz4T815/4Tr9TiqjPouUk8tVLIAqJJewVlIAHDLTfs2ILQPY9LfPBsYBeTfq8d0igkeUgBde+2HZxduAUbkOFOzQNXl/VYhYQFJyYpterzB+mdhecneslqp3rMnmkZ88vcO8/eFdIZRwFmcUOoETE1g0J8KbZjoZSErcYORxESkKXJg5ZTLrr/rJfdgEyEZXWTE4poRRkyQTxHDA4r5mQCqQBDNp3ZJ3xeTFzviI5s1eX4frBsWjBp61QlMImQ/EAInjQ2cFmPXbfaezxdWQ7k/Y28o6wK30w94MGDiGVMIUNBtAFLNMxZWXD4LUV1BuwX32RTGgwZLrq7hG3ePmU1IIM2jmwEdYjVxq4dSvci3ZQOTga0E0JsUJBLa3dCiWFFDXQQtpnAcTPa3xyH+90o2k9Mv/kMD4LygKAzM+Kl/NsExiD3R4Xnpp11lpwHp9pzuJotPlt/g4vCHw4wZpxNyTnYU48VfQgWpc7IFg3VGbVTTG1iz8Ctd6r0WSxg1eaJBMMp3vQ3eq8JBRYPgm1LmUbVpiU90Oe/y85UJbnNsVRphfk6X7RriSRIjB9X6bPcP/f2cxGR/CxBQZR2NxasbCXPJHbIpiVJYZIYiPiBcJ1WxCi6NKsIGBSm1tnec0nhaGrlN0aTHLgY1AKs+os19XCaTn2wyglRUYWD+yxgoPS7cjzdicE+XSOlI7sWKAvs9AXgBuRJCPHFCI82oozLgXU4t29DxpptzARwTkI6RvsdulKy34z5QjIGcsavwTHcSF3++KqsayE9/2B4BbLtY5argzXK5ITrDh1ysluI6dhpslPOHi6UNrvVoWDGyVbYmS/kHnCu+4PO//GwHwuF9JF4I8cmXm/Q==,iv:hGKXmZtGvDOEJNMV+q5rOIoWquikzerG94TC8rlMhr8=,tag:8cc2BP/HMZM4trsiEVjtbQ==,type:str] sops: age: - enc: | @@ -29,6 +32,6 @@ sops: -----END AGE ENCRYPTED FILE----- recipient: age1c9jlhvq47psz20sl2wu7d2jwstjjylfx769tuumree8hdf5klf2sjluulk encrypted_regex: ^(data|stringData)$ - lastmodified: "2026-10-12T19:06:48Z" - mac: ENC[AES256_GCM,data:DRwrlXi3hw8/j+ZL8CM1svQn+qlvS35k1dPz/7b+Fd395RB+idYGUjrAeScLt+Z4NMKKo2aC2UzsMSdramX/MR49Xd2H+F0IcI28JSYi43Ua9/b55YXQQMunm8F1vFV7x1zhjor/m4sTxVCNt3m7PLQ9YDIgWuxxV1WfxevsaIw=,iv:znRQ+qTSd6Qy11T7caQZOr+4eJMq/32HIgiTWPm0sW4=,tag:rGCfzQD0Hlktk6sBN9lHdw==,type:str] + lastmodified: "2026-10-12T19:48:00Z" + mac: ENC[AES256_GCM,data:GHZXUzYb5K+Nb8/jsyHBWuvjEgvT0VhQ0ST3loFpimd5mp/Lsi99HS8H8q769UJpub/v8Xfi+mElhWet169aLQ4z4GtAb1B85pMDKq38/pWcmd0sFH0i0be/quK5ss5qsqnAYr6WXQ6ovBNwmwwvdWuZFZzWNWLKRauIFdYLHNM=,iv:qmKNNL1/TfF2eNYn2nzhbs2lJZMDDW8HeSiK2wbjoAY=,tag:fPHESGX17XXXqa3f7edaYw==,type:str] version: 3.13.3 diff --git a/infrastructure/configs/monitoring/stack.yaml b/infrastructure/configs/monitoring/stack.yaml index 655999d..646b81e 100644 --- a/infrastructure/configs/monitoring/stack.yaml +++ b/infrastructure/configs/monitoring/stack.yaml @@ -152,7 +152,7 @@ spec: passwordKey: admin-password # Bump when the Secret grafana changes: the pod restarts and, with no PVC, recreates its database. podAnnotations: - bulbashenko.com/secret-revision: "2" + bulbashenko.com/secret-revision: "3" envValueFrom: GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: secretKeyRef: { name: grafana, key: oidc-client-secret } From 134cc16c34c98e93c19402133d8b1f341156f0f0 Mon Sep 17 00:00:00 2001 From: Aleksandr Albekov Date: Mon, 12 Oct 2026 21:50:43 +0200 Subject: [PATCH 2/2] Fix the dashboard and rules after checking them on live data - The "primary only" filter multiplied by (in_recovery == 0), whose value is 0, so WAL age, streaming replicas and Postgres connections always read 0: PostgresNoStreamingReplica fired falsely and PostgresWALArchivingStalled could never fire. Use `and on (pod)`. - FluxResourceNotReady: Flux Operator writes suspended="True"/"False". - "Pods not ready" counted completed Job pods. - "Typical deploy duration" from Flux sync times was meaningless (syncs take seconds; the time goes into CI). Replaced by "last deploy, ago" from ReplicaSet creation. - Grafana was OOMKilled at 384Mi (it sits at ~380Mi with the GitHub plugin): request 256Mi, limit 768Mi. Refs #74 Co-Authored-By: Claude Opus 5.5 --- .../configs/monitoring/dashboards/main.json | 16 ++++++++-------- infrastructure/configs/monitoring/rules.yaml | 6 +++--- infrastructure/configs/monitoring/stack.yaml | 5 +++-- scripts/dashboard-main.py | 16 ++++++++++------ 4 files changed, 24 insertions(+), 19 deletions(-) diff --git a/infrastructure/configs/monitoring/dashboards/main.json b/infrastructure/configs/monitoring/dashboards/main.json index 7a290f6..cbb9cde 100644 --- a/infrastructure/configs/monitoring/dashboards/main.json +++ b/infrastructure/configs/monitoring/dashboards/main.json @@ -149,7 +149,7 @@ "id": 4, "type": "stat", "title": "Поды не готовы", - "description": "", + "description": "Без завершённых подов Job/CronJob.", "datasource": { "type": "prometheus", "uid": "VictoriaMetrics" @@ -199,7 +199,7 @@ "type": "prometheus", "uid": "VictoriaMetrics" }, - "expr": "sum(kube_pod_status_ready{condition=\"false\"}) or vector(0)", + "expr": "count((kube_pod_status_ready{condition=\"false\"} == 1) unless on (namespace, pod) (kube_pod_status_phase{phase=\"Succeeded\"} == 1)) or vector(0)", "instant": true } ] @@ -520,8 +520,8 @@ { "id": 11, "type": "stat", - "title": "Обычно длится деплой", - "description": "Медиана синхронизации Kustomization приложений за 7 дней (включая ожидание готовности подов). Не ETA, а ориентир.", + "title": "Последний деплой, назад", + "description": "Когда последний раз выкатилась новая версия приложения (новый ReplicaSet). Время деплоя уходит в основном на CI (таблицы CI/CD ниже), выкатка в кластере — секунды.", "datasource": { "type": "prometheus", "uid": "VictoriaMetrics" @@ -567,7 +567,7 @@ "type": "prometheus", "uid": "VictoriaMetrics" }, - "expr": "histogram_quantile(0.5, sum by (le) (rate(gotk_reconcile_duration_seconds_bucket{kind=\"Kustomization\", name=~\"site|auth|mail|teamspeak\"}[7d])))", + "expr": "time() - max(kube_replicaset_created{namespace=~\"site|auth|mail|teamspeak\"})", "instant": true } ] @@ -1871,7 +1871,7 @@ "type": "prometheus", "uid": "VictoriaMetrics" }, - "expr": "max(cnpg_pg_stat_archiver_seconds_since_last_archival * on (pod) group_left () (cnpg_pg_replication_in_recovery == 0)) / 60", + "expr": "max(cnpg_pg_stat_archiver_seconds_since_last_archival and on (pod) (cnpg_pg_replication_in_recovery == 0)) / 60", "instant": true } ] @@ -2113,7 +2113,7 @@ "type": "prometheus", "uid": "VictoriaMetrics" }, - "expr": "max(cnpg_pg_replication_streaming_replicas * on (pod) group_left () (cnpg_pg_replication_in_recovery == 0))", + "expr": "max(cnpg_pg_replication_streaming_replicas and on (pod) (cnpg_pg_replication_in_recovery == 0))", "instant": true } ] @@ -2607,7 +2607,7 @@ "type": "prometheus", "uid": "VictoriaMetrics" }, - "expr": "sum by (datname) (cnpg_backends_total * on (pod) group_left () (cnpg_pg_replication_in_recovery == 0))", + "expr": "sum by (datname) (cnpg_backends_total and on (pod) (cnpg_pg_replication_in_recovery == 0))", "legendFormat": "{{datname}}" } ] diff --git a/infrastructure/configs/monitoring/rules.yaml b/infrastructure/configs/monitoring/rules.yaml index 90a9985..9b8a5c5 100644 --- a/infrastructure/configs/monitoring/rules.yaml +++ b/infrastructure/configs/monitoring/rules.yaml @@ -25,7 +25,7 @@ spec: summary: "Нет метрики времени бэкапа Postgres" description: "Плагин barman-cloud не отдаёт last_available_backup_timestamp: возраст бэкапа не контролируется." - alert: PostgresWALArchivingStalled - expr: max(cnpg_pg_stat_archiver_seconds_since_last_archival * on (pod) group_left () (cnpg_pg_replication_in_recovery == 0)) > 900 + expr: max(cnpg_pg_stat_archiver_seconds_since_last_archival and on (pod) (cnpg_pg_replication_in_recovery == 0)) > 900 for: 10m labels: { severity: critical } annotations: @@ -56,7 +56,7 @@ spec: summary: "Postgres {{ $labels.pod }} не работает" description: "Экспортер CNPG на {{ $labels.pod }} сообщает, что PostgreSQL недоступен." - alert: PostgresNoStreamingReplica - expr: max(cnpg_pg_replication_streaming_replicas * on (pod) group_left () (cnpg_pg_replication_in_recovery == 0)) < 1 + expr: max(cnpg_pg_replication_streaming_replicas and on (pod) (cnpg_pg_replication_in_recovery == 0)) < 1 for: 10m labels: { severity: warning } annotations: @@ -143,7 +143,7 @@ spec: summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) убит по памяти" description: "Контейнер упёрся в memory limit и был перезапущен. Подними limit или найди утечку." - alert: FluxResourceNotReady - expr: flux_resource_info{ready="False", suspended!="true"} == 1 + expr: flux_resource_info{ready="False", suspended!="True"} == 1 for: 15m labels: { severity: warning } annotations: diff --git a/infrastructure/configs/monitoring/stack.yaml b/infrastructure/configs/monitoring/stack.yaml index 646b81e..78077cb 100644 --- a/infrastructure/configs/monitoring/stack.yaml +++ b/infrastructure/configs/monitoring/stack.yaml @@ -139,9 +139,10 @@ spec: # No PVC: dashboards and datasources are provisioned from git, people sign in through Ory. persistence: enabled: false + # Grafana 13 with the GitHub plugin sits around 380Mi; 384Mi got it OOMKilled. resources: - requests: { cpu: 20m, memory: 128Mi } - limits: { memory: 384Mi } + requests: { cpu: 20m, memory: 256Mi } + limits: { memory: 768Mi } # Local admin: used by the dashboard/datasource sidecars (basic auth against the reload API) and as break-glass # sign-in at /login?disableAutoLogin if Ory is down; the login form is hidden otherwise. # Its login must not match any Ory username: Grafana refuses to create an SSO user whose login or email is diff --git a/scripts/dashboard-main.py b/scripts/dashboard-main.py index 7d8854a..bdd5520 100644 --- a/scripts/dashboard-main.py +++ b/scripts/dashboard-main.py @@ -8,7 +8,8 @@ DS = {"type": "prometheus", "uid": "VictoriaMetrics"} LAYER = ('* on (namespace) group_left (label_bulbashenko_com_layer, label_bulbashenko_com_app) ' 'max by (namespace, label_bulbashenko_com_layer, label_bulbashenko_com_app) (kube_namespace_labels)') -PRIMARY = '* on (pod) group_left () (cnpg_pg_replication_in_recovery == 0)' +# `and`, not `*`: multiplying by the filter would multiply by its value, 0. +PRIMARY = 'and on (pod) (cnpg_pg_replication_in_recovery == 0)' panels = [] y = 0 @@ -94,7 +95,9 @@ def advance(h): 'count(ALERTS{alertstate="firing", alertname!~"Watchdog|InfoInhibitor"}) or vector(0)', 0, desc="Сработавшие алерты, кроме служебных. Список: Alerting > Alert rules."), stat("Таргеты вниз", 'count(up == 0) or vector(0)', 4, desc="Что vmagent не может опросить; таблица внизу."), - stat("Поды не готовы", 'sum(kube_pod_status_ready{condition="false"}) or vector(0)', 8), + stat("Поды не готовы", + 'count((kube_pod_status_ready{condition="false"} == 1) unless on (namespace, pod) (kube_pod_status_phase{phase="Succeeded"} == 1)) or vector(0)', + 8, desc="Без завершённых подов Job/CronJob."), stat("Рестарты за 1 ч", 'round(sum(increase(kube_pod_container_status_restarts_total[1h]))) or vector(0)', 12), stat("OOMKilled сейчас", 'count(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"} == 1) or vector(0)', 16, @@ -122,10 +125,11 @@ def advance(h): ) advance(4) add( - stat("Обычно длится деплой", - 'histogram_quantile(0.5, sum by (le) (rate(gotk_reconcile_duration_seconds_bucket{kind="Kustomization", name=~"site|auth|mail|teamspeak"}[7d])))', - 0, w=4, unit="s", thr=steps((None, "blue")), - desc="Медиана синхронизации Kustomization приложений за 7 дней (включая ожидание готовности подов). Не ETA, а ориентир."), + stat("Последний деплой, назад", + 'time() - max(kube_replicaset_created{namespace=~"site|auth|mail|teamspeak"})', 0, w=4, + unit="s", thr=steps((None, "blue")), + desc="Когда последний раз выкатилась новая версия приложения (новый ReplicaSet). " + "Время деплоя уходит в основном на CI (таблицы CI/CD ниже), выкатка в кластере — секунды."), ) advance(4) add(