Skip to content

Grafana: GitHub App credentials and fixes found on live data #111

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bulbashenko merged 2 commits into from
Oct 12, 2026
Merged

Grafana: GitHub App credentials and fixes found on live data #111

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3734c39
Add the GitHub App credentials for Grafana's GitHub datasource
bulbashenko Oct 12, 2026
134cc16
Fix the dashboard and rules after checking them on live data
bulbashenko Oct 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions infrastructure/configs/monitoring/dashboards/main.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@
"id": 4,
"type": "stat",
"title": "Поды не готовы",
"description": "",
"description": "Без завершённых подов Job/CronJob.",
"datasource": {
"type": "prometheus",
"uid": "VictoriaMetrics"
Expand Down Expand Up @@ -199,7 +199,7 @@
"type": "prometheus",
"uid": "VictoriaMetrics"
},
"expr": "sum(kube_pod_status_ready{condition=\"false\"}) or vector(0)",
"expr": "count((kube_pod_status_ready{condition=\"false\"} == 1) unless on (namespace, pod) (kube_pod_status_phase{phase=\"Succeeded\"} == 1)) or vector(0)",
"instant": true
}
]
Expand Down Expand Up @@ -520,8 +520,8 @@
{
"id": 11,
"type": "stat",
"title": "Обычно длится деплой",
"description": "Медиана синхронизации Kustomization приложений за 7 дней (включая ожидание готовности подов). Не ETA, а ориентир.",
"title": "Последний деплой, назад",
"description": "Когда последний раз выкатилась новая версия приложения (новый ReplicaSet). Время деплоя уходит в основном на CI (таблицы CI/CD ниже), выкатка в кластере — секунды.",
"datasource": {
"type": "prometheus",
"uid": "VictoriaMetrics"
Expand Down Expand Up @@ -567,7 +567,7 @@
"type": "prometheus",
"uid": "VictoriaMetrics"
},
"expr": "histogram_quantile(0.5, sum by (le) (rate(gotk_reconcile_duration_seconds_bucket{kind=\"Kustomization\", name=~\"site|auth|mail|teamspeak\"}[7d])))",
"expr": "time() - max(kube_replicaset_created{namespace=~\"site|auth|mail|teamspeak\"})",
"instant": true
}
]
Expand Down Expand Up @@ -1871,7 +1871,7 @@
"type": "prometheus",
"uid": "VictoriaMetrics"
},
"expr": "max(cnpg_pg_stat_archiver_seconds_since_last_archival * on (pod) group_left () (cnpg_pg_replication_in_recovery == 0)) / 60",
"expr": "max(cnpg_pg_stat_archiver_seconds_since_last_archival and on (pod) (cnpg_pg_replication_in_recovery == 0)) / 60",
"instant": true
}
]
Expand Down Expand Up @@ -2113,7 +2113,7 @@
"type": "prometheus",
"uid": "VictoriaMetrics"
},
"expr": "max(cnpg_pg_replication_streaming_replicas * on (pod) group_left () (cnpg_pg_replication_in_recovery == 0))",
"expr": "max(cnpg_pg_replication_streaming_replicas and on (pod) (cnpg_pg_replication_in_recovery == 0))",
"instant": true
}
]
Expand Down Expand Up @@ -2607,7 +2607,7 @@
"type": "prometheus",
"uid": "VictoriaMetrics"
},
"expr": "sum by (datname) (cnpg_backends_total * on (pod) group_left () (cnpg_pg_replication_in_recovery == 0))",
"expr": "sum by (datname) (cnpg_backends_total and on (pod) (cnpg_pg_replication_in_recovery == 0))",
"legendFormat": "{{datname}}"
}
]
Expand Down
6 changes: 3 additions & 3 deletions infrastructure/configs/monitoring/rules.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ spec:
summary: "Нет метрики времени бэкапа Postgres"
description: "Плагин barman-cloud не отдаёт last_available_backup_timestamp: возраст бэкапа не контролируется."
- alert: PostgresWALArchivingStalled
expr: max(cnpg_pg_stat_archiver_seconds_since_last_archival * on (pod) group_left () (cnpg_pg_replication_in_recovery == 0)) > 900
expr: max(cnpg_pg_stat_archiver_seconds_since_last_archival and on (pod) (cnpg_pg_replication_in_recovery == 0)) > 900
for: 10m
labels: { severity: critical }
annotations:
Expand Down Expand Up @@ -56,7 +56,7 @@ spec:
summary: "Postgres {{ $labels.pod }} не работает"
description: "Экспортер CNPG на {{ $labels.pod }} сообщает, что PostgreSQL недоступен."
- alert: PostgresNoStreamingReplica
expr: max(cnpg_pg_replication_streaming_replicas * on (pod) group_left () (cnpg_pg_replication_in_recovery == 0)) < 1
expr: max(cnpg_pg_replication_streaming_replicas and on (pod) (cnpg_pg_replication_in_recovery == 0)) < 1
for: 10m
labels: { severity: warning }
annotations:
Expand Down Expand Up @@ -143,7 +143,7 @@ spec:
summary: "{{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container }}) убит по памяти"
description: "Контейнер упёрся в memory limit и был перезапущен. Подними limit или найди утечку."
- alert: FluxResourceNotReady
expr: flux_resource_info{ready="False", suspended!="true"} == 1
expr: flux_resource_info{ready="False", suspended!="True"} == 1
for: 15m
labels: { severity: warning }
annotations:
Expand Down
7 changes: 5 additions & 2 deletions infrastructure/configs/monitoring/secret.sops.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ stringData:
admin-user: ENC[AES256_GCM,data:DIZEGbpSfxHB8Qs=,iv:JmBr2VvPEgudEK5xXMCZVbo5QKdnHFAStI1mtxxv/mQ=,tag:Vt3IwD1BgJuQq7jPkB+CIQ==,type:str]
admin-password: ENC[AES256_GCM,data:029sxX8MPFsDCrkHHCYd2e7vZv6beBmXRTDxhtKKa0M=,iv:SJd8Vz1hrMqzxcZ9/kH9q5xNOtUDqmTLltIaHtYpAYw=,tag:gH7g7ra9E2A7XQNTROmSrw==,type:str]
oidc-client-secret: ENC[AES256_GCM,data:vVz8nQRKEdSeVKUvPZjbH3Bgi0TSetfDtoe8LZVNZ2+MN1yIVA2cGjDaLphrODJNozmJOg0Xpux6vUDC5PvKnQ==,iv:ezkCA3ePrCGPhNh9CNCXoageI8X3QIFV5+QXFWvhwOg=,tag:0uP7gekPgeYsvvbiyF6sSA==,type:str]
github-app-id: ENC[AES256_GCM,data:dV0+LIJEuA==,iv:5KJh7oNWYXU9WC4s2stWRZ3v5547WYlNCF3uNI39kqc=,tag:/8DfgCqXZxdDnwJwfm5rhg==,type:str]
github-app-installation-id: ENC[AES256_GCM,data:7FDXDfUH8MW3,iv:kvJ35LolM5MgoDgJqsyw1+N2oTKZGn9AU2G2cYS+rsY=,tag:GQS//aMGWo3tjdtwfsBbww==,type:str]
github-app-private-key: ENC[AES256_GCM,data:/cqK4krS1//pfDdSYzu0rLz/qDrVkcdKfyFXQ/EE1HJLElIhOaUUC9zpD4Dkb+F1U9chuxj+m1n3BdBJdTKUACtPM/6Sz9IZpsGQJIyTFODbIj2ezDb80mO70/jlYTIz8oc+0QAGdwRMoH3xVO5v/7InkNytpbTSEH9qjMkTjVLMRPbMQ4huvquRr1eD20z2JLX4W0zvxzihMOSbCIiV9P5Hf5oWYxEgHC1vLgMg7GBXKXp6taQDAfgrWIS5nG2AD5Pie6jZRWhIkIgYXBvDbWq10Kp/g5u12dIHFviHiruSqJVdkuIpWPWI1xF7sBoIIwsIb//wG4TkRfCHNMqj65+c7W2woK71xTdhQylb9tybzWqqyoXE53b1JFPXmy1+Qn0AAOZIt4/iz8xsqy9pi9Uo6mdtW5cApaj+LpCuEShg5/dfHhhZlEbqGWVmD8SK2ZPWqjrn1SoWU1izRn84Ns7uIHRAmFhRjoj6eFxJvgX2pBWZkoTwvQFQeYrHLiOoAwBNla1G45AsGk8kOXD3zyXSUz9QTVnTBccq362jeykKh4PcOVKr3aglfM7zNHHNjoqd8CM6IOsfaBmXzSirtanNj/dtDtrbpoVRT7z3vM+S0PSdBHeciEZF/hPWSrdDaLDV86PJZhkMvYu0kxl6ZaRZ9didt/E91KgljZM6NbmCptq1dfgcxR44aSiDt8nRkJ3GKT+hNXX1G2dSMN48WYNgHrue3IGz2mTyTs4lLQ8HDv5SkcUjRFhJb2tN67ucZ6DC7n/42RXaUn8XYeQMjeZAlULKAj50BwR5CKWWEoMAv2EDEVm2UmYPcXGMAuzd7BHWktMsI51kKrAeS5LNARvLs6TSe8IM0t5G7JlOgUbBZfnfBE6Hb6WqD23iPAX1/GzKGK077mHOY/oMwUq1VYjrwnSZxBUClXqTYquhaO4rF1i9ibIrqliIdsVpXYInX8lE7oghHGdei5cw5tEyCnva9nxGCRUZafZxOjtspqSd2CyzFDcF8y1ljJzYBb+1fIAEi+5zjOg15OG/V21Hl1LfnRmepLgj5KbRAYWG1DqBs/F4+UawNLMUq3uB87cUaUaLo7WLmvQNZhlnewqF3gAFb0L5WcTlawbqnFvcxF20PsrsFqJlRaDK3BAGfQqQ8FPrEaOI+iZa/DGh139wU4cy7msHpHij6yIgyakTeUwyknGvibIknAKlsciSWx5xI6QKHTQcfUM8yCv9utDmxDEKUv9AbTPk4Dko5enTI2wZ3sjrn4+zDCIt58m8uut7FqVCSrvedPruxqE8bAjeeTNATt/Ty0foQtik0gAL4dSCSmnLszpzT6IQvlp6EjQie2k4eXvaaom8CJiH9AshrhCqvz4T815/4Tr9TiqjPouUk8tVLIAqJJewVlIAHDLTfs2ILQPY9LfPBsYBeTfq8d0igkeUgBde+2HZxduAUbkOFOzQNXl/VYhYQFJyYpterzB+mdhecneslqp3rMnmkZ88vcO8/eFdIZRwFmcUOoETE1g0J8KbZjoZSErcYORxESkKXJg5ZTLrr/rJfdgEyEZXWTE4poRRkyQTxHDA4r5mQCqQBDNp3ZJ3xeTFzviI5s1eX4frBsWjBp61QlMImQ/EAInjQ2cFmPXbfaezxdWQ7k/Y28o6wK30w94MGDiGVMIUNBtAFLNMxZWXD4LUV1BuwX32RTGgwZLrq7hG3ePmU1IIM2jmwEdYjVxq4dSvci3ZQOTga0E0JsUJBLa3dCiWFFDXQQtpnAcTPa3xyH+90o2k9Mv/kMD4LygKAzM+Kl/NsExiD3R4Xnpp11lpwHp9pzuJotPlt/g4vCHw4wZpxNyTnYU48VfQgWpc7IFg3VGbVTTG1iz8Ctd6r0WSxg1eaJBMMp3vQ3eq8JBRYPgm1LmUbVpiU90Oe/y85UJbnNsVRphfk6X7RriSRIjB9X6bPcP/f2cxGR/CxBQZR2NxasbCXPJHbIpiVJYZIYiPiBcJ1WxCi6NKsIGBSm1tnec0nhaGrlN0aTHLgY1AKs+os19XCaTn2wyglRUYWD+yxgoPS7cjzdicE+XSOlI7sWKAvs9AXgBuRJCPHFCI82oozLgXU4t29DxpptzARwTkI6RvsdulKy34z5QjIGcsavwTHcSF3++KqsayE9/2B4BbLtY5argzXK5ITrDh1ysluI6dhpslPOHi6UNrvVoWDGyVbYmS/kHnCu+4PO//GwHwuF9JF4I8cmXm/Q==,iv:hGKXmZtGvDOEJNMV+q5rOIoWquikzerG94TC8rlMhr8=,tag:8cc2BP/HMZM4trsiEVjtbQ==,type:str]
sops:
age:
- enc: |
Expand All @@ -29,6 +32,6 @@ sops:
-----END AGE ENCRYPTED FILE-----
recipient: age1c9jlhvq47psz20sl2wu7d2jwstjjylfx769tuumree8hdf5klf2sjluulk
encrypted_regex: ^(data|stringData)$
lastmodified: "2026-10-12T19:06:48Z"
mac: ENC[AES256_GCM,data:DRwrlXi3hw8/j+ZL8CM1svQn+qlvS35k1dPz/7b+Fd395RB+idYGUjrAeScLt+Z4NMKKo2aC2UzsMSdramX/MR49Xd2H+F0IcI28JSYi43Ua9/b55YXQQMunm8F1vFV7x1zhjor/m4sTxVCNt3m7PLQ9YDIgWuxxV1WfxevsaIw=,iv:znRQ+qTSd6Qy11T7caQZOr+4eJMq/32HIgiTWPm0sW4=,tag:rGCfzQD0Hlktk6sBN9lHdw==,type:str]
lastmodified: "2026-10-12T19:48:00Z"
mac: ENC[AES256_GCM,data:GHZXUzYb5K+Nb8/jsyHBWuvjEgvT0VhQ0ST3loFpimd5mp/Lsi99HS8H8q769UJpub/v8Xfi+mElhWet169aLQ4z4GtAb1B85pMDKq38/pWcmd0sFH0i0be/quK5ss5qsqnAYr6WXQ6ovBNwmwwvdWuZFZzWNWLKRauIFdYLHNM=,iv:qmKNNL1/TfF2eNYn2nzhbs2lJZMDDW8HeSiK2wbjoAY=,tag:fPHESGX17XXXqa3f7edaYw==,type:str]
version: 3.13.3
7 changes: 4 additions & 3 deletions infrastructure/configs/monitoring/stack.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,9 +139,10 @@ spec:
# No PVC: dashboards and datasources are provisioned from git, people sign in through Ory.
persistence:
enabled: false
# Grafana 13 with the GitHub plugin sits around 380Mi; 384Mi got it OOMKilled.
resources:
requests: { cpu: 20m, memory: 128Mi }
limits: { memory: 384Mi }
requests: { cpu: 20m, memory: 256Mi }
limits: { memory: 768Mi }
# Local admin: used by the dashboard/datasource sidecars (basic auth against the reload API) and as break-glass
# sign-in at /login?disableAutoLogin if Ory is down; the login form is hidden otherwise.
# Its login must not match any Ory username: Grafana refuses to create an SSO user whose login or email is
Expand All @@ -152,7 +153,7 @@ spec:
passwordKey: admin-password
# Bump when the Secret grafana changes: the pod restarts and, with no PVC, recreates its database.
podAnnotations:
bulbashenko.com/secret-revision: "2"
bulbashenko.com/secret-revision: "3"
envValueFrom:
GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET:
secretKeyRef: { name: grafana, key: oidc-client-secret }
Expand Down
16 changes: 10 additions & 6 deletions scripts/dashboard-main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@
DS = {"type": "prometheus", "uid": "VictoriaMetrics"}
LAYER = ('* on (namespace) group_left (label_bulbashenko_com_layer, label_bulbashenko_com_app) '
'max by (namespace, label_bulbashenko_com_layer, label_bulbashenko_com_app) (kube_namespace_labels)')
PRIMARY = '* on (pod) group_left () (cnpg_pg_replication_in_recovery == 0)'
# `and`, not `*`: multiplying by the filter would multiply by its value, 0.
PRIMARY = 'and on (pod) (cnpg_pg_replication_in_recovery == 0)'

panels = []
y = 0
Expand Down Expand Up @@ -94,7 +95,9 @@ def advance(h):
'count(ALERTS{alertstate="firing", alertname!~"Watchdog|InfoInhibitor"}) or vector(0)', 0,
desc="Сработавшие алерты, кроме служебных. Список: Alerting > Alert rules."),
stat("Таргеты вниз", 'count(up == 0) or vector(0)', 4, desc="Что vmagent не может опросить; таблица внизу."),
stat("Поды не готовы", 'sum(kube_pod_status_ready{condition="false"}) or vector(0)', 8),
stat("Поды не готовы",
'count((kube_pod_status_ready{condition="false"} == 1) unless on (namespace, pod) (kube_pod_status_phase{phase="Succeeded"} == 1)) or vector(0)',
8, desc="Без завершённых подов Job/CronJob."),
stat("Рестарты за 1 ч", 'round(sum(increase(kube_pod_container_status_restarts_total[1h]))) or vector(0)', 12),
stat("OOMKilled сейчас",
'count(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"} == 1) or vector(0)', 16,
Expand Down Expand Up @@ -122,10 +125,11 @@ def advance(h):
)
advance(4)
add(
stat("Обычно длится деплой",
'histogram_quantile(0.5, sum by (le) (rate(gotk_reconcile_duration_seconds_bucket{kind="Kustomization", name=~"site|auth|mail|teamspeak"}[7d])))',
0, w=4, unit="s", thr=steps((None, "blue")),
desc="Медиана синхронизации Kustomization приложений за 7 дней (включая ожидание готовности подов). Не ETA, а ориентир."),
stat("Последний деплой, назад",
'time() - max(kube_replicaset_created{namespace=~"site|auth|mail|teamspeak"})', 0, w=4,
unit="s", thr=steps((None, "blue")),
desc="Когда последний раз выкатилась новая версия приложения (новый ReplicaSet). "
"Время деплоя уходит в основном на CI (таблицы CI/CD ниже), выкатка в кластере — секунды."),
)
advance(4)
add(
Expand Down
Loading