From 7d3d2fd3bb86d3275ff5a435fe6b3b288e781084 Mon Sep 17 00:00:00 2001
From: operations <operations@hive.kubestellar.io>
Date: Fri, 9 Oct 2026 14:34:59 -0400
Subject: [PATCH] feat(deploy): alert on repeated Dibs container restarts

Adds DibsPodRestarting with a runbook section covering OOM kills and crash loops.

Signed-off-by: operations <operations@hive.kubestellar.io>
---
 deploy/monitoring/prometheusrule.yaml | 10 ++++++++++
 runbooks/incident-response.md         | 12 ++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/deploy/monitoring/prometheusrule.yaml b/deploy/monitoring/prometheusrule.yaml
index a12d701..bc2c525 100644
--- a/deploy/monitoring/prometheusrule.yaml
+++ b/deploy/monitoring/prometheusrule.yaml
@@ -74,3 +74,13 @@ spec:
           annotations:
             runbook_url: https://github.com/hivecommons/dibs/blob/main/runbooks/incident-response.md#6-match-engine-llm-alert
             summary: More than half of Dibs match-engine LLM calls for op {{ $labels.op }} are failing.
+        - alert: DibsPodRestarting
+          # Needs kube-state-metrics. Catches OOM kills and crash loops that
+          # recover between scrapes, which DibsDown (absent up) does not see.
+          expr: increase(kube_pod_container_status_restarts_total{namespace="dibs",container="dibs"}[30m]) >= 3
+          for: 5m
+          labels:
+            severity: warning
+          annotations:
+            runbook_url: https://github.com/hivecommons/dibs/blob/main/runbooks/incident-response.md#7-pod-restart-alert
+            summary: Dibs container has restarted 3 or more times in 30 minutes.
diff --git a/runbooks/incident-response.md b/runbooks/incident-response.md
index d9817e5..a6a484e 100644
--- a/runbooks/incident-response.md
+++ b/runbooks/incident-response.md
@@ -71,3 +71,15 @@ quality degrades.
    credential or model name is wrong, fix the secret and restart with
    `kubectl -n dibs rollout restart deploy/dibs`.
 5. If errors began right after a rollout, follow [release-rollback.md](release-rollback.md).
+
+## 7. Pod restart alert
+
+For `DibsPodRestarting`. A container that restarts and recovers can leave
+`/readyz` green at scrape time while requests in flight are dropped.
+
+1. Find the last termination reason:
+   `kubectl -n dibs describe pod -l app=dibs` (`Last State`, `Reason`, `Exit Code`).
+2. `OOMKilled` means the 256Mi memory limit in `deploy/deployment.yaml` is too
+   low for the current load; raise the limit and roll out.
+3. Otherwise read the crash output: `kubectl -n dibs logs deploy/dibs --previous`.
+4. If restarts began right after a rollout, follow [release-rollback.md](release-rollback.md).