feat(telemetry): observe cluster remote fallback persistence

deploy/helm/sie-cluster/files/alerts/sie-rules.yaml

@@ -77,6 +77,24 @@ groups:
           summary: "SIE Gateway is down"
           description: "Gateway pod {{ $labels.pod }} is unreachable. Traffic cannot be routed."
 
+      - alert: SIERemoteFallbackPersistent
+        expr: |
+          max by (model) (
+            sie_gateway_remote_serving_duration_seconds{namespace="__NAMESPACE__",service="__COLLECTOR_SERVICE__",endpoint="prometheus",producer_service="sie-gateway"}
+            and on (producer_instance, collector_generation, model)
+            (
+              sum by (producer_instance, collector_generation, model) (
+                increase(sie_gateway_remote_fallbacks_total{namespace="__NAMESPACE__",service="__COLLECTOR_SERVICE__",endpoint="prometheus",producer_service="sie-gateway",outcome="committed"}[5m])
+              ) > 0
+            )
+          ) > __REMOTE_FALLBACK_PERSISTENCE_SECONDS__
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "SIE model {{ $labels.model }} continues to serve remotely"
+          description: "Committed fallback bridges for {{ $labels.model }} span more than __REMOTE_FALLBACK_PERSISTENCE_SECONDS__ seconds without observed local success. Check local worker readiness, model loading and admission pressure."
+
       - alert: SIEHighErrorRate
         expr: |
           sum(rate(sie_gateway_requests_total{namespace="__NAMESPACE__",service="__COLLECTOR_SERVICE__",endpoint="prometheus",producer_service="sie-gateway",http_status_code=~"5.."}[5m]))

deploy/helm/sie-cluster/files/dashboards/queue-routing.json

@@ -1738,6 +1738,84 @@
       ],
       "title": "Gateway Queue Worker-Pool Backpressure and Shed",
       "type": "timeseries"
+    },
+    {
+      "description": "Committed HTTP/first-output bridge responses and pre-output refusals. Later stream errors remain in generation telemetry.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps"
+        }
+      },
+      "gridPos": {
+        "x": 0,
+        "y": 125,
+        "w": 12,
+        "h": 8
+      },
+      "id": 119,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "sum by (model, operation, fallback_reason, outcome) (rate(sie_gateway_remote_fallbacks_total{namespace=\"$namespace\", service=\"$collector\", endpoint=\"prometheus\", producer_service=\"sie-gateway\"}[$__rate_interval]))",
+          "legendFormat": "{{model}} {{operation}} {{fallback_reason}} {{outcome}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Remote Fallback Responses",
+      "type": "timeseries"
+    },
+    {
+      "description": "Observed seconds between first and latest committed bridge per gateway replica. Local success resets it. The persistence alert additionally requires recent committed activity from the same replica.",
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s"
+        }
+      },
+      "gridPos": {
+        "x": 12,
+        "y": 125,
+        "w": 12,
+        "h": 8
+      },
+      "id": 120,
+      "options": {
+        "legend": {
+          "calcs": [
+            "mean",
+            "max"
+          ],
+          "displayMode": "table",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "desc"
+        }
+      },
+      "targets": [
+        {
+          "expr": "max by (model) (sie_gateway_remote_serving_duration_seconds{namespace=\"$namespace\", service=\"$collector\", endpoint=\"prometheus\", producer_service=\"sie-gateway\"})",
+          "legendFormat": "{{model}}",
+          "refId": "A"
+        }
+      ],
+      "title": "Remote Serving Duration Since Local Success",
+      "type": "timeseries"
     }
   ],
   "refresh": "10s",

deploy/helm/sie-cluster/templates/_otel-collector-config.tpl

@@ -89,7 +89,7 @@ processors:
     error_mode: propagate
     metrics:
       metric:
-        - 'not IsMatch(name, "^(sie[.]gateway[.]requests|sie[.]gateway[.]request[.]duration|sie[.]gateway[.]admission[.]decisions|sie[.]gateway[.]dispatches|sie[.]gateway[.]dispatch[.]duration|sie[.]gateway[.]config[.]applied_epoch|sie[.]gateway[.]config[.]operations|sie[.]gateway[.]config[.]bootstrap[.]degraded|sie[.]gateway[.]messaging[.]client[.]ready|sie[.]gateway[.]queue[.]publishes|sie[.]gateway[.]queue[.]publish[.]duration|sie[.]gateway[.]queue[.]publish[.]items|sie[.]gateway[.]queue[.]result_waits|sie[.]gateway[.]queue[.]result_wait[.]duration|sie[.]gateway[.]queue[.]result_chunks[.]received|sie[.]gateway[.]queue[.]result_chunk[.]bytes_received|sie[.]gateway[.]queue[.]result_chunk[.]rejections|sie[.]gateway[.]queue[.]result_chunk[.]transfers_completed|sie[.]gateway[.]queue[.]result_chunk[.]duplicates|sie[.]gateway[.]queue[.]result_chunk[.]retry_replacements|sie[.]gateway[.]queue[.]result_chunk[.]stale_retries|sie[.]gateway[.]queue[.]result_chunk[.]reserved_bytes|sie[.]gateway[.]queue[.]events|sie[.]gateway[.]queue[.]lane_admission[.]decisions|sie[.]gateway[.]queue[.]worker_pool[.]events|sie[.]gateway[.]provisioning[.]responses|sie[.]gateway[.]routing[.]unsupported_model_exclusions|sie[.]gateway[.]generation[.]events|sie[.]gateway[.]generation[.]ttft|sie[.]gateway[.]generation[.]tpot|sie[.]gateway[.]generation[.]tokens|sie[.]gateway[.]pool[.]pinned_model[.]loaded|sie[.]gateway[.]pending_demand|sie[.]gateway[.]lane[.]queue[.]depth|sie[.]gateway[.]lane[.]queue[.]snapshot[.]timestamp|sie[.]gateway[.]active_lease[.]gpus|sie[.]gateway[.]pool[.]warm_floor|sie[.]gateway[.]rejected[.]requests|sie[.]gateway[.]capacity[.]snapshot[.]timestamp|sie[.]gateway[.]key_snapshot[.]polls|sie[.]gateway[.]settlement[.]confirms|sie[.]gateway[.]settlement[.]confirm[.]duration)$")'
+        - 'not IsMatch(name, "^(sie[.]gateway[.]requests|sie[.]gateway[.]request[.]duration|sie[.]gateway[.]admission[.]decisions|sie[.]gateway[.]dispatches|sie[.]gateway[.]remote[.]fallbacks|sie[.]gateway[.]dispatch[.]duration|sie[.]gateway[.]remote[.]serving[.]duration|sie[.]gateway[.]config[.]applied_epoch|sie[.]gateway[.]config[.]operations|sie[.]gateway[.]config[.]bootstrap[.]degraded|sie[.]gateway[.]messaging[.]client[.]ready|sie[.]gateway[.]queue[.]publishes|sie[.]gateway[.]queue[.]publish[.]duration|sie[.]gateway[.]queue[.]publish[.]items|sie[.]gateway[.]queue[.]result_waits|sie[.]gateway[.]queue[.]result_wait[.]duration|sie[.]gateway[.]queue[.]result_chunks[.]received|sie[.]gateway[.]queue[.]result_chunk[.]bytes_received|sie[.]gateway[.]queue[.]result_chunk[.]rejections|sie[.]gateway[.]queue[.]result_chunk[.]transfers_completed|sie[.]gateway[.]queue[.]result_chunk[.]duplicates|sie[.]gateway[.]queue[.]result_chunk[.]retry_replacements|sie[.]gateway[.]queue[.]result_chunk[.]stale_retries|sie[.]gateway[.]queue[.]result_chunk[.]reserved_bytes|sie[.]gateway[.]queue[.]events|sie[.]gateway[.]queue[.]lane_admission[.]decisions|sie[.]gateway[.]queue[.]worker_pool[.]events|sie[.]gateway[.]provisioning[.]responses|sie[.]gateway[.]routing[.]unsupported_model_exclusions|sie[.]gateway[.]generation[.]events|sie[.]gateway[.]generation[.]ttft|sie[.]gateway[.]generation[.]tpot|sie[.]gateway[.]generation[.]tokens|sie[.]gateway[.]pool[.]pinned_model[.]loaded|sie[.]gateway[.]pending_demand|sie[.]gateway[.]lane[.]queue[.]depth|sie[.]gateway[.]lane[.]queue[.]snapshot[.]timestamp|sie[.]gateway[.]active_lease[.]gpus|sie[.]gateway[.]pool[.]warm_floor|sie[.]gateway[.]rejected[.]requests|sie[.]gateway[.]capacity[.]snapshot[.]timestamp|sie[.]gateway[.]key_snapshot[.]polls|sie[.]gateway[.]settlement[.]confirms|sie[.]gateway[.]settlement[.]confirm[.]duration)$")'
         - 'resource.attributes["service.name"] != "sie-gateway"'
   filter/prometheus_application_contract:
     error_mode: propagate
@@ -120,8 +120,8 @@ processors:
       - context: metric
         statements:
           - set(description, "")
-          - 'set(unit, "{request}") where IsMatch(name, "^(sie[.]gateway[.]requests|sie[.]gateway[.]admission[.]decisions|sie[.]gateway[.]dispatches|sie[.]config[.]requests|sie[.]worker[.]ipc[.]requests|sie[.]worker[.]ipc[.]capacity|sie[.]worker[.]ipc[.]inflight|sie[.]worker[.]generation[.]inflight|sie[.]worker[.]generation[.]duplicate_prevented|sie[.]worker[.]generation[.]grammar[.]requests|sie[.]worker[.]upstream[.]refusals|sie[.]gateway[.]pending_demand|sie[.]gateway[.]rejected[.]requests)$")'
-          - 'set(unit, "s") where IsMatch(name, "^(sie[.]gateway[.]request[.]duration|sie[.]gateway[.]dispatch[.]duration|sie[.]gateway[.]queue[.]publish[.]duration|sie[.]gateway[.]queue[.]result_wait[.]duration|sie[.]gateway[.]generation[.]ttft|sie[.]gateway[.]generation[.]tpot|sie[.]dispatcher[.]invocation[.]duration|sie[.]dispatcher[.]sealed[.]stage[.]duration|sie[.]dispatcher[.]sealed[.]sandbox[.]seconds|sie[.]dispatcher[.]sealed[.]cold_start[.]seconds|sie[.]config[.]request[.]duration|sie[.]worker[.]queue[.]duration|sie[.]worker[.]scheduler[.]request_batch[.]dispatch_wait|sie[.]worker[.]scheduler[.]request_batch[.]total|sie[.]worker[.]scheduler[.]adaptive[.]wait|sie[.]worker[.]scheduler[.]adaptive[.]p50|sie[.]worker[.]ipc[.]request[.]duration|sie[.]worker[.]payload[.]fetch[.]duration|sie[.]worker[.]ipc[.]acquire[.]duration|sie[.]worker[.]shutdown[.]drain[.]duration|sie[.]worker[.]request[.]duration|sie[.]worker[.]inference[.]duration|sie[.]worker[.]model[.]load[.]duration|sie[.]worker[.]generation[.]worker_wait|sie[.]worker[.]generation[.]ttft|sie[.]worker[.]generation[.]tpot|sie[.]worker[.]generation[.]grammar[.]compile[.]duration|sie[.]worker[.]runtime[.]forward[.]duration|sie[.]worker[.]runtime[.]forward[.]permit[.]wait|sie[.]gateway[.]lane[.]queue[.]snapshot[.]timestamp|sie[.]gateway[.]capacity[.]snapshot[.]timestamp|sie[.]gateway[.]settlement[.]confirm[.]duration|sie[.]worker[.]work_item[.]age)$")'
+          - 'set(unit, "{request}") where IsMatch(name, "^(sie[.]gateway[.]requests|sie[.]gateway[.]admission[.]decisions|sie[.]gateway[.]dispatches|sie[.]gateway[.]remote[.]fallbacks|sie[.]config[.]requests|sie[.]worker[.]ipc[.]requests|sie[.]worker[.]ipc[.]capacity|sie[.]worker[.]ipc[.]inflight|sie[.]worker[.]generation[.]inflight|sie[.]worker[.]generation[.]duplicate_prevented|sie[.]worker[.]generation[.]grammar[.]requests|sie[.]worker[.]upstream[.]refusals|sie[.]gateway[.]pending_demand|sie[.]gateway[.]rejected[.]requests)$")'
+          - 'set(unit, "s") where IsMatch(name, "^(sie[.]gateway[.]request[.]duration|sie[.]gateway[.]dispatch[.]duration|sie[.]gateway[.]remote[.]serving[.]duration|sie[.]gateway[.]queue[.]publish[.]duration|sie[.]gateway[.]queue[.]result_wait[.]duration|sie[.]gateway[.]generation[.]ttft|sie[.]gateway[.]generation[.]tpot|sie[.]dispatcher[.]invocation[.]duration|sie[.]dispatcher[.]sealed[.]stage[.]duration|sie[.]dispatcher[.]sealed[.]sandbox[.]seconds|sie[.]dispatcher[.]sealed[.]cold_start[.]seconds|sie[.]config[.]request[.]duration|sie[.]worker[.]queue[.]duration|sie[.]worker[.]scheduler[.]request_batch[.]dispatch_wait|sie[.]worker[.]scheduler[.]request_batch[.]total|sie[.]worker[.]scheduler[.]adaptive[.]wait|sie[.]worker[.]scheduler[.]adaptive[.]p50|sie[.]worker[.]ipc[.]request[.]duration|sie[.]worker[.]payload[.]fetch[.]duration|sie[.]worker[.]ipc[.]acquire[.]duration|sie[.]worker[.]shutdown[.]drain[.]duration|sie[.]worker[.]request[.]duration|sie[.]worker[.]inference[.]duration|sie[.]worker[.]model[.]load[.]duration|sie[.]worker[.]generation[.]worker_wait|sie[.]worker[.]generation[.]ttft|sie[.]worker[.]generation[.]tpot|sie[.]worker[.]generation[.]grammar[.]compile[.]duration|sie[.]worker[.]runtime[.]forward[.]duration|sie[.]worker[.]runtime[.]forward[.]permit[.]wait|sie[.]gateway[.]lane[.]queue[.]snapshot[.]timestamp|sie[.]gateway[.]capacity[.]snapshot[.]timestamp|sie[.]gateway[.]settlement[.]confirm[.]duration|sie[.]worker[.]work_item[.]age)$")'
           - 'set(unit, "{epoch}") where IsMatch(name, "^(sie[.]gateway[.]config[.]applied_epoch|sie[.]config[.]epoch|sie[.]worker[.]config[.]epoch)$")'
           - 'set(unit, "{operation}") where IsMatch(name, "^(sie[.]gateway[.]config[.]operations|sie[.]config[.]publish|sie[.]config[.]store[.]writes|sie[.]worker[.]nats[.]operations)$")'
           - 'set(unit, "1") where IsMatch(name, "^(sie[.]gateway[.]config[.]bootstrap[.]degraded|sie[.]gateway[.]messaging[.]client[.]ready|sie[.]config[.]messaging[.]ready|sie[.]worker[.]batch[.]fill_ratio|sie[.]worker[.]config[.]degraded|sie[.]worker[.]saturated|sie[.]gateway[.]pool[.]pinned_model[.]loaded)$")'
@@ -168,6 +168,8 @@ processors:
           - 'keep_keys(attributes, []) where IsMatch(metric.name, "^(sie[.]gateway[.]queue[.]result_chunks[.]received|sie[.]gateway[.]queue[.]result_chunk[.]bytes_received|sie[.]gateway[.]queue[.]result_chunk[.]transfers_completed|sie[.]gateway[.]queue[.]result_chunk[.]duplicates|sie[.]gateway[.]queue[.]result_chunk[.]retry_replacements|sie[.]gateway[.]queue[.]result_chunk[.]stale_retries|sie[.]gateway[.]queue[.]result_chunk[.]reserved_bytes)$")'
           - 'keep_keys(attributes, ["reason"]) where metric.name == "sie.gateway.queue.result_chunk.rejections"'
           - 'keep_keys(attributes, ["operation", "dispatch.path", "outcome", "fallback.reason", "lane"]) where IsMatch(metric.name, "^(sie[.]gateway[.]dispatches|sie[.]gateway[.]dispatch[.]duration)$")'
+          - 'keep_keys(attributes, ["operation", "model", "fallback.reason", "outcome"]) where metric.name == "sie.gateway.remote.fallbacks"'
+          - 'keep_keys(attributes, ["model"]) where metric.name == "sie.gateway.remote.serving.duration"'
           - 'keep_keys(attributes, []) where IsMatch(metric.name, "^(sie[.]gateway[.]config[.]applied_epoch|sie[.]gateway[.]config[.]bootstrap[.]degraded|sie[.]config[.]epoch|sie[.]gateway[.]capacity[.]snapshot[.]timestamp)$")'
           - 'keep_keys(attributes, ["transport"]) where IsMatch(metric.name, "^(sie[.]gateway[.]messaging[.]client[.]ready|sie[.]config[.]messaging[.]ready)$")'
           - 'keep_keys(attributes, ["event", "outcome"]) where metric.name == "sie.gateway.queue.events"'
@@ -287,7 +289,7 @@ processors:
     error_mode: propagate
     metrics:
       metric:
-        - 'not IsMatch(name, "^(sie[.]gateway[.]requests|sie[.]gateway[.]request[.]duration|sie[.]gateway[.]admission[.]decisions|sie[.]gateway[.]dispatches|sie[.]gateway[.]dispatch[.]duration|sie[.]gateway[.]config[.]applied_epoch|sie[.]gateway[.]config[.]operations|sie[.]gateway[.]config[.]bootstrap[.]degraded|sie[.]gateway[.]messaging[.]client[.]ready|sie[.]gateway[.]queue[.]publishes|sie[.]gateway[.]queue[.]publish[.]duration|sie[.]gateway[.]queue[.]publish[.]items|sie[.]gateway[.]queue[.]result_waits|sie[.]gateway[.]queue[.]result_wait[.]duration|sie[.]gateway[.]queue[.]result_chunks[.]received|sie[.]gateway[.]queue[.]result_chunk[.]bytes_received|sie[.]gateway[.]queue[.]result_chunk[.]rejections|sie[.]gateway[.]queue[.]result_chunk[.]transfers_completed|sie[.]gateway[.]queue[.]result_chunk[.]duplicates|sie[.]gateway[.]queue[.]result_chunk[.]retry_replacements|sie[.]gateway[.]queue[.]result_chunk[.]stale_retries|sie[.]gateway[.]queue[.]result_chunk[.]reserved_bytes|sie[.]gateway[.]queue[.]events|sie[.]gateway[.]queue[.]lane_admission[.]decisions|sie[.]gateway[.]provisioning[.]responses|sie[.]gateway[.]routing[.]unsupported_model_exclusions|sie[.]gateway[.]generation[.]events|sie[.]gateway[.]generation[.]ttft|sie[.]gateway[.]generation[.]tpot|sie[.]gateway[.]generation[.]tokens|sie[.]gateway[.]pending_demand|sie[.]gateway[.]lane[.]queue[.]depth|sie[.]gateway[.]lane[.]queue[.]snapshot[.]timestamp|sie[.]gateway[.]active_lease[.]gpus|sie[.]gateway[.]pool[.]warm_floor|sie[.]gateway[.]rejected[.]requests|sie[.]gateway[.]key_snapshot[.]polls|sie[.]gateway[.]settlement[.]confirms|sie[.]gateway[.]settlement[.]confirm[.]duration)$")'
+        - 'not IsMatch(name, "^(sie[.]gateway[.]requests|sie[.]gateway[.]request[.]duration|sie[.]gateway[.]admission[.]decisions|sie[.]gateway[.]dispatches|sie[.]gateway[.]remote[.]fallbacks|sie[.]gateway[.]dispatch[.]duration|sie[.]gateway[.]remote[.]serving[.]duration|sie[.]gateway[.]config[.]applied_epoch|sie[.]gateway[.]config[.]operations|sie[.]gateway[.]config[.]bootstrap[.]degraded|sie[.]gateway[.]messaging[.]client[.]ready|sie[.]gateway[.]queue[.]publishes|sie[.]gateway[.]queue[.]publish[.]duration|sie[.]gateway[.]queue[.]publish[.]items|sie[.]gateway[.]queue[.]result_waits|sie[.]gateway[.]queue[.]result_wait[.]duration|sie[.]gateway[.]queue[.]result_chunks[.]received|sie[.]gateway[.]queue[.]result_chunk[.]bytes_received|sie[.]gateway[.]queue[.]result_chunk[.]rejections|sie[.]gateway[.]queue[.]result_chunk[.]transfers_completed|sie[.]gateway[.]queue[.]result_chunk[.]duplicates|sie[.]gateway[.]queue[.]result_chunk[.]retry_replacements|sie[.]gateway[.]queue[.]result_chunk[.]stale_retries|sie[.]gateway[.]queue[.]result_chunk[.]reserved_bytes|sie[.]gateway[.]queue[.]events|sie[.]gateway[.]queue[.]lane_admission[.]decisions|sie[.]gateway[.]provisioning[.]responses|sie[.]gateway[.]routing[.]unsupported_model_exclusions|sie[.]gateway[.]generation[.]events|sie[.]gateway[.]generation[.]ttft|sie[.]gateway[.]generation[.]tpot|sie[.]gateway[.]generation[.]tokens|sie[.]gateway[.]pending_demand|sie[.]gateway[.]lane[.]queue[.]depth|sie[.]gateway[.]lane[.]queue[.]snapshot[.]timestamp|sie[.]gateway[.]active_lease[.]gpus|sie[.]gateway[.]pool[.]warm_floor|sie[.]gateway[.]rejected[.]requests|sie[.]gateway[.]key_snapshot[.]polls|sie[.]gateway[.]settlement[.]confirms|sie[.]gateway[.]settlement[.]confirm[.]duration)$")'
         - 'resource.attributes["service.name"] != "sie-gateway"'
   filter/remote_application_contract:
     error_mode: propagate

deploy/helm/sie-cluster/templates/prometheusrule.yaml

@@ -1,4 +1,8 @@
 {{- if or (index .Values "kube-prometheus-stack" "install") .Values.alertRules.enabled }}
+{{- $remoteSeconds := .Values.alertRules.remoteFallbackPersistenceSeconds -}}
+{{- if not (and (or (kindIs "int" $remoteSeconds) (kindIs "int64" $remoteSeconds) (kindIs "float64" $remoteSeconds)) (eq (float64 $remoteSeconds) (floor (float64 $remoteSeconds))) (gt (float64 $remoteSeconds) 0.0) (le (float64 $remoteSeconds) 86400.0)) -}}
+{{- fail "alertRules.remoteFallbackPersistenceSeconds must be an integer from 1 to 86400" -}}
+{{- end -}}
 {{- $fullname := include "sie-cluster.fullname" . -}}
 # Application rules below read the collector's Prometheus compatibility
 # exporter. The shared applicationPrometheusEnabled gate therefore also
@@ -13,5 +17,5 @@ metadata:
     app.kubernetes.io/component: alert
     prometheus: kube-prometheus
 spec:
-{{ .Files.Get "files/alerts/sie-rules.yaml" | replace "__NAMESPACE__" (include "sie-cluster.namespace" .) | replace "__FULLNAME__" $fullname | replace "__COLLECTOR_SERVICE__" (printf "%s-otel-collector" $fullname) | indent 2 }}
+{{ .Files.Get "files/alerts/sie-rules.yaml" | replace "__NAMESPACE__" (include "sie-cluster.namespace" .) | replace "__FULLNAME__" $fullname | replace "__COLLECTOR_SERVICE__" (printf "%s-otel-collector" $fullname) | replace "__REMOTE_FALLBACK_PERSISTENCE_SECONDS__" (printf "%d" (int64 $remoteSeconds)) | indent 2 }}
 {{- end }}

deploy/helm/sie-cluster/values.yaml

@@ -1490,6 +1490,9 @@ serviceMonitor:
 # using an external Prometheus Operator.
 alertRules:
   enabled: false
+  # -- Alert after committed remote bridges span this many seconds without
+  # observed local success. Requires recent activity from that gateway replica.
+  remoteFallbackPersistenceSeconds: 600
 
 # -- Grafana dashboard ConfigMaps
 dashboards: