diff --git a/.github/scripts/regenerate_local_alloy_values.py b/.github/scripts/regenerate_local_alloy_values.py index f2037e02d..7330d2aad 100644 --- a/.github/scripts/regenerate_local_alloy_values.py +++ b/.github/scripts/regenerate_local_alloy_values.py @@ -11,11 +11,12 @@ ``templates/alloy-config.yaml`` never renders; and a ``loki.process`` stage inserted after the pod log source. -That last one is not an address rewrite but an added behaviour, and it is the only place it -exists. It promotes ``trace_id`` to Loki structured metadata and relabels ``service_name`` from the -log line, which is what makes Grafana's trace-to-logs query resolve. The chart has no equivalent, -so moving the stage into ``helm/alloy/values.yaml`` is what would extend correlation to deployed -clusters, and this substitution would then be dropped. +Two of the substitutions are not address rewrites but added behaviour that exists nowhere else. +The ``loki.process`` stage promotes ``trace_id`` to Loki structured metadata, which is what lets +Grafana's trace-to-logs query filter on it with no parser stage. The +cAdvisor scrape adds the per-container memory and CPU series the chart's kubelet scrape does not +carry. The chart has no equivalent of either, so moving one into ``helm/alloy/values.yaml`` is what +would extend it to deployed clusters, and its substitution would then be dropped. Run it after any change to the chart's configuration: @@ -77,25 +78,74 @@ " forward_to = [loki.write.endpoint.receiver]\n" "\n" " stage.json {\n" - ' expressions = { trace_id = "trace_id", span_id = "span_id", otel_service = "service" }\n' + ' expressions = { trace_id = "trace_id", span_id = "span_id" }\n' " }\n" "\n" " stage.structured_metadata {\n" ' values = { trace_id = "", span_id = "" }\n' " }\n" "\n" - " // Each line reports the service.name its span was recorded under. Using that as\n" - " // the stream label is what keeps logs joinable to traces: Grafana builds its\n" - " // trace-to-logs query from service.name, while Loki would otherwise derive\n" - " // service_name from the pod's `app` label. Those two spellings differ whenever a\n" - " // service names itself with underscores, and the join then silently finds nothing.\n" - " //\n" - " // Lines logged outside a span extract nothing here and keep Loki's derived value.\n" - " stage.labels {\n" - ' values = { service_name = "otel_service" }\n' - " }\n" " }\n", ), + ( + # cAdvisor, reached over the same node proxy the kubelet scrape uses. The chart scrapes + # the kubelet's own /metrics, which carries no per-container memory or CPU, so nothing + # in a cluster deployed from helm/alloy reports what a pod actually uses. + ' // Cluster Events\n', + ' // cAdvisor. Local-only for now; see docs/otel-logs-and-traces.md.\n' + " //\n" + " // The kubelet serves this on a second path, so it needs its own scrape rather than a\n" + " // longer keep list on the kubelet one.\n" + ' discovery.relabel "cadvisor" {\n' + " targets = discovery.kubernetes.nodes.targets\n" + " rule {\n" + ' target_label = "__address__"\n' + ' replacement = "kubernetes.default.svc.cluster.local:443"\n' + " }\n" + " rule {\n" + ' source_labels = ["__meta_kubernetes_node_name"]\n' + ' regex = "(.+)"\n' + ' replacement = "/api/v1/nodes/${1}/proxy/metrics/cadvisor"\n' + ' target_label = "__metrics_path__"\n' + " }\n" + " }\n" + "\n" + ' prometheus.scrape "cadvisor" {\n' + ' job_name = "integrations/kubernetes/cadvisor"\n' + " targets = discovery.relabel.cadvisor.output\n" + ' scheme = "https"\n' + ' scrape_interval = "60s"\n' + ' bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"\n' + " tls_config {\n" + " insecure_skip_verify = true\n" + " }\n" + " clustering {\n" + " enabled = true\n" + " }\n" + " forward_to = [prometheus.relabel.cadvisor.receiver]\n" + " }\n" + "\n" + ' prometheus.relabel "cadvisor" {\n' + " rule {\n" + ' source_labels = ["__name__"]\n' + ' regex = "container_memory_working_set_bytes|container_memory_rss|container_spec_memory_limit_bytes|container_cpu_usage_seconds_total|container_cpu_cfs_periods_total|container_cpu_cfs_throttled_periods_total|container_spec_cpu_quota|container_spec_cpu_period|container_oom_events_total"\n' + ' action = "keep"\n' + " }\n" + "\n" + " // cAdvisor reports one series per cgroup level. The pod and slice rollups carry an\n" + " // empty container label and sum the containers beneath them, so keeping both counts\n" + " // every container twice.\n" + " rule {\n" + ' source_labels = ["container"]\n' + ' regex = ""\n' + ' action = "drop"\n' + " }\n" + "\n" + " forward_to = [prometheus.relabel.metrics_service.receiver]\n" + " }\n" + "\n" + ' // Cluster Events\n', + ), # Once per write endpoint, hence the repeated pairs. (' cluster = "{{ .Values.cluster }}",\n', ' cluster = "local-kind",\n'), (' project = "{{ .Values.project }}",\n', ' project = "local",\n'), @@ -153,9 +203,10 @@ memory: 256Mi # Copied from helm/alloy/values.yaml, changing the three write endpoints and the two external - # labels, and inserting the loki.process stage that follows the pod log source. That stage is - # local-only: the chart has no equivalent, so trace-to-logs correlation works here and not in a - # cluster deployed from helm/alloy. + # labels, and inserting two things the chart has no equivalent of: the loki.process stage that + # follows the pod log source, and the cAdvisor scrape. Both are local-only, so trace-to-logs + # correlation and per-container memory and CPU work here and not in a cluster deployed from + # helm/alloy. See docs/otel-logs-and-traces.md. # # The labels are written out literally on purpose. templates/alloy-config.yaml renders this # with toYaml rather than tpl, so any {{ }} left in here reaches the ConfigMap unrendered. diff --git a/docs/local-observability.md b/docs/local-observability.md index 74774b93d..d3dc54fce 100644 --- a/docs/local-observability.md +++ b/docs/local-observability.md @@ -25,7 +25,8 @@ Use this when you are developing a service and want to see its own metrics, trac profiles. Do not use it as a model for a deployed cluster: - one replica of everything, no high availability -- `emptyDir` storage, so all data is lost when the pod restarts +- `emptyDir` storage for the telemetry, so metrics, logs, traces and profiles are lost when the + pod restarts; only Grafana's own database persists, on a claim - no ingress, no TLS, no authentication beyond Grafana's default `admin` / `admin` The `observability` chart is the deployed-cluster answer. It is sized for EKS - five Mimir @@ -108,6 +109,21 @@ gen3-embeddings: for `grpc` on 4317, and a mismatched pair fails when the first span is exported rather than at startup. +**Logs have to be JSON**, or trace-to-logs cannot resolve. Alloy's `stage.json` reads `trace_id` +off each line, and `gen3logging`'s text formatter buries it in a `[trace_id=...]` suffix that no +JSON parser sees. `gen3logging` emits text unless `GEN3_JSON_LOGS` is truthy, which it reads +straight from the process environment: + +```yaml +gen3-workflow: + extraEnv: + GEN3_JSON_LOGS: "true" +``` + +The variable is only consulted when a service leaves `get_logger`'s `json_logs` argument as +`None`. A service that passes the value explicitly, usually from a config key of its own, ignores +the environment entirely - so check which of the two governs before setting either. + **Profiles need one address.** A service that ships a Pyroscope SDK likely pushes to `PYROSCOPE_SERVER_ADDRESS`, which is something like: @@ -141,6 +157,112 @@ label filter with no parser stage, which resolves only because the Alloy overlay Every sample Alloy forwards carries `cluster="local-kind"` and `project="local"`, set as external labels in the values file. If a metric has those labels it came through this pipeline. +# Step 5. Load the local observability dashboard + +[examples/dashboards/local-services.json](../examples/dashboards/local-services.json) pulls all four +signals onto one board: CPU and memory per service and per pod, request rate and p95 latency and +recent traces, real per-container memory against each container's limit, the metrics the services +publish themselves, CPU and heap flame graphs, and log volume with a log viewer at the bottom. Load +it with: + +```bash +jq '{dashboard: ., overwrite: true}' examples/dashboards/local-services.json \ + | curl -sS -u admin:admin -H 'Content-Type: application/json' \ + -X POST http://localhost:3000/api/dashboards/db -d @- | jq -r '.status, .url' +``` + +Run that once. Grafana's own volume is the one thing in +[examples/local_lgtm.yaml](../examples/local_lgtm.yaml) backed by a claim rather than an `emptyDir`, +because `GF_PATHS_DATA` points inside it - so imported and hand-edited dashboards survive a pod +restart while metrics, logs, traces and profiles start clean. The stack stays ephemeral where it +would otherwise grow without bound locally. Deleting the +`lgtm-grafana` claim resets Grafana; `kind delete cluster` takes it too. + +`examples/dashboards/` is still the source of truth. A dashboard edited in the UI now sticks in that +volume, which means it can drift from the file - export it back after editing for general improvements +to share. + +## Exporting an edited dashboard back over the file + +Grafana is the easier place to drag panels around; the file is what everyone else gets. After +saving in the UI, here's how to export back to the general file: + +```bash +curl -sS -u admin:admin http://localhost:3000/api/dashboards/uid/gen3-local-services \ + | jq '.dashboard | .id = null | del(.version)' > examples/dashboards/local-services.json +``` + +`id` is Grafana's local row number and `version` changes on every save, so dropping both keeps the +diff to what actually changed and lets the file import cleanly with the command above. Dashboard +settings -> JSON Model does the same thing by hand. + +Every panel carries a description. Hover over for what the series is and what it is not: which +signal it came from, whether the pickers reach it, and the way it misleads. + +Some general info on what's on there: + +**Two views of memory.** The Memory row is Pyroscope's sampled heap, which is +useful for which code holds objects. The Container memory row is +`container_memory_working_set_bytes` against each container's limit, which is the footprint +Kubernetes evicts and OOM-kills on. Expect them to disagree by an order of magnitude, and take the +second for footprint. + +Only the local overlay scrapes cAdvisor, so a cluster deployed from +`helm/alloy` has neither container series - see +[otel-logs-and-traces.md](otel-logs-and-traces.md). The Pyroscope memory panels also stay empty +until `PROFILE_MEMORY` is set (step 3). + +**Two pickers at the top, because the label spellings differ.** Service is populated from Pyroscope's +`service_name`; Log stream from Loki's, which holds both spellings of a service because Alloy falls +back to the pod's `app` label when a line carries no `service` field. Panels keyed by `app` or by +`pod` reach neither picker, and say so in their titles. Making the services register one spelling +is what would collapse this back to a single picker. Log level, search and the display toggles live +in each log viewer's own controls rather than at the top of the board; clicking a field in a line's +details adds it to the Log filters control, which narrows every Loki panel at once. + +The Logs row hides cluster plumbing by default through the Exclude streams box, which holds a regex +rather than a list of services: a new Gen3 service is kept without editing anything, a new kube +component is dropped by the same `kube-.*` it already matches. Add `|name` to hide one more, or +clear the box to see everything the cluster logs. + +**Drilldown starts with the time picker.** No panel overrides the dashboard range, so dragging a +selection across a spike on any graph re-scopes every other panel, log viewers included, to those +minutes. From there, clicking a series on the trace panels opens Explore for that service and +window, and each log viewer has a header link to its own query. + +**Profile queries are capped at a week.** Pyroscope rejects any query longer than its +`max_query_length`, which defaults to 1d, and Grafana reports that only as "An error occurred within +the plugin" on every profile panel at once. `examples/local_lgtm.yaml` disables that check with +`PYROSCOPE_EXTRA_ARGS`, leaving `max-query-lookback` at its 1w default - so a wider dashboard window +returns the last week of profiles instead of failing, while metrics, logs and traces keep the whole +range. A stack already running keeps the old limit until its pod is replaced, and replacing it +discards the telemetry collected so far - the dashboards survive, the five `emptyDir` volumes do +not. + +## Building your own panel from a profile + +A Pyroscope query returns a different frame for each `queryType`, and the panel has to match it: + +| `queryType` | frames returned | panel | +| --- | --- | --- | +| `metrics` | one time series, unit already set to `cores`, `bytes` or `binBps` | Time series, Stat | +| `profile` | one flame-graph frame, no time field | Flame graph | +| `both` | both of the above | Explore only | + +Explore queries with `both` and splits the two frames into a Graph above a Flame graph. A dashboard +panel renders whichever frame it understands and drops the other, so "Add to dashboard" appears to +lose the graph. Keep the queries and split them: a Time series panel set to `metrics`, a Flame graph +panel set to `profile`. + +Leave the panel's unit unset. The datasource stamps a unit on each series, and a unit picked in the +panel options overrides it - which is also why one panel holding both CPU and memory queries can +label neither correctly. One unit per panel. + +`groupBy` is not limited to `service_name`. Pyroscope labels each sample with the `span_name` that +was active when it was taken, so grouping by that attributes CPU to a route - the `CPU by endpoint` +panel. Samples taken outside a span carry no `span_name`, so background work and startup drop out of +any panel grouped that way. + ## When a log line or trace link does not show up Alloy tails every pod's containers by default, so absent logs are usually a write or a label diff --git a/docs/otel-logs-and-traces.md b/docs/otel-logs-and-traces.md index eac53ef6a..574480013 100644 --- a/docs/otel-logs-and-traces.md +++ b/docs/otel-logs-and-traces.md @@ -114,10 +114,12 @@ Two problems follow from that. `trace_id` is only text inside the log message, so a LogQL label filter cannot see it and a query built from a trace id matches nothing. -The service name also disagrees with itself. Grafana builds its trace-to-logs stream selector from -the span's `service.name`, while Loki derives `service_name` from the pod's `app` label when the -label is absent. A span says `service.name = gen3_embeddings` and its logs land on a stream -labelled `service_name = gen3-embeddings`, so the join finds nothing. Underscore against hyphen. +A service that names itself differently from its pod compounds it. Grafana builds its trace-to-logs +stream selector from the span's `service.name`, while Loki derives `service_name` from the pod's +`app` label when the label is absent, so a span saying `service.name = a_service` whose logs land on +`service_name = a-service` joins to nothing. That is a naming bug rather than a pipeline one, and the +fix belongs in the service: an `app` label cannot carry an underscore, because the API server rejects +one in an object name, so the name a service reports to OpenTelemetry has to be the hyphenated one. Insert a processing stage between the source and the write: @@ -131,16 +133,12 @@ loki.process "pod_logs" { forward_to = [loki.write.endpoint.receiver] stage.json { - expressions = { trace_id = "trace_id", span_id = "span_id", otel_service = "service" } + expressions = { trace_id = "trace_id", span_id = "span_id" } } stage.structured_metadata { values = { trace_id = "", span_id = "" } } - - stage.labels { - values = { service_name = "otel_service" } - } } ``` @@ -148,9 +146,10 @@ loki.process "pod_logs" { structured metadata is filterable with `| trace_id = "..."` and no parser stage, while a per-request value used as a stream label would multiply Loki's stream cardinality without bound. -`stage.labels` takes `service_name` from the line's own `service` field, so the label matches the -span by construction. Lines logged outside a span have no `service` value and keep Loki's derived -label, which is why both spellings appear in a healthy cluster. +Nothing rewrites `service_name`: Loki's own derivation from the pod's `app` label is correct once a +service reports that same name to OpenTelemetry, and one stream per service is what you want. A +service still reporting a different name needs fixing there, not here - rewriting the label hides the +divergence while leaving every other signal split. Requires Loki on `tsdb` with schema `v13` or later. `helm/observability/values.yaml` already configures `store: tsdb` with `schema: v13`. @@ -162,7 +161,8 @@ This stage currently ships in one place only: the kind overlay, where `.github/scripts/regenerate_local_alloy_values.py` inserts it while generating `examples/local_alloy_values.yaml`. Neither `cluster-level-resources/values.yaml` nor `helm/alloy/values.yaml` contains it, so correlation is a local-development capability until this -lands in one of them. Anything verified against a kind cluster is exercising the overlay rather +lands in one of them. It is now a smaller change than it was: promoting `trace_id`, with no label +rewriting attached. Anything verified against a kind cluster is exercising the overlay rather than the configuration a cluster runs. **Verify:** `{service_name="gen3_embeddings"} | trace_id != ""` returns lines **with no `| json` @@ -327,6 +327,21 @@ Not required for the ArgoCD path, but it should probably not stay broken. Groupe this; only `tpl` evaluates the template. Note that `tpl` evaluates the whole config, so any future stage using Go-style braces, `stage.template` being the common one, would break. - No `labeldrop`, unlike the ArgoCD config, which makes the label limit in section 1 worse. +- No cAdvisor scrape. The chart scrapes the kubelet's own `/metrics`, which carries operational + counters and no per-container resource usage, so no cluster deployed from this chart reports what + a pod's memory or CPU actually is - `container_memory_working_set_bytes`, + `container_spec_memory_limit_bytes`, `container_cpu_usage_seconds_total`, the CFS throttling + counters, `container_spec_cpu_quota` / `container_spec_cpu_period` and + `container_oom_events_total` are all absent, so neither a container's real footprint nor the + reason it was killed or throttled can be seen, and + the `kube_state_metrics` scrape alongside it targets a `monitoring-extras-kube-state-metrics` + service that this repo never creates. The kubelet serves cAdvisor on a second path + (`/metrics/cadvisor`) reached through the same node proxy and the same service account token, so + this is an added scrape rather than a longer keep list. + [examples/local_alloy_values.yaml](../examples/local_alloy_values.yaml) carries that scrape for + local development only, inserted by `.github/scripts/regenerate_local_alloy_values.py` alongside + the `loki.process` stage below. Moving it into `helm/alloy/values.yaml` is what would give + deployed clusters the same view, and that substitution would then be dropped from the script. **Blocks correlation.** @@ -343,4 +358,5 @@ Not required for the ArgoCD path, but it should probably not stay broken. Groupe Metrics. `common.grafanaAnnotations` already puts `prometheus.io/scrape` and `prometheus.io/path` on every Gen3 pod, `global.metricsEnabled` defaults to true, and Alloy's `annotation_autodiscovery_pods` relabel already resolves those to `podIP:/metrics`. -A service only has to serve the endpoint. \ No newline at end of file +A service only has to serve the endpoint. That covers what a service publishes about itself; what +the container consumes is a separate scrape, in the cAdvisor bullet above. \ No newline at end of file diff --git a/examples/dashboards/local-services.json b/examples/dashboards/local-services.json new file mode 100644 index 000000000..c8621c079 --- /dev/null +++ b/examples/dashboards/local-services.json @@ -0,0 +1,2764 @@ +{ + "id": null, + "uid": "gen3-local-services", + "title": "Local services - overview", + "description": "Resources, requests, traces, service metrics and logs for the local kind cluster, one row per signal. The CPU and Memory rows are profiler-reported, so only services running a profiler appear there and sampled heap is not the pod's resident memory - the Container resources row is.", + "tags": [ + "local", + "observability" + ], + "timezone": "browser", + "editable": true, + "graphTooltip": 1, + "schemaVersion": 42, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "service", + "label": "Service (CPU, memory, traces)", + "type": "query", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "query": { + "refId": "PyroscopeVariableQueryEditor-VariableQuery", + "type": "labelValue", + "labelName": "service_name", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds" + }, + "regex": "/^(?!pyroscope$).*$/", + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "options": [], + "sort": 1, + "hide": 0, + "description": "Pyroscope's own service_name label, read live from the datasource. Drives the At a glance, CPU, Memory, Requests and Profiles rows. It does not reach the Container memory or scraped-metrics panels, which are keyed by pod and by the Kubernetes app label." + }, + { + "name": "logservice", + "label": "Log stream (logs)", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": { + "refId": "LokiVariableQueryEditor-VariableQuery", + "type": 1, + "label": "service_name", + "stream": "" + }, + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".+", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "options": [], + "sort": 1, + "hide": 0, + "description": "Loki's service_name label, which is not the same thing as the Service picker. Alloy sets it from each log line's own `service` field and falls back to the pod's `app` label when a line has none, so one service can appear twice under two spellings, and services that never profile appear here too. Drives the Logs row only." + }, + { + "name": "exclude", + "label": "Exclude streams (logs)", + "type": "textbox", + "query": "alloy|ambassador|coredns|etcd|kindnet|kube-.*|ingress-nginx|integrations/.*|revproxy|http", + "current": { + "text": "alloy|ambassador|coredns|etcd|kindnet|kube-.*|ingress-nginx|integrations/.*|revproxy|http", + "value": "alloy|ambassador|coredns|etcd|kindnet|kube-.*|ingress-nginx|integrations/.*|revproxy|http" + }, + "options": [], + "hide": 0, + "skipUrlSync": false, + "description": "Streams dropped from every Logs panel, as one regex. Matches whole names, so `kube-.*` covers every kube component and a new Gen3 service is kept without editing this. To hide one more service add `|its-name`; to see everything, clear the box. A stream excluded here shows nothing even when picked in Log stream." + }, + { + "name": "logfilter", + "label": "Log filters (logs)", + "type": "adhoc", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "filters": [], + "hide": 0, + "description": "Ad hoc label filters appended to every Loki query on the board. Add them by clicking a field in a log line's details, or with the + here." + } + ] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "At a glance", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + } + }, + { + "id": 2, + "type": "stat", + "title": "CPU now", + "description": "Latest CPU the profiler attributed to each service. 0.02 cores means 2% of one core - CPU-seconds consumed per second of wall clock, the same unit as a Kubernetes CPU request or limit, where 100m is 0.1 cores.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "service_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_name}", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "value_and_name", + "justifyMode": "auto", + "orientation": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + } + }, + { + "id": 3, + "type": "stat", + "title": "Heap in use now", + "description": "Latest sampled heap in use: the bytes the profiler saw allocated and still live. It is not what the kernel charges the container, which also covers the runtime itself, its shared libraries, buffers allocated inside C extensions, and memory the allocator has freed but kept back from the OS - for the Python services profiled here that is CPython's allocator, and a Go service would instead hold the runtime's stacks and spans. Expect an order of magnitude between the two figures. Read the Container memory row for what a limit is enforced against, and this panel for which code is holding objects.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "memory:inuse_space:bytes:space:bytes", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "service_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_name}", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "value_and_name", + "justifyMode": "auto", + "orientation": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + } + }, + { + "id": 4, + "type": "row", + "title": "CPU (profiler samples)", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + } + }, + { + "id": 5, + "type": "timeseries", + "title": "CPU by service", + "description": "CPU as sampled by the profiler. 0.02 cores means 2% of one core - CPU-seconds consumed per second of wall clock, the same unit as a Kubernetes CPU request or limit, where 100m is 0.1 cores. It covers the process the profiler is attached to, so it is not the container's total.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "service_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_name}", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": true, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "CPU by pod", + "description": "One series per replica. 0.02 cores means 2% of one core - CPU-seconds consumed per second of wall clock, the same unit as a Kubernetes CPU request or limit, where 100m is 0.1 cores. Pods that have been replaced keep their series for as long as their profiles sit inside the time range.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "pod" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.pod}", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": true, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "type": "row", + "title": "Memory (profiler samples)", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Allocation rate", + "description": "Bytes allocated per second, including memory freed again immediately. High here with a flat heap in use is churn, not a leak.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 18 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "memory:alloc_space:bytes:space:bytes", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "service_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_name}", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": true, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Heap in use", + "description": "Bytes the profiler sampled as allocated and still live - what the program itself is holding. Read it for which code holds objects and whether that is growing. For what the kernel charges the container, which is what a limit is enforced against, see the Container memory row.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 18 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "memory:inuse_space:bytes:space:bytes", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "service_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_name}", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": true, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Objects in use", + "description": "The count of live sampled allocations, where Heap in use is their bytes. Bytes flat while objects climb means many small objects accumulating; bytes divided by objects gives the average object size.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 18 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "memory:inuse_objects:count:space:bytes", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "service_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_name}", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": true, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 30, + "type": "row", + "title": "Container memory (kernel accounting)", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + } + }, + { + "id": 28, + "type": "timeseries", + "title": "Container memory (working set) by pod", + "description": "Working set as the kernel accounts it: pages charged to the container's cgroup, minus the page cache it could reclaim under pressure. This is the number Kubernetes evicts and OOM-kills on, so it decides whether the pod lives - but it counts memory the process is not using, including the interpreter, shared libraries and anything its allocator has held back from the OS. The profiler rows above count what the program allocated and still holds. Neither is wrong; they answer different questions, and the gap between them is the runtime's overhead. Dashed lines are each container's limit; a container without one has no line, because cAdvisor reports its limit as 0 rather than leaving the series out. Not filtered by the Service picker: these series are keyed by pod and container.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 27 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "range": true, + "editorMode": "code", + "expr": "container_memory_working_set_bytes{container!=\"\", namespace=~\"default|monitoring\"}", + "legendFormat": "{{pod}}" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "range": true, + "editorMode": "code", + "expr": "container_spec_memory_limit_bytes{container!=\"\", namespace=~\"default|monitoring\"} > 0", + "legendFormat": "{{pod}} limit" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".* limit" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 10 + ] + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineWidth", + "value": 1 + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 34, + "type": "timeseries", + "title": "OOM kills", + "description": "Out-of-memory kills counted by the kernel. Flat at zero until something is killed, which is the point: a container that vanished mid-request with nothing in its own logs shows up here and nowhere else. Read it beside the memory table - a pod killed at its limit was at 100% there a moment earlier. Not filtered by the Service picker: these series are keyed by pod and container, which carry neither spelling of the name the picker holds.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 27 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (increase(container_oom_events_total{container!=\"\", namespace=~\"default|monitoring\"}[$__rate_interval]))", + "legendFormat": "{{pod}}", + "editorMode": "code", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 60, + "gradientMode": "opacity", + "showPoints": "never", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "barAlignment": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 29, + "type": "table", + "title": "Memory used vs limit", + "description": "Working set now, the pod's total limit, and the ratio. Pods whose containers declare no memory limit read `none` and `no limit` rather than being dropped or shown at zero - cAdvisor reports their limit as 0, and there is nothing for a percentage to be of. Values are summed over the pod's containers, so a sidecar counts toward both columns. Not filtered by the Service picker: these series are keyed by pod, which carries neither spelling of the name the picker holds.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 36 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (container_memory_working_set_bytes{container!=\"\", namespace=~\"default|monitoring\"})", + "instant": true, + "range": false, + "format": "table", + "editorMode": "code" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (container_spec_memory_limit_bytes{container!=\"\", namespace=~\"default|monitoring\"})", + "instant": true, + "range": false, + "format": "table", + "editorMode": "code" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (container_memory_working_set_bytes{container!=\"\", namespace=~\"default|monitoring\"}) / (sum by (pod) (container_spec_memory_limit_bytes{container!=\"\", namespace=~\"default|monitoring\"}) > 0)", + "instant": true, + "range": false, + "format": "table", + "editorMode": "code" + } + ], + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "pod", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time #A": true, + "Time #B": true, + "Time #C": true + }, + "renameByName": { + "pod": "Pod", + "Value #A": "Used", + "Value #B": "Limit", + "Value #C": "Of limit", + "Value 1": "Used", + "Value 2": "Limit", + "Value 3": "Of limit" + }, + "indexByName": {}, + "includeByName": {} + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Limit" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.width", + "value": 110 + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "0": { + "text": "none", + "index": 0 + } + } + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Of limit" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 1 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "gradient", + "valueDisplayMode": "text" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + } + }, + { + "id": "mappings", + "value": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "no limit", + "index": 0 + } + } + } + ] + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Used", + "desc": true + } + ] + } + }, + { + "id": 40, + "type": "row", + "title": "Container CPU (kernel accounting)", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + } + }, + { + "id": 31, + "type": "timeseries", + "title": "Container CPU by pod", + "description": "CPU the container actually consumed, from cAdvisor - the counterpart to the profiler's CPU row above, and the pair that shows how much CPU the profiler never sees. 0.02 cores means 2% of one core. Not filtered by the Service picker: these series are keyed by pod and container, which carry neither spelling of the name the picker holds.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 48 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{container!=\"\", namespace=~\"default|monitoring\"}[$__rate_interval]))", + "legendFormat": "{{pod}}", + "editorMode": "code", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "never", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "barAlignment": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "cores" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 32, + "type": "timeseries", + "title": "CPU used vs limit", + "description": "Consumption as a share of the container's own CPU limit. Only containers that declare a limit appear; the rest are absent rather than shown at zero, since there is nothing to be a share of. Check this panel before reading CPU throttling: a flat zero there means headroom if a limit exists here, and nothing at all if it does not. Not filtered by the Service picker: these series are keyed by pod and container.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 48 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{container!=\"\", namespace=~\"default|monitoring\"}[$__rate_interval])) / (sum by (pod) (container_spec_cpu_quota{container!=\"\", namespace=~\"default|monitoring\"} / container_spec_cpu_period{container!=\"\", namespace=~\"default|monitoring\"}) > 0)", + "legendFormat": "{{pod}}", + "editorMode": "code", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "never", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "barAlignment": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 33, + "type": "timeseries", + "title": "CPU throttling", + "description": "The share of scheduling periods in which the kernel stopped the container after it spent its quota. Only a container with a CPU limit can throttle, so a flat zero here means either headroom or no limit - check CPU used vs limit to tell which. This is the usual reason a service is slow while its own CPU graph looks unremarkable. Not filtered by the Service picker: these series are keyed by pod and container, which carry neither spelling of the name the picker holds.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 57 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (rate(container_cpu_cfs_throttled_periods_total{container!=\"\", namespace=~\"default|monitoring\"}[$__rate_interval])) / sum by (pod) (rate(container_cpu_cfs_periods_total{container!=\"\", namespace=~\"default|monitoring\"}[$__rate_interval]))", + "legendFormat": "{{pod}}", + "editorMode": "code", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "never", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "barAlignment": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 14, + "type": "row", + "title": "Requests (from traces)", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 64 + } + }, + { + "id": 15, + "type": "timeseries", + "title": "Request rate by endpoint", + "description": "Inbound server spans per second, by route template. Derived by Tempo's metrics generator from the spans it receives, not exported by the service. The series exist only while traces arrive, so a quiet service leaves a gap rather than a zero.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 65 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (service, span_name) (rate(traces_spanmetrics_calls_total{service=~\"${service:pipe}\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))", + "legendFormat": "{{service}} {{span_name}}", + "range": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "title": "Traces for this service (Explore)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22tempo%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22tempo%22%2C%22uid%22%3A%22tempo%22%7D%2C%22queryType%22%3A%22traceql%22%2C%22query%22%3A%22%7Bresource.service.name%3D%5C%22${__field.labels.service}%5C%22%7D%22%2C%22limit%22%3A50%2C%22tableType%22%3A%22traces%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + } + ] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 16, + "type": "timeseries", + "title": "Latency p95 by endpoint", + "description": "95th percentile of inbound server span duration. The marks are exemplars: individual requests carrying their trace ID, so a spike can be opened as the trace that caused it. Derived by Tempo's metrics generator from the spans it receives, not exported by the service, so the series exist only while traces arrive and a quiet service leaves a gap rather than a zero.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 65 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (service, span_name, le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:pipe}\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "legendFormat": "{{service}} {{span_name}}", + "range": true, + "editorMode": "code", + "exemplar": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "title": "Traces for this service (Explore)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22tempo%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22tempo%22%2C%22uid%22%3A%22tempo%22%7D%2C%22queryType%22%3A%22traceql%22%2C%22query%22%3A%22%7Bresource.service.name%3D%5C%22${__field.labels.service}%5C%22%7D%22%2C%22limit%22%3A50%2C%22tableType%22%3A%22traces%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + } + ] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 36, + "type": "heatmap", + "title": "Latency heatmap", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The full distribution of inbound span duration, not just its 95th percentile: two bands rather than one is the shape a p95 line hides, typically a fast path and a slow one. The magenta marks are exemplars - each one is a real request that landed in that cell, carrying its trace ID, so a hot spot can be opened as a trace rather than guessed at. Hovering names the service and span; clicking follows it to Tempo. Buckets and exemplars both come from Tempo's metrics generator, so they exist only while traces arrive.", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 74 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:pipe}\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))", + "legendFormat": "{{le}}", + "editorMode": "code", + "range": true, + "instant": false, + "format": "heatmap", + "exemplar": true + } + ], + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "short" + }, + "color": { + "mode": "scheme", + "scheme": "Spectral", + "steps": 64, + "reverse": false, + "exponent": 0.5, + "fill": "dark-orange" + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + } + }, + { + "id": 37, + "type": "table", + "title": "Slowest endpoints", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "p95 per route across the whole time range shown, ranked, so the worst endpoint names itself instead of being picked out of a crowded legend. It reads the range rather than a rate interval on purpose: a p95 over the last few minutes is empty whenever traffic is not arriving right now. Routes with no samples in the range are dropped rather than listed blank. Server spans only.", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 74 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, histogram_quantile(0.95, sum by (service, span_name, le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:pipe}\", span_kind=\"SPAN_KIND_SERVER\"}[$__range]))) > 0)", + "legendFormat": "", + "editorMode": "code", + "range": false, + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "service": "Service", + "span_name": "Endpoint", + "Value": "p95", + "Value #A": "p95" + }, + "indexByName": {}, + "includeByName": {} + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.width", + "value": 110 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "p95", + "desc": true + } + ] + } + }, + { + "id": 17, + "type": "timeseries", + "title": "Error rate by span", + "description": "Spans whose status is ERROR, of every kind - not just endpoints. A server span is a request that failed, a client span is a call this service made that failed, and an internal span is a step inside it, which is how a failure that propagates up a call chain shows as several series at once. Empty until a span actually carries that status: a handled 500 that never marks its span is invisible here. Derived by Tempo's metrics generator from the spans it receives, not exported by the service, so the series exist only while traces arrive.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 83 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (service, span_kind, span_name) (rate(traces_spanmetrics_calls_total{service=~\"${service:pipe}\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval]))", + "legendFormat": "{{service}} {{span_kind}} {{span_name}}", + "range": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "title": "Error traces for this service (Explore)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22tempo%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22tempo%22%2C%22uid%22%3A%22tempo%22%7D%2C%22queryType%22%3A%22traceql%22%2C%22query%22%3A%22%7Bresource.service.name%3D%5C%22${__field.labels.service}%5C%22%20%26%26%20status%3Derror%7D%22%2C%22limit%22%3A50%2C%22tableType%22%3A%22traces%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + }, + { + "title": "Error logs for this service (Explore)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22loki%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22loki%22%2C%22uid%22%3A%22loki%22%7D%2C%22expr%22%3A%22%7Bservice_name%3D~%5C%22${__field.labels.service}%5C%22%7D%20%7C%20detected_level%20%3D~%20%5C%22error%7Cwarn%5C%22%22%2C%22queryType%22%3A%22range%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + } + ] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 18, + "type": "table", + "title": "Recent traces", + "description": "Newest traces for the selected services. Click a Trace ID to open it in Tempo; its logs link resolves through Loki's trace_id metadata.", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 90 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceql", + "query": "{resource.service.name=~\"${service:pipe}\"}", + "limit": 20, + "tableType": "traces", + "filters": [] + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + } + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "sortBy": [ + { + "displayName": "Start time", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + } + }, + { + "id": 38, + "type": "nodeGraph", + "title": "Service graph", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "description": "Who calls whom, derived by Tempo from span pairs rather than declared anywhere. Nodes are not all services: a datastore appears as its own node, and `user` is a request that arrived with no parent span, meaning anything entering from outside. Grafana draws those two with a dashed outline because Tempo marks them un-instrumented, which is the only labelling the panel supports - the Service dependencies table beside it names each edge in words. Edges carry request rate and latency, so an N+1 query pattern shows here before it shows in CPU. Needs traffic, and ignores both pickers.", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 99 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "serviceMap" + } + ], + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 41, + "type": "table", + "title": "Service dependencies", + "description": "The same edges the graph draws, named. Kind is Tempo's `connection_type`: a blank one is service to service, `database` is a datastore the client queried, and `virtual_node` is not a caller at all but a request that arrived with no parent span - anything entering from outside the traced system. The graph cannot say this in text: its node subtitle is reserved for the service namespace, and its only marker for a non-service node is a dashed outline. Database names come from the span's database attributes, which for these charts is the `_` name the chart generates, so a service and its own database can appear under two different spellings.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 99 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "instant": true, + "range": false, + "format": "table", + "expr": "sum by (client, server, connection_type) (rate(traces_service_graph_request_total[$__range]))" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "client": "Calls from", + "server": "Calls to", + "connection_type": "Kind", + "Value": "Rate", + "Value #A": "Rate" + }, + "indexByName": {}, + "includeByName": {} + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Kind" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "": { + "text": "service to service", + "index": 0 + }, + "database": { + "text": "database query", + "index": 1 + }, + "virtual_node": { + "text": "entering from outside", + "index": 2 + }, + "messaging_system": { + "text": "queue", + "index": 3 + } + } + } + ] + }, + { + "id": "custom.width", + "value": 190 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Rate" + }, + "properties": [ + { + "id": "unit", + "value": "reqps" + }, + { + "id": "custom.width", + "value": 110 + }, + { + "id": "decimals", + "value": 4 + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Rate", + "desc": true + } + ] + } + }, + { + "id": 19, + "type": "row", + "title": "Service metrics (scraped from /metrics)", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 108 + } + }, + { + "id": 20, + "type": "timeseries", + "title": "Request rate by status (all services)", + "description": "Counted by the service itself through cdispyutils request_metrics and read from its /metrics, so it is independent of the trace pipeline and survives a service that never exports a span. Selected by metric name (`.+_api_requests_total`) rather than by service, so anything using that helper appears without editing this dashboard. That panel ignores the Service picker, which is why the title says so: its series carry `app`, from Kubernetes. The Kubernetes `app` label and the name a service registers with the SDK need not match - hyphen against underscore is the usual difference - and no variable format converts one into the other.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 109 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (app, status_code) (rate({__name__=~\".+_api_requests_total\"}[$__rate_interval]))", + "legendFormat": "{{app}} {{status_code}}", + "range": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "type": "row", + "title": "Profiles ($service)", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 118 + } + }, + { + "id": 35, + "type": "timeseries", + "title": "CPU by endpoint", + "description": "CPU attributed to the route that was executing when each sample was taken, which is the one thing the flame graphs below cannot tell you at a glance: they merge every request together. Only samples taken inside a span carry span_name, so background work and startup are absent here while still appearing in CPU by service.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 119 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "labelSelector": "{service_name=~\"${service:pipe}\", span_name!=\"\"}", + "groupBy": [ + "span_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "never", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "barAlignment": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.span_name}" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 12, + "type": "flamegraph", + "title": "CPU profile", + "description": "Merged across every service selected in the Service picker. Pick a single service to read it. The totals are summed over the dashboard's time range, so they grow as the range widens rather than describing a moment.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 127 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "profile", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [] + } + ], + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 13, + "type": "flamegraph", + "title": "Heap in use profile", + "description": "Merged across every service selected in the Service picker. Pick a single service to read it. The totals are summed over the dashboard's time range, so they grow as the range widens rather than describing a moment.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 127 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "profile", + "profileTypeId": "memory:inuse_space:bytes:space:bytes", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [] + } + ], + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 23, + "type": "row", + "title": "Logs", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 140 + } + }, + { + "id": 24, + "type": "timeseries", + "title": "Log volume by service", + "description": "Lines per second per stream, driven by the Log stream picker rather than by Service. Alloy takes `service_name` from each line's own `service` field and falls back to the pod's `app` label when a line has none, so one service can own two streams under two spellings, and services that never profile appear here too. The Kubernetes `app` label and the name a service registers with the SDK need not match - hyphen against underscore is the usual difference - and no variable format converts one into the other.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 141 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (service_name) (rate({service_name=~\"${logservice:pipe}\", service_name!~\"$exclude\"}[$__auto]))", + "legendFormat": "{{service_name}}", + "queryType": "range", + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cps", + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 60, + "gradientMode": "none", + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "barAlignment": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "normal", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 25, + "type": "timeseries", + "title": "Log volume by level", + "description": "`detected_level` is inferred by Loki per line, not a label the service sets, so `unknown` means it could not find one rather than that the line was unimportant.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 141 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (detected_level) (rate({service_name=~\"${logservice:pipe}\", service_name!~\"$exclude\"}[$__auto]))", + "legendFormat": "{{detected_level}}", + "queryType": "range", + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cps", + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 60, + "gradientMode": "none", + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "barAlignment": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "normal", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "info" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "debug" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 39, + "type": "timeseries", + "title": "Error rate by service", + "description": "Error lines per second per service, which for much of this cluster is the only error signal there is: a service that serves no metrics and exports no spans still logs. Driven by the Log stream and Exclude controls, and by the level Loki inferred - a service logging severity in a shape Loki cannot parse is invisible here.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 141 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "editorMode": "code", + "legendFormat": "{{service_name}}", + "expr": "sum by (service_name) (rate({service_name=~\"${logservice:pipe}\", service_name!~\"$exclude\"} | detected_level =~ \"error\" [$__auto]))" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 60, + "gradientMode": "opacity", + "showPoints": "never", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "barAlignment": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "normal", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "cps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 27, + "type": "logs", + "title": "Recent logs", + "description": "Oldest first, so the newest line is at the bottom and the panel opens scrolled to it. Each line is prefixed with its service. The prefix is added by the query with `line_format`, because the panel's displayed-fields option replaces the log line rather than prefixing it - so a copied line carries the prefix too, and Prettify JSON no longer applies to lines that were pure JSON. Level filtering, search, wrapping and sort live in the panel's own controls on the right. Clicking a field in a line's details adds it to the Log filters control at the top, which narrows every Loki panel. Lines carrying `trace_id` link to the trace in Tempo.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 150 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service_name=~\"${logservice:pipe}\", service_name!~\"$exclude\"} | line_format \"{{ printf \\\"%-16s\\\" .service_name }} {{ __line__ }}\"", + "queryType": "range", + "maxLines": 500, + "editorMode": "code" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Ascending", + "fontSize": "small", + "syntaxHighlighting": true, + "showLevel": true, + "showControls": true, + "controlsStorageKey": "gen3-local-recent" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "links": [ + { + "title": "Open in Explore (Loki)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22loki%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22loki%22%2C%22uid%22%3A%22loki%22%7D%2C%22expr%22%3A%22%7Bservice_name%3D~%5C%22${logservice:pipe}%5C%22%7D%22%2C%22queryType%22%3A%22range%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + } + ] + }, + { + "id": 26, + "type": "logs", + "title": "Errors and warnings", + "description": "The same lines and the same service prefix, filtered to the level Loki inferred - so a service logging its own severity in a format Loki cannot read is missing here even while it fails. Search and the display toggles are in the panel's controls; the level filter there narrows this further rather than widening it, since the query is already fixed to error and warn.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 162 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service_name=~\"${logservice:pipe}\", service_name!~\"$exclude\"} | detected_level =~ \"error|warn\" | line_format \"{{ printf \\\"%-16s\\\" .service_name }} {{ __line__ }}\"", + "queryType": "range", + "maxLines": 500, + "editorMode": "code" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Ascending", + "fontSize": "small", + "syntaxHighlighting": true, + "showLevel": true, + "showControls": true, + "controlsStorageKey": "gen3-local-errors" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "links": [ + { + "title": "Open in Explore (Loki)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22loki%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22loki%22%2C%22uid%22%3A%22loki%22%7D%2C%22expr%22%3A%22%7Bservice_name%3D~%5C%22${logservice:pipe}%5C%22%7D%20%7C%20detected_level%20%3D~%20%5C%22error%7Cwarn%5C%22%22%2C%22queryType%22%3A%22range%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + } + ] + } + ] +} diff --git a/examples/local_alloy_values.yaml b/examples/local_alloy_values.yaml index 767a0c3f8..16bb6fb96 100644 --- a/examples/local_alloy_values.yaml +++ b/examples/local_alloy_values.yaml @@ -47,9 +47,10 @@ alloy: memory: 256Mi # Copied from helm/alloy/values.yaml, changing the three write endpoints and the two external - # labels, and inserting the loki.process stage that follows the pod log source. That stage is - # local-only: the chart has no equivalent, so trace-to-logs correlation works here and not in a - # cluster deployed from helm/alloy. + # labels, and inserting two things the chart has no equivalent of: the loki.process stage that + # follows the pod log source, and the cAdvisor scrape. Both are local-only, so trace-to-logs + # correlation and per-container memory and CPU work here and not in a cluster deployed from + # helm/alloy. See docs/otel-logs-and-traces.md. # # The labels are written out literally on purpose. templates/alloy-config.yaml renders this # with toYaml rather than tpl, so any {{ }} left in here reaches the ConfigMap unrendered. @@ -374,23 +375,13 @@ alloy: forward_to = [loki.write.endpoint.receiver] stage.json { - expressions = { trace_id = "trace_id", span_id = "span_id", otel_service = "service" } + expressions = { trace_id = "trace_id", span_id = "span_id" } } stage.structured_metadata { values = { trace_id = "", span_id = "" } } - // Each line reports the service.name its span was recorded under. Using that as - // the stream label is what keeps logs joinable to traces: Grafana builds its - // trace-to-logs query from service.name, while Loki would otherwise derive - // service_name from the pod's `app` label. Those two spellings differ whenever a - // service names itself with underscores, and the join then silently finds nothing. - // - // Lines logged outside a span extract nothing here and keep Loki's derived value. - stage.labels { - values = { service_name = "otel_service" } - } } // kube-state-metrics @@ -454,6 +445,58 @@ alloy: forward_to = [prometheus.relabel.metrics_service.receiver] } + // cAdvisor. Local-only for now; see docs/otel-logs-and-traces.md. + // + // The kubelet serves this on a second path, so it needs its own scrape rather than a + // longer keep list on the kubelet one. + discovery.relabel "cadvisor" { + targets = discovery.kubernetes.nodes.targets + rule { + target_label = "__address__" + replacement = "kubernetes.default.svc.cluster.local:443" + } + rule { + source_labels = ["__meta_kubernetes_node_name"] + regex = "(.+)" + replacement = "/api/v1/nodes/${1}/proxy/metrics/cadvisor" + target_label = "__metrics_path__" + } + } + + prometheus.scrape "cadvisor" { + job_name = "integrations/kubernetes/cadvisor" + targets = discovery.relabel.cadvisor.output + scheme = "https" + scrape_interval = "60s" + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + tls_config { + insecure_skip_verify = true + } + clustering { + enabled = true + } + forward_to = [prometheus.relabel.cadvisor.receiver] + } + + prometheus.relabel "cadvisor" { + rule { + source_labels = ["__name__"] + regex = "container_memory_working_set_bytes|container_memory_rss|container_spec_memory_limit_bytes|container_cpu_usage_seconds_total|container_cpu_cfs_periods_total|container_cpu_cfs_throttled_periods_total|container_spec_cpu_quota|container_spec_cpu_period|container_oom_events_total" + action = "keep" + } + + // cAdvisor reports one series per cgroup level. The pod and slice rollups carry an + // empty container label and sum the containers beneath them, so keeping both counts + // every container twice. + rule { + source_labels = ["container"] + regex = "" + action = "drop" + } + + forward_to = [prometheus.relabel.metrics_service.receiver] + } + // Cluster Events loki.source.kubernetes_events "cluster_events" { job_name = "integrations/kubernetes/eventhandler" diff --git a/examples/local_lgtm.yaml b/examples/local_lgtm.yaml index a8df42ec6..a89229217 100644 --- a/examples/local_lgtm.yaml +++ b/examples/local_lgtm.yaml @@ -8,7 +8,8 @@ # services can push profiles. Both already listen inside the image; a Service that omits the port # drops the traffic rather than refusing it. # -# For local development only. One replica, emptyDir storage so everything is lost on restart, +# For local development only. One replica, emptyDir storage for the telemetry so it is lost on +# restart while dashboards persist on a small claim, # no ingress, no TLS, and Grafana's default admin/admin. The observability chart is the answer # for a deployed cluster. --- @@ -17,6 +18,23 @@ kind: Namespace metadata: name: monitoring --- +# Dashboards outlive the pod; telemetry does not. kind's default local-path StorageClass keeps +# this on the node, so it survives a pod restart and a `docker restart`, and goes away with +# `kind delete cluster` or by deleting the claim - which is how you reset Grafana. +# +# examples/dashboards/ stays the source of truth. Editing a dashboard in the UI now sticks, so +# export it back to the file or the next fresh cluster will not have the change. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: lgtm-grafana + namespace: monitoring +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -45,6 +63,13 @@ spec: # of each pod's log rather than from now. - name: LOKI_EXTRA_ARGS value: "-validation.max-label-names-per-series=32 -validation.reject-old-samples=false" + # Pyroscope refuses any query longer than its default max_query_length of 1d with + # "the query time range exceeds the limit", which in Grafana surfaces only as + # "An error occurred within the plugin" on every profile panel at once. 0 disables + # the check. max-query-lookback stays at its 1w default, so a longer window returns + # the last week rather than failing. + - name: PYROSCOPE_EXTRA_ARGS + value: "-querier.max-query-length=0" ports: - { name: grafana, containerPort: 3000 } - { name: loki, containerPort: 3100 } @@ -71,7 +96,12 @@ spec: - { name: pyroscope-storage, mountPath: /data/pyroscope } volumes: - { name: tempo-data, emptyDir: {} } - - { name: grafana-data, emptyDir: {} } + # Only Grafana's own volume persists: GF_PATHS_DATA is /data/grafana/data, so + # this is the dashboard database and the plugin directory and nothing else. + # Metrics, logs, traces and profiles stay on emptyDir and start clean. + - name: grafana-data + persistentVolumeClaim: + claimName: lgtm-grafana - { name: loki-data, emptyDir: {} } - { name: loki-storage, emptyDir: {} } - { name: p8s-storage, emptyDir: {} } diff --git a/helm/arborist/templates/db-init.yaml b/helm/arborist/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/arborist/templates/db-init.yaml +++ b/helm/arborist/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/argo-wrapper/README.md b/helm/argo-wrapper/README.md index 4a238d390..b9ea266df 100644 --- a/helm/argo-wrapper/README.md +++ b/helm/argo-wrapper/README.md @@ -29,6 +29,7 @@ Published versions of this chart are listed in the | commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | | criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | | environment | string | `"default"` | Environment name. | +| global.argoHost | string | `"http://argo-argo-workflows-server.argo.svc.cluster.local:2746"` | argo server host URL:PORT. Endpoint for the the Argo Workflows service. | | global.autoscaling.averageCPUValue | string | `"500m"` | | | global.autoscaling.averageMemoryValue | string | `"500Mi"` | | | global.autoscaling.enabled | bool | `false` | | diff --git a/helm/argo-wrapper/templates/deployment.yaml b/helm/argo-wrapper/templates/deployment.yaml index ab8c70233..cd45f46d9 100644 --- a/helm/argo-wrapper/templates/deployment.yaml +++ b/helm/argo-wrapper/templates/deployment.yaml @@ -76,4 +76,6 @@ spec: {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - env: \ No newline at end of file + env: + - name: ARGO_HOST + value: {{ .Values.global.argoHost }} diff --git a/helm/argo-wrapper/values.yaml b/helm/argo-wrapper/values.yaml index 8837da919..1153cf32d 100644 --- a/helm/argo-wrapper/values.yaml +++ b/helm/argo-wrapper/values.yaml @@ -4,6 +4,8 @@ # Global configuration global: + # -- (string) argo server host URL:PORT. Endpoint for the the Argo Workflows service. + argoHost: http://argo-argo-workflows-server.argo.svc.cluster.local:2746 # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. environment: default # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. diff --git a/helm/audit/templates/db-init.yaml b/helm/audit/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/audit/templates/db-init.yaml +++ b/helm/audit/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/cluster-level-resources/README.md b/helm/cluster-level-resources/README.md index c194c0699..89d508219 100644 --- a/helm/cluster-level-resources/README.md +++ b/helm/cluster-level-resources/README.md @@ -85,6 +85,9 @@ Published versions of this chart are listed in the | karpenter-crds.default.disruptionBudget | string | `"10%"` | | | karpenter-crds.default.enabled | bool | `true` | | | karpenter-crds.default.expireAfter | string | `"168h"` | | +| karpenter-crds.default.fastImagePull.enabled | bool | `false` | | +| karpenter-crds.default.fastImagePull.iops | int | `3000` | | +| karpenter-crds.default.fastImagePull.throughput | int | `600` | | | karpenter-crds.default.limits.cpu | string | `"1000"` | | | karpenter-crds.default.limits.memory | string | `"1000Gi"` | | | karpenter-crds.default.requirements[0].key | string | `"karpenter.sh/capacity-type"` | | @@ -109,6 +112,10 @@ Published versions of this chart are listed in the | karpenter-crds.gpu.consolidationPolicy | string | `"WhenEmpty"` | | | karpenter-crds.gpu.enabled | bool | `false` | | | karpenter-crds.gpu.expireAfter | string | `"168h"` | | +| karpenter-crds.gpu.fastImagePull.enabled | bool | `false` | | +| karpenter-crds.gpu.fastImagePull.iops | int | `3000` | | +| karpenter-crds.gpu.fastImagePull.throughput | int | `600` | | +| karpenter-crds.gpu.instanceTypes | list | `[]` | Specific EC2 instance types for the gpu NodePools (e.g. ["g6.xlarge"]). If empty, defaults to the g4dn instance family. | | karpenter-crds.gpu.volumeSize | string | `"50Gi"` | | | karpenter-crds.jupyter.additionalTags | object | `{}` | | | karpenter-crds.jupyter.consolidateAfter | string | `"30s"` | | @@ -116,6 +123,9 @@ Published versions of this chart are listed in the | karpenter-crds.jupyter.consolidationPolicy | string | `"WhenEmpty"` | | | karpenter-crds.jupyter.enabled | bool | `true` | | | karpenter-crds.jupyter.expireAfter | string | `"168h"` | | +| karpenter-crds.jupyter.fastImagePull.enabled | bool | `false` | | +| karpenter-crds.jupyter.fastImagePull.iops | int | `3000` | | +| karpenter-crds.jupyter.fastImagePull.throughput | int | `600` | | | karpenter-crds.jupyter.requirements[0].key | string | `"karpenter.sh/capacity-type"` | | | karpenter-crds.jupyter.requirements[0].operator | string | `"In"` | | | karpenter-crds.jupyter.requirements[0].values[0] | string | `"on-demand"` | | @@ -137,6 +147,9 @@ Published versions of this chart are listed in the | karpenter-crds.secondary.disruptionBudget | string | `"10%"` | | | karpenter-crds.secondary.enabled | bool | `false` | | | karpenter-crds.secondary.expireAfter | string | `"168h"` | | +| karpenter-crds.secondary.fastImagePull.enabled | bool | `false` | | +| karpenter-crds.secondary.fastImagePull.iops | int | `3000` | | +| karpenter-crds.secondary.fastImagePull.throughput | int | `600` | | | karpenter-crds.secondary.requirements[0].key | string | `"karpenter.sh/capacity-type"` | | | karpenter-crds.secondary.requirements[0].operator | string | `"In"` | | | karpenter-crds.secondary.requirements[0].values[0] | string | `"on-demand"` | | @@ -161,6 +174,9 @@ Published versions of this chart are listed in the | karpenter-crds.workflow.consolidationPolicy | string | `"WhenEmpty"` | | | karpenter-crds.workflow.enabled | bool | `true` | | | karpenter-crds.workflow.expireAfter | string | `"168h"` | | +| karpenter-crds.workflow.fastImagePull.enabled | bool | `false` | | +| karpenter-crds.workflow.fastImagePull.iops | int | `3000` | | +| karpenter-crds.workflow.fastImagePull.throughput | int | `600` | | | karpenter-crds.workflow.requirements[0].key | string | `"karpenter.sh/capacity-type"` | | | karpenter-crds.workflow.requirements[0].operator | string | `"In"` | | | karpenter-crds.workflow.requirements[0].values[0] | string | `"on-demand"` | | @@ -201,6 +217,17 @@ Published versions of this chart are listed in the | nvidia-device-plugin.enabled | bool | `false` | | | nvidia-device-plugin.targetRevision | string | `"v0.18.0"` | | | project | string | `"unfunded"` | | +| storageclass.allowVolumeExpansion | bool | `true` | | +| storageclass.default | bool | `true` | | +| storageclass.enabled | bool | `false` | | +| storageclass.name | string | `"gp3"` | | +| storageclass.parameters.encrypted | string | `"true"` | | +| storageclass.parameters.type | string | `"gp3"` | | +| storageclass.previousDefault.name | string | `"gp2"` | | +| storageclass.previousDefault.removeDefault | bool | `true` | | +| storageclass.provisioner | string | `"ebs.csi.aws.com"` | | +| storageclass.reclaimPolicy | string | `"Retain"` | | +| storageclass.volumeBindingMode | string | `"WaitForFirstConsumer"` | | | vpc-cni.configuration.enabled | bool | `false` | | | vpc-cni.enabled | bool | `false` | | | vpc-cni.targetRevision | string | `"v1.20.4"` | | diff --git a/helm/cluster-level-resources/templates/karpenter-config-resources-default.yaml b/helm/cluster-level-resources/templates/karpenter-config-resources-default.yaml index df01aa1dc..4876cce02 100644 --- a/helm/cluster-level-resources/templates/karpenter-config-resources-default.yaml +++ b/helm/cluster-level-resources/templates/karpenter-config-resources-default.yaml @@ -28,6 +28,10 @@ spec: encrypted: true volumeSize: {{ index .Values "karpenter-crds" "default" "volumeSize" }} volumeType: gp3 +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "default" "fastImagePull" "enabled") }} + iops: {{ index .Values "karpenter-crds" "default" "fastImagePull" "iops" }} + throughput: {{ index .Values "karpenter-crds" "default" "fastImagePull" "throughput" }} +{{ end }} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: disabled @@ -62,6 +66,18 @@ spec: sysctl -w fs.inotify.max_user_watches=12000 & systemctl stop firewalld systemctl disable firewalld +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "default" "fastImagePull" "enabled") }} + + --BOUNDARY + Content-Type: application/node.eks.aws + + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + featureGates: + FastImagePull: true +{{ end }} --BOUNDARY-- {{- if semverCompare ">=1.0.8" .Values.karpenter.targetRevision }} diff --git a/helm/cluster-level-resources/templates/karpenter-config-resources-gpu.yaml b/helm/cluster-level-resources/templates/karpenter-config-resources-gpu.yaml index dacd6a97b..3860e2dd7 100644 --- a/helm/cluster-level-resources/templates/karpenter-config-resources-gpu.yaml +++ b/helm/cluster-level-resources/templates/karpenter-config-resources-gpu.yaml @@ -19,6 +19,10 @@ spec: encrypted: true volumeSize: {{ index .Values "karpenter-crds" "gpu" "volumeSize" }} volumeType: gp3 +{{ if (index .Values "karpenter-crds" "gpu" "fastImagePull" "enabled") }} + iops: {{ index .Values "karpenter-crds" "gpu" "fastImagePull" "iops" }} + throughput: {{ index .Values "karpenter-crds" "gpu" "fastImagePull" "throughput" }} +{{ end }} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: disabled @@ -53,7 +57,19 @@ spec: curl https://raw.githubusercontent.com/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys & sysctl -w fs.inotify.max_user_watches=12000 & systemctl stop firewalld - systemctl disable firewalld + systemctl disable firewalld +{{ if (index .Values "karpenter-crds" "gpu" "fastImagePull" "enabled") }} + + --BOUNDARY + Content-Type: application/node.eks.aws + + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + featureGates: + FastImagePull: true +{{ end }} --BOUNDARY-- {{- if semverCompare ">=1.0.8" .Values.karpenter.targetRevision }} @@ -92,9 +108,15 @@ spec: value: "present" effect: "NoSchedule" requirements: + {{- with (index .Values "karpenter-crds" "gpu" "instanceTypes") }} + - key: node.kubernetes.io/instance-type + operator: In + values: {{ toJson . }} + {{- else }} - key: karpenter.k8s.aws/instance-family operator: In values: ["g4dn"] + {{- end }} - key: karpenter.sh/capacity-type operator: In values: ["on-demand"] @@ -150,9 +172,15 @@ spec: value: "present" effect: "NoSchedule" requirements: + {{- with (index .Values "karpenter-crds" "gpu" "instanceTypes") }} + - key: node.kubernetes.io/instance-type + operator: In + values: {{ toJson . }} + {{- else }} - key: karpenter.k8s.aws/instance-family operator: In values: ["g4dn"] + {{- end }} - key: karpenter.sh/capacity-type operator: In values: ["on-demand"] @@ -187,9 +215,15 @@ spec: value: "present" effect: "NoSchedule" requirements: + {{- with (index .Values "karpenter-crds" "gpu" "instanceTypes") }} + - key: node.kubernetes.io/instance-type + operator: In + values: {{ toJson . }} + {{- else }} - key: karpenter.k8s.aws/instance-family operator: In values: ["g4dn"] + {{- end }} - key: karpenter.sh/capacity-type operator: In values: ["spot","on-demand"] @@ -250,9 +284,15 @@ spec: value: "present" effect: "NoSchedule" requirements: + {{- with (index .Values "karpenter-crds" "gpu" "instanceTypes") }} + - key: node.kubernetes.io/instance-type + operator: In + values: {{ toJson . }} + {{- else }} - key: karpenter.k8s.aws/instance-family operator: In values: ["g4dn"] + {{- end }} - key: karpenter.sh/capacity-type operator: In values: ["spot","on-demand"] diff --git a/helm/cluster-level-resources/templates/karpenter-config-resources-jupyter.yaml b/helm/cluster-level-resources/templates/karpenter-config-resources-jupyter.yaml index 21186d37f..bf4538d17 100644 --- a/helm/cluster-level-resources/templates/karpenter-config-resources-jupyter.yaml +++ b/helm/cluster-level-resources/templates/karpenter-config-resources-jupyter.yaml @@ -19,7 +19,7 @@ spec: owner: {{ index .Values "karpenter-crds" "amiOwner" | quote }} {{- else }} - alias: {{ index .Values "karpenter-crds" "alias" | quote }} - {{- end }} + {{- end }} blockDeviceMappings: - deviceName: /dev/xvda ebs: @@ -27,6 +27,10 @@ spec: encrypted: true volumeSize: {{ index .Values "karpenter-crds" "jupyter" "volumeSize" }} volumeType: gp3 +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "jupyter" "fastImagePull" "enabled") }} + iops: {{ index .Values "karpenter-crds" "jupyter" "fastImagePull" "iops" }} + throughput: {{ index .Values "karpenter-crds" "jupyter" "fastImagePull" "throughput" }} +{{ end }} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: disabled @@ -61,6 +65,18 @@ spec: sysctl -w fs.inotify.max_user_watches=12000 & systemctl stop firewalld systemctl disable firewalld +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "jupyter" "fastImagePull" "enabled") }} + + --BOUNDARY + Content-Type: application/node.eks.aws + + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + featureGates: + FastImagePull: true +{{ end }} --BOUNDARY-- {{- if semverCompare ">=1.0.8" .Values.karpenter.targetRevision }} diff --git a/helm/cluster-level-resources/templates/karpenter-config-resources-secondary.yaml b/helm/cluster-level-resources/templates/karpenter-config-resources-secondary.yaml index f1a295274..540e2abb6 100644 --- a/helm/cluster-level-resources/templates/karpenter-config-resources-secondary.yaml +++ b/helm/cluster-level-resources/templates/karpenter-config-resources-secondary.yaml @@ -28,6 +28,10 @@ spec: encrypted: true volumeSize: {{ index .Values "karpenter-crds" "secondary" "volumeSize" }} volumeType: gp3 +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "secondary" "fastImagePull" "enabled") }} + iops: {{ index .Values "karpenter-crds" "secondary" "fastImagePull" "iops" }} + throughput: {{ index .Values "karpenter-crds" "secondary" "fastImagePull" "throughput" }} +{{ end }} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: disabled @@ -62,6 +66,18 @@ spec: sysctl -w fs.inotify.max_user_watches=12000 & systemctl stop firewalld systemctl disable firewalld +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "secondary" "fastImagePull" "enabled") }} + + --BOUNDARY + Content-Type: application/node.eks.aws + + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + featureGates: + FastImagePull: true +{{ end }} --BOUNDARY-- {{- if semverCompare ">=1.0.8" .Values.karpenter.targetRevision }} diff --git a/helm/cluster-level-resources/templates/karpenter-config-resources-workflow.yaml b/helm/cluster-level-resources/templates/karpenter-config-resources-workflow.yaml index 5d57392b5..7db504c49 100644 --- a/helm/cluster-level-resources/templates/karpenter-config-resources-workflow.yaml +++ b/helm/cluster-level-resources/templates/karpenter-config-resources-workflow.yaml @@ -28,6 +28,10 @@ spec: encrypted: true volumeSize: {{ index .Values "karpenter-crds" "workflow" "volumeSize" }} volumeType: gp3 +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "workflow" "fastImagePull" "enabled") }} + iops: {{ index .Values "karpenter-crds" "workflow" "fastImagePull" "iops" }} + throughput: {{ index .Values "karpenter-crds" "workflow" "fastImagePull" "throughput" }} +{{ end }} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: disabled @@ -66,6 +70,18 @@ spec: sysctl -w fs.inotify.max_user_watches=12000 & systemctl stop firewalld systemctl disable firewalld +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "workflow" "fastImagePull" "enabled") }} + + --BOUNDARY + Content-Type: application/node.eks.aws + + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + featureGates: + FastImagePull: true +{{ end }} --BOUNDARY-- diff --git a/helm/cluster-level-resources/templates/storage-class.yaml b/helm/cluster-level-resources/templates/storage-class.yaml new file mode 100644 index 000000000..d1133bebf --- /dev/null +++ b/helm/cluster-level-resources/templates/storage-class.yaml @@ -0,0 +1,32 @@ +{{- if .Values.storageclass.enabled }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.storageclass.name }} + annotations: + storageclass.kubernetes.io/is-default-class: {{ .Values.storageclass.default | quote }} + argocd.argoproj.io/sync-wave: "-1" + argocd.argoproj.io/sync-options: Prune=false,Delete=false +provisioner: {{ .Values.storageclass.provisioner }} +parameters: +{{- with .Values.storageclass.parameters }} +{{ toYaml . | indent 2 }} +{{- end }} +reclaimPolicy: {{ .Values.storageclass.reclaimPolicy }} +volumeBindingMode: {{ .Values.storageclass.volumeBindingMode }} +allowVolumeExpansion: {{ .Values.storageclass.allowVolumeExpansion }} +{{- end }} +--- +{{- if and + .Values.storageclass.enabled + .Values.storageclass.previousDefault.removeDefault +}} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.storageclass.previousDefault.name }} + annotations: + storageclass.kubernetes.io/is-default-class: "false" + argocd.argoproj.io/sync-wave: "0" + argocd.argoproj.io/sync-options: ServerSideApply=true,Validate=false,Prune=false,Delete=false +{{- end }} \ No newline at end of file diff --git a/helm/cluster-level-resources/values.yaml b/helm/cluster-level-resources/values.yaml index 7cda90eab..80e5eea02 100644 --- a/helm/cluster-level-resources/values.yaml +++ b/helm/cluster-level-resources/values.yaml @@ -157,6 +157,10 @@ karpenter-crds: cpu: "1000" memory: 1000Gi volumeSize: 50Gi + fastImagePull: + enabled: false + iops: 3000 + throughput: 600 additionalTags: {} requirements: - key: karpenter.sh/capacity-type @@ -182,7 +186,13 @@ karpenter-crds: consolidationPolicy: "WhenEmpty" expireAfter: "168h" volumeSize: 50Gi + fastImagePull: + enabled: false + iops: 3000 + throughput: 600 additionalTags: {} + # -- (list) Specific EC2 instance types for the gpu NodePools (e.g. ["g6.xlarge"]). If empty, defaults to the g4dn instance family. + instanceTypes: [] secondary: enabled: false weight: 1 @@ -192,6 +202,10 @@ karpenter-crds: consolidationPolicy: "WhenEmpty" expireAfter: "168h" volumeSize: 50Gi + fastImagePull: + enabled: false + iops: 3000 + throughput: 600 additionalTags: {} requirements: - key: karpenter.sh/capacity-type @@ -217,6 +231,10 @@ karpenter-crds: consolidationPolicy: "WhenEmpty" expireAfter: "168h" volumeSize: 50Gi + fastImagePull: + enabled: false + iops: 3000 + throughput: 600 additionalTags: {} requirements: - key: karpenter.sh/capacity-type @@ -242,6 +260,10 @@ karpenter-crds: expireAfter: "168h" volumeSize: 50Gi sgSelector: "" + fastImagePull: + enabled: false + iops: 3000 + throughput: 600 additionalTags: {} requirements: - key: karpenter.sh/capacity-type @@ -287,6 +309,25 @@ nvidia-device-plugin: enabled: false targetRevision: v0.18.0 +storageclass: + enabled: false + name: gp3 + default: true + + provisioner: ebs.csi.aws.com + + reclaimPolicy: Retain + volumeBindingMode: WaitForFirstConsumer + allowVolumeExpansion: true + + parameters: + type: gp3 + encrypted: "true" + + previousDefault: + removeDefault: true + name: gp2 + vpc-cni: enabled: false targetRevision: v1.20.4 diff --git a/helm/common/templates/_db_setup_job.tpl b/helm/common/templates/_db_setup_job.tpl index 395721688..5056b85f8 100644 --- a/helm/common/templates/_db_setup_job.tpl +++ b/helm/common/templates/_db_setup_job.tpl @@ -14,6 +14,11 @@ rules: - apiGroups: [""] resources: ["secrets"] verbs: ["*"] +{{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret $.Values.externalSecrets.pushSecret) }} +- apiGroups: ["external-secrets.io"] + resources: ["pushsecrets"] + verbs: ["get", "list", "create", "patch", "update", "delete"] +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -177,6 +182,40 @@ spec: PGPASSWORD=$SERVICE_PGPASS psql -d $SERVICE_PGDB -h $PGHOST -p $PGPORT -U $SERVICE_PGUSER -c "\conninfo" kubectl patch secret/{{ .Chart.Name }}-dbcreds -p '{"data":{"dbcreated":"dHJ1ZQo="}}' fi +{{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret $.Values.externalSecrets.pushSecret) }} + + # Create the PushSecret from within this job (instead of having helm create it as part of + # the release) and wait until the remote secret is populated. This avoids the race condition + # where the job completes before helm/external-secrets has created and processed the + # PushSecret, leaving consumers with an unpopulated remote secret. + echo "Waiting for bootstrap secret {{ $.Chart.Name }}-dbcreds-bootstrap ..." + for i in $(seq 1 60); do + kubectl -n {{ $.Release.Namespace }} get secret {{ $.Chart.Name }}-dbcreds-bootstrap >/dev/null 2>&1 && break + sleep 5 + done + kubectl -n {{ $.Release.Namespace }} get secret {{ $.Chart.Name }}-dbcreds-bootstrap + + # PGHOST is sourced from the Aurora master secret when + # global.postgres.externalSecret is configured. Copy that resolved value into the + # bootstrap secret immediately before creating the PushSecret so the hostname does + # not need to be duplicated in Helm values or rendered into the bootstrap manifest. + if [ -z "$PGHOST" ]; then + echo "ERROR: PGHOST is empty; cannot populate {{ $.Chart.Name }}-dbcreds-bootstrap" + exit 1 + fi + BOOTSTRAP_PGHOST_B64="$(printf '%s' "$PGHOST" | base64 | tr -d '\n')" + kubectl -n {{ $.Release.Namespace }} patch secret {{ $.Chart.Name }}-dbcreds-bootstrap \ + --type merge \ + -p "{\"data\":{\"host\":\"${BOOTSTRAP_PGHOST_B64}\"}}" + + echo "Creating PushSecret {{ $.Chart.Name }}-dbcreds ..." + # kubectl apply is idempotent: on subsequent runs the existing PushSecret is kept as-is + # (updatePolicy is IfNotExists, so the remote secret is never overwritten). + echo '{{ include "common.db-push-secret" . | b64enc }}' | base64 --decode | kubectl -n {{ $.Release.Namespace }} apply -f - + echo "Waiting for PushSecret to sync to the remote secret store ..." + kubectl -n {{ $.Release.Namespace }} wait --for=condition=Ready pushsecret/{{ .Chart.Name }}-dbcreds --timeout=300s + echo "PushSecret is Ready - remote secret has been populated" +{{- end }} {{- end }} {{- end }} @@ -221,16 +260,17 @@ data: username: {{ ( $.Values.postgres.username | default (printf "%s_%s" $.Chart.Name $.Release.Name) ) | b64enc | quote}} port: {{ $.Values.postgres.port | b64enc | quote }} password: {{ include "gen3.service-postgres" (dict "key" "password" "service" $.Chart.Name "context" $) | b64enc | quote }} - {{- if $.Values.global.dev }} - host: {{ (printf "%s-%s" $.Release.Name "postgresql" ) | b64enc | quote }} - {{- else }} - host: {{ ( $.Values.postgres.host | default ( $.Values.global.postgres.master.host)) | b64enc | quote }} - {{- end }} dbcreated: {{ "true" | b64enc | quote }} {{- end }} {{- end -}} +{{/* + PushSecret manifest for the dbcreds bootstrap secret. + NOTE: this is no longer rendered as a standalone helm resource. It is applied by the + {{ .Chart.Name }}-dbcreate job (see common.db_setup_job) so that the job only completes + after the remote secret has been populated. +*/}} {{- define "common.db-push-secret" -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} apiVersion: external-secrets.io/v1alpha1 diff --git a/helm/dicom-server/templates/db-init.yaml b/helm/dicom-server/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/dicom-server/templates/db-init.yaml +++ b/helm/dicom-server/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/fence/templates/db-init.yaml b/helm/fence/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/fence/templates/db-init.yaml +++ b/helm/fence/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/frontend-framework/templates/deployment.yaml b/helm/frontend-framework/templates/deployment.yaml index 8c9c60afa..9c9b64a32 100644 --- a/helm/frontend-framework/templates/deployment.yaml +++ b/helm/frontend-framework/templates/deployment.yaml @@ -155,12 +155,6 @@ spec: value: /portal {{- end }} {{- if .Values.customConfig.enabled }} - - name: DATADOG_APPLICATION_ID - value: {{ .Values.env.DATADOG_APPLICATION_ID | quote }} - - name: DATADOG_CLIENT_TOKEN - value: {{ .Values.env.DATADOG_CLIENT_TOKEN | quote }} - - name: DATACOMMONS - value: {{ .Values.env.DATACOMMONS | quote }} volumeMounts: - name: config-volume mountPath: /gen3/config diff --git a/helm/funnel/README.md b/helm/funnel/README.md index 8b25fa75c..0f7258421 100644 --- a/helm/funnel/README.md +++ b/helm/funnel/README.md @@ -187,8 +187,8 @@ Published versions of this chart are listed in the | image.initContainers[0].pullPolicy | string | `"Always"` | When to pull the image. This value should be "Always" to ensure the latest image is used. | | image.initContainers[0].tag | string | `"main-gen3"` | The Docker image tag for the Funnel init/plugin container. | | image.pullPolicy | string | `"Always"` | When to pull the image. This value should be "Always" to ensure the latest image is used. | -| image.repository | string | `"quay.io/ohsu-comp-bio/funnel"` | The Docker image repository for the Funnel service. | -| image.tag | string | `"develop-2026-07-09-19-49-55Z-97d1df55"` | | +| image.repository | string | `"quay.io/cdis/funnel"` | The Docker image repository for the Funnel service. | +| image.tag | string | `"main-gen3"` | | | labels.app | string | `"funnel"` | | | metricsEnabled | bool | `false` | | | netPolicy | map | `{"egressApps":["gen3-workflow"],"ingressApps":["gen3-workflow"]}` | Configuration for network policies created by this chart. Only relevant if "global.netPolicy.enabled" is set to true | diff --git a/helm/funnel/files/worker-job.yaml b/helm/funnel/files/worker-job.yaml index 7bf13da87..ebf6042bc 100644 --- a/helm/funnel/files/worker-job.yaml +++ b/helm/funnel/files/worker-job.yaml @@ -100,8 +100,7 @@ spec: {{` {{- if .NeedsPVC }} - name: funnel-storage-{{.TaskId}} - mountPath: /opt/funnel/funnel-work-dir/{{.TaskId}} - subPath: {{.TaskId}} + mountPath: /opt/funnel/funnel-work-dir {{- end }} `}} volumes: diff --git a/helm/funnel/files/worker-pv.yaml b/helm/funnel/files/worker-pv.yaml index d118617bf..92d9082b4 100644 --- a/helm/funnel/files/worker-pv.yaml +++ b/helm/funnel/files/worker-pv.yaml @@ -22,7 +22,6 @@ spec: - allow-other # allow non-root users to access the mount - region={{`{{.Region}}`}} - file-mode=0755 - - prefix=funnel-temp-files/ {{`{{- if .KmsKeyID}}`}} - sse aws:kms - sse-kms-key-id={{`{{.KmsKeyID}}`}} diff --git a/helm/funnel/templates/db-init.yaml b/helm/funnel/templates/db-init.yaml index 0de60dbff..b664f2632 100644 --- a/helm/funnel/templates/db-init.yaml +++ b/helm/funnel/templates/db-init.yaml @@ -9,7 +9,5 @@ {{- end -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/funnel/templates/netpolicy.yaml b/helm/funnel/templates/netpolicy.yaml index 297622ec8..a863dda83 100644 --- a/helm/funnel/templates/netpolicy.yaml +++ b/helm/funnel/templates/netpolicy.yaml @@ -137,6 +137,8 @@ spec: # Allow access to the Kubernetes API server in kube-system - to: [] ports: + - protocol: TCP + port: 6443 - protocol: TCP port: 443 --- @@ -170,7 +172,9 @@ spec: {{- end }} ports: - protocol: TCP - port: 443 # Kubernetes API + port: 443 + - protocol: TCP + port: 6443 # Kubernetes API policyTypes: - Egress diff --git a/helm/funnel/values.yaml b/helm/funnel/values.yaml index e49125757..86c9ba43d 100644 --- a/helm/funnel/values.yaml +++ b/helm/funnel/values.yaml @@ -127,8 +127,8 @@ partOf: "Workflow_Execution" oidc_job_enabled: true image: # -- (string) The Docker image repository for the Funnel service. - repository: quay.io/ohsu-comp-bio/funnel - tag: develop-2026-07-09-19-49-55Z-97d1df55 + repository: quay.io/cdis/funnel + tag: main-gen3 # -- (string) When to pull the image. This value should be "Always" to ensure the latest image is used. pullPolicy: Always diff --git a/helm/gen3-embeddings/templates/db-init.yaml b/helm/gen3-embeddings/templates/db-init.yaml index 95297b084..89e99f044 100644 --- a/helm/gen3-embeddings/templates/db-init.yaml +++ b/helm/gen3-embeddings/templates/db-init.yaml @@ -9,7 +9,5 @@ {{- end -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} diff --git a/helm/gen3-user-data-library/templates/db-init.yaml b/helm/gen3-user-data-library/templates/db-init.yaml index 95297b084..89e99f044 100644 --- a/helm/gen3-user-data-library/templates/db-init.yaml +++ b/helm/gen3-user-data-library/templates/db-init.yaml @@ -9,7 +9,5 @@ {{- end -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 62073e924..f10e69835 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -136,6 +136,7 @@ Published versions of this chart are listed in the | gen3-user-data-library.enabled | bool | `false` | Whether to deploy the gen3-user-data-library subchart. | | gen3-workflow | map | `{"enabled":false}` | Configurations for gen3-workflow chart. | | gen3-workflow.enabled | bool | `false` | Whether to deploy the gen3-workflow subchart. | +| global.argoHost | string | `"http://argo-argo-workflows-server.argo.svc.cluster.local:2746"` | argo server host URL:PORT. Endpoint for the the Argo Workflows service. | | global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false,"externalSecrets":{"enabled":false,"externalSecretAwsCreds":null,"pushSecret":false},"region":"us-east-1","secretStoreServiceAccount":{"enabled":false,"name":"secret-store-sa","roleArn":null},"useLocalSecret":{"enabled":false,"localSecretName":null}}` | AWS configuration | | global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | | global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index f76c1e5d2..368cc51ba 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -147,6 +147,8 @@ global: topologyKey: "topology.kubernetes.io/zone" # -- (int) The maxSkew to use for topology spread constraints. Defaults to 1. maxSkew: 1 + # -- (string) argo server host URL:PORT. Endpoint for the the Argo Workflows service. + argoHost: http://argo-argo-workflows-server.argo.svc.cluster.local:2746 # Dependancy Charts diff --git a/helm/hatchery/README.md b/helm/hatchery/README.md index e83cb74c5..23c0e1528 100644 --- a/helm/hatchery/README.md +++ b/helm/hatchery/README.md @@ -131,6 +131,7 @@ Published versions of this chart are listed in the | tolerations | list | `[]` | Tolerations to use for the deployment. | | volumeMounts | list | `[{"mountPath":"/var/hatchery/hatchery.json","name":"hatchery-config","readOnly":true,"subPath":"json"},{"mountPath":"/hatchery.json","name":"hatchery-config","readOnly":true,"subPath":"json"}]` | Volumes to mount to the container. | | volumes | list | `[{"configMap":{"name":"manifest-hatchery"},"name":"hatchery-config"}]` | Volumes to attach to the container. | +| workspaceInternetEgress | bool | `true` | | | workspaceLaunchTest | map | `{"enabled":false,"hostname":"https://example.com","operatorName":"username","schedule":"*/20 * * * *","workspaceImages":"(Generic) Jupyter Lab Notebook with R Kernel"}` | Configuration for workspace launch test crobjob | | workspaceLaunchTest.enabled | bool | `false` | Whether the workspace launch test cron job is enabled. | | workspaceLaunchTest.hostname | string | `"https://example.com"` | Hostname for the workspace launch test operator. | diff --git a/helm/indexd/README.md b/helm/indexd/README.md index 1ab7ac1c0..c43241030 100644 --- a/helm/indexd/README.md +++ b/helm/indexd/README.md @@ -78,6 +78,7 @@ Published versions of this chart are listed in the | image.repository | string | `"quay.io/cdis/indexd"` | The Docker image repository for the indexd service | | image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | | imagePullSecrets | list | `[]` | Docker image pull secrets. | +| maxBulkRequestLength | int | `100` | The maximum number of records that can be processed in a single bulk request. This is used to prevent large requests from overwhelming the system. | | metricsEnabled | bool | `false` | Whether Metrics are enabled. indexd doesn't expose metrics yet. | | migrateToSingleTable.args | string | `nil` | | | nameOverride | string | `""` | Override the name of the chart. | diff --git a/helm/indexd/templates/db-init.yaml b/helm/indexd/templates/db-init.yaml index 0de60dbff..b664f2632 100644 --- a/helm/indexd/templates/db-init.yaml +++ b/helm/indexd/templates/db-init.yaml @@ -9,7 +9,5 @@ {{- end -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/indexd/templates/deployment.yaml b/helm/indexd/templates/deployment.yaml index ab999c487..2baf9cbda 100644 --- a/helm/indexd/templates/deployment.yaml +++ b/helm/indexd/templates/deployment.yaml @@ -98,6 +98,8 @@ spec: value: {{ .Values.trustedIssuers.defaultPreferredType | quote }} - name: CLOUD_PROVIDER_MAP value: {{ .Values.cloudProviderMap | toJson | quote }} + - name: MAX_BULK_REQUEST_LENGTH + value: {{ .Values.maxBulkRequestLength | quote }} {{- toYaml .Values.env | nindent 12 }} {{- with .Values.volumeMounts }} volumeMounts: diff --git a/helm/indexd/values.yaml b/helm/indexd/values.yaml index f5ce02485..b90ea29ac 100644 --- a/helm/indexd/values.yaml +++ b/helm/indexd/values.yaml @@ -294,6 +294,8 @@ cloudProviderMap: gs: gcp az: azure +# -- (int) The maximum number of records that can be processed in a single bulk request. This is used to prevent large requests from overwhelming the system. +maxBulkRequestLength: 100 # -- (map) Maps an arborist resource to the issuers for passports and bearer tokens. trustedIssuers: diff --git a/helm/metadata/templates/db-init.yaml b/helm/metadata/templates/db-init.yaml index 0de60dbff..b664f2632 100644 --- a/helm/metadata/templates/db-init.yaml +++ b/helm/metadata/templates/db-init.yaml @@ -9,7 +9,5 @@ {{- end -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/metadata/templates/deployment.yaml b/helm/metadata/templates/deployment.yaml index 94eb88809..73f3e0e61 100644 --- a/helm/metadata/templates/deployment.yaml +++ b/helm/metadata/templates/deployment.yaml @@ -64,6 +64,8 @@ spec: value: "False" - name: GEN3_ES_ENDPOINT value: {{ .Values.esEndpoint }} + - name: FORCE_AUTHZ_CHECK_FOR_METADATA_QUERIES + value: {{ .Values.forceAuthzCheckForMetadataQueries | quote }} - name: DB_HOST valueFrom: secretKeyRef: @@ -176,4 +178,4 @@ spec: {{- with .Values.args }} args: {{- toYaml . | nindent 12}} - {{- end }} \ No newline at end of file + {{- end }} diff --git a/helm/metadata/templates/secrets.yaml b/helm/metadata/templates/secrets.yaml index 93d4d9632..0bd639d73 100644 --- a/helm/metadata/templates/secrets.yaml +++ b/helm/metadata/templates/secrets.yaml @@ -13,5 +13,4 @@ stringData: DB_PASSWORD={{ include "metadata.postgres.password" . }} DB_DATABASE={{ .Values.postgres.dbname }} ADMIN_LOGINS={{ $randomPass }} - FORCE_AUTHZ_CHECK_FOR_METADATA_QUERIES={{ .Values.forceAuthzCheckForMetadataQueries }} {{- end }} \ No newline at end of file diff --git a/helm/ohdsi-webapi/templates/db-init.yaml b/helm/ohdsi-webapi/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/ohdsi-webapi/templates/db-init.yaml +++ b/helm/ohdsi-webapi/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml b/helm/ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml index dad2fcdae..bc9244ade 100644 --- a/helm/ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml +++ b/helm/ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml @@ -4,6 +4,7 @@ kind: Job metadata: name: ohdsi-webapi-client-job spec: + ttlSecondsAfterFinished: 120 template: metadata: labels: @@ -141,4 +142,6 @@ spec: \"security_oid_clientId\":\"${FENCE_CLIENT_ID}\", \"security_oid_apiSecret\":\"${FENCE_CLIENT_SECRET}\" }}" + echo -e "\n\n>>========== Restarting webapi pods so they pick up new config above ============\n\n" + kubectl delete pod -l app=ohdsi-webapi {{- end }} \ No newline at end of file diff --git a/helm/ohdsi-webapi/templates/serviceaccount.yaml b/helm/ohdsi-webapi/templates/serviceaccount.yaml index 4a3477d7c..9c0c943ff 100644 --- a/helm/ohdsi-webapi/templates/serviceaccount.yaml +++ b/helm/ohdsi-webapi/templates/serviceaccount.yaml @@ -16,6 +16,9 @@ rules: resources: ["secrets"] resourceNames: ["fence-dbcreds"] verbs: ["get"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["list", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding diff --git a/helm/orthanc/templates/db-init.yaml b/helm/orthanc/templates/db-init.yaml index 83f4d86a4..559e62393 100644 --- a/helm/orthanc/templates/db-init.yaml +++ b/helm/orthanc/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_job" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/peregrine/templates/db-init.yaml b/helm/peregrine/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/peregrine/templates/db-init.yaml +++ b/helm/peregrine/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/requestor/templates/db-init.yaml b/helm/requestor/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/requestor/templates/db-init.yaml +++ b/helm/requestor/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/revproxy/README.md b/helm/revproxy/README.md index 4fd02f186..5872ef0fd 100644 --- a/helm/revproxy/README.md +++ b/helm/revproxy/README.md @@ -26,6 +26,7 @@ Published versions of this chart are listed in the | enableRobotsTxt | bool | `false` | Whether to enable robots.txt generation and serving. | | extraServices | map | `nil` | Configuration to add any extra service endpoints outside of gen3 to be served by revproxy | | fullnameOverride | string | `""` | Override the full name of the deployment. | +| global.argoHost | string | `"http://argo-argo-workflows-server.argo.svc.cluster.local:2746"` | argo server host URL:PORT. Endpoint for the the Argo Workflows service. | | global.autoscaling.averageCPUValue | string | `"500m"` | | | global.autoscaling.averageMemoryValue | string | `"500Mi"` | | | global.autoscaling.enabled | bool | `false` | | @@ -81,9 +82,9 @@ Published versions of this chart are listed in the | ingress.tls | list | `[]` | To secure an Ingress by specifying a secret that contains a TLS private key and certificate. | | metricsEnabled | bool | `nil` | Whether Metrics are enabled. | | nameOverride | string | `""` | Override the name of the chart. | -| netPolicy | map | `{"egressApps":["portal","sowerjob"],"ingressApps":["portal","sowerjob"]}` | Configuration for network policies created by this chart. Only relevant if "global.netPolicy.enabled" is set to true | +| netPolicy | map | `{"egressApps":["portal","sowerjob"],"ingressApps":["portal","sowerjob","ssjdispatcherjob"]}` | Configuration for network policies created by this chart. Only relevant if "global.netPolicy.enabled" is set to true | | netPolicy.egressApps | array | `["portal","sowerjob"]` | List of apps that this app requires egress to | -| netPolicy.ingressApps | array | `["portal","sowerjob"]` | List of app labels that require ingress to this service | +| netPolicy.ingressApps | array | `["portal","sowerjob","ssjdispatcherjob"]` | List of app labels that require ingress to this service | | nginx.pidFile | string | `"/var/run/nginx.pid"` | | | nginx.resolver | string | `"kube-dns.kube-system.svc.cluster.local"` | | | nginx.user | string | `"nginx"` | | diff --git a/helm/revproxy/gen3.nginx.conf/argo-server.conf b/helm/revproxy/gen3.nginx.conf/argo-server.conf index 185ce05e4..e0a69cf30 100644 --- a/helm/revproxy/gen3.nginx.conf/argo-server.conf +++ b/helm/revproxy/gen3.nginx.conf/argo-server.conf @@ -7,7 +7,7 @@ location /argo/ { auth_request /gen3-authz; set $proxy_service "argo"; - set $upstream http://argo-argo-workflows-server.argo.svc.cluster.local:2746; + set $upstream $argoHost; rewrite ^/argo/(.*) /$1 break; diff --git a/helm/revproxy/nginx/nginx.conf b/helm/revproxy/nginx/nginx.conf index 920a18f1e..6d8f44f7b 100644 --- a/helm/revproxy/nginx/nginx.conf +++ b/helm/revproxy/nginx/nginx.conf @@ -273,6 +273,7 @@ map $http_user_agent $loggable { # individual service locations should override to "peregrine", ... # set $proxy_service "noproxy"; + set $argoHost {{ .Values.global.argoHost }}; # # Note - need to repeat this line in location blocks that call proxy_set_header, diff --git a/helm/revproxy/values.yaml b/helm/revproxy/values.yaml index a85aa0b73..d77e58912 100644 --- a/helm/revproxy/values.yaml +++ b/helm/revproxy/values.yaml @@ -88,7 +88,8 @@ global: topologyKey: "topology.kubernetes.io/zone" # -- (int) The maxSkew to use for topology spread constraints. Defaults to 1. maxSkew: 1 - + # -- (string) argo server host URL:PORT. Endpoint for the the Argo Workflows service. + argoHost: http://argo-argo-workflows-server.argo.svc.cluster.local:2746 # -- (map) This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ autoscaling: {} diff --git a/helm/sheepdog/README.md b/helm/sheepdog/README.md index d8970578c..a9cbd09aa 100644 --- a/helm/sheepdog/README.md +++ b/helm/sheepdog/README.md @@ -46,6 +46,7 @@ Published versions of this chart are listed in the | global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. | | global.aws.externalSecrets.enabled | bool | `false` | Whether to use External Secrets for aws config. | | global.aws.externalSecrets.externalSecretAwsCreds | String | `nil` | Name of Secrets Manager secret. | +| global.debug | str | `"False"` | True to enable debug logs and other dev only functionality, False if otherwise. | | global.dev | bool | `true` | Whether the deployment is for development purposes. | | global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. | | global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | diff --git a/helm/sheepdog/templates/db-init.yaml b/helm/sheepdog/templates/db-init.yaml index 0de60dbff..b664f2632 100644 --- a/helm/sheepdog/templates/db-init.yaml +++ b/helm/sheepdog/templates/db-init.yaml @@ -9,7 +9,5 @@ {{- end -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/sheepdog/templates/deployment.yaml b/helm/sheepdog/templates/deployment.yaml index e7ed92b63..a750a3217 100644 --- a/helm/sheepdog/templates/deployment.yaml +++ b/helm/sheepdog/templates/deployment.yaml @@ -251,7 +251,7 @@ spec: # # # value: /etc/ssl/certs/ca-certificates.crt - name: GEN3_DEBUG - value: "True" + value: {{ .Values.debug }} {{- with .Values.volumeMounts }} volumeMounts: {{- toYaml . | nindent 10 }} diff --git a/helm/sheepdog/values.yaml b/helm/sheepdog/values.yaml index eb907fb75..93cfa4ec4 100644 --- a/helm/sheepdog/values.yaml +++ b/helm/sheepdog/values.yaml @@ -19,7 +19,8 @@ global: externalSecretAwsCreds: # -- (bool) Whether the deployment is for development purposes. dev: true - + # -- (str) True to enable debug logs and other dev only functionality, False if otherwise. + debug: "False" postgres: # -- (bool) Whether the database should be created. dbCreate: true diff --git a/helm/workspace-proxy/templates/netpol.yaml b/helm/workspace-proxy/templates/netpol.yaml index 5b3efcdac..b962d323b 100644 --- a/helm/workspace-proxy/templates/netpol.yaml +++ b/helm/workspace-proxy/templates/netpol.yaml @@ -45,6 +45,18 @@ spec: port: 80 - protocol: TCP port: 8888 + # Arborist: workspace-proxy calls arborist /auth/proxy to authenticate & + # authorize every request. + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.deploymentNamespace }} + podSelector: + matchLabels: + app: arborist + ports: + - protocol: TCP + port: 8080 {{- range .Values.kubernetesApiServerCIDRs }} - to: - ipBlock: diff --git a/helm/wts/templates/db-init.yaml b/helm/wts/templates/db-init.yaml index ad9bb547d..d6ff85e0c 100644 --- a/helm/wts/templates/db-init.yaml +++ b/helm/wts/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }}