From 3a0597cddc04cdbf8c025b71e649cfbf7484906c Mon Sep 17 00:00:00 2001 From: Guerdon Mukama Date: Thu, 13 Aug 2026 11:44:49 +1000 Subject: [PATCH 01/23] fix(frontend-framework): remove duplicate environment variables --- helm/frontend-framework/templates/deployment.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/helm/frontend-framework/templates/deployment.yaml b/helm/frontend-framework/templates/deployment.yaml index 216b108b8..9199e3635 100644 --- a/helm/frontend-framework/templates/deployment.yaml +++ b/helm/frontend-framework/templates/deployment.yaml @@ -153,12 +153,6 @@ spec: value: /portal {{- end }} {{- if .Values.customConfig.enabled }} - - name: DATADOG_APPLICATION_ID - value: {{ .Values.env.DATADOG_APPLICATION_ID | quote }} - - name: DATADOG_CLIENT_TOKEN - value: {{ .Values.env.DATADOG_CLIENT_TOKEN | quote }} - - name: DATACOMMONS - value: {{ .Values.env.DATACOMMONS | quote }} volumeMounts: - name: config-volume mountPath: /gen3/config From ac710dbc7882d5c2080de5a7a53bd9607597a7f0 Mon Sep 17 00:00:00 2001 From: jacob50231 Date: Wed, 26 Aug 2026 11:40:35 -0500 Subject: [PATCH 02/23] add maxBulkRequestLength for indexd --- helm/indexd/README.md | 1 + helm/indexd/templates/deployment.yaml | 2 ++ helm/indexd/values.yaml | 1 + helm/zendesk-wrapper/README.md | 3 ++- 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/helm/indexd/README.md b/helm/indexd/README.md index 1ab7ac1c0..5fb463976 100644 --- a/helm/indexd/README.md +++ b/helm/indexd/README.md @@ -78,6 +78,7 @@ Published versions of this chart are listed in the | image.repository | string | `"quay.io/cdis/indexd"` | The Docker image repository for the indexd service | | image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | | imagePullSecrets | list | `[]` | Docker image pull secrets. | +| maxBulkRequestLength | int | `100` | | | metricsEnabled | bool | `false` | Whether Metrics are enabled. indexd doesn't expose metrics yet. | | migrateToSingleTable.args | string | `nil` | | | nameOverride | string | `""` | Override the name of the chart. | diff --git a/helm/indexd/templates/deployment.yaml b/helm/indexd/templates/deployment.yaml index ab999c487..2baf9cbda 100644 --- a/helm/indexd/templates/deployment.yaml +++ b/helm/indexd/templates/deployment.yaml @@ -98,6 +98,8 @@ spec: value: {{ .Values.trustedIssuers.defaultPreferredType | quote }} - name: CLOUD_PROVIDER_MAP value: {{ .Values.cloudProviderMap | toJson | quote }} + - name: MAX_BULK_REQUEST_LENGTH + value: {{ .Values.maxBulkRequestLength | quote }} {{- toYaml .Values.env | nindent 12 }} {{- with .Values.volumeMounts }} volumeMounts: diff --git a/helm/indexd/values.yaml b/helm/indexd/values.yaml index f5ce02485..49fba60a4 100644 --- a/helm/indexd/values.yaml +++ b/helm/indexd/values.yaml @@ -294,6 +294,7 @@ cloudProviderMap: gs: gcp az: azure +maxBulkRequestLength: 100 # -- (map) Maps an arborist resource to the issuers for passports and bearer tokens. trustedIssuers: diff --git a/helm/zendesk-wrapper/README.md b/helm/zendesk-wrapper/README.md index bd6ca7d74..fc2e7f9b9 100644 --- a/helm/zendesk-wrapper/README.md +++ b/helm/zendesk-wrapper/README.md @@ -26,8 +26,9 @@ Published versions of this chart are listed in the | autoscaling | object | `{}` | | | commonLabels | string | `nil` | | | criticalService | string | `"false"` | | -| env | map | `[{"name":"GEN3_ZENDESK_URL","value":""}]` | Environment variables for the Zendesk wrapper service | +| env | map | `[{"name":"GEN3_ZENDESK_URL","value":""},{"name":"GEN3_ZENDESK_BRAND_ID","value":""}]` | Environment variables for the Zendesk wrapper service | | env[0] | string | `{"name":"GEN3_ZENDESK_URL","value":""}` | Zendesk instance URL (e.g., https://gen3support.zendesk.com) | +| env[1] | string | `{"name":"GEN3_ZENDESK_BRAND_ID","value":""}` | Zendesk brand ID (e.g., 123456, can be found in Zendesk admin panel) | | externalSecrets | map | `{"name":"zendesk-wrapper-secret"}` | Secret environment variables (referenced from Kubernetes secrets) | | externalSecrets.name | string | `"zendesk-wrapper-secret"` | Name of the Kubernetes secret containing the Zendesk API token | | global.autoscaling.averageCPUValue | string | `"500m"` | | From 867e7bc771572e1c36bb0212ae14059b50a9b71e Mon Sep 17 00:00:00 2001 From: jacob50231 Date: Thu, 27 Aug 2026 09:22:03 -0500 Subject: [PATCH 03/23] add comment --- helm/indexd/README.md | 2 +- helm/indexd/values.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/helm/indexd/README.md b/helm/indexd/README.md index 5fb463976..c43241030 100644 --- a/helm/indexd/README.md +++ b/helm/indexd/README.md @@ -78,7 +78,7 @@ Published versions of this chart are listed in the | image.repository | string | `"quay.io/cdis/indexd"` | The Docker image repository for the indexd service | | image.tag | string | `""` | Overrides the image tag whose default is the chart appVersion. | | imagePullSecrets | list | `[]` | Docker image pull secrets. | -| maxBulkRequestLength | int | `100` | | +| maxBulkRequestLength | int | `100` | The maximum number of records that can be processed in a single bulk request. This is used to prevent large requests from overwhelming the system. | | metricsEnabled | bool | `false` | Whether Metrics are enabled. indexd doesn't expose metrics yet. | | migrateToSingleTable.args | string | `nil` | | | nameOverride | string | `""` | Override the name of the chart. | diff --git a/helm/indexd/values.yaml b/helm/indexd/values.yaml index 49fba60a4..b90ea29ac 100644 --- a/helm/indexd/values.yaml +++ b/helm/indexd/values.yaml @@ -294,6 +294,7 @@ cloudProviderMap: gs: gcp az: azure +# -- (int) The maximum number of records that can be processed in a single bulk request. This is used to prevent large requests from overwhelming the system. maxBulkRequestLength: 100 # -- (map) Maps an arborist resource to the issuers for passports and bearer tokens. From 2f6f6ac031bdbeda81e678b2974ba2a9a0a77606 Mon Sep 17 00:00:00 2001 From: avantol Date: Mon, 31 Aug 2026 10:16:53 -0500 Subject: [PATCH 04/23] feat(observability): default grafana dashboard for local observability, update docs --- .../scripts/regenerate_local_alloy_values.py | 77 +- docs/local-observability.md | 104 +- docs/otel-logs-and-traces.md | 15 +- examples/dashboards/local-services.json | 1914 +++++++++++++++++ examples/local_alloy_values.yaml | 59 +- examples/local_lgtm.yaml | 34 +- helm/hatchery/README.md | 1 + helm/revproxy/README.md | 4 +- 8 files changed, 2191 insertions(+), 17 deletions(-) create mode 100644 examples/dashboards/local-services.json diff --git a/.github/scripts/regenerate_local_alloy_values.py b/.github/scripts/regenerate_local_alloy_values.py index f2037e02d..b0c220ea8 100644 --- a/.github/scripts/regenerate_local_alloy_values.py +++ b/.github/scripts/regenerate_local_alloy_values.py @@ -11,11 +11,12 @@ ``templates/alloy-config.yaml`` never renders; and a ``loki.process`` stage inserted after the pod log source. -That last one is not an address rewrite but an added behaviour, and it is the only place it -exists. It promotes ``trace_id`` to Loki structured metadata and relabels ``service_name`` from the -log line, which is what makes Grafana's trace-to-logs query resolve. The chart has no equivalent, -so moving the stage into ``helm/alloy/values.yaml`` is what would extend correlation to deployed -clusters, and this substitution would then be dropped. +Two of the substitutions are not address rewrites but added behaviour that exists nowhere else. +The ``loki.process`` stage promotes ``trace_id`` to Loki structured metadata and relabels +``service_name`` from the log line, which is what makes Grafana's trace-to-logs query resolve. The +cAdvisor scrape adds the per-container memory and CPU series the chart's kubelet scrape does not +carry. The chart has no equivalent of either, so moving one into ``helm/alloy/values.yaml`` is what +would extend it to deployed clusters, and its substitution would then be dropped. Run it after any change to the chart's configuration: @@ -96,6 +97,65 @@ " }\n" " }\n", ), + ( + # cAdvisor, reached over the same node proxy the kubelet scrape uses. The chart scrapes + # the kubelet's own /metrics, which carries no per-container memory or CPU, so nothing + # in a cluster deployed from helm/alloy reports what a pod actually uses. + ' // Cluster Events\n', + ' // cAdvisor. Local-only; see docs/otel-logs-and-traces.md.\n' + " //\n" + " // The kubelet serves this on a second path, so it needs its own scrape rather than a\n" + " // longer keep list on the kubelet one.\n" + ' discovery.relabel "cadvisor" {\n' + " targets = discovery.kubernetes.nodes.targets\n" + " rule {\n" + ' target_label = "__address__"\n' + ' replacement = "kubernetes.default.svc.cluster.local:443"\n' + " }\n" + " rule {\n" + ' source_labels = ["__meta_kubernetes_node_name"]\n' + ' regex = "(.+)"\n' + ' replacement = "/api/v1/nodes/${1}/proxy/metrics/cadvisor"\n' + ' target_label = "__metrics_path__"\n' + " }\n" + " }\n" + "\n" + ' prometheus.scrape "cadvisor" {\n' + ' job_name = "integrations/kubernetes/cadvisor"\n' + " targets = discovery.relabel.cadvisor.output\n" + ' scheme = "https"\n' + ' scrape_interval = "60s"\n' + ' bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"\n' + " tls_config {\n" + " insecure_skip_verify = true\n" + " }\n" + " clustering {\n" + " enabled = true\n" + " }\n" + " forward_to = [prometheus.relabel.cadvisor.receiver]\n" + " }\n" + "\n" + ' prometheus.relabel "cadvisor" {\n' + " rule {\n" + ' source_labels = ["__name__"]\n' + ' regex = "container_memory_working_set_bytes|container_memory_rss|container_spec_memory_limit_bytes|container_cpu_usage_seconds_total|container_cpu_cfs_periods_total|container_cpu_cfs_throttled_periods_total"\n' + ' action = "keep"\n' + " }\n" + "\n" + " // cAdvisor reports one series per cgroup level. The pod and slice rollups carry an\n" + " // empty container label and sum the containers beneath them, so keeping both counts\n" + " // every container twice.\n" + " rule {\n" + ' source_labels = ["container"]\n' + ' regex = ""\n' + ' action = "drop"\n' + " }\n" + "\n" + " forward_to = [prometheus.relabel.metrics_service.receiver]\n" + " }\n" + "\n" + ' // Cluster Events\n', + ), # Once per write endpoint, hence the repeated pairs. (' cluster = "{{ .Values.cluster }}",\n', ' cluster = "local-kind",\n'), (' project = "{{ .Values.project }}",\n', ' project = "local",\n'), @@ -153,9 +213,10 @@ memory: 256Mi # Copied from helm/alloy/values.yaml, changing the three write endpoints and the two external - # labels, and inserting the loki.process stage that follows the pod log source. That stage is - # local-only: the chart has no equivalent, so trace-to-logs correlation works here and not in a - # cluster deployed from helm/alloy. + # labels, and inserting two things the chart has no equivalent of: the loki.process stage that + # follows the pod log source, and the cAdvisor scrape. Both are local-only, so trace-to-logs + # correlation and per-container memory and CPU work here and not in a cluster deployed from + # helm/alloy. See docs/otel-logs-and-traces.md. # # The labels are written out literally on purpose. templates/alloy-config.yaml renders this # with toYaml rather than tpl, so any {{ }} left in here reaches the ConfigMap unrendered. diff --git a/docs/local-observability.md b/docs/local-observability.md index 74774b93d..867644556 100644 --- a/docs/local-observability.md +++ b/docs/local-observability.md @@ -25,7 +25,8 @@ Use this when you are developing a service and want to see its own metrics, trac profiles. Do not use it as a model for a deployed cluster: - one replica of everything, no high availability -- `emptyDir` storage, so all data is lost when the pod restarts +- `emptyDir` storage for the telemetry, so metrics, logs, traces and profiles are lost when the + pod restarts; only Grafana's own database persists, on a claim - no ingress, no TLS, no authentication beyond Grafana's default `admin` / `admin` The `observability` chart is the deployed-cluster answer. It is sized for EKS - five Mimir @@ -141,6 +142,107 @@ label filter with no parser stage, which resolves only because the Alloy overlay Every sample Alloy forwards carries `cluster="local-kind"` and `project="local"`, set as external labels in the values file. If a metric has those labels it came through this pipeline. +# Step 5. Load the local observability dashboard + +[examples/dashboards/local-services.json](../examples/dashboards/local-services.json) pulls all four +signals onto one board: CPU and memory per service and per pod, request rate and p95 latency and +recent traces, real per-container memory against each container's limit, the metrics the services +publish themselves, CPU and heap flame graphs, and log volume with a log viewer at the bottom. Load +it with: + +```bash +jq '{dashboard: ., overwrite: true}' examples/dashboards/local-services.json \ + | curl -sS -u admin:admin -H 'Content-Type: application/json' \ + -X POST http://localhost:3000/api/dashboards/db -d @- | jq -r '.status, .url' +``` + +Run that once. Grafana's own volume is the one thing in +[examples/local_lgtm.yaml](../examples/local_lgtm.yaml) backed by a claim rather than an `emptyDir`, +because `GF_PATHS_DATA` points inside it - so imported and hand-edited dashboards survive a pod +restart while metrics, logs, traces and profiles start clean. The stack stays ephemeral where it +would otherwise grow without bound locally. Deleting the +`lgtm-grafana` claim resets Grafana; `kind delete cluster` takes it too. + +`examples/dashboards/` is still the source of truth. A dashboard edited in the UI now sticks in that +volume, which means it can drift from the file - export it back after editing for general improvements +to share. + +## Exporting an edited dashboard back over the file + +Grafana is the easier place to drag panels around; the file is what everyone else gets. After +saving in the UI, here's how to export back to the general file: + +```bash +curl -sS -u admin:admin http://localhost:3000/api/dashboards/uid/gen3-local-services \ + | jq '.dashboard | .id = null | del(.version)' > examples/dashboards/local-services.json +``` + +`id` is Grafana's local row number and `version` changes on every save, so dropping both keeps the +diff to what actually changed and lets the file import cleanly with the command above. Dashboard +settings -> JSON Model does the same thing by hand. + +Every panel carries a description. Hover over for what the series is and what it is not: which +signal it came from, whether the pickers reach it, and the way it misleads. + +Some general info on what's on there: + +**Two views of memory.** The Memory row is Pyroscope's sampled heap, which is +useful for which code holds objects. The Container memory row is +`container_memory_working_set_bytes` against each container's limit, which is the footprint +Kubernetes evicts and OOM-kills on. Expect them to disagree by an order of magnitude, and take the +second for footprint. + +Only the local overlay scrapes cAdvisor, so a cluster deployed from +`helm/alloy` has neither container series - see +[otel-logs-and-traces.md](otel-logs-and-traces.md). The Pyroscope memory panels also stay empty +until `PROFILE_MEMORY` is set (step 3). + +**Two pickers at the top, because the label spellings differ.** Service is populated from Pyroscope's +`service_name`; Log stream from Loki's, which holds both spellings of a service because Alloy falls +back to the pod's `app` label when a line carries no `service` field. Panels keyed by `app` or by +`pod` reach neither picker, and say so in their titles. Making the services register one spelling +is what would collapse this back to a single picker. Log level, search and the display toggles live +in each log viewer's own controls rather than at the top of the board; clicking a field in a line's +details adds it to the Log filters control, which narrows every Loki panel at once. + +The Logs row hides cluster plumbing by default through the Exclude streams box, which holds a regex +rather than a list of services: a new Gen3 service is kept without editing anything, a new kube +component is dropped by the same `kube-.*` it already matches. Add `|name` to hide one more, or +clear the box to see everything the cluster logs. + +**Drilldown starts with the time picker.** No panel overrides the dashboard range, so dragging a +selection across a spike on any graph re-scopes every other panel, log viewers included, to those +minutes. From there, clicking a series on the trace panels opens Explore for that service and +window, and each log viewer has a header link to its own query. + +**Profile queries are capped at a week.** Pyroscope rejects any query longer than its +`max_query_length`, which defaults to 1d, and Grafana reports that only as "An error occurred within +the plugin" on every profile panel at once. `examples/local_lgtm.yaml` disables that check with +`PYROSCOPE_EXTRA_ARGS`, leaving `max-query-lookback` at its 1w default - so a wider dashboard window +returns the last week of profiles instead of failing, while metrics, logs and traces keep the whole +range. A stack already running keeps the old limit until its pod is replaced, and replacing it +discards the telemetry collected so far - the dashboards survive, the five `emptyDir` volumes do +not. + +## Building your own panel from a profile + +A Pyroscope query returns a different frame for each `queryType`, and the panel has to match it: + +| `queryType` | frames returned | panel | +| --- | --- | --- | +| `metrics` | one time series, unit already set to `cores`, `bytes` or `binBps` | Time series, Stat | +| `profile` | one flame-graph frame, no time field | Flame graph | +| `both` | both of the above | Explore only | + +Explore queries with `both` and splits the two frames into a Graph above a Flame graph. A dashboard +panel renders whichever frame it understands and drops the other, so "Add to dashboard" appears to +lose the graph. Keep the queries and split them: a Time series panel set to `metrics`, a Flame graph +panel set to `profile`. + +Leave the panel's unit unset. The datasource stamps a unit on each series, and a unit picked in the +panel options overrides it - which is also why one panel holding both CPU and memory queries can +label neither correctly. One unit per panel. + ## When a log line or trace link does not show up Alloy tails every pod's containers by default, so absent logs are usually a write or a label diff --git a/docs/otel-logs-and-traces.md b/docs/otel-logs-and-traces.md index eac53ef6a..5fefe298d 100644 --- a/docs/otel-logs-and-traces.md +++ b/docs/otel-logs-and-traces.md @@ -327,6 +327,18 @@ Not required for the ArgoCD path, but it should probably not stay broken. Groupe this; only `tpl` evaluates the template. Note that `tpl` evaluates the whole config, so any future stage using Go-style braces, `stage.template` being the common one, would break. - No `labeldrop`, unlike the ArgoCD config, which makes the label limit in section 1 worse. +- No cAdvisor scrape. The chart scrapes the kubelet's own `/metrics`, which carries operational + counters and no per-container resource usage, so no cluster deployed from this chart reports what + a pod's memory or CPU actually is - `container_memory_working_set_bytes`, + `container_spec_memory_limit_bytes` and `container_cpu_usage_seconds_total` are all absent, and + the `kube_state_metrics` scrape alongside it targets a `monitoring-extras-kube-state-metrics` + service that this repo never creates. The kubelet serves cAdvisor on a second path + (`/metrics/cadvisor`) reached through the same node proxy and the same service account token, so + this is an added scrape rather than a longer keep list. + [examples/local_alloy_values.yaml](../examples/local_alloy_values.yaml) carries that scrape for + local development only, inserted by `.github/scripts/regenerate_local_alloy_values.py` alongside + the `loki.process` stage below. Moving it into `helm/alloy/values.yaml` is what would give + deployed clusters the same view, and that substitution would then be dropped from the script. **Blocks correlation.** @@ -343,4 +355,5 @@ Not required for the ArgoCD path, but it should probably not stay broken. Groupe Metrics. `common.grafanaAnnotations` already puts `prometheus.io/scrape` and `prometheus.io/path` on every Gen3 pod, `global.metricsEnabled` defaults to true, and Alloy's `annotation_autodiscovery_pods` relabel already resolves those to `podIP:/metrics`. -A service only has to serve the endpoint. \ No newline at end of file +A service only has to serve the endpoint. That covers what a service publishes about itself; what +the container consumes is a separate scrape, in the cAdvisor bullet above. \ No newline at end of file diff --git a/examples/dashboards/local-services.json b/examples/dashboards/local-services.json new file mode 100644 index 000000000..9ba74eec5 --- /dev/null +++ b/examples/dashboards/local-services.json @@ -0,0 +1,1914 @@ +{ + "id": null, + "uid": "gen3-local-services", + "title": "Local services - overview", + "description": "Resources, requests, traces, service metrics and logs for the local kind cluster, one row per signal. CPU and memory are profiler-reported and come from Python services today: only services pushing profiles appear there, and sampled heap is not the pod's resident memory - the Container memory row is.", + "tags": [ + "local", + "observability" + ], + "timezone": "browser", + "editable": true, + "graphTooltip": 1, + "schemaVersion": 42, + "refresh": "30s", + "time": { + "from": "now-1h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "service", + "label": "Service (CPU, memory, traces)", + "type": "query", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "query": { + "refId": "PyroscopeVariableQueryEditor-VariableQuery", + "type": "labelValue", + "labelName": "service_name", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds" + }, + "regex": "/^(?!pyroscope$).*$/", + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".*", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "options": [], + "sort": 1, + "hide": 0, + "description": "Pyroscope's own service_name label, read live from the datasource. Drives the At a glance, CPU, Memory, Requests and Profiles rows. It does not reach the Container memory or scraped-metrics panels, which are keyed by pod and by the Kubernetes app label." + }, + { + "name": "logservice", + "label": "Log stream (logs)", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": { + "refId": "LokiVariableQueryEditor-VariableQuery", + "type": 1, + "label": "service_name", + "stream": "" + }, + "refresh": 2, + "multi": true, + "includeAll": true, + "allValue": ".+", + "current": { + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "options": [], + "sort": 1, + "hide": 0, + "description": "Loki's service_name label, which is not the same thing as the Service picker. Alloy sets it from each log line's own `service` field and falls back to the pod's `app` label when a line has none, so one service can appear twice under two spellings, and services that never profile appear here too. Drives the Logs row only." + }, + { + "name": "exclude", + "label": "Exclude streams (logs)", + "type": "textbox", + "query": "alloy|ambassador|coredns|etcd|kindnet|kube-.*|ingress-nginx|integrations/.*|revproxy|http", + "current": { + "text": "alloy|ambassador|coredns|etcd|kindnet|kube-.*|ingress-nginx|integrations/.*|revproxy|http", + "value": "alloy|ambassador|coredns|etcd|kindnet|kube-.*|ingress-nginx|integrations/.*|revproxy|http" + }, + "options": [], + "hide": 0, + "skipUrlSync": false, + "description": "Streams dropped from every Logs panel, as one regex. Matches whole names, so `kube-.*` covers every kube component and a new Gen3 service is kept without editing this. To hide one more service add `|its-name`; to see everything, clear the box. A stream excluded here shows nothing even when picked in Log stream." + }, + { + "name": "logfilter", + "label": "Log filters (logs)", + "type": "adhoc", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "filters": [], + "hide": 0, + "description": "Ad hoc label filters appended to every Loki query on the board. Add them by clicking a field in a log line's details, or with the + here." + } + ] + }, + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "panels": [ + { + "id": 1, + "type": "row", + "title": "At a glance", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + } + }, + { + "id": 2, + "type": "stat", + "title": "CPU now", + "description": "Latest CPU the profiler attributed to each service. 0.02 cores means 2% of one core - CPU-seconds consumed per second of wall clock, the same unit as a Kubernetes CPU request or limit, where 100m is 0.1 cores.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "service_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_name}", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "value_and_name", + "justifyMode": "auto", + "orientation": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + } + }, + { + "id": 3, + "type": "stat", + "title": "Heap in use now", + "description": "Latest sampled heap in use: the bytes the profiler saw allocated and still live. This is not the pod's memory usage, which is resident set size - the memory the kernel actually holds in RAM for the process. For the Python services profiled here, resident memory also covers the Python interpreter itself, its shared libraries, buffers allocated inside C extensions, and memory CPython's allocator has freed but kept back from the OS, none of which reaches a profile; expect an order of magnitude between the two figures. A Go service would be sampled the same way, with the Go runtime's stacks and spans outside the profile instead. Read the Container memory row for the footprint and this panel for which code is holding objects.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 1 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "memory:inuse_space:bytes:space:bytes", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "service_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_name}", + "color": { + "mode": "palette-classic" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "colorMode": "value", + "graphMode": "area", + "textMode": "value_and_name", + "justifyMode": "auto", + "orientation": "auto", + "wideLayout": true, + "percentChangeColorMode": "standard", + "showPercentChange": false + } + }, + { + "id": 4, + "type": "row", + "title": "CPU", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + } + }, + { + "id": 5, + "type": "timeseries", + "title": "CPU by service", + "description": "CPU as sampled by the profiler. 0.02 cores means 2% of one core - CPU-seconds consumed per second of wall clock, the same unit as a Kubernetes CPU request or limit, where 100m is 0.1 cores. It covers the process the profiler is attached to, so it is not the container's total.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "service_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_name}", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": true, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "mean", + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 6, + "type": "timeseries", + "title": "CPU by pod", + "description": "One series per replica. 0.02 cores means 2% of one core - CPU-seconds consumed per second of wall clock, the same unit as a Kubernetes CPU request or limit, where 100m is 0.1 cores. Pods that have been replaced keep their series for as long as their profiles sit inside the time range.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 7 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "pod" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.pod}", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": true, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true, + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 7, + "type": "row", + "title": "Memory", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + } + }, + { + "id": 9, + "type": "timeseries", + "title": "Allocation rate", + "description": "Bytes allocated per second, including memory freed again immediately. High here with a flat heap in use is churn, not a leak.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 18 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "memory:alloc_space:bytes:space:bytes", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "service_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_name}", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": true, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "Heap in use", + "description": "Bytes the profiler sampled as allocated and still live. Useful for which code holds objects and whether that is growing, not for footprint - the Container memory row below is the footprint, and is normally an order of magnitude larger.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 18 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "memory:inuse_space:bytes:space:bytes", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "service_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_name}", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": true, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 10, + "type": "timeseries", + "title": "Objects in use", + "description": "The count of live sampled allocations, where Heap in use is their bytes. Bytes flat while objects climb means many small objects accumulating; bytes divided by objects gives the average object size.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 18 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "memory:inuse_objects:count:space:bytes", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [ + "service_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "displayName": "${__field.labels.service_name}", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": true, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 30, + "type": "row", + "title": "Container memory (real usage)", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + } + }, + { + "id": 28, + "type": "timeseries", + "title": "Container memory (working set) by pod", + "description": "Working set is the number Kubernetes evicts and OOM-kills on - resident memory minus the page cache it could reclaim - so it is the pod's real footprint, unlike the sampled heap above. Dashed lines are each container's limit. A container with no limit has no line, because cAdvisor reports its limit as 0 rather than leaving the series out. Not filtered by the Service picker: these series are keyed by pod and container, which carry neither spelling of the name the picker holds.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 27 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "range": true, + "editorMode": "code", + "expr": "container_memory_working_set_bytes{container!=\"\", namespace=~\"default|monitoring\"}", + "legendFormat": "{{pod}}" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "range": true, + "editorMode": "code", + "expr": "container_spec_memory_limit_bytes{container!=\"\", namespace=~\"default|monitoring\"} > 0", + "legendFormat": "{{pod}} limit" + } + ], + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "never", + "spanNulls": true, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".* limit" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "fill": "dash", + "dash": [ + 10, + 10 + ] + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + }, + { + "id": "custom.lineWidth", + "value": 1 + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 29, + "type": "table", + "title": "Memory used vs limit", + "description": "Working set now, the pod's total limit, and the ratio. Pods whose containers declare no memory limit read `none` and `no limit` rather than being dropped or shown at zero - cAdvisor reports their limit as 0, and there is nothing for a percentage to be of. Values are summed over the pod's containers, so a sidecar counts toward both columns. Not filtered by the Service picker: these series are keyed by pod, which carries neither spelling of the name the picker holds.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 36 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (container_memory_working_set_bytes{container!=\"\", namespace=~\"default|monitoring\"})", + "instant": true, + "range": false, + "format": "table", + "editorMode": "code" + }, + { + "refId": "B", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (container_spec_memory_limit_bytes{container!=\"\", namespace=~\"default|monitoring\"})", + "instant": true, + "range": false, + "format": "table", + "editorMode": "code" + }, + { + "refId": "C", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (container_memory_working_set_bytes{container!=\"\", namespace=~\"default|monitoring\"}) / (sum by (pod) (container_spec_memory_limit_bytes{container!=\"\", namespace=~\"default|monitoring\"}) > 0)", + "instant": true, + "range": false, + "format": "table", + "editorMode": "code" + } + ], + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "pod", + "mode": "outer" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Time 1": true, + "Time 2": true, + "Time 3": true, + "Time #A": true, + "Time #B": true, + "Time #C": true + }, + "renameByName": { + "pod": "Pod", + "Value #A": "Used", + "Value #B": "Limit", + "Value #C": "Of limit", + "Value 1": "Used", + "Value 2": "Limit", + "Value 3": "Of limit" + }, + "indexByName": {}, + "includeByName": {} + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.width", + "value": 110 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Limit" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + }, + { + "id": "custom.width", + "value": 110 + }, + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "0": { + "text": "none", + "index": 0 + } + } + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Of limit" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 1 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "gauge", + "mode": "gradient", + "valueDisplayMode": "text" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.7 + }, + { + "color": "red", + "value": 0.9 + } + ] + } + }, + { + "id": "mappings", + "value": [ + { + "type": "special", + "options": { + "match": "null", + "result": { + "text": "no limit", + "index": 0 + } + } + } + ] + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Used", + "desc": true + } + ] + } + }, + { + "id": 14, + "type": "row", + "title": "Requests (from traces)", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + } + }, + { + "id": 15, + "type": "timeseries", + "title": "Request rate by endpoint", + "description": "Inbound server spans per second, by route template. Derived by Tempo's metrics generator from the spans it receives, not exported by the service. The series exist only while traces arrive, so a quiet service leaves a gap rather than a zero.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 48 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (service, span_name) (rate(traces_spanmetrics_calls_total{service=~\"${service:pipe}\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))", + "legendFormat": "{{service}} {{span_name}}", + "range": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "title": "Traces for this service (Explore)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22tempo%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22tempo%22%2C%22uid%22%3A%22tempo%22%7D%2C%22queryType%22%3A%22traceql%22%2C%22query%22%3A%22%7Bresource.service.name%3D%5C%22${__field.labels.service}%5C%22%7D%22%2C%22limit%22%3A50%2C%22tableType%22%3A%22traces%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + } + ] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 16, + "type": "timeseries", + "title": "Latency p95 by endpoint", + "description": "95th percentile of inbound server span duration. Derived by Tempo's metrics generator from the spans it receives, not exported by the service. The series exist only while traces arrive, so a quiet service leaves a gap rather than a zero.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 48 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "histogram_quantile(0.95, sum by (service, span_name, le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:pipe}\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "legendFormat": "{{service}} {{span_name}}", + "range": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "title": "Traces for this service (Explore)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22tempo%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22tempo%22%2C%22uid%22%3A%22tempo%22%7D%2C%22queryType%22%3A%22traceql%22%2C%22query%22%3A%22%7Bresource.service.name%3D%5C%22${__field.labels.service}%5C%22%7D%22%2C%22limit%22%3A50%2C%22tableType%22%3A%22traces%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + } + ] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 17, + "type": "timeseries", + "title": "Error rate by endpoint", + "description": "Spans of any kind whose status is ERROR. Empty until a span actually carries that status - an HTTP 500 that the service handles without marking its span is invisible here. Derived by Tempo's metrics generator from the spans it receives, not exported by the service. The series exist only while traces arrive, so a quiet service leaves a gap rather than a zero.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 57 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (service, span_kind, span_name) (rate(traces_spanmetrics_calls_total{service=~\"${service:pipe}\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval]))", + "legendFormat": "{{service}} {{span_kind}} {{span_name}}", + "range": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "title": "Error traces for this service (Explore)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22tempo%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22tempo%22%2C%22uid%22%3A%22tempo%22%7D%2C%22queryType%22%3A%22traceql%22%2C%22query%22%3A%22%7Bresource.service.name%3D%5C%22${__field.labels.service}%5C%22%20%26%26%20status%3Derror%7D%22%2C%22limit%22%3A50%2C%22tableType%22%3A%22traces%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + }, + { + "title": "Error logs for this service (Explore)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22loki%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22loki%22%2C%22uid%22%3A%22loki%22%7D%2C%22expr%22%3A%22%7Bservice_name%3D~%5C%22${__field.labels.service}%5C%22%7D%20%7C%20detected_level%20%3D~%20%5C%22error%7Cwarn%5C%22%22%2C%22queryType%22%3A%22range%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + } + ] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 18, + "type": "table", + "title": "Recent traces", + "description": "Newest traces for the selected services. Click a Trace ID to open it in Tempo; its logs link resolves through Loki's trace_id metadata.", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 64 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceql", + "query": "{resource.service.name=~\"${service:pipe}\"}", + "limit": 20, + "tableType": "traces", + "filters": [] + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + } + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "sortBy": [ + { + "displayName": "Start time", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + } + }, + { + "id": 19, + "type": "row", + "title": "Service metrics (scraped from /metrics)", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 73 + } + }, + { + "id": 20, + "type": "timeseries", + "title": "Request rate by status (all services)", + "description": "Counted by the service itself through cdispyutils request_metrics and read from its /metrics, so it is independent of the trace pipeline and survives a service that never exports a span. Selected by metric name rather than by service, so any service using that helper appears without editing this dashboard. Ignores the Service picker: these series carry `app` from the Kubernetes label (`gen3-embeddings`), which is spelled differently from the `service_name` the picker holds (`gen3_embeddings`), and no variable format converts between the two.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 74 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (app, status_code) (rate({__name__=~\".+_api_requests_total\"}[$__rate_interval]))", + "legendFormat": "{{app}} {{status_code}}", + "range": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "type": "row", + "title": "Profiles ($service)", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 83 + } + }, + { + "id": 12, + "type": "flamegraph", + "title": "CPU profile", + "description": "Merged across every service selected in the Service picker. Pick a single service to read it. The totals are summed over the dashboard's time range, so they grow as the range widens rather than describing a moment.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 84 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "profile", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [] + } + ], + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 13, + "type": "flamegraph", + "title": "Heap in use profile", + "description": "Merged across every service selected in the Service picker. Pick a single service to read it. The totals are summed over the dashboard's time range, so they grow as the range widens rather than describing a moment.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 84 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "profile", + "profileTypeId": "memory:inuse_space:bytes:space:bytes", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [] + } + ], + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 23, + "type": "row", + "title": "Logs", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 97 + } + }, + { + "id": 24, + "type": "timeseries", + "title": "Log volume by service", + "description": "Lines per second per stream. Driven by the Log stream picker, not Service: Alloy takes `service_name` from each line's own `service` field and falls back to the pod's `app` label, so one service can appear under both spellings (`gen3_embeddings` and `gen3-embeddings`) while Pyroscope knows only the one the SDK registered.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 98 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (service_name) (rate({service_name=~\"${logservice:pipe}\", service_name!~\"$exclude\"}[$__auto]))", + "legendFormat": "{{service_name}}", + "queryType": "range", + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cps", + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 60, + "gradientMode": "none", + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "barAlignment": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "normal", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 25, + "type": "timeseries", + "title": "Log volume by level", + "description": "`detected_level` is inferred by Loki per line, not a label the service sets, so `unknown` means it could not find one rather than that the line was unimportant.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 98 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "sum by (detected_level) (rate({service_name=~\"${logservice:pipe}\", service_name!~\"$exclude\"}[$__auto]))", + "legendFormat": "{{detected_level}}", + "queryType": "range", + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "cps", + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 60, + "gradientMode": "none", + "showPoints": "never", + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "barAlignment": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "normal", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "warn" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "orange" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "info" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "blue" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "debug" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "purple" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "unknown" + }, + "properties": [ + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "text" + } + } + ] + } + ] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 27, + "type": "logs", + "title": "Recent logs", + "description": "Oldest first, so the newest line is at the bottom and the panel opens scrolled to it. Each line is prefixed with its service. The prefix is added by the query with `line_format`, because the panel's displayed-fields option replaces the log line rather than prefixing it - so a copied line carries the prefix too, and Prettify JSON no longer applies to lines that were pure JSON. Level filtering, search, wrapping and sort live in the panel's own controls on the right. Clicking a field in a line's details adds it to the Log filters control at the top, which narrows every Loki panel. Lines carrying `trace_id` link to the trace in Tempo.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 12, + "w": 24, + "x": 0, + "y": 107 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service_name=~\"${logservice:pipe}\", service_name!~\"$exclude\"} | line_format \"{{ printf \\\"%-16s\\\" .service_name }} {{ __line__ }}\"", + "queryType": "range", + "maxLines": 500, + "editorMode": "code" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Ascending", + "fontSize": "small", + "syntaxHighlighting": true, + "showLevel": true, + "showControls": true, + "controlsStorageKey": "gen3-local-recent" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "links": [ + { + "title": "Open in Explore (Loki)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22loki%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22loki%22%2C%22uid%22%3A%22loki%22%7D%2C%22expr%22%3A%22%7Bservice_name%3D~%5C%22${logservice:pipe}%5C%22%7D%22%2C%22queryType%22%3A%22range%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + } + ] + }, + { + "id": 26, + "type": "logs", + "title": "Errors and warnings", + "description": "The same lines and the same service prefix, filtered to the level Loki inferred - so a service logging its own severity in a format Loki cannot read is missing here even while it fails. Search and the display toggles are in the panel's controls; the level filter there narrows this further rather than widening it, since the query is already fixed to error and warn.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 119 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "expr": "{service_name=~\"${logservice:pipe}\", service_name!~\"$exclude\"} | detected_level =~ \"error|warn\" | line_format \"{{ printf \\\"%-16s\\\" .service_name }} {{ __line__ }}\"", + "queryType": "range", + "maxLines": 500, + "editorMode": "code" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Ascending", + "fontSize": "small", + "syntaxHighlighting": true, + "showLevel": true, + "showControls": true, + "controlsStorageKey": "gen3-local-errors" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "links": [ + { + "title": "Open in Explore (Loki)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22loki%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22loki%22%2C%22uid%22%3A%22loki%22%7D%2C%22expr%22%3A%22%7Bservice_name%3D~%5C%22${logservice:pipe}%5C%22%7D%20%7C%20detected_level%20%3D~%20%5C%22error%7Cwarn%5C%22%22%2C%22queryType%22%3A%22range%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + } + ] + } + ] +} diff --git a/examples/local_alloy_values.yaml b/examples/local_alloy_values.yaml index 767a0c3f8..a863d8d56 100644 --- a/examples/local_alloy_values.yaml +++ b/examples/local_alloy_values.yaml @@ -47,9 +47,10 @@ alloy: memory: 256Mi # Copied from helm/alloy/values.yaml, changing the three write endpoints and the two external - # labels, and inserting the loki.process stage that follows the pod log source. That stage is - # local-only: the chart has no equivalent, so trace-to-logs correlation works here and not in a - # cluster deployed from helm/alloy. + # labels, and inserting two things the chart has no equivalent of: the loki.process stage that + # follows the pod log source, and the cAdvisor scrape. Both are local-only, so trace-to-logs + # correlation and per-container memory and CPU work here and not in a cluster deployed from + # helm/alloy. See docs/otel-logs-and-traces.md. # # The labels are written out literally on purpose. templates/alloy-config.yaml renders this # with toYaml rather than tpl, so any {{ }} left in here reaches the ConfigMap unrendered. @@ -454,6 +455,58 @@ alloy: forward_to = [prometheus.relabel.metrics_service.receiver] } + // cAdvisor. Local-only for now; see docs/otel-logs-and-traces.md. + // + // The kubelet serves this on a second path, so it needs its own scrape rather than a + // longer keep list on the kubelet one. + discovery.relabel "cadvisor" { + targets = discovery.kubernetes.nodes.targets + rule { + target_label = "__address__" + replacement = "kubernetes.default.svc.cluster.local:443" + } + rule { + source_labels = ["__meta_kubernetes_node_name"] + regex = "(.+)" + replacement = "/api/v1/nodes/${1}/proxy/metrics/cadvisor" + target_label = "__metrics_path__" + } + } + + prometheus.scrape "cadvisor" { + job_name = "integrations/kubernetes/cadvisor" + targets = discovery.relabel.cadvisor.output + scheme = "https" + scrape_interval = "60s" + bearer_token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" + tls_config { + insecure_skip_verify = true + } + clustering { + enabled = true + } + forward_to = [prometheus.relabel.cadvisor.receiver] + } + + prometheus.relabel "cadvisor" { + rule { + source_labels = ["__name__"] + regex = "container_memory_working_set_bytes|container_memory_rss|container_spec_memory_limit_bytes|container_cpu_usage_seconds_total|container_cpu_cfs_periods_total|container_cpu_cfs_throttled_periods_total" + action = "keep" + } + + // cAdvisor reports one series per cgroup level. The pod and slice rollups carry an + // empty container label and sum the containers beneath them, so keeping both counts + // every container twice. + rule { + source_labels = ["container"] + regex = "" + action = "drop" + } + + forward_to = [prometheus.relabel.metrics_service.receiver] + } + // Cluster Events loki.source.kubernetes_events "cluster_events" { job_name = "integrations/kubernetes/eventhandler" diff --git a/examples/local_lgtm.yaml b/examples/local_lgtm.yaml index a8df42ec6..a89229217 100644 --- a/examples/local_lgtm.yaml +++ b/examples/local_lgtm.yaml @@ -8,7 +8,8 @@ # services can push profiles. Both already listen inside the image; a Service that omits the port # drops the traffic rather than refusing it. # -# For local development only. One replica, emptyDir storage so everything is lost on restart, +# For local development only. One replica, emptyDir storage for the telemetry so it is lost on +# restart while dashboards persist on a small claim, # no ingress, no TLS, and Grafana's default admin/admin. The observability chart is the answer # for a deployed cluster. --- @@ -17,6 +18,23 @@ kind: Namespace metadata: name: monitoring --- +# Dashboards outlive the pod; telemetry does not. kind's default local-path StorageClass keeps +# this on the node, so it survives a pod restart and a `docker restart`, and goes away with +# `kind delete cluster` or by deleting the claim - which is how you reset Grafana. +# +# examples/dashboards/ stays the source of truth. Editing a dashboard in the UI now sticks, so +# export it back to the file or the next fresh cluster will not have the change. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: lgtm-grafana + namespace: monitoring +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 1Gi +--- apiVersion: apps/v1 kind: Deployment metadata: @@ -45,6 +63,13 @@ spec: # of each pod's log rather than from now. - name: LOKI_EXTRA_ARGS value: "-validation.max-label-names-per-series=32 -validation.reject-old-samples=false" + # Pyroscope refuses any query longer than its default max_query_length of 1d with + # "the query time range exceeds the limit", which in Grafana surfaces only as + # "An error occurred within the plugin" on every profile panel at once. 0 disables + # the check. max-query-lookback stays at its 1w default, so a longer window returns + # the last week rather than failing. + - name: PYROSCOPE_EXTRA_ARGS + value: "-querier.max-query-length=0" ports: - { name: grafana, containerPort: 3000 } - { name: loki, containerPort: 3100 } @@ -71,7 +96,12 @@ spec: - { name: pyroscope-storage, mountPath: /data/pyroscope } volumes: - { name: tempo-data, emptyDir: {} } - - { name: grafana-data, emptyDir: {} } + # Only Grafana's own volume persists: GF_PATHS_DATA is /data/grafana/data, so + # this is the dashboard database and the plugin directory and nothing else. + # Metrics, logs, traces and profiles stay on emptyDir and start clean. + - name: grafana-data + persistentVolumeClaim: + claimName: lgtm-grafana - { name: loki-data, emptyDir: {} } - { name: loki-storage, emptyDir: {} } - { name: p8s-storage, emptyDir: {} } diff --git a/helm/hatchery/README.md b/helm/hatchery/README.md index e83cb74c5..23c0e1528 100644 --- a/helm/hatchery/README.md +++ b/helm/hatchery/README.md @@ -131,6 +131,7 @@ Published versions of this chart are listed in the | tolerations | list | `[]` | Tolerations to use for the deployment. | | volumeMounts | list | `[{"mountPath":"/var/hatchery/hatchery.json","name":"hatchery-config","readOnly":true,"subPath":"json"},{"mountPath":"/hatchery.json","name":"hatchery-config","readOnly":true,"subPath":"json"}]` | Volumes to mount to the container. | | volumes | list | `[{"configMap":{"name":"manifest-hatchery"},"name":"hatchery-config"}]` | Volumes to attach to the container. | +| workspaceInternetEgress | bool | `true` | | | workspaceLaunchTest | map | `{"enabled":false,"hostname":"https://example.com","operatorName":"username","schedule":"*/20 * * * *","workspaceImages":"(Generic) Jupyter Lab Notebook with R Kernel"}` | Configuration for workspace launch test crobjob | | workspaceLaunchTest.enabled | bool | `false` | Whether the workspace launch test cron job is enabled. | | workspaceLaunchTest.hostname | string | `"https://example.com"` | Hostname for the workspace launch test operator. | diff --git a/helm/revproxy/README.md b/helm/revproxy/README.md index 4fd02f186..c34724fe3 100644 --- a/helm/revproxy/README.md +++ b/helm/revproxy/README.md @@ -81,9 +81,9 @@ Published versions of this chart are listed in the | ingress.tls | list | `[]` | To secure an Ingress by specifying a secret that contains a TLS private key and certificate. | | metricsEnabled | bool | `nil` | Whether Metrics are enabled. | | nameOverride | string | `""` | Override the name of the chart. | -| netPolicy | map | `{"egressApps":["portal","sowerjob"],"ingressApps":["portal","sowerjob"]}` | Configuration for network policies created by this chart. Only relevant if "global.netPolicy.enabled" is set to true | +| netPolicy | map | `{"egressApps":["portal","sowerjob"],"ingressApps":["portal","sowerjob","ssjdispatcherjob"]}` | Configuration for network policies created by this chart. Only relevant if "global.netPolicy.enabled" is set to true | | netPolicy.egressApps | array | `["portal","sowerjob"]` | List of apps that this app requires egress to | -| netPolicy.ingressApps | array | `["portal","sowerjob"]` | List of app labels that require ingress to this service | +| netPolicy.ingressApps | array | `["portal","sowerjob","ssjdispatcherjob"]` | List of app labels that require ingress to this service | | nginx.pidFile | string | `"/var/run/nginx.pid"` | | | nginx.resolver | string | `"kube-dns.kube-system.svc.cluster.local"` | | | nginx.user | string | `"nginx"` | | From 0bfe19f70c75858157b48e11de9ac9808057b17c Mon Sep 17 00:00:00 2001 From: avantol Date: Mon, 31 Aug 2026 15:00:49 -0500 Subject: [PATCH 05/23] chore(observability): update default grafana chart with more info --- .../scripts/regenerate_local_alloy_values.py | 4 +- docs/local-observability.md | 5 + docs/otel-logs-and-traces.md | 5 +- examples/dashboards/local-services.json | 1186 ++++++++++++++--- examples/local_alloy_values.yaml | 2 +- 5 files changed, 1030 insertions(+), 172 deletions(-) diff --git a/.github/scripts/regenerate_local_alloy_values.py b/.github/scripts/regenerate_local_alloy_values.py index b0c220ea8..7d836fa6c 100644 --- a/.github/scripts/regenerate_local_alloy_values.py +++ b/.github/scripts/regenerate_local_alloy_values.py @@ -102,7 +102,7 @@ # the kubelet's own /metrics, which carries no per-container memory or CPU, so nothing # in a cluster deployed from helm/alloy reports what a pod actually uses. ' // Cluster Events\n', - ' // cAdvisor. Local-only; see docs/otel-logs-and-traces.md.\n' + ' // cAdvisor. Local-only for now; see docs/otel-logs-and-traces.md.\n' " //\n" " // The kubelet serves this on a second path, so it needs its own scrape rather than a\n" " // longer keep list on the kubelet one.\n" @@ -138,7 +138,7 @@ ' prometheus.relabel "cadvisor" {\n' " rule {\n" ' source_labels = ["__name__"]\n' - ' regex = "container_memory_working_set_bytes|container_memory_rss|container_spec_memory_limit_bytes|container_cpu_usage_seconds_total|container_cpu_cfs_periods_total|container_cpu_cfs_throttled_periods_total"\n' + ' regex = "container_memory_working_set_bytes|container_memory_rss|container_spec_memory_limit_bytes|container_cpu_usage_seconds_total|container_cpu_cfs_periods_total|container_cpu_cfs_throttled_periods_total|container_spec_cpu_quota|container_spec_cpu_period|container_oom_events_total"\n' ' action = "keep"\n' " }\n" "\n" diff --git a/docs/local-observability.md b/docs/local-observability.md index 867644556..b3abb56c9 100644 --- a/docs/local-observability.md +++ b/docs/local-observability.md @@ -243,6 +243,11 @@ Leave the panel's unit unset. The datasource stamps a unit on each series, and a panel options overrides it - which is also why one panel holding both CPU and memory queries can label neither correctly. One unit per panel. +`groupBy` is not limited to `service_name`. Pyroscope labels each sample with the `span_name` that +was active when it was taken, so grouping by that attributes CPU to a route - the `CPU by endpoint` +panel. Samples taken outside a span carry no `span_name`, so background work and startup drop out of +any panel grouped that way. + ## When a log line or trace link does not show up Alloy tails every pod's containers by default, so absent logs are usually a write or a label diff --git a/docs/otel-logs-and-traces.md b/docs/otel-logs-and-traces.md index 5fefe298d..ec52432a9 100644 --- a/docs/otel-logs-and-traces.md +++ b/docs/otel-logs-and-traces.md @@ -330,7 +330,10 @@ Not required for the ArgoCD path, but it should probably not stay broken. Groupe - No cAdvisor scrape. The chart scrapes the kubelet's own `/metrics`, which carries operational counters and no per-container resource usage, so no cluster deployed from this chart reports what a pod's memory or CPU actually is - `container_memory_working_set_bytes`, - `container_spec_memory_limit_bytes` and `container_cpu_usage_seconds_total` are all absent, and + `container_spec_memory_limit_bytes`, `container_cpu_usage_seconds_total`, the CFS throttling + counters, `container_spec_cpu_quota` / `container_spec_cpu_period` and + `container_oom_events_total` are all absent, so neither a container's real footprint nor the + reason it was killed or throttled can be seen, and the `kube_state_metrics` scrape alongside it targets a `monitoring-extras-kube-state-metrics` service that this repo never creates. The kubelet serves cAdvisor on a second path (`/metrics/cadvisor`) reached through the same node proxy and the same service account token, so diff --git a/examples/dashboards/local-services.json b/examples/dashboards/local-services.json index 9ba74eec5..c8621c079 100644 --- a/examples/dashboards/local-services.json +++ b/examples/dashboards/local-services.json @@ -2,7 +2,7 @@ "id": null, "uid": "gen3-local-services", "title": "Local services - overview", - "description": "Resources, requests, traces, service metrics and logs for the local kind cluster, one row per signal. CPU and memory are profiler-reported and come from Python services today: only services pushing profiles appear there, and sampled heap is not the pod's resident memory - the Container memory row is.", + "description": "Resources, requests, traces, service metrics and logs for the local kind cluster, one row per signal. The CPU and Memory rows are profiler-reported, so only services running a profiler appear there and sampled heap is not the pod's resident memory - the Container resources row is.", "tags": [ "local", "observability" @@ -209,7 +209,7 @@ "id": 3, "type": "stat", "title": "Heap in use now", - "description": "Latest sampled heap in use: the bytes the profiler saw allocated and still live. This is not the pod's memory usage, which is resident set size - the memory the kernel actually holds in RAM for the process. For the Python services profiled here, resident memory also covers the Python interpreter itself, its shared libraries, buffers allocated inside C extensions, and memory CPython's allocator has freed but kept back from the OS, none of which reaches a profile; expect an order of magnitude between the two figures. A Go service would be sampled the same way, with the Go runtime's stacks and spans outside the profile instead. Read the Container memory row for the footprint and this panel for which code is holding objects.", + "description": "Latest sampled heap in use: the bytes the profiler saw allocated and still live. It is not what the kernel charges the container, which also covers the runtime itself, its shared libraries, buffers allocated inside C extensions, and memory the allocator has freed but kept back from the OS - for the Python services profiled here that is CPython's allocator, and a Go service would instead hold the runtime's stacks and spans. Expect an order of magnitude between the two figures. Read the Container memory row for what a limit is enforced against, and this panel for which code is holding objects.", "datasource": { "type": "grafana-pyroscope-datasource", "uid": "pyroscope" @@ -274,7 +274,7 @@ { "id": 4, "type": "row", - "title": "CPU", + "title": "CPU (profiler samples)", "collapsed": false, "panels": [], "gridPos": { @@ -445,7 +445,7 @@ { "id": 7, "type": "row", - "title": "Memory", + "title": "Memory (profiler samples)", "collapsed": false, "panels": [], "gridPos": { @@ -539,7 +539,7 @@ "id": 8, "type": "timeseries", "title": "Heap in use", - "description": "Bytes the profiler sampled as allocated and still live. Useful for which code holds objects and whether that is growing, not for footprint - the Container memory row below is the footprint, and is normally an order of magnitude larger.", + "description": "Bytes the profiler sampled as allocated and still live - what the program itself is holding. Read it for which code holds objects and whether that is growing. For what the kernel charges the container, which is what a limit is enforced against, see the Container memory row.", "datasource": { "type": "grafana-pyroscope-datasource", "uid": "pyroscope" @@ -698,7 +698,7 @@ { "id": 30, "type": "row", - "title": "Container memory (real usage)", + "title": "Container memory (kernel accounting)", "collapsed": false, "panels": [], "gridPos": { @@ -712,14 +712,14 @@ "id": 28, "type": "timeseries", "title": "Container memory (working set) by pod", - "description": "Working set is the number Kubernetes evicts and OOM-kills on - resident memory minus the page cache it could reclaim - so it is the pod's real footprint, unlike the sampled heap above. Dashed lines are each container's limit. A container with no limit has no line, because cAdvisor reports its limit as 0 rather than leaving the series out. Not filtered by the Service picker: these series are keyed by pod and container, which carry neither spelling of the name the picker holds.", + "description": "Working set as the kernel accounts it: pages charged to the container's cgroup, minus the page cache it could reclaim under pressure. This is the number Kubernetes evicts and OOM-kills on, so it decides whether the pod lives - but it counts memory the process is not using, including the interpreter, shared libraries and anything its allocator has held back from the OS. The profiler rows above count what the program allocated and still holds. Neither is wrong; they answer different questions, and the gap between them is the runtime's overhead. Dashed lines are each container's limit; a container without one has no line, because cAdvisor reports its limit as 0 rather than leaving the series out. Not filtered by the Service picker: these series are keyed by pod and container.", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 9, - "w": 24, + "w": 16, "x": 0, "y": 27 }, @@ -822,6 +822,85 @@ } } }, + { + "id": 34, + "type": "timeseries", + "title": "OOM kills", + "description": "Out-of-memory kills counted by the kernel. Flat at zero until something is killed, which is the point: a container that vanished mid-request with nothing in its own logs shows up here and nowhere else. Read it beside the memory table - a pod killed at its limit was at 100% there a moment earlier. Not filtered by the Service picker: these series are keyed by pod and container, which carry neither spelling of the name the picker holds.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 27 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (pod) (increase(container_oom_events_total{container!=\"\", namespace=~\"default|monitoring\"}[$__rate_interval]))", + "legendFormat": "{{pod}}", + "editorMode": "code", + "range": true, + "instant": false + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 60, + "gradientMode": "opacity", + "showPoints": "never", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "barAlignment": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "sum" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, { "id": 29, "type": "table", @@ -1052,9 +1131,9 @@ } }, { - "id": 14, + "id": 40, "type": "row", - "title": "Requests (from traces)", + "title": "Container CPU (kernel accounting)", "collapsed": false, "panels": [], "gridPos": { @@ -1065,10 +1144,10 @@ } }, { - "id": 15, + "id": 31, "type": "timeseries", - "title": "Request rate by endpoint", - "description": "Inbound server spans per second, by route template. Derived by Tempo's metrics generator from the spans it receives, not exported by the service. The series exist only while traces arrive, so a quiet service leaves a gap rather than a zero.", + "title": "Container CPU by pod", + "description": "CPU the container actually consumed, from cAdvisor - the counterpart to the profiler's CPU row above, and the pair that shows how much CPU the profiler never sees. 0.02 cores means 2% of one core. Not filtered by the Service picker: these series are keyed by pod and container, which carry neither spelling of the name the picker holds.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -1086,27 +1165,28 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum by (service, span_name) (rate(traces_spanmetrics_calls_total{service=~\"${service:pipe}\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))", - "legendFormat": "{{service}} {{span_name}}", + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{container!=\"\", namespace=~\"default|monitoring\"}[$__rate_interval]))", + "legendFormat": "{{pod}}", + "editorMode": "code", "range": true, - "editorMode": "code" + "instant": false } ], "fieldConfig": { "defaults": { - "unit": "reqps", "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 8, "gradientMode": "opacity", - "showPoints": "auto", + "showPoints": "never", "pointSize": 4, "spanNulls": false, "axisSoftMin": 0, "axisPlacement": "auto", "axisBorderShow": false, + "barAlignment": 0, "scaleDistribution": { "type": "linear" }, @@ -1123,13 +1203,7 @@ "mode": "off" } }, - "links": [ - { - "title": "Traces for this service (Explore)", - "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22tempo%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22tempo%22%2C%22uid%22%3A%22tempo%22%7D%2C%22queryType%22%3A%22traceql%22%2C%22query%22%3A%22%7Bresource.service.name%3D%5C%22${__field.labels.service}%5C%22%7D%22%2C%22limit%22%3A50%2C%22tableType%22%3A%22traces%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", - "targetBlank": true - } - ] + "unit": "cores" }, "overrides": [] }, @@ -1150,10 +1224,10 @@ } }, { - "id": 16, + "id": 32, "type": "timeseries", - "title": "Latency p95 by endpoint", - "description": "95th percentile of inbound server span duration. Derived by Tempo's metrics generator from the spans it receives, not exported by the service. The series exist only while traces arrive, so a quiet service leaves a gap rather than a zero.", + "title": "CPU used vs limit", + "description": "Consumption as a share of the container's own CPU limit. Only containers that declare a limit appear; the rest are absent rather than shown at zero, since there is nothing to be a share of. Check this panel before reading CPU throttling: a flat zero there means headroom if a limit exists here, and nothing at all if it does not. Not filtered by the Service picker: these series are keyed by pod and container.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -1171,27 +1245,28 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "histogram_quantile(0.95, sum by (service, span_name, le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:pipe}\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", - "legendFormat": "{{service}} {{span_name}}", + "expr": "sum by (pod) (rate(container_cpu_usage_seconds_total{container!=\"\", namespace=~\"default|monitoring\"}[$__rate_interval])) / (sum by (pod) (container_spec_cpu_quota{container!=\"\", namespace=~\"default|monitoring\"} / container_spec_cpu_period{container!=\"\", namespace=~\"default|monitoring\"}) > 0)", + "legendFormat": "{{pod}}", + "editorMode": "code", "range": true, - "editorMode": "code" + "instant": false } ], "fieldConfig": { "defaults": { - "unit": "s", "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 8, "gradientMode": "opacity", - "showPoints": "auto", + "showPoints": "never", "pointSize": 4, "spanNulls": false, "axisSoftMin": 0, "axisPlacement": "auto", "axisBorderShow": false, + "barAlignment": 0, "scaleDistribution": { "type": "linear" }, @@ -1208,13 +1283,7 @@ "mode": "off" } }, - "links": [ - { - "title": "Traces for this service (Explore)", - "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22tempo%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22tempo%22%2C%22uid%22%3A%22tempo%22%7D%2C%22queryType%22%3A%22traceql%22%2C%22query%22%3A%22%7Bresource.service.name%3D%5C%22${__field.labels.service}%5C%22%7D%22%2C%22limit%22%3A50%2C%22tableType%22%3A%22traces%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", - "targetBlank": true - } - ] + "unit": "percentunit" }, "overrides": [] }, @@ -1235,10 +1304,10 @@ } }, { - "id": 17, + "id": 33, "type": "timeseries", - "title": "Error rate by endpoint", - "description": "Spans of any kind whose status is ERROR. Empty until a span actually carries that status - an HTTP 500 that the service handles without marking its span is invisible here. Derived by Tempo's metrics generator from the spans it receives, not exported by the service. The series exist only while traces arrive, so a quiet service leaves a gap rather than a zero.", + "title": "CPU throttling", + "description": "The share of scheduling periods in which the kernel stopped the container after it spent its quota. Only a container with a CPU limit can throttle, so a flat zero here means either headroom or no limit - check CPU used vs limit to tell which. This is the usual reason a service is slow while its own CPU graph looks unremarkable. Not filtered by the Service picker: these series are keyed by pod and container, which carry neither spelling of the name the picker holds.", "datasource": { "type": "prometheus", "uid": "prometheus" @@ -1256,27 +1325,28 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum by (service, span_kind, span_name) (rate(traces_spanmetrics_calls_total{service=~\"${service:pipe}\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval]))", - "legendFormat": "{{service}} {{span_kind}} {{span_name}}", + "expr": "sum by (pod) (rate(container_cpu_cfs_throttled_periods_total{container!=\"\", namespace=~\"default|monitoring\"}[$__rate_interval])) / sum by (pod) (rate(container_cpu_cfs_periods_total{container!=\"\", namespace=~\"default|monitoring\"}[$__rate_interval]))", + "legendFormat": "{{pod}}", + "editorMode": "code", "range": true, - "editorMode": "code" + "instant": false } ], "fieldConfig": { "defaults": { - "unit": "reqps", "custom": { "drawStyle": "line", "lineInterpolation": "linear", "lineWidth": 2, "fillOpacity": 8, "gradientMode": "opacity", - "showPoints": "auto", + "showPoints": "never", "pointSize": 4, "spanNulls": false, "axisSoftMin": 0, "axisPlacement": "auto", "axisBorderShow": false, + "barAlignment": 0, "scaleDistribution": { "type": "linear" }, @@ -1293,18 +1363,7 @@ "mode": "off" } }, - "links": [ - { - "title": "Error traces for this service (Explore)", - "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22tempo%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22tempo%22%2C%22uid%22%3A%22tempo%22%7D%2C%22queryType%22%3A%22traceql%22%2C%22query%22%3A%22%7Bresource.service.name%3D%5C%22${__field.labels.service}%5C%22%20%26%26%20status%3Derror%7D%22%2C%22limit%22%3A50%2C%22tableType%22%3A%22traces%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", - "targetBlank": true - }, - { - "title": "Error logs for this service (Explore)", - "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22loki%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22loki%22%2C%22uid%22%3A%22loki%22%7D%2C%22expr%22%3A%22%7Bservice_name%3D~%5C%22${__field.labels.service}%5C%22%7D%20%7C%20detected_level%20%3D~%20%5C%22error%7Cwarn%5C%22%22%2C%22queryType%22%3A%22range%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", - "targetBlank": true - } - ] + "unit": "percentunit" }, "overrides": [] }, @@ -1325,91 +1384,32 @@ } }, { - "id": 18, - "type": "table", - "title": "Recent traces", - "description": "Newest traces for the selected services. Click a Trace ID to open it in Tempo; its logs link resolves through Loki's trace_id metadata.", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "gridPos": { - "h": 9, - "w": 24, - "x": 0, - "y": 64 - }, - "targets": [ - { - "refId": "A", - "datasource": { - "type": "tempo", - "uid": "tempo" - }, - "queryType": "traceql", - "query": "{resource.service.name=~\"${service:pipe}\"}", - "limit": 20, - "tableType": "traces", - "filters": [] - } - ], - "fieldConfig": { - "defaults": { - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - } - } - }, - "overrides": [] - }, - "options": { - "showHeader": true, - "cellHeight": "sm", - "sortBy": [ - { - "displayName": "Start time", - "desc": true - } - ], - "footer": { - "show": false, - "reducer": [ - "sum" - ], - "countRows": false, - "fields": "" - } - } - }, - { - "id": 19, + "id": 14, "type": "row", - "title": "Service metrics (scraped from /metrics)", + "title": "Requests (from traces)", "collapsed": false, "panels": [], "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 73 + "y": 64 } }, { - "id": 20, + "id": 15, "type": "timeseries", - "title": "Request rate by status (all services)", - "description": "Counted by the service itself through cdispyutils request_metrics and read from its /metrics, so it is independent of the trace pipeline and survives a service that never exports a span. Selected by metric name rather than by service, so any service using that helper appears without editing this dashboard. Ignores the Service picker: these series carry `app` from the Kubernetes label (`gen3-embeddings`), which is spelled differently from the `service_name` the picker holds (`gen3_embeddings`), and no variable format converts between the two.", + "title": "Request rate by endpoint", + "description": "Inbound server spans per second, by route template. Derived by Tempo's metrics generator from the spans it receives, not exported by the service. The series exist only while traces arrive, so a quiet service leaves a gap rather than a zero.", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 9, - "w": 24, + "w": 12, "x": 0, - "y": 74 + "y": 65 }, "targets": [ { @@ -1418,8 +1418,8 @@ "type": "prometheus", "uid": "prometheus" }, - "expr": "sum by (app, status_code) (rate({__name__=~\".+_api_requests_total\"}[$__rate_interval]))", - "legendFormat": "{{app}} {{status_code}}", + "expr": "sum by (service, span_name) (rate(traces_spanmetrics_calls_total{service=~\"${service:pipe}\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))", + "legendFormat": "{{service}} {{span_name}}", "range": true, "editorMode": "code" } @@ -1454,7 +1454,14 @@ "thresholdsStyle": { "mode": "off" } - } + }, + "links": [ + { + "title": "Traces for this service (Explore)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22tempo%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22tempo%22%2C%22uid%22%3A%22tempo%22%7D%2C%22queryType%22%3A%22traceql%22%2C%22query%22%3A%22%7Bresource.service.name%3D%5C%22${__field.labels.service}%5C%22%7D%22%2C%22limit%22%3A50%2C%22tableType%22%3A%22traces%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + } + ] }, "overrides": [] }, @@ -1475,44 +1482,808 @@ } }, { - "id": 11, - "type": "row", - "title": "Profiles ($service)", - "collapsed": false, - "panels": [], - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 83 - } - }, - { - "id": 12, - "type": "flamegraph", - "title": "CPU profile", - "description": "Merged across every service selected in the Service picker. Pick a single service to read it. The totals are summed over the dashboard's time range, so they grow as the range widens rather than describing a moment.", + "id": 16, + "type": "timeseries", + "title": "Latency p95 by endpoint", + "description": "95th percentile of inbound server span duration. The marks are exemplars: individual requests carrying their trace ID, so a spike can be opened as the trace that caused it. Derived by Tempo's metrics generator from the spans it receives, not exported by the service, so the series exist only while traces arrive and a quiet service leaves a gap rather than a zero.", "datasource": { - "type": "grafana-pyroscope-datasource", - "uid": "pyroscope" + "type": "prometheus", + "uid": "prometheus" }, "gridPos": { - "h": 13, + "h": 9, "w": 12, - "x": 0, - "y": 84 + "x": 12, + "y": 65 }, "targets": [ { "refId": "A", "datasource": { - "type": "grafana-pyroscope-datasource", - "uid": "pyroscope" + "type": "prometheus", + "uid": "prometheus" }, - "queryType": "profile", - "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", - "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", - "groupBy": [] + "expr": "histogram_quantile(0.95, sum by (service, span_name, le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:pipe}\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval])))", + "legendFormat": "{{service}} {{span_name}}", + "range": true, + "editorMode": "code", + "exemplar": true + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "title": "Traces for this service (Explore)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22tempo%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22tempo%22%2C%22uid%22%3A%22tempo%22%7D%2C%22queryType%22%3A%22traceql%22%2C%22query%22%3A%22%7Bresource.service.name%3D%5C%22${__field.labels.service}%5C%22%7D%22%2C%22limit%22%3A50%2C%22tableType%22%3A%22traces%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + } + ] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 36, + "type": "heatmap", + "title": "Latency heatmap", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "The full distribution of inbound span duration, not just its 95th percentile: two bands rather than one is the shape a p95 line hides, typically a fast path and a slow one. The magenta marks are exemplars - each one is a real request that landed in that cell, carrying its trace ID, so a hot spot can be opened as a trace rather than guessed at. Hovering names the service and span; clicking follows it to Tempo. Buckets and exemplars both come from Tempo's metrics generator, so they exist only while traces arrive.", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 74 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:pipe}\", span_kind=\"SPAN_KIND_SERVER\"}[$__rate_interval]))", + "legendFormat": "{{le}}", + "editorMode": "code", + "range": true, + "instant": false, + "format": "heatmap", + "exemplar": true + } + ], + "options": { + "calculate": false, + "cellGap": 1, + "cellValues": { + "unit": "short" + }, + "color": { + "mode": "scheme", + "scheme": "Spectral", + "steps": 64, + "reverse": false, + "exponent": 0.5, + "fill": "dark-orange" + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-09 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + } + }, + { + "id": 37, + "type": "table", + "title": "Slowest endpoints", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "p95 per route across the whole time range shown, ranked, so the worst endpoint names itself instead of being picked out of a crowded legend. It reads the range rather than a rate interval on purpose: a p95 over the last few minutes is empty whenever traffic is not arriving right now. Routes with no samples in the range are dropped rather than listed blank. Server spans only.", + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 74 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "topk(10, histogram_quantile(0.95, sum by (service, span_name, le) (rate(traces_spanmetrics_latency_bucket{service=~\"${service:pipe}\", span_kind=\"SPAN_KIND_SERVER\"}[$__range]))) > 0)", + "legendFormat": "", + "editorMode": "code", + "range": false, + "instant": true, + "format": "table" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "service": "Service", + "span_name": "Endpoint", + "Value": "p95", + "Value #A": "p95" + }, + "indexByName": {}, + "includeByName": {} + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.width", + "value": 110 + }, + { + "id": "custom.cellOptions", + "value": { + "type": "color-text" + } + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "p95", + "desc": true + } + ] + } + }, + { + "id": 17, + "type": "timeseries", + "title": "Error rate by span", + "description": "Spans whose status is ERROR, of every kind - not just endpoints. A server span is a request that failed, a client span is a call this service made that failed, and an internal span is a step inside it, which is how a failure that propagates up a call chain shows as several series at once. Empty until a span actually carries that status: a handled 500 that never marks its span is invisible here. Derived by Tempo's metrics generator from the spans it receives, not exported by the service, so the series exist only while traces arrive.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 83 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (service, span_kind, span_name) (rate(traces_spanmetrics_calls_total{service=~\"${service:pipe}\", status_code=\"STATUS_CODE_ERROR\"}[$__rate_interval]))", + "legendFormat": "{{service}} {{span_kind}} {{span_name}}", + "range": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [ + { + "title": "Error traces for this service (Explore)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22tempo%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22tempo%22%2C%22uid%22%3A%22tempo%22%7D%2C%22queryType%22%3A%22traceql%22%2C%22query%22%3A%22%7Bresource.service.name%3D%5C%22${__field.labels.service}%5C%22%20%26%26%20status%3Derror%7D%22%2C%22limit%22%3A50%2C%22tableType%22%3A%22traces%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + }, + { + "title": "Error logs for this service (Explore)", + "url": "/explore?schemaVersion=1&orgId=1&panes=%7B%22dl%22%3A%7B%22datasource%22%3A%22loki%22%2C%22queries%22%3A%5B%7B%22refId%22%3A%22A%22%2C%22datasource%22%3A%7B%22type%22%3A%22loki%22%2C%22uid%22%3A%22loki%22%7D%2C%22expr%22%3A%22%7Bservice_name%3D~%5C%22${__field.labels.service}%5C%22%7D%20%7C%20detected_level%20%3D~%20%5C%22error%7Cwarn%5C%22%22%2C%22queryType%22%3A%22range%22%7D%5D%2C%22range%22%3A%7B%22from%22%3A%22${__from}%22%2C%22to%22%3A%22${__to}%22%7D%7D%7D", + "targetBlank": true + } + ] + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 18, + "type": "table", + "title": "Recent traces", + "description": "Newest traces for the selected services. Click a Trace ID to open it in Tempo; its logs link resolves through Loki's trace_id metadata.", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 90 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "traceql", + "query": "{resource.service.name=~\"${service:pipe}\"}", + "limit": 20, + "tableType": "traces", + "filters": [] + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + } + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "sortBy": [ + { + "displayName": "Start time", + "desc": true + } + ], + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + } + } + }, + { + "id": 38, + "type": "nodeGraph", + "title": "Service graph", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "description": "Who calls whom, derived by Tempo from span pairs rather than declared anywhere. Nodes are not all services: a datastore appears as its own node, and `user` is a request that arrived with no parent span, meaning anything entering from outside. Grafana draws those two with a dashed outline because Tempo marks them un-instrumented, which is the only labelling the panel supports - the Service dependencies table beside it names each edge in words. Edges carry request rate and latency, so an N+1 query pattern shows here before it shows in CPU. Needs traffic, and ignores both pickers.", + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 99 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "tempo", + "uid": "tempo" + }, + "queryType": "serviceMap" + } + ], + "options": {}, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + }, + { + "id": 41, + "type": "table", + "title": "Service dependencies", + "description": "The same edges the graph draws, named. Kind is Tempo's `connection_type`: a blank one is service to service, `database` is a datastore the client queried, and `virtual_node` is not a caller at all but a request that arrived with no parent span - anything entering from outside the traced system. The graph cannot say this in text: its node subtitle is reserved for the service namespace, and its only marker for a non-service node is a dashed outline. Database names come from the span's database attributes, which for these charts is the `_` name the chart generates, so a service and its own database can appear under two different spellings.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 99 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "instant": true, + "range": false, + "format": "table", + "expr": "sum by (client, server, connection_type) (rate(traces_service_graph_request_total[$__range]))" + } + ], + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true + }, + "renameByName": { + "client": "Calls from", + "server": "Calls to", + "connection_type": "Kind", + "Value": "Rate", + "Value #A": "Rate" + }, + "indexByName": {}, + "includeByName": {} + } + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "filterable": true + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Kind" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "type": "value", + "options": { + "": { + "text": "service to service", + "index": 0 + }, + "database": { + "text": "database query", + "index": 1 + }, + "virtual_node": { + "text": "entering from outside", + "index": 2 + }, + "messaging_system": { + "text": "queue", + "index": 3 + } + } + } + ] + }, + { + "id": "custom.width", + "value": 190 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Rate" + }, + "properties": [ + { + "id": "unit", + "value": "reqps" + }, + { + "id": "custom.width", + "value": 110 + }, + { + "id": "decimals", + "value": 4 + } + ] + } + ] + }, + "options": { + "showHeader": true, + "cellHeight": "sm", + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "sortBy": [ + { + "displayName": "Rate", + "desc": true + } + ] + } + }, + { + "id": 19, + "type": "row", + "title": "Service metrics (scraped from /metrics)", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 108 + } + }, + { + "id": 20, + "type": "timeseries", + "title": "Request rate by status (all services)", + "description": "Counted by the service itself through cdispyutils request_metrics and read from its /metrics, so it is independent of the trace pipeline and survives a service that never exports a span. Selected by metric name (`.+_api_requests_total`) rather than by service, so anything using that helper appears without editing this dashboard. That panel ignores the Service picker, which is why the title says so: its series carry `app`, from Kubernetes. The Kubernetes `app` label and the name a service registers with the SDK need not match - hyphen against underscore is the usual difference - and no variable format converts one into the other.", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 109 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "expr": "sum by (app, status_code) (rate({__name__=~\".+_api_requests_total\"}[$__rate_interval]))", + "legendFormat": "{{app}} {{status_code}}", + "range": true, + "editorMode": "code" + } + ], + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "auto", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 11, + "type": "row", + "title": "Profiles ($service)", + "collapsed": false, + "panels": [], + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 118 + } + }, + { + "id": 35, + "type": "timeseries", + "title": "CPU by endpoint", + "description": "CPU attributed to the route that was executing when each sample was taken, which is the one thing the flame graphs below cannot tell you at a glance: they merge every request together. Only samples taken inside a span carry span_name, so background work and startup are absent here while still appearing in CPU by service.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 119 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "metrics", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "labelSelector": "{service_name=~\"${service:pipe}\", span_name!=\"\"}", + "groupBy": [ + "span_name" + ] + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 8, + "gradientMode": "opacity", + "showPoints": "never", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "barAlignment": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "none", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "displayName": "${__field.labels.span_name}" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 12, + "type": "flamegraph", + "title": "CPU profile", + "description": "Merged across every service selected in the Service picker. Pick a single service to read it. The totals are summed over the dashboard's time range, so they grow as the range widens rather than describing a moment.", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 127 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "grafana-pyroscope-datasource", + "uid": "pyroscope" + }, + "queryType": "profile", + "profileTypeId": "process_cpu:cpu:nanoseconds:cpu:nanoseconds", + "labelSelector": "{service_name=~\"${service:pipe}\", service_name!=\"pyroscope\"}", + "groupBy": [] } ], "options": {}, @@ -1534,7 +2305,7 @@ "h": 13, "w": 12, "x": 12, - "y": 84 + "y": 127 }, "targets": [ { @@ -1565,23 +2336,23 @@ "h": 1, "w": 24, "x": 0, - "y": 97 + "y": 140 } }, { "id": 24, "type": "timeseries", "title": "Log volume by service", - "description": "Lines per second per stream. Driven by the Log stream picker, not Service: Alloy takes `service_name` from each line's own `service` field and falls back to the pod's `app` label, so one service can appear under both spellings (`gen3_embeddings` and `gen3-embeddings`) while Pyroscope knows only the one the SDK registered.", + "description": "Lines per second per stream, driven by the Log stream picker rather than by Service. Alloy takes `service_name` from each line's own `service` field and falls back to the pod's `app` label when a line has none, so one service can own two streams under two spellings, and services that never profile appear here too. The Kubernetes `app` label and the name a service registers with the SDK need not match - hyphen against underscore is the usual difference - and no variable format converts one into the other.", "datasource": { "type": "loki", "uid": "loki" }, "gridPos": { "h": 9, - "w": 12, + "w": 8, "x": 0, - "y": 98 + "y": 141 }, "targets": [ { @@ -1657,9 +2428,9 @@ }, "gridPos": { "h": 9, - "w": 12, - "x": 12, - "y": 98 + "w": 8, + "x": 8, + "y": 141 }, "targets": [ { @@ -1800,6 +2571,85 @@ } } }, + { + "id": 39, + "type": "timeseries", + "title": "Error rate by service", + "description": "Error lines per second per service, which for much of this cluster is the only error signal there is: a service that serves no metrics and exports no spans still logs. Driven by the Log stream and Exclude controls, and by the level Loki inferred - a service logging severity in a shape Loki cannot parse is invisible here.", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 141 + }, + "targets": [ + { + "refId": "A", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "queryType": "range", + "editorMode": "code", + "legendFormat": "{{service_name}}", + "expr": "sum by (service_name) (rate({service_name=~\"${logservice:pipe}\", service_name!~\"$exclude\"} | detected_level =~ \"error\" [$__auto]))" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "bars", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 60, + "gradientMode": "opacity", + "showPoints": "never", + "pointSize": 4, + "spanNulls": false, + "axisSoftMin": 0, + "axisPlacement": "auto", + "axisBorderShow": false, + "barAlignment": 0, + "scaleDistribution": { + "type": "linear" + }, + "stacking": { + "mode": "normal", + "group": "A" + }, + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "unit": "cps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "calcs": [ + "max", + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, { "id": 27, "type": "logs", @@ -1813,7 +2663,7 @@ "h": 12, "w": 24, "x": 0, - "y": 107 + "y": 150 }, "targets": [ { @@ -1868,7 +2718,7 @@ "h": 10, "w": 24, "x": 0, - "y": 119 + "y": 162 }, "targets": [ { diff --git a/examples/local_alloy_values.yaml b/examples/local_alloy_values.yaml index a863d8d56..6e440d467 100644 --- a/examples/local_alloy_values.yaml +++ b/examples/local_alloy_values.yaml @@ -491,7 +491,7 @@ alloy: prometheus.relabel "cadvisor" { rule { source_labels = ["__name__"] - regex = "container_memory_working_set_bytes|container_memory_rss|container_spec_memory_limit_bytes|container_cpu_usage_seconds_total|container_cpu_cfs_periods_total|container_cpu_cfs_throttled_periods_total" + regex = "container_memory_working_set_bytes|container_memory_rss|container_spec_memory_limit_bytes|container_cpu_usage_seconds_total|container_cpu_cfs_periods_total|container_cpu_cfs_throttled_periods_total|container_spec_cpu_quota|container_spec_cpu_period|container_oom_events_total" action = "keep" } From 571b237f926aba3d06a4db26b860449a2cfd5f7a Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Tue, 1 Sep 2026 10:23:25 -0500 Subject: [PATCH 06/23] Put default storageclass into argocd --- helm/cluster-level-resources/README.md | 11 +++++++ .../templates/storage-class.yaml | 32 +++++++++++++++++++ helm/cluster-level-resources/values.yaml | 19 +++++++++++ helm/hatchery/README.md | 1 + helm/revproxy/README.md | 4 +-- 5 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 helm/cluster-level-resources/templates/storage-class.yaml diff --git a/helm/cluster-level-resources/README.md b/helm/cluster-level-resources/README.md index c194c0699..3b28b6a3b 100644 --- a/helm/cluster-level-resources/README.md +++ b/helm/cluster-level-resources/README.md @@ -201,6 +201,17 @@ Published versions of this chart are listed in the | nvidia-device-plugin.enabled | bool | `false` | | | nvidia-device-plugin.targetRevision | string | `"v0.18.0"` | | | project | string | `"unfunded"` | | +| storageclass.allowVolumeExpansion | bool | `true` | | +| storageclass.default | bool | `true` | | +| storageclass.enabled | bool | `false` | | +| storageclass.name | string | `"gp3"` | | +| storageclass.parameters.encrypted | string | `"true"` | | +| storageclass.parameters.type | string | `"gp3"` | | +| storageclass.previousDefault.name | string | `"gp2"` | | +| storageclass.previousDefault.removeDefault | bool | `true` | | +| storageclass.provisioner | string | `"ebs.csi.aws.com"` | | +| storageclass.reclaimPolicy | string | `"Retain"` | | +| storageclass.volumeBindingMode | string | `"WaitForFirstConsumer"` | | | vpc-cni.configuration.enabled | bool | `false` | | | vpc-cni.enabled | bool | `false` | | | vpc-cni.targetRevision | string | `"v1.20.4"` | | diff --git a/helm/cluster-level-resources/templates/storage-class.yaml b/helm/cluster-level-resources/templates/storage-class.yaml new file mode 100644 index 000000000..d1133bebf --- /dev/null +++ b/helm/cluster-level-resources/templates/storage-class.yaml @@ -0,0 +1,32 @@ +{{- if .Values.storageclass.enabled }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.storageclass.name }} + annotations: + storageclass.kubernetes.io/is-default-class: {{ .Values.storageclass.default | quote }} + argocd.argoproj.io/sync-wave: "-1" + argocd.argoproj.io/sync-options: Prune=false,Delete=false +provisioner: {{ .Values.storageclass.provisioner }} +parameters: +{{- with .Values.storageclass.parameters }} +{{ toYaml . | indent 2 }} +{{- end }} +reclaimPolicy: {{ .Values.storageclass.reclaimPolicy }} +volumeBindingMode: {{ .Values.storageclass.volumeBindingMode }} +allowVolumeExpansion: {{ .Values.storageclass.allowVolumeExpansion }} +{{- end }} +--- +{{- if and + .Values.storageclass.enabled + .Values.storageclass.previousDefault.removeDefault +}} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Values.storageclass.previousDefault.name }} + annotations: + storageclass.kubernetes.io/is-default-class: "false" + argocd.argoproj.io/sync-wave: "0" + argocd.argoproj.io/sync-options: ServerSideApply=true,Validate=false,Prune=false,Delete=false +{{- end }} \ No newline at end of file diff --git a/helm/cluster-level-resources/values.yaml b/helm/cluster-level-resources/values.yaml index 7cda90eab..5662f6598 100644 --- a/helm/cluster-level-resources/values.yaml +++ b/helm/cluster-level-resources/values.yaml @@ -287,6 +287,25 @@ nvidia-device-plugin: enabled: false targetRevision: v0.18.0 +storageclass: + enabled: false + name: gp3 + default: true + + provisioner: ebs.csi.aws.com + + reclaimPolicy: Retain + volumeBindingMode: WaitForFirstConsumer + allowVolumeExpansion: true + + parameters: + type: gp3 + encrypted: "true" + + previousDefault: + removeDefault: true + name: gp2 + vpc-cni: enabled: false targetRevision: v1.20.4 diff --git a/helm/hatchery/README.md b/helm/hatchery/README.md index e83cb74c5..23c0e1528 100644 --- a/helm/hatchery/README.md +++ b/helm/hatchery/README.md @@ -131,6 +131,7 @@ Published versions of this chart are listed in the | tolerations | list | `[]` | Tolerations to use for the deployment. | | volumeMounts | list | `[{"mountPath":"/var/hatchery/hatchery.json","name":"hatchery-config","readOnly":true,"subPath":"json"},{"mountPath":"/hatchery.json","name":"hatchery-config","readOnly":true,"subPath":"json"}]` | Volumes to mount to the container. | | volumes | list | `[{"configMap":{"name":"manifest-hatchery"},"name":"hatchery-config"}]` | Volumes to attach to the container. | +| workspaceInternetEgress | bool | `true` | | | workspaceLaunchTest | map | `{"enabled":false,"hostname":"https://example.com","operatorName":"username","schedule":"*/20 * * * *","workspaceImages":"(Generic) Jupyter Lab Notebook with R Kernel"}` | Configuration for workspace launch test crobjob | | workspaceLaunchTest.enabled | bool | `false` | Whether the workspace launch test cron job is enabled. | | workspaceLaunchTest.hostname | string | `"https://example.com"` | Hostname for the workspace launch test operator. | diff --git a/helm/revproxy/README.md b/helm/revproxy/README.md index 4fd02f186..c34724fe3 100644 --- a/helm/revproxy/README.md +++ b/helm/revproxy/README.md @@ -81,9 +81,9 @@ Published versions of this chart are listed in the | ingress.tls | list | `[]` | To secure an Ingress by specifying a secret that contains a TLS private key and certificate. | | metricsEnabled | bool | `nil` | Whether Metrics are enabled. | | nameOverride | string | `""` | Override the name of the chart. | -| netPolicy | map | `{"egressApps":["portal","sowerjob"],"ingressApps":["portal","sowerjob"]}` | Configuration for network policies created by this chart. Only relevant if "global.netPolicy.enabled" is set to true | +| netPolicy | map | `{"egressApps":["portal","sowerjob"],"ingressApps":["portal","sowerjob","ssjdispatcherjob"]}` | Configuration for network policies created by this chart. Only relevant if "global.netPolicy.enabled" is set to true | | netPolicy.egressApps | array | `["portal","sowerjob"]` | List of apps that this app requires egress to | -| netPolicy.ingressApps | array | `["portal","sowerjob"]` | List of app labels that require ingress to this service | +| netPolicy.ingressApps | array | `["portal","sowerjob","ssjdispatcherjob"]` | List of app labels that require ingress to this service | | nginx.pidFile | string | `"/var/run/nginx.pid"` | | | nginx.resolver | string | `"kube-dns.kube-system.svc.cluster.local"` | | | nginx.user | string | `"nginx"` | | From c7914736b16275a8f9aad61aa935da994bbd6c85 Mon Sep 17 00:00:00 2001 From: avantol Date: Tue, 1 Sep 2026 11:33:46 -0500 Subject: [PATCH 07/23] don't rely on underscores in names for linking --- .../scripts/regenerate_local_alloy_values.py | 16 +++--------- docs/local-observability.md | 15 +++++++++++ docs/otel-logs-and-traces.md | 26 +++++++++---------- examples/local_alloy_values.yaml | 12 +-------- 4 files changed, 32 insertions(+), 37 deletions(-) diff --git a/.github/scripts/regenerate_local_alloy_values.py b/.github/scripts/regenerate_local_alloy_values.py index 7d836fa6c..7330d2aad 100644 --- a/.github/scripts/regenerate_local_alloy_values.py +++ b/.github/scripts/regenerate_local_alloy_values.py @@ -12,8 +12,8 @@ log source. Two of the substitutions are not address rewrites but added behaviour that exists nowhere else. -The ``loki.process`` stage promotes ``trace_id`` to Loki structured metadata and relabels -``service_name`` from the log line, which is what makes Grafana's trace-to-logs query resolve. The +The ``loki.process`` stage promotes ``trace_id`` to Loki structured metadata, which is what lets +Grafana's trace-to-logs query filter on it with no parser stage. The cAdvisor scrape adds the per-container memory and CPU series the chart's kubelet scrape does not carry. The chart has no equivalent of either, so moving one into ``helm/alloy/values.yaml`` is what would extend it to deployed clusters, and its substitution would then be dropped. @@ -78,23 +78,13 @@ " forward_to = [loki.write.endpoint.receiver]\n" "\n" " stage.json {\n" - ' expressions = { trace_id = "trace_id", span_id = "span_id", otel_service = "service" }\n' + ' expressions = { trace_id = "trace_id", span_id = "span_id" }\n' " }\n" "\n" " stage.structured_metadata {\n" ' values = { trace_id = "", span_id = "" }\n' " }\n" "\n" - " // Each line reports the service.name its span was recorded under. Using that as\n" - " // the stream label is what keeps logs joinable to traces: Grafana builds its\n" - " // trace-to-logs query from service.name, while Loki would otherwise derive\n" - " // service_name from the pod's `app` label. Those two spellings differ whenever a\n" - " // service names itself with underscores, and the join then silently finds nothing.\n" - " //\n" - " // Lines logged outside a span extract nothing here and keep Loki's derived value.\n" - " stage.labels {\n" - ' values = { service_name = "otel_service" }\n' - " }\n" " }\n", ), ( diff --git a/docs/local-observability.md b/docs/local-observability.md index b3abb56c9..d3dc54fce 100644 --- a/docs/local-observability.md +++ b/docs/local-observability.md @@ -109,6 +109,21 @@ gen3-embeddings: for `grpc` on 4317, and a mismatched pair fails when the first span is exported rather than at startup. +**Logs have to be JSON**, or trace-to-logs cannot resolve. Alloy's `stage.json` reads `trace_id` +off each line, and `gen3logging`'s text formatter buries it in a `[trace_id=...]` suffix that no +JSON parser sees. `gen3logging` emits text unless `GEN3_JSON_LOGS` is truthy, which it reads +straight from the process environment: + +```yaml +gen3-workflow: + extraEnv: + GEN3_JSON_LOGS: "true" +``` + +The variable is only consulted when a service leaves `get_logger`'s `json_logs` argument as +`None`. A service that passes the value explicitly, usually from a config key of its own, ignores +the environment entirely - so check which of the two governs before setting either. + **Profiles need one address.** A service that ships a Pyroscope SDK likely pushes to `PYROSCOPE_SERVER_ADDRESS`, which is something like: diff --git a/docs/otel-logs-and-traces.md b/docs/otel-logs-and-traces.md index ec52432a9..574480013 100644 --- a/docs/otel-logs-and-traces.md +++ b/docs/otel-logs-and-traces.md @@ -114,10 +114,12 @@ Two problems follow from that. `trace_id` is only text inside the log message, so a LogQL label filter cannot see it and a query built from a trace id matches nothing. -The service name also disagrees with itself. Grafana builds its trace-to-logs stream selector from -the span's `service.name`, while Loki derives `service_name` from the pod's `app` label when the -label is absent. A span says `service.name = gen3_embeddings` and its logs land on a stream -labelled `service_name = gen3-embeddings`, so the join finds nothing. Underscore against hyphen. +A service that names itself differently from its pod compounds it. Grafana builds its trace-to-logs +stream selector from the span's `service.name`, while Loki derives `service_name` from the pod's +`app` label when the label is absent, so a span saying `service.name = a_service` whose logs land on +`service_name = a-service` joins to nothing. That is a naming bug rather than a pipeline one, and the +fix belongs in the service: an `app` label cannot carry an underscore, because the API server rejects +one in an object name, so the name a service reports to OpenTelemetry has to be the hyphenated one. Insert a processing stage between the source and the write: @@ -131,16 +133,12 @@ loki.process "pod_logs" { forward_to = [loki.write.endpoint.receiver] stage.json { - expressions = { trace_id = "trace_id", span_id = "span_id", otel_service = "service" } + expressions = { trace_id = "trace_id", span_id = "span_id" } } stage.structured_metadata { values = { trace_id = "", span_id = "" } } - - stage.labels { - values = { service_name = "otel_service" } - } } ``` @@ -148,9 +146,10 @@ loki.process "pod_logs" { structured metadata is filterable with `| trace_id = "..."` and no parser stage, while a per-request value used as a stream label would multiply Loki's stream cardinality without bound. -`stage.labels` takes `service_name` from the line's own `service` field, so the label matches the -span by construction. Lines logged outside a span have no `service` value and keep Loki's derived -label, which is why both spellings appear in a healthy cluster. +Nothing rewrites `service_name`: Loki's own derivation from the pod's `app` label is correct once a +service reports that same name to OpenTelemetry, and one stream per service is what you want. A +service still reporting a different name needs fixing there, not here - rewriting the label hides the +divergence while leaving every other signal split. Requires Loki on `tsdb` with schema `v13` or later. `helm/observability/values.yaml` already configures `store: tsdb` with `schema: v13`. @@ -162,7 +161,8 @@ This stage currently ships in one place only: the kind overlay, where `.github/scripts/regenerate_local_alloy_values.py` inserts it while generating `examples/local_alloy_values.yaml`. Neither `cluster-level-resources/values.yaml` nor `helm/alloy/values.yaml` contains it, so correlation is a local-development capability until this -lands in one of them. Anything verified against a kind cluster is exercising the overlay rather +lands in one of them. It is now a smaller change than it was: promoting `trace_id`, with no label +rewriting attached. Anything verified against a kind cluster is exercising the overlay rather than the configuration a cluster runs. **Verify:** `{service_name="gen3_embeddings"} | trace_id != ""` returns lines **with no `| json` diff --git a/examples/local_alloy_values.yaml b/examples/local_alloy_values.yaml index 6e440d467..16bb6fb96 100644 --- a/examples/local_alloy_values.yaml +++ b/examples/local_alloy_values.yaml @@ -375,23 +375,13 @@ alloy: forward_to = [loki.write.endpoint.receiver] stage.json { - expressions = { trace_id = "trace_id", span_id = "span_id", otel_service = "service" } + expressions = { trace_id = "trace_id", span_id = "span_id" } } stage.structured_metadata { values = { trace_id = "", span_id = "" } } - // Each line reports the service.name its span was recorded under. Using that as - // the stream label is what keeps logs joinable to traces: Grafana builds its - // trace-to-logs query from service.name, while Loki would otherwise derive - // service_name from the pod's `app` label. Those two spellings differ whenever a - // service names itself with underscores, and the join then silently finds nothing. - // - // Lines logged outside a span extract nothing here and keep Loki's derived value. - stage.labels { - values = { service_name = "otel_service" } - } } // kube-state-metrics From 8f7956d464b868fc0db184f3965d8ea5b6c6fd61 Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Tue, 1 Sep 2026 11:42:48 -0500 Subject: [PATCH 08/23] Updated db create job to create push secret and avoid race conditions --- helm/arborist/templates/db-init.yaml | 2 -- helm/audit/templates/db-init.yaml | 2 -- helm/common/templates/_db_setup_job.tpl | 33 +++++++++++++++++++ helm/dicom-server/templates/db-init.yaml | 2 -- helm/fence/templates/db-init.yaml | 2 -- helm/funnel/templates/db-init.yaml | 2 -- helm/gen3-embeddings/templates/db-init.yaml | 2 -- .../templates/db-init.yaml | 2 -- helm/indexd/templates/db-init.yaml | 2 -- helm/metadata/templates/db-init.yaml | 2 -- helm/ohdsi-webapi/templates/db-init.yaml | 2 -- helm/orthanc/templates/db-init.yaml | 2 -- helm/peregrine/templates/db-init.yaml | 2 -- helm/requestor/templates/db-init.yaml | 2 -- helm/sheepdog/templates/db-init.yaml | 2 -- helm/wts/templates/db-init.yaml | 2 -- 16 files changed, 33 insertions(+), 30 deletions(-) diff --git a/helm/arborist/templates/db-init.yaml b/helm/arborist/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/arborist/templates/db-init.yaml +++ b/helm/arborist/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/audit/templates/db-init.yaml b/helm/audit/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/audit/templates/db-init.yaml +++ b/helm/audit/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/common/templates/_db_setup_job.tpl b/helm/common/templates/_db_setup_job.tpl index 395721688..1090e07ef 100644 --- a/helm/common/templates/_db_setup_job.tpl +++ b/helm/common/templates/_db_setup_job.tpl @@ -14,6 +14,11 @@ rules: - apiGroups: [""] resources: ["secrets"] verbs: ["*"] +{{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret $.Values.externalSecrets.pushSecret) }} +- apiGroups: ["external-secrets.io"] + resources: ["pushsecrets"] + verbs: ["get", "list", "create", "patch", "update", "delete"] +{{- end }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding @@ -177,6 +182,28 @@ spec: PGPASSWORD=$SERVICE_PGPASS psql -d $SERVICE_PGDB -h $PGHOST -p $PGPORT -U $SERVICE_PGUSER -c "\conninfo" kubectl patch secret/{{ .Chart.Name }}-dbcreds -p '{"data":{"dbcreated":"dHJ1ZQo="}}' fi +{{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret $.Values.externalSecrets.pushSecret) }} + + # Create the PushSecret from within this job (instead of having helm create it as part of + # the release) and wait until the remote secret is populated. This avoids the race condition + # where the job completes before helm/external-secrets has created and processed the + # PushSecret, leaving consumers with an unpopulated remote secret. + echo "Waiting for bootstrap secret {{ $.Chart.Name }}-dbcreds-bootstrap ..." + for i in $(seq 1 60); do + kubectl -n {{ $.Release.Namespace }} get secret {{ $.Chart.Name }}-dbcreds-bootstrap >/dev/null 2>&1 && break + sleep 5 + done + kubectl -n {{ $.Release.Namespace }} get secret {{ $.Chart.Name }}-dbcreds-bootstrap + + echo "Creating PushSecret {{ $.Chart.Name }}-dbcreds ..." + # kubectl apply is idempotent: on subsequent runs the existing PushSecret is kept as-is + # (updatePolicy is IfNotExists, so the remote secret is never overwritten). + echo '{{ include "common.db-push-secret" . | b64enc }}' | base64 --decode | kubectl -n {{ $.Release.Namespace }} apply -f - + + echo "Waiting for PushSecret to sync echo "Waiting for PushSecret to sync to the remote secret store ..." + kubectl -n {{ $.Release.Namespace }} wait --for=condition=Ready pushsecret/{{ .Chart.Name }}-dbcreds --timeout=300s + echo "PushSecret is Ready - remote secret has been populated" +{{- end }} {{- end }} {{- end }} @@ -231,6 +258,12 @@ data: {{- end -}} +{{/* + PushSecret manifest for the dbcreds bootstrap secret. + NOTE: this is no longer rendered as a standalone helm resource. It is applied by the + {{ .Chart.Name }}-dbcreate job (see common.db_setup_job) so that the job only completes + after the remote secret has been populated. +*/}} {{- define "common.db-push-secret" -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} apiVersion: external-secrets.io/v1alpha1 diff --git a/helm/dicom-server/templates/db-init.yaml b/helm/dicom-server/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/dicom-server/templates/db-init.yaml +++ b/helm/dicom-server/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/fence/templates/db-init.yaml b/helm/fence/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/fence/templates/db-init.yaml +++ b/helm/fence/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/funnel/templates/db-init.yaml b/helm/funnel/templates/db-init.yaml index 0de60dbff..b664f2632 100644 --- a/helm/funnel/templates/db-init.yaml +++ b/helm/funnel/templates/db-init.yaml @@ -9,7 +9,5 @@ {{- end -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/gen3-embeddings/templates/db-init.yaml b/helm/gen3-embeddings/templates/db-init.yaml index 95297b084..89e99f044 100644 --- a/helm/gen3-embeddings/templates/db-init.yaml +++ b/helm/gen3-embeddings/templates/db-init.yaml @@ -9,7 +9,5 @@ {{- end -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} diff --git a/helm/gen3-user-data-library/templates/db-init.yaml b/helm/gen3-user-data-library/templates/db-init.yaml index 95297b084..89e99f044 100644 --- a/helm/gen3-user-data-library/templates/db-init.yaml +++ b/helm/gen3-user-data-library/templates/db-init.yaml @@ -9,7 +9,5 @@ {{- end -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} diff --git a/helm/indexd/templates/db-init.yaml b/helm/indexd/templates/db-init.yaml index 0de60dbff..b664f2632 100644 --- a/helm/indexd/templates/db-init.yaml +++ b/helm/indexd/templates/db-init.yaml @@ -9,7 +9,5 @@ {{- end -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/metadata/templates/db-init.yaml b/helm/metadata/templates/db-init.yaml index 0de60dbff..b664f2632 100644 --- a/helm/metadata/templates/db-init.yaml +++ b/helm/metadata/templates/db-init.yaml @@ -9,7 +9,5 @@ {{- end -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/ohdsi-webapi/templates/db-init.yaml b/helm/ohdsi-webapi/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/ohdsi-webapi/templates/db-init.yaml +++ b/helm/ohdsi-webapi/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/orthanc/templates/db-init.yaml b/helm/orthanc/templates/db-init.yaml index 83f4d86a4..559e62393 100644 --- a/helm/orthanc/templates/db-init.yaml +++ b/helm/orthanc/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_job" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/peregrine/templates/db-init.yaml b/helm/peregrine/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/peregrine/templates/db-init.yaml +++ b/helm/peregrine/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/requestor/templates/db-init.yaml b/helm/requestor/templates/db-init.yaml index b1fe92d63..1cef39f9b 100644 --- a/helm/requestor/templates/db-init.yaml +++ b/helm/requestor/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/sheepdog/templates/db-init.yaml b/helm/sheepdog/templates/db-init.yaml index 0de60dbff..b664f2632 100644 --- a/helm/sheepdog/templates/db-init.yaml +++ b/helm/sheepdog/templates/db-init.yaml @@ -9,7 +9,5 @@ {{- end -}} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} \ No newline at end of file diff --git a/helm/wts/templates/db-init.yaml b/helm/wts/templates/db-init.yaml index ad9bb547d..d6ff85e0c 100644 --- a/helm/wts/templates/db-init.yaml +++ b/helm/wts/templates/db-init.yaml @@ -5,7 +5,5 @@ {{ include "common.db_setup_sa" . }} {{- if and $.Values.global.externalSecrets.deploy (or $.Values.global.externalSecrets.pushSecret .Values.externalSecrets.pushSecret) }} --- -{{ include "common.db-push-secret" . }} ---- {{ include "common.secret.db.bootstrap" . }} {{- end }} From ca9fc8fd70a16048cce16f2a155351c2fb8dd081 Mon Sep 17 00:00:00 2001 From: pieterlukasse Date: Thu, 3 Sep 2026 12:40:52 -0400 Subject: [PATCH 09/23] fix: add FORCE_AUTHZ_CHECK_FOR_METADATA_QUERIES to deployment.yaml for MDS --- helm/metadata/templates/deployment.yaml | 2 ++ helm/metadata/templates/secrets.yaml | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/helm/metadata/templates/deployment.yaml b/helm/metadata/templates/deployment.yaml index 94eb88809..2c50982f7 100644 --- a/helm/metadata/templates/deployment.yaml +++ b/helm/metadata/templates/deployment.yaml @@ -64,6 +64,8 @@ spec: value: "False" - name: GEN3_ES_ENDPOINT value: {{ .Values.esEndpoint }} + - name: FORCE_AUTHZ_CHECK_FOR_METADATA_QUERIES + value: {{ .Values.forceAuthzCheckForMetadataQueries }} - name: DB_HOST valueFrom: secretKeyRef: diff --git a/helm/metadata/templates/secrets.yaml b/helm/metadata/templates/secrets.yaml index 93d4d9632..0bd639d73 100644 --- a/helm/metadata/templates/secrets.yaml +++ b/helm/metadata/templates/secrets.yaml @@ -13,5 +13,4 @@ stringData: DB_PASSWORD={{ include "metadata.postgres.password" . }} DB_DATABASE={{ .Values.postgres.dbname }} ADMIN_LOGINS={{ $randomPass }} - FORCE_AUTHZ_CHECK_FOR_METADATA_QUERIES={{ .Values.forceAuthzCheckForMetadataQueries }} {{- end }} \ No newline at end of file From d1a071b975b9b763b19860073039dc333d53c98b Mon Sep 17 00:00:00 2001 From: Justin Barnowski Date: Thu, 3 Sep 2026 13:58:32 -0500 Subject: [PATCH 10/23] fix(quote): add quotes around env var usage --- helm/metadata/templates/deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helm/metadata/templates/deployment.yaml b/helm/metadata/templates/deployment.yaml index 2c50982f7..73f3e0e61 100644 --- a/helm/metadata/templates/deployment.yaml +++ b/helm/metadata/templates/deployment.yaml @@ -65,7 +65,7 @@ spec: - name: GEN3_ES_ENDPOINT value: {{ .Values.esEndpoint }} - name: FORCE_AUTHZ_CHECK_FOR_METADATA_QUERIES - value: {{ .Values.forceAuthzCheckForMetadataQueries }} + value: {{ .Values.forceAuthzCheckForMetadataQueries | quote }} - name: DB_HOST valueFrom: secretKeyRef: @@ -178,4 +178,4 @@ spec: {{- with .Values.args }} args: {{- toYaml . | nindent 12}} - {{- end }} \ No newline at end of file + {{- end }} From 1bc08cd57f84323983a61a220501aea7ab059d55 Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Thu, 3 Sep 2026 15:23:25 -0500 Subject: [PATCH 11/23] Updated cluster-level-resources to allow for setting fast image pulls --- helm/cluster-level-resources/README.md | 15 ++++++++++++++ .../karpenter-config-resources-default.yaml | 16 +++++++++++++++ .../karpenter-config-resources-gpu.yaml | 18 ++++++++++++++++- .../karpenter-config-resources-jupyter.yaml | 18 ++++++++++++++++- .../karpenter-config-resources-secondary.yaml | 16 +++++++++++++++ .../karpenter-config-resources-workflow.yaml | 16 +++++++++++++++ helm/cluster-level-resources/values.yaml | 20 +++++++++++++++++++ 7 files changed, 117 insertions(+), 2 deletions(-) diff --git a/helm/cluster-level-resources/README.md b/helm/cluster-level-resources/README.md index c194c0699..f2de17aba 100644 --- a/helm/cluster-level-resources/README.md +++ b/helm/cluster-level-resources/README.md @@ -85,6 +85,9 @@ Published versions of this chart are listed in the | karpenter-crds.default.disruptionBudget | string | `"10%"` | | | karpenter-crds.default.enabled | bool | `true` | | | karpenter-crds.default.expireAfter | string | `"168h"` | | +| karpenter-crds.default.fastImagePull.enabled | bool | `false` | | +| karpenter-crds.default.fastImagePull.iops | int | `3000` | | +| karpenter-crds.default.fastImagePull.throughput | int | `600` | | | karpenter-crds.default.limits.cpu | string | `"1000"` | | | karpenter-crds.default.limits.memory | string | `"1000Gi"` | | | karpenter-crds.default.requirements[0].key | string | `"karpenter.sh/capacity-type"` | | @@ -109,6 +112,9 @@ Published versions of this chart are listed in the | karpenter-crds.gpu.consolidationPolicy | string | `"WhenEmpty"` | | | karpenter-crds.gpu.enabled | bool | `false` | | | karpenter-crds.gpu.expireAfter | string | `"168h"` | | +| karpenter-crds.gpu.fastImagePull.enabled | bool | `false` | | +| karpenter-crds.gpu.fastImagePull.iops | int | `3000` | | +| karpenter-crds.gpu.fastImagePull.throughput | int | `600` | | | karpenter-crds.gpu.volumeSize | string | `"50Gi"` | | | karpenter-crds.jupyter.additionalTags | object | `{}` | | | karpenter-crds.jupyter.consolidateAfter | string | `"30s"` | | @@ -116,6 +122,9 @@ Published versions of this chart are listed in the | karpenter-crds.jupyter.consolidationPolicy | string | `"WhenEmpty"` | | | karpenter-crds.jupyter.enabled | bool | `true` | | | karpenter-crds.jupyter.expireAfter | string | `"168h"` | | +| karpenter-crds.jupyter.fastImagePull.enabled | bool | `false` | | +| karpenter-crds.jupyter.fastImagePull.iops | int | `3000` | | +| karpenter-crds.jupyter.fastImagePull.throughput | int | `600` | | | karpenter-crds.jupyter.requirements[0].key | string | `"karpenter.sh/capacity-type"` | | | karpenter-crds.jupyter.requirements[0].operator | string | `"In"` | | | karpenter-crds.jupyter.requirements[0].values[0] | string | `"on-demand"` | | @@ -137,6 +146,9 @@ Published versions of this chart are listed in the | karpenter-crds.secondary.disruptionBudget | string | `"10%"` | | | karpenter-crds.secondary.enabled | bool | `false` | | | karpenter-crds.secondary.expireAfter | string | `"168h"` | | +| karpenter-crds.secondary.fastImagePull.enabled | bool | `false` | | +| karpenter-crds.secondary.fastImagePull.iops | int | `3000` | | +| karpenter-crds.secondary.fastImagePull.throughput | int | `600` | | | karpenter-crds.secondary.requirements[0].key | string | `"karpenter.sh/capacity-type"` | | | karpenter-crds.secondary.requirements[0].operator | string | `"In"` | | | karpenter-crds.secondary.requirements[0].values[0] | string | `"on-demand"` | | @@ -161,6 +173,9 @@ Published versions of this chart are listed in the | karpenter-crds.workflow.consolidationPolicy | string | `"WhenEmpty"` | | | karpenter-crds.workflow.enabled | bool | `true` | | | karpenter-crds.workflow.expireAfter | string | `"168h"` | | +| karpenter-crds.workflow.fastImagePull.enabled | bool | `false` | | +| karpenter-crds.workflow.fastImagePull.iops | int | `3000` | | +| karpenter-crds.workflow.fastImagePull.throughput | int | `600` | | | karpenter-crds.workflow.requirements[0].key | string | `"karpenter.sh/capacity-type"` | | | karpenter-crds.workflow.requirements[0].operator | string | `"In"` | | | karpenter-crds.workflow.requirements[0].values[0] | string | `"on-demand"` | | diff --git a/helm/cluster-level-resources/templates/karpenter-config-resources-default.yaml b/helm/cluster-level-resources/templates/karpenter-config-resources-default.yaml index df01aa1dc..4876cce02 100644 --- a/helm/cluster-level-resources/templates/karpenter-config-resources-default.yaml +++ b/helm/cluster-level-resources/templates/karpenter-config-resources-default.yaml @@ -28,6 +28,10 @@ spec: encrypted: true volumeSize: {{ index .Values "karpenter-crds" "default" "volumeSize" }} volumeType: gp3 +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "default" "fastImagePull" "enabled") }} + iops: {{ index .Values "karpenter-crds" "default" "fastImagePull" "iops" }} + throughput: {{ index .Values "karpenter-crds" "default" "fastImagePull" "throughput" }} +{{ end }} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: disabled @@ -62,6 +66,18 @@ spec: sysctl -w fs.inotify.max_user_watches=12000 & systemctl stop firewalld systemctl disable firewalld +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "default" "fastImagePull" "enabled") }} + + --BOUNDARY + Content-Type: application/node.eks.aws + + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + featureGates: + FastImagePull: true +{{ end }} --BOUNDARY-- {{- if semverCompare ">=1.0.8" .Values.karpenter.targetRevision }} diff --git a/helm/cluster-level-resources/templates/karpenter-config-resources-gpu.yaml b/helm/cluster-level-resources/templates/karpenter-config-resources-gpu.yaml index dacd6a97b..e3ad48fab 100644 --- a/helm/cluster-level-resources/templates/karpenter-config-resources-gpu.yaml +++ b/helm/cluster-level-resources/templates/karpenter-config-resources-gpu.yaml @@ -19,6 +19,10 @@ spec: encrypted: true volumeSize: {{ index .Values "karpenter-crds" "gpu" "volumeSize" }} volumeType: gp3 +{{ if (index .Values "karpenter-crds" "gpu" "fastImagePull" "enabled") }} + iops: {{ index .Values "karpenter-crds" "gpu" "fastImagePull" "iops" }} + throughput: {{ index .Values "karpenter-crds" "gpu" "fastImagePull" "throughput" }} +{{ end }} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: disabled @@ -53,7 +57,19 @@ spec: curl https://raw.githubusercontent.com/uc-cdis/cloud-automation/master/files/authorized_keys/ops_team >> /home/ec2-user/.ssh/authorized_keys & sysctl -w fs.inotify.max_user_watches=12000 & systemctl stop firewalld - systemctl disable firewalld + systemctl disable firewalld +{{ if (index .Values "karpenter-crds" "gpu" "fastImagePull" "enabled") }} + + --BOUNDARY + Content-Type: application/node.eks.aws + + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + featureGates: + FastImagePull: true +{{ end }} --BOUNDARY-- {{- if semverCompare ">=1.0.8" .Values.karpenter.targetRevision }} diff --git a/helm/cluster-level-resources/templates/karpenter-config-resources-jupyter.yaml b/helm/cluster-level-resources/templates/karpenter-config-resources-jupyter.yaml index 21186d37f..bf4538d17 100644 --- a/helm/cluster-level-resources/templates/karpenter-config-resources-jupyter.yaml +++ b/helm/cluster-level-resources/templates/karpenter-config-resources-jupyter.yaml @@ -19,7 +19,7 @@ spec: owner: {{ index .Values "karpenter-crds" "amiOwner" | quote }} {{- else }} - alias: {{ index .Values "karpenter-crds" "alias" | quote }} - {{- end }} + {{- end }} blockDeviceMappings: - deviceName: /dev/xvda ebs: @@ -27,6 +27,10 @@ spec: encrypted: true volumeSize: {{ index .Values "karpenter-crds" "jupyter" "volumeSize" }} volumeType: gp3 +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "jupyter" "fastImagePull" "enabled") }} + iops: {{ index .Values "karpenter-crds" "jupyter" "fastImagePull" "iops" }} + throughput: {{ index .Values "karpenter-crds" "jupyter" "fastImagePull" "throughput" }} +{{ end }} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: disabled @@ -61,6 +65,18 @@ spec: sysctl -w fs.inotify.max_user_watches=12000 & systemctl stop firewalld systemctl disable firewalld +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "jupyter" "fastImagePull" "enabled") }} + + --BOUNDARY + Content-Type: application/node.eks.aws + + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + featureGates: + FastImagePull: true +{{ end }} --BOUNDARY-- {{- if semverCompare ">=1.0.8" .Values.karpenter.targetRevision }} diff --git a/helm/cluster-level-resources/templates/karpenter-config-resources-secondary.yaml b/helm/cluster-level-resources/templates/karpenter-config-resources-secondary.yaml index f1a295274..540e2abb6 100644 --- a/helm/cluster-level-resources/templates/karpenter-config-resources-secondary.yaml +++ b/helm/cluster-level-resources/templates/karpenter-config-resources-secondary.yaml @@ -28,6 +28,10 @@ spec: encrypted: true volumeSize: {{ index .Values "karpenter-crds" "secondary" "volumeSize" }} volumeType: gp3 +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "secondary" "fastImagePull" "enabled") }} + iops: {{ index .Values "karpenter-crds" "secondary" "fastImagePull" "iops" }} + throughput: {{ index .Values "karpenter-crds" "secondary" "fastImagePull" "throughput" }} +{{ end }} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: disabled @@ -62,6 +66,18 @@ spec: sysctl -w fs.inotify.max_user_watches=12000 & systemctl stop firewalld systemctl disable firewalld +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "secondary" "fastImagePull" "enabled") }} + + --BOUNDARY + Content-Type: application/node.eks.aws + + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + featureGates: + FastImagePull: true +{{ end }} --BOUNDARY-- {{- if semverCompare ">=1.0.8" .Values.karpenter.targetRevision }} diff --git a/helm/cluster-level-resources/templates/karpenter-config-resources-workflow.yaml b/helm/cluster-level-resources/templates/karpenter-config-resources-workflow.yaml index 5d57392b5..7db504c49 100644 --- a/helm/cluster-level-resources/templates/karpenter-config-resources-workflow.yaml +++ b/helm/cluster-level-resources/templates/karpenter-config-resources-workflow.yaml @@ -28,6 +28,10 @@ spec: encrypted: true volumeSize: {{ index .Values "karpenter-crds" "workflow" "volumeSize" }} volumeType: gp3 +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "workflow" "fastImagePull" "enabled") }} + iops: {{ index .Values "karpenter-crds" "workflow" "fastImagePull" "iops" }} + throughput: {{ index .Values "karpenter-crds" "workflow" "fastImagePull" "throughput" }} +{{ end }} metadataOptions: httpEndpoint: enabled httpProtocolIPv6: disabled @@ -66,6 +70,18 @@ spec: sysctl -w fs.inotify.max_user_watches=12000 & systemctl stop firewalld systemctl disable firewalld +{{ if and (semverCompare ">=1.33" .Capabilities.KubeVersion.Version) (index .Values "karpenter-crds" "workflow" "fastImagePull" "enabled") }} + + --BOUNDARY + Content-Type: application/node.eks.aws + + --- + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + featureGates: + FastImagePull: true +{{ end }} --BOUNDARY-- diff --git a/helm/cluster-level-resources/values.yaml b/helm/cluster-level-resources/values.yaml index 7cda90eab..e0df38198 100644 --- a/helm/cluster-level-resources/values.yaml +++ b/helm/cluster-level-resources/values.yaml @@ -157,6 +157,10 @@ karpenter-crds: cpu: "1000" memory: 1000Gi volumeSize: 50Gi + fastImagePull: + enabled: false + iops: 3000 + throughput: 600 additionalTags: {} requirements: - key: karpenter.sh/capacity-type @@ -182,6 +186,10 @@ karpenter-crds: consolidationPolicy: "WhenEmpty" expireAfter: "168h" volumeSize: 50Gi + fastImagePull: + enabled: false + iops: 3000 + throughput: 600 additionalTags: {} secondary: enabled: false @@ -192,6 +200,10 @@ karpenter-crds: consolidationPolicy: "WhenEmpty" expireAfter: "168h" volumeSize: 50Gi + fastImagePull: + enabled: false + iops: 3000 + throughput: 600 additionalTags: {} requirements: - key: karpenter.sh/capacity-type @@ -217,6 +229,10 @@ karpenter-crds: consolidationPolicy: "WhenEmpty" expireAfter: "168h" volumeSize: 50Gi + fastImagePull: + enabled: false + iops: 3000 + throughput: 600 additionalTags: {} requirements: - key: karpenter.sh/capacity-type @@ -242,6 +258,10 @@ karpenter-crds: expireAfter: "168h" volumeSize: 50Gi sgSelector: "" + fastImagePull: + enabled: false + iops: 3000 + throughput: 600 additionalTags: {} requirements: - key: karpenter.sh/capacity-type From dc5b9d8acd077c9d4d7a530b3084d714a80e3fbf Mon Sep 17 00:00:00 2001 From: Edward Malinowski Date: Tue, 8 Sep 2026 11:24:26 -0500 Subject: [PATCH 12/23] Updated push secret flow to remove need for RDS hostname in values file --- helm/common/templates/_db_setup_job.tpl | 21 ++++++++++++++------- helm/hatchery/README.md | 1 + helm/revproxy/README.md | 4 ++-- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/helm/common/templates/_db_setup_job.tpl b/helm/common/templates/_db_setup_job.tpl index 1090e07ef..5056b85f8 100644 --- a/helm/common/templates/_db_setup_job.tpl +++ b/helm/common/templates/_db_setup_job.tpl @@ -195,12 +195,24 @@ spec: done kubectl -n {{ $.Release.Namespace }} get secret {{ $.Chart.Name }}-dbcreds-bootstrap + # PGHOST is sourced from the Aurora master secret when + # global.postgres.externalSecret is configured. Copy that resolved value into the + # bootstrap secret immediately before creating the PushSecret so the hostname does + # not need to be duplicated in Helm values or rendered into the bootstrap manifest. + if [ -z "$PGHOST" ]; then + echo "ERROR: PGHOST is empty; cannot populate {{ $.Chart.Name }}-dbcreds-bootstrap" + exit 1 + fi + BOOTSTRAP_PGHOST_B64="$(printf '%s' "$PGHOST" | base64 | tr -d '\n')" + kubectl -n {{ $.Release.Namespace }} patch secret {{ $.Chart.Name }}-dbcreds-bootstrap \ + --type merge \ + -p "{\"data\":{\"host\":\"${BOOTSTRAP_PGHOST_B64}\"}}" + echo "Creating PushSecret {{ $.Chart.Name }}-dbcreds ..." # kubectl apply is idempotent: on subsequent runs the existing PushSecret is kept as-is # (updatePolicy is IfNotExists, so the remote secret is never overwritten). echo '{{ include "common.db-push-secret" . | b64enc }}' | base64 --decode | kubectl -n {{ $.Release.Namespace }} apply -f - - - echo "Waiting for PushSecret to sync echo "Waiting for PushSecret to sync to the remote secret store ..." + echo "Waiting for PushSecret to sync to the remote secret store ..." kubectl -n {{ $.Release.Namespace }} wait --for=condition=Ready pushsecret/{{ .Chart.Name }}-dbcreds --timeout=300s echo "PushSecret is Ready - remote secret has been populated" {{- end }} @@ -248,11 +260,6 @@ data: username: {{ ( $.Values.postgres.username | default (printf "%s_%s" $.Chart.Name $.Release.Name) ) | b64enc | quote}} port: {{ $.Values.postgres.port | b64enc | quote }} password: {{ include "gen3.service-postgres" (dict "key" "password" "service" $.Chart.Name "context" $) | b64enc | quote }} - {{- if $.Values.global.dev }} - host: {{ (printf "%s-%s" $.Release.Name "postgresql" ) | b64enc | quote }} - {{- else }} - host: {{ ( $.Values.postgres.host | default ( $.Values.global.postgres.master.host)) | b64enc | quote }} - {{- end }} dbcreated: {{ "true" | b64enc | quote }} {{- end }} {{- end -}} diff --git a/helm/hatchery/README.md b/helm/hatchery/README.md index e83cb74c5..23c0e1528 100644 --- a/helm/hatchery/README.md +++ b/helm/hatchery/README.md @@ -131,6 +131,7 @@ Published versions of this chart are listed in the | tolerations | list | `[]` | Tolerations to use for the deployment. | | volumeMounts | list | `[{"mountPath":"/var/hatchery/hatchery.json","name":"hatchery-config","readOnly":true,"subPath":"json"},{"mountPath":"/hatchery.json","name":"hatchery-config","readOnly":true,"subPath":"json"}]` | Volumes to mount to the container. | | volumes | list | `[{"configMap":{"name":"manifest-hatchery"},"name":"hatchery-config"}]` | Volumes to attach to the container. | +| workspaceInternetEgress | bool | `true` | | | workspaceLaunchTest | map | `{"enabled":false,"hostname":"https://example.com","operatorName":"username","schedule":"*/20 * * * *","workspaceImages":"(Generic) Jupyter Lab Notebook with R Kernel"}` | Configuration for workspace launch test crobjob | | workspaceLaunchTest.enabled | bool | `false` | Whether the workspace launch test cron job is enabled. | | workspaceLaunchTest.hostname | string | `"https://example.com"` | Hostname for the workspace launch test operator. | diff --git a/helm/revproxy/README.md b/helm/revproxy/README.md index 4fd02f186..c34724fe3 100644 --- a/helm/revproxy/README.md +++ b/helm/revproxy/README.md @@ -81,9 +81,9 @@ Published versions of this chart are listed in the | ingress.tls | list | `[]` | To secure an Ingress by specifying a secret that contains a TLS private key and certificate. | | metricsEnabled | bool | `nil` | Whether Metrics are enabled. | | nameOverride | string | `""` | Override the name of the chart. | -| netPolicy | map | `{"egressApps":["portal","sowerjob"],"ingressApps":["portal","sowerjob"]}` | Configuration for network policies created by this chart. Only relevant if "global.netPolicy.enabled" is set to true | +| netPolicy | map | `{"egressApps":["portal","sowerjob"],"ingressApps":["portal","sowerjob","ssjdispatcherjob"]}` | Configuration for network policies created by this chart. Only relevant if "global.netPolicy.enabled" is set to true | | netPolicy.egressApps | array | `["portal","sowerjob"]` | List of apps that this app requires egress to | -| netPolicy.ingressApps | array | `["portal","sowerjob"]` | List of app labels that require ingress to this service | +| netPolicy.ingressApps | array | `["portal","sowerjob","ssjdispatcherjob"]` | List of app labels that require ingress to this service | | nginx.pidFile | string | `"/var/run/nginx.pid"` | | | nginx.resolver | string | `"kube-dns.kube-system.svc.cluster.local"` | | | nginx.user | string | `"nginx"` | | From cb4fa30c8381ca2b404724a589706117977658d8 Mon Sep 17 00:00:00 2001 From: pieterlukasse Date: Wed, 9 Sep 2026 17:28:54 -0400 Subject: [PATCH 13/23] feat: add clean up spec to ensure the ohdsi-webapi-client-job is cleaned up 2 min after finishing ... which ensures it can run again in a next deployment / upgrade of the service. Without this, the job stays there and blocks new/future instances --- helm/ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml b/helm/ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml index dad2fcdae..ae60dfd78 100644 --- a/helm/ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml +++ b/helm/ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml @@ -4,6 +4,7 @@ kind: Job metadata: name: ohdsi-webapi-client-job spec: + ttlSecondsAfterFinished: 120 template: metadata: labels: From f1ef3969c436a3d86e7a0f4a8a6d679325e234ae Mon Sep 17 00:00:00 2001 From: pieterlukasse Date: Thu, 10 Sep 2026 16:54:47 -0400 Subject: [PATCH 14/23] feat: ensure app=ohdsi-webapi is restarted after config is patched --- .../ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml | 2 ++ helm/ohdsi-webapi/templates/serviceaccount.yaml | 3 +++ 2 files changed, 5 insertions(+) diff --git a/helm/ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml b/helm/ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml index ae60dfd78..bc9244ade 100644 --- a/helm/ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml +++ b/helm/ohdsi-webapi/templates/ohdsi-webapi-client-create-job.yaml @@ -142,4 +142,6 @@ spec: \"security_oid_clientId\":\"${FENCE_CLIENT_ID}\", \"security_oid_apiSecret\":\"${FENCE_CLIENT_SECRET}\" }}" + echo -e "\n\n>>========== Restarting webapi pods so they pick up new config above ============\n\n" + kubectl delete pod -l app=ohdsi-webapi {{- end }} \ No newline at end of file diff --git a/helm/ohdsi-webapi/templates/serviceaccount.yaml b/helm/ohdsi-webapi/templates/serviceaccount.yaml index 4a3477d7c..9c0c943ff 100644 --- a/helm/ohdsi-webapi/templates/serviceaccount.yaml +++ b/helm/ohdsi-webapi/templates/serviceaccount.yaml @@ -16,6 +16,9 @@ rules: resources: ["secrets"] resourceNames: ["fence-dbcreds"] verbs: ["get"] +- apiGroups: [""] + resources: ["pods"] + verbs: ["list", "delete"] --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding From d77fbeeb7a1f7a6a9a18cb054ab187ab5c44f872 Mon Sep 17 00:00:00 2001 From: Kyle Burton Date: Tue, 15 Sep 2026 10:54:01 -0500 Subject: [PATCH 15/23] Remove hardcoded DEBUG: True from sheepdog --- helm/sheepdog/README.md | 1 + helm/sheepdog/templates/deployment.yaml | 2 +- helm/sheepdog/values.yaml | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/helm/sheepdog/README.md b/helm/sheepdog/README.md index d8970578c..7db76c224 100644 --- a/helm/sheepdog/README.md +++ b/helm/sheepdog/README.md @@ -32,6 +32,7 @@ Published versions of this chart are listed in the | commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | | criticalService | string | `"true"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | | dataDog | bool | `{"enabled":false,"env":"dev"}` | Whether Datadog is enabled. | +| debug | bool | `false` | True to enable debug logs and other dev only functionality, false if otherwise. | | externalSecrets | map | `{"dbcreds":null,"pushSecret":false}` | External Secrets settings. | | externalSecrets.dbcreds | string | `nil` | Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" | | externalSecrets.pushSecret | bool | `false` | Whether to create the database and Secrets Manager secrets via PushSecret. | diff --git a/helm/sheepdog/templates/deployment.yaml b/helm/sheepdog/templates/deployment.yaml index e7ed92b63..a750a3217 100644 --- a/helm/sheepdog/templates/deployment.yaml +++ b/helm/sheepdog/templates/deployment.yaml @@ -251,7 +251,7 @@ spec: # # # value: /etc/ssl/certs/ca-certificates.crt - name: GEN3_DEBUG - value: "True" + value: {{ .Values.debug }} {{- with .Values.volumeMounts }} volumeMounts: {{- toYaml . | nindent 10 }} diff --git a/helm/sheepdog/values.yaml b/helm/sheepdog/values.yaml index eb907fb75..add38b02d 100644 --- a/helm/sheepdog/values.yaml +++ b/helm/sheepdog/values.yaml @@ -86,7 +86,8 @@ global: # -- (map) This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ autoscaling: {} - +# -- (bool) True to enable debug logs and other dev only functionality, false if otherwise. +debug: false # -- (bool) Whether Metrics are enabled. metricsEnabled: From f50ceced5c2db520b841e9b2d91c167c703f8559 Mon Sep 17 00:00:00 2001 From: Kyle Burton Date: Tue, 15 Sep 2026 11:41:27 -0500 Subject: [PATCH 16/23] Change debug type --- helm/sheepdog/README.md | 2 +- helm/sheepdog/values.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/sheepdog/README.md b/helm/sheepdog/README.md index 7db76c224..a9cbd09aa 100644 --- a/helm/sheepdog/README.md +++ b/helm/sheepdog/README.md @@ -32,7 +32,6 @@ Published versions of this chart are listed in the | commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | | criticalService | string | `"true"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | | dataDog | bool | `{"enabled":false,"env":"dev"}` | Whether Datadog is enabled. | -| debug | bool | `false` | True to enable debug logs and other dev only functionality, false if otherwise. | | externalSecrets | map | `{"dbcreds":null,"pushSecret":false}` | External Secrets settings. | | externalSecrets.dbcreds | string | `nil` | Will override the name of the aws secrets manager secret. Default is "Values.global.environment-.Chart.Name-creds" | | externalSecrets.pushSecret | bool | `false` | Whether to create the database and Secrets Manager secrets via PushSecret. | @@ -47,6 +46,7 @@ Published versions of this chart are listed in the | global.aws.enabled | bool | `false` | Set to true if deploying to AWS. Controls ingress annotations. | | global.aws.externalSecrets.enabled | bool | `false` | Whether to use External Secrets for aws config. | | global.aws.externalSecrets.externalSecretAwsCreds | String | `nil` | Name of Secrets Manager secret. | +| global.debug | str | `"False"` | True to enable debug logs and other dev only functionality, False if otherwise. | | global.dev | bool | `true` | Whether the deployment is for development purposes. | | global.dictionaryUrl | string | `"https://s3.amazonaws.com/dictionary-artifacts/datadictionary/develop/schema.json"` | URL of the data dictionary. | | global.dispatcherJobNum | int | `"10"` | Number of dispatcher jobs. | diff --git a/helm/sheepdog/values.yaml b/helm/sheepdog/values.yaml index add38b02d..93cfa4ec4 100644 --- a/helm/sheepdog/values.yaml +++ b/helm/sheepdog/values.yaml @@ -19,7 +19,8 @@ global: externalSecretAwsCreds: # -- (bool) Whether the deployment is for development purposes. dev: true - + # -- (str) True to enable debug logs and other dev only functionality, False if otherwise. + debug: "False" postgres: # -- (bool) Whether the database should be created. dbCreate: true @@ -86,8 +87,7 @@ global: # -- (map) This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ autoscaling: {} -# -- (bool) True to enable debug logs and other dev only functionality, false if otherwise. -debug: false + # -- (bool) Whether Metrics are enabled. metricsEnabled: From 36a078a516a563a15ae70002097cb1059f78f9d1 Mon Sep 17 00:00:00 2001 From: pieterlukasse Date: Tue, 22 Sep 2026 16:18:08 -0400 Subject: [PATCH 17/23] feat: introduce global argoHost and use in argo-wrapper and revproxy --- helm/argo-wrapper/README.md | 1 + helm/argo-wrapper/templates/deployment.yaml | 4 +++- helm/argo-wrapper/values.yaml | 2 ++ helm/gen3/README.md | 1 + helm/gen3/values.yaml | 2 ++ helm/revproxy/README.md | 1 + helm/revproxy/gen3.nginx.conf/argo-server.conf | 2 +- helm/revproxy/nginx/nginx.conf | 1 + helm/revproxy/values.yaml | 3 ++- 9 files changed, 14 insertions(+), 3 deletions(-) diff --git a/helm/argo-wrapper/README.md b/helm/argo-wrapper/README.md index 4a238d390..0bdeb36b6 100644 --- a/helm/argo-wrapper/README.md +++ b/helm/argo-wrapper/README.md @@ -29,6 +29,7 @@ Published versions of this chart are listed in the | commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | | criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | | environment | string | `"default"` | Environment name. | +| global.argoHost | string | `"http://argo-workflows-server.argo.svc.cluster.local:2746"` | argo server host URL:PORT. Endpoint for the the Argo Workflows service. | | global.autoscaling.averageCPUValue | string | `"500m"` | | | global.autoscaling.averageMemoryValue | string | `"500Mi"` | | | global.autoscaling.enabled | bool | `false` | | diff --git a/helm/argo-wrapper/templates/deployment.yaml b/helm/argo-wrapper/templates/deployment.yaml index ab8c70233..cd45f46d9 100644 --- a/helm/argo-wrapper/templates/deployment.yaml +++ b/helm/argo-wrapper/templates/deployment.yaml @@ -76,4 +76,6 @@ spec: {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} - env: \ No newline at end of file + env: + - name: ARGO_HOST + value: {{ .Values.global.argoHost }} diff --git a/helm/argo-wrapper/values.yaml b/helm/argo-wrapper/values.yaml index 8837da919..04157f9a1 100644 --- a/helm/argo-wrapper/values.yaml +++ b/helm/argo-wrapper/values.yaml @@ -4,6 +4,8 @@ # Global configuration global: + # -- (string) argo server host URL:PORT. Endpoint for the the Argo Workflows service. + argoHost: http://argo-workflows-server.argo.svc.cluster.local:2746 # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. environment: default # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 62073e924..2dc9e4be2 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -136,6 +136,7 @@ Published versions of this chart are listed in the | gen3-user-data-library.enabled | bool | `false` | Whether to deploy the gen3-user-data-library subchart. | | gen3-workflow | map | `{"enabled":false}` | Configurations for gen3-workflow chart. | | gen3-workflow.enabled | bool | `false` | Whether to deploy the gen3-workflow subchart. | +| global.argoHost | string | `"http://argo-workflows-server.argo.svc.cluster.local:2746"` | argo server host URL:PORT. Endpoint for the the Argo Workflows service. | | global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false,"externalSecrets":{"enabled":false,"externalSecretAwsCreds":null,"pushSecret":false},"region":"us-east-1","secretStoreServiceAccount":{"enabled":false,"name":"secret-store-sa","roleArn":null},"useLocalSecret":{"enabled":false,"localSecretName":null}}` | AWS configuration | | global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | | global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index f76c1e5d2..c128523a3 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -147,6 +147,8 @@ global: topologyKey: "topology.kubernetes.io/zone" # -- (int) The maxSkew to use for topology spread constraints. Defaults to 1. maxSkew: 1 + # -- (string) argo server host URL:PORT. Endpoint for the the Argo Workflows service. + argoHost: http://argo-workflows-server.argo.svc.cluster.local:2746 # Dependancy Charts diff --git a/helm/revproxy/README.md b/helm/revproxy/README.md index c34724fe3..dd33aaaa1 100644 --- a/helm/revproxy/README.md +++ b/helm/revproxy/README.md @@ -26,6 +26,7 @@ Published versions of this chart are listed in the | enableRobotsTxt | bool | `false` | Whether to enable robots.txt generation and serving. | | extraServices | map | `nil` | Configuration to add any extra service endpoints outside of gen3 to be served by revproxy | | fullnameOverride | string | `""` | Override the full name of the deployment. | +| global.argoHost | string | `"http://argo-workflows-server.argo.svc.cluster.local:2746"` | argo server host URL:PORT. Endpoint for the the Argo Workflows service. | | global.autoscaling.averageCPUValue | string | `"500m"` | | | global.autoscaling.averageMemoryValue | string | `"500Mi"` | | | global.autoscaling.enabled | bool | `false` | | diff --git a/helm/revproxy/gen3.nginx.conf/argo-server.conf b/helm/revproxy/gen3.nginx.conf/argo-server.conf index 185ce05e4..e0a69cf30 100644 --- a/helm/revproxy/gen3.nginx.conf/argo-server.conf +++ b/helm/revproxy/gen3.nginx.conf/argo-server.conf @@ -7,7 +7,7 @@ location /argo/ { auth_request /gen3-authz; set $proxy_service "argo"; - set $upstream http://argo-argo-workflows-server.argo.svc.cluster.local:2746; + set $upstream $argoHost; rewrite ^/argo/(.*) /$1 break; diff --git a/helm/revproxy/nginx/nginx.conf b/helm/revproxy/nginx/nginx.conf index 920a18f1e..6d8f44f7b 100644 --- a/helm/revproxy/nginx/nginx.conf +++ b/helm/revproxy/nginx/nginx.conf @@ -273,6 +273,7 @@ map $http_user_agent $loggable { # individual service locations should override to "peregrine", ... # set $proxy_service "noproxy"; + set $argoHost {{ .Values.global.argoHost }}; # # Note - need to repeat this line in location blocks that call proxy_set_header, diff --git a/helm/revproxy/values.yaml b/helm/revproxy/values.yaml index a85aa0b73..84165c648 100644 --- a/helm/revproxy/values.yaml +++ b/helm/revproxy/values.yaml @@ -88,7 +88,8 @@ global: topologyKey: "topology.kubernetes.io/zone" # -- (int) The maxSkew to use for topology spread constraints. Defaults to 1. maxSkew: 1 - + # -- (string) argo server host URL:PORT. Endpoint for the the Argo Workflows service. + argoHost: http://argo-workflows-server.argo.svc.cluster.local:2746 # -- (map) This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ autoscaling: {} From e272b7c22e9efa40efefae5f9bd7708955aee4f6 Mon Sep 17 00:00:00 2001 From: pieterlukasse Date: Tue, 22 Sep 2026 16:30:09 -0400 Subject: [PATCH 18/23] feat: leave the argo- double prefix leaving argo-argo-workflows-server.... for now. The stutter is the release name meeting the chart name. --- helm/argo-wrapper/README.md | 2 +- helm/argo-wrapper/values.yaml | 2 +- helm/gen3/README.md | 2 +- helm/gen3/values.yaml | 2 +- helm/revproxy/README.md | 2 +- helm/revproxy/values.yaml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/helm/argo-wrapper/README.md b/helm/argo-wrapper/README.md index 0bdeb36b6..b9ea266df 100644 --- a/helm/argo-wrapper/README.md +++ b/helm/argo-wrapper/README.md @@ -29,7 +29,7 @@ Published versions of this chart are listed in the | commonLabels | map | `nil` | Will completely override the commonLabels defined in the common chart's _label_setup.tpl | | criticalService | string | `"false"` | Valid options are "true" or "false". If invalid option is set- the value will default to "false". | | environment | string | `"default"` | Environment name. | -| global.argoHost | string | `"http://argo-workflows-server.argo.svc.cluster.local:2746"` | argo server host URL:PORT. Endpoint for the the Argo Workflows service. | +| global.argoHost | string | `"http://argo-argo-workflows-server.argo.svc.cluster.local:2746"` | argo server host URL:PORT. Endpoint for the the Argo Workflows service. | | global.autoscaling.averageCPUValue | string | `"500m"` | | | global.autoscaling.averageMemoryValue | string | `"500Mi"` | | | global.autoscaling.enabled | bool | `false` | | diff --git a/helm/argo-wrapper/values.yaml b/helm/argo-wrapper/values.yaml index 04157f9a1..1153cf32d 100644 --- a/helm/argo-wrapper/values.yaml +++ b/helm/argo-wrapper/values.yaml @@ -5,7 +5,7 @@ # Global configuration global: # -- (string) argo server host URL:PORT. Endpoint for the the Argo Workflows service. - argoHost: http://argo-workflows-server.argo.svc.cluster.local:2746 + argoHost: http://argo-argo-workflows-server.argo.svc.cluster.local:2746 # -- (string) Environment name. This should be the same as vpcname if you're doing an AWS deployment. Currently this is being used to share ALB's if you have multiple namespaces. Might be used other places too. environment: default # -- (bool) If the service will be deployed with a Pod Disruption Budget. Note- you need to have more than 2 replicas for the pdb to be deployed. diff --git a/helm/gen3/README.md b/helm/gen3/README.md index 2dc9e4be2..f10e69835 100644 --- a/helm/gen3/README.md +++ b/helm/gen3/README.md @@ -136,7 +136,7 @@ Published versions of this chart are listed in the | gen3-user-data-library.enabled | bool | `false` | Whether to deploy the gen3-user-data-library subchart. | | gen3-workflow | map | `{"enabled":false}` | Configurations for gen3-workflow chart. | | gen3-workflow.enabled | bool | `false` | Whether to deploy the gen3-workflow subchart. | -| global.argoHost | string | `"http://argo-workflows-server.argo.svc.cluster.local:2746"` | argo server host URL:PORT. Endpoint for the the Argo Workflows service. | +| global.argoHost | string | `"http://argo-argo-workflows-server.argo.svc.cluster.local:2746"` | argo server host URL:PORT. Endpoint for the the Argo Workflows service. | | global.aws | map | `{"awsAccessKeyId":null,"awsSecretAccessKey":null,"enabled":false,"externalSecrets":{"enabled":false,"externalSecretAwsCreds":null,"pushSecret":false},"region":"us-east-1","secretStoreServiceAccount":{"enabled":false,"name":"secret-store-sa","roleArn":null},"useLocalSecret":{"enabled":false,"localSecretName":null}}` | AWS configuration | | global.aws.awsAccessKeyId | string | `nil` | Credentials for AWS stuff. | | global.aws.awsSecretAccessKey | string | `nil` | Credentials for AWS stuff. | diff --git a/helm/gen3/values.yaml b/helm/gen3/values.yaml index c128523a3..368cc51ba 100644 --- a/helm/gen3/values.yaml +++ b/helm/gen3/values.yaml @@ -148,7 +148,7 @@ global: # -- (int) The maxSkew to use for topology spread constraints. Defaults to 1. maxSkew: 1 # -- (string) argo server host URL:PORT. Endpoint for the the Argo Workflows service. - argoHost: http://argo-workflows-server.argo.svc.cluster.local:2746 + argoHost: http://argo-argo-workflows-server.argo.svc.cluster.local:2746 # Dependancy Charts diff --git a/helm/revproxy/README.md b/helm/revproxy/README.md index dd33aaaa1..5872ef0fd 100644 --- a/helm/revproxy/README.md +++ b/helm/revproxy/README.md @@ -26,7 +26,7 @@ Published versions of this chart are listed in the | enableRobotsTxt | bool | `false` | Whether to enable robots.txt generation and serving. | | extraServices | map | `nil` | Configuration to add any extra service endpoints outside of gen3 to be served by revproxy | | fullnameOverride | string | `""` | Override the full name of the deployment. | -| global.argoHost | string | `"http://argo-workflows-server.argo.svc.cluster.local:2746"` | argo server host URL:PORT. Endpoint for the the Argo Workflows service. | +| global.argoHost | string | `"http://argo-argo-workflows-server.argo.svc.cluster.local:2746"` | argo server host URL:PORT. Endpoint for the the Argo Workflows service. | | global.autoscaling.averageCPUValue | string | `"500m"` | | | global.autoscaling.averageMemoryValue | string | `"500Mi"` | | | global.autoscaling.enabled | bool | `false` | | diff --git a/helm/revproxy/values.yaml b/helm/revproxy/values.yaml index 84165c648..d77e58912 100644 --- a/helm/revproxy/values.yaml +++ b/helm/revproxy/values.yaml @@ -89,7 +89,7 @@ global: # -- (int) The maxSkew to use for topology spread constraints. Defaults to 1. maxSkew: 1 # -- (string) argo server host URL:PORT. Endpoint for the the Argo Workflows service. - argoHost: http://argo-workflows-server.argo.svc.cluster.local:2746 + argoHost: http://argo-argo-workflows-server.argo.svc.cluster.local:2746 # -- (map) This section is for setting up autoscaling more information can be found here: https://kubernetes.io/docs/concepts/workloads/autoscaling/ autoscaling: {} From afde52f9639f6918e74250c4233e49d817205fa7 Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 24 Sep 2026 14:02:01 -0500 Subject: [PATCH 19/23] Funnel worker mounts the whole bucket (#711) --- helm/funnel/files/worker-job.yaml | 3 +-- helm/funnel/files/worker-pv.yaml | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/helm/funnel/files/worker-job.yaml b/helm/funnel/files/worker-job.yaml index 7bf13da87..ebf6042bc 100644 --- a/helm/funnel/files/worker-job.yaml +++ b/helm/funnel/files/worker-job.yaml @@ -100,8 +100,7 @@ spec: {{` {{- if .NeedsPVC }} - name: funnel-storage-{{.TaskId}} - mountPath: /opt/funnel/funnel-work-dir/{{.TaskId}} - subPath: {{.TaskId}} + mountPath: /opt/funnel/funnel-work-dir {{- end }} `}} volumes: diff --git a/helm/funnel/files/worker-pv.yaml b/helm/funnel/files/worker-pv.yaml index d118617bf..92d9082b4 100644 --- a/helm/funnel/files/worker-pv.yaml +++ b/helm/funnel/files/worker-pv.yaml @@ -22,7 +22,6 @@ spec: - allow-other # allow non-root users to access the mount - region={{`{{.Region}}`}} - file-mode=0755 - - prefix=funnel-temp-files/ {{`{{- if .KmsKeyID}}`}} - sse aws:kms - sse-kms-key-id={{`{{.KmsKeyID}}`}} From c8ed6f4127f23adabe0ce61ccc1504de87c1becd Mon Sep 17 00:00:00 2001 From: Pauline Ribeyre <4224001+paulineribeyre@users.noreply.github.com> Date: Thu, 24 Sep 2026 16:52:23 -0500 Subject: [PATCH 20/23] Default funnel to CTDS fork (#716) --- helm/funnel/README.md | 4 ++-- helm/funnel/values.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/helm/funnel/README.md b/helm/funnel/README.md index 8b25fa75c..0f7258421 100644 --- a/helm/funnel/README.md +++ b/helm/funnel/README.md @@ -187,8 +187,8 @@ Published versions of this chart are listed in the | image.initContainers[0].pullPolicy | string | `"Always"` | When to pull the image. This value should be "Always" to ensure the latest image is used. | | image.initContainers[0].tag | string | `"main-gen3"` | The Docker image tag for the Funnel init/plugin container. | | image.pullPolicy | string | `"Always"` | When to pull the image. This value should be "Always" to ensure the latest image is used. | -| image.repository | string | `"quay.io/ohsu-comp-bio/funnel"` | The Docker image repository for the Funnel service. | -| image.tag | string | `"develop-2026-07-09-19-49-55Z-97d1df55"` | | +| image.repository | string | `"quay.io/cdis/funnel"` | The Docker image repository for the Funnel service. | +| image.tag | string | `"main-gen3"` | | | labels.app | string | `"funnel"` | | | metricsEnabled | bool | `false` | | | netPolicy | map | `{"egressApps":["gen3-workflow"],"ingressApps":["gen3-workflow"]}` | Configuration for network policies created by this chart. Only relevant if "global.netPolicy.enabled" is set to true | diff --git a/helm/funnel/values.yaml b/helm/funnel/values.yaml index e49125757..86c9ba43d 100644 --- a/helm/funnel/values.yaml +++ b/helm/funnel/values.yaml @@ -127,8 +127,8 @@ partOf: "Workflow_Execution" oidc_job_enabled: true image: # -- (string) The Docker image repository for the Funnel service. - repository: quay.io/ohsu-comp-bio/funnel - tag: develop-2026-07-09-19-49-55Z-97d1df55 + repository: quay.io/cdis/funnel + tag: main-gen3 # -- (string) When to pull the image. This value should be "Always" to ensure the latest image is used. pullPolicy: Always From 6d24ef43ba834b00747173c7d3e86046926c9e66 Mon Sep 17 00:00:00 2001 From: pieterlukasse Date: Thu, 1 Oct 2026 17:54:15 -0400 Subject: [PATCH 21/23] feat: add network egress policy to allow arborist to be reached by workspace pod --- helm/workspace-proxy/templates/netpol.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/helm/workspace-proxy/templates/netpol.yaml b/helm/workspace-proxy/templates/netpol.yaml index 5b3efcdac..b962d323b 100644 --- a/helm/workspace-proxy/templates/netpol.yaml +++ b/helm/workspace-proxy/templates/netpol.yaml @@ -45,6 +45,18 @@ spec: port: 80 - protocol: TCP port: 8888 + # Arborist: workspace-proxy calls arborist /auth/proxy to authenticate & + # authorize every request. + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ .Values.deploymentNamespace }} + podSelector: + matchLabels: + app: arborist + ports: + - protocol: TCP + port: 8080 {{- range .Values.kubernetesApiServerCIDRs }} - to: - ipBlock: From e91a8beb448aab5472f77db44682eb26e2b8f0d4 Mon Sep 17 00:00:00 2001 From: nss10 Date: Tue, 6 Oct 2026 14:15:10 -0500 Subject: [PATCH 22/23] Update funnel kubeapi endpoint port for kind --- helm/funnel/templates/netpolicy.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/helm/funnel/templates/netpolicy.yaml b/helm/funnel/templates/netpolicy.yaml index 297622ec8..a863dda83 100644 --- a/helm/funnel/templates/netpolicy.yaml +++ b/helm/funnel/templates/netpolicy.yaml @@ -137,6 +137,8 @@ spec: # Allow access to the Kubernetes API server in kube-system - to: [] ports: + - protocol: TCP + port: 6443 - protocol: TCP port: 443 --- @@ -170,7 +172,9 @@ spec: {{- end }} ports: - protocol: TCP - port: 443 # Kubernetes API + port: 443 + - protocol: TCP + port: 6443 # Kubernetes API policyTypes: - Egress From 2a782e7e0067f7af00b2b29b5735e8b6c9327962 Mon Sep 17 00:00:00 2001 From: Jawad Date: Wed, 7 Oct 2026 11:27:35 -0500 Subject: [PATCH 23/23] feat(cluster-level-resources): allow overriding gpu karpenter instance types Adds an opt-in karpenter-crds.gpu.instanceTypes value. When set, the gpu and gpu-shared NodePools require node.kubernetes.io/instance-type In instead of the hardcoded g4dn instance family. Default (empty) renders identically to before. --- helm/cluster-level-resources/README.md | 1 + .../karpenter-config-resources-gpu.yaml | 24 +++++++++++++++++++ helm/cluster-level-resources/values.yaml | 2 ++ 3 files changed, 27 insertions(+) diff --git a/helm/cluster-level-resources/README.md b/helm/cluster-level-resources/README.md index 8fb3fb012..89d508219 100644 --- a/helm/cluster-level-resources/README.md +++ b/helm/cluster-level-resources/README.md @@ -115,6 +115,7 @@ Published versions of this chart are listed in the | karpenter-crds.gpu.fastImagePull.enabled | bool | `false` | | | karpenter-crds.gpu.fastImagePull.iops | int | `3000` | | | karpenter-crds.gpu.fastImagePull.throughput | int | `600` | | +| karpenter-crds.gpu.instanceTypes | list | `[]` | Specific EC2 instance types for the gpu NodePools (e.g. ["g6.xlarge"]). If empty, defaults to the g4dn instance family. | | karpenter-crds.gpu.volumeSize | string | `"50Gi"` | | | karpenter-crds.jupyter.additionalTags | object | `{}` | | | karpenter-crds.jupyter.consolidateAfter | string | `"30s"` | | diff --git a/helm/cluster-level-resources/templates/karpenter-config-resources-gpu.yaml b/helm/cluster-level-resources/templates/karpenter-config-resources-gpu.yaml index e3ad48fab..3860e2dd7 100644 --- a/helm/cluster-level-resources/templates/karpenter-config-resources-gpu.yaml +++ b/helm/cluster-level-resources/templates/karpenter-config-resources-gpu.yaml @@ -108,9 +108,15 @@ spec: value: "present" effect: "NoSchedule" requirements: + {{- with (index .Values "karpenter-crds" "gpu" "instanceTypes") }} + - key: node.kubernetes.io/instance-type + operator: In + values: {{ toJson . }} + {{- else }} - key: karpenter.k8s.aws/instance-family operator: In values: ["g4dn"] + {{- end }} - key: karpenter.sh/capacity-type operator: In values: ["on-demand"] @@ -166,9 +172,15 @@ spec: value: "present" effect: "NoSchedule" requirements: + {{- with (index .Values "karpenter-crds" "gpu" "instanceTypes") }} + - key: node.kubernetes.io/instance-type + operator: In + values: {{ toJson . }} + {{- else }} - key: karpenter.k8s.aws/instance-family operator: In values: ["g4dn"] + {{- end }} - key: karpenter.sh/capacity-type operator: In values: ["on-demand"] @@ -203,9 +215,15 @@ spec: value: "present" effect: "NoSchedule" requirements: + {{- with (index .Values "karpenter-crds" "gpu" "instanceTypes") }} + - key: node.kubernetes.io/instance-type + operator: In + values: {{ toJson . }} + {{- else }} - key: karpenter.k8s.aws/instance-family operator: In values: ["g4dn"] + {{- end }} - key: karpenter.sh/capacity-type operator: In values: ["spot","on-demand"] @@ -266,9 +284,15 @@ spec: value: "present" effect: "NoSchedule" requirements: + {{- with (index .Values "karpenter-crds" "gpu" "instanceTypes") }} + - key: node.kubernetes.io/instance-type + operator: In + values: {{ toJson . }} + {{- else }} - key: karpenter.k8s.aws/instance-family operator: In values: ["g4dn"] + {{- end }} - key: karpenter.sh/capacity-type operator: In values: ["spot","on-demand"] diff --git a/helm/cluster-level-resources/values.yaml b/helm/cluster-level-resources/values.yaml index b0102eadc..80e5eea02 100644 --- a/helm/cluster-level-resources/values.yaml +++ b/helm/cluster-level-resources/values.yaml @@ -191,6 +191,8 @@ karpenter-crds: iops: 3000 throughput: 600 additionalTags: {} + # -- (list) Specific EC2 instance types for the gpu NodePools (e.g. ["g6.xlarge"]). If empty, defaults to the g4dn instance family. + instanceTypes: [] secondary: enabled: false weight: 1