From b87ed3f66316bccbf08dc6b726d036888c7065e7 Mon Sep 17 00:00:00 2001 From: Jakub Dobry Date: Mon, 15 Dec 2025 19:45:49 +0100 Subject: [PATCH 1/4] feat: add more host metrics --- .../nomad/configs/otel-collector.yaml | 94 ++++++++++++------- 1 file changed, 58 insertions(+), 36 deletions(-) diff --git a/iac/provider-gcp/nomad/configs/otel-collector.yaml b/iac/provider-gcp/nomad/configs/otel-collector.yaml index 4401676920..e7115e9093 100644 --- a/iac/provider-gcp/nomad/configs/otel-collector.yaml +++ b/iac/provider-gcp/nomad/configs/otel-collector.yaml @@ -14,11 +14,11 @@ receivers: - job_name: nomad scrape_interval: 15s scrape_timeout: 5s - metrics_path: '/v1/metrics' + metrics_path: "/v1/metrics" static_configs: - - targets: ['localhost:4646'] + - targets: ["localhost:4646"] params: - format: ['prometheus'] + format: ["prometheus"] hostmetrics: root_path: /hostfs @@ -30,7 +30,7 @@ receivers: scrapers: cpu: metrics: - system.cpu.time: # counter of seconds spent + system.cpu.time: # counter of seconds spent enabled: true system.cpu.frequency: enabled: false @@ -38,7 +38,7 @@ receivers: enabled: true system.cpu.physical.count: enabled: true - system.cpu.utilization: # percentage utilized in the moment + system.cpu.utilization: # percentage utilized in the moment enabled: false # entirely disabled, so we omit the module @@ -89,7 +89,7 @@ receivers: enabled: true system.linux.memory.available: # free memory enabled: true - system.linux.memory.dirty: # data waiting to be written to disk + system.linux.memory.dirty: # data waiting to be written to disk enabled: true system.memory.limit: # total memory enabled: true @@ -156,36 +156,36 @@ receivers: enabled: true # entirely disabled, so we omit the module - #process: - # metrics: - # process.cpu.time: - # enabled: false - # process.disk.io: - # enabled: false - # process.memory.usage: - # enabled: false - # process.memory.virtual: - # enabled: false - # process.context_switches: - # enabled: false - # process.cpu.utilization: - # enabled: false - # process.disk.operations: - # enabled: false - # process.handles: - # enabled: false - # process.memory.utilization: - # enabled: false - # process.open_file_descriptors: - # enabled: false - # process.paging.faults: - # enabled: false - # process.signals_pending: - # enabled: false - # process.threads: - # enabled: false - # process.uptime: - # enabled: false + process: + metrics: + process.cpu.time: + enabled: false + process.disk.io: + enabled: false + process.memory.usage: + enabled: false + process.memory.virtual: + enabled: false + process.context_switches: + enabled: true + process.cpu.utilization: + enabled: false + process.disk.operations: + enabled: false + process.handles: + enabled: false + process.memory.utilization: + enabled: false + process.open_file_descriptors: + enabled: true + process.paging.faults: + enabled: false + process.signals_pending: + enabled: false + process.threads: + enabled: false + process.uptime: + enabled: false # entirely disabled, so we omit the module #system: @@ -253,6 +253,27 @@ processors: - state aggregation_type: sum + metricstransform/aggregate_fd: + transforms: + - include: "process.open_file_descriptors" + match_type: strict + action: update + new_name: "system.open_file_descriptors" + operations: + - action: aggregate_labels + label_set: + - node.id + aggregation_type: sum + - include: "process.context_switches" + match_type: strict + action: update + new_name: "system.context_switches" + operations: + - action: aggregate_labels + label_set: + - node.id + aggregation_type: sum + resourcedetection: # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor detectors: [gcp] @@ -415,6 +436,7 @@ service: processors: - attributes/host_metrics_node - metricstransform/single_cpu + - metricstransform/aggregate_fd - resourcedetection - transform/set-name - batch From e90555c95f35ab51b959abd846005dcb3688be76 Mon Sep 17 00:00:00 2001 From: Jakub Dobry Date: Mon, 15 Dec 2025 19:50:30 +0100 Subject: [PATCH 2/4] add threads count --- iac/provider-gcp/nomad/configs/otel-collector.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/iac/provider-gcp/nomad/configs/otel-collector.yaml b/iac/provider-gcp/nomad/configs/otel-collector.yaml index e7115e9093..6154882029 100644 --- a/iac/provider-gcp/nomad/configs/otel-collector.yaml +++ b/iac/provider-gcp/nomad/configs/otel-collector.yaml @@ -183,7 +183,7 @@ receivers: process.signals_pending: enabled: false process.threads: - enabled: false + enabled: true process.uptime: enabled: false @@ -273,6 +273,15 @@ processors: label_set: - node.id aggregation_type: sum + - include: "process.threads" + match_type: strict + action: update + new_name: "system.threads" + operations: + - action: aggregate_labels + label_set: + - node.id + aggregation_type: sum resourcedetection: # https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/processor/resourcedetectionprocessor From 7eabdbba49cc2debde4990786bc9cf1432021e59 Mon Sep 17 00:00:00 2001 From: Jakub Dobry Date: Mon, 15 Dec 2025 19:51:48 +0100 Subject: [PATCH 3/4] rename metrics aggregation filter --- iac/provider-gcp/nomad/configs/otel-collector.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/iac/provider-gcp/nomad/configs/otel-collector.yaml b/iac/provider-gcp/nomad/configs/otel-collector.yaml index 6154882029..7eb1d24b78 100644 --- a/iac/provider-gcp/nomad/configs/otel-collector.yaml +++ b/iac/provider-gcp/nomad/configs/otel-collector.yaml @@ -253,7 +253,7 @@ processors: - state aggregation_type: sum - metricstransform/aggregate_fd: + metricstransform/aggregate_process_metrics: transforms: - include: "process.open_file_descriptors" match_type: strict @@ -445,7 +445,7 @@ service: processors: - attributes/host_metrics_node - metricstransform/single_cpu - - metricstransform/aggregate_fd + - metricstransform/aggregate_process_metrics - resourcedetection - transform/set-name - batch From e588b2a92557471074bc358c057cc5bee07b3384 Mon Sep 17 00:00:00 2001 From: Jakub Dobry Date: Mon, 15 Dec 2025 20:12:25 +0100 Subject: [PATCH 4/4] mute process scrape errors --- iac/provider-gcp/nomad/configs/otel-collector.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/iac/provider-gcp/nomad/configs/otel-collector.yaml b/iac/provider-gcp/nomad/configs/otel-collector.yaml index 7eb1d24b78..bfe76e5caf 100644 --- a/iac/provider-gcp/nomad/configs/otel-collector.yaml +++ b/iac/provider-gcp/nomad/configs/otel-collector.yaml @@ -157,6 +157,7 @@ receivers: # entirely disabled, so we omit the module process: + mute_process_all_errors: true metrics: process.cpu.time: enabled: false