From c1b4fd210e0e62746536d59198833281229e3d45 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Sun, 13 Sep 2026 14:22:55 +0000 Subject: [PATCH 1/6] add affinity for all system workloads --- fleet/authentik-config/outpost.yaml | 6 ++- .../token-sync-initial-job.yaml | 20 ++++++-- .../authentik-token-sync/token-sync-job.yaml | 20 ++++++-- fleet/authentik/fleet.yaml | 18 ++++--- fleet/bandwidth-operator/fleet.yaml | 6 ++- fleet/bandwidth-scheduler/fleet.yaml | 23 +++++++-- fleet/homepage/deployment.yaml | 6 ++- fleet/longhorn/fleet.yaml | 6 ++- .../victoria-logs-cluster-external/fleet.yaml | 51 ++++++++++++++++--- fleet/victoria-logs-cluster/fleet.yaml | 35 +++++++++++-- fleet/victoria-metrics-stack-ext/fleet.yaml | 45 ++++++++++++++++ fleet/victoria-metrics-stack/fleet.yaml | 28 ++++++++-- 12 files changed, 224 insertions(+), 40 deletions(-) diff --git a/fleet/authentik-config/outpost.yaml b/fleet/authentik-config/outpost.yaml index 459c24a..f5b7c32 100644 --- a/fleet/authentik-config/outpost.yaml +++ b/fleet/authentik-config/outpost.yaml @@ -19,8 +19,10 @@ spec: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/worker - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" diff --git a/fleet/authentik-token-sync/token-sync-initial-job.yaml b/fleet/authentik-token-sync/token-sync-initial-job.yaml index b9caf5e..097d5dc 100644 --- a/fleet/authentik-token-sync/token-sync-initial-job.yaml +++ b/fleet/authentik-token-sync/token-sync-initial-job.yaml @@ -13,11 +13,21 @@ spec: ttlSecondsAfterFinished: 3600 template: spec: - #tolerations: - # - key: "dedicated" - # operator: "Equal" - # value: "monitoring" - # effect: "PreferNoSchedule" + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst serviceAccountName: authentik-outpost-token-sync restartPolicy: OnFailure initContainers: diff --git a/fleet/authentik-token-sync/token-sync-job.yaml b/fleet/authentik-token-sync/token-sync-job.yaml index ccc4308..ffda88c 100644 --- a/fleet/authentik-token-sync/token-sync-job.yaml +++ b/fleet/authentik-token-sync/token-sync-job.yaml @@ -11,11 +11,21 @@ spec: spec: template: spec: - #tolerations: - # - key: "dedicated" - # operator: "Equal" - # value: "monitoring" - # effect: "PreferNoSchedule" + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst serviceAccountName: authentik-outpost-token-sync restartPolicy: OnFailure initContainers: diff --git a/fleet/authentik/fleet.yaml b/fleet/authentik/fleet.yaml index f5bf58a..13fad85 100644 --- a/fleet/authentik/fleet.yaml +++ b/fleet/authentik/fleet.yaml @@ -55,8 +55,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" @@ -170,8 +172,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" @@ -216,8 +220,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" diff --git a/fleet/bandwidth-operator/fleet.yaml b/fleet/bandwidth-operator/fleet.yaml index c474d8f..e4a287e 100644 --- a/fleet/bandwidth-operator/fleet.yaml +++ b/fleet/bandwidth-operator/fleet.yaml @@ -46,8 +46,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" diff --git a/fleet/bandwidth-scheduler/fleet.yaml b/fleet/bandwidth-scheduler/fleet.yaml index 4e8115b..f4e55d0 100644 --- a/fleet/bandwidth-scheduler/fleet.yaml +++ b/fleet/bandwidth-scheduler/fleet.yaml @@ -26,8 +26,21 @@ helm: cpu: 1 memory: 500Mi nodeSelector: {} - affinity: {} - tolerations: [] + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" controller: name: vaclab-scheduler-controller @@ -49,8 +62,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" diff --git a/fleet/homepage/deployment.yaml b/fleet/homepage/deployment.yaml index 64bb9e2..a46b8af 100644 --- a/fleet/homepage/deployment.yaml +++ b/fleet/homepage/deployment.yaml @@ -28,8 +28,10 @@ spec: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" diff --git a/fleet/longhorn/fleet.yaml b/fleet/longhorn/fleet.yaml index ca83a0f..3c1d0db 100644 --- a/fleet/longhorn/fleet.yaml +++ b/fleet/longhorn/fleet.yaml @@ -30,8 +30,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" diff --git a/fleet/victoria-logs-cluster-external/fleet.yaml b/fleet/victoria-logs-cluster-external/fleet.yaml index 3c6e772..43a76f3 100644 --- a/fleet/victoria-logs-cluster-external/fleet.yaml +++ b/fleet/victoria-logs-cluster-external/fleet.yaml @@ -27,7 +27,16 @@ helm: vector: enabled: false vlinsert: - affinity: {} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst annotations: {} containerWorkingDir: "" enabled: true @@ -127,7 +136,11 @@ helm: strategy: {} suppressStorageFQDNsRender: false terminationGracePeriodSeconds: 30 - tolerations: [] + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" topologySpreadConstraints: [] vmServiceScrape: annotations: {} @@ -139,7 +152,16 @@ helm: - port: http useServiceMonitor: false vlselect: - affinity: {} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst annotations: {} containerWorkingDir: "" enabled: true @@ -247,7 +269,11 @@ helm: strategy: {} suppressStorageFQDNsRender: false terminationGracePeriodSeconds: 60 - tolerations: [] + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" topologySpreadConstraints: [] vmServiceScrape: annotations: {} @@ -259,7 +285,16 @@ helm: - port: http useServiceMonitor: false vlstorage: - affinity: {} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst annotations: {} containerWorkingDir: "" emptyDir: {} @@ -351,7 +386,11 @@ helm: targetPort: http type: ClusterIP terminationGracePeriodSeconds: 60 - tolerations: [] + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" topologySpreadConstraints: [] vmServiceScrape: annotations: {} diff --git a/fleet/victoria-logs-cluster/fleet.yaml b/fleet/victoria-logs-cluster/fleet.yaml index b738f6f..d0c57ce 100644 --- a/fleet/victoria-logs-cluster/fleet.yaml +++ b/fleet/victoria-logs-cluster/fleet.yaml @@ -69,7 +69,18 @@ helm: type: internal_metrics k8s: type: kubernetes_logs - # Exclude system namespaces + # Default is 60s: too short for this cluster's high-churn + # simulation workloads (nomos-testnet, nimlibp2p, zerotesting - + # pods created/destroyed constantly). If a pod is deleted before + # Vector gets to its backlogged log lines, metadata annotation + # fails; under sustained churn this compounds into a growing + # multi-hour processing lag (observed: 4 of 6 nodes stuck + # hours behind, continuously emitting annotation_failed errors - + # vectordotdev/vector#10122, #12909, #13467, #21793). Longer + # cache retention gives Vector room to catch up without losing + # the metadata race. + delay_deletion_ms: 300000 + # Exclude system namespaces extra_field_selector: "metadata.namespace!=kube-system,metadata.namespace!=kyverno,metadata.namespace!=homepage,metadata.namespace!=authentik,metadata.namespace!=vaclab-system,metadata.namespace!=cert-manager,metadata.namespace!=cattle-capi-system,metadata.namespace!=cattle-fleet-clusters-system,metadata.namespace!=cattle-fleet-local-system,metadata.namespace!=cattle-fleet-system,metadata.namespace!=cattle-global-data,metadata.namespace!=cattle-impersonation-system,metadata.namespace!=cattle-local-user-passwords,metadata.namespace!=cattle-system,metadata.namespace!=cattle-turtles-system,metadata.namespace!=cattle-ui-plugin-system,metadata.namespace!=cilium-secrets,metadata.namespace!=fleet-default,metadata.namespace!=fleet-local,metadata.namespace!=kube-node-lease,metadata.namespace!=kube-public,metadata.namespace!=local,metadata.namespace!=longhorn-system,metadata.namespace!=policy-reporter,metadata.namespace!=victorialogs,metadata.namespace!=vmetrics,metadata.namespace!=grafana-tempo,metadata.namespace!=grafana-tempo-external" transforms: parser: @@ -100,7 +111,16 @@ helm: service: enabled: false vlinsert: - affinity: {} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst annotations: {} containerWorkingDir: "" enabled: true @@ -216,7 +236,16 @@ helm: - port: http useServiceMonitor: false vlselect: - affinity: {} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst annotations: {} containerWorkingDir: "" enabled: true diff --git a/fleet/victoria-metrics-stack-ext/fleet.yaml b/fleet/victoria-metrics-stack-ext/fleet.yaml index 16ae125..df6fd2c 100644 --- a/fleet/victoria-metrics-stack-ext/fleet.yaml +++ b/fleet/victoria-metrics-stack-ext/fleet.yaml @@ -78,6 +78,22 @@ helm: retentionPeriod: "6" vminsert: enabled: true + priorityClassName: monitoring-critical + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst extraArgs: # Enable OpenTelemetry metrics ingestion with Prometheus-compatible naming # OTLP Protobuf: http://vminsert-vmks-victoria-metrics-k8s-stack.vmetrics:8480/insert/0/opentelemetry/v1/metrics @@ -95,6 +111,22 @@ helm: memory: 4Gi vmselect: enabled: true + priorityClassName: monitoring-critical + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst extraArgs: {} port: "8481" replicaCount: 2 @@ -107,6 +139,19 @@ helm: memory: "8Gi" vmstorage: enabled: true + priorityClassName: monitoring-critical + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: vmstorage + topologyKey: kubernetes.io/hostname replicaCount: 2 resources: limits: diff --git a/fleet/victoria-metrics-stack/fleet.yaml b/fleet/victoria-metrics-stack/fleet.yaml index 204deed..bbd2a95 100644 --- a/fleet/victoria-metrics-stack/fleet.yaml +++ b/fleet/victoria-metrics-stack/fleet.yaml @@ -683,6 +683,16 @@ helm: operator: "Equal" value: "monitoring" effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst inlineScrapeConfig: |- - job_name: 'mix-testnet-0.2' kubernetes_sd_configs: @@ -1036,8 +1046,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" @@ -1131,7 +1143,17 @@ helm: operator: "Equal" value: "monitoring" effect: "PreferNoSchedule" - + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + # -- [Scrape configuration](https://docs.victoriametrics.com/operator/api/#vmservicescrapespec) for Kube State Metrics vmScrape: enabled: true From e986961263758656874d133092fe4f0bdbd15407 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Sun, 13 Sep 2026 16:10:47 +0000 Subject: [PATCH 2/6] Fix Helm values-schema mismatches in metal-01 affinity migration --- fleet/kyverno-reporter-ui/fleet.yaml | 18 +++++++ fleet/kyverno/fleet.yaml | 20 +++----- fleet/victoria-metrics-stack/fleet.yaml | 65 +++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 12 deletions(-) diff --git a/fleet/kyverno-reporter-ui/fleet.yaml b/fleet/kyverno-reporter-ui/fleet.yaml index 853de24..c332fbe 100644 --- a/fleet/kyverno-reporter-ui/fleet.yaml +++ b/fleet/kyverno-reporter-ui/fleet.yaml @@ -9,6 +9,24 @@ helm: releaseName: policy-reporter takeOwnership: true values: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" ui: enabled: true affinity: diff --git a/fleet/kyverno/fleet.yaml b/fleet/kyverno/fleet.yaml index 3a9f075..4ce2b38 100644 --- a/fleet/kyverno/fleet.yaml +++ b/fleet/kyverno/fleet.yaml @@ -9,9 +9,8 @@ helm: values: admissionController: replicas: 2 - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: matchExpressions: @@ -29,9 +28,8 @@ helm: effect: "NoSchedule" backgroundController: replicas: 2 - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: matchExpressions: @@ -49,9 +47,8 @@ helm: effect: "NoSchedule" cleanupController: replicas: 2 - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: matchExpressions: @@ -69,9 +66,8 @@ helm: effect: "NoSchedule" reportsController: replicas: 2 - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: matchExpressions: diff --git a/fleet/victoria-metrics-stack/fleet.yaml b/fleet/victoria-metrics-stack/fleet.yaml index bbd2a95..d887b91 100644 --- a/fleet/victoria-metrics-stack/fleet.yaml +++ b/fleet/victoria-metrics-stack/fleet.yaml @@ -46,6 +46,21 @@ helm: limits: cpu: "2" memory: 2Gi + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" defaultDashboards: # -- Enable custom dashboards installation @@ -339,6 +354,16 @@ helm: operator: "Equal" value: "monitoring" effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst extraArgs: # Enable OpenTelemetry metrics ingestion with Prometheus-compatible naming # OTLP Protobuf: http://vminsert-vmks-victoria-metrics-k8s-stack.vmetrics:8480/insert/0/opentelemetry/v1/metrics @@ -362,6 +387,16 @@ helm: operator: "Equal" value: "monitoring" effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst extraArgs: {} port: "8481" replicaCount: 2 @@ -428,6 +463,21 @@ helm: limits: cpu: "1" memory: 1Gi + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" # -- (string) If this one defined, it will be used for alertmanager configuration and config parameter will be ignored configSecret: "" @@ -528,6 +578,21 @@ helm: limits: cpu: "2" memory: 2Gi + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" # External labels to add to all generated recording rules and alerts externalLabels: {} From 2e7813c45b585951ffd225c3c6e115f2a51a0f29 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Thu, 17 Sep 2026 15:04:45 +0000 Subject: [PATCH 3/6] reduce system workload cpu request --- .fleetignore | 5 +++++ fleet/authentik/fleet.yaml | 4 ++-- fleet/dst-dashboard/backend/backend.yaml | 2 +- fleet/grafana-tempo-external/fleet.yaml | 2 +- fleet/grafana-tempo/fleet.yaml | 2 +- fleet/homepage/deployment.yaml | 2 +- fleet/longhorn/fleet.yaml | 4 ++++ fleet/otlp-collector-external/fleet.yaml | 2 +- fleet/vaclab-ntp/server/ntp-server.yaml | 2 +- fleet/victoria-logs-cluster-external/fleet.yaml | 4 ++-- fleet/victoria-logs-cluster/fleet.yaml | 8 ++++---- fleet/victoria-metrics-stack-ext/fleet.yaml | 8 ++++---- fleet/victoria-metrics-stack/fleet.yaml | 10 +++++----- 13 files changed, 32 insertions(+), 23 deletions(-) create mode 100644 .fleetignore diff --git a/.fleetignore b/.fleetignore new file mode 100644 index 0000000..08be35f --- /dev/null +++ b/.fleetignore @@ -0,0 +1,5 @@ +# Bundles listed here are kept in the repo for reference but are not +# scanned/managed by Fleet, so their live resources get uninstalled. +# Remove a line here to re-enable that bundle. +fleet/bandwidth-scheduler/ +fleet/bandwidth-operator/ diff --git a/fleet/authentik/fleet.yaml b/fleet/authentik/fleet.yaml index 13fad85..41d5f5c 100644 --- a/fleet/authentik/fleet.yaml +++ b/fleet/authentik/fleet.yaml @@ -186,7 +186,7 @@ helm: effect: "NoSchedule" resources: requests: - cpu: "4" + cpu: "2" memory: "4Gi" limits: cpu: "8" @@ -234,7 +234,7 @@ helm: effect: "NoSchedule" resources: requests: - cpu: "2" + cpu: "1" memory: "2Gi" limits: cpu: "4" diff --git a/fleet/dst-dashboard/backend/backend.yaml b/fleet/dst-dashboard/backend/backend.yaml index badc72e..9d6fe2c 100644 --- a/fleet/dst-dashboard/backend/backend.yaml +++ b/fleet/dst-dashboard/backend/backend.yaml @@ -57,7 +57,7 @@ spec: resources: requests: memory: "4Gi" - cpu: "4" + cpu: "1" limits: memory: "16Gi" cpu: "8" diff --git a/fleet/grafana-tempo-external/fleet.yaml b/fleet/grafana-tempo-external/fleet.yaml index c943d19..db3c522 100644 --- a/fleet/grafana-tempo-external/fleet.yaml +++ b/fleet/grafana-tempo-external/fleet.yaml @@ -14,7 +14,7 @@ helm: retention: 4380h # 6 months resources: requests: - cpu: "2" + cpu: "500m" memory: 4Gi limits: cpu: "2" diff --git a/fleet/grafana-tempo/fleet.yaml b/fleet/grafana-tempo/fleet.yaml index 2c47719..8e33697 100644 --- a/fleet/grafana-tempo/fleet.yaml +++ b/fleet/grafana-tempo/fleet.yaml @@ -15,7 +15,7 @@ helm: retention: 4380h # 6 months resources: requests: - cpu: "2" + cpu: "500m" memory: 4Gi limits: cpu: "2" diff --git a/fleet/homepage/deployment.yaml b/fleet/homepage/deployment.yaml index a46b8af..bb57efb 100644 --- a/fleet/homepage/deployment.yaml +++ b/fleet/homepage/deployment.yaml @@ -63,7 +63,7 @@ spec: - ALL resources: requests: - cpu: "4" + cpu: "250m" memory: "2Gi" limits: cpu: "8" diff --git a/fleet/longhorn/fleet.yaml b/fleet/longhorn/fleet.yaml index 3c1d0db..cabb09a 100644 --- a/fleet/longhorn/fleet.yaml +++ b/fleet/longhorn/fleet.yaml @@ -11,6 +11,10 @@ helm: defaultSettings: createDefaultDiskLabeledNodes: true defaultDataPath: /var/lib/longhorn + # Default is 12% of node CPU reserved for instance-manager regardless of + # actual usage (observed ~0.3% used on a 62-core node); lower reservation + # frees real scheduling headroom without affecting the runtime CPU limit. + guaranteedInstanceManagerCPU: "4" longhornManager: tolerations: - key: "dedicated" diff --git a/fleet/otlp-collector-external/fleet.yaml b/fleet/otlp-collector-external/fleet.yaml index 7efac70..8a76c60 100644 --- a/fleet/otlp-collector-external/fleet.yaml +++ b/fleet/otlp-collector-external/fleet.yaml @@ -44,7 +44,7 @@ helm: resources: requests: - cpu: "2" + cpu: "300m" memory: 4Gi limits: cpu: "4" diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index a18cadc..3c563ef 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -38,7 +38,7 @@ spec: resources: requests: memory: "1Gi" - cpu: "2" + cpu: "200m" limits: memory: "1Gi" cpu: "2" diff --git a/fleet/victoria-logs-cluster-external/fleet.yaml b/fleet/victoria-logs-cluster-external/fleet.yaml index 43a76f3..859a845 100644 --- a/fleet/victoria-logs-cluster-external/fleet.yaml +++ b/fleet/victoria-logs-cluster-external/fleet.yaml @@ -110,7 +110,7 @@ helm: cpu: "8" # set to 8 in prod memory: "8Gi" # set to 8Gi in prod requests: - cpu: "2" # set to 2 in prod + cpu: "1" memory: 2Gi securityContext: enabled: true @@ -363,7 +363,7 @@ helm: replicaCount: 1 resources: requests: - cpu: "2" + cpu: "1" memory: "1Gi" retentionDiskSpaceUsage: "" retentionPeriod: 6 diff --git a/fleet/victoria-logs-cluster/fleet.yaml b/fleet/victoria-logs-cluster/fleet.yaml index d0c57ce..ec2465f 100644 --- a/fleet/victoria-logs-cluster/fleet.yaml +++ b/fleet/victoria-logs-cluster/fleet.yaml @@ -97,7 +97,7 @@ helm: - vector-vl-config resources: requests: - cpu: "2" + cpu: "1" memory: 2Gi limits: cpu: "4" @@ -194,7 +194,7 @@ helm: cpu: "8" # set to 8 in prod memory: "8Gi" # set to 8Gi in prod requests: - cpu: "2" # set to 2 in prod + cpu: "1" memory: 2Gi securityContext: enabled: true @@ -329,7 +329,7 @@ helm: replicaCount: 2 resources: requests: - cpu: "2" + cpu: "1" memory: 4Gi limits: cpu: "4" @@ -449,7 +449,7 @@ helm: replicaCount: 3 resources: requests: - cpu: "4" + cpu: "1" memory: "8Gi" limits: cpu: "8" diff --git a/fleet/victoria-metrics-stack-ext/fleet.yaml b/fleet/victoria-metrics-stack-ext/fleet.yaml index df6fd2c..666a3c6 100644 --- a/fleet/victoria-metrics-stack-ext/fleet.yaml +++ b/fleet/victoria-metrics-stack-ext/fleet.yaml @@ -107,7 +107,7 @@ helm: cpu: "8" memory: 8Gi requests: - cpu: "4" + cpu: "1" memory: 4Gi vmselect: enabled: true @@ -132,10 +132,10 @@ helm: replicaCount: 2 resources: limits: - cpu: "8" + cpu: "8" memory: "16Gi" requests: - cpu: "4" + cpu: "1" memory: "8Gi" vmstorage: enabled: true @@ -158,7 +158,7 @@ helm: cpu: "8" # set to 8 in production memory: "8Gi" requests: - cpu: "2" + cpu: "1" memory: "2Gi" storage: volumeClaimTemplate: diff --git a/fleet/victoria-metrics-stack/fleet.yaml b/fleet/victoria-metrics-stack/fleet.yaml index d887b91..392b8cd 100644 --- a/fleet/victoria-metrics-stack/fleet.yaml +++ b/fleet/victoria-metrics-stack/fleet.yaml @@ -377,7 +377,7 @@ helm: cpu: "8" memory: 8Gi requests: - cpu: "4" + cpu: "1" memory: 4Gi vmselect: enabled: true @@ -405,7 +405,7 @@ helm: cpu: "8" memory: "16Gi" requests: - cpu: "4" + cpu: "1" memory: "8Gi" vmstorage: enabled: true @@ -428,7 +428,7 @@ helm: cpu: "8" # set to 8 in production memory: "8Gi" requests: - cpu: "2" + cpu: "1" memory: "6Gi" storage: volumeClaimTemplate: @@ -849,7 +849,7 @@ helm: promscrape.dropOriginalLabels: "true" resources: requests: - cpu: "4" + cpu: "2" memory: 4Gi limits: cpu: "16" @@ -1123,7 +1123,7 @@ helm: resources: requests: - cpu: "8" + cpu: "1" memory: 8Gi limits: cpu: "16" From 9b8d63b1b64c346adcf6c6c7f119323f36cca7cb Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Thu, 17 Sep 2026 21:52:54 +0000 Subject: [PATCH 4/6] Fix .fleetignore location so bandwidth-scheduler/operator get disabled --- .fleetignore => fleet/.fleetignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename .fleetignore => fleet/.fleetignore (77%) diff --git a/.fleetignore b/fleet/.fleetignore similarity index 77% rename from .fleetignore rename to fleet/.fleetignore index 08be35f..b4d6fea 100644 --- a/.fleetignore +++ b/fleet/.fleetignore @@ -1,5 +1,5 @@ # Bundles listed here are kept in the repo for reference but are not # scanned/managed by Fleet, so their live resources get uninstalled. # Remove a line here to re-enable that bundle. -fleet/bandwidth-scheduler/ -fleet/bandwidth-operator/ +bandwidth-scheduler/ +bandwidth-operator/ From 210ede27688accf49b1d46f45ab8d15ed57a3390 Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Thu, 17 Sep 2026 21:58:01 +0000 Subject: [PATCH 5/6] Disable bandwidth-scheduler/operator bundles --- fleet/bandwidth-operator/{fleet.yaml => fleet.yaml.disabled} | 0 fleet/bandwidth-scheduler/{fleet.yaml => fleet.yaml.disabled} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename fleet/bandwidth-operator/{fleet.yaml => fleet.yaml.disabled} (100%) rename fleet/bandwidth-scheduler/{fleet.yaml => fleet.yaml.disabled} (100%) diff --git a/fleet/bandwidth-operator/fleet.yaml b/fleet/bandwidth-operator/fleet.yaml.disabled similarity index 100% rename from fleet/bandwidth-operator/fleet.yaml rename to fleet/bandwidth-operator/fleet.yaml.disabled diff --git a/fleet/bandwidth-scheduler/fleet.yaml b/fleet/bandwidth-scheduler/fleet.yaml.disabled similarity index 100% rename from fleet/bandwidth-scheduler/fleet.yaml rename to fleet/bandwidth-scheduler/fleet.yaml.disabled From 4adce46b3299544e425629d69871912c87c515ee Mon Sep 17 00:00:00 2001 From: mamoutou-diarra Date: Wed, 23 Sep 2026 15:14:08 +0000 Subject: [PATCH 6/6] set authentik oupost token sync periodicity to 10m instead of 2m --- ansible/playbooks/tasks/cilium.yaml | 72 +++++++++++++++++++ ansible/playbooks/tasks/rancher_fleet.yaml | 30 +++++++- fleet/.fleetignore | 5 -- .../authentik-token-sync/token-sync-job.yaml | 2 +- 4 files changed, 101 insertions(+), 8 deletions(-) delete mode 100644 fleet/.fleetignore diff --git a/ansible/playbooks/tasks/cilium.yaml b/ansible/playbooks/tasks/cilium.yaml index cf01361..c05f310 100644 --- a/ansible/playbooks/tasks/cilium.yaml +++ b/ansible/playbooks/tasks/cilium.yaml @@ -114,8 +114,45 @@ enabled: true serviceMonitor: enabled: false + # Keep the chart's default required podAffinity to a local cilium + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" ui: enabled: true + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" backend: resources: limits: @@ -156,6 +193,41 @@ requests: cpu: "2" memory: "512Mi" + # Keep the chart's default required podAntiAffinity (spreads the 3 + # replicas across 3 different nodes for HA -- only one is ever the + # elected leader) and add a soft preference for metal-01 alongside + # it, so at most one of the 3 replicas can land there. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + io.cilium/app: operator + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + # Preserve the chart's default tolerations (control-plane, master, + # not-ready, uninitialized) alongside our own. + tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: Exists + - key: "node-role.kubernetes.io/master" + operator: Exists + - key: "node.kubernetes.io/not-ready" + operator: Exists + - key: "node.cloudprovider.kubernetes.io/uninitialized" + operator: Exists + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" # Cilium agent resources resources: diff --git a/ansible/playbooks/tasks/rancher_fleet.yaml b/ansible/playbooks/tasks/rancher_fleet.yaml index c27c2b5..965c27f 100644 --- a/ansible/playbooks/tasks/rancher_fleet.yaml +++ b/ansible/playbooks/tasks/rancher_fleet.yaml @@ -17,6 +17,27 @@ values: crds: enabled: true + tolerations: &cert_manager_tolerations + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + affinity: &cert_manager_affinity + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + webhook: + tolerations: *cert_manager_tolerations + affinity: *cert_manager_affinity + cainjector: + tolerations: *cert_manager_tolerations + affinity: *cert_manager_affinity - name: Wait for cert-manager to be ready command: kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=cert-manager -n cert-manager --timeout=300s @@ -152,9 +173,14 @@ operator: "Equal" value: "monitoring" effect: "PreferNoSchedule" + # Rancher's chart only supports hard (requiredDuringScheduling) node + # affinity -- there is no soft/preferred option in this chart + # anywhere else until this is reverted. extraNodeSelectorTerms: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst ingress: enabled: true tls: diff --git a/fleet/.fleetignore b/fleet/.fleetignore deleted file mode 100644 index b4d6fea..0000000 --- a/fleet/.fleetignore +++ /dev/null @@ -1,5 +0,0 @@ -# Bundles listed here are kept in the repo for reference but are not -# scanned/managed by Fleet, so their live resources get uninstalled. -# Remove a line here to re-enable that bundle. -bandwidth-scheduler/ -bandwidth-operator/ diff --git a/fleet/authentik-token-sync/token-sync-job.yaml b/fleet/authentik-token-sync/token-sync-job.yaml index ffda88c..8602f6c 100644 --- a/fleet/authentik-token-sync/token-sync-job.yaml +++ b/fleet/authentik-token-sync/token-sync-job.yaml @@ -4,7 +4,7 @@ metadata: name: authentik-outpost-token-sync namespace: authentik spec: - schedule: "*/2 * * * *" # Every 2 minutes + schedule: "*/10 * * * *" # Every 10 minutes (token rotates ~hourly) successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 1 jobTemplate: