diff --git a/ansible/playbooks/tasks/cilium.yaml b/ansible/playbooks/tasks/cilium.yaml index cf01361..c05f310 100644 --- a/ansible/playbooks/tasks/cilium.yaml +++ b/ansible/playbooks/tasks/cilium.yaml @@ -114,8 +114,45 @@ enabled: true serviceMonitor: enabled: false + # Keep the chart's default required podAffinity to a local cilium + affinity: + podAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + k8s-app: cilium + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" ui: enabled: true + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" backend: resources: limits: @@ -156,6 +193,41 @@ requests: cpu: "2" memory: "512Mi" + # Keep the chart's default required podAntiAffinity (spreads the 3 + # replicas across 3 different nodes for HA -- only one is ever the + # elected leader) and add a soft preference for metal-01 alongside + # it, so at most one of the 3 replicas can land there. + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + io.cilium/app: operator + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + # Preserve the chart's default tolerations (control-plane, master, + # not-ready, uninitialized) alongside our own. + tolerations: + - key: "node-role.kubernetes.io/control-plane" + operator: Exists + - key: "node-role.kubernetes.io/master" + operator: Exists + - key: "node.kubernetes.io/not-ready" + operator: Exists + - key: "node.cloudprovider.kubernetes.io/uninitialized" + operator: Exists + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" # Cilium agent resources resources: diff --git a/ansible/playbooks/tasks/rancher_fleet.yaml b/ansible/playbooks/tasks/rancher_fleet.yaml index c27c2b5..965c27f 100644 --- a/ansible/playbooks/tasks/rancher_fleet.yaml +++ b/ansible/playbooks/tasks/rancher_fleet.yaml @@ -17,6 +17,27 @@ values: crds: enabled: true + tolerations: &cert_manager_tolerations + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + affinity: &cert_manager_affinity + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + webhook: + tolerations: *cert_manager_tolerations + affinity: *cert_manager_affinity + cainjector: + tolerations: *cert_manager_tolerations + affinity: *cert_manager_affinity - name: Wait for cert-manager to be ready command: kubectl wait --for=condition=ready pod -l app.kubernetes.io/instance=cert-manager -n cert-manager --timeout=300s @@ -152,9 +173,14 @@ operator: "Equal" value: "monitoring" effect: "PreferNoSchedule" + # Rancher's chart only supports hard (requiredDuringScheduling) node + # affinity -- there is no soft/preferred option in this chart + # anywhere else until this is reverted. extraNodeSelectorTerms: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst ingress: enabled: true tls: diff --git a/fleet/authentik-config/outpost.yaml b/fleet/authentik-config/outpost.yaml index 459c24a..f5b7c32 100644 --- a/fleet/authentik-config/outpost.yaml +++ b/fleet/authentik-config/outpost.yaml @@ -19,8 +19,10 @@ spec: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/worker - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" diff --git a/fleet/authentik-token-sync/token-sync-initial-job.yaml b/fleet/authentik-token-sync/token-sync-initial-job.yaml index b9caf5e..097d5dc 100644 --- a/fleet/authentik-token-sync/token-sync-initial-job.yaml +++ b/fleet/authentik-token-sync/token-sync-initial-job.yaml @@ -13,11 +13,21 @@ spec: ttlSecondsAfterFinished: 3600 template: spec: - #tolerations: - # - key: "dedicated" - # operator: "Equal" - # value: "monitoring" - # effect: "PreferNoSchedule" + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst serviceAccountName: authentik-outpost-token-sync restartPolicy: OnFailure initContainers: diff --git a/fleet/authentik-token-sync/token-sync-job.yaml b/fleet/authentik-token-sync/token-sync-job.yaml index ccc4308..8602f6c 100644 --- a/fleet/authentik-token-sync/token-sync-job.yaml +++ b/fleet/authentik-token-sync/token-sync-job.yaml @@ -4,18 +4,28 @@ metadata: name: authentik-outpost-token-sync namespace: authentik spec: - schedule: "*/2 * * * *" # Every 2 minutes + schedule: "*/10 * * * *" # Every 10 minutes (token rotates ~hourly) successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 1 jobTemplate: spec: template: spec: - #tolerations: - # - key: "dedicated" - # operator: "Equal" - # value: "monitoring" - # effect: "PreferNoSchedule" + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst serviceAccountName: authentik-outpost-token-sync restartPolicy: OnFailure initContainers: diff --git a/fleet/authentik/fleet.yaml b/fleet/authentik/fleet.yaml index f5bf58a..41d5f5c 100644 --- a/fleet/authentik/fleet.yaml +++ b/fleet/authentik/fleet.yaml @@ -55,8 +55,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" @@ -170,8 +172,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" @@ -182,7 +186,7 @@ helm: effect: "NoSchedule" resources: requests: - cpu: "4" + cpu: "2" memory: "4Gi" limits: cpu: "8" @@ -216,8 +220,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" @@ -228,7 +234,7 @@ helm: effect: "NoSchedule" resources: requests: - cpu: "2" + cpu: "1" memory: "2Gi" limits: cpu: "4" diff --git a/fleet/bandwidth-operator/fleet.yaml b/fleet/bandwidth-operator/fleet.yaml.disabled similarity index 90% rename from fleet/bandwidth-operator/fleet.yaml rename to fleet/bandwidth-operator/fleet.yaml.disabled index c474d8f..e4a287e 100644 --- a/fleet/bandwidth-operator/fleet.yaml +++ b/fleet/bandwidth-operator/fleet.yaml.disabled @@ -46,8 +46,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" diff --git a/fleet/bandwidth-scheduler/fleet.yaml b/fleet/bandwidth-scheduler/fleet.yaml.disabled similarity index 72% rename from fleet/bandwidth-scheduler/fleet.yaml rename to fleet/bandwidth-scheduler/fleet.yaml.disabled index 4e8115b..f4e55d0 100644 --- a/fleet/bandwidth-scheduler/fleet.yaml +++ b/fleet/bandwidth-scheduler/fleet.yaml.disabled @@ -26,8 +26,21 @@ helm: cpu: 1 memory: 500Mi nodeSelector: {} - affinity: {} - tolerations: [] + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" controller: name: vaclab-scheduler-controller @@ -49,8 +62,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" diff --git a/fleet/dst-dashboard/backend/backend.yaml b/fleet/dst-dashboard/backend/backend.yaml index badc72e..9d6fe2c 100644 --- a/fleet/dst-dashboard/backend/backend.yaml +++ b/fleet/dst-dashboard/backend/backend.yaml @@ -57,7 +57,7 @@ spec: resources: requests: memory: "4Gi" - cpu: "4" + cpu: "1" limits: memory: "16Gi" cpu: "8" diff --git a/fleet/grafana-tempo-external/fleet.yaml b/fleet/grafana-tempo-external/fleet.yaml index c943d19..db3c522 100644 --- a/fleet/grafana-tempo-external/fleet.yaml +++ b/fleet/grafana-tempo-external/fleet.yaml @@ -14,7 +14,7 @@ helm: retention: 4380h # 6 months resources: requests: - cpu: "2" + cpu: "500m" memory: 4Gi limits: cpu: "2" diff --git a/fleet/grafana-tempo/fleet.yaml b/fleet/grafana-tempo/fleet.yaml index 2c47719..8e33697 100644 --- a/fleet/grafana-tempo/fleet.yaml +++ b/fleet/grafana-tempo/fleet.yaml @@ -15,7 +15,7 @@ helm: retention: 4380h # 6 months resources: requests: - cpu: "2" + cpu: "500m" memory: 4Gi limits: cpu: "2" diff --git a/fleet/homepage/deployment.yaml b/fleet/homepage/deployment.yaml index 64bb9e2..bb57efb 100644 --- a/fleet/homepage/deployment.yaml +++ b/fleet/homepage/deployment.yaml @@ -28,8 +28,10 @@ spec: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" @@ -61,7 +63,7 @@ spec: - ALL resources: requests: - cpu: "4" + cpu: "250m" memory: "2Gi" limits: cpu: "8" diff --git a/fleet/kyverno-reporter-ui/fleet.yaml b/fleet/kyverno-reporter-ui/fleet.yaml index 853de24..c332fbe 100644 --- a/fleet/kyverno-reporter-ui/fleet.yaml +++ b/fleet/kyverno-reporter-ui/fleet.yaml @@ -9,6 +9,24 @@ helm: releaseName: policy-reporter takeOwnership: true values: + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" ui: enabled: true affinity: diff --git a/fleet/kyverno/fleet.yaml b/fleet/kyverno/fleet.yaml index 3a9f075..4ce2b38 100644 --- a/fleet/kyverno/fleet.yaml +++ b/fleet/kyverno/fleet.yaml @@ -9,9 +9,8 @@ helm: values: admissionController: replicas: 2 - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: matchExpressions: @@ -29,9 +28,8 @@ helm: effect: "NoSchedule" backgroundController: replicas: 2 - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: matchExpressions: @@ -49,9 +47,8 @@ helm: effect: "NoSchedule" cleanupController: replicas: 2 - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: matchExpressions: @@ -69,9 +66,8 @@ helm: effect: "NoSchedule" reportsController: replicas: 2 - affinity: - nodeAffinity: - preferredDuringSchedulingIgnoredDuringExecution: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: - weight: 100 preference: matchExpressions: diff --git a/fleet/longhorn/fleet.yaml b/fleet/longhorn/fleet.yaml index ca83a0f..cabb09a 100644 --- a/fleet/longhorn/fleet.yaml +++ b/fleet/longhorn/fleet.yaml @@ -11,6 +11,10 @@ helm: defaultSettings: createDefaultDiskLabeledNodes: true defaultDataPath: /var/lib/longhorn + # Default is 12% of node CPU reserved for instance-manager regardless of + # actual usage (observed ~0.3% used on a 62-core node); lower reservation + # frees real scheduling headroom without affecting the runtime CPU limit. + guaranteedInstanceManagerCPU: "4" longhornManager: tolerations: - key: "dedicated" @@ -30,8 +34,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" diff --git a/fleet/otlp-collector-external/fleet.yaml b/fleet/otlp-collector-external/fleet.yaml index 7efac70..8a76c60 100644 --- a/fleet/otlp-collector-external/fleet.yaml +++ b/fleet/otlp-collector-external/fleet.yaml @@ -44,7 +44,7 @@ helm: resources: requests: - cpu: "2" + cpu: "300m" memory: 4Gi limits: cpu: "4" diff --git a/fleet/vaclab-ntp/server/ntp-server.yaml b/fleet/vaclab-ntp/server/ntp-server.yaml index a18cadc..3c563ef 100644 --- a/fleet/vaclab-ntp/server/ntp-server.yaml +++ b/fleet/vaclab-ntp/server/ntp-server.yaml @@ -38,7 +38,7 @@ spec: resources: requests: memory: "1Gi" - cpu: "2" + cpu: "200m" limits: memory: "1Gi" cpu: "2" diff --git a/fleet/victoria-logs-cluster-external/fleet.yaml b/fleet/victoria-logs-cluster-external/fleet.yaml index 3c6e772..859a845 100644 --- a/fleet/victoria-logs-cluster-external/fleet.yaml +++ b/fleet/victoria-logs-cluster-external/fleet.yaml @@ -27,7 +27,16 @@ helm: vector: enabled: false vlinsert: - affinity: {} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst annotations: {} containerWorkingDir: "" enabled: true @@ -101,7 +110,7 @@ helm: cpu: "8" # set to 8 in prod memory: "8Gi" # set to 8Gi in prod requests: - cpu: "2" # set to 2 in prod + cpu: "1" memory: 2Gi securityContext: enabled: true @@ -127,7 +136,11 @@ helm: strategy: {} suppressStorageFQDNsRender: false terminationGracePeriodSeconds: 30 - tolerations: [] + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" topologySpreadConstraints: [] vmServiceScrape: annotations: {} @@ -139,7 +152,16 @@ helm: - port: http useServiceMonitor: false vlselect: - affinity: {} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst annotations: {} containerWorkingDir: "" enabled: true @@ -247,7 +269,11 @@ helm: strategy: {} suppressStorageFQDNsRender: false terminationGracePeriodSeconds: 60 - tolerations: [] + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" topologySpreadConstraints: [] vmServiceScrape: annotations: {} @@ -259,7 +285,16 @@ helm: - port: http useServiceMonitor: false vlstorage: - affinity: {} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst annotations: {} containerWorkingDir: "" emptyDir: {} @@ -328,7 +363,7 @@ helm: replicaCount: 1 resources: requests: - cpu: "2" + cpu: "1" memory: "1Gi" retentionDiskSpaceUsage: "" retentionPeriod: 6 @@ -351,7 +386,11 @@ helm: targetPort: http type: ClusterIP terminationGracePeriodSeconds: 60 - tolerations: [] + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" topologySpreadConstraints: [] vmServiceScrape: annotations: {} diff --git a/fleet/victoria-logs-cluster/fleet.yaml b/fleet/victoria-logs-cluster/fleet.yaml index b738f6f..ec2465f 100644 --- a/fleet/victoria-logs-cluster/fleet.yaml +++ b/fleet/victoria-logs-cluster/fleet.yaml @@ -69,7 +69,18 @@ helm: type: internal_metrics k8s: type: kubernetes_logs - # Exclude system namespaces + # Default is 60s: too short for this cluster's high-churn + # simulation workloads (nomos-testnet, nimlibp2p, zerotesting - + # pods created/destroyed constantly). If a pod is deleted before + # Vector gets to its backlogged log lines, metadata annotation + # fails; under sustained churn this compounds into a growing + # multi-hour processing lag (observed: 4 of 6 nodes stuck + # hours behind, continuously emitting annotation_failed errors - + # vectordotdev/vector#10122, #12909, #13467, #21793). Longer + # cache retention gives Vector room to catch up without losing + # the metadata race. + delay_deletion_ms: 300000 + # Exclude system namespaces extra_field_selector: "metadata.namespace!=kube-system,metadata.namespace!=kyverno,metadata.namespace!=homepage,metadata.namespace!=authentik,metadata.namespace!=vaclab-system,metadata.namespace!=cert-manager,metadata.namespace!=cattle-capi-system,metadata.namespace!=cattle-fleet-clusters-system,metadata.namespace!=cattle-fleet-local-system,metadata.namespace!=cattle-fleet-system,metadata.namespace!=cattle-global-data,metadata.namespace!=cattle-impersonation-system,metadata.namespace!=cattle-local-user-passwords,metadata.namespace!=cattle-system,metadata.namespace!=cattle-turtles-system,metadata.namespace!=cattle-ui-plugin-system,metadata.namespace!=cilium-secrets,metadata.namespace!=fleet-default,metadata.namespace!=fleet-local,metadata.namespace!=kube-node-lease,metadata.namespace!=kube-public,metadata.namespace!=local,metadata.namespace!=longhorn-system,metadata.namespace!=policy-reporter,metadata.namespace!=victorialogs,metadata.namespace!=vmetrics,metadata.namespace!=grafana-tempo,metadata.namespace!=grafana-tempo-external" transforms: parser: @@ -86,7 +97,7 @@ helm: - vector-vl-config resources: requests: - cpu: "2" + cpu: "1" memory: 2Gi limits: cpu: "4" @@ -100,7 +111,16 @@ helm: service: enabled: false vlinsert: - affinity: {} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst annotations: {} containerWorkingDir: "" enabled: true @@ -174,7 +194,7 @@ helm: cpu: "8" # set to 8 in prod memory: "8Gi" # set to 8Gi in prod requests: - cpu: "2" # set to 2 in prod + cpu: "1" memory: 2Gi securityContext: enabled: true @@ -216,7 +236,16 @@ helm: - port: http useServiceMonitor: false vlselect: - affinity: {} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst annotations: {} containerWorkingDir: "" enabled: true @@ -300,7 +329,7 @@ helm: replicaCount: 2 resources: requests: - cpu: "2" + cpu: "1" memory: 4Gi limits: cpu: "4" @@ -420,7 +449,7 @@ helm: replicaCount: 3 resources: requests: - cpu: "4" + cpu: "1" memory: "8Gi" limits: cpu: "8" diff --git a/fleet/victoria-metrics-stack-ext/fleet.yaml b/fleet/victoria-metrics-stack-ext/fleet.yaml index 16ae125..666a3c6 100644 --- a/fleet/victoria-metrics-stack-ext/fleet.yaml +++ b/fleet/victoria-metrics-stack-ext/fleet.yaml @@ -78,6 +78,22 @@ helm: retentionPeriod: "6" vminsert: enabled: true + priorityClassName: monitoring-critical + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst extraArgs: # Enable OpenTelemetry metrics ingestion with Prometheus-compatible naming # OTLP Protobuf: http://vminsert-vmks-victoria-metrics-k8s-stack.vmetrics:8480/insert/0/opentelemetry/v1/metrics @@ -91,29 +107,58 @@ helm: cpu: "8" memory: 8Gi requests: - cpu: "4" + cpu: "1" memory: 4Gi vmselect: enabled: true + priorityClassName: monitoring-critical + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst extraArgs: {} port: "8481" replicaCount: 2 resources: limits: - cpu: "8" + cpu: "8" memory: "16Gi" requests: - cpu: "4" + cpu: "1" memory: "8Gi" vmstorage: enabled: true + priorityClassName: monitoring-critical + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: vmstorage + topologyKey: kubernetes.io/hostname replicaCount: 2 resources: limits: cpu: "8" # set to 8 in production memory: "8Gi" requests: - cpu: "2" + cpu: "1" memory: "2Gi" storage: volumeClaimTemplate: diff --git a/fleet/victoria-metrics-stack/fleet.yaml b/fleet/victoria-metrics-stack/fleet.yaml index 204deed..392b8cd 100644 --- a/fleet/victoria-metrics-stack/fleet.yaml +++ b/fleet/victoria-metrics-stack/fleet.yaml @@ -46,6 +46,21 @@ helm: limits: cpu: "2" memory: 2Gi + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" defaultDashboards: # -- Enable custom dashboards installation @@ -339,6 +354,16 @@ helm: operator: "Equal" value: "monitoring" effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst extraArgs: # Enable OpenTelemetry metrics ingestion with Prometheus-compatible naming # OTLP Protobuf: http://vminsert-vmks-victoria-metrics-k8s-stack.vmetrics:8480/insert/0/opentelemetry/v1/metrics @@ -352,7 +377,7 @@ helm: cpu: "8" memory: 8Gi requests: - cpu: "4" + cpu: "1" memory: 4Gi vmselect: enabled: true @@ -362,6 +387,16 @@ helm: operator: "Equal" value: "monitoring" effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst extraArgs: {} port: "8481" replicaCount: 2 @@ -370,7 +405,7 @@ helm: cpu: "8" memory: "16Gi" requests: - cpu: "4" + cpu: "1" memory: "8Gi" vmstorage: enabled: true @@ -393,7 +428,7 @@ helm: cpu: "8" # set to 8 in production memory: "8Gi" requests: - cpu: "2" + cpu: "1" memory: "6Gi" storage: volumeClaimTemplate: @@ -428,6 +463,21 @@ helm: limits: cpu: "1" memory: 1Gi + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" # -- (string) If this one defined, it will be used for alertmanager configuration and config parameter will be ignored configSecret: "" @@ -528,6 +578,21 @@ helm: limits: cpu: "2" memory: 2Gi + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + tolerations: + - key: "dedicated" + operator: "Equal" + value: "monitoring" + effect: "PreferNoSchedule" # External labels to add to all generated recording rules and alerts externalLabels: {} @@ -683,6 +748,16 @@ helm: operator: "Equal" value: "monitoring" effect: "PreferNoSchedule" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst inlineScrapeConfig: |- - job_name: 'mix-testnet-0.2' kubernetes_sd_configs: @@ -774,7 +849,7 @@ helm: promscrape.dropOriginalLabels: "true" resources: requests: - cpu: "4" + cpu: "2" memory: 4Gi limits: cpu: "16" @@ -1036,8 +1111,10 @@ helm: - weight: 100 preference: matchExpressions: - - key: node-role.kubernetes.io/control-plane - operator: Exists + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst tolerations: - key: "dedicated" operator: "Equal" @@ -1046,7 +1123,7 @@ helm: resources: requests: - cpu: "8" + cpu: "1" memory: 8Gi limits: cpu: "16" @@ -1131,7 +1208,17 @@ helm: operator: "Equal" value: "monitoring" effect: "PreferNoSchedule" - + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: kubernetes.io/hostname + operator: In + values: + - metal-01.he-eu-hel1.misc.vacdst + # -- [Scrape configuration](https://docs.victoriametrics.com/operator/api/#vmservicescrapespec) for Kube State Metrics vmScrape: enabled: true