diff --git a/deploy/helm/sie-cluster/values-aks.yaml b/deploy/helm/sie-cluster/values-aks.yaml index 849357da8..a84ce611e 100644 --- a/deploy/helm/sie-cluster/values-aks.yaml +++ b/deploy/helm/sie-cluster/values-aks.yaml @@ -178,5 +178,6 @@ serviceAccount: # azure.workload.identity/client-id: # inject via --set at deploy time dcgm-exporter: - # AKS GPU nodes use taints; no node selector required - nodeSelector: {} + # GPU node label set by the terraform-azure-sie GPU node pools + nodeSelector: + sie.superlinked.com/node-type: gpu diff --git a/deploy/helm/sie-cluster/values-aws.yaml b/deploy/helm/sie-cluster/values-aws.yaml index ac7772060..692c42c93 100644 --- a/deploy/helm/sie-cluster/values-aws.yaml +++ b/deploy/helm/sie-cluster/values-aws.yaml @@ -177,5 +177,6 @@ serviceAccount: # eks.amazonaws.com/role-arn: # inject via --set at deploy time dcgm-exporter: - # AWS GPU nodes use taints, not the GCP accelerator label - nodeSelector: {} + # GPU node label set by the terraform-aws-sie GPU node groups + nodeSelector: + sie.superlinked.com/node-type: gpu diff --git a/deploy/helm/sie-cluster/values.yaml b/deploy/helm/sie-cluster/values.yaml index b72af4176..b5d07d621 100644 --- a/deploy/helm/sie-cluster/values.yaml +++ b/deploy/helm/sie-cluster/values.yaml @@ -2105,11 +2105,8 @@ dcgm-exporter: interval: "15s" additionalLabels: app.kubernetes.io/part-of: sie - # -- Run on GPU nodes only. Override per cloud: - # GCP: cloud.google.com/gke-accelerator-count: "1" - # AWS: add dcgm-exporter.nodeSelector: {} in values-aws.yaml - nodeSelector: - cloud.google.com/gke-accelerator-count: "1" + # -- GPU node labels; set per cloud in the values-.yaml overlays. + nodeSelector: {} tolerations: - key: nvidia.com/gpu operator: Exists diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index e8210cb99..57ec7c48a 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -3996,3 +3996,36 @@ def test_worker_auth_off_preserves_existing_image_compatibility(tmp_path: Path, assert "SIE_AUTH_TOKEN" not in env assert WORKER_AUTH_TEST_JOB not in hook_jobs(docs) assert worker_auth_requirements(docs) == {} + + +GKE_ACCELERATOR_LABEL = "cloud.google.com/gke-accelerator-count" + + +def dcgm_exporter_node_selector(tmp_path: Path, overlays: tuple[str, ...] = ()) -> dict: + docs = rendered_documents(tmp_path, {"dcgm-exporter": {"install": True}}, overlays) + (daemonset,) = [doc for doc in docs if doc["kind"] == "DaemonSet" and "dcgm-exporter" in doc["metadata"]["name"]] + return daemonset["spec"]["template"]["spec"].get("nodeSelector") or {} + + +@pytest.mark.parametrize( + ("overlay", "expected"), + [ + ("values-aws.yaml", {"sie.superlinked.com/node-type": "gpu"}), + ("values-aks.yaml", {"sie.superlinked.com/node-type": "gpu"}), + ( + "values-ack.yaml", + { + "ack.node.gpu.schedule": "default", + "sie.superlinked.com/gpu-type": "a10", + "sie.superlinked.com/node-type": "gpu", + }, + ), + ("values-gke.yaml", {GKE_ACCELERATOR_LABEL: "1"}), + ], +) +def test_dcgm_exporter_selects_only_the_cloud_overlay_gpu_labels(tmp_path: Path, overlay: str, expected: dict) -> None: + assert dcgm_exporter_node_selector(tmp_path, (overlay,)) == expected + + +def test_dcgm_exporter_has_no_cloud_specific_default_selector(tmp_path: Path) -> None: + assert GKE_ACCELERATOR_LABEL not in dcgm_exporter_node_selector(tmp_path)