From 47c1529f630c193053d9a5043cc456b8040a9e84 Mon Sep 17 00:00:00 2001 From: Jaya Venkatesh Date: Tue, 31 Mar 2026 12:58:38 -0700 Subject: [PATCH 1/5] change conf to PEP440 standards Signed-off-by: Jaya Venkatesh --- source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/conf.py b/source/conf.py index 3064871f..11da287e 100644 --- a/source/conf.py +++ b/source/conf.py @@ -49,7 +49,7 @@ "rapids_conda_channels": "-c rapidsai-nightly -c conda-forge", "rapids_conda_packages": f"rapids={nightly_version} python=3.13 'cuda-version>=12.0,<=12.9'", "rapids_pip_index": "https://pypi.anaconda.org/rapidsai-wheels-nightly/simple", - "rapids_pip_version": f"{nightly_version}.*,>=0.0.0a0", + "rapids_pip_version": f"~={nightly_version}.0a0", # SageMaker Notebook Instance examples need to stay pinned to an older RAPIDS until this is resolved: # https://github.com/rapidsai/deployment/issues/520 "rapids_sagemaker_conda_packages": f"rapids={nightly_version} python=3.12 cuda-version=13", From 1b8caeafc18d723fb9a34496d950336c4627ecbe Mon Sep 17 00:00:00 2001 From: Jaya Venkatesh Date: Tue, 2 Jun 2026 23:21:11 -0700 Subject: [PATCH 2/5] added nvidia gpu operator instructions Signed-off-by: Jaya Venkatesh --- source/cloud/gcp/gke.md | 90 +++++++++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 12 deletions(-) diff --git a/source/cloud/gcp/gke.md b/source/cloud/gcp/gke.md index 5516d214..764f8294 100644 --- a/source/cloud/gcp/gke.md +++ b/source/cloud/gcp/gke.md @@ -24,11 +24,12 @@ Now we can launch a GPU enabled GKE cluster. ```bash $ gcloud container clusters create rapids-gpu-kubeflow \ - --accelerator type=nvidia-tesla-a100,count=2,gpu-driver-version=latest --machine-type a2-highgpu-2g \ - --zone us-central1-c --release-channel stable + --accelerator type=nvidia-tesla-a100,count=2,gpu-driver-version=disabled --machine-type a2-highgpu-2g \ + --zone us-central1-c --release-channel stable \ + --node-labels="gke-no-default-nvidia-gpu-device-plugin=true" ``` -With this command, you’ve launched a GKE cluster called `rapids-gpu-kubeflow`. You’ve specified that it should use nodes of type a2-highgpu-2g, each with two A100 GPUs, along with the latest GPU drivers for the current GKE version. +With this command, you’ve launched a GKE cluster called `rapids-gpu-kubeflow` with nodes of type `a2-highgpu-2g`, which has two A100 GPUs. GKE's automatic GPU driver installation and default NVIDIA GPU device plugin are disabled so that the [NVIDIA GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/google-gke.html) can configure the GPU stack that RAPIDS needs. ````{note} After creating your cluster, if you get a message saying @@ -37,7 +38,7 @@ After creating your cluster, if you get a message saying CRITICAL: ACTION REQUIRED: gke-gcloud-auth-plugin, which is needed for continued use of kubectl, was not found or is not executable. Install gke-gcloud-auth-plugin for use with kubectl by following https://cloud.google.com/kubernetes-engine/docs/how-to/cluster-access-for-kubectl#install_plugin ``` -you will need to install the `gke-gcloud-auth-plugin` to be able to get the credentials. To do so, +You will need to install the `gke-gcloud-auth-plugin` to be able to get the credentials. To do so, ```bash $ gcloud components install gke-gcloud-auth-plugin @@ -48,23 +49,88 @@ $ gcloud components install gke-gcloud-auth-plugin ```bash $ gcloud container clusters get-credentials rapids-gpu-kubeflow \ - --region=us-central1-c + --zone us-central1-c ``` With this command, your `kubeconfig` is updated with credentials and endpoint information for the `rapids-gpu-kubeflow` cluster. -## Verify drivers +## Install GPU drivers and Operator -Verify that the NVIDIA drivers are successfully installed. +Create a namespace for the NVIDIA GPU Operator. + +```bash +$ kubectl create ns gpu-operator +``` + +Create a resource quota for critical GPU Operator Pods. + +```bash +kubectl apply -n gpu-operator -f - << EOF +apiVersion: v1 +kind: ResourceQuota +metadata: + name: gpu-operator-quota +spec: + hard: + pods: 100 + scopeSelector: + matchExpressions: + - operator: In + scopeName: PriorityClass + values: + - system-node-critical + - system-cluster-critical +EOF +``` + +Install the Google driver installer DaemonSet. This command is for Container-Optimized OS nodes, which GKE uses by default. For Ubuntu nodes, use the Ubuntu driver installer manifest from the [GKE manual driver installation documentation](https://cloud.google.com/kubernetes-engine/docs/how-to/gpus#installing_drivers). + +```bash +$ kubectl apply -f https://raw.githubusercontent.com/GoogleCloudPlatform/container-engine-accelerators/master/nvidia-driver-installer/cos/daemonset-preloaded.yaml +$ kubectl rollout status daemonset/nvidia-driver-installer -n kube-system --timeout=300s +``` + +Install the NVIDIA GPU Operator with the GKE-specific driver and toolkit paths. + +```bash +$ helm repo add nvidia https://helm.ngc.nvidia.com/nvidia +$ helm repo update +$ helm install --wait gpu-operator \ + -n gpu-operator \ + nvidia/gpu-operator \ + --version=v26.3.2 \ + --set hostPaths.driverInstallDir=/home/kubernetes/bin/nvidia \ + --set toolkit.installDir=/home/kubernetes/bin/nvidia \ + --set cdi.enabled=true \ + --set cdi.default=true \ + --set driver.enabled=false +``` + +````{note} +On GKE 1.33 and later, NVIDIA documents a known `containerd` configuration issue that can prevent GPU Operator toolkit Pods from starting. If you hit this, follow NVIDIA's `RUNTIME_CONFIG_SOURCE=file` ClusterPolicy workaround in the [NVIDIA GPU Operator with Google GKE prerequisites](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/google-gke.html#prerequisites). +```` + +Verify that the GPU Operator Pods are running. + +```console +$ kubectl get pods -n gpu-operator +NAME READY STATUS RESTARTS AGE +gpu-operator-5c7cf8b4f6-bx4rg 1/1 Running 0 11m +nvidia-container-toolkit-daemonset-vr8fv 1/1 Running 0 8m +nvidia-cuda-validator-4nljj 0/1 Completed 0 2m +nvidia-device-plugin-daemonset-jfbcj 1/1 Running 0 8m +nvidia-operator-validator-fcrr6 1/1 Running 0 8m +``` + +Verify that GPUs are allocatable on the node. ```console -$ kubectl get po -A --watch | grep nvidia -kube-system nvidia-gpu-device-plugin-medium-cos-h5kkz 2/2 Running 0 3m42s -kube-system nvidia-gpu-device-plugin-medium-cos-pw89w 2/2 Running 0 3m42s -kube-system nvidia-gpu-device-plugin-medium-cos-wdnm9 2/2 Running 0 3m42s +$ kubectl get nodes -o=custom-columns='NAME:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu' +NAME GPU +gke-rapids-gpu-kubeflow-default-pool-00000000-0000 2 ``` -After GPU device plugin pods are in running state, you are ready to test your cluster. +Once the GPU Operator Pods are running and GPUs are allocatable, you are ready to test your cluster. ```{include} ../../_includes/check-gpu-pod-works.md From 76de762f1db1e866418fb9ff226de2565df64a4c Mon Sep 17 00:00:00 2001 From: Jaya Venkatesh Date: Tue, 2 Jun 2026 23:23:44 -0700 Subject: [PATCH 3/5] run precommit Signed-off-by: Jaya Venkatesh --- source/cloud/gcp/gke.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/cloud/gcp/gke.md b/source/cloud/gcp/gke.md index 764f8294..e9338e80 100644 --- a/source/cloud/gcp/gke.md +++ b/source/cloud/gcp/gke.md @@ -106,9 +106,9 @@ $ helm install --wait gpu-operator \ --set driver.enabled=false ``` -````{note} +```{note} On GKE 1.33 and later, NVIDIA documents a known `containerd` configuration issue that can prevent GPU Operator toolkit Pods from starting. If you hit this, follow NVIDIA's `RUNTIME_CONFIG_SOURCE=file` ClusterPolicy workaround in the [NVIDIA GPU Operator with Google GKE prerequisites](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/google-gke.html#prerequisites). -```` +``` Verify that the GPU Operator Pods are running. From 49c78881d68b194a1aaad7cfc33707bc00ebf319 Mon Sep 17 00:00:00 2001 From: Jaya Venkatesh Date: Wed, 3 Jun 2026 10:37:57 -0700 Subject: [PATCH 4/5] changed wording Signed-off-by: Jaya Venkatesh --- source/cloud/gcp/gke.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/cloud/gcp/gke.md b/source/cloud/gcp/gke.md index e9338e80..6a29e837 100644 --- a/source/cloud/gcp/gke.md +++ b/source/cloud/gcp/gke.md @@ -110,7 +110,7 @@ $ helm install --wait gpu-operator \ On GKE 1.33 and later, NVIDIA documents a known `containerd` configuration issue that can prevent GPU Operator toolkit Pods from starting. If you hit this, follow NVIDIA's `RUNTIME_CONFIG_SOURCE=file` ClusterPolicy workaround in the [NVIDIA GPU Operator with Google GKE prerequisites](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/google-gke.html#prerequisites). ``` -Verify that the GPU Operator Pods are running. +Verify that the GPU Operator Pods and the Operator DaemonSet Pods are `Running` and validator Pods are `Running` or `Completed`. ```console $ kubectl get pods -n gpu-operator From 44a15385dc0436037e1d1ab44eb010974b83d668 Mon Sep 17 00:00:00 2001 From: Jaya Venkatesh Date: Wed, 3 Jun 2026 10:40:49 -0700 Subject: [PATCH 5/5] run pre-commit Signed-off-by: Jaya Venkatesh --- source/cloud/gcp/gke.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/cloud/gcp/gke.md b/source/cloud/gcp/gke.md index 6a29e837..e4b3fd17 100644 --- a/source/cloud/gcp/gke.md +++ b/source/cloud/gcp/gke.md @@ -110,7 +110,7 @@ $ helm install --wait gpu-operator \ On GKE 1.33 and later, NVIDIA documents a known `containerd` configuration issue that can prevent GPU Operator toolkit Pods from starting. If you hit this, follow NVIDIA's `RUNTIME_CONFIG_SOURCE=file` ClusterPolicy workaround in the [NVIDIA GPU Operator with Google GKE prerequisites](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/google-gke.html#prerequisites). ``` -Verify that the GPU Operator Pods and the Operator DaemonSet Pods are `Running` and validator Pods are `Running` or `Completed`. +Verify that the GPU Operator Pods and the Operator DaemonSet Pods are `Running` and validator Pods are `Running` or `Completed`. ```console $ kubectl get pods -n gpu-operator