From c8c1cc022c017d45503b06644019966e912044d8 Mon Sep 17 00:00:00 2001 From: Vico Chu Date: Fri, 24 Jul 2026 16:31:04 +0800 Subject: [PATCH 1/4] fix(ci): pin kubernetes<36 in all Python CI workflows kubernetes 36.0.0 regenerated the sync client with Pydantic models, breaking the FakeResponse deserialization hack used in SDK unit tests. Pin kubernetes<36 in all CI workflows that install the Python SDK (test-python, integration-tests, e2e-test-train-api, test-example-notebooks) to avoid impacting SDK release. Add a TODO for future Pydantic work. Signed-off-by: Vico Chu --- .github/workflows/e2e-test-train-api.yaml | 2 +- .github/workflows/integration-tests.yaml | 2 +- .github/workflows/test-example-notebooks.yaml | 2 +- .github/workflows/test-python.yaml | 7 ++++++- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-test-train-api.yaml b/.github/workflows/e2e-test-train-api.yaml index 045c3b19e2..8d4ca0e734 100644 --- a/.github/workflows/e2e-test-train-api.yaml +++ b/.github/workflows/e2e-test-train-api.yaml @@ -53,7 +53,7 @@ jobs: - name: Run tests run: | - pip install pytest + pip install pytest "kubernetes<36" python3 -m pip install -e sdk/python[huggingface] pytest -s sdk/python/test/e2e-fine-tune-llm/test_e2e_pytorch_fine_tune_llm.py --log-cli-level=debug env: diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml index a450a76b16..9626d8216b 100644 --- a/.github/workflows/integration-tests.yaml +++ b/.github/workflows/integration-tests.yaml @@ -80,7 +80,7 @@ jobs: - name: Run tests run: | - pip install pytest + pip install pytest "kubernetes<36" python3 -m pip install -e sdk/python; pytest -s sdk/python/test/e2e --log-cli-level=debug --namespace=default env: GANG_SCHEDULER_NAME: ${{ matrix.gang-scheduler-name }} diff --git a/.github/workflows/test-example-notebooks.yaml b/.github/workflows/test-example-notebooks.yaml index 0ee767e165..61338ef2ea 100644 --- a/.github/workflows/test-example-notebooks.yaml +++ b/.github/workflows/test-example-notebooks.yaml @@ -28,7 +28,7 @@ jobs: - name: Install Python Dependencies run: | - pip install papermill==2.6.0 jupyter==1.1.1 ipykernel==6.29.5 + pip install papermill==2.6.0 jupyter==1.1.1 ipykernel==6.29.5 "kubernetes<36" - name: Run Jupyter Notebook with Papermill shell: bash diff --git a/.github/workflows/test-python.yaml b/.github/workflows/test-python.yaml index 9a706461b7..11ebce3928 100644 --- a/.github/workflows/test-python.yaml +++ b/.github/workflows/test-python.yaml @@ -28,7 +28,12 @@ jobs: - name: Install dependencies run: | - pip install pytest python-dateutil urllib3 kubernetes + # TODO: kubernetes 36.0.0 switched to Pydantic models, breaking the + # FakeResponse deserialization hack in the SDK unit tests. To remove + # this pin, either adapt FakeResponse in + # sdk/python/kubeflow/training/utils/utils.py for Pydantic or mock + # ApiClient.deserialize directly in the test suite. + pip install pytest python-dateutil urllib3 "kubernetes<36" pip install -U './sdk/python[huggingface]' - name: Run unit test for training sdk From 82b2b71ee3b18ec16ff36a28c73b68fe165753a7 Mon Sep 17 00:00:00 2001 From: Vico Chu Date: Fri, 24 Jul 2026 19:42:09 +0800 Subject: [PATCH 2/4] fix(test): increase JAX e2e test memory limit to avoid OOMKilled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit JAX worker pods were OOMKilled (exit 137) with 3Gi memory limit. Increase to 5Gi to accommodate XLA compilation and training memory usage. GitHub-hosted runners have 16GB RAM, so 2 replicas × 5Gi is feasible. Signed-off-by: Vico Chu --- sdk/python/test/e2e/test_e2e_jaxjob.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/test/e2e/test_e2e_jaxjob.py b/sdk/python/test/e2e/test_e2e_jaxjob.py index 7471f67338..9ddb18cfd4 100644 --- a/sdk/python/test/e2e/test_e2e_jaxjob.py +++ b/sdk/python/test/e2e/test_e2e_jaxjob.py @@ -156,5 +156,5 @@ def generate_container() -> V1Container: return V1Container( name=CONTAINER_NAME, image=os.getenv("JAX_JOB_IMAGE", "docker.io/kubeflow/jaxjob-dist-spmd-mnist:latest"), - resources=V1ResourceRequirements(limits={"memory": "3Gi", "cpu": "1.2"}), + resources=V1ResourceRequirements(limits={"memory": "5Gi", "cpu": "1.2"}), ) From 650f606297b099a01e7a8c875bbe4384ab3769e7 Mon Sep 17 00:00:00 2001 From: Vico Chu Date: Wed, 15 Jul 2026 19:21:11 +0800 Subject: [PATCH 3/4] fix(docker): make kubectl-delivery base image and k8s version configurable Add ALPINE_VERSION (default 3.22) and K8S_VERSION (default v1.30.7) as build args so they can be overridden via --build-arg without editing the Dockerfile. Signed-off-by: Vico Chu --- build/images/kubectl-delivery/Dockerfile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build/images/kubectl-delivery/Dockerfile b/build/images/kubectl-delivery/Dockerfile index 7ad21aa273..abe6d31772 100644 --- a/build/images/kubectl-delivery/Dockerfile +++ b/build/images/kubectl-delivery/Dockerfile @@ -1,16 +1,16 @@ -FROM alpine:3.17 AS build +ARG ALPINE_VERSION=3.22 -ARG TARGETARCH +FROM alpine:${ALPINE_VERSION} AS build -# Install kubectl. -ENV K8S_VERSION v1.30.7 +ARG TARGETARCH +ARG K8S_VERSION=v1.30.7 RUN apk add --no-cache wget RUN wget -q https://dl.k8s.io/release/${K8S_VERSION}/bin/linux/${TARGETARCH}/kubectl RUN chmod +x ./kubectl RUN mv ./kubectl /bin/kubectl -FROM alpine:3.17 +FROM alpine:${ALPINE_VERSION} COPY --from=build /bin/kubectl /bin/kubectl RUN apk add --no-cache bash From f2ada4d6547bb6acb9df4c370ab9a956e4a0c2ca Mon Sep 17 00:00:00 2001 From: Vico Chu Date: Thu, 23 Jul 2026 19:34:50 +0800 Subject: [PATCH 4/4] fix(docker): verify kubectl sha256 checksum in kubectl-delivery image Download the official .sha256 from dl.k8s.io and verify the binary before installing, preventing use of a tampered kubectl. --- build/images/kubectl-delivery/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build/images/kubectl-delivery/Dockerfile b/build/images/kubectl-delivery/Dockerfile index abe6d31772..7957bb18c1 100644 --- a/build/images/kubectl-delivery/Dockerfile +++ b/build/images/kubectl-delivery/Dockerfile @@ -7,6 +7,8 @@ ARG K8S_VERSION=v1.30.7 RUN apk add --no-cache wget RUN wget -q https://dl.k8s.io/release/${K8S_VERSION}/bin/linux/${TARGETARCH}/kubectl +RUN wget -q https://dl.k8s.io/release/${K8S_VERSION}/bin/linux/${TARGETARCH}/kubectl.sha256 +RUN test "$(cat kubectl.sha256)" = "$(sha256sum kubectl | cut -d' ' -f1)" RUN chmod +x ./kubectl RUN mv ./kubectl /bin/kubectl