From edb9a734ce86fc8f5d7742d82a14046474b04a91 Mon Sep 17 00:00:00 2001 From: Hung Cao Hiep Date: Mon, 19 Oct 2026 10:24:45 +0200 Subject: [PATCH] feat(helm): add extraInitContainers and extraVolumes to worker pods Worker pods had no way to run a step before the worker starts or to mount extra volumes without patching the rendered StatefulSet. Both lists are rendered on every worker lane except remote lanes, and extra volume names cannot reuse a chart worker volume name. --- deploy/helm/sie-cluster/README.md | 26 +++++++++ .../templates/worker-statefulset.yaml | 12 ++++ deploy/helm/sie-cluster/values.yaml | 19 +++++++ tools/ci/tests/test_helm_render.py | 55 +++++++++++++++++++ 4 files changed, 112 insertions(+) diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index 9ff8048ae..1999d9513 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -572,6 +572,32 @@ autoscaling: cooldownPeriod: 600 # 10 min before scale-down ``` +### Worker init containers and volumes + +`workers.common.extraInitContainers` adds init containers to every worker pod, +run before the worker starts. `workers.common.extraVolumes` adds volumes for +them. Both are standard Kubernetes lists and are not rendered on remote lanes. +Init containers can also mount the worker's `shm` and `model-cache` volumes, plus +`sie-ipc` and `kernel-cache` when those features are on. Extra volume names +must not reuse a chart worker volume name (`shm`, `model-cache`, `sie-ipc`, +`upstreams`, `equivalence`, `kernel-cache`); the render fails if they do. + +```yaml +workers: + common: + extraInitContainers: + - name: wait-for-dependency + image: busybox:1.37.0 + command: ["sh", "-c", "until wget -q -O /dev/null http://my-dependency/readyz; do sleep 2; done"] + volumeMounts: + - name: extra-config + mountPath: /etc/extra + extraVolumes: + - name: extra-config + configMap: + name: my-extra-config +``` + ### Multi-GPU Worker Shapes Req6 separates the Kubernetes allocation shape from the worker execution shape. diff --git a/deploy/helm/sie-cluster/templates/worker-statefulset.yaml b/deploy/helm/sie-cluster/templates/worker-statefulset.yaml index 2dfc87d12..1ab944cd7 100644 --- a/deploy/helm/sie-cluster/templates/worker-statefulset.yaml +++ b/deploy/helm/sie-cluster/templates/worker-statefulset.yaml @@ -55,6 +55,11 @@ fast at render time. {{- end -}} {{- include "sie-cluster.env.assertNoReserved" (dict "path" "workers.common.extraEnv" "entries" $root.Values.workers.common.extraEnv "reserved" $reservedWorkerEnv) -}} {{- include "sie-cluster.env.assertNoReserved" (dict "path" "workers.common.workerSidecar.extraEnv" "entries" $root.Values.workers.common.workerSidecar.extraEnv "reserved" $reservedSidecarEnv) -}} +{{- range $volume := (default (list) $root.Values.workers.common.extraVolumes) -}} +{{- if has $volume.name (list "shm" "model-cache" "sie-ipc" "upstreams" "equivalence" "kernel-cache") -}} +{{- fail (printf "workers.common.extraVolumes: %q is a chart-owned worker volume name" $volume.name) -}} +{{- end -}} +{{- end -}} {{- range $poolName, $pool := .Values.workers.pools }} {{- if $pool.enabled }} {{- if not $pool.bundles }} @@ -295,6 +300,10 @@ spec: fsGroup: 1000 runAsUser: 1000 runAsGroup: 1000 + {{- with (and (not $remoteLane) $root.Values.workers.common.extraInitContainers) }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} containers: {{- range $workerIndex := $workerIndexes }} {{- $workerSocket := "/var/run/sie/ipc.sock" }} @@ -883,6 +892,9 @@ spec: name: {{ $evidenceConfigMap | quote }} {{- end }} {{- end }} + {{- with (and (not $remoteLane) $root.Values.workers.common.extraVolumes) }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/deploy/helm/sie-cluster/values.yaml b/deploy/helm/sie-cluster/values.yaml index b72af4176..5c8b979a0 100644 --- a/deploy/helm/sie-cluster/values.yaml +++ b/deploy/helm/sie-cluster/values.yaml @@ -698,6 +698,25 @@ workers: # scope. The sidecar additionally reserves its health/capacity identity. extraEnv: [] + # -- Extra init containers for worker pods (standard Kubernetes container + # list), run before the worker starts. Not rendered on remote lanes. They + # can mount `extraVolumes`, `shm`, `model-cache`, and `sie-ipc` / + # `kernel-cache` when enabled: + # + # - name: wait-for-dependency + # image: busybox:1.37.0 + # command: ["sh", "-c", "until wget -q -O /dev/null http://my-dependency/readyz; do sleep 2; done"] + # volumeMounts: + # - name: extra-config + # mountPath: /etc/extra + extraInitContainers: [] + + # -- Extra volumes for worker pods (standard Kubernetes volume list), e.g. + # for `extraInitContainers`. Not rendered on remote lanes. Names must not + # clash with the chart's worker volumes (shm, model-cache, sie-ipc, + # upstreams, equivalence, kernel-cache). + extraVolumes: [] + # -- Shared model cold-start/ready timeout in seconds. The chart renders # this into both containers as SIE_MODEL_READY_TIMEOUT_S: the adapter worker # uses it as an SGLang startup-health timeout fallback, and the Rust diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index e8210cb99..c7b9fd22d 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -3996,3 +3996,58 @@ def test_worker_auth_off_preserves_existing_image_compatibility(tmp_path: Path, assert "SIE_AUTH_TOKEN" not in env assert WORKER_AUTH_TEST_JOB not in hook_jobs(docs) assert worker_auth_requirements(docs) == {} + + +EXTRA_INIT_CONTAINER = { + "name": "wait-for-dependency", + "image": "busybox:1.37.0", + "command": ["true"], + "volumeMounts": [{"name": "extra-config", "mountPath": "/etc/extra"}], +} +EXTRA_VOLUME = {"name": "extra-config", "configMap": {"name": "my-extra-config"}} + + +def extra_init_values(**common: object) -> dict: + return {"workers": {"common": common, "pools": {"l4": {"enabled": True}}}} + + +def test_workers_render_no_init_containers_by_default(tmp_path: Path) -> None: + for statefulset in worker_statefulsets(tmp_path, L4_POOL): + pod = statefulset["spec"]["template"]["spec"] + assert "initContainers" not in pod + assert "extra-config" not in {volume["name"] for volume in pod["volumes"]} + + +def test_extra_init_containers_and_volumes_reach_every_worker(tmp_path: Path) -> None: + values = extra_init_values(extraInitContainers=[EXTRA_INIT_CONTAINER], extraVolumes=[EXTRA_VOLUME]) + statefulsets = worker_statefulsets(tmp_path, values) + assert statefulsets + for statefulset in statefulsets: + pod = statefulset["spec"]["template"]["spec"] + assert pod["initContainers"] == [EXTRA_INIT_CONTAINER] + volumes = {volume["name"]: volume for volume in pod["volumes"]} + assert volumes["extra-config"] == EXTRA_VOLUME + assert pod["containers"][0]["name"] == "worker" + + +def test_extra_init_containers_and_volumes_skip_remote_lanes(tmp_path: Path) -> None: + values = remote_pool_values(l4={"enabled": True}) + values["workers"]["common"] = {"extraInitContainers": [EXTRA_INIT_CONTAINER], "extraVolumes": [EXTRA_VOLUME]} + pods = { + doc["metadata"]["name"]: doc["spec"]["template"]["spec"] + for doc in rendered_documents(tmp_path, values) + if doc["kind"] == "StatefulSet" and "-worker-" in doc["metadata"]["name"] + } + remote = pods.pop(REMOTE_WORKER[1]) + assert "initContainers" not in remote + assert "extra-config" not in {volume["name"] for volume in remote["volumes"]} + assert pods + for pod in pods.values(): + assert pod["initContainers"] == [EXTRA_INIT_CONTAINER] + + +@pytest.mark.parametrize("name", ["shm", "model-cache", "sie-ipc", "upstreams", "equivalence", "kernel-cache"]) +def test_extra_volumes_cannot_reuse_a_chart_worker_volume_name(tmp_path: Path, name: str) -> None: + result = render_workers(tmp_path, extra_init_values(extraVolumes=[{"name": name, "emptyDir": {}}])) + assert result.returncode != 0 + assert f'workers.common.extraVolumes: "{name}" is a chart-owned worker volume name' in result.stderr