diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index 9ff8048ae..1999d9513 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -572,6 +572,32 @@ autoscaling: cooldownPeriod: 600 # 10 min before scale-down ``` +### Worker init containers and volumes + +`workers.common.extraInitContainers` adds init containers to every worker pod, +run before the worker starts. `workers.common.extraVolumes` adds volumes for +them. Both are standard Kubernetes lists and are not rendered on remote lanes. +Init containers can also mount the worker's `shm` and `model-cache` volumes, plus +`sie-ipc` and `kernel-cache` when those features are on. Extra volume names +must not reuse a chart worker volume name (`shm`, `model-cache`, `sie-ipc`, +`upstreams`, `equivalence`, `kernel-cache`); the render fails if they do. + +```yaml +workers: + common: + extraInitContainers: + - name: wait-for-dependency + image: busybox:1.37.0 + command: ["sh", "-c", "until wget -q -O /dev/null http://my-dependency/readyz; do sleep 2; done"] + volumeMounts: + - name: extra-config + mountPath: /etc/extra + extraVolumes: + - name: extra-config + configMap: + name: my-extra-config +``` + ### Multi-GPU Worker Shapes Req6 separates the Kubernetes allocation shape from the worker execution shape. diff --git a/deploy/helm/sie-cluster/templates/worker-statefulset.yaml b/deploy/helm/sie-cluster/templates/worker-statefulset.yaml index 2dfc87d12..1ab944cd7 100644 --- a/deploy/helm/sie-cluster/templates/worker-statefulset.yaml +++ b/deploy/helm/sie-cluster/templates/worker-statefulset.yaml @@ -55,6 +55,11 @@ fast at render time. {{- end -}} {{- include "sie-cluster.env.assertNoReserved" (dict "path" "workers.common.extraEnv" "entries" $root.Values.workers.common.extraEnv "reserved" $reservedWorkerEnv) -}} {{- include "sie-cluster.env.assertNoReserved" (dict "path" "workers.common.workerSidecar.extraEnv" "entries" $root.Values.workers.common.workerSidecar.extraEnv "reserved" $reservedSidecarEnv) -}} +{{- range $volume := (default (list) $root.Values.workers.common.extraVolumes) -}} +{{- if has $volume.name (list "shm" "model-cache" "sie-ipc" "upstreams" "equivalence" "kernel-cache") -}} +{{- fail (printf "workers.common.extraVolumes: %q is a chart-owned worker volume name" $volume.name) -}} +{{- end -}} +{{- end -}} {{- range $poolName, $pool := .Values.workers.pools }} {{- if $pool.enabled }} {{- if not $pool.bundles }} @@ -295,6 +300,10 @@ spec: fsGroup: 1000 runAsUser: 1000 runAsGroup: 1000 + {{- with (and (not $remoteLane) $root.Values.workers.common.extraInitContainers) }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} containers: {{- range $workerIndex := $workerIndexes }} {{- $workerSocket := "/var/run/sie/ipc.sock" }} @@ -883,6 +892,9 @@ spec: name: {{ $evidenceConfigMap | quote }} {{- end }} {{- end }} + {{- with (and (not $remoteLane) $root.Values.workers.common.extraVolumes) }} + {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/deploy/helm/sie-cluster/values.yaml b/deploy/helm/sie-cluster/values.yaml index b72af4176..5c8b979a0 100644 --- a/deploy/helm/sie-cluster/values.yaml +++ b/deploy/helm/sie-cluster/values.yaml @@ -698,6 +698,25 @@ workers: # scope. The sidecar additionally reserves its health/capacity identity. extraEnv: [] + # -- Extra init containers for worker pods (standard Kubernetes container + # list), run before the worker starts. Not rendered on remote lanes. They + # can mount `extraVolumes`, `shm`, `model-cache`, and `sie-ipc` / + # `kernel-cache` when enabled: + # + # - name: wait-for-dependency + # image: busybox:1.37.0 + # command: ["sh", "-c", "until wget -q -O /dev/null http://my-dependency/readyz; do sleep 2; done"] + # volumeMounts: + # - name: extra-config + # mountPath: /etc/extra + extraInitContainers: [] + + # -- Extra volumes for worker pods (standard Kubernetes volume list), e.g. + # for `extraInitContainers`. Not rendered on remote lanes. Names must not + # clash with the chart's worker volumes (shm, model-cache, sie-ipc, + # upstreams, equivalence, kernel-cache). + extraVolumes: [] + # -- Shared model cold-start/ready timeout in seconds. The chart renders # this into both containers as SIE_MODEL_READY_TIMEOUT_S: the adapter worker # uses it as an SGLang startup-health timeout fallback, and the Rust diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index e8210cb99..c7b9fd22d 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -3996,3 +3996,58 @@ def test_worker_auth_off_preserves_existing_image_compatibility(tmp_path: Path, assert "SIE_AUTH_TOKEN" not in env assert WORKER_AUTH_TEST_JOB not in hook_jobs(docs) assert worker_auth_requirements(docs) == {} + + +EXTRA_INIT_CONTAINER = { + "name": "wait-for-dependency", + "image": "busybox:1.37.0", + "command": ["true"], + "volumeMounts": [{"name": "extra-config", "mountPath": "/etc/extra"}], +} +EXTRA_VOLUME = {"name": "extra-config", "configMap": {"name": "my-extra-config"}} + + +def extra_init_values(**common: object) -> dict: + return {"workers": {"common": common, "pools": {"l4": {"enabled": True}}}} + + +def test_workers_render_no_init_containers_by_default(tmp_path: Path) -> None: + for statefulset in worker_statefulsets(tmp_path, L4_POOL): + pod = statefulset["spec"]["template"]["spec"] + assert "initContainers" not in pod + assert "extra-config" not in {volume["name"] for volume in pod["volumes"]} + + +def test_extra_init_containers_and_volumes_reach_every_worker(tmp_path: Path) -> None: + values = extra_init_values(extraInitContainers=[EXTRA_INIT_CONTAINER], extraVolumes=[EXTRA_VOLUME]) + statefulsets = worker_statefulsets(tmp_path, values) + assert statefulsets + for statefulset in statefulsets: + pod = statefulset["spec"]["template"]["spec"] + assert pod["initContainers"] == [EXTRA_INIT_CONTAINER] + volumes = {volume["name"]: volume for volume in pod["volumes"]} + assert volumes["extra-config"] == EXTRA_VOLUME + assert pod["containers"][0]["name"] == "worker" + + +def test_extra_init_containers_and_volumes_skip_remote_lanes(tmp_path: Path) -> None: + values = remote_pool_values(l4={"enabled": True}) + values["workers"]["common"] = {"extraInitContainers": [EXTRA_INIT_CONTAINER], "extraVolumes": [EXTRA_VOLUME]} + pods = { + doc["metadata"]["name"]: doc["spec"]["template"]["spec"] + for doc in rendered_documents(tmp_path, values) + if doc["kind"] == "StatefulSet" and "-worker-" in doc["metadata"]["name"] + } + remote = pods.pop(REMOTE_WORKER[1]) + assert "initContainers" not in remote + assert "extra-config" not in {volume["name"] for volume in remote["volumes"]} + assert pods + for pod in pods.values(): + assert pod["initContainers"] == [EXTRA_INIT_CONTAINER] + + +@pytest.mark.parametrize("name", ["shm", "model-cache", "sie-ipc", "upstreams", "equivalence", "kernel-cache"]) +def test_extra_volumes_cannot_reuse_a_chart_worker_volume_name(tmp_path: Path, name: str) -> None: + result = render_workers(tmp_path, extra_init_values(extraVolumes=[{"name": name, "emptyDir": {}}])) + assert result.returncode != 0 + assert f'workers.common.extraVolumes: "{name}" is a chart-owned worker volume name' in result.stderr