Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions deploy/helm/sie-cluster/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -572,6 +572,32 @@ autoscaling:
cooldownPeriod: 600 # 10 min before scale-down
```

### Worker init containers and volumes

`workers.common.extraInitContainers` adds init containers to every worker pod,
run before the worker starts. `workers.common.extraVolumes` adds volumes for
them. Both are standard Kubernetes lists and are not rendered on remote lanes.
Init containers can also mount the worker's `shm` and `model-cache` volumes, plus
`sie-ipc` and `kernel-cache` when those features are on. Extra volume names
must not reuse a chart worker volume name (`shm`, `model-cache`, `sie-ipc`,
`upstreams`, `equivalence`, `kernel-cache`); the render fails if they do.

```yaml
workers:
common:
extraInitContainers:
- name: wait-for-dependency
image: busybox:1.37.0
command: ["sh", "-c", "until wget -q -O /dev/null http://my-dependency/readyz; do sleep 2; done"]
volumeMounts:
- name: extra-config
mountPath: /etc/extra
extraVolumes:
- name: extra-config
configMap:
name: my-extra-config
```

### Multi-GPU Worker Shapes

Req6 separates the Kubernetes allocation shape from the worker execution shape.
Expand Down
12 changes: 12 additions & 0 deletions deploy/helm/sie-cluster/templates/worker-statefulset.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,11 @@ fast at render time.
{{- end -}}
{{- include "sie-cluster.env.assertNoReserved" (dict "path" "workers.common.extraEnv" "entries" $root.Values.workers.common.extraEnv "reserved" $reservedWorkerEnv) -}}
{{- include "sie-cluster.env.assertNoReserved" (dict "path" "workers.common.workerSidecar.extraEnv" "entries" $root.Values.workers.common.workerSidecar.extraEnv "reserved" $reservedSidecarEnv) -}}
{{- range $volume := (default (list) $root.Values.workers.common.extraVolumes) -}}
{{- if has $volume.name (list "shm" "model-cache" "sie-ipc" "upstreams" "equivalence" "kernel-cache") -}}
{{- fail (printf "workers.common.extraVolumes: %q is a chart-owned worker volume name" $volume.name) -}}
{{- end -}}
{{- end -}}
{{- range $poolName, $pool := .Values.workers.pools }}
{{- if $pool.enabled }}
{{- if not $pool.bundles }}
Expand Down Expand Up @@ -295,6 +300,10 @@ spec:
fsGroup: 1000
runAsUser: 1000
runAsGroup: 1000
{{- with (and (not $remoteLane) $root.Values.workers.common.extraInitContainers) }}
initContainers:
{{- toYaml . | nindent 8 }}
{{- end }}
containers:
{{- range $workerIndex := $workerIndexes }}
{{- $workerSocket := "/var/run/sie/ipc.sock" }}
Expand Down Expand Up @@ -883,6 +892,9 @@ spec:
name: {{ $evidenceConfigMap | quote }}
{{- end }}
{{- end }}
{{- with (and (not $remoteLane) $root.Values.workers.common.extraVolumes) }}
{{- toYaml . | nindent 8 }}
{{- end }}
{{- end }}
{{- end }}
{{- end }}
Expand Down
19 changes: 19 additions & 0 deletions deploy/helm/sie-cluster/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -698,6 +698,25 @@ workers:
# scope. The sidecar additionally reserves its health/capacity identity.
extraEnv: []

# -- Extra init containers for worker pods (standard Kubernetes container
# list), run before the worker starts. Not rendered on remote lanes. They
# can mount `extraVolumes`, `shm`, `model-cache`, and `sie-ipc` /
# `kernel-cache` when enabled:
#
# - name: wait-for-dependency
# image: busybox:1.37.0
# command: ["sh", "-c", "until wget -q -O /dev/null http://my-dependency/readyz; do sleep 2; done"]
# volumeMounts:
# - name: extra-config
# mountPath: /etc/extra
extraInitContainers: []

# -- Extra volumes for worker pods (standard Kubernetes volume list), e.g.
# for `extraInitContainers`. Not rendered on remote lanes. Names must not
# clash with the chart's worker volumes (shm, model-cache, sie-ipc,
# upstreams, equivalence, kernel-cache).
extraVolumes: []

# -- Shared model cold-start/ready timeout in seconds. The chart renders
# this into both containers as SIE_MODEL_READY_TIMEOUT_S: the adapter worker
# uses it as an SGLang startup-health timeout fallback, and the Rust
Expand Down
55 changes: 55 additions & 0 deletions tools/ci/tests/test_helm_render.py
Original file line number Diff line number Diff line change
Expand Up @@ -3996,3 +3996,58 @@ def test_worker_auth_off_preserves_existing_image_compatibility(tmp_path: Path,
assert "SIE_AUTH_TOKEN" not in env
assert WORKER_AUTH_TEST_JOB not in hook_jobs(docs)
assert worker_auth_requirements(docs) == {}


EXTRA_INIT_CONTAINER = {
"name": "wait-for-dependency",
"image": "busybox:1.37.0",
"command": ["true"],
"volumeMounts": [{"name": "extra-config", "mountPath": "/etc/extra"}],
}
EXTRA_VOLUME = {"name": "extra-config", "configMap": {"name": "my-extra-config"}}


def extra_init_values(**common: object) -> dict:
return {"workers": {"common": common, "pools": {"l4": {"enabled": True}}}}


def test_workers_render_no_init_containers_by_default(tmp_path: Path) -> None:
for statefulset in worker_statefulsets(tmp_path, L4_POOL):
pod = statefulset["spec"]["template"]["spec"]
assert "initContainers" not in pod
assert "extra-config" not in {volume["name"] for volume in pod["volumes"]}


def test_extra_init_containers_and_volumes_reach_every_worker(tmp_path: Path) -> None:
values = extra_init_values(extraInitContainers=[EXTRA_INIT_CONTAINER], extraVolumes=[EXTRA_VOLUME])
statefulsets = worker_statefulsets(tmp_path, values)
assert statefulsets
for statefulset in statefulsets:
pod = statefulset["spec"]["template"]["spec"]
assert pod["initContainers"] == [EXTRA_INIT_CONTAINER]
volumes = {volume["name"]: volume for volume in pod["volumes"]}
assert volumes["extra-config"] == EXTRA_VOLUME
assert pod["containers"][0]["name"] == "worker"


def test_extra_init_containers_and_volumes_skip_remote_lanes(tmp_path: Path) -> None:
values = remote_pool_values(l4={"enabled": True})
values["workers"]["common"] = {"extraInitContainers": [EXTRA_INIT_CONTAINER], "extraVolumes": [EXTRA_VOLUME]}
pods = {
doc["metadata"]["name"]: doc["spec"]["template"]["spec"]
for doc in rendered_documents(tmp_path, values)
if doc["kind"] == "StatefulSet" and "-worker-" in doc["metadata"]["name"]
}
remote = pods.pop(REMOTE_WORKER[1])
assert "initContainers" not in remote
assert "extra-config" not in {volume["name"] for volume in remote["volumes"]}
assert pods
for pod in pods.values():
assert pod["initContainers"] == [EXTRA_INIT_CONTAINER]


@pytest.mark.parametrize("name", ["shm", "model-cache", "sie-ipc", "upstreams", "equivalence", "kernel-cache"])
def test_extra_volumes_cannot_reuse_a_chart_worker_volume_name(tmp_path: Path, name: str) -> None:
result = render_workers(tmp_path, extra_init_values(extraVolumes=[{"name": name, "emptyDir": {}}]))
assert result.returncode != 0
assert f'workers.common.extraVolumes: "{name}" is a chart-owned worker volume name' in result.stderr
Loading