From dd67b708087ff2d782a4f948e45c0e6470b62456 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Thu, 8 Oct 2026 12:26:01 +0200 Subject: [PATCH] feat(server): run the remote bundle on transformers 5 so hybrid counting loads transformers-5 tokenizers A worker counts a hybrid model's upstream generation with the model's own tokenizer. Tokenizer configs in the transformers-5 format, such as GLM-5.3-Flash's (tokenizer_class: TokenizersBackend), do not load on transformers 4, so the worker fell back to the upstream's counts for them. The remote bundle now declares transformers >=5.14,<6, the pin the transformers5 bundle already carries, and the chart runs the remote lane on the cpu-transformers5 image, which provides it, instead of cpu-default. The render still refuses imageBundle: remote, and its message now names transformers5. The no-dependencies invariant for the remote bundle becomes a check that the chart's remote lane image provides every remote bundle dependency. The kind remote smoke keeps imageBundle: default: it serves a remote_only model, which needs no tokenizer. --- deploy/helm/sie-cluster/README.md | 15 ++++-- deploy/helm/sie-cluster/templates/NOTES.txt | 2 + .../templates/worker-statefulset.yaml | 2 +- deploy/helm/sie-cluster/values.yaml | 9 ++-- packages/sie_server/REMOTE_BACKENDS.md | 7 +++ packages/sie_server/bundles/remote.yaml | 7 ++- .../tests/config/test_remote_bundle.py | 6 +-- .../test_tokenizer_transformers5_format.py | 47 +++++++++++++++++++ tools/ci/kind_remote_smoke.py | 20 +++++--- tools/ci/tests/test_helm_render.py | 16 ++++++- 10 files changed, 110 insertions(+), 21 deletions(-) create mode 100644 packages/sie_server/tests/core/test_tokenizer_transformers5_format.py diff --git a/deploy/helm/sie-cluster/README.md b/deploy/helm/sie-cluster/README.md index 82d33ffb8..1dbedddc6 100644 --- a/deploy/helm/sie-cluster/README.md +++ b/deploy/helm/sie-cluster/README.md @@ -837,15 +837,24 @@ Below, `` is the chart's full name: `-sie-cluster`, or - `workers.remote.serving: false` keeps remote lanes running but refuses every remote profile and sends nothing upstream. The lanes then receive neither the upstreams file nor the credentials. -- A remote lane runs no model on an accelerator. It runs the `cpu-default` - worker image (`imageBundle: default`), which contains the remote adapters, and - the render fails when its resolved engine is not `pytorch`, its pool sets +- A remote lane runs no model on an accelerator. It runs the + `cpu-transformers5` worker image (`imageBundle: transformers5`), which + contains the remote adapters and the transformers 5 release that the `remote` + bundle requires to count a hybrid model's generation with tokenizers in the + transformers-5 format. The render fails when its resolved engine is not + `pytorch`, its pool sets `gpu.count` above 0, requests any resource other than `cpu`, `memory` and `ephemeral-storage`, or sets a `runtimeClassName`. It does not inherit `workers.common.runtimeClassName`. - The worker image must ship the `remote` bundle, which server images do from the first release that includes #492. With an older image the worker exits at start (`Bundle file not found`) and the lane restarts in a loop. +- Behaviour change: the remote lane's default image moved from `cpu-default` + to `cpu-transformers5`. An install that pins + `workers.pools.remote.bundles.remote.imageBundle: default` keeps running, + but its worker cannot load tokenizer configs in the transformers-5 format, so + a hybrid model whose tokenizer uses that format reports the upstream's counts + instead of the worker's own. - Workers read their environment at start. After rotating an upstream's Secret, restart the remote lane: `kubectl rollout restart statefulset/-worker-remote-remote -n `. diff --git a/deploy/helm/sie-cluster/templates/NOTES.txt b/deploy/helm/sie-cluster/templates/NOTES.txt index db84cad8a..f3143bb91 100644 --- a/deploy/helm/sie-cluster/templates/NOTES.txt +++ b/deploy/helm/sie-cluster/templates/NOTES.txt @@ -47,6 +47,8 @@ KEDA: enabled {{- if eq (include "sie-cluster.worker.remoteLaneEnabled" .) "true" }} Remote backends: the remote lane is the only worker that holds upstream credentials. +It runs the cpu-transformers5 image by default (it ran cpu-default before), so it can count +hybrid generation with tokenizers in the transformers-5 format. {{- if dig "remote" "networkPolicy" "enabled" true .Values.workers }} Its NetworkPolicy admits only the gateway and limits egress by address and port, not by host name. It works only with a CNI that enforces NetworkPolicy. See the chart README section "Network policy diff --git a/deploy/helm/sie-cluster/templates/worker-statefulset.yaml b/deploy/helm/sie-cluster/templates/worker-statefulset.yaml index d986d1c72..31b1cc3f6 100644 --- a/deploy/helm/sie-cluster/templates/worker-statefulset.yaml +++ b/deploy/helm/sie-cluster/templates/worker-statefulset.yaml @@ -147,7 +147,7 @@ fast at render time. {{- fail (printf "workers.pools.%s.bundles.remote: a remote lane holds upstream credentials and runs no model on an accelerator; set workers.pools.%s.gpu.count to 0" $poolName $poolName) }} {{- end }} {{- if and $remoteLane (eq $imageBundle "remote") }} -{{- fail (printf "workers.pools.%s.bundles.remote: no remote worker image is published; set imageBundle: default to run the remote adapters on the %s-default image" $poolName $platform) }} +{{- fail (printf "workers.pools.%s.bundles.remote: no remote worker image is published; set imageBundle: transformers5 to run the remote adapters on the %s-transformers5 image" $poolName $platform) }} {{- end }} {{- if $remoteLane }} {{- range $resourceKind := list "requests" "limits" }} diff --git a/deploy/helm/sie-cluster/values.yaml b/deploy/helm/sie-cluster/values.yaml index bfdd1e894..f10f7afb9 100644 --- a/deploy/helm/sie-cluster/values.yaml +++ b/deploy/helm/sie-cluster/values.yaml @@ -1226,9 +1226,10 @@ workers: maxReplicas: 3 # Remote profiles (CPU, no GPU). The only lane that receives the upstreams - # above and their credentials. It runs the cpu-default image, which - # contains the remote adapters, with the remote bundle; the server image - # must ship the remote bundle. It requests only cpu, memory and + # above and their credentials. It runs the cpu-transformers5 image, which + # contains the remote adapters and the transformers 5 release the remote + # bundle requires, with the remote bundle; the server image must ship the + # remote bundle. It requests only cpu, memory and # ephemeral-storage, sets no runtimeClassName and does not inherit # workers.common.runtimeClassName. It requires workers.networkPolicy or # workers.remote.networkPolicy. The former excludes remote lanes when the @@ -1254,7 +1255,7 @@ workers: tolerations: [] bundles: remote: - imageBundle: default + imageBundle: transformers5 minReplicas: 1 maxReplicas: 2 diff --git a/packages/sie_server/REMOTE_BACKENDS.md b/packages/sie_server/REMOTE_BACKENDS.md index 4f011aaf8..916aec307 100644 --- a/packages/sie_server/REMOTE_BACKENDS.md +++ b/packages/sie_server/REMOTE_BACKENDS.md @@ -150,6 +150,13 @@ chat request, the upstream's counts are reported instead. An `sie` upstream serves the same model and counts with its tokenizer, so its counts are reported as they are. Single-node serving reports the upstream's counts. +The `remote` bundle installs transformers 5, so the worker can load tokenizer +configs in the transformers-5 format (`tokenizer_class: TokenizersBackend`), +such as GLM-5.3-Flash's. In a cluster the chart runs the remote lane on the +`cpu-transformers5` image, which provides it. A remote lane pinned to another +image keeps serving, but a model whose tokenizer uses that format reports the +upstream's counts there. + ## OpenAI-compatible embeddings and rerank Define a `kind: openai` upstream with a base URL that includes the provider's diff --git a/packages/sie_server/bundles/remote.yaml b/packages/sie_server/bundles/remote.yaml index 24a065679..0c3502646 100644 --- a/packages/sie_server/bundles/remote.yaml +++ b/packages/sie_server/bundles/remote.yaml @@ -8,4 +8,9 @@ default: false adapters: - sie_server.adapters.remote.openai - sie_server.adapters.remote.sie -deps: {} +deps: + # The worker counts a hybrid model's upstream generation with the model's + # own tokenizer. Tokenizer configs in the transformers-5 format (for example + # GLM-5.3-Flash's TokenizersBackend) load only on transformers 5. The chart's + # remote lane runs the transformers5 image, which provides this pin. + transformers: '>=5.14,<6' diff --git a/packages/sie_server/tests/config/test_remote_bundle.py b/packages/sie_server/tests/config/test_remote_bundle.py index fdccb6aa6..f718183ca 100644 --- a/packages/sie_server/tests/config/test_remote_bundle.py +++ b/packages/sie_server/tests/config/test_remote_bundle.py @@ -5,7 +5,8 @@ credentials, so every remote adapter must be in that bundle and the bundle must outrank any other bundle that lists a remote adapter. A single server serves the default bundle when no bundle is named, so the default bundle keeps -the remote adapters too. +the remote adapters too. The chart runs the remote bundle on another bundle's +image; that image's dependencies are checked in tools/ci/tests/test_helm_render.py. """ from __future__ import annotations @@ -32,13 +33,12 @@ def remote_adapters(bundle: dict[str, Any]) -> set[str]: return {adapter for adapter in bundle.get("adapters") or [] if adapter.startswith(REMOTE_ADAPTER_PREFIX)} -def test_the_remote_bundle_lists_only_existing_remote_adapters_and_no_dependencies() -> None: +def test_the_remote_bundle_lists_only_existing_remote_adapters() -> None: remote = load_bundles()["remote"] assert remote["adapters"] assert remote_adapters(remote) == set(remote["adapters"]) assert all(importlib.util.find_spec(adapter) is not None for adapter in remote["adapters"]) - assert not remote.get("deps") def test_every_remote_adapter_in_any_bundle_is_in_the_remote_bundle() -> None: diff --git a/packages/sie_server/tests/core/test_tokenizer_transformers5_format.py b/packages/sie_server/tests/core/test_tokenizer_transformers5_format.py new file mode 100644 index 000000000..033dca789 --- /dev/null +++ b/packages/sie_server/tests/core/test_tokenizer_transformers5_format.py @@ -0,0 +1,47 @@ +"""A tokenizer config in the transformers-5 format loads through the server's loader. + +The remote bundle pins transformers 5 so that a worker can count a hybrid +model's generation with such a tokenizer. This runs only where transformers 5 +is installed, as on the image the chart runs the remote lane on. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import transformers +from sie_server.core.tokenizer import load_tokenizer +from tokenizers import Tokenizer, models, pre_tokenizers + +pytestmark = pytest.mark.skipif( + int(transformers.__version__.split(".", 1)[0]) < 5, + reason="the transformers-5 tokenizer format needs transformers 5, which the remote bundle installs", +) + + +def test_a_transformers5_format_tokenizer_loads_and_counts_a_rendered_chat(tmp_path: Path) -> None: + vocab = {"[UNK]": 0, "<|user|>": 1, "<|assistant|>": 2, "hello": 3, "world": 4} + tokenizer = Tokenizer(models.WordLevel(vocab, "[UNK]")) + tokenizer.pre_tokenizer = pre_tokenizers.Whitespace() + tokenizer.add_special_tokens(["<|user|>", "<|assistant|>"]) + tokenizer.save(str(tmp_path / "tokenizer.json")) + (tmp_path / "tokenizer_config.json").write_text( + json.dumps({"backend": "tokenizers", "tokenizer_class": "TokenizersBackend", "unk_token": "[UNK]"}), + encoding="utf-8", + ) + (tmp_path / "chat_template.jinja").write_text( + "{% for message in messages %}<|{{ message.role }}|>{{ message.content }}{% endfor %}" + "{% if add_generation_prompt %}<|assistant|>{% endif %}", + encoding="utf-8", + ) + + loaded = load_tokenizer(tmp_path) + rendered = loaded.apply_chat_template( + [{"role": "user", "content": "hello world"}], tokenize=False, add_generation_prompt=True + ) + + assert type(loaded).__name__ == "TokenizersBackend" + assert rendered == "<|user|>hello world<|assistant|>" + assert loaded.encode(rendered, add_special_tokens=False) == [1, 3, 4, 2] diff --git a/tools/ci/kind_remote_smoke.py b/tools/ci/kind_remote_smoke.py index 93b9b8588..92be626b8 100644 --- a/tools/ci/kind_remote_smoke.py +++ b/tools/ci/kind_remote_smoke.py @@ -18,13 +18,15 @@ own, separate network namespace. A successful call therefore could only have come from a process inside the worker's pod. -Chart note: the remote lane's `imageBundle` is pinned to `default`, and the -chart refuses `imageBundle: remote` at render time ("no remote worker image -is published; set imageBundle: default to run the remote adapters on the -%s-default image", templates/worker-statefulset.yaml). The default bundle -(packages/sie_server/bundles/default.yaml) lists the same remote adapters as -the remote bundle, at a higher priority, so a single built image serves both; -this script therefore builds `--bundle default`, not `--bundle remote`. +Chart note: the chart runs the remote lane on the transformers5 image by +default and refuses `imageBundle: remote` at render time ("no remote worker +image is published", templates/worker-statefulset.yaml). This smoke overrides +the lane to `imageBundle: default`: it serves only a remote_only model, which +needs no tokenizer, so it does not have to build the transformers5 image. The +default bundle (packages/sie_server/bundles/default.yaml) lists the same remote +adapters as the remote bundle, at a higher priority, so a single built image +serves both; this script therefore builds `--bundle default`, not +`--bundle remote`. Usage: mise exec -- uv run --frozen --project . --no-sync python -m tools.ci.kind_remote_smoke @@ -158,6 +160,10 @@ def repo(image: str) -> str: pools: remote: enabled: true + bundles: + remote: + # A remote_only model needs no tokenizer, so the default image serves it. + imageBundle: default gateway: image: repository: {repo(images["sie-gateway"])} diff --git a/tools/ci/tests/test_helm_render.py b/tools/ci/tests/test_helm_render.py index ff03ae0c2..28be4de15 100644 --- a/tools/ci/tests/test_helm_render.py +++ b/tools/ci/tests/test_helm_render.py @@ -1509,11 +1509,23 @@ def test_only_the_remote_lane_worker_receives_upstreams_and_credentials(tmp_path doc for doc in docs if doc["kind"] == "StatefulSet" and doc["metadata"]["name"] == REMOTE_WORKER[1] ] (worker,) = [c for c in remote_lane["spec"]["template"]["spec"]["containers"] if c["name"] == "worker"] - assert worker["image"].endswith("-cpu-default") + assert worker["image"].endswith("-cpu-transformers5") assert "--bundle=remote" in worker["args"] assert "nvidia.com/gpu" not in worker["resources"]["limits"] +def test_the_remote_lane_image_provides_every_remote_bundle_dependency() -> None: + values = yaml.safe_load((ROOT / helm.CHART_DIR / "values.yaml").read_text(encoding="utf-8")) + image_bundle = values["workers"]["pools"]["remote"]["bundles"]["remote"]["imageBundle"] + bundles = ROOT / "packages/sie_server/bundles" + remote = yaml.safe_load((bundles / "remote.yaml").read_text(encoding="utf-8")) + image = yaml.safe_load((bundles / f"{image_bundle}.yaml").read_text(encoding="utf-8")) + + assert image_bundle == "transformers5" + assert remote["deps"] + assert remote["deps"].items() <= (image.get("deps") or {}).items() + + def test_every_lane_names_a_bundle_the_server_ships(tmp_path: Path) -> None: pools = yaml.safe_load((ROOT / helm.CHART_DIR / "values.yaml").read_text(encoding="utf-8"))["workers"]["pools"] docs = rendered_documents(tmp_path, remote_pool_values(**{name: {"enabled": True} for name in pools})) @@ -1921,7 +1933,7 @@ def test_the_mcp_ingress_certificate_cannot_be_sent_upstream(tmp_path: Path) -> ), ( remote_lane(bundles={"remote": {"imageBundle": "remote", "minReplicas": 1, "maxReplicas": 1}}), - "no remote worker image is published", + "no remote worker image is published; set imageBundle: transformers5", ), ( remote_lane(engine="candle"),