feat(server): run the remote bundle on transformers 5 so hybrid counting loads transformers-5 tokenizers

deploy/helm/sie-cluster/README.md

@@ -837,15 +837,24 @@ Below, `<fullname>` is the chart's full name: `<release>-sie-cluster`, or
 - `workers.remote.serving: false` keeps remote lanes running but refuses every
   remote profile and sends nothing upstream. The lanes then receive neither
   the upstreams file nor the credentials.
-- A remote lane runs no model on an accelerator. It runs the `cpu-default`
-  worker image (`imageBundle: default`), which contains the remote adapters, and
-  the render fails when its resolved engine is not `pytorch`, its pool sets
+- A remote lane runs no model on an accelerator. It runs the
+  `cpu-transformers5` worker image (`imageBundle: transformers5`), which
+  contains the remote adapters and the transformers 5 release that the `remote`
+  bundle requires to count a hybrid model's generation with tokenizers in the
+  transformers-5 format. The render fails when its resolved engine is not
+  `pytorch`, its pool sets
   `gpu.count` above 0, requests any resource other than `cpu`, `memory` and
   `ephemeral-storage`, or sets a `runtimeClassName`. It does not inherit
   `workers.common.runtimeClassName`.
 - The worker image must ship the `remote` bundle, which server images do from
   the first release that includes #492. With an older image the worker exits
   at start (`Bundle file not found`) and the lane restarts in a loop.
+- Behaviour change: the remote lane's default image moved from `cpu-default`
+  to `cpu-transformers5`. An install that pins
+  `workers.pools.remote.bundles.remote.imageBundle: default` keeps running,
+  but its worker cannot load tokenizer configs in the transformers-5 format, so
+  a hybrid model whose tokenizer uses that format reports the upstream's counts
+  instead of the worker's own.
 - Workers read their environment at start. After rotating an upstream's
   Secret, restart the remote lane:
   `kubectl rollout restart statefulset/<fullname>-worker-remote-remote -n <NAMESPACE>`.

deploy/helm/sie-cluster/templates/NOTES.txt

@@ -47,6 +47,8 @@ KEDA:       enabled
 {{- if eq (include "sie-cluster.worker.remoteLaneEnabled" .) "true" }}
 
 Remote backends: the remote lane is the only worker that holds upstream credentials.
+It runs the cpu-transformers5 image by default (it ran cpu-default before), so it can count
+hybrid generation with tokenizers in the transformers-5 format.
 {{- if dig "remote" "networkPolicy" "enabled" true .Values.workers }}
 Its NetworkPolicy admits only the gateway and limits egress by address and port, not by host name.
 It works only with a CNI that enforces NetworkPolicy. See the chart README section "Network policy

deploy/helm/sie-cluster/templates/worker-statefulset.yaml

@@ -147,7 +147,7 @@ fast at render time.
 {{- fail (printf "workers.pools.%s.bundles.remote: a remote lane holds upstream credentials and runs no model on an accelerator; set workers.pools.%s.gpu.count to 0" $poolName $poolName) }}
 {{- end }}
 {{- if and $remoteLane (eq $imageBundle "remote") }}
-{{- fail (printf "workers.pools.%s.bundles.remote: no remote worker image is published; set imageBundle: default to run the remote adapters on the %s-default image" $poolName $platform) }}
+{{- fail (printf "workers.pools.%s.bundles.remote: no remote worker image is published; set imageBundle: transformers5 to run the remote adapters on the %s-transformers5 image" $poolName $platform) }}
 {{- end }}
 {{- if $remoteLane }}
 {{- range $resourceKind := list "requests" "limits" }}

deploy/helm/sie-cluster/values.yaml

@@ -1226,9 +1226,10 @@ workers:
           maxReplicas: 3
 
     # Remote profiles (CPU, no GPU). The only lane that receives the upstreams
-    # above and their credentials. It runs the cpu-default image, which
-    # contains the remote adapters, with the remote bundle; the server image
-    # must ship the remote bundle. It requests only cpu, memory and
+    # above and their credentials. It runs the cpu-transformers5 image, which
+    # contains the remote adapters and the transformers 5 release the remote
+    # bundle requires, with the remote bundle; the server image must ship the
+    # remote bundle. It requests only cpu, memory and
     # ephemeral-storage, sets no runtimeClassName and does not inherit
     # workers.common.runtimeClassName. It requires workers.networkPolicy or
     # workers.remote.networkPolicy. The former excludes remote lanes when the
@@ -1254,7 +1255,7 @@ workers:
       tolerations: []
       bundles:
         remote:
-          imageBundle: default
+          imageBundle: transformers5
           minReplicas: 1
           maxReplicas: 2
 

packages/sie_server/REMOTE_BACKENDS.md

@@ -150,6 +150,13 @@ chat request, the upstream's counts are reported instead. An `sie` upstream
 serves the same model and counts with its tokenizer, so its counts are reported
 as they are. Single-node serving reports the upstream's counts.
 
+The `remote` bundle installs transformers 5, so the worker can load tokenizer
+configs in the transformers-5 format (`tokenizer_class: TokenizersBackend`),
+such as GLM-5.3-Flash's. In a cluster the chart runs the remote lane on the
+`cpu-transformers5` image, which provides it. A remote lane pinned to another
+image keeps serving, but a model whose tokenizer uses that format reports the
+upstream's counts there.
+
 ## OpenAI-compatible embeddings and rerank
 
 Define a `kind: openai` upstream with a base URL that includes the provider's

packages/sie_server/bundles/remote.yaml

@@ -8,4 +8,9 @@ default: false
 adapters:
 - sie_server.adapters.remote.openai
 - sie_server.adapters.remote.sie
-deps: {}
+deps:
+  # The worker counts a hybrid model's upstream generation with the model's
+  # own tokenizer. Tokenizer configs in the transformers-5 format (for example
+  # GLM-5.3-Flash's TokenizersBackend) load only on transformers 5. The chart's
+  # remote lane runs the transformers5 image, which provides this pin.
+  transformers: '>=5.14,<6'

packages/sie_server/tests/config/test_remote_bundle.py

@@ -5,7 +5,8 @@
 credentials, so every remote adapter must be in that bundle and the bundle
 must outrank any other bundle that lists a remote adapter. A single server
 serves the default bundle when no bundle is named, so the default bundle keeps
-the remote adapters too.
+the remote adapters too. The chart runs the remote bundle on another bundle's
+image; that image's dependencies are checked in tools/ci/tests/test_helm_render.py.
 """
 
 from __future__ import annotations
@@ -32,13 +33,12 @@ def remote_adapters(bundle: dict[str, Any]) -> set[str]:
     return {adapter for adapter in bundle.get("adapters") or [] if adapter.startswith(REMOTE_ADAPTER_PREFIX)}
 
 
-def test_the_remote_bundle_lists_only_existing_remote_adapters_and_no_dependencies() -> None:
+def test_the_remote_bundle_lists_only_existing_remote_adapters() -> None:
     remote = load_bundles()["remote"]
 
     assert remote["adapters"]
     assert remote_adapters(remote) == set(remote["adapters"])
     assert all(importlib.util.find_spec(adapter) is not None for adapter in remote["adapters"])
-    assert not remote.get("deps")
 
 
 def test_every_remote_adapter_in_any_bundle_is_in_the_remote_bundle() -> None:

packages/sie_server/tests/core/test_tokenizer_transformers5_format.py

@@ -0,0 +1,47 @@
+"""A tokenizer config in the transformers-5 format loads through the server's loader.
+
+The remote bundle pins transformers 5 so that a worker can count a hybrid
+model's generation with such a tokenizer. This runs only where transformers 5
+is installed, as on the image the chart runs the remote lane on.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+import transformers
+from sie_server.core.tokenizer import load_tokenizer
+from tokenizers import Tokenizer, models, pre_tokenizers
+
+pytestmark = pytest.mark.skipif(
+    int(transformers.__version__.split(".", 1)[0]) < 5,
+    reason="the transformers-5 tokenizer format needs transformers 5, which the remote bundle installs",
+)
+
+
+def test_a_transformers5_format_tokenizer_loads_and_counts_a_rendered_chat(tmp_path: Path) -> None:
+    vocab = {"[UNK]": 0, "<|user|>": 1, "<|assistant|>": 2, "hello": 3, "world": 4}
+    tokenizer = Tokenizer(models.WordLevel(vocab, "[UNK]"))
+    tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
+    tokenizer.add_special_tokens(["<|user|>", "<|assistant|>"])
+    tokenizer.save(str(tmp_path / "tokenizer.json"))
+    (tmp_path / "tokenizer_config.json").write_text(
+        json.dumps({"backend": "tokenizers", "tokenizer_class": "TokenizersBackend", "unk_token": "[UNK]"}),
+        encoding="utf-8",
+    )
+    (tmp_path / "chat_template.jinja").write_text(
+        "{% for message in messages %}<|{{ message.role }}|>{{ message.content }}{% endfor %}"
+        "{% if add_generation_prompt %}<|assistant|>{% endif %}",
+        encoding="utf-8",
+    )
+
+    loaded = load_tokenizer(tmp_path)
+    rendered = loaded.apply_chat_template(
+        [{"role": "user", "content": "hello world"}], tokenize=False, add_generation_prompt=True
+    )
+
+    assert type(loaded).__name__ == "TokenizersBackend"
+    assert rendered == "<|user|>hello world<|assistant|>"
+    assert loaded.encode(rendered, add_special_tokens=False) == [1, 3, 4, 2]

tools/ci/kind_remote_smoke.py

@@ -18,13 +18,15 @@
 own, separate network namespace. A successful call therefore could only have
 come from a process inside the worker's pod.
 
-Chart note: the remote lane's `imageBundle` is pinned to `default`, and the
-chart refuses `imageBundle: remote` at render time ("no remote worker image
-is published; set imageBundle: default to run the remote adapters on the
-%s-default image", templates/worker-statefulset.yaml). The default bundle
-(packages/sie_server/bundles/default.yaml) lists the same remote adapters as
-the remote bundle, at a higher priority, so a single built image serves both;
-this script therefore builds `--bundle default`, not `--bundle remote`.
+Chart note: the chart runs the remote lane on the transformers5 image by
+default and refuses `imageBundle: remote` at render time ("no remote worker
+image is published", templates/worker-statefulset.yaml). This smoke overrides
+the lane to `imageBundle: default`: it serves only a remote_only model, which
+needs no tokenizer, so it does not have to build the transformers5 image. The
+default bundle (packages/sie_server/bundles/default.yaml) lists the same remote
+adapters as the remote bundle, at a higher priority, so a single built image
+serves both; this script therefore builds `--bundle default`, not
+`--bundle remote`.
 
 Usage:
     mise exec -- uv run --frozen --project . --no-sync python -m tools.ci.kind_remote_smoke
@@ -158,6 +160,10 @@ def repo(image: str) -> str:
   pools:
     remote:
       enabled: true
+      bundles:
+        remote:
+          # A remote_only model needs no tokenizer, so the default image serves it.
+          imageBundle: default
 gateway:
   image:
     repository: {repo(images["sie-gateway"])}

tools/ci/tests/test_helm_render.py

@@ -1509,11 +1509,23 @@ def test_only_the_remote_lane_worker_receives_upstreams_and_credentials(tmp_path
         doc for doc in docs if doc["kind"] == "StatefulSet" and doc["metadata"]["name"] == REMOTE_WORKER[1]
     ]
     (worker,) = [c for c in remote_lane["spec"]["template"]["spec"]["containers"] if c["name"] == "worker"]
-    assert worker["image"].endswith("-cpu-default")
+    assert worker["image"].endswith("-cpu-transformers5")
     assert "--bundle=remote" in worker["args"]
     assert "nvidia.com/gpu" not in worker["resources"]["limits"]
 
 
+def test_the_remote_lane_image_provides_every_remote_bundle_dependency() -> None:
+    values = yaml.safe_load((ROOT / helm.CHART_DIR / "values.yaml").read_text(encoding="utf-8"))
+    image_bundle = values["workers"]["pools"]["remote"]["bundles"]["remote"]["imageBundle"]
+    bundles = ROOT / "packages/sie_server/bundles"
+    remote = yaml.safe_load((bundles / "remote.yaml").read_text(encoding="utf-8"))
+    image = yaml.safe_load((bundles / f"{image_bundle}.yaml").read_text(encoding="utf-8"))
+
+    assert image_bundle == "transformers5"
+    assert remote["deps"]
+    assert remote["deps"].items() <= (image.get("deps") or {}).items()
+
+
 def test_every_lane_names_a_bundle_the_server_ships(tmp_path: Path) -> None:
     pools = yaml.safe_load((ROOT / helm.CHART_DIR / "values.yaml").read_text(encoding="utf-8"))["workers"]["pools"]
     docs = rendered_documents(tmp_path, remote_pool_values(**{name: {"enabled": True} for name in pools}))
@@ -1921,7 +1933,7 @@ def test_the_mcp_ingress_certificate_cannot_be_sent_upstream(tmp_path: Path) ->
         ),
         (
             remote_lane(bundles={"remote": {"imageBundle": "remote", "minReplicas": 1, "maxReplicas": 1}}),
-            "no remote worker image is published",
+            "no remote worker image is published; set imageBundle: transformers5",
         ),
         (
             remote_lane(engine="candle"),