From d3d839f9cbb294f6a294e0df6eaaf3ee90c9ebb8 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Mon, 19 Oct 2026 13:24:01 +0200 Subject: [PATCH] fix(server): fit the Qwen3.6-27B h100 NEXTN profile on one H100 The h100 profile could not start on an H100 80GB: SGLang exited with "Not enough memory" while sizing its pools at mem_fraction_static 0.93. NEXTN reserves intermediate mamba state for every running request and draft token, about 0.14 GB per request per token for this model. With SGLang's speculative default of 48 running requests and four draft tokens that is about 27.5 GB, more than the 26.93 GB left after the 51.05 GB of BF16 weights, so no fraction fits. Capping running requests at 8 lets the pools fit at 0.93, but the NEXTN draft then needs 5.53 GB to load (its embedding and head are allocated before they are shared) and only 4.64 GB remained. Measured on one H100 80GB with SGLang 0.5.10.post1, the profile now runs with mem_fraction_static 0.88 and --max-running-requests 8: 42 mamba slots, a 113,971-token KV pool, the draft loaded with 2.54 GB to spare, and through sie-server it served 8 concurrent requests and a 7,027-token prompt alongside 7 short ones. The model-card 3/4 draft, the 8K context and the KV budget are unchanged. The measurement is recorded in the YAML. --- .../sie_server/models/Qwen__Qwen3.6-27B.yaml | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/packages/sie_server/models/Qwen__Qwen3.6-27B.yaml b/packages/sie_server/models/Qwen__Qwen3.6-27B.yaml index d10240287..bec605887 100644 --- a/packages/sie_server/models/Qwen__Qwen3.6-27B.yaml +++ b/packages/sie_server/models/Qwen__Qwen3.6-27B.yaml @@ -138,10 +138,23 @@ profiles: # Same H100-80GB target as ``default`` with the model-card 3/4 NEXTN draft # preserved from main. Keep the public H100 context at the 8K # base-model window until a separate 32K H100 smoke lands. + # + # Memory fit, measured on one H100 80GB with SGLang 0.5.10.post1: the + # BF16 target weights take 51.05 GB of 77.98 GB. NEXTN reserves + # intermediate mamba state for every running request and draft token + # (about 0.14 GB per request per token), so SGLang's default of 48 + # running requests needs about 27.5 GB and the pool cannot be sized at + # any fraction. With 8 running requests at 0.93 the pool fits, but the + # draft model then needs 5.53 GB (its embedding and head are loaded + # before they are shared) with only 4.64 GB left. At 0.88 with 8 running + # requests: mamba cache 42 slots (6.17 GB) plus 5.16 GB of speculative + # state, a 113,971-token KV pool and the draft loaded with 2.54 GB to + # spare. It served 8 concurrent requests, and a 7,027-token prompt + # alongside 7 short ones. kv_budget_tokens: 32768 adapter_options: loadtime: - mem_fraction_static: 0.93 + mem_fraction_static: 0.88 served_model_name: Qwen/Qwen3.6-27B disable_cuda_graph: true attention_backend: triton @@ -163,6 +176,8 @@ profiles: - "--mamba-scheduler-strategy" - "extra_buffer" - "--disable-overlap-schedule" + - "--max-running-requests" + - "8" runtime: first_chunk_timeout_s: 90 inter_chunk_timeout_s: 15