From 16162ff369ce359d82c865006651a75cf6d7b610 Mon Sep 17 00:00:00 2001 From: krisztian-gajdar Date: Mon, 19 Oct 2026 13:32:05 +0200 Subject: [PATCH] fix(server): declare the H100 class NV-Embed-v2 needs NV-Embed-v2 runs out of memory on an L4 and on an A100 40GB and serves on an H100. The checkpoint's code builds its Mistral body from a float32 text config, so its weights take about 28 GB on the GPU, and a forward pass under the checkpoint's float16 autocast adds about 14 GB. The compatibility matrix reads a profile's GPU class from its name, and the bare model's default profile carries none. Add an h100 profile that extends default, with the memory measured through sie-server on one H100 80GB recorded beside it, so the matrix and GPU smoke tests have a declared class for the shape. The served shape is unchanged. --- packages/sie_server/models/nvidia__NV-Embed-v2.yaml | 12 ++++++++++++ tools/ci/tests/test_compatibility_matrix.py | 1 + 2 files changed, 13 insertions(+) diff --git a/packages/sie_server/models/nvidia__NV-Embed-v2.yaml b/packages/sie_server/models/nvidia__NV-Embed-v2.yaml index bca54367d..eb258f30e 100644 --- a/packages/sie_server/models/nvidia__NV-Embed-v2.yaml +++ b/packages/sie_server/models/nvidia__NV-Embed-v2.yaml @@ -45,3 +45,15 @@ profiles: # The rendered prefix is passed as the model's `instruction`, so # is_mask_instruction excludes it from the pooling mean. query_template: "Instruct: {instruction}\nQuery: {text}" + # NV-Embed-v2 needs an 80 GB card. The checkpoint's code builds its Mistral + # body from a float32 text config, so the weights take about 28 GB on the + # GPU, and a forward pass under the checkpoint's float16 autocast adds about + # 14 GB. Measured through sie-server on one H100 80GB: 29,065 MiB after + # load, 42,993 MiB after a one-sentence query, 67,113 MiB for 32 documents + # of 642 tokens, and 75,041 MiB for one 21,122-token document; a + # 41,602-token document did not raise the peak. It does not load on a + # 24 GB L4 and runs out of memory on its first request on an A100 40GB. + # The hardware name records the tested pairing; callers and operators + # still select the machine profile. + h100: + extends: default diff --git a/tools/ci/tests/test_compatibility_matrix.py b/tools/ci/tests/test_compatibility_matrix.py index e39d32e04..64121ba99 100644 --- a/tools/ci/tests/test_compatibility_matrix.py +++ b/tools/ci/tests/test_compatibility_matrix.py @@ -122,6 +122,7 @@ def test_profile_variants_carry_their_profile_and_width(): assert (variant["model"], variant["profile"], variant["gpu_class"]) == ("Qwen/Qwen3.8-27B-FP8", "h100-256k", "h100") assert cells["cuda13-sglang-cu130", "zai-org/GLM-5.3-Flash"]["tp_width"] == 8 assert cells["cuda12-sglang", "Qwen/Qwen3-0.6B"]["profile"] == "default" + assert cells["cuda12-default", "nvidia/NV-Embed-v2:h100"]["gpu_class"] == "h100" def run(lane, route, image, result="passed", **extra):