diff --git a/packages/sie_server/models/nvidia__NV-Embed-v2.yaml b/packages/sie_server/models/nvidia__NV-Embed-v2.yaml index bca54367d..eb258f30e 100644 --- a/packages/sie_server/models/nvidia__NV-Embed-v2.yaml +++ b/packages/sie_server/models/nvidia__NV-Embed-v2.yaml @@ -45,3 +45,15 @@ profiles: # The rendered prefix is passed as the model's `instruction`, so # is_mask_instruction excludes it from the pooling mean. query_template: "Instruct: {instruction}\nQuery: {text}" + # NV-Embed-v2 needs an 80 GB card. The checkpoint's code builds its Mistral + # body from a float32 text config, so the weights take about 28 GB on the + # GPU, and a forward pass under the checkpoint's float16 autocast adds about + # 14 GB. Measured through sie-server on one H100 80GB: 29,065 MiB after + # load, 42,993 MiB after a one-sentence query, 67,113 MiB for 32 documents + # of 642 tokens, and 75,041 MiB for one 21,122-token document; a + # 41,602-token document did not raise the peak. It does not load on a + # 24 GB L4 and runs out of memory on its first request on an A100 40GB. + # The hardware name records the tested pairing; callers and operators + # still select the machine profile. + h100: + extends: default diff --git a/tools/ci/tests/test_compatibility_matrix.py b/tools/ci/tests/test_compatibility_matrix.py index e39d32e04..64121ba99 100644 --- a/tools/ci/tests/test_compatibility_matrix.py +++ b/tools/ci/tests/test_compatibility_matrix.py @@ -122,6 +122,7 @@ def test_profile_variants_carry_their_profile_and_width(): assert (variant["model"], variant["profile"], variant["gpu_class"]) == ("Qwen/Qwen3.8-27B-FP8", "h100-256k", "h100") assert cells["cuda13-sglang-cu130", "zai-org/GLM-5.3-Flash"]["tp_width"] == 8 assert cells["cuda12-sglang", "Qwen/Qwen3-0.6B"]["profile"] == "default" + assert cells["cuda12-default", "nvidia/NV-Embed-v2:h100"]["gpu_class"] == "h100" def run(lane, route, image, result="passed", **extra):