diff --git a/docs/inference/set-up-llama-cpp.mdx b/docs/inference/set-up-llama-cpp.mdx index 6c0aee14291..c6fbff705ad 100644 --- a/docs/inference/set-up-llama-cpp.mdx +++ b/docs/inference/set-up-llama-cpp.mdx @@ -5,7 +5,7 @@ title: "Set Up llama.cpp" sidebar-title: "Set Up llama.cpp" description: "Attach an existing llama.cpp server or select an experimental managed llama.cpp profile." description-agent: "Sets up llama.cpp for NemoClaw. Use for an authenticated existing server or an experimental declarative managed profile." -keywords: ["nemoclaw llama.cpp", "llama.cpp dgx spark", "managed llama.cpp", "muse glimmer gguf", "nemotron gguf"] +keywords: ["nemoclaw llama.cpp", "llama.cpp dgx spark", "managed llama.cpp", "muse glimmer gguf", "nemotron gguf", "qwen3.8 gguf"] content: type: "how_to" --- @@ -88,6 +88,8 @@ That path supports operator-selected endpoints and ports without claiming llama. Use this experimental path when you want NemoClaw to manage one declarative llama.cpp recipe on one DGX Spark. The recommended recipe serves Meta Muse Glimmer 30B through the OpenAI Chat Completions API. The NVIDIA Nemotron 3 Nano 30B-A3B recipe remains available as a lower-priority profile. +The Qwen3.8 27B recipe is explicit-only: it never appears in the interactive menu and is never selected automatically. +Select it by exact recipe ID. This path is an experimental implementation. @@ -96,6 +98,7 @@ The protected qualification runner requires Docker Engine `28.3.3` or newer and That requirement supersedes older Docker 27 qualification evidence. Ordinary managed onboarding does not publish a Docker port: it retains its no-publication container contract and uses the host-owned private bridge, so this runner-specific version floor does not apply to onboarding. The Muse Glimmer profile remains Experimental after bounded physical DGX Spark qualification of text, separated reasoning, and one structured tool call. +The Qwen3.8 27B profile has not completed physical DGX Spark qualification. Before you start, confirm these prerequisites: @@ -143,6 +146,7 @@ NEMOCLAW_SANDBOX_NAME=my-assistant \ ``` Use `llama-cpp.nemotron-3-nano-30b-a3b.spark-single.v1` to select the lower-priority NVIDIA Nemotron recipe explicitly. +Use `llama-cpp.qwen3-8-27b.spark-single.v1` to select the Qwen3.8 27B recipe, which is only reachable this way. For non-interactive onboarding or an explicit `install-llama-cpp` provider request, an unset `NEMOCLAW_LLAMACPP_RECIPE` selects the unique highest-priority compatible automatic profile. If more than one compatible automatic profile has the highest priority, automatic non-interactive selection stops instead of selecting by catalog order. @@ -182,6 +186,19 @@ serve: The typed contract accepts `low`, `medium`, `high`, or `xhigh`, but the selected recipe fixes the value and does not expose an onboarding override. +The Qwen3.8 27B recipe also uses the Jinja chat template embedded in its GGUF file, and declares `xhigh`: + +```yaml +serve: + chatTemplate: model-embedded-jinja + chatTemplateArguments: + reasoningStrength: xhigh +``` + +That checkpoint's embedded template reads `reasoning_effort` rather than the `reasoning_strength` argument the runtime supplies, so it ignores the supplied value and resolves to its own `xhigh` default. +The declared value therefore records what the server serves; it does not change it. +The template accepts only `xhigh`, `medium`, and `low`, and raises an error for any other effort value. + Onboarding performs these actions: - Resolves the selected recipe and profile against a fresh host readiness report. @@ -199,6 +216,9 @@ The API key enters the container through an owner-only read-only file, not a pro The Muse Glimmer profile enables text responses, reasoning, and structured tool calls. It does not load a multimodal projector or a DFlash draft model because multimodal projection and speculative decoding remain disabled. +The Qwen3.8 27B profile enables text responses, reasoning, and structured tool calls. +Its source repository publishes multimodal projector and draft-model files, and the recipe pins neither, because multimodal projection and speculative decoding remain disabled. + Podman and Kubernetes do not provide the required host-local-inference capability for this path. Selecting either runtime fails before model acquisition or runtime mutation and never falls back to Docker. diff --git a/managed-inference/presets/llama-cpp.dgx-spark-gb10.single.qwen3-8-27b.yaml b/managed-inference/presets/llama-cpp.dgx-spark-gb10.single.qwen3-8-27b.yaml new file mode 100644 index 00000000000..aac32d12bd0 --- /dev/null +++ b/managed-inference/presets/llama-cpp.dgx-spark-gb10.single.qwen3-8-27b.yaml @@ -0,0 +1,106 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nemoclaw.nvidia.com/managed-inference/v1 +kind: ServingPreset + +metadata: + id: llama-cpp.dgx-spark-gb10.single.qwen3-8-27b + displayName: Qwen3.8 27B on one DGX Spark + supportState: experimental + +spec: + selection: explicit-only + priority: 430 + + requirements: + all: + - readiness: + scope: everyNode + kind: qualification + id: host.platform.dgx_spark + status: qualified + - readiness: + scope: everyNode + kind: capability + id: host.platform.supported + state: present + - readiness: + scope: everyNode + kind: capability + id: host.platform.dgx_spark + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.daemon_reachable + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.runtime_supported + state: present + - readiness: + scope: everyNode + kind: capability + id: host.docker.storage_compatible + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.nvidia_available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.container_toolkit_available + state: present + - readiness: + scope: everyNode + kind: capability + id: host.gpu.cdi_healthy + state: present + - readiness: + scope: everyNode + kind: observation + id: host.os.platform + comparison: + operator: equals + value: linux + - readiness: + scope: everyNode + kind: observation + id: host.os.architecture + comparison: + operator: equals + value: arm64 + - readiness: + scope: everyNode + kind: observation + id: host.docker.runtime + comparison: + operator: equals + value: docker + - readiness: + scope: everyNode + kind: observation + id: host.gpu.count + comparison: + operator: at-least + value: 1 + - readiness: + scope: everyNode + kind: observation + id: host.gpu.driver_version + comparison: + operator: version-at-least + value: 580.65.06 + + plan: + backend: install-llama-cpp + recipeRef: llama-cpp.qwen3-8-27b.spark-single.v1 diff --git a/managed-inference/recipes/llama-cpp.qwen3-8-27b.spark-single.v1.yaml b/managed-inference/recipes/llama-cpp.qwen3-8-27b.spark-single.v1.yaml new file mode 100644 index 00000000000..49a6c91c5a0 --- /dev/null +++ b/managed-inference/recipes/llama-cpp.qwen3-8-27b.spark-single.v1.yaml @@ -0,0 +1,140 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +apiVersion: nemoclaw.nvidia.com/managed-inference/v1 +kind: ServingRecipe + +metadata: + id: llama-cpp.qwen3-8-27b.spark-single.v1 + displayName: Qwen3.8 27B with llama.cpp + +spec: + backend: install-llama-cpp + providerId: llama-cpp-local + + server: + technology: llama.cpp + source: + repository: ggml-org/llama.cpp + revision: 22dc605c4ead20e36f447cc67b55ef87e523bd55 + + model: + id: unsloth/Qwen3.8-27B-GGUF + revision: 27af057ecb382ddfea5d12837360a8980560e3ed + servedName: qwen3-8-27b + files: + - path: Qwen3.8-27B-UD-Q4_K_XL.gguf + digest: sha256:3f227079003add2511437e5b1e94812e363385225bf6a9b47b0054a72bc8b01e + sizeBytes: 17559178144 + format: gguf + quantization: UD-Q4_K_XL + license: Apache-2.0 + acquisition: + ref: hugging-face-exact-file/v1 + downloaderImage: nvcr.io/nvidia/vllm@sha256:94e21552f644e0c1627464ba89d2f7a4ce7442e196f72afa0bb5d7fba23cbb03 + authentication: + mode: optional + environment: HF_TOKEN + cache: + ref: hugging-face-shared-cache/v1 + root: user-cache + reuse: verify-exact-file + sharing: host-user + cleanup: preserve + + runtime: + image: ghcr.io/ggml-org/llama.cpp@sha256:866ad568474de9e835e487ae841ad6ace1a494b5eab4f292cbd45adb6180f711 + imageDownloadSizeBytes: 2181958990 + platforms: + - linux/amd64 + - linux/arm64 + containerRuntime: docker + networkExposure: loopback + restartPolicy: unless-stopped + hosts: 1 + cuda: + baseImage: docker.io/nvidia/cuda@sha256:789e629e49401647e22b7054ae9c6c4f6427dba68010ba428deb4cc6b063676e + minimumDriverVersion: 580.65.06 + gpu: + vendor: nvidia + count: 1 + offload: full + cpuFallback: reject + resources: + memoryBytes: 51539607552 + writableStorageBytes: 42949672960 + pidsLimit: 256 + + execution: + receiptRef: llama-cpp.host-local.receipt/v1 + materializerRef: llama-cpp.host-local/v1 + lifecycleRef: llama-cpp.host-local.lifecycle/v1 + + serve: + protocol: openai-completions + authentication: bearer + port: 8081 + chatTemplate: model-embedded-jinja + # This checkpoint's embedded template reads `reasoning_effort` (default + # `xhigh`; it accepts only `xhigh`, `medium`, `low` and raises otherwise). + # The llama.cpp runtime emits `reasoning_strength`, so the template ignores + # it and resolves to its own default. `xhigh` is declared here because it is + # what the server actually serves; it is not delivered through the kwarg. + chatTemplateArguments: + reasoningStrength: xhigh + contextSize: 262144 + slots: 1 + idleSleepSeconds: -1 + batchSize: 2048 + microBatchSize: 512 + flashAttention: enabled + kvCache: + key: f16 + value: f16 + speculativeDecoding: disabled + limits: + maxRequestBodyBytes: 1048576 + maxRequestHeaderBytes: 32768 + maxOutputTokens: 4096 + requestTimeoutSeconds: 900 + shutdownTimeoutSeconds: 25 + requestGuard: + upstreamPort: 8082 + + readiness: + contractRef: llama-cpp.server-readiness/v1 + timeoutSeconds: 1800 + expectedModel: qwen3-8-27b + probeImage: nvcr.io/nvidia/vllm@sha256:94e21552f644e0c1627464ba89d2f7a4ce7442e196f72afa0bb5d7fba23cbb03 + probes: + models: true + health: true + properties: true + metrics: true + + policy: + egress: disabled + modelSource: verified-local + modelDownloads: disabled + + surfaces: + ui: disabled + slotInspection: disabled + router: disabled + mcpProxy: disabled + serverTools: disabled + agentMode: disabled + multimodalProjection: disabled + + capabilities: + agents: [] + protocols: + - openai-completions + streaming: true + toolCalls: true + structuredOutputs: true + parallelToolCalls: false + responsesApi: false + embeddings: false + reranking: false + multimodal: false diff --git a/src/lib/inference/llama-cpp/managed-selection.test.ts b/src/lib/inference/llama-cpp/managed-selection.test.ts index 3e18398fd1c..975ea50209d 100644 --- a/src/lib/inference/llama-cpp/managed-selection.test.ts +++ b/src/lib/inference/llama-cpp/managed-selection.test.ts @@ -17,6 +17,7 @@ const GENERIC_PRESET_ID = "llama-cpp.linux-amd64-nvidia.single.nemotron-3-nano-3 const SPARK_PRESET_ID = "llama-cpp.dgx-spark-gb10.single.nemotron-3-nano-30b-a3b"; const MUSE_RECIPE_ID = "llama-cpp.muse-glimmer-30b.spark-single.v1"; const MUSE_PRESET_ID = "llama-cpp.dgx-spark-gb10.single.muse-glimmer-30b"; +const QWEN_RECIPE_ID = "llama-cpp.qwen3-8-27b.spark-single.v1"; function readinessReport( preset: ManagedInferenceServingPreset, @@ -313,6 +314,21 @@ describe("managed llama.cpp selection", () => { }); }); + it("keeps the explicit-only Qwen recipe out of automatic choices but selectable by name", () => { + const { catalog, report } = fixture(); + + expect( + listManagedLlamaCppSelectionChoices(catalog, report).map( + ({ selection }) => selection.recipe.metadata.id, + ), + ).not.toContain(QWEN_RECIPE_ID); + + expect( + resolveManagedLlamaCppSelection({ [LLAMA_CPP_RECIPE_ENV]: QWEN_RECIPE_ID }, catalog, report) + .kind, + ).toBe("selected"); + }); + it("selects an explicitly named shipped recipe", () => { const { catalog, report } = fixture();