diff --git a/CHANGELOG.md b/CHANGELOG.md index d8f7c38..fc7739d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,18 @@ the fact. ### Added +- Grace Hopper and Grace Blackwell (GH200, GB200, GB300, including the DGX + Station GB300): a mixture-of-experts model too big for the GPU's own memory + can keep part of its experts in the Grace CPU's memory. `inspect` and the + panel recommend the largest copy that fits on the GPU, and only when none + does, the largest that fits with offload. `new` gives such a profile + vLLM's `--cpu-offload-gb` and `--cpu-offload-params experts`; llama.cpp + places experts itself. `doctor` and `setup` print the CPU memory available + for it. Engines declare this in a new `offload` field. +- On a machine whose GPUs are different models, such as a DGX Station's GB300 + and its display GPU, `setup` records the GPUs with the most memory in + `config.env` as `CUDA_VISIBLE_DEVICES`, by UUID, and `doctor` warns until + it is set. - Warm-up: a profile's `WARMUP_FILE` names a chat request that llm-helper sends when `up`, `restart` or `switch` sees the server healthy, and that the router sends when a server comes back, so the server has the client's fixed prompt cached @@ -103,6 +115,10 @@ the fact. ### Changed +- Grace Hopper and Grace Blackwell GPUs are sized by their own HBM rather + than as one pool with system memory, which promised a GH200 about 480 GB + for a 96 GB GPU, and `--gpu-memory-utilization` is no longer capped on + them as on a DGX Spark. - gpt-oss profiles now carry `TIKTOKEN_ENCODINGS_BASE` themselves, and `setup` no longer writes it into `config.env`. Existing configs keep the line, and existing profiles serve as before. diff --git a/catalogue/engines/llamacpp.json b/catalogue/engines/llamacpp.json index a3040d1..0ddf582 100644 --- a/catalogue/engines/llamacpp.json +++ b/catalogue/engines/llamacpp.json @@ -39,6 +39,10 @@ "capabilities": { "gguf_quant": true }, + "offload": { + "automatic": true, + "why": "llama-server's --fit, on by default, moves a mixture-of-experts model's expert tensors to CPU memory first when the weights outgrow the GPU, then fills the GPU back up layer by layer, so a profile needs no flags." + }, "notes": { "hybrid": "llama.cpp can reuse a cached prompt on this model only up to the start of a user message, so keep the system prompt and tools identical between chats." } diff --git a/catalogue/engines/vllm.json b/catalogue/engines/vllm.json index baf3e3c..a6bf793 100644 --- a/catalogue/engines/vllm.json +++ b/catalogue/engines/vllm.json @@ -113,5 +113,17 @@ "vllm_args": true, "gguf_quant": true, "chat_template_kwargs_flag": "--default-chat-template-kwargs" + }, + "offload": { + "args": [ + "--cpu-offload-gb", + "{offload_gib}", + "--cpu-offload-params", + "experts" + ], + "env": { + "VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY": "1" + }, + "why": "--cpu-offload-gb keeps that many GiB of weights in CPU memory, which the GPU reads in place on every forward pass, and --cpu-offload-params experts limits that to expert weights, so attention and shared layers stay on the GPU. Pinning is off because on a coherent GPU pinned memory takes from GPU memory, as vLLM's own offloader notes; NVIDIA's DGX Station recipe for Nemotron 3 Ultra sets the same three." } } diff --git a/catalogue/hardware/machines/nvidia_grace_blackwell.json b/catalogue/hardware/machines/nvidia_grace_blackwell.json index ecb017c..515ed42 100644 --- a/catalogue/hardware/machines/nvidia_grace_blackwell.json +++ b/catalogue/hardware/machines/nvidia_grace_blackwell.json @@ -7,5 +7,5 @@ "gpu_name": "\\b(GB200|GB300|Grace Blackwell)\\b" }, "memory": "coherent", - "why": "The GPU has HBM of its own, with the Grace CPU's memory coherently attached over NVLink-C2C. Sized as one shared pool, as before, until that is measured on the hardware. The name matches with or without the GB200 or GB300 code, so a name that spells out Grace Blackwell lands here." + "why": "The GPU has HBM of its own, with the Grace CPU's memory coherently attached over NVLink-C2C. Weights are sized against the HBM, and a mixture-of-experts model too big for it can keep experts in the CPU's memory. The name matches with or without the GB200 or GB300 code, so a name that spells out Grace Blackwell lands here." } diff --git a/catalogue/hardware/machines/nvidia_grace_hopper.json b/catalogue/hardware/machines/nvidia_grace_hopper.json index ea04ca1..ffcd793 100644 --- a/catalogue/hardware/machines/nvidia_grace_hopper.json +++ b/catalogue/hardware/machines/nvidia_grace_hopper.json @@ -8,5 +8,5 @@ }, "memory": "coherent", "arch": "sm90", - "why": "The GPU has HBM of its own, with the Grace CPU's memory coherently attached over NVLink-C2C. Sized as one shared pool, as before, until that is measured on the hardware. The name matches with or without the GH200 code, so a name that spells out Grace Hopper lands here and gets sm90." + "why": "The GPU has HBM of its own, with the Grace CPU's memory coherently attached over NVLink-C2C. Weights are sized against the HBM, and a mixture-of-experts model too big for it can keep experts in the CPU's memory. The name matches with or without the GH200 code, so a name that spells out Grace Hopper lands here and gets sm90." } diff --git a/catalogue/hardware/reports/dgx-station-gb300.json b/catalogue/hardware/reports/dgx-station-gb300.json new file mode 100644 index 0000000..4314e6c --- /dev/null +++ b/catalogue/hardware/reports/dgx-station-gb300.json @@ -0,0 +1,23 @@ +{ + "provenance": "synthetic: not measured; a DGX Station GB300 (Dell Pro Max with GB300) from NVIDIA's published specifications: a GB300 with 252 GB of HBM3e, 496 GB of LPDDR5X, and an RTX PRO 2000 Blackwell display GPU, listed after the GB300", + "readings": { + "schema_version": 1, + "os": "linux", + "arch": "arm64", + "mem_total_bytes": 496000000000, + "nvidia_device": true, + "nvidia_smi": true, + "gpus": [ + { "index": 0, "name": "NVIDIA GB300", "mem_bytes": 269172604928, "compute_cap": "10.3", "addressing_mode": "ATS" }, + { "index": 1, "name": "NVIDIA RTX PRO 2000 Blackwell", "mem_bytes": 17103323136, "compute_cap": "12.0", "addressing_mode": "HMM" } + ] + }, + "detected": { + "hardware_id": "nvidia_grace_blackwell", + "gpu_name": "NVIDIA GB300", + "memory": "coherent", + "memory_source": "definition", + "arch": "sm103", + "cuda": true + } +} diff --git a/docs/hardware.md b/docs/hardware.md index 5c12e40..f8e3552 100644 --- a/docs/hardware.md +++ b/docs/hardware.md @@ -35,7 +35,8 @@ A definition's memory is one of three kinds: - **unified** — one pool shared with the CPU, such as a DGX Spark, a Jetson or a Mac. - **discrete** — memory of the GPU's own. -- **coherent** — memory of the GPU's own with the CPU's memory coherently attached, as on Grace Hopper and Grace Blackwell. It is sized as unified until that has been measured on the hardware. +- **coherent** — memory of the GPU's own with the CPU's memory coherently attached, as on Grace Hopper and Grace Blackwell. + Weights are sized against the GPU's own memory, and a mixture-of-experts model too big for it can keep part of its experts in the CPU's; see "Grace Hopper and Grace Blackwell" in [Memory budget](./memory.md). ## Flags that crash an architecture diff --git a/docs/memory.md b/docs/memory.md index 36b1fa8..7a04df6 100644 --- a/docs/memory.md +++ b/docs/memory.md @@ -24,6 +24,36 @@ A model larger than one GPU needs `--tensor-parallel-size` in the profile's `EXT How llm-helper decides whether a GPU has memory of its own is described in [Hardware detection](./hardware.md). +## Grace Hopper and Grace Blackwell + +A GH200, GB200 or GB300 has HBM of its own, and the Grace CPU's LPDDR5X attached over NVLink-C2C, which the GPU reads coherently. +The budget is the HBM alone, because in the driver's default memory mode CUDA allocates only from it, so that is where the weights and the KV cache go. +The CPU's memory is a second pool, and `doctor` and `setup` print how much of it a model can use on their `offload:` line: system memory, less what the OS is using and 8 GiB for the server's own process. + +The second pool is offered only to mixture-of-experts models. +Each token reads only a few of their experts, where a dense model would read every offloaded weight for every token. +`inspect` and the panel recommend the largest copy of a model that fits on the GPU. +Only when none does do they recommend the largest copy that fits with part of its experts in CPU memory, and say so. +Every copy that fits either way stays selectable. + +A profile `new` creates for such a copy keeps the shortfall, plus 4 GiB more for the KV cache, in CPU memory. +For vLLM that is `--cpu-offload-gb --cpu-offload-params experts` in `EXTRA_ARGS`, and `VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1` in the profile, which are the settings NVIDIA's DGX Station recipes use. +Pinning is off because on these machines pinned memory is taken from the GPU's. +llama.cpp's `llama-server` moves experts to CPU memory by itself when the weights outgrow the GPU, so its profile gets nothing added. +Each engine says which it does in the `offload` field of its definition; see [Providers](./providers.md). + +Offloaded experts are read at the CPU memory's bandwidth, 396 GB/s on a DGX Station GB300 against 7.1 TB/s for its HBM, so decode is slower than for a copy that fits on the GPU. +How much slower depends on the model and has not yet been measured on a GB300. + +A DGX Station also has a small display GPU, an RTX PRO 2000. +DGX OS names the GB300 in `CUDA_VISIBLE_DEVICES` in `/etc/environment`, which login shells read and services started by systemd do not. +On a machine whose GPUs are different models, `setup` writes `CUDA_VISIBLE_DEVICES` to `config.env`, naming the GPUs with the most memory by UUID, unless `config.env` already sets it. +`doctor` warns, with the line to add, until it is set. + +Older drivers default to a NUMA memory mode instead, in which the HBM is also system memory. +There the OS can fill the HBM with page cache and vLLM's free-memory check can fail at startup, so update the driver before serving. +Leave MIG off: llm-helper does not size MIG instances. + ## AMD Ryzen AI Max (Strix Halo) A Ryzen AI Max GPU (Radeon 8060S, 8050S or 8040S) has no memory of its own beyond a carve-out the firmware sets aside, 512 MB by default. diff --git a/docs/providers.md b/docs/providers.md index bbebfdd..44f7a6e 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -285,6 +285,10 @@ A definition holds `schema_version` and the engine's fields, and says: `chat_template_kwargs_flag` names the flag that sets default chat template kwargs, which is how a profile turns a model's thinking on or off; see [Model types](./model-types.md). - `notes`: optional sentences `inspect` prints for some kinds of model. `hybrid` is shown for a model whose layers are not all attention, such as one with linear-attention or Mamba layers. +- `offload`: optional, for a process engine that can keep a mixture-of-experts model's expert weights in CPU memory when they do not fit on a GPU that reads CPU memory coherently, such as a Grace Blackwell's; see "Grace Hopper and Grace Blackwell" in [Memory budget](./memory.md). + `args` are added to the profile's `EXTRA_ARGS` and must use `{offload_gib}`, the GiB to keep in CPU memory; `env` is added to the profile. + `automatic` instead says the server places the weights itself, so nothing is added. + Like everything else, it needs a `why`. The definition is checked when it loads. An unknown field, an unknown placeholder, a key that would land on the command line, or a container with no role that serves HTTP is refused with the reason. diff --git a/internal/cli/detectcmds.go b/internal/cli/detectcmds.go index 1ef0b23..17db7ef 100644 --- a/internal/cli/detectcmds.go +++ b/internal/cli/detectcmds.go @@ -174,6 +174,12 @@ func gatherDoctor() doctorReport { } r.Warnings = append(r.Warnings, hardwareWarnings(r.Hardware, running)...) r.Warnings = append(r.Warnings, gpuDeviceWarnings(r.Hardware.Accelerators)...) + configPinned := false + if cfg != nil { + configPinned = cfg.File.Has("CUDA_VISIBLE_DEVICES") || (cfg.Secrets != nil && cfg.Secrets.Has("CUDA_VISIBLE_DEVICES")) + } + shell, shellSet := os.LookupEnv("CUDA_VISIBLE_DEVICES") + r.Warnings = append(r.Warnings, gpuPinWarnings(r.Hardware, configPinned, shell, shellSet)...) if cfg != nil { r.Warnings = append(r.Warnings, serviceScopeWarnings(cfg)...) } @@ -355,6 +361,27 @@ func hardwareWarnings(hw detect.Hardware, running int) []string { return w } +// gpuPinWarnings says when a server could load onto the wrong GPU: the +// machine's GPUs are different models and config.env either names none, so +// each service sees them all, or names a mix of them. +func gpuPinWarnings(hw detect.Hardware, configPinned bool, shell string, shellSet bool) []string { + switch { + case hw.GPUPin == "": + return nil + case configPinned && !hw.SameGPUs(): + return []string{fmt.Sprintf( + "CUDA_VISIBLE_DEVICES gives the server different GPUs (%s), and it loads onto the first. Set CUDA_VISIBLE_DEVICES=%s in config.env to use the one with the most memory.", + hw.GPUSummary(), hw.GPUPin)} + case configPinned: + return nil + } + msg := "this machine has GPUs of different models and config.env does not set CUDA_VISIBLE_DEVICES, so each model's service sees all of them and loads onto the first in PCI order" + if shellSet { + msg += fmt.Sprintf(". This shell's CUDA_VISIBLE_DEVICES=%s reaches a service only if systemd sets it too", shell) + } + return []string{fmt.Sprintf("%s. Run '%s setup' to record CUDA_VISIBLE_DEVICES=%s in config.env, or add it by hand.", msg, cmdName, hw.GPUPin)} +} + // overlapWarnings names the definitions that claim one of this machine's // GPUs when more than one does. The shipped ones are tested never to // overlap, so the others come from hardware.d, and only the first is used. @@ -398,6 +425,26 @@ func overlapWarnings(reg *hardware.Registry, hw detect.Hardware) []string { return w } +// memoryKind names how the GPU's memory relates to the CPU's. +func memoryKind(hw detect.Hardware) string { + switch { + case hw.Unified: + return "unified memory" + case hw.Memory == string(hardware.MemoryCoherent): + return "its own memory, and the CPU's over NVLink-C2C" + } + return "discrete VRAM" +} + +// offloadLine says how much CPU memory a coherent GPU can keep a +// mixture-of-experts model's experts in, or "" when it cannot. +func offloadLine(hw detect.Hardware) string { + if hw.OffloadGB <= 0 { + return "" + } + return fmt.Sprintf("~%d GB more of a mixture-of-experts model's experts in CPU memory, when it does not fit the GPU (slower)", hw.OffloadGB) +} + // hardwareLine says which hardware definition recognised the GPU, its // architecture, and what decided its memory kind when no definition did. func hardwareLine(hw detect.Hardware) string { @@ -424,11 +471,7 @@ func renderDoctor(r doctorReport, yes bool) { fmt.Println("HARDWARE") fmt.Printf(" memory: %d GB total\n", hw.TotalMemGB) if hw.GPUName != "" { - kind := "discrete VRAM" - if hw.Unified { - kind = "unified memory" - } - fmt.Printf(" gpu: %s (%s)\n", hw.GPUSummary(), kind) + fmt.Printf(" gpu: %s (%s)\n", hw.GPUSummary(), memoryKind(hw)) if line := hardwareLine(hw); line != "" { fmt.Printf(" hardware: %s\n", line) } @@ -465,6 +508,9 @@ func renderDoctor(r doctorReport, yes bool) { fmt.Printf(" in use: %.1f GB by the OS and everything else running\n", hw.BaselineGB) } fmt.Printf(" budget: ~%d GB of model weights (%s)\n", hw.BudgetGB, hw.BudgetReason) + if line := offloadLine(hw); line != "" { + fmt.Printf(" offload: %s\n", line) + } // An override silently replaces the figure above everywhere else, so // showing only the measured one would make doctor disagree with search. if r.Config != nil && r.Config.BudgetOverride != "" { diff --git a/internal/cli/gpupin_internal_test.go b/internal/cli/gpupin_internal_test.go new file mode 100644 index 0000000..a5aeead --- /dev/null +++ b/internal/cli/gpupin_internal_test.go @@ -0,0 +1,113 @@ +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/detect" + "github.com/coder/llm-helper/internal/engine" + "github.com/coder/llm-helper/internal/hardware" + "github.com/coder/llm-helper/internal/hf" + "github.com/coder/llm-helper/internal/resolve" +) + +// station is a DGX Station GB300 whose display GPU the server can also see. +var station = detect.Hardware{ + GPUName: "NVIDIA GB300", Memory: string(hardware.MemoryCoherent), OffloadGB: 478, + GPUs: []detect.GPU{{Name: "NVIDIA GB300", UUID: "GPU-aaaa"}, {Name: "NVIDIA RTX PRO 2000 Blackwell", UUID: "GPU-bbbb"}}, + GPUPin: "GPU-aaaa", +} + +// A new config on a machine with different GPUs names the biggest, and a +// kept one gets it only when it names no GPUs itself. +func TestSetupPinsTheBiggestGPUWhenTheyDiffer(t *testing.T) { + prov, err := engine.Lookup(engine.VLLM) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "config.env") + if err := os.WriteFile(path, []byte(newConfigEnv(prov, "/opt/vllm/bin/vllm", "user", station)), 0o600); err != nil { + t.Fatal(err) + } + env, err := config.ParseEnvFile(path) + if err != nil { + t.Fatal(err) + } + if got := env.Get("CUDA_VISIBLE_DEVICES"); got != "GPU-aaaa" { + t.Errorf("new config CUDA_VISIBLE_DEVICES = %q, want the GB300's UUID", got) + } + if strings.Contains(newConfigEnv(prov, "/opt/vllm/bin/vllm", "user", detect.Hardware{}), "CUDA_VISIBLE_DEVICES") { + t.Error("a machine with one kind of GPU got a CUDA_VISIBLE_DEVICES line") + } + + kept := filepath.Join(t.TempDir(), "config.env") + if err := os.WriteFile(kept, []byte("PORT=8001\n"), 0o600); err != nil { + t.Fatal(err) + } + if filled, err := fillMissingGPUPin(kept, "GPU-aaaa"); err != nil || !filled { + t.Fatalf("fillMissingGPUPin = %v, %v; want filled", filled, err) + } + if filled, err := fillMissingGPUPin(kept, "GPU-cccc"); err != nil || filled { + t.Errorf("a second fill replaced the first: %v, %v", filled, err) + } + env, err = config.ParseEnvFile(kept) + if err != nil { + t.Fatal(err) + } + if got := env.Get("CUDA_VISIBLE_DEVICES"); got != "GPU-aaaa" || env.Get("PORT") != "8001" { + t.Errorf("kept config: CUDA_VISIBLE_DEVICES=%q PORT=%q", got, env.Get("PORT")) + } +} + +func TestDoctorWarnsWhenTheServerCouldPickTheWrongGPU(t *testing.T) { + t.Parallel() + if w := gpuPinWarnings(station, false, "GPU-aaaa", true); len(w) != 1 || + !strings.Contains(w[0], "CUDA_VISIBLE_DEVICES=GPU-aaaa in config.env") || !strings.Contains(w[0], "only if systemd sets it") { + t.Errorf("unpinned, with the shell's setting: %q", w) + } + if w := gpuPinWarnings(station, true, "", false); len(w) != 1 || !strings.Contains(w[0], "different GPUs") { + t.Errorf("config.env names both GPUs: %q", w) + } + pinned := station + pinned.GPUs = pinned.GPUs[:1] + if w := gpuPinWarnings(pinned, true, "", false); len(w) != 0 { + t.Errorf("config.env names the GB300 alone: %q", w) + } + if w := gpuPinWarnings(detect.Hardware{GPUName: "NVIDIA GB10"}, false, "", false); len(w) != 0 { + t.Errorf("one GPU: %q", w) + } +} + +func TestACoherentGPUIsDescribedWithBothPools(t *testing.T) { + t.Parallel() + if got := memoryKind(station); !strings.Contains(got, "NVLink-C2C") { + t.Errorf("memoryKind = %q", got) + } + if got := offloadLine(station); !strings.Contains(got, "478 GB") { + t.Errorf("offloadLine = %q", got) + } + if got := offloadLine(detect.Hardware{Unified: true}); got != "" { + t.Errorf("unified offloadLine = %q", got) + } +} + +// A mixture-of-experts model too big for a Grace GPU is shown as fitting with +// offload, and its copies say which fit how. +func TestInspectShowsAModelThatFitsWithOffload(t *testing.T) { + d := resolve.Detail{ + SizeGB: 395, BudgetGB: 244, BudgetKnown: true, FitsBudget: new(false), + MoE: true, OffloadGB: 478, FitsWithOffload: new(true), OffloadNeededGB: 151, + All: []hf.Variant{{Name: "big", Label: "BIG", SizeGB: 900}, {Name: "nvfp4", Label: "NVFP4", SizeGB: 395}}, + } + d.Best = &d.All[1] + d.Variants = 2 + out, _ := captureStdout(t, func() error { printModelDetail(d, nil); return nil }) + for _, want := range []string{"fits with ~151 GB of its experts in CPU memory", "fits with experts in CPU memory, slower", "too big for this machine"} { + if !strings.Contains(out, want) { + t.Errorf("output lacks %q:\n%s", want, out) + } + } +} diff --git a/internal/cli/search.go b/internal/cli/search.go index 84807ea..92dd828 100644 --- a/internal/cli/search.go +++ b/internal/cli/search.go @@ -166,6 +166,8 @@ func runInspect(ctx context.Context, args []string, jsonOut bool) error { fit := "" switch { case !d.BudgetKnown: + case d.FitsOffload(picked.SizeGB): + fit = fmt.Sprintf(", fits with ~%.0f GB of its experts in CPU memory", picked.SizeGB-d.BudgetGB) case picked.SizeGB > d.BudgetGB: fit = fmt.Sprintf(", too big for the ~%.0f GB budget", d.BudgetGB) default: @@ -265,6 +267,13 @@ func printModelDetail(d resolve.Detail, cfg *config.Config) { d.Best.SizeGB, display(*d.Best), d.Variants, d.BudgetGB, d.SizeGB) case *d.FitsBudget: fmt.Printf(" size: %.1f GB (fits under the ~%.0fGB budget%s)\n", d.SizeGB, d.BudgetGB, of) + case d.FitsWithOffload != nil && *d.FitsWithOffload: + size, at := d.SizeGB, "" + if d.Best != nil { + size, at = d.Best.SizeGB, " at "+display(*d.Best) + } + fmt.Printf(" size: %.1f GB%s (more than the GPU's ~%.0f GB: fits with ~%.0f GB of its experts in CPU memory, which decodes slower)\n", + size, at, d.BudgetGB, d.OffloadNeededGB) case d.Variants > 1: // Every quant was measured and the smallest still doesn't fit, so // there is no smaller one to go and look for. @@ -342,6 +351,11 @@ func printVariants(d resolve.Detail) { mark, note = " -", " needs the GGUF plugin" case d.Best != nil && v.Name == d.Best.Name: mark = " >" + if d.FitsOffload(v.SizeGB) { + note = " fits with experts in CPU memory, slower" + } + case d.FitsOffload(v.SizeGB): + note = " fits with experts in CPU memory, slower" case d.BudgetKnown && v.SizeGB > d.BudgetGB: mark, note = " x", " too big for this machine" } diff --git a/internal/cli/setup.go b/internal/cli/setup.go index ef91490..86f66fe 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -90,11 +90,7 @@ func runSetup(providerName, scope string) error { hw := detect.Probe() fmt.Printf("Hardware: %d GB memory", hw.TotalMemGB) if hw.GPUName != "" { - kind := "discrete VRAM" - if hw.Unified { - kind = "unified memory" - } - fmt.Printf(", %s (%s)", hw.GPUSummary(), kind) + fmt.Printf(", %s (%s)", hw.GPUSummary(), memoryKind(hw)) } if hw.Metal { fmt.Print(", serving on the GPU through vllm-metal") @@ -102,7 +98,11 @@ func runSetup(providerName, scope string) error { fmt.Printf(" (%d GB Metal working set)", hw.MetalWorkingSetGB) } } - fmt.Printf("\nBudget: ~%d GB of model weights (%s)\n\n", hw.BudgetGB, hw.BudgetReason) + fmt.Printf("\nBudget: ~%d GB of model weights (%s)\n", hw.BudgetGB, hw.BudgetReason) + if line := offloadLine(hw); line != "" { + fmt.Printf("Offload: %s\n", line) + } + fmt.Println() for _, d := range []string{dirs.Config, dirs.Profiles, dirs.State} { if err := os.MkdirAll(d, 0o755); err != nil { @@ -132,12 +132,20 @@ func runSetup(providerName, scope string) error { fmt.Println(note) } } + if filled, err := fillMissingGPUPin(dirs.ConfigFile(), hw.GPUPin); err != nil { + return err + } else if filled { + fmt.Printf("Recorded CUDA_VISIBLE_DEVICES=%s in %s: this machine's GPUs differ, and models load onto the one with the most memory.\n", hw.GPUPin, dirs.ConfigFile()) + } } else { content := newConfigEnv(prov, bin, scope, hw) if err := os.WriteFile(dirs.ConfigFile(), []byte(content), 0o644); err != nil { return err } fmt.Println("Wrote", dirs.ConfigFile()) + if hw.GPUPin != "" { + fmt.Printf("Set CUDA_VISIBLE_DEVICES=%s in it: this machine's GPUs differ, and models load onto the one with the most memory.\n", hw.GPUPin) + } if bin == "" { fmt.Println() if prov.Name() == engine.VLLM { @@ -248,7 +256,41 @@ SERVICE_SCOPE=%s `, strings.Join(engine.ProcessNames(), ", "), prov.Name(), prov.BinName(), prov.BinKey(), bin, config.ServiceManager, scope, hw.BudgetGB, hw.BudgetReason, hw.BudgetGB, - detect.DefaultContextTarget, detect.DefaultContextTarget) + detect.DefaultContextTarget, detect.DefaultContextTarget) + gpuPinSection(hw.GPUPin) +} + +// gpuPinSection is the config.env lines naming the GPU every model uses, on a +// machine whose GPUs differ. +func gpuPinSection(pin string) string { + if pin == "" { + return "" + } + return fmt.Sprintf(` +# This machine's GPUs are different models, so every model is served on the +# one with the most memory, named by UUID. A shell's setting reaches that +# service only if systemd sets it too. +CUDA_VISIBLE_DEVICES=%s +`, pin) +} + +// fillMissingGPUPin records pin in a config.env that names no GPUs. A value +// already there is the user's choice and is never replaced. +func fillMissingGPUPin(path, pin string) (bool, error) { + if pin == "" { + return false, nil + } + f, err := config.ParseEnvFile(path) + if err != nil { + return false, fmt.Errorf("reading %s: %w", path, err) + } + if f.Has("CUDA_VISIBLE_DEVICES") { + return false, nil + } + f.Set("CUDA_VISIBLE_DEVICES", pin) + if err := f.Write(path, 0o644); err != nil { + return false, fmt.Errorf("recording CUDA_VISIBLE_DEVICES in %s: %w", path, err) + } + return true, nil } // fillMissingVLLMBin records a vllm installed since config.env was written diff --git a/internal/detect/budget.go b/internal/detect/budget.go index 093b5d3..c5057c6 100644 --- a/internal/detect/budget.go +++ b/internal/detect/budget.go @@ -5,6 +5,8 @@ import ( "math" "strconv" "strings" + + "github.com/coder/llm-helper/internal/hardware" ) // Budget sizing, from vLLM's memory profiler across 14 starts spanning a 125M @@ -47,6 +49,9 @@ const ( // A discrete card holds only what CUDA puts there. discreteBaseline = 0.5 * gibibyte discreteRuntime = 1.5 * gibibyte + + // The server's own process lives in CPU memory beside offloaded weights. + offloadHostReserve = 8 * gibibyte ) // weightsBudget solves w + baseline + runtime + graphs + activationFraction*w @@ -141,11 +146,16 @@ func (h Hardware) Budget(usedBytes int64) (int64, string) { runtime, graphs = discreteRuntime, float64(graphReserve) baseline = discreteBaseline reason = "discrete VRAM, minus CUDA's overhead and room for a KV cache" + if h.Memory == string(hardware.MemoryCoherent) { + reason = "the GPU's own memory, not the CPU memory attached to it, minus CUDA's overhead and room for a KV cache" + } // One GPU's worth, since that is what a server without // --tensor-parallel-size uses. Saying so is the difference between // "this box is small" and "this box needs a flag". - if n := len(h.GPUs); n > 1 { + if n := len(h.GPUs); n > 1 && h.SameGPUs() { reason += fmt.Sprintf(", on one of %d GPUs; a larger model needs --tensor-parallel-size", n) + } else if n > 1 { + reason += fmt.Sprintf(", on the first of %d GPUs", n) } case poolKind == "gtt": // The GTT limit is a share of system memory, which the OS lives in @@ -187,6 +197,49 @@ func (h Hardware) Budget(usedBytes int64) (int64, string) { return b, reason } +// offloadPool is the CPU memory a coherent GPU can keep weights in when they +// do not all fit in its own: system memory, less the OS and the server's own +// process. It reads that memory over NVLink-C2C, where a PCIe card would +// read it at a fraction of the speed, so any other GPU gets 0. +func (h Hardware) offloadPool(usedBytes int64) int64 { + if h.Memory != string(hardware.MemoryCoherent) || !h.CUDA || h.GPUMemBytes <= 0 || h.AllGPUsHidden() { + return 0 + } + ram := float64(h.TotalMemBytes) + if h.MemLimited() { + ram = float64(h.MemLimitBytes) + } + baseline := defaultBaseline(ram) + if usedBytes > 0 && float64(usedBytes) < baselineCeiling*ram { + baseline = float64(usedBytes) + } + free := ram - baseline - offloadHostReserve + if free <= 0 { + return 0 + } + return int64(free) +} + +// offloadMargin is moved off the GPU beyond the shortfall, so the KV cache +// gets more than its floor. +const offloadMargin = 4 * gibibyte + +// OffloadFor reports how many GiB of a model's weights to keep in CPU memory +// so the rest fit on the GPU: the shortfall against the budget plus a margin, +// in whole GiB, the unit vLLM's --cpu-offload-gb takes. ok is false when the +// weights already fit on the GPU, or do not fit even with offload. +func (h Hardware) OffloadFor(weightBytes int64) (gib int, ok bool) { + short := weightBytes - h.BudgetBytes + if h.OffloadBytes <= 0 || h.BudgetBytes <= 0 || short <= 0 || short > h.OffloadBytes { + return 0, false + } + gib = int(math.Ceil(float64(short+offloadMargin) / gibibyte)) + if most := int(h.OffloadBytes / gibibyte); gib > most { + gib = most + } + return gib, int64(gib)*gibibyte >= short +} + // BudgetKnown reports whether the budget is a measurement. Zero can be one: // every GPU hidden, or too little memory for vLLM's own overhead. That is a // different answer from a machine whose memory could not be read at all. @@ -205,6 +258,9 @@ type WeightsBudget struct { // could not be measured. A known budget of 0 means nothing fits. Known bool Reason string + // OffloadGB is how much more CPU memory can hold for a coherent GPU, + // for models whose experts can live there; 0 elsewhere. + OffloadGB float64 } // ModelBudget reports the weights budget, honouring an override from @@ -215,7 +271,7 @@ func ModelBudget(override string) WeightsBudget { return WeightsBudget{GB: v, Known: true, Reason: "MODEL_BUDGET_GB in config.env"} } hw := Probe() - return WeightsBudget{GB: float64(hw.BudgetBytes) / gigabyte, Known: hw.BudgetKnown(), Reason: hw.BudgetReason} + return WeightsBudget{GB: float64(hw.BudgetBytes) / gigabyte, Known: hw.BudgetKnown(), Reason: hw.BudgetReason, OffloadGB: float64(hw.OffloadBytes) / gigabyte} } // overrideGB reads a configured budget, reporting false when there is none to diff --git a/internal/detect/classify.go b/internal/detect/classify.go index e187cef..2d444c7 100644 --- a/internal/detect/classify.go +++ b/internal/detect/classify.go @@ -41,6 +41,7 @@ func (f facts) describeGPUs(hw *Hardware) { } hw.GPUs = visibleGPUs(all, f.visible, f.visibleSet) hw.GPUsHidden = len(all) - len(hw.GPUs) + hw.GPUPin = gpuPin(all) } // A Jetson whose nvidia-smi lists nothing still has one GPU, and // CUDA_VISIBLE_DEVICES can hide it as surely as a listed one. @@ -76,12 +77,37 @@ func (f facts) describeGPUs(hw *Hardware) { f.describePCIGPU(reg, hw) } hw.GPUMemGB = decimalGB(hw.GPUMemBytes) - hw.Unified = hw.Memory == string(hardware.MemoryUnified) || hw.Memory == string(hardware.MemoryCoherent) + hw.Unified = hw.Memory == string(hardware.MemoryUnified) if d, ok := reg.ByID(hw.HardwareID); ok { hw.HardwareName = d.DisplayName } } +// gpuPin names the GPUs with the most memory by UUID, comma-separated, when +// the GPUs are not all one model. "" when they are, or when a UUID is missing. +func gpuPin(all []GPU) string { + if (Hardware{GPUs: all}).SameGPUs() { + return "" + } + big := all[0] + for _, g := range all[1:] { + if g.MemBytes > big.MemBytes { + big = g + } + } + var uuids []string + for _, g := range all { + if g.Name != big.Name { + continue + } + if g.UUID == "" { + return "" + } + uuids = append(uuids, g.UUID) + } + return strings.Join(uuids, ",") +} + // describePCIGPU names a GPU that no tool lists but a hardware definition // recognises on the PCI bus by its ids, such as an AMD APU under amdgpu. A // GPU no definition names is left alone: it is only listed as an diff --git a/internal/detect/coherent_internal_test.go b/internal/detect/coherent_internal_test.go new file mode 100644 index 0000000..e5fb24d --- /dev/null +++ b/internal/detect/coherent_internal_test.go @@ -0,0 +1,116 @@ +package detect + +import ( + "strings" + "testing" + + "github.com/coder/llm-helper/internal/hardware" +) + +// stationReadings is a DGX Station GB300 as the synthetic report records it. +func stationReadings(t *testing.T) Readings { + t.Helper() + for _, f := range loadFixtures(t) { + if f.name == "dgx-station-gb300.json" { + return f.Readings + } + } + t.Fatal("no dgx-station-gb300.json fixture") + return Readings{} +} + +// A GB300 is sized by its own HBM, with the Grace CPU's memory offered only +// for offloaded experts. +func TestAGraceGPUIsSizedByItsOwnMemory(t *testing.T) { + hw := FromReadings(stationReadings(t)) + if hw.Unified { + t.Fatal("a GB300 was read as unified memory") + } + if pool, kind := hw.Pool(); kind != "vram" || pool != hw.GPUMemBytes { + t.Errorf("Pool() = %d bytes of %s, want the GPU's %d bytes of vram", pool, kind, hw.GPUMemBytes) + } + if hw.BudgetBytes <= 200e9 || hw.BudgetBytes >= hw.GPUMemBytes { + t.Errorf("budget %d GB for a %d GB GPU", hw.BudgetGB, hw.GPUMemGB) + } + if !strings.Contains(hw.BudgetReason, "first of 2 GPUs") || strings.Contains(hw.BudgetReason, "--tensor-parallel-size") { + t.Errorf("reason %q should name the first of two different GPUs, not tensor parallelism", hw.BudgetReason) + } + want := hw.TotalMemBytes - int64(defaultBaseline(float64(hw.TotalMemBytes))) - offloadHostReserve + if hw.OffloadBytes != want { + t.Errorf("offload pool %d bytes, want system memory less the OS and the server: %d", hw.OffloadBytes, want) + } + if plan := hw.PlanFor(200e9, 100_000, 131072, 0); plan.KVCacheBytes <= 0 || float64(plan.KVCacheBytes)+200e9 > float64(hw.GPUMemBytes) { + t.Errorf("planned %d bytes of KV cache beside 200 GB of weights on a %d GB GPU", plan.KVCacheBytes, hw.GPUMemGB) + } +} + +// Offloading is for a GPU that reads CPU memory coherently. A PCIe card or a +// unified-memory machine has nowhere faster to put the weights. +func TestOnlyACoherentGPUOffloads(t *testing.T) { + for _, f := range loadFixtures(t) { + hw := FromReadings(f.Readings) + coherent := hw.Memory == string(hardware.MemoryCoherent) + if got := hw.OffloadBytes > 0; got != coherent { + t.Errorf("%s: offload %d bytes with %q memory", f.name, hw.OffloadBytes, hw.Memory) + } + } + hidden := Hardware{Memory: string(hardware.MemoryCoherent), CUDA: true, GPUMemBytes: bi(252), TotalMemBytes: bi(480), GPUsHidden: 1} + if got := hidden.offloadPool(0); got != 0 { + t.Errorf("offload pool %d bytes with every GPU hidden", got) + } +} + +func TestOffloadForCoversTheShortfall(t *testing.T) { + t.Parallel() + hw := Hardware{BudgetBytes: 240e9, OffloadBytes: 450e9} + tests := []struct { + name string + weights int64 + ok bool + }{ + {"fits on the GPU", 200e9, false}, + {"fits with offload", 395e9, true}, + {"too big even with offload", 700e9, false}, + } + for _, tt := range tests { + gib, ok := hw.OffloadFor(tt.weights) + if ok != tt.ok { + t.Errorf("%s: OffloadFor(%d) ok = %v, want %v", tt.name, tt.weights, ok, tt.ok) + continue + } + if !ok { + continue + } + short := tt.weights - hw.BudgetBytes + if moved := int64(gib) * gibibyte; moved < short || moved > short+offloadMargin+gibibyte { + t.Errorf("%s: %d GiB offloaded for a %d byte shortfall", tt.name, gib, short) + } + } + if gib, ok := (Hardware{BudgetBytes: 240e9, OffloadBytes: 160e9}).OffloadFor(398e9); !ok || int64(gib)*gibibyte > 160e9 { + t.Errorf("OffloadFor beyond the pool's margin = %d GiB, %v; want capped at the pool", gib, ok) + } +} + +func TestTheGPUPinNamesTheBiggestModelByUUID(t *testing.T) { + t.Parallel() + gb300 := GPU{Name: "NVIDIA GB300", MemBytes: 269e9, UUID: "GPU-aaaa"} + rtx := GPU{Name: "NVIDIA RTX PRO 2000 Blackwell", MemBytes: 17e9, UUID: "GPU-bbbb"} + h100 := GPU{Name: "NVIDIA H100", MemBytes: 85e9, UUID: "GPU-cccc"} + h100b := GPU{Name: "NVIDIA H100", MemBytes: 85e9, UUID: "GPU-dddd"} + tests := []struct { + name string + gpus []GPU + want string + }{ + {"one GPU", []GPU{gb300}, ""}, + {"all one model", []GPU{h100, h100b}, ""}, + {"display GPU listed first", []GPU{rtx, gb300}, "GPU-aaaa"}, + {"every GPU of the biggest model", []GPU{rtx, h100, h100b}, "GPU-cccc,GPU-dddd"}, + {"no UUID to name it by", []GPU{rtx, {Name: "NVIDIA GB300", MemBytes: 269e9}}, ""}, + } + for _, tt := range tests { + if got := gpuPin(tt.gpus); got != tt.want { + t.Errorf("%s: gpuPin = %q, want %q", tt.name, got, tt.want) + } + } +} diff --git a/internal/detect/hardware.go b/internal/detect/hardware.go index cd3cc1f..f4ef633 100644 --- a/internal/detect/hardware.go +++ b/internal/detect/hardware.go @@ -58,6 +58,10 @@ type Hardware struct { // GPUsHidden counts GPUs nvidia-smi listed that CUDA_VISIBLE_DEVICES // keeps from the server. GPUsHidden int `json:"gpus_hidden,omitempty"` + // GPUPin is a CUDA_VISIBLE_DEVICES value naming, by UUID, the GPUs with + // the most memory, set only when the machine's GPUs are not all the + // same model, such as a DGX Station's GB300 beside its display GPU. + GPUPin string `json:"gpu_pin,omitempty"` // HardwareID is the hardware definition that recognised the GPU, such // as "dgx_spark_gb10", and HardwareName its display name. Both are "" // when no definition did. @@ -71,7 +75,7 @@ type Hardware struct { // Arch is the GPU architecture, such as "sm121", from its compute // capability or its definition. Flags known to abort there are refused. Arch string `json:"arch,omitempty"` - Unified bool `json:"unified"` // GPU shares system memory (DGX Spark / GB10 / Grace) + Unified bool `json:"unified"` // GPU shares system memory (DGX Spark / GB10 / Jetson) // CUDA reports an NVIDIA GPU vLLM could actually use. Without one it // runs its CPU backend, which holds no CUDA context and captures no // graphs, so it is budgeted differently. @@ -107,6 +111,10 @@ type Hardware struct { BudgetBytes int64 `json:"budget_bytes"` // recommended max model weights BudgetGB int `json:"budget_gb"` // the same, for display BudgetReason string `json:"budget_reason"` // one-liner explaining the budget for the user + // OffloadBytes is how much more of a model's weights CPU memory can hold + // for a GPU it is coherently attached to, 0 for any other GPU. + OffloadBytes int64 `json:"offload_bytes,omitempty"` + OffloadGB int `json:"offload_gb,omitempty"` } // Pool reports the memory weights are actually loaded into, and which pool @@ -143,18 +151,26 @@ func (h Hardware) GPUSummary() string { if len(h.GPUs) <= 1 { return h.GPUName } + if h.SameGPUs() { + return strconv.Itoa(len(h.GPUs)) + "x " + h.GPUs[0].Name + } names := make([]string, len(h.GPUs)) - same := true for i, g := range h.GPUs { names[i] = g.Name - same = same && g.Name == h.GPUs[0].Name - } - if same { - return strconv.Itoa(len(h.GPUs)) + "x " + names[0] } return strings.Join(names, " + ") } +// SameGPUs reports whether every GPU the server can see is the same model. +func (h Hardware) SameGPUs() bool { + for _, g := range h.GPUs { + if g.Name != h.GPUs[0].Name { + return false + } + } + return true +} + // TotalMemMB reports installed RAM in mebibytes, the unit the CPU-backend // clamp works in because that is how vLLM's own memory readings arrive. func (h Hardware) TotalMemMB() int { return int(h.TotalMemBytes / (1 << 20)) } @@ -465,5 +481,7 @@ func (f facts) hardware(metal *metalState) Hardware { hw.BudgetBytes, hw.BudgetReason = hw.Budget(used) hw.BudgetGB = decimalGB(hw.BudgetBytes) + hw.OffloadBytes = hw.offloadPool(used) + hw.OffloadGB = decimalGB(hw.OffloadBytes) return hw } diff --git a/internal/detect/hardware_internal_test.go b/internal/detect/hardware_internal_test.go index 441d325..a7ab15a 100644 --- a/internal/detect/hardware_internal_test.go +++ b/internal/detect/hardware_internal_test.go @@ -11,7 +11,7 @@ import ( func unifiedFor(name string, gpuMemGB, totalMemGB int, addressing string) (bool, GPU) { g := GPU{Name: name, MemBytes: int64(gpuMemGB) * 1e9, AddressingMode: addressing} classifyGPU(hardware.Default(), &g, int64(totalMemGB)*1e9, false) - return g.Memory == string(hardware.MemoryUnified) || g.Memory == string(hardware.MemoryCoherent), g + return g.Memory == string(hardware.MemoryUnified), g } // Which machines share one memory pool with the CPU. This decides whether @@ -32,8 +32,10 @@ func TestIsUnified(t *testing.T) { // nvidia-smi reports "[N/A]" for memory.total on a GB10, so the name // is the only thing that identifies it. {name: "DGX Spark", gpuName: "NVIDIA GB10", totalMemGB: 121, want: true, source: MemoryFromDefinition}, - {name: "GH200", gpuName: "NVIDIA GH200 480GB", totalMemGB: 480, want: true, source: MemoryFromDefinition}, - {name: "Grace-Blackwell", gpuName: "NVIDIA GB200", totalMemGB: 384, want: true, source: MemoryFromDefinition}, + // Grace Hopper and Grace Blackwell have memory of their own, sized + // on its own and not capped as a shared pool. + {name: "GH200", gpuName: "NVIDIA GH200 480GB", gpuMemGB: 97, totalMemGB: 480, want: false}, + {name: "Grace-Blackwell", gpuName: "NVIDIA GB200", gpuMemGB: 186, totalMemGB: 384, want: false}, {name: "Jetson", gpuName: "Orin (nvgpu)", totalMemGB: 62, want: true, source: MemoryFromDefinition}, {name: "Thor", gpuName: "NVIDIA Thor", totalMemGB: 128, want: true, source: MemoryFromDefinition}, { @@ -96,6 +98,18 @@ func TestIsUnified(t *testing.T) { } } +// A Grace CPU's GPU is coherent: memory of its own, plus the CPU's reachable +// over NVLink-C2C. +func TestGraceGPUsAreCoherent(t *testing.T) { + t.Parallel() + for _, name := range []string{"NVIDIA GH200 480GB", "NVIDIA GB200", "NVIDIA GB300"} { + _, g := unifiedFor(name, 186, 480, "ATS") + if g.Memory != string(hardware.MemoryCoherent) || g.MemorySource != MemoryFromDefinition { + t.Errorf("%s: memory %q from %q, want coherent from the definition", name, g.Memory, g.MemorySource) + } + } +} + // A definition that names a card as discrete outranks the size heuristic, // which would otherwise read 96 GB of VRAM in a 96 GB host as a shared pool. func TestADefinitionOutranksTheSizeHeuristic(t *testing.T) { diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 80cb08c..aa68a06 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -11,6 +11,7 @@ import ( "path" "regexp" "slices" + "strconv" "strings" "github.com/coder/llm-helper/catalogue" @@ -77,6 +78,9 @@ type Engine struct { // Container describes the container runtime, which recipes launch. Container *Container `json:"container,omitempty"` Caps Capabilities `json:"capabilities"` + // Offload is how the server keeps part of a model's weights in CPU + // memory when they do not fit on the GPU, or nil when it cannot. + Offload *Offload `json:"offload,omitempty"` // Notes are sentences inspect prints about serving some kinds of model. Notes Notes `json:"notes,omitempty"` @@ -203,8 +207,28 @@ type Capabilities struct { // ChatTemplateKwargsFlag is the server flag that sets default chat // template kwargs from a JSON object, such as a model's thinking switch. ChatTemplateKwargsFlag string `json:"chat_template_kwargs_flag,omitempty"` + // WeightOffload reports whether the server can keep part of a model's + // weights in CPU memory. It follows from offload. + WeightOffload bool `json:"-"` } +// Offload is how a process-runtime server serves a mixture-of-experts model +// whose weights outgrow the GPU, from a GPU that reads CPU memory coherently. +type Offload struct { + // Args are added to the profile's EXTRA_ARGS. {offload_gib} is how many + // GiB of weights to keep in CPU memory. + Args []string `json:"args,omitempty"` + // Env is added to the profile. + Env map[string]string `json:"env,omitempty"` + // Automatic means the server places weights across GPU and CPU memory + // by itself, so a profile needs nothing added. + Automatic bool `json:"automatic,omitempty"` + Why string `json:"why"` +} + +// offloadVars are the placeholders offload args and env may use. +var offloadVars = []string{"offload_gib"} + // Notes are sentences about how the engine serves a kind of model. type Notes struct { // Hybrid is shown for models whose layers are not all attention. @@ -303,6 +327,7 @@ func (e Engine) BinName() string { // Capabilities reports what shared code has to act on. func (e Engine) Capabilities() Capabilities { c := e.Caps + c.WeightOffload = e.Offload != nil if e.Has(RuntimeProcess) { c.APIKey = e.keyMode().kind != "none" } else { @@ -314,6 +339,21 @@ func (e Engine) Capabilities() Capabilities { // HybridNote is what inspect says about serving a hybrid model. func (e Engine) HybridNote() string { return e.Notes.Hybrid } +// OffloadSettings is what a profile adds to keep gib GiB of weights in CPU +// memory: args for EXTRA_ARGS and KEY=value env lines. Both are empty for an +// engine that offloads by itself or not at all. +func (e Engine) OffloadSettings(gib int) (args, env []string) { + if e.Offload == nil || e.Offload.Automatic { + return nil, nil + } + vals := map[string]string{"offload_gib": strconv.Itoa(gib)} + args = expandAll(e.Offload.Args, vals) + for _, k := range slices.Sorted(maps.Keys(e.Offload.Env)) { + env = append(env, k+"="+expand(e.Offload.Env[k], vals)) + } + return args, env +} + // InstallHint says how to get the server on goos. func (e Engine) InstallHint(goos string) string { if h := e.Install[goos]; h != "" { @@ -891,8 +931,9 @@ func validate(e Engine, hw *hardware.Registry) []error { if len(e.Install) == 0 { add("the process runtime needs an install hint") } - } else if e.Bin != nil || len(e.Serve) > 0 || len(e.OptionalArgs) > 0 || len(e.EnvVars) > 0 || e.ModelArg != nil || e.Bind != nil { - add("bin, serve, optional_args, env, model_arg and bind belong to the process runtime, which this engine does not declare") + validateOffload(e.Offload, check, add) + } else if e.Bin != nil || len(e.Serve) > 0 || len(e.OptionalArgs) > 0 || len(e.EnvVars) > 0 || e.ModelArg != nil || e.Bind != nil || e.Offload != nil { + add("bin, serve, optional_args, env, model_arg, bind and offload belong to the process runtime, which this engine does not declare") } if e.Has(RuntimeContainer) { @@ -911,6 +952,32 @@ func validate(e Engine, hw *hardware.Registry) []error { return errs } +// validateOffload checks an offload block: a reason, and either args that +// say how much to offload or automatic, never both. +func validateOffload(o *Offload, check func(string, []string, map[string]bool), add func(string, ...any)) { + if o == nil { + return + } + if strings.TrimSpace(o.Why) == "" { + add("offload has no why") + } + switch { + case o.Automatic && (len(o.Args) > 0 || len(o.Env) > 0): + add("offload is automatic, so it takes no args or env") + case !o.Automatic && len(o.Args) == 0: + add("offload needs args, or automatic") + } + ok := map[string]bool{} + for _, v := range offloadVars { + ok[v] = true + } + check("offload.args", o.Args, ok) + if len(o.Args) > 0 && !slices.ContainsFunc(o.Args, func(t string) bool { return strings.Contains(t, "{offload_gib}") }) { + add("offload.args never use {offload_gib}, so the amount would not reach the server") + } + checkEnv("offload.env", o.Env, ok, check, add) +} + func checkEnv(where string, env map[string]string, ok map[string]bool, check func(string, []string, map[string]bool), add func(string, ...any)) { for _, k := range slices.Sorted(maps.Keys(env)) { if !envNamePattern.MatchString(k) { diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go index 8ecd8cb..3fa8274 100644 --- a/internal/engine/engine_test.go +++ b/internal/engine/engine_test.go @@ -175,6 +175,12 @@ func TestParseIsStrict(t *testing.T) { {"process engine without install hint", toyProcess, `"install": { "any": "pip install toy" },`, ``, "install hint"}, {"trailing data", toyContainer, ``, ``, "after the top-level"}, {"kwargs flag that is not a flag", toyContainer, `"capabilities": {}`, `"capabilities": {"chat_template_kwargs_flag": "kwargs"}`, "chat_template_kwargs_flag"}, + {"offload on a container engine", toyContainer, `"capabilities": {}`, `"capabilities": {}, "offload": {"automatic": true, "why": "x"}`, "offload belong to the process runtime"}, + {"offload without a why", toyProcess, `"capabilities": {}`, `"capabilities": {}, "offload": {"automatic": true}`, "offload has no why"}, + {"automatic offload with args", toyProcess, `"capabilities": {}`, `"capabilities": {}, "offload": {"automatic": true, "args": ["--x", "{offload_gib}"], "why": "x"}`, "takes no args"}, + {"offload with nothing to do", toyProcess, `"capabilities": {}`, `"capabilities": {}, "offload": {"why": "x"}`, "needs args"}, + {"offload args without the amount", toyProcess, `"capabilities": {}`, `"capabilities": {}, "offload": {"args": ["--offload"], "why": "x"}`, "never use {offload_gib}"}, + {"offload with an unknown placeholder", toyProcess, `"capabilities": {}`, `"capabilities": {}, "offload": {"args": ["--x", "{offload_gib}", "{port}"], "why": "x"}`, "{port}"}, } { data := strings.Replace(tc.base, tc.old, tc.new, 1) if tc.name == "trailing data" { @@ -338,3 +344,30 @@ func TestUseLocalSkipsAndReportsBadFiles(t *testing.T) { t.Error("UseLocal kept errors from a directory it no longer reads") } } + +// Offload settings name the amount, and an engine that places weights itself +// adds nothing. +func TestOffloadSettingsFillTheAmount(t *testing.T) { + t.Parallel() + vl, err := engine.Lookup(engine.VLLM) + if err != nil { + t.Fatal(err) + } + if !vl.Capabilities().WeightOffload { + t.Fatal("vLLM cannot offload") + } + args, env := vl.OffloadSettings(145) + if !slices.Contains(args, "145") || slices.ContainsFunc(args, func(a string) bool { return strings.Contains(a, "{") }) { + t.Errorf("args %v do not carry 145 GiB", args) + } + if len(env) == 0 { + t.Error("no env") + } + llama, err := engine.Lookup(engine.LlamaCPP) + if err != nil { + t.Fatal(err) + } + if args, env := llama.OffloadSettings(145); !llama.Capabilities().WeightOffload || len(args)+len(env) > 0 { + t.Errorf("llama.cpp offload: %v %v", args, env) + } +} diff --git a/internal/engine/registry.go b/internal/engine/registry.go index 0e00a12..e76079d 100644 --- a/internal/engine/registry.go +++ b/internal/engine/registry.go @@ -61,6 +61,9 @@ type Provider interface { // HybridNote is what inspect says about serving a hybrid model, if // anything. HybridNote() string + // OffloadSettings is what a profile adds to keep gib GiB of weights in + // CPU memory; nothing when the server does that itself or cannot. + OffloadSettings(gib int) (args, env []string) // Has reports whether the engine can run in a runtime. Has(runtime string) bool // InstallHint says how to get the server on an OS, by GOOS. diff --git a/internal/hardware/hardware.go b/internal/hardware/hardware.go index a3ed60c..8e91ec1 100644 --- a/internal/hardware/hardware.go +++ b/internal/hardware/hardware.go @@ -50,7 +50,8 @@ const ( MemoryDiscrete Memory = "discrete" // MemoryCoherent is a GPU with memory of its own that the CPU's memory // is coherently attached to, as on Grace Hopper and Grace Blackwell. - // It is sized as unified until that has been measured on the hardware. + // Weights come out of the GPU's own memory, since cudaMalloc allocates + // only there. MemoryCoherent Memory = "coherent" ) @@ -120,9 +121,9 @@ type Definition struct { } // Unified reports whether weights for this machine come out of system -// memory, which is true of a coherent pool too until that is measured. +// memory. func (d Definition) Unified() bool { - return d.Memory == MemoryUnified || d.Memory == MemoryCoherent + return d.Memory == MemoryUnified } // Match is how a definition recognises a GPU. Every rule given has to hold, diff --git a/internal/hardware/hardware_test.go b/internal/hardware/hardware_test.go index e45fa58..a03248f 100644 --- a/internal/hardware/hardware_test.go +++ b/internal/hardware/hardware_test.go @@ -17,13 +17,15 @@ import ( // of the CPU backend's live clamp (issue #13), and one wrongly called discrete // leaves a Spark uncapped. // -// Two differences are deliberate. The pattern matched anywhere in the name, so -// "Integrated" read as Tegra; the definitions match whole words. And it took +// Three differences are deliberate. The pattern matched anywhere in the name, +// so "Integrated" read as Tegra; the definitions match whole words. It took // "Grace" alone as a superchip; the definitions need Grace Hopper or Grace // Blackwell, the only GPUs Grace is paired with, so no catch-all overlaps them. +// And those superchips are coherent, not unified: weights go to their HBM. func TestShippedDefinitionsMatchTheUnifiedNamesTheyReplaced(t *testing.T) { t.Parallel() replaced := regexp.MustCompile(`(?i)\b(GB10|GB200|GB300|GH200|Grace Hopper|Grace Blackwell|Orin|Thor|Tegra)\b`) + coherent := regexp.MustCompile(`(?i)\b(GB200|GB300|GH200|Grace Hopper|Grace Blackwell)\b`) names := []string{ "NVIDIA GB10", "NVIDIA GB200", @@ -56,6 +58,12 @@ func TestShippedDefinitionsMatchTheUnifiedNamesTheyReplaced(t *testing.T) { t.Errorf("%q matches %d shipped machines; a GPU may match only one", name, len(m)) } d, ok := reg.Match(o) + if name != "" && coherent.MatchString(name) { + if !ok || d.Memory != hardware.MemoryCoherent { + t.Errorf("%q: memory %q (definition %q), want coherent", name, d.Memory, d.ID) + } + continue + } got := ok && d.Unified() want := name != "" && replaced.MatchString(name) if got != want { diff --git a/internal/resolve/offload.go b/internal/resolve/offload.go new file mode 100644 index 0000000..a641010 --- /dev/null +++ b/internal/resolve/offload.go @@ -0,0 +1,52 @@ +package resolve + +import ( + "fmt" + "strings" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/detect" + "github.com/coder/llm-helper/internal/engine" + "github.com/coder/llm-helper/internal/hf" + "github.com/coder/llm-helper/internal/vllm" +) + +// offloadPlan is how much of a model's weights a profile keeps in CPU memory. +type offloadPlan struct { + GiB int + Args []string + Env []string + Note string +} + +// planOffload decides whether a mixture-of-experts model too big for the GPU +// is served with part of its experts in CPU memory. Only a GPU that reads CPU +// memory coherently has room for that, and only an engine that can is asked. +func planOffload(hw detect.Hardware, arch hf.Arch, weights int64, prov engine.Provider, caps vllm.Caps) offloadPlan { + if prov == nil || !prov.Capabilities().WeightOffload || !arch.MoE() || weights <= hw.BudgetBytes { + return offloadPlan{} + } + gib, ok := hw.OffloadFor(weights) + if !ok { + return offloadPlan{} + } + size := fmt.Sprintf("This model is %.0f GB, more than the %d GB its GPU holds", float64(weights)/1e9, hw.BudgetGB) + args, env := prov.OffloadSettings(gib) + if len(args) == 0 { + return offloadPlan{Note: size + "; " + prov.Name() + " keeps its expert weights in CPU memory by itself, which decodes slower than from the GPU's own memory."} + } + quoted := make([]string, len(args)) + for i, a := range args { + quoted[i] = config.Quote(a) + } + flags := strings.Join(quoted, " ") + if prov.Capabilities().VLLMArgs { + if _, dropped := vllm.Check(flags, caps); len(dropped) > 0 { + return offloadPlan{Note: size + ", and the installed vLLM cannot keep experts in CPU memory: " + strings.Join(dropped, "; ")} + } + } + return offloadPlan{GiB: gib, Args: quoted, Env: env, Note: fmt.Sprintf( + "%s, so %d GiB of its expert weights stay in CPU memory (%s). "+ + "The GPU reads them from there on every token, which is slower than its own memory; a smaller copy that fits on the GPU decodes faster.", + size, gib, flags)} +} diff --git a/internal/resolve/offload_internal_test.go b/internal/resolve/offload_internal_test.go new file mode 100644 index 0000000..0a1d3f6 --- /dev/null +++ b/internal/resolve/offload_internal_test.go @@ -0,0 +1,131 @@ +package resolve + +import ( + "slices" + "strconv" + "strings" + "testing" + + "github.com/coder/llm-helper/internal/detect" + "github.com/coder/llm-helper/internal/engine" + "github.com/coder/llm-helper/internal/hardware" + "github.com/coder/llm-helper/internal/hf" + "github.com/coder/llm-helper/internal/vllm" +) + +// station is a DGX Station GB300 as detect sizes it: about 244 GB of weights +// on the GPU and 478 GB more of CPU memory for offloaded experts. +var station = detect.Hardware{ + Memory: string(hardware.MemoryCoherent), CUDA: true, + GPUMemBytes: 269172604928, TotalMemBytes: 496e9, + BudgetBytes: 244e9, BudgetGB: 244, OffloadBytes: 478e9, OffloadGB: 478, +} + +var deepseek = hf.Arch{ModelType: "deepseek_v3", NRoutedExperts: 256} + +func TestTheBestCopyFitsOnTheGPUBeforeAnyIsOffloaded(t *testing.T) { + t.Parallel() + fp8 := hf.Variant{Name: "fp8", Label: "FP8", SizeGB: 641} + nvfp4 := hf.Variant{Name: "nvfp4", Label: "NVFP4", SizeGB: 395} + q2 := hf.Variant{Name: "q2", Label: "Q2", SizeGB: 149} + tests := []struct { + name string + all []hf.Variant + size float64 + moe bool + offloadGB float64 + wantBest string + wantOffload *bool + }{ + {"one copy fits on the GPU", []hf.Variant{fp8, nvfp4, q2}, 149, true, 478, "Q2", nil}, + {"none fits on the GPU", []hf.Variant{fp8, nvfp4}, 395, true, 478, "FP8", new(true)}, + {"a dense model is not offloaded", []hf.Variant{fp8, nvfp4}, 395, false, 478, "", nil}, + {"no CPU memory to offload into", []hf.Variant{fp8, nvfp4}, 395, true, 0, "", nil}, + {"one copy, offloaded", []hf.Variant{nvfp4}, 395, true, 478, "", new(true)}, + {"one copy, too big even offloaded", []hf.Variant{{Name: "pro", SizeGB: 805}}, 805, true, 478, "", new(false)}, + } + for _, tt := range tests { + d := Detail{All: tt.all, SizeGB: tt.size, BudgetGB: 244, BudgetKnown: true, MoE: tt.moe, OffloadGB: tt.offloadGB} + d.rankVariants(true) + best := "" + if d.Best != nil { + best = d.Best.Label + } + if best != tt.wantBest { + t.Errorf("%s: best %q, want %q", tt.name, best, tt.wantBest) + } + if (d.FitsWithOffload == nil) != (tt.wantOffload == nil) || (d.FitsWithOffload != nil && *d.FitsWithOffload != *tt.wantOffload) { + t.Errorf("%s: fits with offload %v, want %v", tt.name, d.FitsWithOffload, tt.wantOffload) + } + if d.FitsWithOffload != nil && *d.FitsWithOffload && d.OffloadNeededGB <= 0 { + t.Errorf("%s: fits with offload but needs %v GB of it", tt.name, d.OffloadNeededGB) + } + } +} + +// An engine with no way to offload ranks only what fits on the GPU. +func TestAnEngineThatCannotOffloadIsNotOfferedIt(t *testing.T) { + t.Parallel() + mlx, err := engine.Lookup(engine.MLX) + if err != nil { + t.Fatal(err) + } + d := Detail{ + All: []hf.Variant{{Name: "a", Label: "A", SizeGB: 641}, {Name: "b", Label: "B", SizeGB: 395}}, + SizeGB: 395, BudgetGB: 244, BudgetKnown: true, MoE: true, OffloadGB: 478, + } + d.rankVariants(true) + got := ForProvider(d, mlx) + if got.OffloadGB != 0 || got.FitsWithOffload != nil || got.Best != nil { + t.Errorf("mlx-lm was offered offload: %+v", got) + } +} + +func TestOffloadIsPlannedForAMoEModelTooBigForTheGPU(t *testing.T) { + t.Parallel() + vl, err := engine.Lookup(engine.VLLM) + if err != nil { + t.Fatal(err) + } + off := planOffload(station, deepseek, 395e9, vl, vllm.Caps{}) + if off.GiB <= 0 || !slices.Contains(off.Args, "--cpu-offload-gb") || !slices.Contains(off.Args, "experts") { + t.Fatalf("no offload planned for a 395 GB MoE model: %+v", off) + } + if i := slices.Index(off.Args, "--cpu-offload-gb"); off.Args[i+1] != strconv.Itoa(off.GiB) { + t.Errorf("--cpu-offload-gb %s, want %d", off.Args[i+1], off.GiB) + } + if !slices.Contains(off.Env, "VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY=1") { + t.Errorf("env %v does not turn off pinning", off.Env) + } + if off.Note == "" { + t.Error("no note explaining the offload") + } + + for name, o := range map[string]offloadPlan{ + "fits on the GPU": planOffload(station, deepseek, 149e9, vl, vllm.Caps{}), + "dense": planOffload(station, hf.Arch{ModelType: "llama"}, 395e9, vl, vllm.Caps{}), + "discrete GPU": planOffload(detect.Hardware{CUDA: true, GPUMemBytes: 85e9, BudgetBytes: 70e9}, deepseek, 395e9, vl, vllm.Caps{}), + } { + if len(o.Args) > 0 || o.Note != "" { + t.Errorf("%s: planned %+v", name, o) + } + } + + old := vllm.Caps{ReasoningParsers: []string{"deepseek_r1"}, Flags: []string{"--cpu-offload-gb"}} + if o := planOffload(station, deepseek, 395e9, vl, old); len(o.Args) > 0 || !strings.Contains(o.Note, "cannot") { + t.Errorf("a vLLM without --cpu-offload-params got %+v", o) + } +} + +// llama-server places experts itself, so the profile gets a note and no flags. +func TestAnAutomaticEngineGetsOnlyANote(t *testing.T) { + t.Parallel() + llama, err := engine.Lookup(engine.LlamaCPP) + if err != nil { + t.Fatal(err) + } + off := planOffload(station, deepseek, 395e9, llama, vllm.Caps{}) + if len(off.Args) > 0 || len(off.Env) > 0 || off.Note == "" { + t.Errorf("llama.cpp offload plan %+v, want only a note", off) + } +} diff --git a/internal/resolve/resolve.go b/internal/resolve/resolve.go index 35c66d7..c523e75 100644 --- a/internal/resolve/resolve.go +++ b/internal/resolve/resolve.go @@ -76,8 +76,9 @@ type Detail struct { // since the smallest quant in a repo is also its worst. All []hf.Variant `json:"all_variants,omitempty"` // Best is the largest copy that fits the budget -- what someone would - // choose by hand, and what the verdict should be about. Absent when - // nothing fits or there is no budget to judge against. + // choose by hand, and what the verdict should be about. When none does + // and FitsWithOffload is set, it is the largest that fits with offload. + // Absent when nothing fits or there is no budget to judge against. Best *hf.Variant `json:"best_variant,omitempty"` // NeedsGGUF is true when every copy in this repo is GGUF, so the repo as // a whole is out of reach without vLLM's out-of-tree plugin. @@ -94,11 +95,22 @@ type Detail struct { BudgetGB float64 `json:"budget_gb"` // BudgetKnown separates a measured budget of 0, where nothing fits, // from no budget at all. BudgetReason says where the figure came from. - BudgetKnown bool `json:"budget_known"` - BudgetReason string `json:"budget_reason,omitempty"` - FitsBudget *bool `json:"fits_budget,omitempty"` // absent when either side is unknown - DetectedFlags string `json:"detected_flags"` - MaxModelLen string `json:"detected_max_model_len,omitempty"` + BudgetKnown bool `json:"budget_known"` + BudgetReason string `json:"budget_reason,omitempty"` + FitsBudget *bool `json:"fits_budget,omitempty"` // absent when either side is unknown + // OffloadGB is how much more of a model's weights CPU memory can hold, + // for a GPU it is coherently attached to; 0 elsewhere. + OffloadGB float64 `json:"offload_gb,omitempty"` + // MoE is set for a mixture-of-experts model, the only kind offered + // offload: a dense one would read every offloaded weight for every token. + MoE bool `json:"moe,omitempty"` + // FitsWithOffload is set for a MoE model that does not fit on the GPU, + // saying whether it does with OffloadNeededGB of its experts in CPU + // memory. That is the headline size, or Best's when there is one. + FitsWithOffload *bool `json:"fits_with_offload,omitempty"` + OffloadNeededGB float64 `json:"offload_needed_gb,omitempty"` + DetectedFlags string `json:"detected_flags"` + MaxModelLen string `json:"detected_max_model_len,omitempty"` // DroppedFlags explains anything the card asked for that the installed // vLLM won't take, so it's visible before pulling rather than as a // crash loop afterwards. @@ -149,6 +161,7 @@ func Inspect(ctx context.Context, m hf.Model, budget detect.WeightsBudget, caps BudgetGB: budget.GB, BudgetKnown: budget.Known, BudgetReason: budget.Reason, + OffloadGB: budget.OffloadGB, BaseModel: m.BaseModel(), // Only claim the plugin is missing when the registry was actually // readable. An unreadable one means "we could not ask", and refusing @@ -211,6 +224,7 @@ func Inspect(ctx context.Context, m hf.Model, budget detect.WeightsBudget, caps fits := w.SizeGB <= d.BudgetGB d.FitsBudget = &fits } + d.MoE = arch.MoE() // BestFit reads a budget of 0 as unknown and offers the smallest, // which on a machine with room for nothing would be a promise. if d.BudgetKnown && d.BudgetGB > 0 { @@ -221,9 +235,7 @@ func Inspect(ctx context.Context, m hf.Model, budget detect.WeightsBudget, caps // "we could not ask", and every other path here offers those // variants. Asking the stricter question only here would rank // nothing on a machine that is about to be offered the lot. - if best, ok := w.BestFit(d.BudgetGB, d.GGUFReady); ok && len(w.All) > 1 { - d.Best = &best - } + d.rankVariants(d.GGUFReady) } // How much context this machine can actually hold for it, which is @@ -233,7 +245,7 @@ func Inspect(ctx context.Context, m hf.Model, budget detect.WeightsBudget, caps if d.Best != nil { size = d.Best.SizeGB } - plan := detect.Probe().PlanFor(int64(size*1e9), d.KVBytesPerToken, d.MaxContext, contextTarget) + plan := detect.Probe().PlanFor(int64((size-d.OffloadNeededGB)*1e9), d.KVBytesPerToken, d.MaxContext, contextTarget) d.ContextTokens = plan.Tokens d.KVCacheGB = float64(plan.KVCacheBytes) / 1e9 } @@ -281,7 +293,13 @@ func ForProvider(d Detail, prov engine.Provider) Detail { if d.Hybrid { d.HybridNote = prov.HybridNote() } + if !caps.WeightOffload { + d.OffloadGB = 0 + } if caps.VLLMArgs { + if !caps.WeightOffload && d.BudgetKnown && d.BudgetGB > 0 { + d.rankVariants(d.GGUFReady) + } return d } // GGUFReady and ArchBlocked are about vLLM's plugin, and installing it @@ -307,12 +325,47 @@ func ForProvider(d Detail, prov engine.Provider) Detail { } } d.Best = nil - if d.BudgetGB > 0 && len(d.All) > 1 { - if best, ok := (hf.Weights{All: d.All}).BestFit(d.BudgetGB, true); ok { + if d.BudgetGB > 0 { + d.rankVariants(true) + } + return d +} + +// FitsOffload reports whether a copy of sizeGB too big for the GPU fits +// with its experts in CPU memory. +func (d Detail) FitsOffload(sizeGB float64) bool { + return d.MoE && d.OffloadGB > 0 && sizeGB > d.BudgetGB && sizeGB <= d.BudgetGB+d.OffloadGB +} + +// rankVariants picks Best and whether the model fits with offload. Best is +// the largest copy that fits on the GPU; only when none does is it the +// largest that fits with its experts in CPU memory, since the GPU reads its +// own memory many times faster. +func (d *Detail) rankVariants(ggufOK bool) { + d.Best, d.FitsWithOffload, d.OffloadNeededGB = nil, nil, 0 + w := hf.Weights{All: d.All} + many := len(d.All) > 1 + if many { + if best, ok := w.BestFit(d.BudgetGB, ggufOK); ok { d.Best = &best + return } + } else if d.SizeGB <= d.BudgetGB { + return + } + if !d.MoE || d.OffloadGB <= 0 || d.SizeGB <= 0 { + return + } + size := d.SizeGB + if best, ok := w.BestFit(d.BudgetGB+d.OffloadGB, ggufOK); ok && many { + d.Best = &best + size = best.SizeGB + } + fits := size <= d.BudgetGB+d.OffloadGB + d.FitsWithOffload = &fits + if fits { + d.OffloadNeededGB = size - d.BudgetGB } - return d } // Plan is the settings a new profile should be created with, plus what was @@ -351,6 +404,18 @@ func PrepareFor(ctx context.Context, dirs config.Dirs, prov engine.Provider, rep env, notes := otherEngineFiles(ctx, dirs.Data, prov.Name(), cfgRepo) p.Opts.Env = env p.Notes = append(p.Notes, notes...) + // Only a machine with CPU memory to offload into asks the Hub about size. + if hw := detect.Probe(); hw.OffloadBytes > 0 && prov.Capabilities().WeightOffload { + arch, _ := hf.Config(ctx, cfgRepo) + off := planOffload(hw, arch, weightBytes(ctx, repo, choice), prov, caps) + if len(off.Args) > 0 { + p.Opts.ExtraArgs = strings.Join(off.Args, " ") + p.Opts.Env = append(p.Opts.Env, off.Env...) + } + if off.Note != "" { + p.Notes = append(p.Notes, off.Note) + } + } if flag != "" { arch, _ := hf.Config(ctx, cfgRepo) p.Thinking = detectThinking(ctx, arch.ModelType, repo, cfgRepo) @@ -447,7 +512,18 @@ func PrepareProfile(ctx context.Context, dirs config.Dirs, engineID, repo string if maxLen == 0 { maxLen = profile.DefaultMaxModelLen } - cache, clamped := planCache(detect.Probe(), arch, weightBytes(ctx, repo, choice), maxLen, contextTarget) + hw := detect.Probe() + weights := weightBytes(ctx, repo, choice) + prov, _ := engine.Lookup(engineID) + off := planOffload(hw, arch, weights, prov, caps) + if len(off.Args) > 0 { + p.Opts.ExtraArgs = strings.TrimSpace(p.Opts.ExtraArgs + " " + strings.Join(off.Args, " ")) + weights -= int64(off.GiB) << 30 + } + if off.Note != "" { + p.Notes = append(p.Notes, off.Note) + } + cache, clamped := planCache(hw, arch, weights, maxLen, contextTarget) if clamped != maxLen { p.Opts.MaxModelLen = strconv.Itoa(clamped) } @@ -464,7 +540,7 @@ func PrepareProfile(ctx context.Context, dirs config.Dirs, engineID, repo string } env, notes := modelTypeFiles(dirs.Data, rules.Files) - p.Opts.Env = env + p.Opts.Env = append(env, off.Env...) p.Notes = append(p.Notes, notes...) if flags, notes := modelTypeArgs(rules.Args, caps, p.Opts.ExtraArgs); len(flags) > 0 { p.Opts.ExtraArgs = strings.TrimSpace(p.Opts.ExtraArgs + " " + strings.Join(flags, " ")) diff --git a/internal/web/static/machine.js b/internal/web/static/machine.js index fd1e9a7..319c268 100644 --- a/internal/web/static/machine.js +++ b/internal/web/static/machine.js @@ -275,6 +275,9 @@ function renderMachine(m) { el.totalLabel.textContent = measured ? "total" : m.pool; el.gpu.textContent = m.gpu_name || "none detected"; el.gpu.title = m.budget_reason || ""; + if (m.offload_gb) { + el.gpu.title += `; ~${m.offload_gb} GB more in CPU memory for mixture-of-experts weights`; + } } function renderProfiles(list) { diff --git a/internal/web/static/models.js b/internal/web/static/models.js index 83d54a0..a0be2d7 100644 --- a/internal/web/static/models.js +++ b/internal/web/static/models.js @@ -218,6 +218,17 @@ function renderVerdict(d) { `Under this machine's ~${Math.round(d.budget_gb)} GB budget${of}.`, ); } + } else if (d.fits_with_offload) { + // A mixture-of-experts model too big for a Grace GPU's own memory can + // keep part of its experts in the CPU memory the GPU reads coherently. + const best = d.best_variant; + const size = best ? best.size_gb : d.size_gb; + const at = best ? ` at ${best.display || best.label}` : ""; + set( + "yes", + `Fits with offload — ${size.toFixed(1)} GB${at}`, + `More than the GPU's ~${Math.round(d.budget_gb)} GB, so about ${Math.round(d.offload_needed_gb)} GB of its experts stay in CPU memory. That decodes slower than a copy that fits on the GPU.`, + ); } else { set( "no", @@ -227,6 +238,16 @@ function renderVerdict(d) { } } +// fitsOffload mirrors resolve.Detail.FitsOffload. +function fitsOffload(d, size) { + return ( + d.moe && + d.offload_gb > 0 && + size > d.budget_gb && + size <= d.budget_gb + d.offload_gb + ); +} + // The quants a repo holds, as a control rather than a readout: a repo with a // 2-bit and an 8-bit copy should not have that decided silently. The best fit // is pre-selected so the common case is still one click. @@ -260,6 +281,8 @@ function renderQuants(d) { } else if (v.gguf && !d.gguf_ready) { opt.disabled = true; opt.textContent = `${name} ${size} needs the GGUF plugin`; + } else if (fitsOffload(d, v.size_gb)) { + opt.textContent = `${name} ${size} fits with experts in CPU memory, slower`; } else if (d.budget_known && v.size_gb > d.budget_gb) { opt.textContent = `${name} ${size} too big for this machine`; } else { @@ -272,7 +295,7 @@ function renderQuants(d) { } el.quantNote.textContent = d.best_variant - ? `${d.best_variant.display || d.best_variant.label} is the biggest this machine can serve.` + ? `${d.best_variant.display || d.best_variant.label} is the biggest this machine can serve${fitsOffload(d, d.best_variant.size_gb) ? ", with part of its experts in CPU memory" : ""}.` : "None of these can be served on this machine."; }