From 1ca011e957735b7fc708ecc0aa57f0f7b534a5a4 Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Fri, 13 Nov 2026 20:53:12 +0000 Subject: [PATCH 01/15] hardware: match a machine by PCI device id amdgpu gives a GPU no name: an AMD device is only its PCI vendor and device ids, so a machine file had no way to recognise one, and an AMD APU that shares system memory was sized as a CPU-only box. A machine's match can now list pci_devices, which must hold together with its vendor (device ids are only unique within one) and with gpu_name when both are given. Matching stays data: no GPU is named in Go. --- catalogue/hardware/machines/README.md | 3 +- docs/hardware.md | 3 +- internal/hardware/hardware.go | 42 ++++++++++++++++++++++----- internal/hardware/hardware_test.go | 39 +++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 9 deletions(-) diff --git a/catalogue/hardware/machines/README.md b/catalogue/hardware/machines/README.md index 49eb295..a1966bc 100644 --- a/catalogue/hardware/machines/README.md +++ b/catalogue/hardware/machines/README.md @@ -2,11 +2,12 @@ Each file here is one kind of machine, named after its id, such as `dgx_spark_gb10.json`. A machine matches a GPU by vendor and by a regular expression over the name the GPU reports, case-insensitively. +A GPU whose driver reports no name, such as an AMD GPU under `amdgpu`, matches by vendor and by a list of PCI device ids in `pci_devices`, such as `0x1586`. It says how the GPU's memory relates to the CPU's: `unified`, `discrete` or `coherent`. Recipes record a machine's id as the hardware they were measured on, so an id never changes once shipped. Add one when llm-helper gets a GPU wrong without it, for example a GPU that shares system memory but reports none of its own. -Keep `gpu_name` narrow enough to match only your GPU, because no two machines may match the same one. +Keep `gpu_name` or `pci_devices` narrow enough to match only your GPU, because no two machines may match the same one. Describe one chip family, not a catch-all: a GPU no machine names already gets its architecture from its compute capability and its memory from what it reports. Give every claim a `why`, and add a report to [`reports/`](../reports) that detects the machine. diff --git a/docs/hardware.md b/docs/hardware.md index 35e5257..bd889ca 100644 --- a/docs/hardware.md +++ b/docs/hardware.md @@ -55,7 +55,8 @@ Each file is named after the id it defines, starts with the format version this `llm-helper hardware list` shows the machines in effect, including any added locally. A machine matches a vendor and a regular expression over the GPU's name, case-insensitively. -No two machines may match the same GPU, so keep the pattern narrow enough to match only yours; the tests run every report against every machine and fail on an overlap. +A GPU that its driver gives no name, such as an AMD GPU under `amdgpu`, is matched by its vendor and its PCI device ids instead, with `"pci_devices": ["0x1586"]`; `llm-helper hardware report` prints them. +No two machines may match the same GPU, so keep the pattern or the list narrow enough to match only yours; the tests run every report against every machine and fail on an overlap. Every definition needs a `why`, because an entry nobody can explain cannot be reviewed, and cannot be removed when the reason stops holding. ## Local definitions diff --git a/internal/hardware/hardware.go b/internal/hardware/hardware.go index b8ee8ba..a3ed60c 100644 --- a/internal/hardware/hardware.go +++ b/internal/hardware/hardware.go @@ -125,13 +125,19 @@ func (d Definition) Unified() bool { return d.Memory == MemoryUnified || d.Memory == MemoryCoherent } -// Match is how a definition recognises a GPU. Every rule given has to hold. +// Match is how a definition recognises a GPU. Every rule given has to hold, +// and a definition needs a gpu_name or pci_devices. type Match struct { // Vendor is a vendor id, such as "nvidia", or "" for any vendor. Vendor string `json:"vendor,omitempty"` // GPUName is a regular expression matched case-insensitively anywhere // in the name the probe reported, such as "NVIDIA GB10". GPUName string `json:"gpu_name"` + // PCIDevices are PCI device ids, such as "0x1586", for a GPU found on + // the PCI bus that no driver gives a name: an AMD GPU under amdgpu + // reports only its ids. Device ids are only unique within a vendor, so + // a definition that lists them names its vendor too. + PCIDevices []string `json:"pci_devices,omitempty"` } // Observed is what a probe saw of one GPU, which is all a definition is @@ -139,6 +145,9 @@ type Match struct { type Observed struct { Vendor string // vendor id, e.g. "nvidia" GPUName string + // PCIDevice is the GPU's PCI device id, such as "0x1586", or "" when + // the GPU was not found on the PCI bus. + PCIDevice string } // Kind is one kind of hardware entry. Each kind is a folder of its own, in @@ -355,8 +364,18 @@ func build(f set) (*Registry, error) { if d.Match.Vendor != "" && !vendors[d.Match.Vendor] { errs = append(errs, fmt.Errorf("definition %q names vendor %q, which is not defined", d.ID, d.Match.Vendor)) } + for _, id := range d.Match.PCIDevices { + if !pciIDPattern.MatchString(id) { + errs = append(errs, fmt.Errorf("definition %q: PCI device %q must look like 0x1586", d.ID, id)) + } + } + if len(d.Match.PCIDevices) > 0 && d.Match.Vendor == "" { + errs = append(errs, fmt.Errorf("definition %q matches PCI devices but names no vendor; device ids are only unique within one", d.ID)) + } if d.Match.GPUName == "" { - errs = append(errs, fmt.Errorf("definition %q has no match rule", d.ID)) + if len(d.Match.PCIDevices) == 0 { + errs = append(errs, fmt.Errorf("definition %q has no match rule", d.ID)) + } continue } re, err := regexp.Compile("(?i)" + d.Match.GPUName) @@ -474,8 +493,9 @@ func (r *Registry) ByID(id string) (Definition, bool) { } // Match returns the definition that recognises a GPU: the first one, with -// shipped machines tried before local ones. An empty name matches nothing: a -// GPU nobody could describe is not evidence of any particular machine. +// shipped machines tried before local ones. A GPU with neither a name nor a +// PCI device id matches nothing: a GPU nobody could describe is not evidence +// of any particular machine. func (r *Registry) Match(o Observed) (Definition, bool) { if m := r.Matches(o); len(m) > 0 { return m[0], true @@ -487,7 +507,8 @@ func (r *Registry) Match(o Observed) (Definition, bool) { // tries them. More than one means two definitions claim the same GPU. The // tests keep the shipped ones from ever doing that, so only a local file can. func (r *Registry) Matches(o Observed) []Definition { - if o.GPUName == "" { + device := strings.ToLower(strings.TrimSpace(o.PCIDevice)) + if o.GPUName == "" && device == "" { return nil } var out []Definition @@ -495,9 +516,16 @@ func (r *Registry) Matches(o Observed) []Definition { if d.Match.Vendor != "" && d.Match.Vendor != o.Vendor { continue } - if d.gpuName != nil && d.gpuName.MatchString(o.GPUName) { - out = append(out, d) + if d.gpuName != nil && (o.GPUName == "" || !d.gpuName.MatchString(o.GPUName)) { + continue + } + if len(d.Match.PCIDevices) > 0 && (device == "" || !slices.Contains(d.Match.PCIDevices, device)) { + continue + } + if d.gpuName == nil && len(d.Match.PCIDevices) == 0 { + continue } + out = append(out, d) } return out } diff --git a/internal/hardware/hardware_test.go b/internal/hardware/hardware_test.go index 8192dc1..e45fa58 100644 --- a/internal/hardware/hardware_test.go +++ b/internal/hardware/hardware_test.go @@ -193,6 +193,8 @@ func TestLoadRejectsInvalidEntries(t *testing.T) { {"no memory", box(`"display_name":"Box","match":{"gpu_name":"Box"},"why":"w"`), "unified, discrete or coherent"}, {"bad pattern", box(`"display_name":"Box","match":{"gpu_name":"("},"memory":"unified","why":"w"`), "match.gpu_name"}, {"no match rule", box(`"display_name":"Box","match":{},"memory":"unified","why":"w"`), "no match rule"}, + {"bad PCI device", box(`"display_name":"Box","match":{"vendor":"acme","pci_devices":["1586"]},"memory":"unified","why":"w"`), "must look like 0x1586"}, + {"PCI devices without a vendor", box(`"display_name":"Box","match":{"pci_devices":["0x1586"]},"memory":"unified","why":"w"`), "names no vendor"}, {"no why", box(`"display_name":"Box","match":{"gpu_name":"Box"},"memory":"unified"`), "no why"}, {"bad id", map[string]string{"machines/Big Box.json": `{"schema_version":1,"id":"Big Box","display_name":"Box","match":{"gpu_name":"Box"},"memory":"unified","why":"w"}`}, "lowercase"}, {"forbidden flag without dashes", map[string]string{"architectures/a.json": `{"schema_version":1,"arch":"a","generation":"g","forbidden_args":[{"flag":"x","value":"y","why":"w"}]}`}, "does not start with --"}, @@ -252,6 +254,43 @@ func TestMatchHonoursVendorAndDiscrete(t *testing.T) { } } +// A GPU the driver gives no name, as amdgpu does, is matched by its PCI ids. +// The device id has to be the vendor's: the same number from another maker is +// another chip. A definition that also names the GPU needs both to hold. +func TestMatchByPCIDevice(t *testing.T) { + t.Parallel() + reg, err := load(map[string]string{ + "vendors/acme.json": `{"schema_version":1,"id":"acme","name":"Acme","pci_ids":["0x1234"]}`, + "vendors/other.json": `{"schema_version":1,"id":"other","name":"Other","pci_ids":["0x5678"]}`, + "machines/acme_apu.json": `{"schema_version":1,"id":"acme_apu","display_name":"Acme APU","match":{"vendor":"acme","pci_devices":["0x1586","0x1587"]},"memory":"unified","why":"shares memory"}`, + "machines/acme_named.json": `{"schema_version":1,"id":"acme_named","display_name":"Acme named",` + + `"match":{"vendor":"acme","gpu_name":"Named","pci_devices":["0x2000"]},"memory":"unified","why":"both rules"}`, + }) + if err != nil { + t.Fatalf("Load: %v", err) + } + for _, dev := range []string{"0x1586", "0x1587", "0X1586 "} { + if d, ok := reg.Match(hardware.Observed{Vendor: "acme", PCIDevice: dev}); !ok || d.ID != "acme_apu" { + t.Errorf("acme %s matched %q, %v; want acme_apu", dev, d.ID, ok) + } + } + if d, ok := reg.Match(hardware.Observed{Vendor: "other", PCIDevice: "0x1586"}); ok { + t.Errorf("another vendor's 0x1586 matched %q", d.ID) + } + if d, ok := reg.Match(hardware.Observed{Vendor: "acme", PCIDevice: "0x1588"}); ok { + t.Errorf("an unlisted device matched %q", d.ID) + } + if d, ok := reg.Match(hardware.Observed{Vendor: "acme", GPUName: "Acme APU"}); ok { + t.Errorf("a name alone matched %q, which matches by device", d.ID) + } + if d, ok := reg.Match(hardware.Observed{Vendor: "acme", PCIDevice: "0x2000"}); ok { + t.Errorf("a device without the name matched %q, which needs both", d.ID) + } + if d, ok := reg.Match(hardware.Observed{Vendor: "acme", GPUName: "Named GPU", PCIDevice: "0x2000"}); !ok || d.ID != "acme_named" { + t.Errorf("name and device matched %q, %v; want acme_named", d.ID, ok) + } +} + // A local file teaches llm-helper about a new machine. It must not be able to // change what is already known, because that would let a stale file turn a // Spark discrete or drop a flag that aborts it. From 0341e2be702ed812b52c3eaeceb3b79bd0aee3fa Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Fri, 13 Nov 2026 20:58:25 +0000 Subject: [PATCH 02/15] detect: size AMD Strix Halo from its GTT limit A Ryzen AI Max read as a CPU-only box: amdgpu reports only a 512 MB carve-out as VRAM, nothing names the GPU, so doctor warned that only NVIDIA and Apple GPUs are sized and the budget was the CPU backend's share of RAM. The GPU actually loads weights into GTT, system memory up to the limit the ttm module sets, which amdgpu reports in mem_info_gtt_total. The amd_strix_halo machine matches PCI device 0x1586 (one id for the 8060S, 8050S and 8040S) and is unified. The probe now reads mem_info_gtt_total, locally and over SSH, and a unified GPU with a GTT reading is sized against the limit plus its carve-out, held to RAM less the OS when the limit is raised close to all of it. With the box's numbers, the stock 101 GB limit budgets about 87 GB and the raised 120 GB one about 105 GB, which matches a 103 GB model needing the raise. The report is the box's readings from 0.6.1-dev.63314bb with gtt_bytes added from sysfs, since that build did not read it. A carve-out under a gigabyte is shown in MB instead of "0 GB". --- CHANGELOG.md | 4 ++ .../hardware/machines/amd_strix_halo.json | 11 ++++ .../hardware/reports/amd_strix_halo.json | 20 +++++++ docs/hardware.md | 3 +- docs/memory.md | 22 ++++++++ internal/cli/detectcmds.go | 21 +++++-- internal/cli/doctor_internal_test.go | 20 +++++++ internal/detect/budget.go | 25 ++++++++- internal/detect/budget_internal_test.go | 56 +++++++++++++++++++ internal/detect/classify.go | 54 +++++++++++++++++- internal/detect/fixtures_internal_test.go | 5 +- internal/detect/gpus_internal_test.go | 10 +++- internal/detect/hardware.go | 17 +++++- internal/detect/hardware_linux.go | 3 + .../detect/hardware_linux_internal_test.go | 3 +- internal/detect/remote.go | 12 +++- 16 files changed, 269 insertions(+), 17 deletions(-) create mode 100644 catalogue/hardware/machines/amd_strix_halo.json create mode 100644 catalogue/hardware/reports/amd_strix_halo.json diff --git a/CHANGELOG.md b/CHANGELOG.md index c652de9..2635dc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,6 +79,10 @@ the fact. - `recipes install-service` follows `SERVICE_SCOPE`. With `user` it installs the recipe unit and its watchdog in `~/.config/systemd/user`, enables lingering so they start at boot, and needs no sudo. +- AMD Ryzen AI Max (Strix Halo): the machine is recognised by its PCI id and + sized against the GPU's GTT limit, the system memory amdgpu lets it map, + rather than as a CPU-only box. A hardware definition can now match by + `pci_devices`, and `hardware report` records `gtt_bytes`. ### Changed diff --git a/catalogue/hardware/machines/amd_strix_halo.json b/catalogue/hardware/machines/amd_strix_halo.json new file mode 100644 index 0000000..325bdac --- /dev/null +++ b/catalogue/hardware/machines/amd_strix_halo.json @@ -0,0 +1,11 @@ +{ + "schema_version": 1, + "id": "amd_strix_halo", + "display_name": "AMD Ryzen AI Max (Strix Halo)", + "match": { + "vendor": "amd", + "pci_devices": ["0x1586"] + }, + "memory": "unified", + "why": "The Radeon 8060S, 8050S and 8040S in a Ryzen AI Max are one PCI device, 0x1586, and amdgpu reports no name for it, so the device id is what identifies it. The GPU has only the carve-out the firmware sets aside (mem_info_vram_total, 512 MB by default) and maps the rest from system memory through GTT, up to mem_info_gtt_total, which the ttm module's pages_limit sets. The budget comes from that limit, not from VRAM or all of RAM." +} diff --git a/catalogue/hardware/reports/amd_strix_halo.json b/catalogue/hardware/reports/amd_strix_halo.json new file mode 100644 index 0000000..9d604c6 --- /dev/null +++ b/catalogue/hardware/reports/amd_strix_halo.json @@ -0,0 +1,20 @@ +{ + "provenance": "measured: 'llm-helper hardware report' 0.6.1-dev.63314bb on a Ryzen AI Max with the Radeon 8060S (gfx1151) and 128 GB, amdgpu driver, 2026-11-13. That build did not read the GTT limit, so gtt_bytes is mem_info_gtt_total read from sysfs by hand at the stock ttm pages_limit; detected is what llm-helper concludes from these readings now.", + "readings": { + "schema_version": 1, + "os": "linux", + "arch": "amd64", + "system": "Advanced Micro Devices, Inc. (AMD) RAH-001", + "mem_total_bytes": 134314070016, + "nvidia_smi": false, + "pci": [{ "vendor": "0x1002", "device": "0x1586", "class": "0x038000", "driver": "amdgpu", "vram_bytes": 536870912, "gtt_bytes": 101139496960 }] + }, + "detected": { + "hardware_id": "amd_strix_halo", + "gpu_name": "AMD Ryzen AI Max (Strix Halo)", + "memory": "unified", + "memory_source": "definition", + "cuda": false, + "accelerators": ["AMD gpu (0x1002:0x1586, driver amdgpu, 536 MB)"] + } +} diff --git a/docs/hardware.md b/docs/hardware.md index bd889ca..5c12e40 100644 --- a/docs/hardware.md +++ b/docs/hardware.md @@ -13,7 +13,8 @@ An NVIDIA GPU with no definition still works: its memory comes from `nvidia-smi` `nvidia-smi` gets 10 seconds; a wedged driver is treated as `nvidia-smi` failing, rather than hanging the command. - **`CUDA_VISIBLE_DEVICES`** — from the environment and from `config.env`, `secrets.env` and the profile being served, so the GPUs sized are the ones the server will get. - **Jetson markers** — `/etc/nv_tegra_release`, `/sys/class/tegra-firmware` and the device-tree model. -- **The PCI bus** — every display controller from a known vendor, and every processing accelerator, so an AMD GPU or an NPU is named even though llm-helper does not size for it yet. +- **The PCI bus** — every display controller from a known vendor, and every processing accelerator, so an AMD GPU or an NPU is named even when llm-helper does not size for it. + A GPU that a hardware definition recognises by its PCI ids, such as the AMD Ryzen AI Max (Strix Halo), is sized from what amdgpu lets it map: its GTT limit (`mem_info_gtt_total`) plus its firmware carve-out (`mem_info_vram_total`). See "AMD Ryzen AI Max (Strix Halo)" in [Memory budget](./memory.md). - **The cgroup memory limit** — in a container or a limited systemd unit, the budget is held to the limit rather than the host's RAM. - **Apple Silicon** — the SoC's brand string, which names its GPU. diff --git a/docs/memory.md b/docs/memory.md index a0ac5e5..03d7d38 100644 --- a/docs/memory.md +++ b/docs/memory.md @@ -24,6 +24,28 @@ A model larger than one GPU needs `--tensor-parallel-size` in the profile's `EXT How llm-helper decides whether a GPU has memory of its own is described in [Hardware detection](./hardware.md). +## AMD Ryzen AI Max (Strix Halo) + +A Ryzen AI Max GPU (Radeon 8060S, 8050S or 8040S) has no memory of its own beyond a carve-out the firmware sets aside, 512 MB by default. +It maps the rest from system memory through GTT, up to a limit the `ttm` kernel module sets, which `amdgpu` reports in `/sys/class/drm/card0/device/mem_info_gtt_total`. +The budget is measured against that limit plus the carve-out, not against all of RAM. +llama.cpp sees the same memory: its ROCm build reports the GTT limit as the device's memory, and its Vulkan build reports the limit plus the carve-out. + +The default limit leaves memory idle on a machine that only serves models. +On a 128 GB machine it is 101 GB, and a 103 GB model does not fit. +To raise it, set `pages_limit` in 4 KiB pages, rebuild the initramfs so the module reads it at boot, and reboot: + +```sh +echo 'options ttm pages_limit=29360128' | sudo tee /etc/modprobe.d/ttm.conf +sudo update-initramfs -u +sudo reboot +``` + +29360128 pages is 112 GiB. +On a 128 GB machine that raised the limit to 120 GB, and with a 103 GB model loaded about 15 GB of RAM was still free. +`update-initramfs` is the Debian and Ubuntu command; on Fedora run `sudo dracut -f` instead. +The OS shares the same memory, so the budget never counts more than RAM less what the OS is using, however high the limit is set. + ## Next steps - [Profiles](./profiles.md) — where `KV_CACHE_MEMORY_BYTES` comes from. diff --git a/internal/cli/detectcmds.go b/internal/cli/detectcmds.go index 8bdb231..f70f988 100644 --- a/internal/cli/detectcmds.go +++ b/internal/cli/detectcmds.go @@ -333,9 +333,11 @@ func hardwareWarnings(hw detect.Hardware, running int) []string { return w } // Sized for the CPU backend with a GPU on the bus: say why, because - // otherwise the budget reads as the machine's limit. + // otherwise the budget reads as the machine's limit. A GPU a hardware + // definition recognised on the bus is sized already. for _, a := range hw.Accelerators { switch { + case hw.GPUPCIID != "" && a.PCIID == hw.GPUPCIID: case a.Vendor == "nvidia": w = append(w, fmt.Sprintf( "an NVIDIA GPU is on the PCI bus (%s), but no NVIDIA driver answers, so this machine is sized for vLLM's CPU backend. Install or load the NVIDIA driver to serve on it.", @@ -352,14 +354,17 @@ func hardwareWarnings(hw detect.Hardware, running int) []string { // overlapWarnings names the definitions that claim one of this machine's // GPUs when more than one does. The shipped ones are tested never to // overlap, so the others come from hardware.d, and only the first is used. -// Every GPU nvidia-smi lists is checked, and the one the machine is named -// after too, which is how an Apple SoC or a Jetson without nvidia-smi is -// seen. +// Every GPU nvidia-smi lists is checked, the one the machine is named after, +// which is how an Apple SoC or a Jetson without nvidia-smi is seen, and every +// GPU on the PCI bus, which is how one with no name is. func overlapWarnings(reg *hardware.Registry, hw detect.Hardware) []string { observed := []hardware.Observed{{Vendor: hw.GPUVendor, GPUName: hw.GPUName}} for _, g := range hw.GPUs { observed = append(observed, hardware.Observed{Vendor: "nvidia", GPUName: g.Name}) } + for _, a := range hw.Accelerators { + observed = append(observed, a.Observed()) + } var w []string seen := map[hardware.Observed]bool{} for _, o := range observed { @@ -379,8 +384,12 @@ func overlapWarnings(reg *hardware.Registry, hw detect.Hardware) []string { } others = append(others, fmt.Sprintf("%s (%s)", d.ID, src)) } - w = append(w, fmt.Sprintf("%s matches more than one hardware definition. llm-helper uses %s and ignores %s; narrow the gpu_name of the one that is wrong.", - o.GPUName, m[0].ID, strings.Join(others, ", "))) + name := o.GPUName + if name == "" { + name = "PCI device " + o.PCIDevice + } + w = append(w, fmt.Sprintf("%s matches more than one hardware definition. llm-helper uses %s and ignores %s; narrow the gpu_name or pci_devices of the one that is wrong.", + name, m[0].ID, strings.Join(others, ", "))) } return w } diff --git a/internal/cli/doctor_internal_test.go b/internal/cli/doctor_internal_test.go index a06f4aa..e77816f 100644 --- a/internal/cli/doctor_internal_test.go +++ b/internal/cli/doctor_internal_test.go @@ -655,6 +655,12 @@ func TestHardwareWarnings(t *testing.T) { if w := hardwareWarnings(amd, 0); len(w) != 1 || !strings.Contains(w[0], "AMD gpu (0x1002:0x744c, driver amdgpu, 24 GB) is present") { t.Errorf("AMD card: %q, want one warning and none for the integrated Intel GPU", w) } + // An APU a definition recognises on the bus is sized for, so there is + // nothing to explain. + halo := detect.ParseRemote("memtotal 131166084\npci 0x1002 0x1586 0x038000 amdgpu 536870912 120259084288\n") + if w := hardwareWarnings(halo, 0); len(w) != 0 { + t.Errorf("Strix Halo: %q", w) + } nodriver := detect.Hardware{TotalMemBytes: 64e9, Accelerators: []detect.Accelerator{{Vendor: "nvidia", VendorName: "NVIDIA", PCIID: "0x10de:0x2684", Kind: "gpu", Driver: "nouveau"}}} if w := hardwareWarnings(nodriver, 0); len(w) != 1 || !strings.Contains(w[0], "no NVIDIA driver answers") { t.Errorf("NVIDIA card without its driver: %q", w) @@ -697,4 +703,18 @@ func TestOverlapWarningsNameTheIgnoredDefinition(t *testing.T) { if w := overlapWarnings(reg, mixed); len(w) != 1 || !strings.Contains(w[0], "NVIDIA GB10 matches") { t.Errorf("an overlap on the second GPU: %q", w) } + + // A GPU with no name overlaps by its PCI device id. + halo := detect.ParseRemote("memtotal 131166084\npci 0x1002 0x1586 0x038000 amdgpu 536870912 120259084288\n") + if w := overlapWarnings(hardware.Shipped(), halo); len(w) != 0 { + t.Errorf("Strix Halo with the shipped definitions alone: %q", w) + } + reg, err = hardware.Shipped().Extend(hardware.KindMachine, "/cfg/hardware.d/machines/my_apu.json", + []byte(`{"schema_version":1,"id":"my_apu","display_name":"Mine","match":{"vendor":"amd","pci_devices":["0x1586"]},"memory":"unified","why":"w"}`)) + if err != nil { + t.Fatal(err) + } + if w := overlapWarnings(reg, halo); len(w) != 1 || !strings.Contains(w[0], "PCI device 0x1586 matches") || !strings.Contains(w[0], "uses amd_strix_halo") { + t.Errorf("an overlap on a PCI device: %q", w) + } } diff --git a/internal/detect/budget.go b/internal/detect/budget.go index 16b8467..093b5d3 100644 --- a/internal/detect/budget.go +++ b/internal/detect/budget.go @@ -34,6 +34,11 @@ const ( // vllm-metal holds MLX and a torch MPS context, and captures no graphs. metalRuntimeReserve = 2 * gibibyte + // A GPU that maps system memory through GTT holds the server's compute + // buffers and the driver's own allocations there, with no CUDA context + // or captured graphs. An estimate, sized like the CPU backend's. + gttRuntimeReserve = 2 * gibibyte + // A model you cannot serve does not fit, so hold back a usable cache -- // capped by fraction so a small card is not budgeted to zero. kvFloor = 6 * gibibyte @@ -88,7 +93,7 @@ const baselineCeiling = 0.15 // should be told. func (h Hardware) Busy() bool { pool, kind := h.Pool() - return kind == "ram" && h.InUseBytes > 0 && float64(h.InUseBytes) >= baselineCeiling*float64(pool) + return (kind == "ram" || kind == "gtt") && h.InUseBytes > 0 && float64(h.InUseBytes) >= baselineCeiling*float64(pool) } // AllGPUsHidden reports that nvidia-smi lists GPUs but CUDA_VISIBLE_DEVICES @@ -142,6 +147,24 @@ func (h Hardware) Budget(usedBytes int64) (int64, string) { if n := len(h.GPUs); n > 1 { reason += fmt.Sprintf(", on one of %d GPUs; a larger model needs --tensor-parallel-size", n) } + case poolKind == "gtt": + // The GTT limit is a share of system memory, which the OS lives in + // too, so it only bounds the weights while it leaves the OS its + // baseline. Raised close to all of RAM, the OS bounds them instead. + runtime, graphs = gttRuntimeReserve, 0 + reason = "the GPU's GTT limit, the system memory amdgpu lets it map, minus runtime overhead and room for a KV cache" + ram := h.TotalMemBytes + if h.MemLimited() { + ram = h.MemLimitBytes + } + osBaseline := defaultBaseline(float64(ram)) + if usedBytes > 0 && float64(usedBytes) < baselineCeiling*float64(ram) { + osBaseline = float64(usedBytes) + } + if room := float64(ram) - osBaseline; ram > 0 && float64(h.GTTBytes) > room { + pool = float64(h.GPUCarveOutBytes) + room + reason = "system memory minus the OS, which is less than the GPU's GTT limit, minus runtime overhead and room for a KV cache" + } case usedBytes > 0 && float64(usedBytes) < baselineCeiling*pool: baseline = float64(usedBytes) reason += ", against a measured OS baseline" diff --git a/internal/detect/budget_internal_test.go b/internal/detect/budget_internal_test.go index 6286038..2bb2f34 100644 --- a/internal/detect/budget_internal_test.go +++ b/internal/detect/budget_internal_test.go @@ -233,3 +233,59 @@ func TestAMachineThatCannotServeSaysSo(t *testing.T) { t.Errorf("reason = %q, want it to say the machine cannot serve a model", reason) } } + +// An APU that maps system memory through GTT, as a Strix Halo does, is sized +// against that limit and its carve-out, not against all of RAM and not as the +// CPU backend. Raising the limit raises the budget, until the OS's share of +// RAM is what bounds it. The figures are a Ryzen AI Max with 128 GB at the +// stock ttm pages_limit and at the one its 103 GB model needed. +func TestAGTTLimitBoundsTheBudget(t *testing.T) { + const ram, carve = 134314070016, 536870912 + stock := FromReadings(strixHaloReadings(ram, carve, 101139496960)) + raised := FromReadings(strixHaloReadings(ram, carve, 120259084288)) + if pool, kind := stock.Pool(); kind != "gtt" || pool != carve+101139496960 { + t.Errorf("stock pool = %d %s, want the GTT limit plus the carve-out", pool, kind) + } + if !strings.Contains(stock.BudgetReason, "GTT limit") || strings.Contains(stock.BudgetReason, "CPU-backend") { + t.Errorf("reason = %q, want it to name the GTT limit", stock.BudgetReason) + } + const model = 103e9 + if stock.BudgetBytes >= model || raised.BudgetBytes <= model { + t.Errorf("budgets %d (stock) and %d (raised); want the 103 GB model to fit only after the raise", stock.BudgetBytes, raised.BudgetBytes) + } + if raised.BudgetBytes > raised.GTTBytes { + t.Errorf("budget %d exceeds the GTT limit %d", raised.BudgetBytes, raised.GTTBytes) + } + + // A limit at all of RAM would leave the OS nothing, so RAM less the + // OS bounds the weights instead. + all := FromReadings(strixHaloReadings(ram, carve, ram)) + if all.BudgetBytes >= int64(ram-defaultBaseline(ram)) || !strings.Contains(all.BudgetReason, "minus the OS") { + t.Errorf("GTT at all of RAM: budget %d (%s)", all.BudgetBytes, all.BudgetReason) + } + + // A large firmware carve-out is memory the OS does not see, and the GPU + // still uses it. + carved := FromReadings(strixHaloReadings(32e9, 96e9, 16e9)) + if carved.BudgetBytes <= 96e9 { + t.Errorf("96 GB carve-out with a 16 GB GTT: budget %d, want more than the carve-out", carved.BudgetBytes) + } +} + +// Without a GTT reading, as from an llm-helper that did not read it, the +// machine is still recognised and sized from system memory as before. +func TestAUnifiedAPUWithoutAGTTReadingFallsBackToRAM(t *testing.T) { + hw := FromReadings(strixHaloReadings(134314070016, 536870912, 0)) + if hw.HardwareID != "amd_strix_halo" || !hw.Unified { + t.Fatalf("hardware = %+v", hw) + } + if _, kind := hw.Pool(); kind != "ram" { + t.Errorf("pool kind = %s, want ram", kind) + } +} + +func strixHaloReadings(ram, carve, gtt int64) Readings { + return Readings{SchemaVersion: 1, OS: "linux", Arch: "amd64", MemTotalBytes: ram, PCI: []PCIDevice{ + {Vendor: "0x1002", Device: "0x1586", Class: "0x038000", Driver: "amdgpu", VRAMBytes: carve, GTTBytes: gtt}, + }} +} diff --git a/internal/detect/classify.go b/internal/detect/classify.go index 04fbfd1..9f6ee6f 100644 --- a/internal/detect/classify.go +++ b/internal/detect/classify.go @@ -72,6 +72,8 @@ func (f facts) describeGPUs(hw *Hardware) { if d, ok := reg.Match(hardware.Observed{Vendor: "apple", GPUName: f.cpuBrand}); ok { hw.HardwareID, hw.Memory, hw.MemorySource, hw.Arch = d.ID, string(d.Memory), MemoryFromDefinition, d.Arch } + default: + f.describePCIGPU(reg, hw) } hw.GPUMemGB = decimalGB(hw.GPUMemBytes) hw.Unified = hw.Memory == string(hardware.MemoryUnified) || hw.Memory == string(hardware.MemoryCoherent) @@ -80,6 +82,37 @@ func (f facts) describeGPUs(hw *Hardware) { } } +// describePCIGPU names a GPU that no tool lists but a hardware definition +// recognises on the PCI bus by its ids, such as an AMD APU under amdgpu. A +// GPU no definition names is left alone: it is only listed as an +// accelerator, and the machine is sized as it was. The definition's display +// name stands in for the GPU's, since the bus reports none. +// +// A unified GPU that reports a GTT limit loads weights into that, plus any +// carve-out the firmware set aside for it, rather than into all of RAM. +func (f facts) describePCIGPU(reg *hardware.Registry, hw *Hardware) { + for _, dev := range f.pci { + if !strings.HasPrefix(strings.ToLower(dev.Class), "0x03") { + continue + } + v, ok := reg.VendorByPCI(dev.Vendor) + if !ok { + continue + } + d, ok := reg.Match(hardware.Observed{Vendor: v.ID, PCIDevice: dev.Device}) + if !ok { + continue + } + hw.GPUName, hw.GPUVendor, hw.GPUPCIID = d.DisplayName, v.ID, dev.Vendor+":"+dev.Device + hw.HardwareID, hw.Memory, hw.MemorySource, hw.Arch = d.ID, string(d.Memory), MemoryFromDefinition, d.Arch + if d.Unified() && dev.GTTBytes > 0 { + hw.GTTBytes, hw.GPUCarveOutBytes = dev.GTTBytes, dev.VRAMBytes + hw.GTTGB = decimalGB(dev.GTTBytes) + } + return + } +} + // tegraVisible reports whether CUDA_VISIBLE_DEVICES leaves a Jetson its one // GPU, which CUDA numbers 0. Only the first entry matters, as it does for // visibleGPUs. A UUID cannot be checked without nvidia-smi listing the GPU, @@ -146,8 +179,13 @@ type PCIDevice struct { Device string `json:"device"` // e.g. "0x2e12" Class string `json:"class"` // e.g. "0x030200" Driver string `json:"driver,omitempty"` - // VRAMBytes is the memory amdgpu reports for the card, 0 elsewhere. + // VRAMBytes is the memory amdgpu reports for the card, 0 elsewhere. On + // an integrated GPU it is the carve-out the firmware set aside. VRAMBytes int64 `json:"vram_bytes,omitempty"` + // GTTBytes is how much system memory amdgpu lets the GPU map + // (mem_info_gtt_total), 0 elsewhere. It is what an integrated GPU loads + // weights into, and the ttm module's pages_limit sets it. + GTTBytes int64 `json:"gtt_bytes,omitempty"` } // Accelerator is a GPU or compute accelerator found on the PCI bus. @@ -170,12 +208,24 @@ func (a Accelerator) Describe() string { } else { parts = append(parts, "no driver") } - if a.VRAMBytes > 0 { + switch { + case a.VRAMBytes >= gigabyte: parts = append(parts, fmt.Sprintf("%d GB", decimalGB(a.VRAMBytes))) + case a.VRAMBytes > 0: + // An integrated GPU's carve-out is often under a gigabyte, which + // whole gigabytes would show as 0. + parts = append(parts, fmt.Sprintf("%d MB", a.VRAMBytes/1e6)) } return fmt.Sprintf("%s %s (%s)", a.VendorName, a.Kind, strings.Join(parts, ", ")) } +// Observed is the accelerator as a hardware definition is matched against +// it: its vendor and PCI device id, since the PCI bus gives no name. +func (a Accelerator) Observed() hardware.Observed { + _, device, _ := strings.Cut(a.PCIID, ":") + return hardware.Observed{Vendor: a.Vendor, PCIDevice: device} +} + // accelerators picks the GPUs and accelerators out of a PCI listing. A // display controller counts only from a vendor the definitions name, since // a server's BMC shows up as one too; a processing accelerator counts from diff --git a/internal/detect/fixtures_internal_test.go b/internal/detect/fixtures_internal_test.go index 15a8216..8302533 100644 --- a/internal/detect/fixtures_internal_test.go +++ b/internal/detect/fixtures_internal_test.go @@ -95,13 +95,16 @@ func TestNoGPUMatchesTwoMachines(t *testing.T) { for _, g := range f.Readings.GPUs { observed[hardware.Observed{Vendor: "nvidia", GPUName: g.Name}] = true } + for _, a := range hw.Accelerators { + observed[a.Observed()] = true + } for o := range observed { if m := hardware.Shipped().Matches(o); len(m) > 1 { var ids []string for _, d := range m { ids = append(ids, d.ID) } - t.Errorf("%s: %q matches %s; narrow a gpu_name so it matches one", f.name, o.GPUName, strings.Join(ids, ", ")) + t.Errorf("%s: %q matches %s; narrow a gpu_name or pci_devices so it matches one", f.name, o.GPUName+o.PCIDevice, strings.Join(ids, ", ")) } } } diff --git a/internal/detect/gpus_internal_test.go b/internal/detect/gpus_internal_test.go index 288f75d..fbfb14e 100644 --- a/internal/detect/gpus_internal_test.go +++ b/internal/detect/gpus_internal_test.go @@ -243,6 +243,14 @@ func TestParseRemote(t *testing.T) { if !jetson.CUDA || !jetson.Unified || jetson.HardwareID != "nvidia_tegra" || jetson.GPUName != "NVIDIA Jetson AGX Orin Developer Kit" { t.Errorf("Jetson without nvidia-smi = %+v", jetson) } + // An APU's GTT limit is the sixth field, and sizes the node's pool. + halo := ParseRemote("memtotal 131166084\npci 0x1002 0x1586 0x038000 amdgpu 536870912 120259084288\n") + if pool, kind := halo.Pool(); halo.HardwareID != "amd_strix_halo" || kind != "gtt" || pool != 536870912+120259084288 { + t.Errorf("Strix Halo node = %+v, pool %d %s", halo, pool, kind) + } + if old := ParseRemote("memtotal 131166084\npci 0x1002 0x1586 0x038000 amdgpu 536870912\n"); old.GTTBytes != 0 || old.HardwareID != "amd_strix_halo" { + t.Errorf("a line without GTT = %+v", old) + } } // An old driver fails the whole capability query on addressing_mode, so the @@ -348,7 +356,7 @@ func TestACgroupLimitCapsTheSystemPool(t *testing.T) { func TestRemoteScriptEmitsWhatParseRemoteReads(t *testing.T) { t.Parallel() s := RemoteScript() - for _, key := range []string{`"memtotal `, `"memavail `, `"nvidiactl `, `"smi ok"`, `s/^/gpu /`, `s/^/gpucap /`, `"tegra `, `"pci `, + for _, key := range []string{`"memtotal `, `"memavail `, `"nvidiactl `, `"smi ok"`, `s/^/gpu /`, `s/^/gpucap /`, `"tegra `, `"pci `, `mem_info_gtt_total`, strings.Join(nvidiaSMIArgs, " "), strings.Join(nvidiaSMICapArgs, " "), strings.Join(nvidiaSMICapFallback, " ")} { if !strings.Contains(s, key) { t.Errorf("RemoteScript does not contain %q", key) diff --git a/internal/detect/hardware.go b/internal/detect/hardware.go index fe6475d..cd3cc1f 100644 --- a/internal/detect/hardware.go +++ b/internal/detect/hardware.go @@ -40,6 +40,17 @@ type Hardware struct { // GPUVendor is the vendor id of the GPU GPUName names, such as "nvidia" // or "apple". GPUVendor string `json:"gpu_vendor,omitempty"` + // GPUPCIID is the vendor:device id of a GPU recognised on the PCI bus + // rather than by a driver tool, such as "0x1002:0x1586", and "" for + // any other. + GPUPCIID string `json:"gpu_pci_id,omitempty"` + // GTTBytes is how much system memory amdgpu lets an integrated GPU map, + // its GTT limit, and GPUCarveOutBytes the memory the firmware set aside + // for it. Together they are what the GPU can load weights into. Both + // are 0 on any other GPU. + GTTBytes int64 `json:"gtt_bytes,omitempty"` + GTTGB int `json:"gtt_gb,omitempty"` + GPUCarveOutBytes int64 `json:"gpu_carve_out_bytes,omitempty"` // GPUs is every NVIDIA GPU the server can see, in CUDA order. GPUName // and GPUMemBytes describe the first, because that is the one a server // without --tensor-parallel-size loads onto. @@ -99,7 +110,8 @@ type Hardware struct { } // Pool reports the memory weights are actually loaded into, and which pool -// that is. A discrete GPU uses its own VRAM; everything else uses system +// that is. A discrete GPU uses its own VRAM; a unified GPU with a GTT limit +// uses what that limit lets it map, plus its carve-out; everything else uses system // memory, as much of it as this process's cgroup allows. One definition, // because a caller drawing installed RAM under a budget measured against // VRAM is comparing two different machines. @@ -110,6 +122,9 @@ func (h Hardware) Pool() (int64, string) { if !h.Unified && h.GPUMemBytes > 0 { return h.GPUMemBytes, "vram" } + if h.GTTBytes > 0 { + return h.GPUCarveOutBytes + h.GTTBytes, "gtt" + } if h.MemLimited() { return h.MemLimitBytes, "ram" } diff --git a/internal/detect/hardware_linux.go b/internal/detect/hardware_linux.go index 8c7298d..32722a6 100644 --- a/internal/detect/hardware_linux.go +++ b/internal/detect/hardware_linux.go @@ -105,6 +105,9 @@ func scanPCI() []PCIDevice { if n, err := strconv.ParseInt(read("mem_info_vram_total"), 10, 64); err == nil && n > 0 { d.VRAMBytes = n } + if n, err := strconv.ParseInt(read("mem_info_gtt_total"), 10, 64); err == nil && n > 0 { + d.GTTBytes = n + } out = append(out, d) } sort.Slice(out, func(i, j int) bool { diff --git a/internal/detect/hardware_linux_internal_test.go b/internal/detect/hardware_linux_internal_test.go index 8eed0b5..9312050 100644 --- a/internal/detect/hardware_linux_internal_test.go +++ b/internal/detect/hardware_linux_internal_test.go @@ -53,6 +53,7 @@ func TestPlatformFactsReadsSysfs(t *testing.T) { amd + "device": "0x744c\n", amd + "class": "0x030000\n", amd + "mem_info_vram_total": "25753026560\n", + amd + "mem_info_gtt_total": "33554432000\n", nic + "vendor": "0x15b3\n", nic + "device": "0x1021\n", nic + "class": "0x020000\n", @@ -71,7 +72,7 @@ func TestPlatformFactsReadsSysfs(t *testing.T) { if !f.tegra || f.tegraModel != "NVIDIA Jetson AGX Orin Developer Kit" { t.Errorf("tegra = %v, %q", f.tegra, f.tegraModel) } - if len(f.pci) != 1 || f.pci[0] != (PCIDevice{Vendor: "0x1002", Device: "0x744c", Class: "0x030000", Driver: "amdgpu", VRAMBytes: 25753026560}) { + if len(f.pci) != 1 || f.pci[0] != (PCIDevice{Vendor: "0x1002", Device: "0x744c", Class: "0x030000", Driver: "amdgpu", VRAMBytes: 25753026560, GTTBytes: 33554432000}) { t.Errorf("pci = %+v, want the AMD card and not the NIC", f.pci) } } diff --git a/internal/detect/remote.go b/internal/detect/remote.go index dea3b9f..28ae979 100644 --- a/internal/detect/remote.go +++ b/internal/detect/remote.go @@ -31,7 +31,7 @@ for d in /sys/bus/pci/devices/*; do case "$class" in 0x03*|0x12*) ;; *) continue ;; esac drv=- if [ -L "$d/driver" ]; then drv=$(basename "$(readlink "$d/driver")"); fi - echo "pci $(cat "$d/vendor") $(cat "$d/device") $class $drv $(cat "$d/mem_info_vram_total" 2>/dev/null || echo 0)" + echo "pci $(cat "$d/vendor") $(cat "$d/device") $class $drv $(cat "$d/mem_info_vram_total" 2>/dev/null || echo 0) $(cat "$d/mem_info_gtt_total" 2>/dev/null || echo 0)" done ` } @@ -70,8 +70,9 @@ func ParseRemote(out string) Hardware { return f.hardware(nil) } -// parsePCILine reads "vendor device class driver vram" as RemoteScript -// prints it, with "-" for no driver. +// parsePCILine reads "vendor device class driver vram gtt" as RemoteScript +// prints it, with "-" for no driver. A size that is missing or 0 is left +// unset. func parsePCILine(s string) (PCIDevice, bool) { fields := strings.Fields(s) if len(fields) < 3 { @@ -86,6 +87,11 @@ func parsePCILine(s string) (PCIDevice, bool) { d.VRAMBytes = n } } + if len(fields) > 5 { + if n, err := strconv.ParseInt(fields[5], 10, 64); err == nil && n > 0 { + d.GTTBytes = n + } + } return d, true } From 33fd308a2d573bcb1307a2f57defd03b12cd07f7 Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Fri, 13 Nov 2026 20:59:14 +0000 Subject: [PATCH 03/15] doctor: show the GTT limit and how to raise it On a Strix Halo the GTT limit is what bounds the model, and the stock one (101 GB on a 128 GB box) leaves memory idle: a 103 GB model only fit after raising ttm's pages_limit by hand. Doctor now prints the limit beside the GPU and, when it is more than 4 GiB below 90% of RAM, the modprobe line, initramfs rebuild and reboot that raise it, the way metal-limit does for the wired limit on macOS. 90% is what served the 103 GB model with about 15 GB still free. It only prints the steps. Writing modprobe.d needs root and a bad value is found out at the next boot, so llm-helper does not do it. --- CHANGELOG.md | 2 + docs/memory.md | 3 +- internal/cli/detectcmds.go | 3 ++ internal/cli/gttlimit.go | 57 ++++++++++++++++++++++++++ internal/cli/gttlimit_internal_test.go | 38 +++++++++++++++++ 5 files changed, 102 insertions(+), 1 deletion(-) create mode 100644 internal/cli/gttlimit.go create mode 100644 internal/cli/gttlimit_internal_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2635dc5..cfef282 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,8 @@ the fact. sized against the GPU's GTT limit, the system memory amdgpu lets it map, rather than as a CPU-only box. A hardware definition can now match by `pci_devices`, and `hardware report` records `gtt_bytes`. +- `doctor` shows the GTT limit on a Strix Halo and, when it is well below + 90% of RAM, the ttm `pages_limit` that raises it; see docs/memory.md. ### Changed diff --git a/docs/memory.md b/docs/memory.md index 03d7d38..36b1fa8 100644 --- a/docs/memory.md +++ b/docs/memory.md @@ -28,7 +28,7 @@ How llm-helper decides whether a GPU has memory of its own is described in [Hard A Ryzen AI Max GPU (Radeon 8060S, 8050S or 8040S) has no memory of its own beyond a carve-out the firmware sets aside, 512 MB by default. It maps the rest from system memory through GTT, up to a limit the `ttm` kernel module sets, which `amdgpu` reports in `/sys/class/drm/card0/device/mem_info_gtt_total`. -The budget is measured against that limit plus the carve-out, not against all of RAM. +The budget is measured against that limit plus the carve-out, not against all of RAM, and `doctor` prints the limit on its `gtt:` line. llama.cpp sees the same memory: its ROCm build reports the GTT limit as the device's memory, and its Vulkan build reports the limit plus the carve-out. The default limit leaves memory idle on a machine that only serves models. @@ -43,6 +43,7 @@ sudo reboot 29360128 pages is 112 GiB. On a 128 GB machine that raised the limit to 120 GB, and with a 103 GB model loaded about 15 GB of RAM was still free. +`doctor` suggests a value for its machine, 90% of RAM in whole GiB, until the limit is within 4 GiB of it. `update-initramfs` is the Debian and Ubuntu command; on Fedora run `sudo dracut -f` instead. The OS shares the same memory, so the budget never counts more than RAM less what the OS is using, however high the limit is set. diff --git a/internal/cli/detectcmds.go b/internal/cli/detectcmds.go index f70f988..5f83966 100644 --- a/internal/cli/detectcmds.go +++ b/internal/cli/detectcmds.go @@ -443,6 +443,9 @@ func renderDoctor(r doctorReport, yes bool) { fmt.Println(line) } } + for _, line := range doctorGTTLines(hw, os.Getpagesize()) { + fmt.Println(line) + } } else if hw.GPUsHidden > 0 { fmt.Printf(" gpu: %d NVIDIA GPU(s) present, but CUDA_VISIBLE_DEVICES hides all of them from the server\n", hw.GPUsHidden) } else { diff --git a/internal/cli/gttlimit.go b/internal/cli/gttlimit.go new file mode 100644 index 0000000..97838f0 --- /dev/null +++ b/internal/cli/gttlimit.go @@ -0,0 +1,57 @@ +package cli + +import ( + "fmt" + + "github.com/coder/llm-helper/internal/detect" +) + +// GTT is how much system memory amdgpu lets an integrated GPU map. The ttm +// kernel module sets it from its pages_limit option, which is read when the +// module loads, so raising it means a modprobe.d file, a rebuilt initramfs +// and a reboot. llm-helper only says how: writing a modprobe file needs root, +// and a wrong value is only found out at the next boot. +const ( + // gttSuggestShare is the share of RAM doctor suggests for the limit. + // On a 128 GB Ryzen AI Max, 90% (112 GiB) served a 103 GB model with + // about 15 GB of RAM still free. + gttSuggestShare = 0.9 + // gttMinRaise is the smallest raise worth suggesting, since it costs a + // reboot. + gttMinRaise = 4 << 30 + // ttmConf is where the pages_limit option goes. + ttmConf = "/etc/modprobe.d/ttm.conf" +) + +// suggestGTTLimit is the GTT limit to suggest for a machine with ramBytes of +// RAM, rounded down to whole GiB, or 0 when RAM is unknown. +func suggestGTTLimit(ramBytes int64) int64 { + if ramBytes <= 0 { + return 0 + } + return int64(float64(ramBytes)*gttSuggestShare) >> 30 << 30 +} + +// doctorGTTLines are the HARDWARE lines doctor prints for an integrated GPU's +// GTT limit, nil on any other GPU. pageSize is the kernel's page size, the +// unit pages_limit is in. +func doctorGTTLines(hw detect.Hardware, pageSize int) []string { + if hw.GTTBytes <= 0 { + return nil + } + line := fmt.Sprintf(" gtt: %d GB the GPU can map (amdgpu's GTT limit)", hw.GTTGB) + switch c := hw.GPUCarveOutBytes; { + case c >= 1e9: + line += fmt.Sprintf(", plus a %d GB carve-out", c/1e9) + case c > 0: + line += fmt.Sprintf(", plus a %d MB carve-out", c/1e6) + } + lines := []string{line} + want := suggestGTTLimit(hw.TotalMemBytes) + if pageSize <= 0 || want-hw.GTTBytes < gttMinRaise { + return lines + } + return append(lines, + fmt.Sprintf(" raise it to %d GiB with 'options ttm pages_limit=%d' in %s,", want>>30, want/int64(pageSize), ttmConf), + " then 'sudo update-initramfs -u' and a reboot (see docs/memory.md)") +} diff --git a/internal/cli/gttlimit_internal_test.go b/internal/cli/gttlimit_internal_test.go new file mode 100644 index 0000000..0191795 --- /dev/null +++ b/internal/cli/gttlimit_internal_test.go @@ -0,0 +1,38 @@ +package cli + +import ( + "strings" + "testing" + + "github.com/coder/llm-helper/internal/detect" +) + +// The figures are a 128 GB Ryzen AI Max at the stock ttm pages_limit and at +// the one its 103 GB model needed, whose 29360128 pages are what doctor +// suggests. +func TestDoctorGTTLines(t *testing.T) { + halo := func(gtt int64) detect.Hardware { + return detect.FromReadings(detect.Readings{SchemaVersion: 1, OS: "linux", MemTotalBytes: 134314070016, PCI: []detect.PCIDevice{ + {Vendor: "0x1002", Device: "0x1586", Class: "0x038000", Driver: "amdgpu", VRAMBytes: 536870912, GTTBytes: gtt}, + }}) + } + + stock := doctorGTTLines(halo(101139496960), 4096) + if len(stock) != 3 { + t.Fatalf("stock limit: %q, want the limit and how to raise it", stock) + } + if !strings.Contains(stock[0], "101 GB the GPU can map") || !strings.Contains(stock[0], "536 MB carve-out") { + t.Errorf("limit line = %q", stock[0]) + } + if !strings.Contains(stock[1], "raise it to 112 GiB with 'options ttm pages_limit=29360128' in /etc/modprobe.d/ttm.conf") || + !strings.Contains(stock[2], "update-initramfs -u") { + t.Errorf("raise lines = %q", stock[1:]) + } + + if raised := doctorGTTLines(halo(120259084288), 4096); len(raised) != 1 || !strings.Contains(raised[0], "120 GB the GPU can map") { + t.Errorf("raised limit: %q, want only the limit", raised) + } + if lines := doctorGTTLines(detect.Hardware{TotalMemBytes: 128e9, GPUName: "NVIDIA GB10"}, 4096); lines != nil { + t.Errorf("a GPU without GTT: %q", lines) + } +} From 91c8acdd3dffe70a1d6f43abeeb32c925783a35b Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Fri, 13 Nov 2026 21:00:13 +0000 Subject: [PATCH 04/15] doctor: warn when the GPU's device nodes cannot be opened llama.cpp's ROCm build failed on a Strix Halo with "no ROCm-capable device is detected", which says nothing about the cause: the user was not in the render group, and /dev/kfd and /dev/dri/renderD128 are root:render 0660. On Linux, when an amdgpu GPU is on the bus, doctor now checks those nodes with access(2), which needs no open (opening /dev/kfd sets up a GPU context), and names the ones this user cannot use with the usermod fix and the re-login it needs. A node that does not exist is not reported, since Vulkan serves without KFD. The rule takes the device directory and the check as arguments, so it is tested without devices. --- CHANGELOG.md | 3 ++ internal/cli/detectcmds.go | 1 + internal/cli/devaccess.go | 43 +++++++++++++++++++ internal/cli/devaccess_internal_test.go | 57 +++++++++++++++++++++++++ internal/cli/devaccess_linux.go | 21 +++++++++ internal/cli/devaccess_other.go | 9 ++++ 6 files changed, 134 insertions(+) create mode 100644 internal/cli/devaccess.go create mode 100644 internal/cli/devaccess_internal_test.go create mode 100644 internal/cli/devaccess_linux.go create mode 100644 internal/cli/devaccess_other.go diff --git a/CHANGELOG.md b/CHANGELOG.md index cfef282..63744dd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,9 @@ the fact. `pci_devices`, and `hardware report` records `gtt_bytes`. - `doctor` shows the GTT limit on a Strix Halo and, when it is well below 90% of RAM, the ttm `pages_limit` that raises it; see docs/memory.md. +- `doctor` warns when an amdgpu GPU is present but the user cannot open + `/dev/kfd` or `/dev/dri/renderD*`, which makes llama.cpp's ROCm build fail + with "no ROCm-capable device is detected", and names the `usermod` fix. ### Changed diff --git a/internal/cli/detectcmds.go b/internal/cli/detectcmds.go index 5f83966..5fe4c4d 100644 --- a/internal/cli/detectcmds.go +++ b/internal/cli/detectcmds.go @@ -173,6 +173,7 @@ func gatherDoctor() doctorReport { running = len(r.Config.Instances) } r.Warnings = append(r.Warnings, hardwareWarnings(r.Hardware, running)...) + r.Warnings = append(r.Warnings, gpuDeviceWarnings(r.Hardware.Accelerators)...) // Drift checks: the part a human wouldn't spot at a glance. if cfg != nil { diff --git a/internal/cli/devaccess.go b/internal/cli/devaccess.go new file mode 100644 index 0000000..a1fe4bf --- /dev/null +++ b/internal/cli/devaccess.go @@ -0,0 +1,43 @@ +package cli + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/coder/llm-helper/internal/detect" +) + +// amdDeviceWarnings warns when an amdgpu GPU is on the bus but this user +// cannot open the device nodes a server reaches it through: /dev/kfd for +// ROCm and /dev/dri/renderD* for ROCm and Vulkan. Both are root:render 0660 +// on the usual distributions, so a user outside the render group gets a +// server that finds no GPU ("no ROCm-capable device is detected") rather than +// a permission error. A node that does not exist is not reported: a kernel +// without KFD still serves through Vulkan. +// +// devDir stands in for /dev and canOpen for the access check, so the rule can +// be tested without the devices. +func amdDeviceWarnings(accels []detect.Accelerator, devDir string, canOpen func(path string) bool) []string { + if !slices.ContainsFunc(accels, func(a detect.Accelerator) bool { return a.Driver == "amdgpu" }) { + return nil + } + nodes, _ := filepath.Glob(filepath.Join(devDir, "dri", "renderD*")) + nodes = append([]string{filepath.Join(devDir, "kfd")}, nodes...) + var denied []string + for _, n := range nodes { + if _, err := os.Stat(n); err == nil && !canOpen(n) { + denied = append(denied, n) + } + } + if len(denied) == 0 { + return nil + } + return []string{fmt.Sprintf( + "an amdgpu GPU is present, but this user cannot open %s, so a server on ROCm or Vulkan will not find it. "+ + "Add the user to the groups that own them with 'sudo usermod -aG render,video $USER', then log out and back in "+ + "(with SERVICE_SCOPE=user and lingering, reboot so the user's service manager picks up the groups).", + strings.Join(denied, ", "))} +} diff --git a/internal/cli/devaccess_internal_test.go b/internal/cli/devaccess_internal_test.go new file mode 100644 index 0000000..6712696 --- /dev/null +++ b/internal/cli/devaccess_internal_test.go @@ -0,0 +1,57 @@ +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/coder/llm-helper/internal/detect" +) + +// A user outside the render group gets a server that finds no GPU, with no +// permission error to say why, so doctor names the nodes and the fix. Only +// nodes that exist count, and only on a machine with an amdgpu GPU. +func TestAMDDeviceWarnings(t *testing.T) { + dev := t.TempDir() + for _, n := range []string{"kfd", "dri/renderD128", "dri/card0"} { + p := filepath.Join(dev, n) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, nil, 0o600); err != nil { + t.Fatal(err) + } + } + amd := []detect.Accelerator{{Vendor: "amd", VendorName: "AMD", PCIID: "0x1002:0x1586", Kind: "gpu", Driver: "amdgpu"}} + none := func(string) bool { return false } + all := func(string) bool { return true } + + w := amdDeviceWarnings(amd, dev, none) + if len(w) != 1 { + t.Fatalf("no access: %q, want one warning", w) + } + for _, want := range []string{filepath.Join(dev, "kfd"), filepath.Join(dev, "dri", "renderD128"), "sudo usermod -aG render,video $USER"} { + if !strings.Contains(w[0], want) { + t.Errorf("warning %q does not name %q", w[0], want) + } + } + if strings.Contains(w[0], "card0") { + t.Errorf("warning %q names a node servers do not open", w[0]) + } + + if w := amdDeviceWarnings(amd, dev, all); len(w) != 0 { + t.Errorf("full access: %q", w) + } + onlyRender := func(p string) bool { return strings.Contains(p, "renderD") } + if w := amdDeviceWarnings(amd, dev, onlyRender); len(w) != 1 || strings.Contains(w[0], "renderD128") || !strings.Contains(w[0], "kfd") { + t.Errorf("render node only: %q, want /dev/kfd alone", w) + } + if w := amdDeviceWarnings(amd, t.TempDir(), none); len(w) != 0 { + t.Errorf("no device nodes at all: %q", w) + } + nvidia := []detect.Accelerator{{Vendor: "nvidia", VendorName: "NVIDIA", PCIID: "0x10de:0x2e12", Kind: "gpu", Driver: "nvidia"}} + if w := amdDeviceWarnings(nvidia, dev, none); len(w) != 0 { + t.Errorf("no amdgpu GPU: %q", w) + } +} diff --git a/internal/cli/devaccess_linux.go b/internal/cli/devaccess_linux.go new file mode 100644 index 0000000..2109de4 --- /dev/null +++ b/internal/cli/devaccess_linux.go @@ -0,0 +1,21 @@ +package cli + +import ( + "syscall" + + "github.com/coder/llm-helper/internal/detect" +) + +// gpuDeviceWarnings checks this machine's GPU device nodes for the user +// running doctor. +func gpuDeviceWarnings(accels []detect.Accelerator) []string { + return amdDeviceWarnings(accels, "/dev", canReadWrite) +} + +// canReadWrite asks the kernel whether this process's user and groups may +// open path for reading and writing, without opening it: opening /dev/kfd +// sets up a GPU context. +func canReadWrite(path string) bool { + const rOK, wOK = 4, 2 + return syscall.Access(path, rOK|wOK) == nil +} diff --git a/internal/cli/devaccess_other.go b/internal/cli/devaccess_other.go new file mode 100644 index 0000000..d3bc240 --- /dev/null +++ b/internal/cli/devaccess_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package cli + +import "github.com/coder/llm-helper/internal/detect" + +// gpuDeviceWarnings has nothing to check off Linux, where no amdgpu device +// nodes exist. +func gpuDeviceWarnings([]detect.Accelerator) []string { return nil } From b9835dd16782568df6ce8ca63b88dfd753cf9501 Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Fri, 13 Nov 2026 21:01:53 +0000 Subject: [PATCH 05/15] setup: honor the provider's bin key from the environment On a Debian box setup found /usr/bin/llama-server, the distribution's 0.2.0 build, and ignored LLAMA_SERVER_BIN in the environment, so the official release had to be put in config.env by hand. lookBin now takes the server the environment names under the engine definition's bin.key first, for every engine, including VLLM_BIN when a rerun fills in a missing one. A bare name is looked up on PATH and recorded as a full path. A key that names nothing executable stops setup before it writes anything, rather than falling back to the server the user set it to avoid. An environment variable rather than a flag, because the key is already how config.env names the server. --- CHANGELOG.md | 3 ++ docs/install.md | 3 +- docs/reference/cli/setup.md | 6 +++ internal/cli/setup.go | 59 ++++++++++++++++++++--- internal/cli/setup_internal_test.go | 74 +++++++++++++++++++++++++++++ 5 files changed, 138 insertions(+), 7 deletions(-) create mode 100644 internal/cli/setup_internal_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 63744dd..9ac950c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,9 @@ the fact. sized against the GPU's GTT limit, the system memory amdgpu lets it map, rather than as a CPU-only box. A hardware definition can now match by `pci_devices`, and `hardware report` records `gtt_bytes`. +- `setup` records the server named by the provider's key in the environment, + such as `LLAMA_SERVER_BIN`, instead of the one on PATH, and refuses to run + when that key names something that is not an executable. - `doctor` shows the GTT limit on a Strix Halo and, when it is well below 90% of RAM, the ttm `pages_limit` that raises it; see docs/memory.md. - `doctor` warns when an amdgpu GPU is present but the user cannot open diff --git a/docs/install.md b/docs/install.md index 55682a0..ab55b44 100644 --- a/docs/install.md +++ b/docs/install.md @@ -22,7 +22,8 @@ On Linux that means an NVIDIA GPU with a vLLM usually living in a venv, or llama 3. Run `llm-helper setup`, or `llm-helper setup --provider mlx` or `--provider llamacpp` to serve with mlx-lm or llama.cpp. -If `vllm` isn't on your PATH, set `VLLM_BIN` in `~/.config/vllm-helper/config.env` to its full path. +If `vllm` isn't on your PATH, run setup with `VLLM_BIN` set to its full path in the environment, or set `VLLM_BIN` in `~/.config/vllm-helper/config.env` afterwards. +The other providers' keys, such as `LLAMA_SERVER_BIN`, work the same way, and one set in the environment wins over a server on PATH. ## What setup writes diff --git a/docs/reference/cli/setup.md b/docs/reference/cli/setup.md index 19939b7..034552f 100644 --- a/docs/reference/cli/setup.md +++ b/docs/reference/cli/setup.md @@ -6,6 +6,12 @@ description: "First-time setup: detect hardware and the inference server, write +First-time setup: detect hardware and the inference server, write config.env and +secrets.env, and offer to adopt a running server. It never overwrites a config. + +The server's path comes from the provider's key in the environment when it is set, +such as LLAMA_SERVER_BIN or VLLM_BIN, and otherwise from PATH and where installers put it. + ## Usage ```console diff --git a/internal/cli/setup.go b/internal/cli/setup.go index 7129d4a..2b91d46 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "runtime" "strings" @@ -20,8 +21,14 @@ import ( func newSetupCmd() *cobra.Command { var providerName string c := &cobra.Command{ - Use: "setup", - Short: "First-time setup: detect hardware and the inference server, write config, offer to install the service", + Use: "setup", + Short: "First-time setup: detect hardware and the inference server, write config, offer to install the service", + Long: "First-time setup: detect hardware and the inference server, write config.env and\n" + + "secrets.env, and offer to adopt a running server. It never overwrites a config.\n\n" + + "The server's path comes from the provider's key in the environment when it is set,\n" + + "such as LLAMA_SERVER_BIN or VLLM_BIN, and otherwise from PATH and where installers put it.", + Example: " llm-helper setup\n" + + " LLAMA_SERVER_BIN=$HOME/llama.cpp/build/bin/llama-server llm-helper setup --provider llamacpp", GroupID: groupMachine, Args: cobra.NoArgs, RunE: func(*cobra.Command, []string) error { return runSetup(providerName) }, @@ -52,11 +59,14 @@ func runSetup(providerName string) error { _, statErr := os.Stat(dirs.ConfigFile()) bin := "" if statErr != nil { + if _, err := envBin(prov); err != nil { + return err + } bin = lookBin(prov) if prov.Name() == engine.VLLM { useMetalProbe(bin) } - } else if found, filled, err := fillMissingVLLMBin(dirs.ConfigFile(), findVLLM); err != nil { + } else if found, filled, err := fillMissingVLLMBin(dirs.ConfigFile(), findVLLMOrEnv); err != nil { return err } else if filled { fmt.Printf("Detected vllm at %s and recorded it as VLLM_BIN in %s\n\n", found, dirs.ConfigFile()) @@ -157,6 +167,8 @@ SERVICE_SCOPE=%s fmt.Printf("%s=%s in config.env.\n", other.BinKey(), lookBin(other)) } fmt.Println() + } else if p, _ := envBin(prov); p == bin { + fmt.Printf("Using %s at %s, from %s in the environment\n", prov.BinName(), bin, prov.BinKey()) } else { fmt.Printf("Detected %s at %s\n", prov.BinName(), bin) } @@ -233,12 +245,18 @@ func fillMissingVLLMBin(path string, find func() string) (string, bool, error) { return bin, true, nil } -// lookBin finds a provider's server, "" when there is none. vLLM's is looked -// for by findVLLM, which resolves symlinks because its venv's python is looked -// up beside it. Any other server is kept as found on PATH, so a package +// lookBin finds a provider's server, "" when there is none. An executable the +// environment names under the provider's bin key, such as LLAMA_SERVER_BIN, +// wins, because a distribution's package on PATH can be far older than the +// build the user means to serve with. vLLM's is otherwise looked for by +// findVLLM, which resolves symlinks because its venv's python is looked up +// beside it. Any other server is kept as found on PATH, so a package // manager's stable link is what config.env records and an upgrade does not // break it. func lookBin(prov engine.Provider) string { + if p, err := envBin(prov); err == nil && p != "" { + return p + } if prov.Name() == engine.VLLM { return findVLLM() } @@ -248,3 +266,32 @@ func lookBin(prov engine.Provider) string { } return p } + +// envBin is the server the environment names under prov's bin key, as an +// absolute path, or "" when it names none. A name rather than a path is +// looked up on PATH. It is an error for the variable to name something that +// is not an executable: setup would otherwise quietly record the server the +// user set it to avoid. +func envBin(prov engine.Provider) (string, error) { + v := strings.TrimSpace(os.Getenv(prov.BinKey())) + if v == "" { + return "", nil + } + p, err := exec.LookPath(v) + if err == nil { + p, err = filepath.Abs(p) + } + if err != nil { + return "", fmt.Errorf("%s=%s in the environment is not an executable (%w); fix it, or unset it to have setup look for %s", prov.BinKey(), v, err, prov.BinName()) + } + return p, nil +} + +// findVLLMOrEnv is findVLLM, with VLLM_BIN in the environment winning as +// lookBin lets it. +func findVLLMOrEnv() string { + if p, err := engine.Lookup(engine.VLLM); err == nil { + return lookBin(p) + } + return findVLLM() +} diff --git a/internal/cli/setup_internal_test.go b/internal/cli/setup_internal_test.go new file mode 100644 index 0000000..299d150 --- /dev/null +++ b/internal/cli/setup_internal_test.go @@ -0,0 +1,74 @@ +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/coder/llm-helper/internal/engine" +) + +// fakeExecutable writes an executable file called name in dir. +func fakeExecutable(t *testing.T, dir, name string) string { + t.Helper() + p := filepath.Join(dir, name) + if err := os.WriteFile(p, []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + return p +} + +// A distribution's llama-server on PATH can be years older than the build the +// user means to serve with; setup found /usr/bin/llama-server (0.2.0) and +// ignored LLAMA_SERVER_BIN. The engine's bin key in the environment wins, for +// every engine, and one that names nothing runnable is an error rather than a +// quiet fall back to the server the user was avoiding. +func TestSetupPrefersTheBinKeyFromTheEnvironment(t *testing.T) { + prov, err := engine.Lookup("llamacpp") + if err != nil { + t.Fatal(err) + } + onPath, mine := t.TempDir(), t.TempDir() + old := fakeExecutable(t, onPath, "llama-server") + built := fakeExecutable(t, mine, "llama-server") + t.Setenv("PATH", onPath) + + t.Setenv(prov.BinKey(), "") + if got := lookBin(prov); got != old { + t.Errorf("unset: lookBin = %q, want the one on PATH, %q", got, old) + } + + t.Setenv(prov.BinKey(), built) + if got := lookBin(prov); got != built { + t.Errorf("%s=%s: lookBin = %q", prov.BinKey(), built, got) + } + + // A bare name is looked up on PATH and recorded as a full path. + t.Setenv(prov.BinKey(), "llama-server") + if got, err := envBin(prov); err != nil || got != old { + t.Errorf("a bare name: envBin = %q, %v; want %q", got, err, old) + } + + notExec := filepath.Join(mine, "llama-server.txt") + if err := os.WriteFile(notExec, nil, 0o644); err != nil { + t.Fatal(err) + } + for _, bad := range []string{notExec, filepath.Join(mine, "missing")} { + t.Setenv(prov.BinKey(), bad) + if _, err := envBin(prov); err == nil || !strings.Contains(err.Error(), prov.BinKey()+"="+bad) { + t.Errorf("%s: envBin error = %v, want one naming the variable", bad, err) + } + } + + // vLLM's key is honored the same way, ahead of its own search. + vllm, err := engine.Lookup(engine.VLLM) + if err != nil { + t.Fatal(err) + } + venv := fakeExecutable(t, mine, "vllm") + t.Setenv(vllm.BinKey(), venv) + if got := findVLLMOrEnv(); got != venv { + t.Errorf("VLLM_BIN=%s: findVLLMOrEnv = %q", venv, got) + } +} From 2ec6e062131849ba7524fc2719aeab900e6eb044 Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Fri, 13 Nov 2026 21:05:04 +0000 Subject: [PATCH 06/15] setup: add --service-scope setup always wrote SERVICE_SCOPE=system on Linux. On a box where sudo asks for a password, every system-scope unit operation fails when run without a terminal, so the scope had to be changed in config.env by hand before the first up. --service-scope system|user picks it for a new config, like --provider. Left empty it is still the platform's default, which keeps the generated CLI reference the same on Linux and macOS. On a kept config that says otherwise, setup names the line to change rather than rewriting the file. The config text moves into newConfigEnv so what it writes can be tested. --- CHANGELOG.md | 2 + docs/install.md | 2 +- docs/reference/cli/setup.md | 10 +++ internal/cli/setup.go | 117 +++++++++++++++++----------- internal/cli/setup_internal_test.go | 35 +++++++++ 5 files changed, 121 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ac950c..93d455a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -86,6 +86,8 @@ the fact. - `setup` records the server named by the provider's key in the environment, such as `LLAMA_SERVER_BIN`, instead of the one on PATH, and refuses to run when that key names something that is not an executable. +- `setup --service-scope user` writes a config whose model services run in + the user's service manager, which needs no sudo. - `doctor` shows the GTT limit on a Strix Halo and, when it is well below 90% of RAM, the ttm `pages_limit` that raises it; see docs/memory.md. - `doctor` warns when an amdgpu GPU is present but the user cannot open diff --git a/docs/install.md b/docs/install.md index ab55b44..993c96c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -43,7 +43,7 @@ See [Hardware detection](./hardware.md). | `MLX_LM_BIN` | config.env | Full path to the `mlx_lm.server` executable. Required for MLX profiles. | | `LLAMA_SERVER_BIN` | config.env | Full path to the `llama-server` executable. Required for llama.cpp profiles. | | `SERVICE_NAME` | config.env | The single service unit earlier releases ran: `vllm.service` on Linux, `com.coder.vllm` on macOS. Read only by `migrate`. | -| `SERVICE_SCOPE` | config.env | `system` on Linux (needs sudo to control) or `user`. macOS defaults to `user`; use `system` on a headless Mac. | +| `SERVICE_SCOPE` | config.env | `system` on Linux (needs sudo) or `user`, as `setup --service-scope` sets it. macOS defaults to `user`; use `system` on a headless Mac. | | `DEFAULT_PROFILE` | config.env | The model Coder selects first. Written by `llm-helper default`. See [Several models at once](./multi-instance.md). | | `ROUTER_PORT` | config.env | Port the model router listens on. Written by `llm-helper router install-service`. | | `MODEL_BUDGET_GB` | config.env | Overrides the measured model budget. See [Memory budget](./memory.md). | diff --git a/docs/reference/cli/setup.md b/docs/reference/cli/setup.md index 034552f..4077c14 100644 --- a/docs/reference/cli/setup.md +++ b/docs/reference/cli/setup.md @@ -12,6 +12,8 @@ secrets.env, and offer to adopt a running server. It never overwrites a config. The server's path comes from the provider's key in the environment when it is set, such as LLAMA_SERVER_BIN or VLLM_BIN, and otherwise from PATH and where installers put it. +--service-scope user runs each model's service under this user, which needs no sudo. + ## Usage ```console @@ -28,3 +30,11 @@ llm-helper setup [flags] | Default | `vllm` | inference server new profiles use: llamacpp, mlx, vllm (only written to a new config) + +### --service-scope + +| | | +| ---- | -------- | +| Type | `string` | + +where each model's service runs: system (needs sudo) or user; default system on Linux, user on macOS (only written to a new config) diff --git a/internal/cli/setup.go b/internal/cli/setup.go index 2b91d46..2538880 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -19,30 +19,36 @@ import ( ) func newSetupCmd() *cobra.Command { - var providerName string + var providerName, scope string c := &cobra.Command{ Use: "setup", Short: "First-time setup: detect hardware and the inference server, write config, offer to install the service", Long: "First-time setup: detect hardware and the inference server, write config.env and\n" + "secrets.env, and offer to adopt a running server. It never overwrites a config.\n\n" + "The server's path comes from the provider's key in the environment when it is set,\n" + - "such as LLAMA_SERVER_BIN or VLLM_BIN, and otherwise from PATH and where installers put it.", + "such as LLAMA_SERVER_BIN or VLLM_BIN, and otherwise from PATH and where installers put it.\n\n" + + "--service-scope user runs each model's service under this user, which needs no sudo.", Example: " llm-helper setup\n" + - " LLAMA_SERVER_BIN=$HOME/llama.cpp/build/bin/llama-server llm-helper setup --provider llamacpp", + " LLAMA_SERVER_BIN=$HOME/llama.cpp/build/bin/llama-server llm-helper setup --provider llamacpp --service-scope user", GroupID: groupMachine, Args: cobra.NoArgs, - RunE: func(*cobra.Command, []string) error { return runSetup(providerName) }, + RunE: func(*cobra.Command, []string) error { + return runSetup(providerName, scope) + }, } c.Flags().StringVar(&providerName, "provider", engine.VLLM, "inference server new profiles use: "+strings.Join(engine.ProcessNames(), ", ")+" (only written to a new config)") + c.Flags().StringVar(&scope, "service-scope", "", + "where each model's service runs: system (needs sudo) or user; default system on Linux, user on macOS (only written to a new config)") return c } // runSetup is first-time setup for binary installs: detect hardware and any // existing vLLM, write config + secrets, install the service unit, and // offer to adopt a running server. Idempotent; never overwrites existing -// config and never restarts anything. -func runSetup(providerName string) error { +// config and never restarts anything. scope is "" for the platform's +// default. +func runSetup(providerName, scope string) error { prov, err := engine.Lookup(providerName) if err != nil { return err @@ -50,6 +56,14 @@ func runSetup(providerName string) error { if !prov.Has(engine.RuntimeProcess) { return fmt.Errorf("%s runs only in a container, so the service cannot serve it; launch it with 'llm-helper recipes launch'", prov.Name()) } + scopeAsked := scope != "" + switch scope { + case "": + scope = config.DefaultServiceScope + case "system", "user": + default: + return fmt.Errorf("--service-scope is system or user, not %q", scope) + } dirs := config.ResolveDirs() fmt.Printf("== %s setup ==\n", cmdName) fmt.Println() @@ -104,45 +118,17 @@ func runSetup(providerName string) error { if prov.Name() != engine.VLLM { fmt.Printf("--provider only shapes a new config; set PROVIDER=%s and %s in it by hand.\n", prov.Name(), prov.BinKey()) } + if kept, err := config.ParseEnvFile(dirs.ConfigFile()); err == nil && scopeAsked { + have := kept.Get("SERVICE_SCOPE") + if have == "" { + have = config.DefaultServiceScope + } + if have != scope { + fmt.Printf("--service-scope only shapes a new config; set SERVICE_SCOPE=%s in it by hand.\n", scope) + } + } } else { - content := fmt.Sprintf(`# llm-helper global config. Sourced (exported) into the server's -# environment whenever a model starts -- values here apply to every profile. - -# First port a model's OpenAI-compatible endpoint listens on. Each model -# running at the same time takes the next free port from here. -PORT=8001 - -# The inference server new profiles are made for: %s. Each profile -# records its own, so changing this leaves existing profiles alone. -PROVIDER=%s - -# Full path to the %s executable (usually inside a venv). REQUIRED. -%s=%s - -# How each model's %s unit is managed: "system" (needs sudo) or "user". -SERVICE_SCOPE=%s - -# Budget for model weights, used by 'llm-helper search' to flag models that -# won't fit. Measured from this machine on every run -- currently ~%d GB, being -# %s. Uncomment only to override that. -#MODEL_BUDGET_GB=%d - -# How many full-length requests each profile's KV cache is sized to hold. -# Defaults to %d. A property of the workload rather than the machine: lower it -# on a single-user box, raise it for a shared endpoint. -#KV_CONTEXT_TARGET=%d - -# --- Optional: Coder Agents sync ------------------------------------------- -# If all three are set (token in secrets.env), a Coder deployment's AI-model -# picker is kept in sync with the models running here. -#CODER_URL=https://coder.example.com -#CODER_AI_PROVIDER_ID= -# Only needed if the deployment has more than one organization. -#CODER_ORGANIZATION=default -`, strings.Join(engine.ProcessNames(), ", "), prov.Name(), prov.BinName(), prov.BinKey(), bin, - config.ServiceManager, config.DefaultServiceScope, - hw.BudgetGB, hw.BudgetReason, hw.BudgetGB, - detect.DefaultContextTarget, detect.DefaultContextTarget) + content := newConfigEnv(prov, bin, scope, hw) if err := os.WriteFile(dirs.ConfigFile(), []byte(content), 0o644); err != nil { return err } @@ -217,6 +203,49 @@ VLLM_API_KEY=%s return nil } +// newConfigEnv is the config.env setup writes for a new install: new +// profiles for prov, served by bin, with each model's service in scope. +func newConfigEnv(prov engine.Provider, bin, scope string, hw detect.Hardware) string { + return fmt.Sprintf(`# llm-helper global config. Sourced (exported) into the server's +# environment whenever a model starts -- values here apply to every profile. + +# First port a model's OpenAI-compatible endpoint listens on. Each model +# running at the same time takes the next free port from here. +PORT=8001 + +# The inference server new profiles are made for: %s. Each profile +# records its own, so changing this leaves existing profiles alone. +PROVIDER=%s + +# Full path to the %s executable (usually inside a venv). REQUIRED. +%s=%s + +# How each model's %s unit is managed: "system" (needs sudo) or "user". +SERVICE_SCOPE=%s + +# Budget for model weights, used by 'llm-helper search' to flag models that +# won't fit. Measured from this machine on every run -- currently ~%d GB, being +# %s. Uncomment only to override that. +#MODEL_BUDGET_GB=%d + +# How many full-length requests each profile's KV cache is sized to hold. +# Defaults to %d. A property of the workload rather than the machine: lower it +# on a single-user box, raise it for a shared endpoint. +#KV_CONTEXT_TARGET=%d + +# --- Optional: Coder Agents sync ------------------------------------------- +# If all three are set (token in secrets.env), a Coder deployment's AI-model +# picker is kept in sync with the models running here. +#CODER_URL=https://coder.example.com +#CODER_AI_PROVIDER_ID= +# Only needed if the deployment has more than one organization. +#CODER_ORGANIZATION=default +`, strings.Join(engine.ProcessNames(), ", "), prov.Name(), prov.BinName(), prov.BinKey(), bin, + config.ServiceManager, scope, + hw.BudgetGB, hw.BudgetReason, hw.BudgetGB, + detect.DefaultContextTarget, detect.DefaultContextTarget) +} + // fillMissingVLLMBin records a vllm installed since config.env was written // without one. The first setup leaves VLLM_BIN empty when it finds nothing and // tells the user to install vLLM and run setup again, so the rerun has to fill diff --git a/internal/cli/setup_internal_test.go b/internal/cli/setup_internal_test.go index 299d150..fa746e2 100644 --- a/internal/cli/setup_internal_test.go +++ b/internal/cli/setup_internal_test.go @@ -6,6 +6,8 @@ import ( "strings" "testing" + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/detect" "github.com/coder/llm-helper/internal/engine" ) @@ -72,3 +74,36 @@ func TestSetupPrefersTheBinKeyFromTheEnvironment(t *testing.T) { t.Errorf("VLLM_BIN=%s: findVLLMOrEnv = %q", venv, got) } } + +// A system-scope unit needs sudo, which fails without a terminal on a box +// where it asks for a password, so setup can write a user-scope config. A +// scope that is neither is refused before anything is written. +func TestSetupWritesTheServiceScopeAskedFor(t *testing.T) { + prov, err := engine.Lookup("llamacpp") + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "config.env") + if err := os.WriteFile(path, []byte(newConfigEnv(prov, "/opt/llama/llama-server", "user", detect.Hardware{})), 0o600); err != nil { + t.Fatal(err) + } + env, err := config.ParseEnvFile(path) + if err != nil { + t.Fatal(err) + } + if got := env.Get("SERVICE_SCOPE"); got != "user" { + t.Errorf("SERVICE_SCOPE = %q, want user", got) + } + if got := env.Get("LLAMA_SERVER_BIN"); got != "/opt/llama/llama-server" { + t.Errorf("LLAMA_SERVER_BIN = %q", got) + } + + dir := t.TempDir() + t.Setenv("VLLM_HELPER_CONFIG_DIR", dir) + if err := runSetup("llamacpp", "users"); err == nil || !strings.Contains(err.Error(), "system or user") { + t.Errorf("--service-scope users: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, "config.env")); err == nil { + t.Error("config.env was written for a refused scope") + } +} From 00d4a9d9ca9ff851cf5913a1ea0c680670641685 Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Fri, 13 Nov 2026 21:05:46 +0000 Subject: [PATCH 07/15] doctor: warn when a user-scope service will not start at boot With SERVICE_SCOPE=user, each model's unit lives in the user's systemd manager, which systemd starts at the user's first login and stops at the last logout unless the user lingers. A box set up that way served nothing after a reboot until 'sudo loginctl enable-linger' was run. up already tries to enable lingering, but without sudo that can fail with only a note. Doctor on Linux now checks /var/lib/systemd/linger for the user, which needs no loginctl call, and names the command when the file is missing. A machine without /var/lib/systemd is not checked. --- CHANGELOG.md | 4 ++- docs/install.md | 3 ++ internal/cli/detectcmds.go | 3 ++ internal/cli/linger.go | 28 ++++++++++++++++++ internal/cli/linger_internal_test.go | 44 ++++++++++++++++++++++++++++ internal/cli/linger_linux.go | 18 ++++++++++++ internal/cli/linger_other.go | 9 ++++++ 7 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 internal/cli/linger.go create mode 100644 internal/cli/linger_internal_test.go create mode 100644 internal/cli/linger_linux.go create mode 100644 internal/cli/linger_other.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 93d455a..9658a25 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,7 +87,9 @@ the fact. such as `LLAMA_SERVER_BIN`, instead of the one on PATH, and refuses to run when that key names something that is not an executable. - `setup --service-scope user` writes a config whose model services run in - the user's service manager, which needs no sudo. + the user's service manager, which needs no sudo. On Linux, `doctor` warns + when lingering is off for that user, since the models then do not start at + boot. - `doctor` shows the GTT limit on a Strix Halo and, when it is well below 90% of RAM, the ttm `pages_limit` that raises it; see docs/memory.md. - `doctor` warns when an amdgpu GPU is present but the user cannot open diff --git a/docs/install.md b/docs/install.md index 993c96c..81a4a35 100644 --- a/docs/install.md +++ b/docs/install.md @@ -59,6 +59,9 @@ See [Hardware detection](./hardware.md). | `HF_TOKEN` | either | HuggingFace token, needed only for gated or private models. A value in `secrets.env` wins. | | `GITHUB_TOKEN` | either | Token for reading releases while the repository is private. A value in `secrets.env` wins; `upgrade` also accepts it in the environment. | +With `SERVICE_SCOPE=user` on Linux, each model's unit runs in the user's systemd manager, which starts at boot only when lingering is on for the user. +`up` tries to turn it on, and `doctor` warns while it is off; turn it on yourself with `sudo loginctl enable-linger $USER`. + ## Upgrade ```sh diff --git a/internal/cli/detectcmds.go b/internal/cli/detectcmds.go index 5fe4c4d..1ef0b23 100644 --- a/internal/cli/detectcmds.go +++ b/internal/cli/detectcmds.go @@ -174,6 +174,9 @@ func gatherDoctor() doctorReport { } r.Warnings = append(r.Warnings, hardwareWarnings(r.Hardware, running)...) r.Warnings = append(r.Warnings, gpuDeviceWarnings(r.Hardware.Accelerators)...) + if cfg != nil { + r.Warnings = append(r.Warnings, serviceScopeWarnings(cfg)...) + } // Drift checks: the part a human wouldn't spot at a glance. if cfg != nil { diff --git a/internal/cli/linger.go b/internal/cli/linger.go new file mode 100644 index 0000000..2ae8e3e --- /dev/null +++ b/internal/cli/linger.go @@ -0,0 +1,28 @@ +package cli + +import ( + "fmt" + "os" + "path/filepath" +) + +// lingerWarnings warns when each model's unit is in user's systemd manager +// but user does not linger. Without lingering, systemd starts that manager at +// the user's first login and stops it at the last logout, so the models do +// not start at boot. lingerDir is /var/lib/systemd/linger, which holds a file +// per lingering user; a machine without its parent directory does not run +// systemd and gets no warning. +func lingerWarnings(scope, user, lingerDir string) []string { + if scope != "user" || user == "" { + return nil + } + if _, err := os.Stat(filepath.Dir(lingerDir)); err != nil { + return nil + } + if _, err := os.Stat(filepath.Join(lingerDir, user)); err == nil { + return nil + } + return []string{fmt.Sprintf( + "SERVICE_SCOPE=user, but lingering is off for %s, so systemd runs the model units only while %s is logged in and they do not start at boot. "+ + "Turn it on with 'sudo loginctl enable-linger %s'.", user, user, user)} +} diff --git a/internal/cli/linger_internal_test.go b/internal/cli/linger_internal_test.go new file mode 100644 index 0000000..da082a4 --- /dev/null +++ b/internal/cli/linger_internal_test.go @@ -0,0 +1,44 @@ +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// A user-scope unit without lingering waits for a login, so a machine that +// reboots unattended serves nothing. Only the user scope is checked, and only +// on a machine that runs systemd. +func TestLingerWarnings(t *testing.T) { + systemd := filepath.Join(t.TempDir(), "systemd") + linger := filepath.Join(systemd, "linger") + if err := os.MkdirAll(systemd, 0o755); err != nil { + t.Fatal(err) + } + + // No user lingers yet, so the folder itself is missing. + w := lingerWarnings("user", "amd", linger) + if len(w) != 1 || !strings.Contains(w[0], "sudo loginctl enable-linger amd") || !strings.Contains(w[0], "do not start at boot") { + t.Errorf("user scope without lingering: %q", w) + } + if w := lingerWarnings("system", "amd", linger); len(w) != 0 { + t.Errorf("system scope: %q", w) + } + + if err := os.MkdirAll(linger, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(linger, "amd"), nil, 0o644); err != nil { + t.Fatal(err) + } + if w := lingerWarnings("user", "amd", linger); len(w) != 0 { + t.Errorf("lingering on: %q", w) + } + if w := lingerWarnings("user", "other", linger); len(w) != 1 { + t.Errorf("another user lingers, this one does not: %q", w) + } + if w := lingerWarnings("user", "amd", filepath.Join(t.TempDir(), "none", "linger")); len(w) != 0 { + t.Errorf("no systemd: %q", w) + } +} diff --git a/internal/cli/linger_linux.go b/internal/cli/linger_linux.go new file mode 100644 index 0000000..8e9af3b --- /dev/null +++ b/internal/cli/linger_linux.go @@ -0,0 +1,18 @@ +package cli + +import ( + "os" + "os/user" + + "github.com/coder/llm-helper/internal/config" +) + +// serviceScopeWarnings checks that the configured service scope starts the +// models at boot on this machine. +func serviceScopeWarnings(cfg *config.Config) []string { + name := os.Getenv("USER") + if u, err := user.Current(); err == nil { + name = u.Username + } + return lingerWarnings(cfg.ServiceScope(), name, "/var/lib/systemd/linger") +} diff --git a/internal/cli/linger_other.go b/internal/cli/linger_other.go new file mode 100644 index 0000000..2237b52 --- /dev/null +++ b/internal/cli/linger_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package cli + +import "github.com/coder/llm-helper/internal/config" + +// serviceScopeWarnings has nothing to check off Linux: a LaunchAgent starts +// at login by design, and a headless Mac is told to use the system scope. +func serviceScopeWarnings(*config.Config) []string { return nil } From 654e08779c90b2c31c4070070dcd2f9df7720612 Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Fri, 13 Nov 2026 21:07:07 +0000 Subject: [PATCH 08/15] inspect: accept : A llama.cpp profile's MODEL is :, and pasting it into inspect sent the whole string to the Hub as the repo id, which answered with "hub lookup: HTTP 401". inspect now looks up the repo and, given a quant, checks the repo holds it and prints its size against the budget, or names the quants it does hold. --json prints the repo's detail as before. --- CHANGELOG.md | 3 ++ internal/cli/inspect_internal_test.go | 61 +++++++++++++++++++++++++++ internal/cli/search.go | 29 ++++++++++++- 3 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 internal/cli/inspect_internal_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 9658a25..214e177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -140,6 +140,9 @@ the fact. ### Fixed +- `inspect :`, the form a llama.cpp profile's `MODEL` takes, + failed with "hub lookup: HTTP 401" because the quant was sent as part of + the repo id. It now looks up the repo and reports that quant's size. - `inspect`, `new` and the panel sized the KV cache of GLM-5.3-Flash about 50 times too large: its 34 linear-attention layers counted as full attention, and its 11 latent-attention layers as 64 full heads. A diff --git a/internal/cli/inspect_internal_test.go b/internal/cli/inspect_internal_test.go new file mode 100644 index 0000000..8e2d589 --- /dev/null +++ b/internal/cli/inspect_internal_test.go @@ -0,0 +1,61 @@ +package cli + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "github.com/coder/llm-helper/internal/hf" +) + +// A llama.cpp profile's MODEL is :, and inspect sent all of it +// to the Hub as the repo id, which answered with a 401. The repo alone is +// looked up. +func TestInspectLooksUpTheRepoOfARepoQuant(t *testing.T) { + var mu sync.Mutex + var paths []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + paths = append(paths, r.URL.Path) + mu.Unlock() + http.NotFound(w, r) + })) + defer srv.Close() + defer hf.SetBaseURLForTest(srv.URL)() + + err := runInspect(context.Background(), []string{"unsloth/DeepSeek-V4-Flash-Vision-Exp-GGUF:UD-IQ3_XXS"}, true) + if err == nil { + t.Fatal("inspect succeeded against a Hub that knows no model") + } + mu.Lock() + defer mu.Unlock() + if len(paths) == 0 || paths[0] != "/api/models/unsloth/DeepSeek-V4-Flash-Vision-Exp-GGUF" { + t.Errorf("asked the Hub for %q, want the repo without its quant", paths) + } +} + +// A quant the repo does not hold is named as such, with the ones it does. +func TestInspectRefusesAQuantTheRepoDoesNotHold(t *testing.T) { + t.Setenv("VLLM_HELPER_CONFIG_DIR", t.TempDir()) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/models/org/m-GGUF": + fmt.Fprint(w, `{"id":"org/m-GGUF","tags":["gguf"]}`) + case "/api/models/org/m-GGUF/tree/main": + fmt.Fprint(w, `[{"type":"file","path":"m-Q4_K_M.gguf","size":4000000000},{"type":"file","path":"m-Q8_0.gguf","size":8000000000}]`) + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + defer hf.SetBaseURLForTest(srv.URL)() + + err := runInspect(context.Background(), []string{"org/m-GGUF:Q5_K_M"}, true) + if err == nil || !strings.Contains(err.Error(), `org/m-GGUF has no quant called "Q5_K_M"`) || !strings.Contains(err.Error(), "Q8_0") { + t.Errorf("err = %v, want the missing quant and the ones there are", err) + } +} diff --git a/internal/cli/search.go b/internal/cli/search.go index 4abbb34..84807ea 100644 --- a/internal/cli/search.go +++ b/internal/cli/search.go @@ -132,7 +132,11 @@ func runInspect(ctx context.Context, args []string, jsonOut bool) error { if len(args) < 1 { return errors.New("usage: llm-helper inspect [--json]") } - m, err := hf.Lookup(ctx, args[0]) + // A profile's MODEL names a quant as :, so inspect takes it + // in that form too. The quant is not part of the repo id the Hub knows, + // which answers an unknown id with a 401. + repo, quant, _ := strings.Cut(args[0], ":") + m, err := hf.Lookup(ctx, repo) if err != nil { return err } @@ -142,10 +146,33 @@ func runInspect(ctx context.Context, args []string, jsonOut bool) error { return err } d := inspectModel(ctx, m, cfg) + var picked *hf.Variant + if quant != "" { + v, ok := resolve.FindVariant(d.All, quant) + if !ok { + have := make([]string, 0, len(d.All)) + for _, x := range d.All { + have = append(have, x.Label) + } + return fmt.Errorf("%s has no quant called %q -- it has: %s", repo, quant, strings.Join(have, ", ")) + } + picked = &v + } if jsonOut { return printJSON(d) } printModelDetail(d, cfg) + if picked != nil { + fit := "" + switch { + case !d.BudgetKnown: + case picked.SizeGB > d.BudgetGB: + fit = fmt.Sprintf(", too big for the ~%.0f GB budget", d.BudgetGB) + default: + fit = fmt.Sprintf(", fits the ~%.0f GB budget", d.BudgetGB) + } + fmt.Printf("\n %s:%s is %.1f GB%s\n", d.ID, picked.Label, picked.SizeGB, fit) + } return nil } From 9adb7afbc883a6cd54f3199573c59d68876fd2ff Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Fri, 13 Nov 2026 21:09:02 +0000 Subject: [PATCH 09/15] docs: llama.cpp on AMD GPUs under Linux, measured on a Ryzen AI Max The llama.cpp section said Metal, CUDA or the CPU and offered only Homebrew. On a Strix Halo box the working setup was the release's Vulkan or ROCm tarball, the render group, LLAMA_SERVER_BIN at setup, the user scope with lingering, and a raised GTT limit; each of those cost a manual step, so they are now written down in order. The example records the box's numbers for DeepSeek V4 Flash Vision UD-IQ3_XXS with both builds, beside the Mac Studio ones, and the note that llama.cpp's token counters do not move while it reads a long prompt. --- CHANGELOG.md | 3 ++ catalogue/engines/llamacpp.json | 2 +- docs/providers.md | 67 ++++++++++++++++++++++++++++++++- 3 files changed, 69 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 214e177..908d1e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -90,6 +90,9 @@ the fact. the user's service manager, which needs no sudo. On Linux, `doctor` warns when lingering is off for that user, since the models then do not start at boot. +- Docs: llama.cpp on an AMD GPU under Linux (the release's Vulkan and ROCm + builds, and the `render` group), and DeepSeek V4 Flash Vision measured on a + Ryzen AI Max with both. - `doctor` shows the GTT limit on a Strix Halo and, when it is well below 90% of RAM, the ttm `pages_limit` that raises it; see docs/memory.md. - `doctor` warns when an amdgpu GPU is present but the user cannot open diff --git a/catalogue/engines/llamacpp.json b/catalogue/engines/llamacpp.json index a66a4b2..a3040d1 100644 --- a/catalogue/engines/llamacpp.json +++ b/catalogue/engines/llamacpp.json @@ -2,7 +2,7 @@ "schema_version": 1, "id": "llamacpp", "display_name": "llama.cpp", - "why": "llama.cpp's OpenAI-compatible server, llama-server. It serves GGUF on Metal, CUDA or the CPU, and sizes its own context and GPU layers to fit the machine, so a profile needs only the model. MODEL is passed as --alias too, so /v1/models reports exactly MODEL whether the model comes from the Hub or a file. It checks an API key, which it reads from LLAMA_API_KEY so the key is on no command line; with one set it binds every interface, and without one it stays on 127.0.0.1. It downloads a repo's GGUF itself, and its /health answers 200 only once the model is loaded.", + "why": "llama.cpp's OpenAI-compatible server, llama-server. It serves GGUF on Metal, CUDA, Vulkan, ROCm or the CPU, and sizes its own context and GPU layers to fit the machine, so a profile needs only the model. MODEL is passed as --alias too, so /v1/models reports exactly MODEL whether the model comes from the Hub or a file. It checks an API key, which it reads from LLAMA_API_KEY so the key is on no command line; with one set it binds every interface, and without one it stays on 127.0.0.1. It downloads a repo's GGUF itself, and its /health answers 200 only once the model is loaded.", "runtime": ["process"], "install": { "any": "brew install llama.cpp" diff --git a/docs/providers.md b/docs/providers.md index 99be24a..bbebfdd 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -90,11 +90,11 @@ llm-helper set -- EXTRA_ARGS='--host 0.0.0.0 --max-tokens 32768' ## llama.cpp -[llama.cpp](https://github.com/ggml-org/llama.cpp)'s `llama-server` serves GGUF models on Metal, CUDA or the CPU. +[llama.cpp](https://github.com/ggml-org/llama.cpp)'s `llama-server` serves GGUF models on Metal, CUDA, Vulkan, ROCm or the CPU. It sizes its context and GPU layers to fit the machine, so a llama.cpp profile needs only the model. Use a repo of GGUF files, such as one from [bartowski](https://huggingface.co/bartowski) or [unsloth](https://huggingface.co/unsloth). -1. Install llama.cpp. Homebrew has it on macOS and Linux, and the [releases](https://github.com/ggml-org/llama.cpp/releases) have prebuilt binaries: +1. Install llama.cpp. Homebrew has it on macOS and Linux, and the [releases](https://github.com/ggml-org/llama.cpp/releases) have prebuilt binaries; for an AMD GPU on Linux, see [AMD GPUs on Linux](#amd-gpus-on-linux): ```sh brew install llama.cpp @@ -129,11 +129,74 @@ Things to know about `llama-server`: - **It downloads the model when it starts.** The first start of a profile takes as long as the download, and the GGUF goes to the same Hugging Face cache vLLM uses. - **Its health check waits for the model.** `status` and `up --wait` report the endpoint up only once the model is loaded and it can serve. - **Context comes from the model,** then shrinks to fit memory. Set it with `-c ` in `EXTRA_ARGS`. +- **Its token counters stand still while it reads a prompt.** `llamacpp:prompt_tokens_total` and `llamacpp:tokens_predicted_total` in `/metrics` stayed flat for 181 seconds while b12364 read a 45,000-token prompt, so a flat counter during a long prompt is not a stuck server. ```sh llm-helper set -- EXTRA_ARGS='-c 65536' ``` +### AMD GPUs on Linux + +llama.cpp serves an AMD GPU through Vulkan or ROCm, and every [release](https://github.com/ggml-org/llama.cpp/releases) has a prebuilt Ubuntu x64 build of each. +A distribution's own `llama-server` can be far older: Debian's `/usr/bin/llama-server` is a 0.2.0 build, and `setup` finds it on PATH unless told otherwise. + +1. Download a build and unpack it. + The Vulkan one, for example: + + ```sh + curl -LO https://github.com/ggml-org/llama.cpp/releases/download/b12364/llama-b12364-bin-ubuntu-vulkan-x64.tar.gz + mkdir -p ~/llama.cpp + tar xzf llama-b12364-bin-ubuntu-vulkan-x64.tar.gz -C ~/llama.cpp --strip-components=1 + ``` + +2. Let your user open the GPU. + `/dev/kfd` and `/dev/dri/renderD*` belong to the `render` group, and without it the ROCm build fails with "no ROCm-capable device is detected". + `doctor` warns when you cannot open them. + + ```sh + sudo usermod -aG render,video $USER + ``` + + Log out and back in for the groups to apply. + A user's systemd manager that lingers keeps the groups it started with, so reboot if your models run with `SERVICE_SCOPE=user`. + +3. Run setup with the server named in `LLAMA_SERVER_BIN`. + The user scope needs no sudo, and lingering starts the models at boot: + + ```sh + LLAMA_SERVER_BIN=$HOME/llama.cpp/llama-server llm-helper setup --provider llamacpp --service-scope user + sudo loginctl enable-linger $USER + ``` + +On a Ryzen AI Max (Strix Halo), the GPU loads the model into GTT memory, which is capped well below RAM by default; raise the cap first, as "AMD Ryzen AI Max (Strix Halo)" in [Memory budget](./memory.md) describes. + +### Example: DeepSeek V4 Flash Vision on a Ryzen AI Max + +This ran on a Ryzen AI Max with the Radeon 8060S and 128 GB, with the GTT limit raised to 120 GB, and llama.cpp b12364's Vulkan and ROCm 10 builds from the releases. +The model is Unsloth's `UD-IQ3_XXS` GGUF (103 GB), served as a shared endpoint with 4 slots. +llama-server finds the vision file (`mmproj`) in the same repo by itself, with `--offline` too. + +```sh +llm-helper new unsloth/DeepSeek-V4-Flash-Vision-Exp-GGUF --quant UD-IQ3_XXS +llm-helper set deepseek-v4-flash-vision-exp-gguf-ud-iq3_xxs -- \ + EXTRA_ARGS="-c 1048576 -np 4 -ngl all --fit off -fa on -b 4096 -ub 2048 --temp 1.0 --top-p 0.95 --min-p 0.0 --metrics --offline" +``` + +| | ROCm 10 | Vulkan (RADV) | +| -------------------------- | ------------------------ | ------------------------ | +| Load to healthy | 22 s | 21 s | +| One request, short prompt | 15.2 tokens/sec | 16.9 tokens/sec | +| Prompt read, 10,449 tokens | 154 tokens/sec | 201 tokens/sec | +| Prompt read, 38,968 tokens | 118 tokens/sec | 128 tokens/sec | +| Two requests at once | 12.0 each | 12.0 each | +| Four requests at once | about 8.3 each, 33 total | about 8.2 each, 33 total | +| GTT memory in use | 113.7 GB | 114.5 GB | + +- Vulkan writes and reads prompts faster than ROCm on one request, and the two are level with several at once. +- With the model loaded, about 15 GB of the 128 GB was still free. + At the stock GTT limit of 101 GB, the model does not fit. +- A tool call came back as correct `tool_calls`, a needle in a long context was found, and a red image was answered "Red." + ### Prompt cache `llama-server` keeps the prompts of finished requests in host memory, up to `--cache-ram` MiB (8192 by default), and reuses the longest matching start of a new prompt instead of reading it again. From 8651eb0c4c1693da088408894e7080501b2e400d Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Fri, 13 Nov 2026 21:15:52 +0000 Subject: [PATCH 10/15] setup: say when a kept config.env ignores the bin key Setup never edits an existing config.env, so a second run with LLAMA_SERVER_BIN (or any provider's bin key) set in the environment did nothing and printed nothing, even when the key named a path that does not exist. It looked as if the new server had been recorded. Setup now prints a note for each provider bin key set in the environment that differs from config.env, or that names nothing runnable, with the line to put in config.env by hand. It still does not change the file. --- CHANGELOG.md | 4 ++- docs/install.md | 1 + internal/cli/setup.go | 49 +++++++++++++++++++++---- internal/cli/setup_internal_test.go | 56 +++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 908d1e8..d8f7c38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,7 +85,9 @@ the fact. `pci_devices`, and `hardware report` records `gtt_bytes`. - `setup` records the server named by the provider's key in the environment, such as `LLAMA_SERVER_BIN`, instead of the one on PATH, and refuses to run - when that key names something that is not an executable. + when that key names something that is not an executable. With a + `config.env` already in place, it keeps the file and says when the key in + the environment differs from it. - `setup --service-scope user` writes a config whose model services run in the user's service manager, which needs no sudo. On Linux, `doctor` warns when lingering is off for that user, since the models then do not start at diff --git a/docs/install.md b/docs/install.md index 81a4a35..254fb8f 100644 --- a/docs/install.md +++ b/docs/install.md @@ -24,6 +24,7 @@ On Linux that means an NVIDIA GPU with a vLLM usually living in a venv, or llama If `vllm` isn't on your PATH, run setup with `VLLM_BIN` set to its full path in the environment, or set `VLLM_BIN` in `~/.config/vllm-helper/config.env` afterwards. The other providers' keys, such as `LLAMA_SERVER_BIN`, work the same way, and one set in the environment wins over a server on PATH. +When `config.env` already exists, setup keeps it, and says when a key in the environment differs from the one in the file so you can change the file by hand. ## What setup writes diff --git a/internal/cli/setup.go b/internal/cli/setup.go index 2538880..1cef14e 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -118,13 +118,18 @@ func runSetup(providerName, scope string) error { if prov.Name() != engine.VLLM { fmt.Printf("--provider only shapes a new config; set PROVIDER=%s and %s in it by hand.\n", prov.Name(), prov.BinKey()) } - if kept, err := config.ParseEnvFile(dirs.ConfigFile()); err == nil && scopeAsked { - have := kept.Get("SERVICE_SCOPE") - if have == "" { - have = config.DefaultServiceScope + if kept, err := config.ParseEnvFile(dirs.ConfigFile()); err == nil { + if scopeAsked { + have := kept.Get("SERVICE_SCOPE") + if have == "" { + have = config.DefaultServiceScope + } + if have != scope { + fmt.Printf("--service-scope only shapes a new config; set SERVICE_SCOPE=%s in it by hand.\n", scope) + } } - if have != scope { - fmt.Printf("--service-scope only shapes a new config; set SERVICE_SCOPE=%s in it by hand.\n", scope) + for _, note := range envBinNotes(kept) { + fmt.Println(note) } } } else { @@ -316,6 +321,38 @@ func envBin(prov engine.Provider) (string, error) { return p, nil } +// envBinNotes says which server bin keys set in the environment a kept +// config.env does not follow. Setup never edits a kept config, so without a +// note LLAMA_SERVER_BIN=... on a second run looks as if it took effect. +func envBinNotes(kept *config.EnvFile) []string { + var notes []string + for _, name := range engine.ProcessNames() { + prov, err := engine.Lookup(name) + if err != nil || prov.BinKey() == "" { + continue + } + key := prov.BinKey() + v := strings.TrimSpace(os.Getenv(key)) + if v == "" { + continue + } + p, err := envBin(prov) + if err != nil { + notes = append(notes, fmt.Sprintf("%s=%s in the environment is not an executable; setup keeps config.env as it is.", key, v)) + continue + } + have := kept.Get(key) + if have == p { + continue + } + if have == "" { + have = "unset" + } + notes = append(notes, fmt.Sprintf("%s in the environment (%s) differs from config.env (%s); setup keeps config.env, so set %s=%s in it by hand.", key, p, have, key, p)) + } + return notes +} + // findVLLMOrEnv is findVLLM, with VLLM_BIN in the environment winning as // lookBin lets it. func findVLLMOrEnv() string { diff --git a/internal/cli/setup_internal_test.go b/internal/cli/setup_internal_test.go index fa746e2..85d4402 100644 --- a/internal/cli/setup_internal_test.go +++ b/internal/cli/setup_internal_test.go @@ -107,3 +107,59 @@ func TestSetupWritesTheServiceScopeAskedFor(t *testing.T) { t.Error("config.env was written for a refused scope") } } + +// Setup keeps an existing config.env, so a bin key set in the environment on +// a second run did nothing and said nothing. It now says when the key differs +// from the kept config or names nothing runnable, and stays quiet otherwise. +func TestSetupSaysWhenAKeptConfigIgnoresTheBinKey(t *testing.T) { + prov, err := engine.Lookup("llamacpp") + if err != nil { + t.Fatal(err) + } + key := prov.BinKey() + for _, name := range engine.ProcessNames() { + if other, err := engine.Lookup(name); err == nil && other.BinKey() != "" { + t.Setenv(other.BinKey(), "") + } + } + dir := t.TempDir() + recorded := fakeExecutable(t, t.TempDir(), "llama-server") + built := fakeExecutable(t, dir, "llama-server") + cfg := filepath.Join(dir, "config.env") + if err := os.WriteFile(cfg, []byte(key+"="+recorded+"\n"), 0o644); err != nil { + t.Fatal(err) + } + kept, err := config.ParseEnvFile(cfg) + if err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + name, env string + want []string + }{ + {"unset", "", nil}, + {"the same", recorded, nil}, + {"another", built, []string{key + " in the environment (" + built + ") differs from config.env (" + recorded + ")", "set " + key + "=" + built}}, + {"not an executable", filepath.Join(dir, "missing"), []string{key + "=" + filepath.Join(dir, "missing") + " in the environment is not an executable", "keeps config.env"}}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(key, tc.env) + notes := envBinNotes(kept) + if tc.want == nil { + if len(notes) != 0 { + t.Errorf("notes = %q, want none", notes) + } + return + } + if len(notes) != 1 { + t.Fatalf("notes = %q, want one", notes) + } + for _, w := range tc.want { + if !strings.Contains(notes[0], w) { + t.Errorf("note %q lacks %q", notes[0], w) + } + } + }) + } +} From 29898c672359d8dee9c244ce0c90d713fed59ba3 Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Fri, 13 Nov 2026 21:16:04 +0000 Subject: [PATCH 11/15] hardware: replace the Strix Halo report with a measured one The report's gtt_bytes was read from sysfs by hand, because the build that wrote it did not record the GTT limit. This one is the output of a build that does, on the same machine with the ttm pages_limit raised, so every reading comes from llm-helper itself. The stock limit stays covered by the unit tests, which build their readings rather than load this file. --- catalogue/hardware/reports/amd_strix_halo.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalogue/hardware/reports/amd_strix_halo.json b/catalogue/hardware/reports/amd_strix_halo.json index 9d604c6..465a63b 100644 --- a/catalogue/hardware/reports/amd_strix_halo.json +++ b/catalogue/hardware/reports/amd_strix_halo.json @@ -1,5 +1,5 @@ { - "provenance": "measured: 'llm-helper hardware report' 0.6.1-dev.63314bb on a Ryzen AI Max with the Radeon 8060S (gfx1151) and 128 GB, amdgpu driver, 2026-11-13. That build did not read the GTT limit, so gtt_bytes is mem_info_gtt_total read from sysfs by hand at the stock ttm pages_limit; detected is what llm-helper concludes from these readings now.", + "provenance": "measured: llm-helper built from 9adb7af on an AMD Ryzen AI Max (Strix Halo, Radeon 8060S) with 128 GB, ttm pages_limit raised to 29360128", "readings": { "schema_version": 1, "os": "linux", @@ -7,7 +7,7 @@ "system": "Advanced Micro Devices, Inc. (AMD) RAH-001", "mem_total_bytes": 134314070016, "nvidia_smi": false, - "pci": [{ "vendor": "0x1002", "device": "0x1586", "class": "0x038000", "driver": "amdgpu", "vram_bytes": 536870912, "gtt_bytes": 101139496960 }] + "pci": [{ "vendor": "0x1002", "device": "0x1586", "class": "0x038000", "driver": "amdgpu", "vram_bytes": 536870912, "gtt_bytes": 120259084288 }] }, "detected": { "hardware_id": "amd_strix_halo", From fa9b0bcfa0cb06098bc7c43c369727c475063210 Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Sun, 15 Nov 2026 06:58:31 -0600 Subject: [PATCH 12/15] detect: size a discrete PCI-matched GPU from its VRAM A machine file that matches by PCI id and says memory is discrete set the GPU's identity but left GPUMemBytes at 0, so Pool() fell back to system RAM. A 24 GB AMD card in a 128 GB host got a RAM budget, and doctor no longer warned about it. Use the VRAM amdgpu reports for the card. Generated by Coder Agents on behalf of @bpmct. --- internal/detect/budget_internal_test.go | 23 +++++++++++++++++++++++ internal/detect/classify.go | 6 +++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/internal/detect/budget_internal_test.go b/internal/detect/budget_internal_test.go index 2bb2f34..af1bcc9 100644 --- a/internal/detect/budget_internal_test.go +++ b/internal/detect/budget_internal_test.go @@ -3,6 +3,8 @@ package detect import ( "strings" "testing" + + "github.com/coder/llm-helper/internal/hardware" ) // Figures below come from vLLM's memory profiler on a DGX Spark (GB10, 121.69 @@ -289,3 +291,24 @@ func strixHaloReadings(ram, carve, gtt int64) Readings { {Vendor: "0x1002", Device: "0x1586", Class: "0x038000", Driver: "amdgpu", VRAMBytes: carve, GTTBytes: gtt}, }} } + +// A card a local definition recognises on the PCI bus as discrete loads +// weights into its own VRAM, not into the host's RAM. +func TestADiscretePCIDefinitionIsSizedFromVRAM(t *testing.T) { + t.Parallel() + reg, err := hardware.Shipped().Extend(hardware.KindMachine, "machines/acme_card.json", + []byte(`{"schema_version":1,"id":"acme_card","display_name":"ACME card","match":{"vendor":"amd","pci_devices":["0x7fff"]},"memory":"discrete","why":"w"}`)) + if err != nil { + t.Fatal(err) + } + const vram = 24e9 + f := facts{totalMem: 128e9, pci: []PCIDevice{{Vendor: "0x1002", Device: "0x7fff", Class: "0x030000", Driver: "amdgpu", VRAMBytes: vram}}} + var hw Hardware + f.describePCIGPU(reg, &hw) + if hw.HardwareID != "acme_card" || hw.GPUMemBytes != vram { + t.Fatalf("hardware = %+v, want acme_card with %d bytes of VRAM", hw, int64(vram)) + } + if pool, kind := hw.Pool(); kind != "vram" || pool != vram { + t.Errorf("pool = %d (%s), want %d (vram)", pool, kind, int64(vram)) + } +} diff --git a/internal/detect/classify.go b/internal/detect/classify.go index 9f6ee6f..e187cef 100644 --- a/internal/detect/classify.go +++ b/internal/detect/classify.go @@ -88,7 +88,8 @@ func (f facts) describeGPUs(hw *Hardware) { // accelerator, and the machine is sized as it was. The definition's display // name stands in for the GPU's, since the bus reports none. // -// A unified GPU that reports a GTT limit loads weights into that, plus any +// A discrete GPU loads weights into the VRAM amdgpu reports for it. A +// unified GPU that reports a GTT limit loads weights into that, plus any // carve-out the firmware set aside for it, rather than into all of RAM. func (f facts) describePCIGPU(reg *hardware.Registry, hw *Hardware) { for _, dev := range f.pci { @@ -105,6 +106,9 @@ func (f facts) describePCIGPU(reg *hardware.Registry, hw *Hardware) { } hw.GPUName, hw.GPUVendor, hw.GPUPCIID = d.DisplayName, v.ID, dev.Vendor+":"+dev.Device hw.HardwareID, hw.Memory, hw.MemorySource, hw.Arch = d.ID, string(d.Memory), MemoryFromDefinition, d.Arch + if d.Memory == hardware.MemoryDiscrete { + hw.GPUMemBytes = dev.VRAMBytes + } if d.Unified() && dev.GTTBytes > 0 { hw.GTTBytes, hw.GPUCarveOutBytes = dev.GTTBytes, dev.VRAMBytes hw.GTTGB = decimalGB(dev.GTTBytes) From c01373d1f00ac42ccc2a111c086b2a38d2aaec03 Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Sun, 15 Nov 2026 06:59:11 -0600 Subject: [PATCH 13/15] doctor: say Vulkan still works when only /dev/kfd is denied The render-group warning said a server on ROCm or Vulkan would not find the GPU whenever any node was denied. Vulkan only needs the render nodes, so a user who can open those but not /dev/kfd was told Vulkan, the backend we recommend on these boxes, would fail. Name ROCm alone then. Generated by Coder Agents on behalf of @bpmct. --- internal/cli/devaccess.go | 36 ++++++++++++++++++------- internal/cli/devaccess_internal_test.go | 8 ++++-- 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/internal/cli/devaccess.go b/internal/cli/devaccess.go index a1fe4bf..8321ce7 100644 --- a/internal/cli/devaccess.go +++ b/internal/cli/devaccess.go @@ -16,7 +16,8 @@ import ( // on the usual distributions, so a user outside the render group gets a // server that finds no GPU ("no ROCm-capable device is detected") rather than // a permission error. A node that does not exist is not reported: a kernel -// without KFD still serves through Vulkan. +// without KFD still serves through Vulkan. When only /dev/kfd is denied, the +// warning says Vulkan is unaffected, since Vulkan is the usual backend there. // // devDir stands in for /dev and canOpen for the access check, so the rule can // be tested without the devices. @@ -24,20 +25,37 @@ func amdDeviceWarnings(accels []detect.Accelerator, devDir string, canOpen func( if !slices.ContainsFunc(accels, func(a detect.Accelerator) bool { return a.Driver == "amdgpu" }) { return nil } - nodes, _ := filepath.Glob(filepath.Join(devDir, "dri", "renderD*")) - nodes = append([]string{filepath.Join(devDir, "kfd")}, nodes...) - var denied []string - for _, n := range nodes { - if _, err := os.Stat(n); err == nil && !canOpen(n) { - denied = append(denied, n) + kfd := filepath.Join(devDir, "kfd") + render, _ := filepath.Glob(filepath.Join(devDir, "dri", "renderD*")) + deniedKFD := isDenied(kfd, canOpen) + var deniedRender []string + for _, n := range render { + if isDenied(n, canOpen) { + deniedRender = append(deniedRender, n) } } + var denied []string + if deniedKFD { + denied = append(denied, kfd) + } + denied = append(denied, deniedRender...) if len(denied) == 0 { return nil } + // ROCm needs /dev/kfd and a render node; Vulkan needs only a render node. + blocked := "a server on ROCm or Vulkan will not find it" + if len(deniedRender) == 0 { + blocked = "a server on ROCm will not find it (Vulkan only needs the render nodes, which this user can open)" + } return []string{fmt.Sprintf( - "an amdgpu GPU is present, but this user cannot open %s, so a server on ROCm or Vulkan will not find it. "+ + "an amdgpu GPU is present, but this user cannot open %s, so %s. "+ "Add the user to the groups that own them with 'sudo usermod -aG render,video $USER', then log out and back in "+ "(with SERVICE_SCOPE=user and lingering, reboot so the user's service manager picks up the groups).", - strings.Join(denied, ", "))} + strings.Join(denied, ", "), blocked)} +} + +// isDenied reports a device node that exists but that canOpen refuses. +func isDenied(path string, canOpen func(path string) bool) bool { + _, err := os.Stat(path) + return err == nil && !canOpen(path) } diff --git a/internal/cli/devaccess_internal_test.go b/internal/cli/devaccess_internal_test.go index 6712696..63a111b 100644 --- a/internal/cli/devaccess_internal_test.go +++ b/internal/cli/devaccess_internal_test.go @@ -39,13 +39,17 @@ func TestAMDDeviceWarnings(t *testing.T) { if strings.Contains(w[0], "card0") { t.Errorf("warning %q names a node servers do not open", w[0]) } + if !strings.Contains(w[0], "ROCm or Vulkan") { + t.Errorf("no access: %q, want both backends named", w[0]) + } if w := amdDeviceWarnings(amd, dev, all); len(w) != 0 { t.Errorf("full access: %q", w) } onlyRender := func(p string) bool { return strings.Contains(p, "renderD") } - if w := amdDeviceWarnings(amd, dev, onlyRender); len(w) != 1 || strings.Contains(w[0], "renderD128") || !strings.Contains(w[0], "kfd") { - t.Errorf("render node only: %q, want /dev/kfd alone", w) + if w := amdDeviceWarnings(amd, dev, onlyRender); len(w) != 1 || strings.Contains(w[0], "renderD128") || !strings.Contains(w[0], "kfd") || + strings.Contains(w[0], "ROCm or Vulkan") || !strings.Contains(w[0], "a server on ROCm will not find it") { + t.Errorf("render node only: %q, want /dev/kfd alone, blocking ROCm but not Vulkan", w) } if w := amdDeviceWarnings(amd, t.TempDir(), none); len(w) != 0 { t.Errorf("no device nodes at all: %q", w) From a6d1f1511a1c6c9c14305733ce5d53175a855be0 Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Sun, 15 Nov 2026 07:00:36 -0600 Subject: [PATCH 14/15] setup: resolve a VLLM_BIN link from the environment as findVLLM does envBin recorded a VLLM_BIN symlink as given, while findVLLM resolves it. Shard commands look for the venv's python and ray beside VLLM_BIN, so a link into a venv broke them. Run vLLM paths through stableVLLMPath, and keep other servers' links as named so a package manager's link survives upgrades. Generated by Coder Agents on behalf of @bpmct. --- internal/cli/setup.go | 7 ++++++- internal/cli/setup_internal_test.go | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/internal/cli/setup.go b/internal/cli/setup.go index 1cef14e..ef91490 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -305,7 +305,9 @@ func lookBin(prov engine.Provider) string { // absolute path, or "" when it names none. A name rather than a path is // looked up on PATH. It is an error for the variable to name something that // is not an executable: setup would otherwise quietly record the server the -// user set it to avoid. +// user set it to avoid. vLLM's path is normalized as findVLLM's is, because +// shard commands look for its venv's python and ray beside it; any other +// server's link is kept as named. func envBin(prov engine.Provider) (string, error) { v := strings.TrimSpace(os.Getenv(prov.BinKey())) if v == "" { @@ -318,6 +320,9 @@ func envBin(prov engine.Provider) (string, error) { if err != nil { return "", fmt.Errorf("%s=%s in the environment is not an executable (%w); fix it, or unset it to have setup look for %s", prov.BinKey(), v, err, prov.BinName()) } + if prov.Name() == engine.VLLM { + p = stableVLLMPath(p) + } return p, nil } diff --git a/internal/cli/setup_internal_test.go b/internal/cli/setup_internal_test.go index 85d4402..f28b19f 100644 --- a/internal/cli/setup_internal_test.go +++ b/internal/cli/setup_internal_test.go @@ -73,6 +73,26 @@ func TestSetupPrefersTheBinKeyFromTheEnvironment(t *testing.T) { if got := findVLLMOrEnv(); got != venv { t.Errorf("VLLM_BIN=%s: findVLLMOrEnv = %q", venv, got) } + + // A VLLM_BIN that links into a venv records the venv's vllm, since + // shard commands look for python and ray beside it. Other servers keep + // the link they were given. + link := filepath.Join(t.TempDir(), "vllm") + if err := os.Symlink(venv, link); err != nil { + t.Fatal(err) + } + t.Setenv(vllm.BinKey(), link) + if got, err := envBin(vllm); err != nil || got != stableVLLMPath(venv) { + t.Errorf("VLLM_BIN=%s (a link): envBin = %q, %v; want %q", link, got, err, stableVLLMPath(venv)) + } + llamaLink := filepath.Join(t.TempDir(), "llama-server") + if err := os.Symlink(built, llamaLink); err != nil { + t.Fatal(err) + } + t.Setenv(prov.BinKey(), llamaLink) + if got, err := envBin(prov); err != nil || got != llamaLink { + t.Errorf("%s=%s (a link): envBin = %q, %v; want the link", prov.BinKey(), llamaLink, got, err) + } } // A system-scope unit needs sudo, which fails without a terminal on a box From 06fd0f43a9a163285274db96b379f4b3efe1430a Mon Sep 17 00:00:00 2001 From: Ben Potter Date: Sun, 15 Nov 2026 07:06:01 -0600 Subject: [PATCH 15/15] cli: expect a resolved VLLM_BIN in the setup test envBin now resolves a VLLM_BIN path, and on macOS t.TempDir() is under /var, a link to /private/var, so the test's unresolved path no longer matched and go (macos-latest) failed. Compare against the resolved path. Generated by Coder Agents on behalf of @bpmct. --- internal/cli/setup_internal_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/cli/setup_internal_test.go b/internal/cli/setup_internal_test.go index f28b19f..c0c7c1e 100644 --- a/internal/cli/setup_internal_test.go +++ b/internal/cli/setup_internal_test.go @@ -70,8 +70,8 @@ func TestSetupPrefersTheBinKeyFromTheEnvironment(t *testing.T) { } venv := fakeExecutable(t, mine, "vllm") t.Setenv(vllm.BinKey(), venv) - if got := findVLLMOrEnv(); got != venv { - t.Errorf("VLLM_BIN=%s: findVLLMOrEnv = %q", venv, got) + if got := findVLLMOrEnv(); got != stableVLLMPath(venv) { + t.Errorf("VLLM_BIN=%s: findVLLMOrEnv = %q, want %q", venv, got, stableVLLMPath(venv)) } // A VLLM_BIN that links into a venv records the venv's vllm, since