diff --git a/CHANGELOG.md b/CHANGELOG.md index 08555d2..89b53bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,11 @@ the fact. `--default-chat-template-kwargs`. Covers Qwen3, Qwen3.5, GLM-4.5 and later, DeepSeek V3.1 and V4, and SmolLM3; the switch lives in each model type's catalogue file. +- `inspect` prints the engine's note on hybrid models, whose layers are not + all attention. llama.cpp's note + says to keep the system prompt and tools identical between chats, since it + reuses a cached prompt on these models only up to the start of a user + message. - `glm-5.3-flash-tf-ablit-2spark` recipe: the GLM-5.3 Flash TensorFold launch across two DGX Sparks with the abliterated checkpoint, thinking kept on. - Several models at once: every running model is an instance with its own diff --git a/catalogue/engines/README.md b/catalogue/engines/README.md index 8057ff1..39fcd44 100644 --- a/catalogue/engines/README.md +++ b/catalogue/engines/README.md @@ -6,6 +6,7 @@ Each file here is one inference server llm-helper can launch, named after the en To add an engine, copy the closest file to `.json` and change it. No Go code is needed: shared code acts on the engine's `capabilities`, never on its name. `chat_template_kwargs_flag` names the flag that sets default chat template kwargs, which is how a profile turns a model's thinking on or off; leave it out if the engine has none. +`notes.hybrid` is a sentence `inspect` prints for a model whose layers are not all attention; leave it out if there is nothing to say. The file is checked when it loads, and the tests in `internal/engine` render every file here for every rank. The fields are described in "Contributing an engine" in [docs/providers.md](../../docs/providers.md). diff --git a/catalogue/engines/llamacpp.json b/catalogue/engines/llamacpp.json index 906b2d1..a66a4b2 100644 --- a/catalogue/engines/llamacpp.json +++ b/catalogue/engines/llamacpp.json @@ -38,5 +38,8 @@ }, "capabilities": { "gguf_quant": true + }, + "notes": { + "hybrid": "llama.cpp can reuse a cached prompt on this model only up to the start of a user message, so keep the system prompt and tools identical between chats." } } diff --git a/docs/providers.md b/docs/providers.md index 163b959..b03e12d 100644 --- a/docs/providers.md +++ b/docs/providers.md @@ -201,6 +201,8 @@ A definition holds `schema_version` and the engine's fields, and says: - `health`: the path that answers 200, and whether it does so only once the model is loaded (`ready`) or as soon as the process listens (`live`). - `capabilities`: the few differences shared code acts on, such as `vllm_args` for vLLM's sizing and flag checks. Launching and health checks act on a capability, never on an engine's name. `chat_template_kwargs_flag` names the flag that sets default chat template kwargs, which is how a profile turns a model's thinking on or off; see [Model types](./model-types.md). +- `notes`: optional sentences `inspect` prints for some kinds of model. + `hybrid` is shown for a model whose layers are not all attention, such as one with linear-attention or Mamba layers. The definition is checked when it loads. An unknown field, an unknown placeholder, a key that would land on the command line, or a container with no role that serves HTTP is refused with the reason. diff --git a/internal/cli/search.go b/internal/cli/search.go index 9f3813e..4abbb34 100644 --- a/internal/cli/search.go +++ b/internal/cli/search.go @@ -262,6 +262,9 @@ func printModelDetail(d resolve.Detail, cfg *config.Config) { fmt.Println(" thinking: off by default ('new --thinking on' turns it on)") } } + if d.HybridNote != "" { + fmt.Printf(" hybrid: %s\n", d.HybridNote) + } if d.DetectedFlags != "" { fmt.Printf(" vllm flags: %s (auto-detected, will be pre-filled)\n", d.DetectedFlags) } else { diff --git a/internal/engine/engine.go b/internal/engine/engine.go index fd06da9..80cb08c 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -77,6 +77,8 @@ type Engine struct { // Container describes the container runtime, which recipes launch. Container *Container `json:"container,omitempty"` Caps Capabilities `json:"capabilities"` + // Notes are sentences inspect prints about serving some kinds of model. + Notes Notes `json:"notes,omitempty"` // Source is the engines.d file a local definition came from, and "" for // one shipped in the binary. @@ -203,6 +205,12 @@ type Capabilities struct { ChatTemplateKwargsFlag string `json:"chat_template_kwargs_flag,omitempty"` } +// Notes are sentences about how the engine serves a kind of model. +type Notes struct { + // Hybrid is shown for models whose layers are not all attention. + Hybrid string `json:"hybrid,omitempty"` +} + // Placeholders the templates may use, besides an engine's own params. var ( processVars = []string{"bin", "model", "model_flag", "host", "port", "max_model_len", "mem_utilization", "kv_cache_bytes", "api_key"} @@ -303,6 +311,9 @@ func (e Engine) Capabilities() Capabilities { return c } +// HybridNote is what inspect says about serving a hybrid model. +func (e Engine) HybridNote() string { return e.Notes.Hybrid } + // InstallHint says how to get the server on goos. func (e Engine) InstallHint(goos string) string { if h := e.Install[goos]; h != "" { diff --git a/internal/engine/registry.go b/internal/engine/registry.go index 3638443..0e00a12 100644 --- a/internal/engine/registry.go +++ b/internal/engine/registry.go @@ -58,6 +58,9 @@ type Provider interface { BinKey() string BinName() string Capabilities() Capabilities + // HybridNote is what inspect says about serving a hybrid model, if + // anything. + HybridNote() string // Has reports whether the engine can run in a runtime. Has(runtime string) bool // InstallHint says how to get the server on an OS, by GOOS. diff --git a/internal/hf/arch.go b/internal/hf/arch.go index 3b1b850..1abdd80 100644 --- a/internal/hf/arch.go +++ b/internal/hf/arch.go @@ -40,6 +40,12 @@ type Arch struct { IndexHeadDim int `json:"index_head_dim"` // LayerTypes names each layer for hybrids, where only some attend. LayerTypes []string `json:"layer_types"` + // Hybrids that name no layer types say so with one of these instead: + // every Nth layer attends, linear-attention heads, or a per-layer pattern + // where M is a Mamba layer. + FullAttentionInterval int `json:"full_attention_interval"` + LinearNumKeyHeads int `json:"linear_num_key_heads"` + HybridOverridePattern string `json:"hybrid_override_pattern"` // SlidingWindow caps how far a sliding-attention layer looks back, so // those layers hold that many tokens rather than the whole context. SlidingWindow int `json:"sliding_window"` @@ -77,6 +83,22 @@ func (a Arch) attentionLayers() (full, sliding int) { return full, sliding } +// Hybrid reports whether some layers are linear attention or a recurrent +// state rather than attention. +func (a Arch) Hybrid() bool { + c := a.text() + if c.FullAttentionInterval > 1 || c.LinearNumKeyHeads > 0 || strings.Contains(c.HybridOverridePattern, "M") { + return true + } + for _, t := range c.LayerTypes { + t = strings.ToLower(t) + if strings.Contains(t, "linear") || !strings.Contains(t, "attention") { + return true + } + } + return false +} + // kvDtypeBytes is the width of one cached element. 2 unless the cache itself // is quantised, which is a separate flag we do not set. func kvDtypeBytes(dtype string) int { diff --git a/internal/hf/kv_internal_test.go b/internal/hf/kv_internal_test.go index 96154fe..c9586bd 100644 --- a/internal/hf/kv_internal_test.go +++ b/internal/hf/kv_internal_test.go @@ -222,3 +222,42 @@ func TestLinearAttentionLayersHoldNoCache(t *testing.T) { t.Errorf("KVBytesPerToken() = %d, want %d (12 full layers of 48)", got, want) } } + +func TestHybrid(t *testing.T) { + cases := []struct { + types []string + want bool + }{ + {nil, false}, + {[]string{"full_attention", "sliding_attention"}, false}, + {[]string{"linear_attention", "full_attention"}, true}, + {[]string{"mamba", "attention"}, true}, + } + for _, c := range cases { + if got := (Arch{LayerTypes: c.types}).Hybrid(); got != c.want { + t.Errorf("Hybrid(%v) = %v, want %v", c.types, got, c.want) + } + } + if !(Arch{TextConfig: &Arch{NumHiddenLayers: 2, LayerTypes: []string{"linear_attention", "full_attention"}}}).Hybrid() { + t.Error("Hybrid() did not read the text config") + } + + // Configs that name no layer types: Qwen3-Next and Nemotron-H. + other := []struct { + name string + a Arch + want bool + }{ + {"full_attention_interval 4", Arch{FullAttentionInterval: 4}, true}, + {"full_attention_interval 1", Arch{FullAttentionInterval: 1}, false}, + {"linear heads", Arch{LinearNumKeyHeads: 16}, true}, + {"pattern with Mamba", Arch{HybridOverridePattern: "MEMEM*EMEM"}, true}, + {"pattern without Mamba", Arch{HybridOverridePattern: "*-*-"}, false}, + {"in text config", Arch{TextConfig: &Arch{NumHiddenLayers: 24, FullAttentionInterval: 4}}, true}, + } + for _, c := range other { + if got := c.a.Hybrid(); got != c.want { + t.Errorf("%s: Hybrid() = %v, want %v", c.name, got, c.want) + } + } +} diff --git a/internal/resolve/resolve.go b/internal/resolve/resolve.go index b3e7de2..35c66d7 100644 --- a/internal/resolve/resolve.go +++ b/internal/resolve/resolve.go @@ -116,6 +116,10 @@ type Detail struct { // Thinking is the model's switch for reasoning before it answers, when // its chat template has one. Thinking *Thinking `json:"thinking,omitempty"` + // Hybrid is set when some layers are not attention, and HybridNote is + // what the serving engine says about that. + Hybrid bool `json:"hybrid,omitempty"` + HybridNote string `json:"hybrid_note,omitempty"` } // ToResult converts a Hub model into a search hit. @@ -237,6 +241,7 @@ func Inspect(ctx context.Context, m hf.Model, budget detect.WeightsBudget, caps if !archRead { arch = <-archCh } + d.Hybrid = arch.Hybrid() d.Thinking = detectThinking(ctx, arch.ModelType, m.ID, d.BaseModel) if flags, err := hf.DetectFlags(ctx, m.ID); err == nil { // Validate here too: seeing that a card's parser won't work on this @@ -272,6 +277,10 @@ func cardFlags(flags hf.DetectedFlags, caps vllm.Caps) (args, note string) { // reading as detected -- a pull would not have copied them either. func ForProvider(d Detail, prov engine.Provider) Detail { caps := prov.Capabilities() + d.HybridNote = "" + if d.Hybrid { + d.HybridNote = prov.HybridNote() + } if caps.VLLMArgs { return d } diff --git a/internal/resolve/resolve_internal_test.go b/internal/resolve/resolve_internal_test.go index 53e4a25..ebd2edd 100644 --- a/internal/resolve/resolve_internal_test.go +++ b/internal/resolve/resolve_internal_test.go @@ -319,6 +319,17 @@ func TestForProvider(t *testing.T) { } } + if llama.HybridNote != "" { + t.Errorf("llamacpp: HybridNote %q on a model that is not hybrid", llama.HybridNote) + } + inspected.Hybrid = true + if got := ForProvider(inspected, get(engine.LlamaCPP)); got.HybridNote == "" { + t.Error("llamacpp: no HybridNote on a hybrid model") + } + if got := ForProvider(inspected, get(engine.VLLM)); got.HybridNote != "" { + t.Errorf("vllm: HybridNote %q, want none", got.HybridNote) + } + // A repo with both kinds: llama.cpp can only take the GGUF, and MLX only // the safetensors. mixed := Detail{BudgetGB: 80, All: []hf.Variant{