Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@ the fact.
`--default-chat-template-kwargs`. Covers Qwen3, Qwen3.5, GLM-4.5 and later,
DeepSeek V3.1 and V4, and SmolLM3; the switch lives in each model type's
catalogue file.
- `inspect` prints the engine's note on hybrid models, whose layers are not
all attention. llama.cpp's note
says to keep the system prompt and tools identical between chats, since it
reuses a cached prompt on these models only up to the start of a user
message.
- `glm-5.3-flash-tf-ablit-2spark` recipe: the GLM-5.3 Flash TensorFold launch
across two DGX Sparks with the abliterated checkpoint, thinking kept on.
- Several models at once: every running model is an instance with its own
Expand Down
1 change: 1 addition & 0 deletions catalogue/engines/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ Each file here is one inference server llm-helper can launch, named after the en
To add an engine, copy the closest file to `<id>.json` and change it.
No Go code is needed: shared code acts on the engine's `capabilities`, never on its name.
`chat_template_kwargs_flag` names the flag that sets default chat template kwargs, which is how a profile turns a model's thinking on or off; leave it out if the engine has none.
`notes.hybrid` is a sentence `inspect` prints for a model whose layers are not all attention; leave it out if there is nothing to say.
The file is checked when it loads, and the tests in `internal/engine` render every file here for every rank.

The fields are described in "Contributing an engine" in [docs/providers.md](../../docs/providers.md).
Expand Down
3 changes: 3 additions & 0 deletions catalogue/engines/llamacpp.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,8 @@
},
"capabilities": {
"gguf_quant": true
},
"notes": {
"hybrid": "llama.cpp can reuse a cached prompt on this model only up to the start of a user message, so keep the system prompt and tools identical between chats."
}
}
2 changes: 2 additions & 0 deletions docs/providers.md
Original file line number Diff line number Diff line change
Expand Up @@ -201,6 +201,8 @@ A definition holds `schema_version` and the engine's fields, and says:
- `health`: the path that answers 200, and whether it does so only once the model is loaded (`ready`) or as soon as the process listens (`live`).
- `capabilities`: the few differences shared code acts on, such as `vllm_args` for vLLM's sizing and flag checks. Launching and health checks act on a capability, never on an engine's name.
`chat_template_kwargs_flag` names the flag that sets default chat template kwargs, which is how a profile turns a model's thinking on or off; see [Model types](./model-types.md).
- `notes`: optional sentences `inspect` prints for some kinds of model.
`hybrid` is shown for a model whose layers are not all attention, such as one with linear-attention or Mamba layers.

The definition is checked when it loads.
An unknown field, an unknown placeholder, a key that would land on the command line, or a container with no role that serves HTTP is refused with the reason.
Expand Down
3 changes: 3 additions & 0 deletions internal/cli/search.go
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,9 @@ func printModelDetail(d resolve.Detail, cfg *config.Config) {
fmt.Println(" thinking: off by default ('new --thinking on' turns it on)")
}
}
if d.HybridNote != "" {
fmt.Printf(" hybrid: %s\n", d.HybridNote)
}
if d.DetectedFlags != "" {
fmt.Printf(" vllm flags: %s (auto-detected, will be pre-filled)\n", d.DetectedFlags)
} else {
Expand Down
11 changes: 11 additions & 0 deletions internal/engine/engine.go
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,8 @@ type Engine struct {
// Container describes the container runtime, which recipes launch.
Container *Container `json:"container,omitempty"`
Caps Capabilities `json:"capabilities"`
// Notes are sentences inspect prints about serving some kinds of model.
Notes Notes `json:"notes,omitempty"`

// Source is the engines.d file a local definition came from, and "" for
// one shipped in the binary.
Expand Down Expand Up @@ -203,6 +205,12 @@ type Capabilities struct {
ChatTemplateKwargsFlag string `json:"chat_template_kwargs_flag,omitempty"`
}

// Notes are sentences about how the engine serves a kind of model.
type Notes struct {
// Hybrid is shown for models whose layers are not all attention.
Hybrid string `json:"hybrid,omitempty"`
}

// Placeholders the templates may use, besides an engine's own params.
var (
processVars = []string{"bin", "model", "model_flag", "host", "port", "max_model_len", "mem_utilization", "kv_cache_bytes", "api_key"}
Expand Down Expand Up @@ -303,6 +311,9 @@ func (e Engine) Capabilities() Capabilities {
return c
}

// HybridNote is what inspect says about serving a hybrid model.
func (e Engine) HybridNote() string { return e.Notes.Hybrid }

// InstallHint says how to get the server on goos.
func (e Engine) InstallHint(goos string) string {
if h := e.Install[goos]; h != "" {
Expand Down
3 changes: 3 additions & 0 deletions internal/engine/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ type Provider interface {
BinKey() string
BinName() string
Capabilities() Capabilities
// HybridNote is what inspect says about serving a hybrid model, if
// anything.
HybridNote() string
// Has reports whether the engine can run in a runtime.
Has(runtime string) bool
// InstallHint says how to get the server on an OS, by GOOS.
Expand Down
22 changes: 22 additions & 0 deletions internal/hf/arch.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@ type Arch struct {
IndexHeadDim int `json:"index_head_dim"`
// LayerTypes names each layer for hybrids, where only some attend.
LayerTypes []string `json:"layer_types"`
// Hybrids that name no layer types say so with one of these instead:
// every Nth layer attends, linear-attention heads, or a per-layer pattern
// where M is a Mamba layer.
FullAttentionInterval int `json:"full_attention_interval"`
LinearNumKeyHeads int `json:"linear_num_key_heads"`
HybridOverridePattern string `json:"hybrid_override_pattern"`
// SlidingWindow caps how far a sliding-attention layer looks back, so
// those layers hold that many tokens rather than the whole context.
SlidingWindow int `json:"sliding_window"`
Expand Down Expand Up @@ -77,6 +83,22 @@ func (a Arch) attentionLayers() (full, sliding int) {
return full, sliding
}

// Hybrid reports whether some layers are linear attention or a recurrent
// state rather than attention.
func (a Arch) Hybrid() bool {
c := a.text()
if c.FullAttentionInterval > 1 || c.LinearNumKeyHeads > 0 || strings.Contains(c.HybridOverridePattern, "M") {
return true
}
for _, t := range c.LayerTypes {
t = strings.ToLower(t)
if strings.Contains(t, "linear") || !strings.Contains(t, "attention") {
return true
}
}
return false
}

// kvDtypeBytes is the width of one cached element. 2 unless the cache itself
// is quantised, which is a separate flag we do not set.
func kvDtypeBytes(dtype string) int {
Expand Down
39 changes: 39 additions & 0 deletions internal/hf/kv_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -222,3 +222,42 @@ func TestLinearAttentionLayersHoldNoCache(t *testing.T) {
t.Errorf("KVBytesPerToken() = %d, want %d (12 full layers of 48)", got, want)
}
}

func TestHybrid(t *testing.T) {
cases := []struct {
types []string
want bool
}{
{nil, false},
{[]string{"full_attention", "sliding_attention"}, false},
{[]string{"linear_attention", "full_attention"}, true},
{[]string{"mamba", "attention"}, true},
}
for _, c := range cases {
if got := (Arch{LayerTypes: c.types}).Hybrid(); got != c.want {
t.Errorf("Hybrid(%v) = %v, want %v", c.types, got, c.want)
}
}
if !(Arch{TextConfig: &Arch{NumHiddenLayers: 2, LayerTypes: []string{"linear_attention", "full_attention"}}}).Hybrid() {
t.Error("Hybrid() did not read the text config")
}

// Configs that name no layer types: Qwen3-Next and Nemotron-H.
other := []struct {
name string
a Arch
want bool
}{
{"full_attention_interval 4", Arch{FullAttentionInterval: 4}, true},
{"full_attention_interval 1", Arch{FullAttentionInterval: 1}, false},
{"linear heads", Arch{LinearNumKeyHeads: 16}, true},
{"pattern with Mamba", Arch{HybridOverridePattern: "MEMEM*EMEM"}, true},
{"pattern without Mamba", Arch{HybridOverridePattern: "*-*-"}, false},
{"in text config", Arch{TextConfig: &Arch{NumHiddenLayers: 24, FullAttentionInterval: 4}}, true},
}
for _, c := range other {
if got := c.a.Hybrid(); got != c.want {
t.Errorf("%s: Hybrid() = %v, want %v", c.name, got, c.want)
}
}
}
9 changes: 9 additions & 0 deletions internal/resolve/resolve.go
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,10 @@ type Detail struct {
// Thinking is the model's switch for reasoning before it answers, when
// its chat template has one.
Thinking *Thinking `json:"thinking,omitempty"`
// Hybrid is set when some layers are not attention, and HybridNote is
// what the serving engine says about that.
Hybrid bool `json:"hybrid,omitempty"`
HybridNote string `json:"hybrid_note,omitempty"`
}

// ToResult converts a Hub model into a search hit.
Expand Down Expand Up @@ -237,6 +241,7 @@ func Inspect(ctx context.Context, m hf.Model, budget detect.WeightsBudget, caps
if !archRead {
arch = <-archCh
}
d.Hybrid = arch.Hybrid()
d.Thinking = detectThinking(ctx, arch.ModelType, m.ID, d.BaseModel)
if flags, err := hf.DetectFlags(ctx, m.ID); err == nil {
// Validate here too: seeing that a card's parser won't work on this
Expand Down Expand Up @@ -272,6 +277,10 @@ func cardFlags(flags hf.DetectedFlags, caps vllm.Caps) (args, note string) {
// reading as detected -- a pull would not have copied them either.
func ForProvider(d Detail, prov engine.Provider) Detail {
caps := prov.Capabilities()
d.HybridNote = ""
if d.Hybrid {
d.HybridNote = prov.HybridNote()
}
if caps.VLLMArgs {
return d
}
Expand Down
11 changes: 11 additions & 0 deletions internal/resolve/resolve_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,17 @@ func TestForProvider(t *testing.T) {
}
}

if llama.HybridNote != "" {
t.Errorf("llamacpp: HybridNote %q on a model that is not hybrid", llama.HybridNote)
}
inspected.Hybrid = true
if got := ForProvider(inspected, get(engine.LlamaCPP)); got.HybridNote == "" {
t.Error("llamacpp: no HybridNote on a hybrid model")
}
if got := ForProvider(inspected, get(engine.VLLM)); got.HybridNote != "" {
t.Errorf("vllm: HybridNote %q, want none", got.HybridNote)
}

// A repo with both kinds: llama.cpp can only take the GGUF, and MLX only
// the safetensors.
mixed := Detail{BudgetGB: 80, All: []hf.Variant{
Expand Down
Loading