From 811fea5fe11f56fc32b749765d2dedabd154a7c0 Mon Sep 17 00:00:00 2001 From: DevelopmentCats Date: Tue, 10 Nov 2026 01:48:31 +0000 Subject: [PATCH 1/4] instances: run several models at once, each with its own service Every running model is an instance keyed by its profile, with its own port, unit (llm-helper@.service on Linux, one plist per instance on macOS) and log. Adds up, down, default, migrate and an opt-in model router; switch is deprecated. Coder sync offers every running instance. --- docs/reference/cli/stop.md | 13 - docs/reference/cli/switch.md | 39 -- internal/cli/clusterrunner.go | 9 +- internal/cli/clusterrunner_internal_test.go | 33 +- internal/cli/detectcmds.go | 202 ++++++----- internal/cli/doctor_internal_test.go | 157 ++++---- internal/cli/editset.go | 7 +- internal/cli/ggufplugin.go | 2 +- internal/cli/instances.go | 342 ++++++++++++++++++ internal/cli/memfit.go | 4 +- internal/cli/plugins.go | 15 - internal/cli/profiles.go | 274 ++++++-------- internal/cli/recipehealth.go | 38 +- internal/cli/recipehealth_internal_test.go | 4 +- internal/cli/recipelaunch.go | 74 ++-- internal/cli/recipelaunch_internal_test.go | 3 +- internal/cli/recipes.go | 3 +- internal/cli/recipeservice.go | 120 +++++- internal/cli/recipeservice_internal_test.go | 25 +- internal/cli/root.go | 145 ++++---- internal/cli/root_internal_test.go | 17 +- internal/cli/router.go | 219 +++++++++++ internal/cli/router_internal_test.go | 121 +++++++ internal/cli/run_test.go | 175 ++++++--- internal/cli/search.go | 6 +- internal/cli/serve.go | 27 +- internal/cli/service.go | 100 ++--- internal/cli/setup.go | 56 +-- internal/cli/start.go | 126 +++++-- internal/cli/status_internal_test.go | 2 +- internal/cli/vllmargs.go | 4 +- internal/cli/vllmargs_internal_test.go | 6 +- internal/cli/watch.go | 16 +- internal/cli/watch_internal_test.go | 22 +- internal/config/config.go | 33 +- internal/config/config_test.go | 18 +- internal/config/envfile.go | 2 +- internal/control/control.go | 162 +++------ internal/control/control_internal_test.go | 143 ++------ internal/control/fit.go | 113 ++++++ internal/control/fit_internal_test.go | 40 ++ internal/control/gguf.go | 3 +- internal/control/gpus.go | 19 +- internal/control/instances.go | 277 ++++++++++++++ internal/control/instances_internal_test.go | 167 +++++++++ internal/engine/registry.go | 2 +- internal/plugin/coder/sync.go | 140 ++++--- internal/plugin/coder/sync_internal_test.go | 2 +- internal/plugin/plugin.go | 18 +- internal/plugin/registry_internal_test.go | 24 +- internal/profile/profile.go | 17 + internal/profile/profile_test.go | 14 + internal/router/models.go | 207 +++++++++++ internal/router/models_internal_test.go | 121 +++++++ internal/service/service.go | 10 +- internal/service/service_darwin.go | 50 ++- .../service/service_darwin_internal_test.go | 42 ++- internal/service/service_linux.go | 116 ++++-- .../service/service_linux_internal_test.go | 38 +- internal/service/service_other.go | 7 + internal/web/activate.go | 62 +++- internal/web/api.go | 266 +++++++++----- internal/web/docssite.go | 2 +- internal/web/models.go | 39 +- internal/web/server.go | 178 ++++----- internal/web/server_internal_test.go | 195 +++++++--- internal/web/static/app.css | 57 ++- internal/web/static/index.html | 14 +- internal/web/static/machine.js | 307 +++++++++++----- internal/web/static/models.html | 2 +- internal/web/static/models.js | 16 +- 71 files changed, 3712 insertions(+), 1617 deletions(-) delete mode 100644 docs/reference/cli/stop.md delete mode 100644 docs/reference/cli/switch.md create mode 100644 internal/cli/instances.go create mode 100644 internal/cli/router.go create mode 100644 internal/cli/router_internal_test.go create mode 100644 internal/control/fit.go create mode 100644 internal/control/fit_internal_test.go create mode 100644 internal/control/instances.go create mode 100644 internal/control/instances_internal_test.go create mode 100644 internal/router/models.go create mode 100644 internal/router/models_internal_test.go diff --git a/docs/reference/cli/stop.md b/docs/reference/cli/stop.md deleted file mode 100644 index 26257a5..0000000 --- a/docs/reference/cli/stop.md +++ /dev/null @@ -1,13 +0,0 @@ ---- -# Code generated by go run ./scripts/gendocs. DO NOT EDIT. -title: "llm-helper stop" -description: "Stop the server, keeping profiles and config" ---- - - - -## Usage - -```console -llm-helper stop -``` diff --git a/docs/reference/cli/switch.md b/docs/reference/cli/switch.md deleted file mode 100644 index b696f5a..0000000 --- a/docs/reference/cli/switch.md +++ /dev/null @@ -1,39 +0,0 @@ ---- -# Code generated by go run ./scripts/gendocs. DO NOT EDIT. -title: "llm-helper switch" -description: "Switch to a saved profile, restarting the server" ---- - - - -## Usage - -```console -llm-helper switch [flags] -``` - -## Options - -### --no-follow - -| | | -| ---- | ------ | -| Type | `bool` | - -return as soon as the server is started instead of watching the boot - -### --timeout - -| | | -| ---- | ---------- | -| Type | `duration` | - -give up waiting after this long (default: wait forever) - -### --wait - -| | | -| ---- | ------ | -| Type | `bool` | - -block until the endpoint is healthy, without streaming logs diff --git a/internal/cli/clusterrunner.go b/internal/cli/clusterrunner.go index 7c0bf17..1a76e4a 100644 --- a/internal/cli/clusterrunner.go +++ b/internal/cli/clusterrunner.go @@ -64,7 +64,7 @@ func clusterRunnerFor(cfg *config.Config, p *profile.Profile) (*clusterRunner, e // would start or stop containers on machines nobody named. func (c *clusterRunner) needNodes() error { if len(c.opts.nodeNames) == 0 { - return fmt.Errorf("the active profile does not record the nodes %s was launched on; relaunch it with 'llm-helper recipes launch %s --nodes '", c.recipe.ID, c.recipe.ID) + return fmt.Errorf("the profile does not record the nodes %s was launched on; relaunch it with 'llm-helper recipes launch %s --nodes '", c.recipe.ID, c.recipe.ID) } return nil } @@ -160,13 +160,6 @@ echo "$code" `, strings.Join(quoted, " "), get, recipe.ShellQuote(port), recipe.ShellQuote(path)) } -// identity names what is holding the GPUs, so a switch can tell whether the -// runner it is leaving is the one it is about to start. Two recipes on the -// same nodes and container name replace each other; anything else does not. -func (c *clusterRunner) identity() string { - return "cluster:" + c.opts.container + "@" + strings.Join(c.opts.nodeNames, ",") -} - // head is the node serving the API: rank 0. func (c *clusterRunner) head() (recipeMember, error) { members, err := resolveRecipeMembers(c.recipe, c.opts) diff --git a/internal/cli/clusterrunner_internal_test.go b/internal/cli/clusterrunner_internal_test.go index 296401a..ce67da9 100644 --- a/internal/cli/clusterrunner_internal_test.go +++ b/internal/cli/clusterrunner_internal_test.go @@ -110,38 +110,7 @@ func TestEndpointURLFollowsTheHead(t *testing.T) { } } -// Switching runners has to stop the one being left: the incoming runner only -// replaces its own containers or unit, so a recipe kept the GPUs after a -// switch to a native profile and vice versa. -func TestOutgoingNeedsStop(t *testing.T) { - cases := []struct { - name string - out, in string - cluster bool - state string - wantStop bool - }{ - {"same unit restarts itself", "native:vllm.service", "native:vllm.service", false, "active", false}, - {"same cluster replaces its own containers", "cluster:vllm-node@a,b", "cluster:vllm-node@a,b", true, "", false}, - {"native to recipe stops the unit", "native:vllm.service", "cluster:vllm-node@a,b", false, "active", true}, - {"a loading unit is still holding the GPUs", "native:vllm.service", "cluster:vllm-node@a,b", false, "activating", true}, - {"an idle unit is left alone", "native:vllm.service", "cluster:vllm-node@a,b", false, "inactive", false}, - // Workers can outlive a crashed head, so the head's state says - // nothing about whether there is anything to stop. - {"recipe to native always stops the cluster", "cluster:vllm-node@a,b", "native:vllm.service", true, "", true}, - {"a recipe on other nodes stops the old one", "cluster:vllm-node@a,b", "cluster:vllm-node@c,d", true, "", true}, - } - for _, c := range cases { - if got := outgoingNeedsStop(c.out, c.in, c.cluster, c.state); got != c.wantStop { - t.Errorf("%s: outgoingNeedsStop = %v, want %v", c.name, got, c.wantStop) - } - } -} - -// A restart relaunches the recipe the profile already describes. It used to -// print the --no-activate warning that the profile and Coder "still describe -// whatever ran before", which is false there; the warning stays for a user who -// actually passed --no-activate. +// A restart prints no --no-activate warning; a real --no-activate does. func TestRestartDoesNotWarnAboutSkippedActivation(t *testing.T) { c := &clusterRunner{opts: recipeLaunchOptions{nodeNames: []string{"a"}}} o := c.restartOpts() diff --git a/internal/cli/detectcmds.go b/internal/cli/detectcmds.go index 4d816b7..8bdb231 100644 --- a/internal/cli/detectcmds.go +++ b/internal/cli/detectcmds.go @@ -19,6 +19,7 @@ import ( "github.com/coder/llm-helper/internal/plugin" "github.com/coder/llm-helper/internal/profile" "github.com/coder/llm-helper/internal/recipe" + "github.com/coder/llm-helper/internal/service" "github.com/coder/llm-helper/internal/toolchain" ) @@ -47,17 +48,25 @@ type doctorConfig struct { // Provider is what new profiles are made for. Its server is reported // beside VLLM_BIN when it is not vLLM, since that is then the binary // whose absence stops a new profile serving. - Provider string `json:"provider"` - ProviderBin string `json:"provider_bin,omitempty"` - ProviderBinOK bool `json:"provider_bin_ok,omitempty"` - ProviderError string `json:"provider_error,omitempty"` - Service string `json:"service"` - Scope string `json:"scope"` - ServiceState string `json:"service_state"` - ServiceCrashed bool `json:"service_crashed"` - ServiceRestarts int `json:"service_restarts"` - BudgetOverride string `json:"budget_override,omitempty"` - CoderConfigured bool `json:"coder_sync_configured"` + Provider string `json:"provider"` + ProviderBin string `json:"provider_bin,omitempty"` + ProviderBinOK bool `json:"provider_bin_ok,omitempty"` + ProviderError string `json:"provider_error,omitempty"` + Instances []doctorInstance `json:"instances"` + LegacyProfile string `json:"legacy_profile,omitempty"` + BudgetOverride string `json:"budget_override,omitempty"` + CoderConfigured bool `json:"coder_sync_configured"` +} + +type doctorInstance struct { + Profile string `json:"profile"` + Model string `json:"model"` + Port string `json:"port"` + Service string `json:"service"` + Scope string `json:"scope,omitempty"` + State string `json:"state"` + Crashed bool `json:"crashed"` + Restarts int `json:"restarts"` } type coderStatus struct { @@ -115,27 +124,21 @@ func gatherDoctor() doctorReport { binOK = true } } - runner := svc(cfg) - health := runner.Health() - // Name what is actually serving. Reporting the configured unit beside - // the cluster's state described a dead unit as healthy. - service, scope := cfg.ServiceName(), cfg.ServiceScope() - if c, ok := runner.(*clusterRunner); ok { - service, scope = c.Label(), "" - } r.Config = &doctorConfig{ Path: cfg.Dirs.ConfigFile(), Port: cfg.Port(), VLLMBin: bin, VLLMBinOK: binOK, - Service: service, - Scope: scope, - ServiceState: health.State, - ServiceCrashed: health.Crashed, - ServiceRestarts: health.NRestarts, + Instances: gatherDoctorInstances(cfg), BudgetOverride: cfg.ModelBudgetGB(), CoderConfigured: cfg.CoderConfigured(), } + if legacyActive(cfg) { + r.Config.LegacyProfile = cfg.LegacyActiveProfile() + r.Warnings = append(r.Warnings, fmt.Sprintf( + "%s still runs under the single-model service from an older release. Run '%s migrate' to move it to an instance.", + r.Config.LegacyProfile, cmdName)) + } if prov, err := engine.Lookup(cfg.Provider()); err != nil { r.Config.Provider = cfg.Provider() r.Config.ProviderError = err.Error() @@ -165,36 +168,71 @@ func gatherDoctor() doctorReport { r.Coder = gatherCoderStatus(cfg) r.HFToken = gatherHFToken(cfg) r.WiredLimit = gatherWiredLimit(cfg, r.Hardware) - r.Warnings = append(r.Warnings, hardwareWarnings(r.Hardware)...) + running := 0 + if r.Config != nil { + running = len(r.Config.Instances) + } + r.Warnings = append(r.Warnings, hardwareWarnings(r.Hardware, running)...) // Drift checks: the part a human wouldn't spot at a glance. if cfg != nil { - activeModel := "" - if active, err := config.ParseEnvFile(cfg.Dirs.ActiveEnvFile()); err == nil { - activeModel = active.Get("MODEL") + byPort := map[string]doctorInstance{} + units := map[string]bool{cfg.ServiceName(): true} + for _, in := range r.Config.Instances { + byPort[in.Port] = in + units[in.Service] = true } for _, s := range r.Servers { if s.InContainer { continue } - if s.Port == cfg.Port() && activeModel != "" && s.Model != "" && s.Model != activeModel { + if in, ok := byPort[s.Port]; ok && in.Model != "" && s.Model != "" && s.Model != in.Model { r.Warnings = append(r.Warnings, fmt.Sprintf( - "config says model %s but port %s is serving %s -- was the server started outside llm-helper? Run 'llm-helper adopt'.", - activeModel, s.Port, s.Model)) + "%s should serve %s on port %s, but %s is there -- was the server started outside llm-helper? Run 'llm-helper adopt'.", + in.Profile, in.Model, s.Port, s.Model)) } - if s.SystemdUnit != "" && s.SystemdUnit != cfg.ServiceName() { + if s.SystemdUnit != "" && !units[s.SystemdUnit] { r.Warnings = append(r.Warnings, fmt.Sprintf( - "running server belongs to %s unit %s but config manages %s -- run 'llm-helper adopt' to take it over.", - config.ServiceManager, s.SystemdUnit, cfg.ServiceName())) + "running server belongs to %s unit %s, which llm-helper does not manage -- run 'llm-helper adopt' to take it over.", + config.ServiceManager, s.SystemdUnit)) } } } return r } +// gatherDoctorInstances names what actually serves each instance: a recipe's +// cluster, or the instance's unit. +func gatherDoctorInstances(cfg *config.Config) []doctorInstance { + list, err := control.ListInstances(cfg) + if err != nil { + return nil + } + out := make([]doctorInstance, 0, len(list)) + for _, inst := range list { + run := instanceRunner(cfg, inst.Name) + h := run.Health() + d := doctorInstance{ + Profile: inst.Name, + Model: inst.Env.Get("MODEL"), + Port: inst.Port, + Service: service.InstanceName(inst.Name), + Scope: cfg.ServiceScope(), + State: h.State, + Crashed: h.Crashed, + Restarts: h.NRestarts, + } + if c, ok := run.(*clusterRunner); ok { + d.Service, d.Scope = c.Label(), "" + } + out = append(out, d) + } + return out +} + // gatherCoderStatus goes beyond "the variables are set": it calls the API // so an expired token or wrong URL shows up here instead of on the next -// switch. +// up. func gatherCoderStatus(cfg *config.Config) coderStatus { st := coderStatus{} if cfg == nil { @@ -258,7 +296,7 @@ func gatherHFToken(cfg *config.Config) hfTokenStatus { // hardwareWarnings is what doctor has to say about the machine itself: the // cases where the budget it prints is right by its own rules but would // still mislead someone choosing a model. -func hardwareWarnings(hw detect.Hardware) []string { +func hardwareWarnings(hw detect.Hardware, running int) []string { var w []string for _, err := range hardware.LocalErrors() { w = append(w, "a local hardware definition was skipped: "+err.Error()) @@ -278,7 +316,12 @@ func hardwareWarnings(hw detect.Hardware) []string { "CUDA_VISIBLE_DEVICES hides all %d GPU(s) from the server, and vLLM will not start without one. Unset it, or name a GPU in config.env or the profile.", hw.GPUsHidden)) } - if hw.Unified && hw.Busy() { + switch { + case hw.Unified && hw.Busy() && running > 0: + w = append(w, fmt.Sprintf( + "%d GB of this machine's shared memory is in use, including %d running instance(s). '%s up' checks a new model against the instances, but not against anything else holding memory (another server, a container).", + int(float64(hw.InUseBytes)/1e9), running, cmdName)) + case hw.Unified && hw.Busy(): w = append(w, fmt.Sprintf( "%d GB of this machine's shared memory is in use. The budget assumes it is free when a model starts, and vLLM sizes its cache from what is free then, so anything still holding it (another server, a container) takes room from the next model.", int(float64(hw.InUseBytes)/1e9))) @@ -452,17 +495,26 @@ func renderDoctor(r doctorReport, yes bool) { default: fmt.Printf(" vllm bin: %s\n", c.VLLMBin) } - state := c.ServiceState - if c.ServiceCrashed { - state += ", failing to start" - if c.ServiceRestarts > 0 { - state += fmt.Sprintf(" (%d restarts)", c.ServiceRestarts) - } + if len(c.Instances) == 0 { + fmt.Println(" models: none running") } - if c.Scope == "" { - fmt.Printf(" service: %s -- %s\n", c.Service, state) - } else { - fmt.Printf(" service: %s (%s scope) -- %s\n", c.Service, c.Scope, state) + for i, in := range c.Instances { + state := in.State + if in.Crashed { + state += ", failing to start" + if in.Restarts > 0 { + state += fmt.Sprintf(" (%d restarts)", in.Restarts) + } + } + label := " models: " + if i > 0 { + label = " " + } + unit := in.Service + if in.Scope != "" { + unit += " (" + in.Scope + " scope)" + } + fmt.Printf("%s %s on port %s, %s -- %s\n", label, in.Profile, in.Port, unit, state) } fmt.Println() } @@ -644,7 +696,7 @@ func adoptInto(cfg *config.Config) error { return nil } fmt.Println("No running vLLM server found to adopt.") - fmt.Println("Use 'llm-helper switch ' to start one, or 'llm-helper doctor' to see what was scanned.") + fmt.Println("Use 'llm-helper up ' to start one, or 'llm-helper doctor' to see what was scanned.") return nil } @@ -674,8 +726,8 @@ func adoptInto(cfg *config.Config) error { fmt.Printf(" binary: %s\n", s.VLLMBin) } fmt.Println() - fmt.Println("Adopting will save this as a profile and make llm-helper manage future") - fmt.Println("switches. The running server is NOT restarted or interrupted.") + fmt.Println("Adopting saves this as a profile llm-helper can run. The running server") + fmt.Println("is NOT restarted or interrupted.") if !confirm("Adopt this setup?", true) { fmt.Println("Left everything as-is.") return nil @@ -688,9 +740,8 @@ func adoptInto(cfg *config.Config) error { // Profile: reuse an existing one for this model or create from the // observed flags. name := profile.NameFromRepo(s.Model) - p, err := profile.Load(cfg.Dirs, name) - if err != nil { - p, err = profile.Create(cfg.Dirs, profile.CreateOpts{ + if _, err := profile.Load(cfg.Dirs, name); err != nil { + _, err = profile.Create(cfg.Dirs, profile.CreateOpts{ Repo: s.Model, MaxModelLen: s.MaxModelLen, MemUtilization: s.MemUtilization, @@ -718,58 +769,27 @@ func adoptInto(cfg *config.Config) error { changed = true fmt.Printf("Config: VLLM_BIN -> %s\n", s.VLLMBin) } - if adoptService(cfg, s) { - changed = true - } if changed { if err := cfg.File.Write(cfg.Dirs.ConfigFile(), 0o644); err != nil { return err } } - if err := control.WriteActiveEnv(cfg, p); err != nil { - return err - } - fmt.Printf("Marked '%s' as the active profile.\n", name) - - if s.SystemdUnit == "" { - fmt.Println() - fmt.Printf("NOTE: the running server is a bare process (no %s unit), so\n", config.ServiceManager) - fmt.Println("'llm-helper switch' can't restart it. Stop it yourself when ready and") - fmt.Println("run 'llm-helper setup' to install the service, then switch normally.") + fmt.Println() + if s.SystemdUnit != "" { + fmt.Printf("To let llm-helper run it, stop and disable %s unit %s,\n", config.ServiceManager, s.SystemdUnit) + } else { + fmt.Printf("To let llm-helper run it, stop the bare process (pid %d),\n", s.PID) } + fmt.Printf("then run '%s up %s'. Until then llm-helper leaves it alone.\n", cmdName, name) if !s.APIKeySet { fmt.Println() - fmt.Println("NOTE: the running server has NO --api-key. After the next switch it will") + fmt.Println("NOTE: the running server has NO --api-key. Once llm-helper runs it, it will") fmt.Printf("require the key in %s -- update your clients.\n", cfg.Dirs.SecretsFile()) } - fmt.Println() - fmt.Println("Adoption complete. 'llm-helper status' now reflects this server, and the") - fmt.Println("next 'llm-helper switch' will manage it.") return nil } -// adoptService points SERVICE_NAME and SERVICE_SCOPE at the unit running s, -// and reports whether either changed. Both are compared: a system-scope job -// with the default name, adopted from a user-scope config, would otherwise -// keep status and restart aimed at the wrong domain. -func adoptService(cfg *config.Config, s detect.Server) bool { - if s.SystemdUnit == "" { - return false - } - scope := "system" - if s.UserScope { - scope = "user" - } - if cfg.ServiceName() == s.SystemdUnit && cfg.ServiceScope() == scope { - return false - } - cfg.File.Set("SERVICE_NAME", s.SystemdUnit) - cfg.File.Set("SERVICE_SCOPE", scope) - fmt.Printf("Config: SERVICE_NAME -> %s (%s scope)\n", s.SystemdUnit, scope) - return true -} - // appleBackend says which vLLM backend an Apple Silicon Mac serves with, since // the same machine runs on its GPU or its CPU depending on what is installed. // Empty on anything else, and when new profiles use another provider: that diff --git a/internal/cli/doctor_internal_test.go b/internal/cli/doctor_internal_test.go index 5a2eb69..a06f4aa 100644 --- a/internal/cli/doctor_internal_test.go +++ b/internal/cli/doctor_internal_test.go @@ -14,9 +14,11 @@ import ( "time" "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/control" "github.com/coder/llm-helper/internal/detect" "github.com/coder/llm-helper/internal/hardware" "github.com/coder/llm-helper/internal/profile" + "github.com/coder/llm-helper/internal/service" ) // Nothing in this package's tests runs in parallel: they swap os.Stdout and @@ -136,7 +138,13 @@ func TestGatherDoctorReportsTheConfig(t *testing.T) { if err := os.WriteFile(bin, []byte("#!/bin/sh\n"), 0o755); err != nil { t.Fatal(err) } - writeConfig(t, dirs, "PORT=9101\nVLLM_BIN="+bin+"\nSERVICE_NAME=vllm-test\nSERVICE_SCOPE=user\n") + writeConfig(t, dirs, "PORT=9101\nVLLM_BIN="+bin+"\nSERVICE_SCOPE=user\n") + if err := os.MkdirAll(dirs.InstancesDir(), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dirs.InstanceEnvFile("demo"), []byte("PORT=9102\nMODEL=org/demo\n"), 0o600); err != nil { + t.Fatal(err) + } r := gatherDoctor() if r.Config == nil { @@ -150,8 +158,6 @@ func TestGatherDoctorReportsTheConfig(t *testing.T) { {"path", r.Config.Path, dirs.ConfigFile()}, {"port", r.Config.Port, "9101"}, {"vllm bin", r.Config.VLLMBin, bin}, - {"service", r.Config.Service, "vllm-test"}, - {"scope", r.Config.Scope, "user"}, } for _, tt := range tests { if tt.got != tt.want { @@ -164,8 +170,18 @@ func TestGatherDoctorReportsTheConfig(t *testing.T) { if r.Config.CoderConfigured { t.Error("CoderConfigured should be false without Coder settings") } - if r.Config.ServiceState == "" { - t.Error("ServiceState should always say something") + if len(r.Config.Instances) != 1 { + t.Fatalf("Instances = %+v, want the one instance", r.Config.Instances) + } + in := r.Config.Instances[0] + if in.Profile != "demo" || in.Port != "9102" || in.Model != "org/demo" { + t.Errorf("instance = %+v", in) + } + if in.Service != service.InstanceName("demo") || in.Scope != "user" { + t.Errorf("instance unit = %s (%s)", in.Service, in.Scope) + } + if in.State == "" { + t.Error("State should always say something") } } @@ -241,12 +257,7 @@ func TestGatherDoctorWarnsWhenThePortServesAnotherModel(t *testing.T) { dirs := newTree(t) const port = "18201" writeConfig(t, dirs, "PORT="+port+"\n") - if err := os.MkdirAll(dirs.State, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(dirs.ActiveEnvFile(), []byte("MODEL=org/configured-model\n"), 0o600); err != nil { - t.Fatal(err) - } + writeInstance(t, dirs, "configured", "PORT="+port+"\nMODEL=org/configured-model\n") startFakeServer(t, port, "org/running-model") var warning string @@ -410,6 +421,16 @@ func TestGatherHFToken(t *testing.T) { }) } +func writeInstance(t *testing.T, dirs config.Dirs, name, content string) { + t.Helper() + if err := os.MkdirAll(dirs.InstancesDir(), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dirs.InstanceEnvFile(name), []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + func TestGatherStatus(t *testing.T) { t.Run("healthy endpoint", func(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})) @@ -417,43 +438,44 @@ func TestGatherStatus(t *testing.T) { port := portOf(t, srv.URL) dirs := newTree(t) - writeConfig(t, dirs, "PORT="+port+"\nSERVICE_NAME=vllm-test\nSERVICE_SCOPE=user\n") - if err := os.MkdirAll(dirs.State, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(dirs.ActiveEnvFile(), []byte("MODEL=org/model\n"), 0o600); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(dirs.ActiveProfileFile(), []byte("qwen\n"), 0o644); err != nil { + writeConfig(t, dirs, "SERVICE_SCOPE=user\n") + writeInstance(t, dirs, "qwen", "PORT="+port+"\nMODEL=org/model\n") + cfg, err := config.Load() + if err != nil { t.Fatal(err) } - cfg, err := config.Load() + inst, err := control.LoadInstance(cfg, "qwen") if err != nil { t.Fatal(err) } - r := gatherStatus(cfg) + r := gatherStatus(cfg, inst, "qwen") if !r.Healthy || r.HTTPStatus != http.StatusOK { t.Errorf("healthy = %v, status = %d, want true and 200", r.Healthy, r.HTTPStatus) } - if r.Model != "org/model" || r.Profile != "qwen" || r.Port != port { - t.Errorf("report = %+v, want the active model, profile and port", r) + if r.Model != "org/model" || r.Profile != "qwen" || r.Port != port || !r.Default { + t.Errorf("report = %+v, want the instance's model, profile, port and default", r) } }) t.Run("nothing listening", func(t *testing.T) { dirs := newTree(t) - writeConfig(t, dirs, "PORT="+closedPort(t)+"\nSERVICE_NAME=vllm-test\nSERVICE_SCOPE=user\n") + writeConfig(t, dirs, "SERVICE_SCOPE=user\n") + writeInstance(t, dirs, "qwen", "PORT="+closedPort(t)+"\n") cfg, err := config.Load() if err != nil { t.Fatal(err) } - r := gatherStatus(cfg) + inst, err := control.LoadInstance(cfg, "qwen") + if err != nil { + t.Fatal(err) + } + r := gatherStatus(cfg, inst, "other") if r.Healthy || r.HTTPStatus != 0 { t.Errorf("healthy = %v, status = %d, want false and 0", r.Healthy, r.HTTPStatus) } - if r.Profile != "unknown" || r.Model != "unknown" { - t.Errorf("report = %+v, want unknown where nothing is set yet", r) + if r.Model != "unknown" || r.Default { + t.Errorf("report = %+v, want an unknown model and not the default", r) } }) } @@ -568,66 +590,6 @@ func TestFitMemUtilizationStaysInRange(t *testing.T) { } } -// Adoption follows the running unit's scope as well as its name: a -// system-scope job under the default name, adopted from a user-scope config, -// would otherwise leave status and restart aimed at the wrong domain. -func TestAdoptServiceFollowsTheScope(t *testing.T) { - tests := []struct { - name string - config string - server detect.Server - wantChanged bool - wantName string - wantScope string - }{ - { - name: "same name, other scope", - config: "SERVICE_NAME=" + config.DefaultServiceName + "\nSERVICE_SCOPE=user\n", - server: detect.Server{SystemdUnit: config.DefaultServiceName}, - wantChanged: true, wantName: config.DefaultServiceName, wantScope: "system", - }, - { - name: "other name", - config: "SERVICE_NAME=vllm-old\nSERVICE_SCOPE=system\n", - server: detect.Server{SystemdUnit: "vllm-new", UserScope: true}, - wantChanged: true, wantName: "vllm-new", wantScope: "user", - }, - { - name: "already matching", - config: "SERVICE_NAME=vllm-test\nSERVICE_SCOPE=user\n", - server: detect.Server{SystemdUnit: "vllm-test", UserScope: true}, - wantChanged: false, wantName: "vllm-test", wantScope: "user", - }, - { - name: "bare process", - config: "SERVICE_NAME=vllm-test\nSERVICE_SCOPE=user\n", - server: detect.Server{}, - wantChanged: false, wantName: "vllm-test", wantScope: "user", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - dirs := newTree(t) - writeConfig(t, dirs, tt.config) - cfg, err := config.Load() - if err != nil { - t.Fatal(err) - } - if got := adoptService(cfg, tt.server); got != tt.wantChanged { - t.Errorf("adoptService() = %v, want %v", got, tt.wantChanged) - } - if got := cfg.ServiceName(); got != tt.wantName { - t.Errorf("SERVICE_NAME = %q, want %q", got, tt.wantName) - } - if got := cfg.ServiceScope(); got != tt.wantScope { - t.Errorf("SERVICE_SCOPE = %q, want %q", got, tt.wantScope) - } - }) - } -} - -// The backend line is about vLLM, so it stays out of a doctor whose new -// profiles go to another server. func TestAppleBackend(t *testing.T) { t.Parallel() @@ -662,22 +624,25 @@ func TestAppleBackend(t *testing.T) { // still send someone after a model this machine cannot start. func TestHardwareWarnings(t *testing.T) { hidden := detect.Hardware{TotalMemBytes: 128e9, CUDA: true, GPUsHidden: 1} - if w := hardwareWarnings(hidden); len(w) != 1 || !strings.Contains(w[0], "CUDA_VISIBLE_DEVICES hides all 1 GPU") { + if w := hardwareWarnings(hidden, 0); len(w) != 1 || !strings.Contains(w[0], "CUDA_VISIBLE_DEVICES hides all 1 GPU") { t.Errorf("every GPU hidden: %q", w) } spark := detect.Hardware{TotalMemBytes: 128e9, CUDA: true, Unified: true, GPUName: "NVIDIA GB10", GPUs: []detect.GPU{{Name: "NVIDIA GB10"}}, InUseBytes: 120e9} - if w := hardwareWarnings(spark); len(w) != 1 || !strings.Contains(w[0], "120 GB of this machine's shared memory is in use") { + if w := hardwareWarnings(spark, 0); len(w) != 1 || !strings.Contains(w[0], "120 GB of this machine's shared memory is in use") { t.Errorf("loaded Spark: %q", w) } + if w := hardwareWarnings(spark, 2); len(w) != 1 || !strings.Contains(w[0], "including 2 running instance(s)") { + t.Errorf("Spark with our instances: %q", w) + } spark.InUseBytes = 8e9 - if w := hardwareWarnings(spark); len(w) != 0 { + if w := hardwareWarnings(spark, 0); len(w) != 0 { t.Errorf("idle Spark: %q", w) } // A discrete card's own memory is what a model loads into, so the // host's being busy says nothing about it. card := detect.Hardware{TotalMemBytes: 128e9, CUDA: true, GPUName: "NVIDIA RTX A6000", GPUMemBytes: 48e9, GPUs: []detect.GPU{{Name: "NVIDIA RTX A6000"}}, InUseBytes: 120e9} - if w := hardwareWarnings(card); len(w) != 0 { + if w := hardwareWarnings(card, 0); len(w) != 0 { t.Errorf("discrete card on a busy host: %q", w) } @@ -687,25 +652,25 @@ func TestHardwareWarnings(t *testing.T) { {Vendor: "amd", VendorName: "AMD", PCIID: "0x1002:0x744c", Kind: "gpu", Driver: "amdgpu", VRAMBytes: 24e9}, {Vendor: "intel", VendorName: "Intel", PCIID: "0x8086:0x46a6", Kind: "gpu", Driver: "i915"}, }} - if w := hardwareWarnings(amd); len(w) != 1 || !strings.Contains(w[0], "AMD gpu (0x1002:0x744c, driver amdgpu, 24 GB) is present") { + if w := hardwareWarnings(amd, 0); len(w) != 1 || !strings.Contains(w[0], "AMD gpu (0x1002:0x744c, driver amdgpu, 24 GB) is present") { t.Errorf("AMD card: %q, want one warning and none for the integrated Intel GPU", w) } nodriver := detect.Hardware{TotalMemBytes: 64e9, Accelerators: []detect.Accelerator{{Vendor: "nvidia", VendorName: "NVIDIA", PCIID: "0x10de:0x2684", Kind: "gpu", Driver: "nouveau"}}} - if w := hardwareWarnings(nodriver); len(w) != 1 || !strings.Contains(w[0], "no NVIDIA driver answers") { + if w := hardwareWarnings(nodriver, 0); len(w) != 1 || !strings.Contains(w[0], "no NVIDIA driver answers") { t.Errorf("NVIDIA card without its driver: %q", w) } nodriver.CUDA = true - if w := hardwareWarnings(nodriver); len(w) != 0 { + if w := hardwareWarnings(nodriver, 0); len(w) != 0 { t.Errorf("an NVIDIA card the driver serves: %q", w) } // The driver's device node makes the machine a CUDA one even when // nvidia-smi fails, so that failure needs a warning of its own. mismatch := detect.ParseRemote("memtotal 64000000\nnvidiactl 1\n") - if w := hardwareWarnings(mismatch); len(w) != 1 || !strings.Contains(w[0], "nvidia-smi does not answer") { + if w := hardwareWarnings(mismatch, 0); len(w) != 1 || !strings.Contains(w[0], "nvidia-smi does not answer") { t.Errorf("nvidia-smi failing beside the driver's device node: %q", w) } - if w := hardwareWarnings(detect.ParseRemote("memtotal 64000000\nnvidiactl 1\nsmi ok\ngpu 0, GPU-1, NVIDIA RTX A6000, 49140\n")); len(w) != 0 { + if w := hardwareWarnings(detect.ParseRemote("memtotal 64000000\nnvidiactl 1\nsmi ok\ngpu 0, GPU-1, NVIDIA RTX A6000, 49140\n"), 0); len(w) != 0 { t.Errorf("a working nvidia-smi: %q", w) } } diff --git a/internal/cli/editset.go b/internal/cli/editset.go index 6c3f014..50e8afa 100644 --- a/internal/cli/editset.go +++ b/internal/cli/editset.go @@ -11,6 +11,7 @@ import ( "github.com/spf13/cobra" "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/control" "github.com/coder/llm-helper/internal/engine" "github.com/coder/llm-helper/internal/profile" ) @@ -203,9 +204,9 @@ func profileChangeHint(cfg *config.Config, name string) { } return } - if name == cfg.ActiveProfile() { - fmt.Printf("'%s' is the active profile -- run 'llm-helper restart' to apply the change.\n", name) + if _, err := control.LoadInstance(cfg, name); err == nil { + fmt.Printf("'%s' is running -- run 'llm-helper restart %s' to apply the change.\n", name, name) } else { - fmt.Printf("Saved. The change applies the next time you switch to '%s'.\n", name) + fmt.Printf("Saved. The change applies the next time you run 'llm-helper up %s'.\n", name) } } diff --git a/internal/cli/ggufplugin.go b/internal/cli/ggufplugin.go index a2cb2a7..40843b7 100644 --- a/internal/cli/ggufplugin.go +++ b/internal/cli/ggufplugin.go @@ -81,7 +81,7 @@ func runInstallGGUF(yes bool) error { fmt.Println() if vllmCaps(cfg).SupportsGGUF() { fmt.Println("Done -- this vLLM can now load GGUF models.") - fmt.Println("It takes effect on the next switch; nothing running was disturbed.") + fmt.Println("It takes effect when an instance next starts; nothing running was disturbed.") return nil } return fmt.Errorf("installed, but vLLM still doesn't register 'gguf'.\n" + diff --git a/internal/cli/instances.go b/internal/cli/instances.go new file mode 100644 index 0000000..cca09c1 --- /dev/null +++ b/internal/cli/instances.go @@ -0,0 +1,342 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "strconv" + + "github.com/spf13/cobra" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/control" + "github.com/coder/llm-helper/internal/profile" +) + +type upOpts struct { + port string + memUtil string + kvCache string +} + +func newUpCmd() *cobra.Command { + var w waitFlags + var o upOpts + c := &cobra.Command{ + Use: "up ", + Short: "Start serving a profile alongside anything already running", + GroupID: groupModels, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + if len(args) == 0 { + printProfiles(cfg) + fmt.Println() + fmt.Println("usage: " + cmdName + " up [--port N] [--mem-util F | --kv-cache SIZE]") + return errors.New("missing profile name") + } + p, err := profile.Load(cfg.Dirs, args[0]) + if err != nil { + return err + } + return upProfile(cfg, p, o, w.opts()) + }, + } + c.Flags().StringVar(&o.port, "port", "", "port to serve on (default: the next free one from PORT)") + c.Flags().StringVar(&o.memUtil, "mem-util", "", "fraction of GPU memory this model may use, e.g. 0.4") + c.Flags().StringVar(&o.kvCache, "kv-cache", "", "fixed KV cache size in bytes, e.g. 20G") + c.MarkFlagsMutuallyExclusive("mem-util", "kv-cache") + w.register(c) + return c +} + +// upProfile starts p as an instance. Starting one that already runs restarts +// it with the new settings. +func upProfile(cfg *config.Config, p *profile.Profile, o upOpts, opts watchOpts) error { + if err := profile.ValidInstanceName(p.Name); err != nil { + return fmt.Errorf("can't run '%s': %w; rename the profile", p.Name, err) + } + if o.memUtil != "" { + if f, err := strconv.ParseFloat(o.memUtil, 64); err != nil || f <= 0 || f > 1 { + return fmt.Errorf("--mem-util %q: want a fraction between 0 and 1", o.memUtil) + } + } + if legacyActive(cfg) { + return fmt.Errorf("this machine still runs a model the old single-model way; run '%s migrate' first", cmdName) + } + var cluster *clusterRunner + if p.Generated() { + c, err := clusterRunnerFor(cfg, p) + if err != nil { + return err + } + if err := c.preflight(cfg); err != nil { + return err + } + cluster = c + } + + if cluster == nil && (o.memUtil != "" || o.kvCache == "") { + if err := control.CheckFitFor(cfg, p.Name, o.memUtil, p.File, cfg.File); err != nil { + return err + } + } + + want := o.port + if want == "" && cluster != nil { + want = strconv.Itoa(cluster.recipe.Port) + } + port, err := control.AllocatePort(cfg, p.Name, want) + if err != nil { + return err + } + if err := control.WriteInstanceEnv(cfg, p, control.InstanceOpts{Port: port, MemUtil: o.memUtil, KVCache: o.kvCache}); err != nil { + return err + } + inst, err := control.LoadInstance(cfg, p.Name) + if err != nil { + return err + } + if cluster == nil { + self, err := os.Executable() + if err != nil { + return err + } + if _, err := instanceService(cfg, p.Name).Install(self); err != nil { + _ = control.RemoveInstance(cfg, p.Name) + return err + } + } + if prov, err := profileProvider(p); err == nil && prov.Capabilities().VLLMArgs { + warnExtraArgs(p, vllmCaps(cfg)) + } + + fmt.Printf("Starting profile %s on port %s\n", p.Name, port) + fmt.Println() + for i, line := range p.File.LeadingComments() { + if i >= 8 { + break + } + fmt.Println(line) + } + fmt.Println() + run := instanceRunner(cfg, p.Name) + restart := func() error { + fmt.Println("Loading the model (this can take several minutes, longer if it") + fmt.Println("isn't cached on disk yet)...") + return run.Restart() + } + if err := control.NotifyAndRestart(context.Background(), cfg, p, inst, os.Stderr, restart); err != nil { + return err + } + if !opts.follow && !opts.wait { + fmt.Println() + fmt.Println("Not waiting for the model to load. Check progress with 'llm-helper status'") + fmt.Println("or 'llm-helper logs " + p.Name + "'.") + return nil + } + fmt.Println() + return watchBoot(cfg, p.Name, opts) +} + +func newDownCmd() *cobra.Command { + return &cobra.Command{ + Use: "down [profile]", + Aliases: []string{"stop"}, + Short: "Stop serving a profile (the only one running when none is named)", + GroupID: groupServer, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + inst, err := pickInstance(cfg, args) + if len(args) == 0 && errors.Is(err, errNoInstances) { + fmt.Println("No model is running; nothing to stop.") + return nil + } + if err != nil { + return err + } + if err := downInstance(cfg, inst.Name); err != nil { + return err + } + fmt.Printf("Stopped %s.\n", inst.Name) + return nil + }, + } +} + +func downInstance(cfg *config.Config, name string) error { + if c := clusterRunnerForName(cfg, name); c != nil { + if err := c.Stop(); err != nil { + return err + } + } else if err := instanceService(cfg, name).Uninstall(); err != nil { + return err + } + if err := control.RemoveInstance(cfg, name); err != nil { + return err + } + control.NotifyStopped(context.Background(), cfg, name, os.Stderr) + return nil +} + +func newDefaultCmd() *cobra.Command { + return &cobra.Command{ + Use: "default [profile]", + Short: "Show or set the model Coder uses by default", + GroupID: groupModels, + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + live := control.LiveProfiles(cfg) + if len(args) == 0 { + if d := control.DefaultInstance(cfg, live); d != "" { + fmt.Println(d) + return nil + } + return errNoInstances + } + name := args[0] + if _, err := control.LoadInstance(cfg, name); err != nil { + if errors.Is(err, fs.ErrNotExist) { + return fmt.Errorf("'%s' is not running; start it with '%s up %s'", name, cmdName, name) + } + return err + } + cfg.File.Set("DEFAULT_PROFILE", name) + if err := cfg.File.Write(cfg.Dirs.ConfigFile(), 0o644); err != nil { + return err + } + control.NotifyDefault(context.Background(), cfg, os.Stderr) + fmt.Printf("%s is now the default model.\n", name) + return nil + }, + } +} + +// legacyActive reports a model still recorded the pre-instance way. +func legacyActive(cfg *config.Config) bool { + _, err := os.Stat(cfg.Dirs.LegacyActiveEnvFile()) + return err == nil +} + +func newMigrateCmd() *cobra.Command { + return &cobra.Command{ + Use: "migrate", + Short: "Move a model started by an older release to the per-model services", + GroupID: groupServer, + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + return runMigrate(cfg) + }, + } +} + +func runMigrate(cfg *config.Config) error { + name := cfg.LegacyActiveProfile() + if !legacyActive(cfg) && name == "" { + fmt.Println("Nothing to migrate.") + return nil + } + legacy := legacyService(cfg) + wasRunning := legacy.Installed() && legacy.IsActive() == "active" + if name != "" { + p, err := profile.Load(cfg.Dirs, name) + if err != nil { + return err + } + port := cfg.Port() + if old, err := config.ParseEnvFile(cfg.Dirs.LegacyActiveEnvFile()); err == nil && old.Get("PORT") != "" { + port = old.Get("PORT") + } + if err := profile.ValidInstanceName(p.Name); err != nil { + return fmt.Errorf("can't migrate '%s': %w; rename the profile and run '%s up' on it", p.Name, err, cmdName) + } + if err := control.WriteInstanceEnv(cfg, p, control.InstanceOpts{Port: port}); err != nil { + return err + } + if !p.Generated() { + self, err := os.Executable() + if err != nil { + return err + } + if legacy.Installed() { + if err := legacy.Uninstall(); err != nil { + return err + } + } + if _, err := instanceService(cfg, p.Name).Install(self); err != nil { + return err + } + if wasRunning { + if err := instanceService(cfg, p.Name).Restart(); err != nil { + return err + } + } + } + fmt.Printf("%s now runs as its own service on port %s.\n", p.Name, port) + } else if legacy.Installed() { + if err := legacy.Uninstall(); err != nil { + return err + } + } + for _, f := range []string{cfg.Dirs.LegacyActiveEnvFile(), cfg.Dirs.LegacyActiveProfileFile()} { + if err := os.Remove(f); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + } + return nil +} + +// newSwitchCmd is the pre-instance verb: stop everything else, run this. +func newSwitchCmd() *cobra.Command { + var w waitFlags + c := &cobra.Command{ + Use: "switch ", + Short: "Stop every other model and run this one", + Hidden: true, + Deprecated: "use 'up' and 'down'; several models can run at once", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + p, err := profile.Load(cfg.Dirs, args[0]) + if err != nil { + return err + } + list, err := control.ListInstances(cfg) + if err != nil { + return err + } + for _, inst := range list { + if inst.Name == p.Name { + continue + } + if err := downInstance(cfg, inst.Name); err != nil { + return err + } + fmt.Printf("Stopped %s.\n", inst.Name) + } + return upProfile(cfg, p, upOpts{}, w.opts()) + }, + } + w.register(c) + return c +} diff --git a/internal/cli/memfit.go b/internal/cli/memfit.go index 2574ae8..dc10a9e 100644 --- a/internal/cli/memfit.go +++ b/internal/cli/memfit.go @@ -24,7 +24,7 @@ const memHeadroom = 0.85 // the KV cache is left to vLLM's profiling, where this number alone decides how // much of the machine gets eaten. Both vLLM's own GB10 guidance and sparkrun's // platform default land on 0.85. -const maxUnifiedMemUtilization = 0.85 +const maxUnifiedMemUtilization = control.MaxUnifiedMemUtilization // capMemUtilization limits --gpu-memory-utilization to ceiling, returning the // value to use plus a line for the user ("" when the request was already fine). @@ -92,7 +92,7 @@ func clampMemUtilization(want string, totalMB, availMB int) (value, msg string) lowered := math.Floor(limit*100) / 100 if lowered < 0.05 { return want, fmt.Sprintf( - "warning: only %d MiB of %d MiB is free -- vLLM will likely refuse to start. Close some apps and switch again.\n", + "warning: only %d MiB of %d MiB is free -- vLLM will likely refuse to start. Close some apps and try again.\n", availMB, totalMB) } return strconv.FormatFloat(lowered, 'f', 2, 64), fmt.Sprintf( diff --git a/internal/cli/plugins.go b/internal/cli/plugins.go index b3b7915..54f98e4 100644 --- a/internal/cli/plugins.go +++ b/internal/cli/plugins.go @@ -1,24 +1,9 @@ package cli import ( - "fmt" - - "github.com/coder/llm-helper/internal/config" - // Integrations register themselves. Importing them here, rather than // from the plugin package, keeps the dependency pointing one way: // plugins may use the core, the core never imports a plugin. _ "github.com/coder/llm-helper/internal/plugin/coder" _ "github.com/coder/llm-helper/internal/plugin/spark" ) - -// localBaseURL is where this machine serves the OpenAI-compatible API. -// -// It is deliberately localhost: that is the only address vllm-helper can -// vouch for, since vLLM binds one port and what reaches it from elsewhere -// depends on the network (a LAN address, Tailscale, or a WSL port proxy). -// A plugin that registers an endpoint somebody else has to reach should -// take that address from its own configuration rather than guess from here. -func localBaseURL(cfg *config.Config) string { - return fmt.Sprintf("http://localhost:%s/v1", cfg.Port()) -} diff --git a/internal/cli/profiles.go b/internal/cli/profiles.go index 70ff815..a7c6238 100644 --- a/internal/cli/profiles.go +++ b/internal/cli/profiles.go @@ -28,7 +28,7 @@ type profileInfo struct { ExtraArgs string `json:"extra_args"` Status string `json:"status"` Thinking string `json:"thinking,omitempty"` - Active bool `json:"active"` + Running bool `json:"running"` } func newListCmd() *cobra.Command { @@ -51,7 +51,7 @@ func runList(jsonOut bool) error { } if jsonOut { profiles, _ := profile.List(cfg.Dirs) - active := cfg.ActiveProfile() + live := liveSet(cfg) out := make([]profileInfo, 0, len(profiles)) for _, p := range profiles { out = append(out, profileInfo{ @@ -62,7 +62,7 @@ func runList(jsonOut bool) error { ExtraArgs: p.ExtraArgs(), Status: p.Status(), Thinking: p.Thinking(), - Active: p.Name == active, + Running: live[p.Name] != "", }) } return printJSON(out) @@ -78,14 +78,14 @@ func printProfiles(cfg *config.Config) { fmt.Println(" (none yet -- add one with 'llm-helper new ')") return } - active := cfg.ActiveProfile() + live := liveSet(cfg) for _, p := range profiles { marker := "" if p.Thinking() == resolve.ThinkingOn { marker = " [thinking]" } - if p.Name == active { - marker += " [active]" + if port := live[p.Name]; port != "" { + marker += " [running :" + port + "]" } // A recipe profile with edits is no longer the thing we measured, so // say so wherever it is listed. @@ -98,7 +98,7 @@ func printProfiles(cfg *config.Config) { } } -// waitFlags are shared by switch/new/restart: how long to stay attached +// waitFlags are shared by up/new/restart: how long to stay attached // while the model loads. type waitFlags struct { noFollow bool @@ -128,47 +128,12 @@ func (w *waitFlags) opts() watchOpts { return opts } -func newSwitchCmd() *cobra.Command { - var w waitFlags - c := &cobra.Command{ - Use: "switch ", - Short: "Switch to a saved profile, restarting the server", - GroupID: groupModels, - // Not ExactArgs: with no profile named, runSwitch lists what's - // available, which beats cobra's count of missing arguments. - Args: cobra.MaximumNArgs(1), - RunE: func(_ *cobra.Command, args []string) error { - return runSwitch(args, w.opts()) - }, - } - w.register(c) - return c -} - -func runSwitch(args []string, opts watchOpts) error { - cfg, err := loadConfig() - if err != nil { - return err - } - if len(args) < 1 { - printProfiles(cfg) - fmt.Println("\nusage: llm-helper switch [--wait|--no-follow] [--timeout 30m]") - return errors.New("missing profile name") - } - p, err := profile.Load(cfg.Dirs, args[0]) - if err != nil { - printProfiles(cfg) - return err - } - return activateProfile(cfg, p, opts) -} - func newNewCmd() *cobra.Command { var w waitFlags var quant, thinking string c := &cobra.Command{ Use: "new ", - Short: "Pull a new HuggingFace model, save it as a profile, and switch to it", + Short: "Pull a new HuggingFace model, save it as a profile, and start it", GroupID: groupModels, Args: cobra.MaximumNArgs(1), Example: " llm-helper new Qwen/Qwen3-32B\n" + @@ -189,7 +154,7 @@ func newNewCmd() *cobra.Command { if err != nil { return err } - return pullAndSwitch(cmd.Context(), cfg, args[0], choice, thinking, w.opts()) + return pullAndUp(cmd.Context(), cfg, args[0], choice, thinking, w.opts()) }, } c.Flags().StringVar(&quant, "quant", "", @@ -203,9 +168,8 @@ func newNewCmd() *cobra.Command { // resolveQuant turns a --quant name into a choice, refusing anything this // machine could not actually serve. // -// Refusing here rather than at startup matters: activating a profile restarts -// the service, so a model that cannot load takes the running one down with it -// and leaves the machine serving nothing. +// Refused here rather than at startup, so a model that cannot load never gets +// as far as a service. func resolveQuant(ctx context.Context, cfg *config.Config, repo, want string) (*resolve.Choice, error) { if want == "" { return nil, nil @@ -245,8 +209,6 @@ func resolveQuant(ctx context.Context, cfg *config.Config, repo, want string) (* if !v.Servable() { return nil, fmt.Errorf("%s is %s", v.Label, v.Blocked) } - // Refused here rather than at startup: activating a profile restarts the - // service, so a model that cannot load takes the running one down with it. if v.GGUF && d.ArchBlocked != "" { return nil, fmt.Errorf("%s can't be served as GGUF -- it is %s", repo, d.ArchBlocked) } @@ -256,9 +218,9 @@ func resolveQuant(ctx context.Context, cfg *config.Config, repo, want string) (* return &resolve.Choice{Variant: v, BaseModel: d.BaseModel}, nil } -// pullAndSwitch is `new` minus the flag parsing, so `search` can funnel +// pullAndUp is `new` minus the flag parsing, so `search` can funnel // into it with its own options. -func pullAndSwitch(ctx context.Context, cfg *config.Config, repo string, choice *resolve.Choice, thinking string, opts watchOpts) error { +func pullAndUp(ctx context.Context, cfg *config.Config, repo string, choice *resolve.Choice, thinking string, opts watchOpts) error { name := profile.NameFromRepo(repo) if choice != nil { if _, ok := choice.Variant.Selector(); ok { @@ -273,7 +235,7 @@ func pullAndSwitch(ctx context.Context, cfg *config.Config, repo string, choice } fmt.Printf("Profile '%s' already exists for %s -- using its saved settings.\n", name, repo) fmt.Printf("(change it with 'llm-helper edit %s' if it needs different flags)\n", name) - return activateProfile(cfg, p, opts) + return upProfile(cfg, p, upOpts{}, opts) } fmt.Println("Pulling new model:", repo) @@ -301,7 +263,7 @@ func pullAndSwitch(ctx context.Context, cfg *config.Config, repo string, choice if err != nil { return err } - return activateProfile(cfg, p, opts) + return upProfile(cfg, p, upOpts{}, opts) } func newRemoveCmd() *cobra.Command { @@ -327,8 +289,8 @@ func runRemove(args []string) error { if _, err := profile.Load(cfg.Dirs, name); err != nil { return err } - if name == cfg.ActiveProfile() { - return fmt.Errorf("'%s' is the currently active model -- switch to something else first", name) + if liveSet(cfg)[name] != "" { + return fmt.Errorf("'%s' is running -- stop it first with 'llm-helper down %s'", name, name) } control.NotifyRemoved(context.Background(), cfg, name, os.Stderr) if err := profile.Remove(cfg.Dirs, name); err != nil { @@ -338,84 +300,24 @@ func runRemove(args []string) error { return nil } -// activateProfile is the core of switch/new: assemble the runtime env, -// record the active profile, sync Coder, restart the service, and watch -// the boot according to opts. -func activateProfile(cfg *config.Config, p *profile.Profile, opts watchOpts) error { - // WriteActiveEnv records the active profile as its last step, so the restart - // below resolves the runner for the profile being switched to rather than - // the one being left. That ordering is what lets a recipe profile go through - // here at all: it used to be refused because the restart could only ever - // mean the init system. - // - // Check the runner can act BEFORE publishing anything. The Coder notify - // happens before the restart, so a failure after this point leaves Coder - // advertising a model that is not being served -- switching to a recipe - // whose profile had no recorded topology did exactly that. - if p.Generated() { - c, err := clusterRunnerFor(cfg, p) - if err != nil { - return err - } - if err := c.preflight(cfg); err != nil { - return err - } - } - // Captured before WriteActiveEnv changes which runner svc resolves to. - outgoing := svc(cfg) - if err := control.WriteActiveEnv(cfg, p); err != nil { - return err - } - incoming := svc(cfg) - // Warn, never refuse: profiles get hand-edited and vLLM gets upgraded - // underneath one that used to work, but the user asked for this model - // and it is not this command's place to overrule that. - if prov, err := profileProvider(p); err == nil && prov.Capabilities().VLLMArgs { - warnExtraArgs(p, vllmCaps(cfg)) - } - - fmt.Println("Switched to profile:", p.Name) - fmt.Println() - for i, line := range p.File.LeadingComments() { - if i >= 8 { - break - } - fmt.Println(line) - } - - fmt.Println() - // The banner belongs to the restart rather than before it, so that a - // plugin taking its time doesn't leave "Restarting..." on screen with - // nothing happening yet. - restart := func() error { - if err := stopOutgoing(outgoing, incoming); err != nil { - return err - } - fmt.Println() - fmt.Println("Reloading the model (this can take several minutes, longer if it") - fmt.Println("isn't cached on disk yet)...") - return incoming.Restart() - } - if err := control.NotifyAndRestart(context.Background(), cfg, p, os.Stderr, restart); err != nil { - return err - } - - if !opts.follow && !opts.wait { - fmt.Println() - fmt.Println("Not waiting for the model to load. Check progress with 'llm-helper status'") - fmt.Println("or 'llm-helper logs'.") - return nil - } - fmt.Println() - return watchBoot(cfg, p.Name, opts) +// liveSet maps each running profile to its port. +func liveSet(cfg *config.Config) map[string]string { + out := map[string]string{} + list, _ := control.ListInstances(cfg) + for _, inst := range list { + out[inst.Name] = inst.Port + } + return out } -// statusReport is the machine-readable shape of `status`. +// statusReport is the machine-readable shape of `status`, one per instance. type statusReport struct { Service string `json:"service"` Profile string `json:"profile"` + Engine string `json:"engine,omitempty"` Model string `json:"model"` Port string `json:"port"` + Default bool `json:"default,omitempty"` Healthy bool `json:"endpoint_healthy"` HTTPStatus int `json:"http_status"` Crashed bool `json:"crashed"` @@ -423,38 +325,39 @@ type statusReport struct { FailReason string `json:"fail_reason,omitempty"` LastLog string `json:"last_log,omitempty"` Thinking string `json:"thinking,omitempty"` + MemUtil string `json:"mem_util,omitempty"` } -func gatherStatus(cfg *config.Config) statusReport { - run := svc(cfg) +func gatherStatus(cfg *config.Config, inst *control.Instance, def string) statusReport { + run := instanceRunner(cfg, inst.Name) health := run.Health() r := statusReport{ Service: health.State, - Profile: cfg.ActiveProfile(), - Model: "unknown", - Port: cfg.Port(), + Profile: inst.Name, + Engine: inst.Env.Get(engine.Key), + Model: inst.Env.Get("MODEL"), + Port: inst.Port, + Default: inst.Name == def, Crashed: health.Crashed, Restarts: health.NRestarts, FailReason: health.Result, + MemUtil: inst.Env.Get("GPU_MEMORY_UTILIZATION"), } - if r.Profile == "" { - r.Profile = "unknown" + if r.Engine == "" { + r.Engine = engine.VLLM } - if active, err := config.ParseEnvFile(cfg.Dirs.ActiveEnvFile()); err == nil { - if m := active.Get("MODEL"); m != "" { - r.Model = m - } + if r.Model == "" { + r.Model = "unknown" } - if p, err := profile.Load(cfg.Dirs, r.Profile); err == nil { + if p, err := profile.Load(cfg.Dirs, inst.Name); err == nil { r.Thinking = p.Thinking() } if c, ok := run.(*clusterRunner); ok { r.HTTPStatus, r.Healthy = c.probeHealth() } else { - r.HTTPStatus, r.Healthy = control.ProbeActive(cfg) + r.HTTPStatus, r.Healthy = inst.Probe() } if r.Crashed { - // Only worth the journalctl call when there's a failure to explain. r.LastLog = control.LastMeaningfulLine(run.LastLogs(40)) } return r @@ -463,27 +366,87 @@ func gatherStatus(cfg *config.Config) statusReport { func newStatusCmd() *cobra.Command { var jsonOut bool c := &cobra.Command{ - Use: "status", - Short: "Server state, active model, and whether the endpoint responds", + Use: "status [profile]", + Short: "Running models, and whether each endpoint responds", GroupID: groupServer, - Args: cobra.NoArgs, - RunE: func(*cobra.Command, []string) error { return runStatus(jsonOut) }, + Args: cobra.MaximumNArgs(1), + RunE: func(_ *cobra.Command, args []string) error { return runStatus(args, jsonOut) }, } c.Flags().BoolVar(&jsonOut, "json", false, "print status as JSON") return c } -func runStatus(jsonOut bool) error { +func runStatus(args []string, jsonOut bool) error { cfg, err := loadConfig() if err != nil { return err } - r := gatherStatus(cfg) + var list []*control.Instance + if len(args) > 0 { + inst, err := pickInstance(cfg, args) + if err != nil { + return err + } + list = []*control.Instance{inst} + } else if list, err = control.ListInstances(cfg); err != nil { + return err + } + def := control.DefaultInstance(cfg, control.LiveProfiles(cfg)) + reports := make([]statusReport, len(list)) + for i, inst := range list { + reports[i] = gatherStatus(cfg, inst, def) + } if jsonOut { - return printJSON(r) + if len(args) > 0 { + return printJSON(reports[0]) + } + return printJSON(reports) } - fmt.Printf("%-9s %s\n", svc(cfg).Label()+":", r.Service) - fmt.Printf("profile: %s\n", r.Profile) + switch len(reports) { + case 0: + if legacyActive(cfg) { + fmt.Printf("A model was started by an older release; run '%s migrate' to manage it.\n", cmdName) + return nil + } + fmt.Println(errNoInstances.Error()) + return nil + case 1: + printStatusDetail(cfg, reports[0]) + return nil + } + fmt.Printf("%-24s %-10s %-6s %-10s %-6s %s\n", "PROFILE", "ENGINE", "PORT", "STATE", "MEMORY", "") + for _, r := range reports { + state := "loading" + switch { + case r.Healthy: + state = "healthy" + case r.Crashed: + state = "failing" + case r.Service == "inactive" || r.Service == "unknown": + state = "stopped" + } + mark := "" + if r.Default { + mark = "default" + } + mem := r.MemUtil + if mem == "" { + mem = "-" + } + fmt.Printf("%-24s %-10s %-6s %-10s %-6s %s\n", r.Profile, r.Engine, r.Port, state, mem, mark) + } + fmt.Printf("\nDetails for one: %s status \n", cmdName) + return nil +} + +func printStatusDetail(cfg *config.Config, r statusReport) { + run := instanceRunner(cfg, r.Profile) + fmt.Printf("%-9s %s\n", run.Label()+":", r.Service) + def := "" + if r.Default { + def = " (default)" + } + fmt.Printf("profile: %s%s\n", r.Profile, def) fmt.Printf("model: %s\n", r.Model) switch r.Thinking { case resolve.ThinkingOn: @@ -493,12 +456,8 @@ func runStatus(jsonOut bool) error { } if r.Healthy { fmt.Printf("endpoint: healthy (http %d on port %s)\n", r.HTTPStatus, r.Port) - return nil + return } - - // An unresponsive endpoint means one of two very different things, and - // telling someone to wait for a server that keeps dying wastes their - // afternoon. Say which one it is. if r.Crashed { fmt.Printf("endpoint: not responding (http %03d on port %s) -- the service is failing to start, not loading\n", r.HTTPStatus, r.Port) detail := r.FailReason @@ -513,17 +472,14 @@ func runStatus(jsonOut bool) error { fmt.Printf("last log: %s\n", r.LastLog) } } else { - // "still loading" is wrong when nothing is running: a stopped model is - // waiting for someone to start it, not making progress. switch r.Service { case "inactive", "unknown": - fmt.Printf("endpoint: not responding (http %03d on port %s) -- nothing is running; start it with 'llm-helper restart'\n", r.HTTPStatus, r.Port) + fmt.Printf("endpoint: not responding (http %03d on port %s) -- nothing is running; start it with 'llm-helper restart %s'\n", r.HTTPStatus, r.Port, r.Profile) default: fmt.Printf("endpoint: not responding (http %03d on port %s) -- still loading, or check the logs\n", r.HTTPStatus, r.Port) } } - if hint := svc(cfg).LogHint(); hint != "" { + if hint := run.LogHint(); hint != "" { fmt.Printf("logs: %s\n", hint) } - return nil } diff --git a/internal/cli/recipehealth.go b/internal/cli/recipehealth.go index 88b7042..06dee86 100644 --- a/internal/cli/recipehealth.go +++ b/internal/cli/recipehealth.go @@ -12,6 +12,7 @@ import ( "github.com/spf13/cobra" "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/control" "github.com/coder/llm-helper/internal/detect" ) @@ -27,6 +28,7 @@ const ( func newRecipesHealthCheckCmd() *cobra.Command { var ( unit string + prof string threshold int statePath string interval int @@ -44,10 +46,11 @@ func newRecipesHealthCheckCmd() *cobra.Command { if statePath == "" { statePath = healthStatePath(unit) } - return runHealthCheck(unit, threshold, statePath, time.Duration(interval)*time.Minute) + return runHealthCheck(unit, prof, threshold, statePath, time.Duration(interval)*time.Minute) }, } c.Flags().StringVar(&unit, "unit", defaultRecipeServiceName, "unit to relaunch") + c.Flags().StringVar(&prof, "profile", "", "recipe profile whose endpoint to check (default: any running)") c.Flags().IntVar(&threshold, "misses", defaultHealthMisses, "consecutive failures before relaunching") c.Flags().StringVar(&statePath, "state", "", "where the miss count is kept") c.Flags().IntVar(&interval, "interval", defaultHealthInterval, "minutes between checks, which is also how long a fresh launch marker is trusted") @@ -72,7 +75,7 @@ const launchStandOff = 15 * time.Minute // markLaunching announces that a launch is under way. Without it the watchdog // reads the silence of a loading model as an outage and restarts the unit, -// which relaunches the OLD active profile over the top of the new one. +// which relaunches the old launch over the top of the new one. // // The marker outlives the launcher on purpose: "recipes launch" returns // minutes before the endpoint answers. It is cleared by the first health @@ -203,10 +206,10 @@ func healthDecision(unitState string, launching, stopped, healthy bool, misses, return healthCount, misses } -func runHealthCheck(unit string, threshold int, statePath string, interval time.Duration) error { +func runHealthCheck(unit, prof string, threshold int, statePath string, interval time.Duration) error { cfg, _ := loadConfig() state := strings.TrimSpace(systemctlOutput("is-active", unit)) - healthy := endpointHealthy(cfg) + healthy := endpointHealthy(cfg, prof) misses := readMisses(statePath) action, next := healthDecision(state, launchInProgress(cfg), stoppedOnPurpose(cfg), healthy, misses, threshold) @@ -233,13 +236,28 @@ func runHealthCheck(unit string, threshold int, statePath string, interval time. // endpointHealthy asks the endpoint status reports on, so the watchdog and // the operator agree on what healthy means. For a recipe that is the head // node on the recipe's port, which need not be this machine. -func endpointHealthy(cfg *config.Config) bool { +func endpointHealthy(cfg *config.Config, prof string) bool { if cfg == nil { return false } - if c := activeClusterRunner(cfg); c != nil { - _, ok := c.probeHealth() - return ok + if prof != "" { + if c := clusterRunnerForName(cfg, prof); c != nil { + _, ok := c.probeHealth() + return ok + } + } + list, _ := control.ListInstances(cfg) + for _, inst := range list { + if c := clusterRunnerForName(cfg, inst.Name); c != nil { + if _, ok := c.probeHealth(); ok { + return true + } + } else if _, ok := inst.Probe(); ok { + return true + } + } + if len(list) > 0 { + return false } for _, ep := range detect.ProbePorts([]string{cfg.Port()}, cfg.APIKey()) { if ep.Healthy { @@ -284,8 +302,8 @@ func renderHealthUnits(u recipeSystemdUnit, unitName string, intervalMin int) (s "[Service]\n"+ "Type=oneshot\n"+ "Environment=HOME=%s\n"+ - "ExecStart=%s recipes health-check --unit %s --interval %d\n", - unitName, u.Home, u.Self, unitName, intervalMin) + "ExecStart=%s recipes health-check --unit %s --profile %s --interval %d\n", + unitName, u.Home, u.Self, unitName, u.Profile, intervalMin) timer = fmt.Sprintf("[Unit]\n"+ "Description=Run the health check for %s every %d minutes\n\n"+ "[Timer]\n"+ diff --git a/internal/cli/recipehealth_internal_test.go b/internal/cli/recipehealth_internal_test.go index 94fc3e9..97b69a8 100644 --- a/internal/cli/recipehealth_internal_test.go +++ b/internal/cli/recipehealth_internal_test.go @@ -113,10 +113,10 @@ func TestHealthNamesFollowTheUnit(t *testing.T) { func TestHealthUnitsRunTheCheckerAgainstTheirUnit(t *testing.T) { svc, timer := renderHealthUnits( - recipeSystemdUnit{Home: "/home/serve", Self: "/usr/local/bin/vllm-helper"}, + recipeSystemdUnit{Home: "/home/serve", Self: "/usr/local/bin/vllm-helper", Profile: "x"}, "vllm-cluster.service", 2) for _, want := range []string{ - "ExecStart=/usr/local/bin/vllm-helper recipes health-check --unit vllm-cluster.service --interval 2", + "ExecStart=/usr/local/bin/vllm-helper recipes health-check --unit vllm-cluster.service --profile x --interval 2", "Environment=HOME=/home/serve", "Type=oneshot", } { diff --git a/internal/cli/recipelaunch.go b/internal/cli/recipelaunch.go index 345aed5..4f30660 100644 --- a/internal/cli/recipelaunch.go +++ b/internal/cli/recipelaunch.go @@ -44,8 +44,7 @@ type recipeLaunchOptions struct { sshKey string dryRun bool skipCoder bool - // relaunch marks a restart of the recipe the active profile already - // describes, as opposed to a user's --no-activate. + // relaunch marks a restart of an existing instance, not --no-activate. relaunch bool } @@ -56,16 +55,16 @@ func newRecipesLaunchCmd() *cobra.Command { ) c := &cobra.Command{ Use: "launch ", - Short: "Launch a tested recipe, and point the active profile and Coder at it", - Long: `Launch a tested recipe, and point the active profile and Coder at it. + Short: "Launch a tested recipe as an instance, and tell Coder about it", + Long: `Launch a tested recipe as an instance, and tell Coder about it. -Like switch and restart, it then watches the load until the head node answers +Like up and restart, it then watches the load until the head node answers its engine's health check: following the log on a terminal, waiting quietly otherwise. Pass --no-follow to return as soon as the containers have started. -To bring the current model back up without switching, use "llm-helper restart": -it drives whichever runner the active profile belongs to.`, +To bring a running instance back up, use "llm-helper restart ": +it drives whichever runner that instance belongs to.`, Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, a []string) error { if err := runRecipeLaunch(cmd.Context(), a[0], o); err != nil || o.dryRun { @@ -82,7 +81,7 @@ it drives whichever runner the active profile belongs to.`, f.StringVar(&o.container, "container-name", defaultRecipeContainer, "container name on each node") f.StringVar(&o.sshKey, "ssh-key", "", "private key for reaching non-head nodes") f.BoolVar(&o.dryRun, "dry-run", false, "print what would run and change nothing") - f.BoolVar(&o.skipCoder, "no-activate", false, "launch only; leave the active profile and Coder alone") + f.BoolVar(&o.skipCoder, "no-activate", false, "launch only; record no instance and leave Coder alone") return c } @@ -399,21 +398,14 @@ func runRecipeLaunch(ctx context.Context, id string, o recipeLaunchOptions) erro return activateRecipeProfile(ctx, cfg, r, local, o) } -// waitForLaunch watches a launch the way switch and restart watch theirs. It -// used to return as soon as the containers started, minutes before the -// endpoint answered, while restart on the same recipe waited for it. -// -// The runner is built from the launch itself rather than the active profile, -// so a --no-activate launch is watched too. +// waitForLaunch watches a launch until its endpoint answers, as up and restart +// do. The runner is built from the launch itself, so a --no-activate launch is +// watched too. func waitForLaunch(id string, o recipeLaunchOptions, w watchOpts) error { if !w.follow && !w.wait { fmt.Println(notWaitingNote(o)) return nil } - cfg, err := config.Load() - if err != nil { - return err - } tested, err := recipe.LoadTested() if err != nil { return err @@ -422,7 +414,7 @@ func waitForLaunch(id string, o recipeLaunchOptions, w watchOpts) error { if !ok { return fmt.Errorf("no tested recipe %q", id) } - t, err := bootTargetFor(cfg, &clusterRunner{recipe: r, opts: o}) + t, err := bootTargetFor(&clusterRunner{recipe: r, opts: o}, nil) if err != nil { return err } @@ -430,24 +422,20 @@ func waitForLaunch(id string, o recipeLaunchOptions, w watchOpts) error { return watchTarget(t, r.ID, w) } -// notWaitingNote says where to look after a --no-follow launch. Status reads -// the active profile, which a --no-activate launch left pointing elsewhere. +// notWaitingNote says where to look after a --no-follow launch. func notWaitingNote(o recipeLaunchOptions) string { if o.skipCoder { - return "Not waiting for the model to load. The active profile does not describe this launch, so follow it with 'docker logs -f " + o.container + "' on the head node." + return "Not waiting for the model to load. No instance records this launch, so follow it with 'docker logs -f " + o.container + "' on the head node." } return "Not waiting for the model to load. Check progress with 'llm-helper status'." } -// skippedActivationNote is what a launch that left the profile alone tells -// the user. A restart relaunches the recipe the profile already describes, so -// warning that the profile still describes "whatever ran before" would be -// false there. +// skippedActivationNote is what a --no-activate launch tells the user. func skippedActivationNote(o recipeLaunchOptions) string { if o.relaunch { return "" } - return "skipped profile activation, so the active profile and Coder still describe whatever ran before" + return "skipped recording an instance, so status and Coder do not list this launch" } // recipeOverrides reads the local override layer for a recipe: the profile of @@ -563,8 +551,7 @@ func recipeFlag(r recipe.TestedRecipe, flag, fallback string) string { } // newRecipeProfile writes the profile a recipe's first launch is recorded -// as. It names the engine whenever that is not vLLM, so switch, status and -// the watchdog act on the engine that is serving. +// as. It names the engine whenever that is not vLLM. func newRecipeProfile(dirs config.Dirs, r recipe.TestedRecipe, limit string) (*profile.Profile, error) { opts := profile.CreateOpts{ Repo: r.Model, @@ -595,16 +582,10 @@ func newRecipeProfile(dirs config.Dirs, r recipe.TestedRecipe, limit string) (*p return p, nil } -// activateRecipeProfile records the launch as a profile and runs the same -// activation switch does, so the active profile, active.env and Coder all -// describe what is actually serving. The raw docker path bypassed all three. +// activateRecipeProfile records the launch as a profile and an instance and +// tells Coder, so all three describe what is actually serving. func activateRecipeProfile(ctx context.Context, cfg *config.Config, r recipe.TestedRecipe, local overrideLayer, o recipeLaunchOptions) error { - // The context Coder advertises is what the server is configured to serve, - // so the recipe's own context flag is the answer whenever it has one. - // Requiring a verified context here refused to activate an entry that - // had not been measured yet, which left the launch serving a model the - // active profile did not name -- the exact incoherence this path exists - // to prevent. + // Coder advertises the context the server is configured to serve. limit := r.ContextLimit() if limit == "" { return fmt.Errorf("%s sets no context limit in its args and records no verified context, so there is no context limit to advertise", r.ID) @@ -638,15 +619,22 @@ func activateRecipeProfile(ctx context.Context, cfg *config.Config, r recipe.Tes if err := p.File.Write(cfg.Dirs.ProfileFile(p.Name), 0o644); err != nil { return err } - if err := control.WriteActiveEnv(cfg, p); err != nil { + port, err := control.AllocatePort(cfg, p.Name, strconv.Itoa(r.Port)) + if err != nil { + return err + } + if err := control.WriteInstanceEnv(cfg, p, control.InstanceOpts{Port: port}); err != nil { + return err + } + inst, err := control.LoadInstance(cfg, p.Name) + if err != nil { return err } - // The containers are already up, so there is no service to restart here; - // the notify half is what syncs Coder. - if err := control.NotifyAndRestart(ctx, cfg, p, os.Stderr, func() error { return nil }); err != nil { + // The containers are already up; the notify half is what syncs Coder. + if err := control.NotifyAndRestart(ctx, cfg, p, inst, os.Stderr, func() error { return nil }); err != nil { return err } - fmt.Printf("active profile is now %s\n", p.Name) + fmt.Printf("%s is now serving\n", p.Name) if local.pruned { fmt.Println("EXTRA_ARGS: dropped the flags that only restated the recipe.") } diff --git a/internal/cli/recipelaunch_internal_test.go b/internal/cli/recipelaunch_internal_test.go index 9ee3086..2d4a605 100644 --- a/internal/cli/recipelaunch_internal_test.go +++ b/internal/cli/recipelaunch_internal_test.go @@ -241,7 +241,6 @@ func TestLaunchWaitsLikeRestart(t *testing.T) { t.Errorf("recipes launch has no --%s", name) } } - // status reads the active profile, which --no-activate leaves alone. if note := notWaitingNote(recipeLaunchOptions{skipCoder: true, container: "vllm-node"}); !strings.Contains(note, "docker logs -f vllm-node") { t.Errorf("--no-activate --no-follow points at %q", note) } @@ -249,7 +248,7 @@ func TestLaunchWaitsLikeRestart(t *testing.T) { // A recipe's profile names its engine unless that is vLLM, including an // engine that takes vLLM's args and so keeps the sizing keys. Without the -// name, switch and the watchdog would treat it as plain vLLM. +// name, restart and the watchdog would treat it as plain vLLM. // // Not parallel: UseLocal swaps the engines every Lookup reads. func TestNewRecipeProfileNamesTheEngine(t *testing.T) { diff --git a/internal/cli/recipes.go b/internal/cli/recipes.go index 47d2360..5cd4713 100644 --- a/internal/cli/recipes.go +++ b/internal/cli/recipes.go @@ -25,7 +25,8 @@ func newRecipesCmd() *cobra.Command { GroupID: groupModels, } c.AddCommand(newRecipesListCmd(), newRecipesShowCmd(), newRecipesLaunchCmd(), newRecipesStopCmd(), - newRecipesInstallServiceCmd(), newRecipesHealthCheckCmd()) + newRecipesInstallServiceCmd(), newRecipesHealthCheckCmd(), + newRecipesServiceStartCmd(), newRecipesServiceStopCmd()) return c } diff --git a/internal/cli/recipeservice.go b/internal/cli/recipeservice.go index 2066824..aee16ee 100644 --- a/internal/cli/recipeservice.go +++ b/internal/cli/recipeservice.go @@ -3,11 +3,16 @@ package cli import ( "errors" "fmt" + "io/fs" "os" "runtime" "strings" "github.com/spf13/cobra" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/control" + "github.com/coder/llm-helper/internal/profile" ) const defaultRecipeServiceName = "vllm-cluster.service" @@ -20,6 +25,7 @@ const defaultRecipeServiceTimeout = 1800 func newRecipesInstallServiceCmd() *cobra.Command { var ( name string + prof string user string timeout int now bool @@ -29,26 +35,28 @@ func newRecipesInstallServiceCmd() *cobra.Command { ) c := &cobra.Command{ Use: "install-service", - Short: "Install a systemd service that keeps the active recipe serving", - Long: "Installs a unit that runs 'llm-helper restart' at boot and 'llm-helper stop' on\n" + - "shutdown. The unit names no recipe: which one runs comes from the active profile,\n" + - "so switching recipes does not mean editing the unit. Linux hosts only.", + Short: "Install a systemd service that keeps a recipe profile serving", + Long: "Installs a unit that brings a recipe profile back at boot and stops its containers\n" + + "on shutdown. --profile names it; without it, the only recipe profile running is used.\n" + + "Install one unit per recipe profile, each with its own --name. Linux hosts only.", Args: cobra.NoArgs, RunE: func(*cobra.Command, []string) error { - // The unit this replaces is whichever one config names, so a - // machine that renamed its service still has the right one stood - // down rather than a guess. - native := "" - if cfg, err := loadConfig(); err == nil { - native = cfg.ServiceName() + cfg, err := loadConfig() + if err != nil { + return err + } + prof, err := recipeServiceProfile(cfg, prof) + if err != nil { + return err } return installRecipeService(recipeServiceOptions{ Name: name, + Profile: prof, User: userOrInvoker(user), Timeout: timeout, Start: now, Supersede: supersede, - Native: native, + Native: cfg.ServiceName(), Watchdog: watchdog, Interval: interval, Service: defaultRecipeServiceRunner{}, @@ -56,6 +64,7 @@ func newRecipesInstallServiceCmd() *cobra.Command { }, } c.Flags().StringVar(&name, "name", defaultRecipeServiceName, "systemd unit name") + c.Flags().StringVar(&prof, "profile", "", "recipe profile the unit serves (default: the only one running)") c.Flags().StringVar(&user, "user", "", "user the launcher runs as (default: $USER)") c.Flags().IntVar(&timeout, "timeout", defaultRecipeServiceTimeout, "seconds to allow for the model to load") c.Flags().BoolVar(&now, "now", false, "enable and start the service immediately") @@ -65,8 +74,40 @@ func newRecipesInstallServiceCmd() *cobra.Command { return c } +// recipeServiceProfile resolves the profile a recipe unit serves. +func recipeServiceProfile(cfg *config.Config, name string) (string, error) { + if name != "" { + p, err := profile.Load(cfg.Dirs, name) + if err != nil { + return "", err + } + if !p.Generated() { + return "", fmt.Errorf("'%s' is not a recipe profile; '%s up %s' already installs its service", name, cmdName, name) + } + return name, nil + } + list, err := control.ListInstances(cfg) + if err != nil { + return "", err + } + var found []string + for _, inst := range list { + if clusterRunnerForName(cfg, inst.Name) != nil { + found = append(found, inst.Name) + } + } + switch len(found) { + case 1: + return found[0], nil + case 0: + return "", errors.New("no recipe profile is running; launch one first or pass --profile") + } + return "", fmt.Errorf("%d recipe profiles are running (%s); pass --profile", len(found), strings.Join(found, ", ")) +} + type recipeServiceOptions struct { Name string + Profile string User string Timeout int Start bool @@ -97,6 +138,9 @@ func installRecipeService(o recipeServiceOptions) error { if strings.TrimSpace(o.Name) == "" { return errors.New("--name is required") } + if strings.TrimSpace(o.Profile) == "" { + return errors.New("--profile is required") + } if o.Timeout <= 0 { return errors.New("--timeout must be a number of seconds") } @@ -112,6 +156,7 @@ func installRecipeService(o recipeServiceOptions) error { User: o.User, Home: home, Self: self, + Profile: o.Profile, Timeout: o.Timeout, }) path, err := o.Service.InstallSystemd(o.Name, unit) @@ -141,7 +186,7 @@ func installRecipeService(o recipeServiceOptions) error { } healthService, timer := healthUnitNames(o.Name) healthTimer = timer - svcUnit, timerUnit := renderHealthUnits(recipeSystemdUnit{Home: home, Self: self}, o.Name, interval) + svcUnit, timerUnit := renderHealthUnits(recipeSystemdUnit{Home: home, Self: self, Profile: o.Profile}, o.Name, interval) if _, err := o.Service.InstallSystemd(healthService, svcUnit); err != nil { return err } @@ -178,6 +223,7 @@ type recipeSystemdUnit struct { User string Home string Self string + Profile string Timeout int } @@ -189,7 +235,7 @@ type recipeSystemdUnit struct { // cleared, not just the head. func renderRecipeSystemdUnit(u recipeSystemdUnit) string { return fmt.Sprintf("[Unit]\n"+ - "Description=LLM Helper cluster (the active recipe)\n"+ + "Description=LLM Helper cluster (%s)\n"+ "After=network-online.target docker.service\n"+ "Wants=network-online.target\n\n"+ "[Service]\n"+ @@ -198,12 +244,54 @@ func renderRecipeSystemdUnit(u recipeSystemdUnit) string { "User=%s\n"+ "Environment=HOME=%s\n"+ "WorkingDirectory=%s\n"+ - "ExecStart=%s restart\n"+ - "ExecStop=%s stop\n"+ + "ExecStart=%s recipes service-start %s\n"+ + "ExecStop=%s recipes service-stop %s\n"+ "TimeoutStartSec=%d\n\n"+ "[Install]\n"+ "WantedBy=multi-user.target\n", - u.User, u.Home, u.Home, u.Self, u.Self, u.Timeout) + u.Profile, u.User, u.Home, u.Home, u.Self, u.Profile, u.Self, u.Profile, u.Timeout) +} + +// newRecipesServiceStartCmd is the unit's ExecStart. A profile taken down on +// purpose stays down: there is no instance left to bring back. +func newRecipesServiceStartCmd() *cobra.Command { + return &cobra.Command{ + Use: "service-start ", + Hidden: true, + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + if _, err := control.LoadInstance(cfg, args[0]); errors.Is(err, fs.ErrNotExist) { + fmt.Printf("%s is not running; leaving it stopped.\n", args[0]) + return nil + } + return runRestart(args, watchOpts{wait: true}) + }, + } +} + +// newRecipesServiceStopCmd is the unit's ExecStop. It clears the containers +// but keeps the instance, so the next boot brings it back. +func newRecipesServiceStopCmd() *cobra.Command { + return &cobra.Command{ + Use: "service-stop ", + Hidden: true, + Args: cobra.ExactArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + c := clusterRunnerForName(cfg, args[0]) + if c == nil { + return fmt.Errorf("'%s' is not a recipe profile", args[0]) + } + return c.Stop() + }, + } } type defaultRecipeServiceRunner struct{} diff --git a/internal/cli/recipeservice_internal_test.go b/internal/cli/recipeservice_internal_test.go index 2a46073..faa836e 100644 --- a/internal/cli/recipeservice_internal_test.go +++ b/internal/cli/recipeservice_internal_test.go @@ -37,15 +37,14 @@ func (f *fakeRecipeService) Supersede(n string) error { return nil } -// The unit must not name a recipe: which one runs comes from the active -// profile, so switching recipes cannot mean editing the unit. +// The unit names the profile it serves, not the recipe behind it. func TestRecipeUnitNamesNoRecipe(t *testing.T) { unit := renderRecipeSystemdUnit(recipeSystemdUnit{ - User: "serve", Home: "/home/serve", Self: "/usr/local/bin/vllm-helper", Timeout: 1800, + User: "serve", Home: "/home/serve", Self: "/usr/local/bin/vllm-helper", Timeout: 1800, Profile: "x", }) for _, want := range []string{ - "ExecStart=/usr/local/bin/vllm-helper restart", - "ExecStop=/usr/local/bin/vllm-helper stop", + "ExecStart=/usr/local/bin/vllm-helper recipes service-start x", + "ExecStop=/usr/local/bin/vllm-helper recipes service-stop x", "Type=oneshot", "RemainAfterExit=yes", "TimeoutStartSec=1800", @@ -56,7 +55,7 @@ func TestRecipeUnitNamesNoRecipe(t *testing.T) { } } if strings.Contains(unit, "recipes launch") { - t.Errorf("the unit pins a recipe, which the active profile is supposed to decide:\n%s", unit) + t.Errorf("the unit pins a recipe instead of a profile:\n%s", unit) } } @@ -80,25 +79,25 @@ func TestInstallRecipeServiceStandsDownTheNativeUnit(t *testing.T) { }{ { name: "supersedes the configured unit", - opts: recipeServiceOptions{Name: "vllm-cluster.service", User: "serve", Timeout: 60, Supersede: true, Native: "vllm.service"}, + opts: recipeServiceOptions{Name: "vllm-cluster.service", Profile: "x", User: "serve", Timeout: 60, Supersede: true, Native: "vllm.service"}, superseded: "vllm.service", }, { name: "leaves it alone when asked", - opts: recipeServiceOptions{Name: "vllm-cluster.service", User: "serve", Timeout: 60, Supersede: false, Native: "vllm.service"}, + opts: recipeServiceOptions{Name: "vllm-cluster.service", Profile: "x", User: "serve", Timeout: 60, Supersede: false, Native: "vllm.service"}, }, { name: "never stands down the unit it just installed", - opts: recipeServiceOptions{Name: "vllm.service", User: "serve", Timeout: 60, Supersede: true, Native: "vllm.service"}, + opts: recipeServiceOptions{Name: "vllm.service", Profile: "x", User: "serve", Timeout: 60, Supersede: true, Native: "vllm.service"}, }, { name: "starts only when asked", - opts: recipeServiceOptions{Name: "vllm-cluster.service", User: "serve", Timeout: 60, Start: true}, + opts: recipeServiceOptions{Name: "vllm-cluster.service", Profile: "x", User: "serve", Timeout: 60, Start: true}, enabled: []string{"vllm-cluster.service"}, }, { name: "enables the watchdog timer alongside the unit", - opts: recipeServiceOptions{Name: "vllm-cluster.service", User: "serve", Timeout: 60, Start: true, Watchdog: true, Interval: 2}, + opts: recipeServiceOptions{Name: "vllm-cluster.service", Profile: "x", User: "serve", Timeout: 60, Start: true, Watchdog: true, Interval: 2}, enabled: []string{"vllm-cluster.service", "vllm-cluster-health.timer"}, }, } @@ -133,7 +132,7 @@ func TestInstallRecipeServiceSurvivesAFailedSupersede(t *testing.T) { } fake := &fakeRecipeService{failStop: true} err := installRecipeService(recipeServiceOptions{ - Name: "vllm-cluster.service", User: "serve", Timeout: 60, + Name: "vllm-cluster.service", Profile: "x", User: "serve", Timeout: 60, Supersede: true, Native: "vllm.service", Service: fake, }) if err != nil { @@ -160,7 +159,7 @@ func TestInstallRecipeServiceHintStartsTheWatchdogToo(t *testing.T) { } { out, err := captureStdout(t, func() error { return installRecipeService(recipeServiceOptions{ - Name: "vllm-cluster.service", User: "serve", Timeout: 60, + Name: "vllm-cluster.service", Profile: "x", User: "serve", Timeout: 60, Watchdog: c.watchdog, Interval: 2, Service: &fakeRecipeService{}, }) }) diff --git a/internal/cli/root.go b/internal/cli/root.go index b65a446..d81dea8 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -3,7 +3,9 @@ package cli import ( "bufio" + "errors" "fmt" + "io/fs" "os" "os/exec" "path/filepath" @@ -52,9 +54,8 @@ func newRootCmd() *cobra.Command { Use: cmdName, Short: "Manage an inference server on one GPU machine or several", Long: cmdName + " (formerly vllm-helper) -- serve large models with vLLM, mlx-lm or llama.cpp\n" + - "on one GPU machine or several (models, switching, sharding, replica pools,\n" + - "recipes, optional Coder Agents sync). Per machine, only one native model is\n" + - "loaded at a time.", + "on one GPU machine or several (models, several at once if memory allows,\n" + + "sharding, replica pools, recipes, optional Coder Agents sync).", SilenceUsage: true, SilenceErrors: true, // describeBuild, not Version: on the dev channel every build shares @@ -84,8 +85,7 @@ func newRootCmd() *cobra.Command { recipe.UseLocal(filepath.Join(config.ResolveDirs().Config, recipe.LocalDir)) if cfg, err := loadConfig(); err == nil { useMetalProbe(cfg.VLLMBin()) - // The active profile's environment, not only config.env's: a - // profile can set CUDA_VISIBLE_DEVICES, and sizing every GPU + // config.env can set CUDA_VISIBLE_DEVICES, and sizing every GPU // for a server allowed one of them overstates the budget. control.UseServerGPUs(cfg) } @@ -102,6 +102,9 @@ func newRootCmd() *cobra.Command { root.AddCommand( newListCmd(), + newUpCmd(), + newDownCmd(), + newDefaultCmd(), newSwitchCmd(), newNewCmd(), newRecipesCmd(), @@ -116,15 +119,16 @@ func newRootCmd() *cobra.Command { newStatusCmd(), newRestartCmd(), - newStopCmd(), newLogsCmd(), newServeCmd(), + newRouterCmd(), newSetupCmd(), newDoctorCmd(), newHardwareCmd(), newMetalLimitCmd(), newAdoptCmd(), + newMigrateCmd(), newUpgradeCmd(), newStartCmd(), @@ -158,11 +162,19 @@ func Run(args []string) int { root.SetArgs(args) if err := root.Execute(); err != nil { fmt.Fprintln(os.Stderr, cmdName+":", err) - return 1 + return exitCode(err) } return 0 } +func exitCode(err error) int { + var ee *exitError + if errors.As(err, &ee) { + return ee.code + } + return 1 +} + // pflag reads a single-dash long flag as a cluster of shorthands, so the // "-json" that stdlib flag used to accept now fails naming a letter nobody // typed. Say what to write instead, but only when the letters actually spell @@ -192,7 +204,7 @@ func loadConfig() (*config.Config, error) { return config.Load() } -// runner is how the CLI drives whatever is actually serving the active model: +// runner is how the CLI drives whatever is actually serving an instance: // the init-system service, or a recipe's container cluster. Label names which // one, so a report about a two-node cluster does not say "systemd". // @@ -211,77 +223,37 @@ type runner interface { Label() string } -// svc is the runner for the active model: the cluster when the active profile -// is owned by a recipe, otherwise the init system. Picking on the profile is -// the whole point -- status and logs used to report systemd regardless. -func svc(cfg *config.Config) runner { - if c := activeClusterRunner(cfg); c != nil { - return c - } - return systemdService(cfg) -} +// errNoInstances is what a command that acts on a running model says when +// nothing runs. +var errNoInstances = errors.New("no model is running; start one with '" + cmdName + " up '") -// runnerIdentity names what a runner holds the GPUs and port with, so a switch -// can tell when it is about to leave one runner for another. -func runnerIdentity(r runner) string { - if c, ok := r.(*clusterRunner); ok { - return c.identity() - } - if s, ok := r.(service.Service); ok { - return "native:" + s.Name +// instanceService is the init-system unit that serves one instance. +func instanceService(cfg *config.Config, name string) service.Service { + return service.Service{ + Name: service.InstanceName(name), + Scope: cfg.ServiceScope(), + // Without a terminal, sudo must fail fast instead of prompting + // for a password nothing can type. + NonInteractive: !stdinIsTTY(), } - return r.Label() } -// stopOutgoing stops the runner a switch is leaving when the one taking over -// is a different runner. The incoming runner only replaces its own -// containers or unit, so without this a recipe kept the GPUs and the API port -// after a switch to a native profile, and the reverse left vLLM running. -// -// A cluster is always told to stop, even with its head down, because -// headless workers can outlive the head and still hold GPU memory. A native -// unit that is not running is left alone, which also keeps platforms without -// a service manager working. -func stopOutgoing(outgoing, incoming runner) error { - if runnerIdentity(outgoing) == runnerIdentity(incoming) { - return nil - } - c, cluster := outgoing.(*clusterRunner) - if cluster && c.needNodes() != nil { - // A profile from before the topology was recorded: there is no node - // list to act on, and refusing would trap the user on it. - fmt.Fprintf(os.Stderr, "warning: the previous profile does not record its nodes, so any %s containers are still running; stop them with 'llm-helper recipes stop %s --nodes '\n", c.recipe.ID, c.recipe.ID) - return nil - } - state := "" - if !cluster { - state = outgoing.IsActive() - } - if !outgoingNeedsStop(runnerIdentity(outgoing), runnerIdentity(incoming), cluster, state) { - return nil - } - fmt.Printf("Stopping the %s the previous profile ran on...\n", outgoing.Label()) - if err := outgoing.Stop(); err != nil { - return fmt.Errorf("stopping the outgoing %s, which would keep the GPUs and port: %w", outgoing.Label(), err) - } - return nil +// legacyService is the single unit used before instances existed. +func legacyService(cfg *config.Config) service.Service { + return service.Service{Name: cfg.ServiceName(), Scope: cfg.ServiceScope(), NonInteractive: !stdinIsTTY()} } -// outgoingNeedsStop is stopOutgoing's decision, kept pure for testing. -func outgoingNeedsStop(outgoingID, incomingID string, outgoingIsCluster bool, outgoingState string) bool { - if outgoingID == incomingID { - return false +// instanceRunner drives one instance: its recipe's cluster when a recipe owns +// the profile, otherwise its unit. +func instanceRunner(cfg *config.Config, name string) runner { + if c := clusterRunnerForName(cfg, name); c != nil { + return c } - return outgoingIsCluster || outgoingState == "active" || outgoingState == "activating" + return instanceService(cfg, name) } -// activeClusterRunner returns nil when the active model is not a recipe, which -// is the signal to fall back rather than an error worth surfacing here. -func activeClusterRunner(cfg *config.Config) *clusterRunner { - name := cfg.ActiveProfile() - if name == "" { - return nil - } +// clusterRunnerForName is nil when the profile is not a recipe's. +func clusterRunnerForName(cfg *config.Config, name string) *clusterRunner { p, err := profile.Load(cfg.Dirs, name) if err != nil || !p.Generated() { return nil @@ -293,16 +265,31 @@ func activeClusterRunner(cfg *config.Config) *clusterRunner { return c } -// systemdService is the init-system runner. Setup needs the concrete type -// because installing a unit is not part of the runner contract. -func systemdService(cfg *config.Config) service.Service { - return service.Service{ - Name: cfg.ServiceName(), - Scope: cfg.ServiceScope(), - // Without a terminal, sudo must fail fast instead of prompting - // for a password nothing can type. - NonInteractive: !stdinIsTTY(), +// pickInstance resolves the instance a command acts on: the one named, or the +// only one running. +func pickInstance(cfg *config.Config, args []string) (*control.Instance, error) { + if len(args) > 0 && args[0] != "" { + inst, err := control.LoadInstance(cfg, args[0]) + if errors.Is(err, fs.ErrNotExist) { + return nil, fmt.Errorf("'%s' is not running; start it with '%s up %s'", args[0], cmdName, args[0]) + } + return inst, err + } + list, err := control.ListInstances(cfg) + if err != nil { + return nil, err + } + switch len(list) { + case 0: + return nil, errNoInstances + case 1: + return list[0], nil + } + names := make([]string, len(list)) + for i, inst := range list { + names[i] = inst.Name } + return nil, fmt.Errorf("%d models are running (%s); name one", len(list), strings.Join(names, ", ")) } // ask prompts for a line of input with a default. Without a terminal it diff --git a/internal/cli/root_internal_test.go b/internal/cli/root_internal_test.go index 0800fa7..bb50033 100644 --- a/internal/cli/root_internal_test.go +++ b/internal/cli/root_internal_test.go @@ -13,9 +13,9 @@ func TestTheCommandSetIsPinned(t *testing.T) { // help is not here: cobra attaches it during Execute, not registration. // completion is not here either, because the root disables it. want := []string{ - "adopt", "coder-sync", "doctor", "edit", "engines", "hardware", "inspect", "install-gguf", "list", "logs", "metal-limit", - "model-types", "new", "node", "pool", "recipes", "remove", "restart", "search", "serve", "set", "setup", "shard", "spark", "start", - "status", "stop", "switch", "upgrade", "version", + "adopt", "coder-sync", "default", "doctor", "down", "edit", "engines", "hardware", "inspect", "install-gguf", "list", "logs", + "metal-limit", "migrate", "model-types", "new", "node", "pool", "recipes", "remove", "restart", "router", "search", "serve", "set", + "setup", "shard", "spark", "start", "status", "switch", "up", "upgrade", "version", } var got []string @@ -29,14 +29,14 @@ func TestTheCommandSetIsPinned(t *testing.T) { } } -// start is the service manager's ExecStart. It execs vLLM over the calling -// process, so listing it in help would invite someone to run it by hand. +// start is the service manager's ExecStart and switch is deprecated, so +// neither belongs in help. func TestStartIsHiddenAndTheRestAreGrouped(t *testing.T) { for _, c := range newRootCmd().Commands() { switch c.Name() { - case "start": + case "start", "switch": if !c.Hidden { - t.Error("start should stay out of the help listing") + t.Errorf("%s should stay out of the help listing", c.Name()) } case "version": // Sits with help under Additional Commands, not with the @@ -57,7 +57,8 @@ func TestStartIsHiddenAndTheRestAreGrouped(t *testing.T) { // missing operand reads as a typo in the help. func TestUsageLinesNameTheirOperands(t *testing.T) { wantOperands := map[string]string{ - "switch": "", + "up": "", + "default": "[profile]", "new": "", "search": "", "inspect": "", diff --git a/internal/cli/router.go b/internal/cli/router.go new file mode 100644 index 0000000..6eee614 --- /dev/null +++ b/internal/cli/router.go @@ -0,0 +1,219 @@ +package cli + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/spf13/cobra" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/control" + "github.com/coder/llm-helper/internal/router" + "github.com/coder/llm-helper/internal/service" +) + +const defaultRouterPort = "8000" + +func newRouterCmd() *cobra.Command { + c := &cobra.Command{ + Use: "router", + Short: "Serve every running model behind one port", + Long: "The router answers on one port and sends each request to the instance serving the model it names. Coder then offers every running model, not just one.", + GroupID: groupServer, + } + c.AddCommand(newRouterServeCmd(), newRouterInstallCmd(), newRouterUninstallCmd()) + return c +} + +// routerPort is the port the router uses: the flag, the installed one, or +// the default. +func routerPort(cfg *config.Config, flag string) string { + if flag != "" { + return flag + } + if p := cfg.RouterPort(); p != "" { + return p + } + return defaultRouterPort +} + +// routerBind keeps the router on loopback unless the instances check a key, +// the same rule the engines follow. +func routerBind(cfg *config.Config, port string) string { + if cfg.APIKey() != "" { + return net.JoinHostPort("", port) + } + return net.JoinHostPort("127.0.0.1", port) +} + +func instanceBackends(cfg *config.Config) func() []string { + return func() []string { + list, err := control.ListInstances(cfg) + if err != nil { + return nil + } + out := make([]string, 0, len(list)) + for _, inst := range list { + if inst.Port != "" { + out = append(out, "http://localhost:"+inst.Port) + } + } + return out + } +} + +func newRouterServeCmd() *cobra.Command { + var port, bind string + c := &cobra.Command{ + Use: "serve [--port N]", + Short: "Run the router in the foreground", + Args: cobra.NoArgs, + RunE: func(*cobra.Command, []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + addr := bind + if addr == "" { + addr = routerBind(cfg, routerPort(cfg, port)) + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + models := router.NewModels(instanceBackends(cfg)) + models.StartRefresh(ctx, 5*time.Second) + + mux := http.NewServeMux() + mux.Handle("/v1/", models) + mux.HandleFunc("/router/status", func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(models.Routes()) + }) + + ln, err := net.Listen("tcp", addr) + if err != nil { + return fmt.Errorf("listening on %s: %w", addr, err) + } + fmt.Printf("Router listening on http://%s/v1\n", ln.Addr()) + + srv := &http.Server{Handler: mux, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 2 * time.Minute} + errs := make(chan error, 1) + go func() { + if err := srv.Serve(ln); err != nil && err != http.ErrServerClosed { + errs <- err + } + }() + stop := make(chan os.Signal, 1) + signal.Notify(stop, os.Interrupt, syscall.SIGTERM) + select { + case err := <-errs: + return err + case <-stop: + _ = srv.Close() + return nil + } + }, + } + c.Flags().StringVar(&port, "port", "", "port to listen on (default: ROUTER_PORT, else "+defaultRouterPort+")") + c.Flags().StringVar(&bind, "bind", "", "address to listen on, overriding --port (default: loopback, or every interface when VLLM_API_KEY is set)") + return c +} + +type routerService interface { + Install(self string) (string, error) + Restart() error + Uninstall() error +} + +func routerUnit(cfg *config.Config, port string) service.Service { + return service.Service{ + Name: service.RouterName, + Scope: cfg.ServiceScope(), + NonInteractive: !stdinIsTTY(), + Args: []string{"router", "serve", "--port", port}, + } +} + +// installRouter checks port, installs and restarts svc, and records the port +// so integrations point at the router. +func installRouter(cfg *config.Config, port, self string, svc routerService) error { + if err := control.CheckRouterPort(cfg, port); err != nil { + return err + } + path, err := svc.Install(self) + if err != nil { + return err + } + if err := svc.Restart(); err != nil { + return err + } + cfg.File.Set("ROUTER_PORT", port) + if err := cfg.File.Write(cfg.Dirs.ConfigFile(), 0o644); err != nil { + return err + } + fmt.Printf("Installed %s\n", path) + fmt.Printf("Router: http://localhost:%s/v1\n", port) + fmt.Println("Point your Coder AI provider's base URL at this port.") + return nil +} + +func newRouterInstallCmd() *cobra.Command { + var port string + c := &cobra.Command{ + Use: "install-service [--port N]", + Short: "Run the router as a service, so Coder can offer every running model", + Args: cobra.NoArgs, + RunE: func(*cobra.Command, []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + self, err := os.Executable() + if err != nil { + return err + } + p := routerPort(cfg, port) + if err := installRouter(cfg, p, self, routerUnit(cfg, p)); err != nil { + return err + } + control.NotifyDefault(context.Background(), cfg, os.Stderr) + return nil + }, + } + c.Flags().StringVar(&port, "port", "", "port to listen on (default: ROUTER_PORT, else "+defaultRouterPort+")") + return c +} + +func uninstallRouter(cfg *config.Config, svc routerService) error { + if err := svc.Uninstall(); err != nil { + return err + } + cfg.File.Delete("ROUTER_PORT") + return cfg.File.Write(cfg.Dirs.ConfigFile(), 0o644) +} + +func newRouterUninstallCmd() *cobra.Command { + return &cobra.Command{ + Use: "uninstall-service", + Short: "Remove the router service; Coder then offers only the model on PORT", + Args: cobra.NoArgs, + RunE: func(*cobra.Command, []string) error { + cfg, err := loadConfig() + if err != nil { + return err + } + if err := uninstallRouter(cfg, routerUnit(cfg, routerPort(cfg, ""))); err != nil { + return err + } + fmt.Println("Router removed.") + control.NotifyDefault(context.Background(), cfg, os.Stderr) + return nil + }, + } +} diff --git a/internal/cli/router_internal_test.go b/internal/cli/router_internal_test.go new file mode 100644 index 0000000..0a9cc65 --- /dev/null +++ b/internal/cli/router_internal_test.go @@ -0,0 +1,121 @@ +package cli + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/service" +) + +type fakeRouterService struct { + installed, restarted, removed bool + err error +} + +func (f *fakeRouterService) Install(string) (string, error) { + f.installed = true + return "/unit", f.err +} +func (f *fakeRouterService) Restart() error { f.restarted = true; return nil } +func (f *fakeRouterService) Uninstall() error { f.removed = true; return nil } + +func routerTestConfig(t *testing.T, conf string) *config.Config { + t.Helper() + dirs := newTree(t) + writeConfig(t, dirs, conf) + cfg, err := config.Load() + if err != nil { + t.Fatal(err) + } + return cfg +} + +func TestRouterPort(t *testing.T) { + cfg := routerTestConfig(t, "PORT=8001\n") + if got := routerPort(cfg, ""); got != defaultRouterPort { + t.Errorf("unset: %q", got) + } + cfg.File.Set("ROUTER_PORT", "9000") + if got := routerPort(cfg, ""); got != "9000" { + t.Errorf("installed: %q", got) + } + if got := routerPort(cfg, "9100"); got != "9100" { + t.Errorf("flag: %q", got) + } +} + +func TestRouterBind(t *testing.T) { + cfg := routerTestConfig(t, "PORT=8001\n") + if got := routerBind(cfg, "8000"); got != "127.0.0.1:8000" { + t.Errorf("no key: %q", got) + } + cfg.File.Set("VLLM_API_KEY", "k") + if got := routerBind(cfg, "8000"); got != ":8000" { + t.Errorf("with key: %q", got) + } +} + +func TestInstallRouterRecordsPort(t *testing.T) { + cfg := routerTestConfig(t, "PORT=8001\n") + svc := &fakeRouterService{} + if err := installRouter(cfg, "18765", "/bin/llm-helper", svc); err != nil { + t.Fatal(err) + } + if !svc.installed || !svc.restarted { + t.Errorf("service = %+v", svc) + } + reloaded, err := config.Load() + if err != nil { + t.Fatal(err) + } + if got := reloaded.RouterPort(); got != "18765" { + t.Errorf("ROUTER_PORT = %q", got) + } + + if err := uninstallRouter(reloaded, svc); err != nil { + t.Fatal(err) + } + reloaded, _ = config.Load() + if !svc.removed || reloaded.RouterPort() != "" { + t.Errorf("after uninstall: removed=%v ROUTER_PORT=%q", svc.removed, reloaded.RouterPort()) + } +} + +func TestInstallRouterRefusesBadPortOrFailure(t *testing.T) { + cfg := routerTestConfig(t, "PORT=8001\n") + svc := &fakeRouterService{} + if err := installRouter(cfg, "nope", "/bin/llm-helper", svc); err == nil || svc.installed { + t.Errorf("bad port: err %v, installed %v", err, svc.installed) + } + svc = &fakeRouterService{err: errors.New("boom")} + if err := installRouter(cfg, "18766", "/bin/llm-helper", svc); err == nil { + t.Fatal("want the install error") + } + if cfg.RouterPort() != "" { + t.Errorf("ROUTER_PORT recorded after a failed install: %q", cfg.RouterPort()) + } +} + +func TestExitCode(t *testing.T) { + if got := exitCode(errors.New("plain")); got != 1 { + t.Errorf("plain error: %d", got) + } + wrapped := fmt.Errorf("start: %w", &exitError{code: service.ExitNoFit, err: errors.New("no fit")}) + if got := exitCode(wrapped); got != service.ExitNoFit { + t.Errorf("exitError: %d, want %d", got, service.ExitNoFit) + } +} + +func TestRecipeServiceProfileRefusesAPlainProfile(t *testing.T) { + cfg := writeProfile(t, "MODEL=org/m\n") + if _, err := recipeServiceProfile(cfg, "deepseek-v4-flash-2spark"); err == nil || !strings.Contains(err.Error(), "not a recipe profile") { + t.Errorf("plain profile: %v", err) + } + cfg.Dirs.State = t.TempDir() + if _, err := recipeServiceProfile(cfg, ""); err == nil || !strings.Contains(err.Error(), "no recipe profile is running") { + t.Errorf("nothing running: %v", err) + } +} diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 8db57d3..afef112 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -85,6 +85,18 @@ func closedPort(t *testing.T) string { // newTree gives the CLI a config tree of its own and one saved profile. The // network is closed off for the whole package in TestMain. +// writeInstanceEnv records a running instance the way 'up' does. +func writeInstanceEnv(t *testing.T, name, content string) { + t.Helper() + dir := filepath.Join(os.Getenv("VLLM_HELPER_STATE_DIR"), "instances") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, name+".env"), []byte(content), 0o600); err != nil { + t.Fatal(err) + } +} + func newTree(t *testing.T) (configDir, profilePath string) { t.Helper() root := t.TempDir() @@ -294,18 +306,18 @@ func TestRunStatusJSON(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})) t.Cleanup(srv.Close) - configDir, _ := newTree(t) - port := portOf(t, srv.URL) - config := "PORT=" + port + "\nSERVICE_NAME=vllm-test\nSERVICE_SCOPE=user\n" - if err := os.WriteFile(filepath.Join(configDir, "config.env"), []byte(config), 0o600); err != nil { - t.Fatal(err) + newTree(t) + if _, stdout, _ := runCLI(t, "status", "--json"); strings.TrimSpace(stdout) != "[]" { + t.Errorf("status --json with nothing running = %q, want []", stdout) } + port := portOf(t, srv.URL) + writeInstanceEnv(t, "qwen", "MODEL=Qwen/Qwen3-8B-FP8\nPORT="+port+"\n") code, stdout, stderr := runCLI(t, "status", "--json") if code != 0 { t.Fatalf("exited %d (%s), want 0", code, stderr) } - var got struct { + var list []struct { Service string `json:"service"` Profile string `json:"profile"` Model string `json:"model"` @@ -313,9 +325,10 @@ func TestRunStatusJSON(t *testing.T) { Healthy bool `json:"endpoint_healthy"` HTTPStatus int `json:"http_status"` } - if err := json.Unmarshal([]byte(stdout), &got); err != nil { - t.Fatalf("status --json isn't JSON: %v\n%s", err, stdout) + if err := json.Unmarshal([]byte(stdout), &list); err != nil || len(list) != 1 { + t.Fatalf("status --json isn't a list of one: %v\n%s", err, stdout) } + got := list[0] if got.Port != port { t.Errorf("port = %q, want %q", got.Port, port) } @@ -325,8 +338,8 @@ func TestRunStatusJSON(t *testing.T) { if got.Service == "" { t.Error("service should always report a state") } - if got.Profile != "unknown" || got.Model != "unknown" { - t.Errorf("profile/model = %q/%q, want unknown before the first switch", got.Profile, got.Model) + if got.Profile != "qwen" || got.Model != "Qwen/Qwen3-8B-FP8" { + t.Errorf("profile/model = %q/%q, want qwen/Qwen/Qwen3-8B-FP8", got.Profile, got.Model) } } @@ -334,21 +347,11 @@ func TestRunListAndStatusShowThinking(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {})) t.Cleanup(srv.Close) - configDir, profilePath := newTree(t) - config := "PORT=" + portOf(t, srv.URL) + "\nSERVICE_NAME=vllm-test\nSERVICE_SCOPE=user\n" - if err := os.WriteFile(filepath.Join(configDir, "config.env"), []byte(config), 0o600); err != nil { - t.Fatal(err) - } + _, profilePath := newTree(t) if err := os.WriteFile(profilePath, []byte(sampleProfile+"THINKING=on\n"), 0o644); err != nil { t.Fatal(err) } - state := os.Getenv("VLLM_HELPER_STATE_DIR") - if err := os.MkdirAll(state, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(state, "active-profile"), []byte("qwen\n"), 0o600); err != nil { - t.Fatal(err) - } + writeInstanceEnv(t, "qwen", sampleProfile+"THINKING=on\nPORT="+portOf(t, srv.URL)+"\n") if _, stdout, _ := runCLI(t, "list"); !strings.Contains(stdout, "[thinking]") { t.Errorf("list = %q, want the profile marked [thinking]", stdout) @@ -364,7 +367,7 @@ func TestRunListAndStatusShowThinking(t *testing.T) { if _, stdout, _ := runCLI(t, "status"); !strings.Contains(stdout, "thinking: on") { t.Errorf("status = %q, want a thinking line", stdout) } - _, stdout, _ = runCLI(t, "status", "--json") + _, stdout, _ = runCLI(t, "status", "qwen", "--json") var status struct { Thinking string `json:"thinking"` } @@ -547,36 +550,35 @@ func TestRunSearchAndInspectNeedAnArgument(t *testing.T) { } } -// start is what the service manager runs. Before the first switch there is no -// active model, and exiting non-zero here would send systemd's -// Restart=on-failure -- or launchd's KeepAlive -- into a crash loop serving -// nothing. -func TestRunStartWithoutAnActiveModel(t *testing.T) { +// start is what the service manager runs. For an instance that was taken +// down, exiting non-zero would send systemd's Restart=on-failure -- or +// launchd's KeepAlive -- into a crash loop serving nothing. +func TestRunStartWithoutTheInstance(t *testing.T) { newTree(t) - code, stdout, stderr := runCLI(t, "start") + code, stdout, stderr := runCLI(t, "start", "--instance", "qwen") if code != 0 { t.Fatalf("exited %d (%s), want 0 so the service manager doesn't restart it", code, stderr) } - if !strings.Contains(stdout, "switch") { - t.Errorf("stdout = %q, want it to point at 'llm-helper switch'", stdout) + if !strings.Contains(stdout, "up qwen") { + t.Errorf("stdout = %q, want it to point at 'llm-helper up qwen'", stdout) } } -// Only a missing active.env means "not set up yet". A file that exists and +// Only a missing instance file means "not set up". A file that exists and // can't be read is a fault, and swallowing it would hide the breakage behind // the same quiet exit. -func TestRunStartReportsAnUnreadableActiveModel(t *testing.T) { +func TestRunStartReportsAnUnreadableInstance(t *testing.T) { newTree(t) state := os.Getenv("VLLM_HELPER_STATE_DIR") // A directory in its place fails to read for every user, including root. - if err := os.MkdirAll(filepath.Join(state, "active.env"), 0o755); err != nil { + if err := os.MkdirAll(filepath.Join(state, "instances", "qwen.env"), 0o755); err != nil { t.Fatal(err) } - code, _, stderr := runCLI(t, "start") + code, _, stderr := runCLI(t, "start", "--instance", "qwen") if code == 0 { - t.Fatal("exited 0, want non-zero when the active model config can't be read") + t.Fatal("exited 0, want non-zero when the instance file can't be read") } - if !strings.Contains(stderr, "active.env") { + if !strings.Contains(stderr, "qwen.env") { t.Errorf("stderr = %q, want it to name the file it couldn't read", stderr) } } @@ -584,12 +586,34 @@ func TestRunStartReportsAnUnreadableActiveModel(t *testing.T) { // A provider this build doesn't know is a fault to report, never a reason to // fall back to vLLM and serve the profile with the wrong server. func TestRunStartRefusesAnUnknownProvider(t *testing.T) { + newTree(t) + writeInstanceEnv(t, "qwen", "PROVIDER=nope\nMODEL=m\nPORT=1\n") + code, _, stderr := runCLI(t, "start", "--instance", "qwen") + if code == 0 || !strings.Contains(stderr, "nope") { + t.Fatalf("exited %d, stderr %q; want a failure naming the provider", code, stderr) + } +} + +// A pre-upgrade unit runs start with no --instance; it must not crash-loop +// before migrate has been run. +func TestRunLegacyStartWithNothingToServe(t *testing.T) { + newTree(t) + code, stdout, stderr := runCLI(t, "start") + if code != 0 { + t.Fatalf("exited %d (%s), want 0", code, stderr) + } + if !strings.Contains(stdout, "migrate") { + t.Errorf("stdout = %q, want it to point at migrate", stdout) + } +} + +func TestRunLegacyStartRefusesAnUnknownProvider(t *testing.T) { newTree(t) state := os.Getenv("VLLM_HELPER_STATE_DIR") if err := os.MkdirAll(state, 0o755); err != nil { t.Fatal(err) } - if err := os.WriteFile(filepath.Join(state, "active.env"), []byte("PROVIDER=nope\nMODEL=m\nPORT=1\n"), 0o600); err != nil { + if err := os.WriteFile(filepath.Join(state, "active.env"), []byte("PROVIDER=nope\nMODEL=m\nPORT=1\n"), 0o644); err != nil { t.Fatal(err) } code, _, stderr := runCLI(t, "start") @@ -650,10 +674,12 @@ func TestRunRejectsExtraOperands(t *testing.T) { {"edit", "qwen", "typo"}, {"inspect", "org/model", "typo"}, {"coder-sync", "register", "qwen", "typo"}, - {"status", "typo"}, + {"status", "qwen", "typo"}, {"list", "typo"}, - {"stop", "typo"}, - {"logs", "typo"}, + {"down", "qwen", "typo"}, + {"stop", "qwen", "typo"}, + {"logs", "qwen", "typo"}, + {"up", "qwen", "typo"}, {"version", "typo"}, } for _, args := range tests { @@ -666,6 +692,7 @@ func TestRunRejectsExtraOperands(t *testing.T) { // Commands with an operand of their own report a count; // commands with none read the extra word as a subcommand. if !strings.Contains(stderr, "accepts at most") && + !strings.Contains(stderr, "accepts 1 arg") && !strings.Contains(stderr, `unknown command "typo"`) { t.Errorf("stderr = %q, want the operand rejected by name or count", stderr) } @@ -711,8 +738,7 @@ func TestRunRejectsSingleDashLongFlags(t *testing.T) { } } -// stop is safe to run against a service that was never installed, and says -// so rather than failing. +// A bare stop with nothing running says so rather than failing. func TestRunStopWhenNothingIsRunning(t *testing.T) { newTree(t) code, stdout, stderr := runCLI(t, "stop") @@ -724,17 +750,68 @@ func TestRunStopWhenNothingIsRunning(t *testing.T) { } } -func TestRunRestartWithoutAnActiveProfile(t *testing.T) { +func TestRunRestartWithNothingRunning(t *testing.T) { newTree(t) code, _, stderr := runCLI(t, "restart") if code == 0 { t.Error("exited 0, want non-zero") } - if !strings.Contains(stderr, "no active profile") { + if !strings.Contains(stderr, "no model is running") { t.Errorf("stderr = %q, want it to explain there's nothing to restart", stderr) } } +func TestRunUpRefusesBadSettings(t *testing.T) { + newTree(t) + code, _, stderr := runCLI(t, "up", "qwen", "--mem-util", "2") + if code == 0 || !strings.Contains(stderr, "--mem-util") { + t.Errorf("up --mem-util 2 exited %d, stderr %q; want a refusal naming the flag", code, stderr) + } + code, _, stderr = runCLI(t, "up", "qwen", "--mem-util", "0.4", "--kv-cache", "20G") + if code == 0 { + t.Errorf("up with both --mem-util and --kv-cache exited 0 (%s)", stderr) + } +} + +func TestRunUpAsksForMigrateFirst(t *testing.T) { + newTree(t) + state := os.Getenv("VLLM_HELPER_STATE_DIR") + if err := os.MkdirAll(state, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(state, "active.env"), []byte("MODEL=x\n"), 0o600); err != nil { + t.Fatal(err) + } + code, _, stderr := runCLI(t, "up", "qwen") + if code == 0 || !strings.Contains(stderr, "migrate") { + t.Errorf("exited %d, stderr %q; want it to ask for migrate", code, stderr) + } +} + +func TestRunDefault(t *testing.T) { + configDir, _ := newTree(t) + code, _, stderr := runCLI(t, "default", "qwen") + if code == 0 || !strings.Contains(stderr, "is not running") { + t.Errorf("default on a stopped profile exited %d, stderr %q", code, stderr) + } + + writeInstanceEnv(t, "qwen", "MODEL=Qwen/Qwen3-8B\nPORT=8001\n") + code, stdout, stderr := runCLI(t, "default", "qwen") + if code != 0 { + t.Fatalf("exited %d (%s), want 0", code, stderr) + } + if !strings.Contains(stdout, "qwen is now the default") { + t.Errorf("stdout = %q", stdout) + } + b, err := os.ReadFile(filepath.Join(configDir, "config.env")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), "DEFAULT_PROFILE=qwen") { + t.Errorf("config.env = %q, want DEFAULT_PROFILE=qwen", b) + } +} + func TestRunDoctor(t *testing.T) { newTree(t) code, stdout, stderr := runCLI(t, "doctor") @@ -923,17 +1000,15 @@ esac t.Setenv("PATH", dir+string(os.PathListSeparator)+os.Getenv("PATH")) } -// A shard serves the profile it is given, not the active one, and the head -// runs it on whichever GPU that profile names. Checking the active profile's -// GPU let a flag that aborts on the head's own GB10 through, and checking the -// head only when its command was built let every node be stopped first. +// A shard serves the profile it is given, and the head runs it on whichever +// GPU that profile names, checked before any node is stopped. func TestRunShardStartChecksTheRequestedProfilesGPUFirst(t *testing.T) { configDir, _ := newTree(t) fakeNVIDIASMI(t) t.Setenv("CUDA_VISIBLE_DEVICES", "0") state := filepath.Join(filepath.Dir(configDir), "state") files := map[string]string{ - filepath.Join(state, "active.env"): "MODEL=Qwen/Qwen3-8B-FP8\nCUDA_VISIBLE_DEVICES=0\n", + filepath.Join(state, "instances", "qwen.env"): "MODEL=Qwen/Qwen3-8B-FP8\nCUDA_VISIBLE_DEVICES=0\n", filepath.Join(configDir, "models", "big.env"): "MODEL=org/big\nCUDA_VISIBLE_DEVICES=1\nEXTRA_ARGS=\"--moe-backend deep_gemm_mega_moe\"\n", filepath.Join(configDir, "nodes.json"): `[{"name":"head","host":"http://127.0.0.1:1","labels":{},"caps":{}},{"name":"worker","host":"http://127.0.0.1:1","ssh":"me@worker","labels":{},"caps":{}}]`, } diff --git a/internal/cli/search.go b/internal/cli/search.go index 6af61c8..9f3813e 100644 --- a/internal/cli/search.go +++ b/internal/cli/search.go @@ -25,7 +25,7 @@ func newSearchCmd() *cobra.Command { var jsonOut bool c := &cobra.Command{ Use: "search ", - Short: "Search HuggingFace and pick a model to pull and switch to", + Short: "Search HuggingFace and pick a model to pull and run", GroupID: groupModels, RunE: func(cmd *cobra.Command, args []string) error { return runSearch(cmd.Context(), args, jsonOut) }, } @@ -106,8 +106,8 @@ func runSearch(ctx context.Context, args []string, jsonOut bool) error { if choice != nil { label += " (" + choice.Variant.Label + ")" } - if confirm(fmt.Sprintf("Pull %s and switch to it?", label), false) { - return pullAndSwitch(ctx, cfg, m.ID, choice, "", watchOpts{follow: stdoutIsTTY(), wait: true}) + if confirm(fmt.Sprintf("Pull %s and run it?", label), false) { + return pullAndUp(ctx, cfg, m.ID, choice, "", watchOpts{follow: stdoutIsTTY(), wait: true}) } fmt.Println() } diff --git a/internal/cli/serve.go b/internal/cli/serve.go index 03dde50..ebf3d82 100644 --- a/internal/cli/serve.go +++ b/internal/cli/serve.go @@ -21,14 +21,14 @@ func newServeCmd() *cobra.Command { Use: "serve", Short: "Run a web panel for this machine", GroupID: groupServer, - Long: "Serve a control panel: what model is loaded, whether it is healthy,\n" + - "and what else could be loaded instead.\n\n" + - "It listens on localhost unless told otherwise. The panel can restart the\n" + - "server and change which model runs, so every request needs the token\n" + - "printed below -- open the URL it gives you.\n\n" + - "The panel drives the native service (systemd or launchd) only. While a\n" + - "recipe container cluster serves, it shows the cluster's status and logs\n" + - "but leaves switch, restart and stop to the command line.", + Long: "Serve a control panel: which models are running, whether they are\n" + + "healthy, and what else could be run.\n\n" + + "It listens on localhost unless told otherwise. The panel can start, stop\n" + + "and restart models, so every request needs the token printed below --\n" + + "open the URL it gives you.\n\n" + + "The panel drives the native service (systemd or launchd) only. For a\n" + + "recipe container cluster it shows status and logs but leaves run,\n" + + "restart and stop to the command line.", Example: " llm-helper serve\n" + " llm-helper serve --listen 0.0.0.0:8080", Args: cobra.NoArgs, @@ -48,10 +48,9 @@ func runServe(cmd *cobra.Command, addr string) error { if err != nil { return err } - // The panel reports a recipe cluster instead of the idle native unit. The - // explicit nil keeps a nil *clusterRunner from becoming a non-nil interface. - srv.SetCluster(func() web.Cluster { - if c := activeClusterRunner(cfg); c != nil { + // The explicit nil keeps a nil *clusterRunner from becoming a non-nil interface. + srv.SetCluster(func(name string) web.Cluster { + if c := clusterRunnerForName(cfg, name); c != nil { return c } return nil @@ -88,8 +87,8 @@ func runServe(cmd *cobra.Command, addr string) error { // self-signed certificate teaches people to click through warnings, // and a real one belongs to whatever already terminates it. fmt.Println("Reachable from the network over plain HTTP. Anyone who has that URL, or") - fmt.Println("who can read traffic on the way to it, can switch models and stop the") - fmt.Println("server. Treat it like a password, and put it behind a TLS proxy or a") + fmt.Println("who can read traffic on the way to it, can run and stop models on this") + fmt.Println("machine. Treat it like a password, and put it behind a TLS proxy or a") fmt.Println("private network rather than exposing it directly.") fmt.Println() } diff --git a/internal/cli/service.go b/internal/cli/service.go index ace27f8..6d83b55 100644 --- a/internal/cli/service.go +++ b/internal/cli/service.go @@ -1,7 +1,6 @@ package cli import ( - "errors" "fmt" "github.com/spf13/cobra" @@ -10,81 +9,41 @@ import ( "github.com/coder/llm-helper/internal/profile" ) -func newStopCmd() *cobra.Command { - return &cobra.Command{ - Use: "stop", - Short: "Stop the server, keeping profiles and config", - GroupID: groupServer, - Args: cobra.NoArgs, - RunE: func(*cobra.Command, []string) error { return runStop() }, - } -} - -// runStop stops the vLLM service without touching profiles or config. -func runStop() error { - cfg, err := loadConfig() - if err != nil { - return err - } - // Name the runner, not a unit: a container recipe has no systemd unit, and - // saying "stopped vllm.service" about a two-node cluster is a lie. - label := svc(cfg).Label() - // A cluster is stopped whatever its head reports: headless workers can - // outlive a crashed head and keep holding GPU memory. - _, cluster := svc(cfg).(*clusterRunner) - if state := svc(cfg).IsActive(); !cluster && state != "active" && state != "activating" { - fmt.Printf("the %s is not running (state: %s) -- nothing to stop.\n", label, state) - return nil - } - if err := svc(cfg).Stop(); err != nil { - return fmt.Errorf("stopping the %s: %w", label, err) - } - fmt.Printf("Stopped the %s. Start it again with 'llm-helper restart' or by\n", label) - fmt.Println("switching to a profile.") - if cfg.CoderConfigured() { - fmt.Println("Note: the active model stays enabled in Coder's picker until the next switch.") - } - return nil -} - func newRestartCmd() *cobra.Command { var w waitFlags c := &cobra.Command{ - Use: "restart", - Short: "Re-apply the active profile and restart the server", + Use: "restart [profile]", + Short: "Re-apply a running profile and restart its server", GroupID: groupServer, - Args: cobra.NoArgs, - RunE: func(*cobra.Command, []string) error { return runRestart(w.opts()) }, + Args: cobra.MaximumNArgs(1), + RunE: func(_ *cobra.Command, args []string) error { return runRestart(args, w.opts()) }, } w.register(c) return c } -// runRestart re-applies the active profile (picking up any edits to it or to -// config.env) and restarts it through whichever runner owns it. Unlike switch -// it never changes which profile is active and never touches Coder, which is -// what makes it the right verb for a supervisor: a recovery should bring the -// same model back, not re-publish it. -func runRestart(o watchOpts) error { +// runRestart re-applies a running profile, picking up edits to it or to +// config.env, and restarts it. It never touches Coder, which makes it the +// right verb for a supervisor: a recovery brings the same model back. +func runRestart(args []string, o watchOpts) error { cfg, err := loadConfig() if err != nil { return err } - name := cfg.ActiveProfile() - if name == "" { - return errors.New("no active profile -- use 'llm-helper switch ' first") + inst, err := pickInstance(cfg, args) + if err != nil { + return err } - p, err := profile.Load(cfg.Dirs, name) + p, err := profile.Load(cfg.Dirs, inst.Name) if err != nil { - return fmt.Errorf("the active profile can't be loaded (%w) -- switch to another one", err) + return fmt.Errorf("profile '%s' can't be loaded (%w); stop it with '%s down %s'", inst.Name, err, cmdName, inst.Name) } - // Regenerate active.env so 'edit'/'set' changes take effect. - if err := control.WriteActiveEnv(cfg, p); err != nil { + if err := control.WriteInstanceEnv(cfg, p, inst.Opts()); err != nil { return err } - - fmt.Printf("Reloading profile '%s' (%s)...\n", name, svc(cfg).Label()) - if err := svc(cfg).Restart(); err != nil { + r := instanceRunner(cfg, inst.Name) + fmt.Printf("Reloading profile '%s' (%s)...\n", inst.Name, r.Label()) + if err := r.Restart(); err != nil { return fmt.Errorf("restarting service: %w", err) } if !o.follow && !o.wait { @@ -92,7 +51,7 @@ func runRestart(o watchOpts) error { return nil } fmt.Println() - return watchBoot(cfg, name, o) + return watchBoot(cfg, inst.Name, o) } func newLogsCmd() *cobra.Command { @@ -102,11 +61,13 @@ func newLogsCmd() *cobra.Command { noFollow bool ) c := &cobra.Command{ - Use: "logs", - Short: "Show the server log, following by default on a terminal", + Use: "logs [profile]", + Short: "Show a running model's server log, following by default on a terminal", GroupID: groupServer, - Args: cobra.NoArgs, - RunE: func(*cobra.Command, []string) error { return runLogs(lines, follow, noFollow) }, + Args: cobra.MaximumNArgs(1), + RunE: func(_ *cobra.Command, args []string) error { + return runLogs(args, lines, follow, noFollow) + }, } c.Flags().IntVarP(&lines, "lines", "n", 50, "how many past lines to show") c.Flags().BoolVar(&follow, "follow", false, "keep following even when output is piped") @@ -114,14 +75,18 @@ func newLogsCmd() *cobra.Command { return c } -// runLogs shows the service journal: last -n lines, following by default +// runLogs shows an instance's log: the last -n lines, following by default // on a terminal. Piped output prints and exits so scripts never hang. -func runLogs(lines int, follow, noFollow bool) error { +func runLogs(args []string, lines int, follow, noFollow bool) error { cfg, err := loadConfig() if err != nil { return err } - s := svc(cfg) + inst, err := pickInstance(cfg, args) + if err != nil { + return err + } + s := instanceRunner(cfg, inst.Name) doFollow := stdoutIsTTY() if follow { doFollow = true @@ -137,8 +102,7 @@ func runLogs(lines int, follow, noFollow bool) error { if err != nil { return err } - // Ctrl+C interrupts the log reader too -- that's the normal way to leave, - // not an error worth reporting. + // Ctrl+C is the normal way to leave, not an error. _ = cmd.Wait() return nil } diff --git a/internal/cli/setup.go b/internal/cli/setup.go index 8404f25..7129d4a 100644 --- a/internal/cli/setup.go +++ b/internal/cli/setup.go @@ -7,7 +7,6 @@ import ( "fmt" "os" "os/exec" - "path/filepath" "runtime" "strings" @@ -97,9 +96,10 @@ func runSetup(providerName string) error { } } else { content := fmt.Sprintf(`# llm-helper global config. Sourced (exported) into the server's -# environment on every switch -- values here apply to every profile. +# environment whenever a model starts -- values here apply to every profile. -# Port the OpenAI-compatible endpoint listens on. +# First port a model's OpenAI-compatible endpoint listens on. Each model +# running at the same time takes the next free port from here. PORT=8001 # The inference server new profiles are made for: %s. Each profile @@ -109,9 +109,8 @@ PROVIDER=%s # Full path to the %s executable (usually inside a venv). REQUIRED. %s=%s -# How the %s unit is managed: "system" (needs sudo) or "user". +# How each model's %s unit is managed: "system" (needs sudo) or "user". SERVICE_SCOPE=%s -SERVICE_NAME=%s # Budget for model weights, used by 'llm-helper search' to flag models that # won't fit. Measured from this machine on every run -- currently ~%d GB, being @@ -124,14 +123,14 @@ SERVICE_NAME=%s #KV_CONTEXT_TARGET=%d # --- Optional: Coder Agents sync ------------------------------------------- -# If all three are set (token in secrets.env), every switch keeps a Coder -# deployment's AI-model picker in sync with the active profile. +# If all three are set (token in secrets.env), a Coder deployment's AI-model +# picker is kept in sync with the models running here. #CODER_URL=https://coder.example.com #CODER_AI_PROVIDER_ID= # Only needed if the deployment has more than one organization. #CODER_ORGANIZATION=default `, strings.Join(engine.ProcessNames(), ", "), prov.Name(), prov.BinName(), prov.BinKey(), bin, - config.ServiceManager, config.DefaultServiceScope, config.DefaultServiceName, + config.ServiceManager, config.DefaultServiceScope, hw.BudgetGB, hw.BudgetReason, hw.BudgetGB, detect.DefaultContextTarget, detect.DefaultContextTarget) if err := os.WriteFile(dirs.ConfigFile(), []byte(content), 0o644); err != nil { @@ -194,30 +193,16 @@ VLLM_API_KEY=%s return err } - // Offer adoption before installing a unit -- if a server already runs - // under its own unit, adoption records it and we skip installing ours. + // Offer to adopt a server that is already running. fmt.Println() if err := adoptInto(cfg); err != nil && !errors.Is(err, os.ErrNotExist) { fmt.Fprintln(os.Stderr, "adoption scan failed (continuing):", err) } - // Reload config -- adoption may have changed SERVICE_NAME/SCOPE. - cfg, err = loadConfig() - if err != nil { - return err - } - if systemdService(cfg).Installed() { - fmt.Println() - fmt.Printf("Service unit %s already installed -- setup complete.\n", cfg.ServiceName()) - return nil - } - fmt.Println() - if !confirm(fmt.Sprintf("Install the %s-scope %s unit (%s)?", cfg.ServiceScope(), config.ServiceManager, cfg.ServiceName()), true) { - fmt.Println("Skipped unit install. Re-run 'llm-helper setup' when ready.") - return nil - } - return installUnit(cfg) + fmt.Println("Setup complete. Next: 'llm-helper new ' or 'llm-helper search '.") + fmt.Println("'llm-helper up ' installs the service that runs each model.") + return nil } // fillMissingVLLMBin records a vllm installed since config.env was written @@ -263,22 +248,3 @@ func lookBin(prov engine.Provider) string { } return p } - -// installUnit installs the host's service unit pointing at this binary's -// `start` command. -func installUnit(cfg *config.Config) error { - self, err := os.Executable() - if err != nil { - return err - } - self, _ = filepath.EvalSymlinks(self) - - path, err := systemdService(cfg).Install(self) - if err != nil { - return err - } - fmt.Printf("Installed %s unit at %s\n", cfg.ServiceScope(), path) - fmt.Println() - fmt.Println("Setup complete. Next: 'llm-helper new ' or 'llm-helper search '.") - return nil -} diff --git a/internal/cli/start.go b/internal/cli/start.go index d364095..cd50ac2 100644 --- a/internal/cli/start.go +++ b/internal/cli/start.go @@ -3,75 +3,137 @@ package cli import ( "errors" "fmt" + "io/fs" "os" "strings" "syscall" + "time" "github.com/spf13/cobra" "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/control" "github.com/coder/llm-helper/internal/profile" + "github.com/coder/llm-helper/internal/service" ) +// exitError makes Run exit with code instead of 1. +type exitError struct { + code int + err error +} + +func (e *exitError) Error() string { return e.err.Error() } +func (e *exitError) Unwrap() error { return e.err } + func newStartCmd() *cobra.Command { - return &cobra.Command{ + var name string + c := &cobra.Command{ Use: "start", - Short: "Run the active model in the foreground", - // The service manager's ExecStart, not something to type: it execs - // vLLM over this process and never returns. + Short: "Run one instance's server in the foreground", + // The service manager's ExecStart: it execs the server over this + // process and never returns. Hidden: true, Args: cobra.NoArgs, - RunE: func(*cobra.Command, []string) error { return runStart() }, + RunE: func(*cobra.Command, []string) error { return runStart(name) }, } + c.Flags().StringVar(&name, "instance", "", "the instance (profile) to serve; without it, the pre-instance unit's model") + return c } -// runStart is what the service manager runs: source the generated active.env -// and exec the active profile's server, replacing this process so signals flow -// straight to it. -func runStart() error { +// startOrderTimeout bounds how long an instance waits for older ones at boot. +const startOrderTimeout = 20 * time.Minute + +func runStart(name string) error { cfg, err := loadConfig() if err != nil { return err } - active, err := config.ParseEnvFile(cfg.Dirs.ActiveEnvFile()) + if name == "" { + return runLegacyStart(cfg) + } + inst, err := control.LoadInstance(cfg, name) switch { - case errors.Is(err, os.ErrNotExist): - // Exit cleanly instead of failing. The unit gets installed before the - // first switch, and a non-zero exit here would send systemd's - // Restart=on-failure -- or launchd's KeepAlive -- into a crash loop - // with no model to serve. Only a missing file means "not set up yet"; - // anything else is a fault worth reporting. - fmt.Println("No active model yet -- run 'llm-helper switch ' to load one.") + case errors.Is(err, fs.ErrNotExist): + // A clean exit: a failure would send the service manager into a + // restart loop with nothing to serve. + fmt.Printf("'%s' is not set up to run -- start it with '%s up %s'.\n", name, cmdName, name) return nil case err != nil: - return fmt.Errorf("reading the active model config at %s: %w", cfg.Dirs.ActiveEnvFile(), err) + return fmt.Errorf("reading %s: %w", cfg.Dirs.InstanceEnvFile(name), err) } - // A recipe-generated profile describes a container launch, so serving it - // natively here would start vLLM without the image's flags and quietly - // produce a different server than the one that was verified. - if id := active.Get(profile.RecipeKey); strings.TrimSpace(id) != "" { - return fmt.Errorf("the active profile is generated from recipe %q and runs in a container; start it with 'llm-helper recipes launch %s'", strings.TrimSpace(id), strings.TrimSpace(id)) + if id := strings.TrimSpace(inst.Env.Get(profile.RecipeKey)); id != "" { + return fmt.Errorf("'%s' is generated from recipe %q and runs in a container; start it with '%s recipes launch %s'", name, id, cmdName, id) } - spec, err := serveSpecFromActive(active) + waitForOlder(cfg, inst) + + if inst.Env.Get("INSTANCE_KV_CACHE") == "" { + if err := control.CheckFitFor(cfg, name, "", inst.Env); err != nil { + return &exitError{code: service.ExitNoFit, err: err} + } + } + return execServer(cfg, inst.Env) +} + +// runLegacyStart keeps a unit from before instances serving after an upgrade, +// until 'migrate' replaces it. +func runLegacyStart(cfg *config.Config) error { + env, err := config.ParseEnvFile(cfg.Dirs.LegacyActiveEnvFile()) + switch { + case errors.Is(err, fs.ErrNotExist): + fmt.Printf("Nothing to serve -- run '%s migrate', then '%s up '.\n", cmdName, cmdName) + return nil + case err != nil: + return fmt.Errorf("reading %s: %w", cfg.Dirs.LegacyActiveEnvFile(), err) + } + if id := strings.TrimSpace(env.Get(profile.RecipeKey)); id != "" { + return fmt.Errorf("the profile is generated from recipe %q and runs in a container; start it with '%s recipes launch %s'", id, cmdName, id) + } + fmt.Fprintf(os.Stderr, "This unit predates instances; run '%s migrate' to move its model to one.\n", cmdName) + return execServer(cfg, env) +} + +func execServer(cfg *config.Config, env *config.EnvFile) error { + spec, err := serveSpecFromEnv(env) if err != nil { return err } - // The root command registered config.env's VLLM_BIN, but a profile can - // override it, and the binary exec'd here decides whether the CPU - // backend's free-RAM clamp applies. Only a vllm is worth asking: any - // other server is neither vLLM's backend nor fitted to its flag. + // A profile can override VLLM_BIN, and the binary decides whether the + // CPU backend's free-RAM clamp applies. if spec.provider().Capabilities().VLLMArgs { useMetalProbe(spec.Bin) } - // The same goes for the GPUs it may use: a profile can set - // CUDA_VISIBLE_DEVICES, and the first GPU it leaves is what gets sized. - useServerGPUs(active) + useServerGPUs(env) args, err := spec.args() if err != nil { return err } awaitWiredLimit(cfg) - return syscall.Exec(spec.Bin, args, withProviderEnv(serveEnvFromActive(active), spec)) + return syscall.Exec(spec.Bin, args, withProviderEnv(serveEnvFromInstance(env), spec)) +} + +// waitForOlder lets instances started earlier load first after a reboot, so +// they claim the memory they had before. +func waitForOlder(cfg *config.Config, inst *control.Instance) { + others, err := control.OtherInstances(cfg, inst.Name) + if err != nil { + return + } + deadline := time.Now().Add(startOrderTimeout) + for _, o := range others { + if !o.Since.Before(inst.Since) || clusterRunnerForName(cfg, o.Name) != nil { + continue + } + for time.Now().Before(deadline) { + if _, ok := o.Probe(); ok { + break + } + if s := instanceService(cfg, o.Name).IsActive(); s != "active" && s != "activating" { + break + } + time.Sleep(5 * time.Second) + } + } } diff --git a/internal/cli/status_internal_test.go b/internal/cli/status_internal_test.go index 5433781..057407f 100644 --- a/internal/cli/status_internal_test.go +++ b/internal/cli/status_internal_test.go @@ -10,7 +10,7 @@ func TestCrashReason(t *testing.T) { now, baseline int want string }{ - {"counts from the baseline", 5, 2, "the service is crash-looping (3 restarts since the switch)"}, + {"counts from the baseline", 5, 2, "the service is crash-looping (3 restarts since it started)"}, {"no baseline to count from", 5, -1, "the service is crash-looping"}, {"counter did not move", 2, 2, "the service is crash-looping"}, {"counter unreadable", -1, 0, "the service is crash-looping"}, diff --git a/internal/cli/vllmargs.go b/internal/cli/vllmargs.go index 9b37363..a42e49b 100644 --- a/internal/cli/vllmargs.go +++ b/internal/cli/vllmargs.go @@ -20,7 +20,7 @@ type serveSpec struct { engine.Spec } -func serveSpecFromActive(active *config.EnvFile) (serveSpec, error) { +func serveSpecFromEnv(active *config.EnvFile) (serveSpec, error) { prov, err := engine.Lookup(active.Get(engine.Key)) if err != nil { return serveSpec{}, err @@ -149,7 +149,7 @@ func requireVLLM(p *profile.Profile, what string) error { return nil } -func serveEnvFromActive(active *config.EnvFile) []string { +func serveEnvFromInstance(active *config.EnvFile) []string { env := os.Environ() for _, k := range active.Keys() { env = append(env, k+"="+active.Get(k)) diff --git a/internal/cli/vllmargs_internal_test.go b/internal/cli/vllmargs_internal_test.go index aa7c4b0..1233e48 100644 --- a/internal/cli/vllmargs_internal_test.go +++ b/internal/cli/vllmargs_internal_test.go @@ -93,7 +93,7 @@ func TestServeSpecFromActivePicksTheProvider(t *testing.T) { if err != nil { t.Fatal(err) } - spec, err := serveSpecFromActive(active) + spec, err := serveSpecFromEnv(active) if err != nil { t.Fatal(err) } @@ -124,7 +124,7 @@ func TestServeSpecFromActiveRejectsAnUnknownProvider(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := serveSpecFromActive(active); err == nil || !strings.Contains(err.Error(), "ollama") { + if _, err := serveSpecFromEnv(active); err == nil || !strings.Contains(err.Error(), "ollama") { t.Fatalf("err = %v, want it to name the unknown provider", err) } } @@ -271,7 +271,7 @@ func TestServeEnvNumbersGPUsInPCIBusOrder(t *testing.T) { active := write("active.env", "MODEL=m\nCUDA_DEVICE_ORDER=FASTEST_FIRST\nCUDA_VISIBLE_DEVICES=1\n") cfg := &config.Config{File: write("config.env", "PORT=8001\n"), Secrets: write("secrets.env", "")} p := &profile.Profile{Name: "p", File: write("p.env", "MODEL=m\nCUDA_DEVICE_ORDER=FASTEST_FIRST\n")} - for name, env := range map[string][]string{"active": serveEnvFromActive(active), "profile": serveEnvFromProfile(cfg, p)} { + for name, env := range map[string][]string{"active": serveEnvFromInstance(active), "profile": serveEnvFromProfile(cfg, p)} { var got []string for _, kv := range env { if strings.HasPrefix(kv, "CUDA_DEVICE_ORDER=") { diff --git a/internal/cli/watch.go b/internal/cli/watch.go index 529c598..8ae6c61 100644 --- a/internal/cli/watch.go +++ b/internal/cli/watch.go @@ -19,7 +19,7 @@ type watchOpts struct { func crashReason(now, baseline int) string { if baseline >= 0 && now > baseline { - return fmt.Sprintf("the service is crash-looping (%d restarts since the switch)", now-baseline) + return fmt.Sprintf("the service is crash-looping (%d restarts since it started)", now-baseline) } return "the service is crash-looping" } @@ -28,12 +28,16 @@ func crashReason(now, baseline int) string { // the journal, polls /health until it answers, and detects failed units // and crash loops instead of letting the user stare at a stalled log. // Returns nil once the endpoint is healthy. -func watchBoot(cfg *config.Config, label string, opts watchOpts) error { - t, err := bootTargetFor(cfg, svc(cfg)) +func watchBoot(cfg *config.Config, name string, opts watchOpts) error { + inst, err := control.LoadInstance(cfg, name) if err != nil { return err } - return watchTarget(t, label, opts) + t, err := bootTargetFor(instanceRunner(cfg, name), inst) + if err != nil { + return err + } + return watchTarget(t, name, opts) } // bootTarget is what a boot is watched through: the runner that reports @@ -62,7 +66,7 @@ func (t bootTarget) healthy() bool { // cluster serves from its head node on the recipe's port; probing localhost on // the configured port instead never succeeds when the head is another machine, // so the wait hung with the model already up. -func bootTargetFor(cfg *config.Config, r runner) (bootTarget, error) { +func bootTargetFor(r runner, inst *control.Instance) (bootTarget, error) { if c, ok := r.(*clusterRunner); ok { base, err := c.endpointURL() if err != nil { @@ -70,7 +74,7 @@ func bootTargetFor(cfg *config.Config, r runner) (bootTarget, error) { } return bootTarget{runner: r, base: base, where: base, health: c.recipe.HealthPath(), probe: c.probeHealth}, nil } - return bootTarget{runner: r, base: "http://localhost:" + cfg.Port(), where: "port " + cfg.Port(), health: control.ActiveHealthPath(cfg)}, nil + return bootTarget{runner: r, base: "http://localhost:" + inst.Port, where: "port " + inst.Port, health: inst.HealthPath()}, nil } func watchTarget(t bootTarget, label string, opts watchOpts) error { diff --git a/internal/cli/watch_internal_test.go b/internal/cli/watch_internal_test.go index c646c51..950bd50 100644 --- a/internal/cli/watch_internal_test.go +++ b/internal/cli/watch_internal_test.go @@ -6,13 +6,13 @@ import ( "net/url" "os" "os/exec" - "path/filepath" "strconv" "strings" "testing" "time" "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/control" "github.com/coder/llm-helper/internal/nodes" "github.com/coder/llm-helper/internal/service" ) @@ -29,7 +29,7 @@ func TestBootTargetFollowsTheServer(t *testing.T) { cfg := &config.Config{File: &config.EnvFile{}} cluster := &clusterRunner{recipe: r, opts: recipeLaunchOptions{nodeNames: []string{"there", "here"}}} - got, err := bootTargetFor(cfg, cluster) + got, err := bootTargetFor(cluster, nil) if err != nil { t.Fatal(err) } @@ -44,20 +44,26 @@ func TestBootTargetFollowsTheServer(t *testing.T) { } // A native server is asked on its engine's own health path. - state := t.TempDir() - if err := os.WriteFile(filepath.Join(state, "active.env"), []byte("PROVIDER=tensorfold-python\n"), 0o644); err != nil { + cfg.Dirs.State = t.TempDir() + if err := os.MkdirAll(cfg.Dirs.InstancesDir(), 0o755); err != nil { t.Fatal(err) } - cfg.Dirs.State = state - got, err = bootTargetFor(cfg, systemdService(cfg)) + if err := os.WriteFile(cfg.Dirs.InstanceEnvFile("tf"), []byte("PROVIDER=tensorfold-python\nPORT=8003\n"), 0o644); err != nil { + t.Fatal(err) + } + inst, err := control.LoadInstance(cfg, "tf") + if err != nil { + t.Fatal(err) + } + got, err = bootTargetFor(instanceService(cfg, "tf"), inst) if err != nil { t.Fatal(err) } - if want := "http://localhost:" + cfg.Port(); got.base != want { + if want := "http://localhost:8003"; got.base != want { t.Errorf("native probe = %s, want %s", got.base, want) } if got.health != "/v1/models" { - t.Errorf("native health path = %q, want the active engine's /v1/models", got.health) + t.Errorf("native health path = %q, want the instance engine's /v1/models", got.health) } } diff --git a/internal/config/config.go b/internal/config/config.go index 17b722d..a793b62 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,6 +1,6 @@ // Package config resolves vllm-helper's directories (XDG layout, matching // the bash implementation exactly) and loads the layered configuration: -// config.env + secrets.env + the active profile. +// config.env + secrets.env + a profile. package config import ( @@ -43,11 +43,19 @@ func ResolveDirs() Dirs { return d } -func (d Dirs) ConfigFile() string { return filepath.Join(d.Config, "config.env") } -func (d Dirs) SecretsFile() string { return filepath.Join(d.Config, "secrets.env") } -func (d Dirs) ActiveEnvFile() string { return filepath.Join(d.State, "active.env") } -func (d Dirs) ActiveProfileFile() string { return filepath.Join(d.State, "active-profile") } -func (d Dirs) CoderIDsFile() string { return filepath.Join(d.State, "coder-ids.env") } +func (d Dirs) ConfigFile() string { return filepath.Join(d.Config, "config.env") } +func (d Dirs) SecretsFile() string { return filepath.Join(d.Config, "secrets.env") } +func (d Dirs) CoderIDsFile() string { return filepath.Join(d.State, "coder-ids.env") } + +// InstancesDir holds one env file per running model. +func (d Dirs) InstancesDir() string { return filepath.Join(d.State, "instances") } +func (d Dirs) InstanceEnvFile(name string) string { + return filepath.Join(d.InstancesDir(), name+".env") +} + +// The single-model state of releases before instances; read only by migrate. +func (d Dirs) LegacyActiveEnvFile() string { return filepath.Join(d.State, "active.env") } +func (d Dirs) LegacyActiveProfileFile() string { return filepath.Join(d.State, "active-profile") } // PluginStateFile is where an integration keeps its own state, namespaced by // plugin name so two of them can't collide. @@ -177,9 +185,16 @@ func (c *Config) CoderConfigured() bool { return c.CoderURL() != "" && c.CoderToken() != "" && c.CoderProviderID() != "" } -// ActiveProfile returns the name of the currently active profile ("" if none). -func (c *Config) ActiveProfile() string { - b, err := os.ReadFile(c.Dirs.ActiveProfileFile()) +// DefaultProfile is the instance Coder offers as its default model, "" when +// none was chosen. +func (c *Config) DefaultProfile() string { return c.get("DEFAULT_PROFILE", "") } + +// RouterPort is where the model router listens, "" when it is not installed. +func (c *Config) RouterPort() string { return c.get("ROUTER_PORT", "") } + +// LegacyActiveProfile is the profile a pre-instance release was serving. +func (c *Config) LegacyActiveProfile() string { + b, err := os.ReadFile(c.Dirs.LegacyActiveProfileFile()) if err != nil { return "" } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index e9d1c60..f69d11c 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -73,8 +73,9 @@ func TestDirsLayout(t *testing.T) { }{ {"config file", d.ConfigFile(), "/c/config.env"}, {"secrets file", d.SecretsFile(), "/c/secrets.env"}, - {"active env", d.ActiveEnvFile(), "/s/active.env"}, - {"active profile", d.ActiveProfileFile(), "/s/active-profile"}, + {"legacy active env", d.LegacyActiveEnvFile(), "/s/active.env"}, + {"legacy active profile", d.LegacyActiveProfileFile(), "/s/active-profile"}, + {"instance file", d.InstanceEnvFile("qwen"), "/s/instances/qwen.env"}, {"coder ids", d.CoderIDsFile(), "/s/coder-ids.env"}, {"profile file", d.ProfileFile("qwen"), "/c/models/qwen.env"}, } @@ -189,7 +190,7 @@ func TestCoderConfiguredNeedsEveryField(t *testing.T) { } } -func TestActiveProfileReadsStateFile(t *testing.T) { +func TestLegacyActiveProfileReadsStateFile(t *testing.T) { dirs := newDirs(t) writeFile(t, dirs.ConfigFile(), "PORT=8001\n") @@ -197,14 +198,13 @@ func TestActiveProfileReadsStateFile(t *testing.T) { if err != nil { t.Fatalf("Load: %v", err) } - if got := cfg.ActiveProfile(); got != "" { - t.Errorf("ActiveProfile() = %q, want empty before anything is switched to", got) + if got := cfg.LegacyActiveProfile(); got != "" { + t.Errorf("LegacyActiveProfile() = %q, want empty", got) } - // Trailing newline, as "echo name >" leaves. - writeFile(t, dirs.ActiveProfileFile(), "qwen3-0.6b\n") - if got, want := cfg.ActiveProfile(), "qwen3-0.6b"; got != want { - t.Errorf("ActiveProfile() = %q, want %q", got, want) + writeFile(t, dirs.LegacyActiveProfileFile(), "qwen3-0.6b\n") + if got, want := cfg.LegacyActiveProfile(), "qwen3-0.6b"; got != want { + t.Errorf("LegacyActiveProfile() = %q, want %q", got, want) } } diff --git a/internal/config/envfile.go b/internal/config/envfile.go index 4f81d4f..4bb38f7 100644 --- a/internal/config/envfile.go +++ b/internal/config/envfile.go @@ -9,7 +9,7 @@ import ( // EnvFile is an ordered KEY=VALUE file, bash-sourceable, with comments // preserved on read-modify-write. It is the shared on-disk format for -// config.env, secrets.env, model profiles, and the generated active.env -- +// config.env, secrets.env, model profiles, and the generated instance files -- // both this binary and the legacy bash scripts read the same files. type EnvFile struct { // Lines preserves the original file verbatim (comments, blanks, order). diff --git a/internal/control/control.go b/internal/control/control.go index 404ee41..909627e 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -1,13 +1,6 @@ // Package control performs the operations that change or report what this -// machine is serving. It is the layer under both the CLI and the web panel. -// -// The two interfaces exist to present this work differently -- one prints to a -// terminal, the other returns JSON to a browser -- and nothing about *what the -// work is* should differ between them. Keeping that here is not tidiness: an -// earlier second copy of WriteActiveEnv had already drifted into sorting the -// file, dropping empty values, and re-quoting values that were fine as -// written, so the same profile produced a different environment depending on -// which interface switched to it. +// machine is serving. It is the layer under both the CLI and the web panel, +// so the two cannot drift apart on what the work is. package control import ( @@ -15,115 +8,70 @@ import ( "fmt" "io" "net/http" - "os" "strings" "time" "github.com/coder/llm-helper/internal/config" - "github.com/coder/llm-helper/internal/engine" "github.com/coder/llm-helper/internal/plugin" "github.com/coder/llm-helper/internal/profile" ) -// BaseURL is where the model server answers on this machine. +// BaseURL is the address a single client such as the Coder provider should +// use: the router when it is installed, the first instance port otherwise. func BaseURL(cfg *config.Config) string { - return fmt.Sprintf("http://localhost:%s/v1", cfg.Port()) -} - -// WriteActiveEnv assembles state/active.env from config.env + secrets.env + -// the profile, and records the active profile name. -// -// Lines are copied through exactly as written, which is the whole contract. -// The file is sourced by the service, so re-serialising it would mean deciding -// how to quote values someone else already quoted; a value with spaces that -// round-trips through a quoting function is not guaranteed to produce the same -// argv. Dropping empty values would be a second such decision -- set-to-empty -// and unset are not the same thing to a program reading its environment. -// -// Together this is the pair that changes what is served: WriteActiveEnv puts -// the settings in place, NotifyAndRestart makes them live. Call them in that -// order; the gap between them is where a caller does its own reporting. -func WriteActiveEnv(cfg *config.Config, p *profile.Profile) error { - // An unknown provider would otherwise surface only when the service tried - // to start it, by which time the outgoing model is already down. - if _, err := engine.Lookup(p.Provider()); err != nil { - return fmt.Errorf("profile '%s': %w", p.Name, err) - } - if err := os.MkdirAll(cfg.Dirs.State, 0o755); err != nil { - return err - } - var b strings.Builder - appendFile := func(f *config.EnvFile) { - if f == nil { - return - } - for _, line := range f.Lines { - trimmed := strings.TrimSpace(line) - if trimmed == "" || strings.HasPrefix(trimmed, "#") { - continue - } - b.WriteString(line + "\n") - } + port := cfg.Port() + if rp := cfg.RouterPort(); rp != "" { + port = rp } - appendFile(cfg.File) - appendFile(cfg.Secrets) - appendFile(p.File) - // config.env's PROVIDER is the default for new profiles, not for this - // one: a profile that names no provider is vLLM's. Without this line the - // default would leak in through config.env's lines above and serve an - // old vLLM profile with another server. Only written when there is a - // default to override, so an install that never set one gets the file - // it always got. - if p.Provider() == "" && cfg.Provider() != "" { - b.WriteString(engine.Key + "=" + engine.VLLM + "\n") - } - - // Written and renamed rather than truncated in place: a crash halfway - // through must not leave the service with half an environment. - tmp := cfg.Dirs.ActiveEnvFile() + ".tmp" - if err := os.WriteFile(tmp, []byte(b.String()), 0o600); err != nil { - return err - } - if err := os.Rename(tmp, cfg.Dirs.ActiveEnvFile()); err != nil { - return err - } - return os.WriteFile(cfg.Dirs.ActiveProfileFile(), []byte(p.Name+"\n"), 0o644) + return fmt.Sprintf("http://localhost:%s/v1", port) } -// NotifyAndRestart tells the plugins what is happening and restarts the -// service. Plugin output goes to out, so a terminal caller can stream it and -// a browser caller can capture it. -// -// restart is supplied by the caller because the two have different ideas of -// what a restart may cost: the CLI blocks on it, while the panel bounds it so -// an HTTP request cannot hang on systemd. -// -// Plugins run before the restart deliberately. Coder's model entry should -// describe the model that is coming up by the time it is up -- doing it after -// would leave a window where the picker offers something the server no longer -// serves. -func NotifyAndRestart(ctx context.Context, cfg *config.Config, p *profile.Profile, out io.Writer, restart func() error) error { - plugin.Dispatch(ctx, out, plugin.EventActivating, plugin.State{ +func state(cfg *config.Config, name string, p *profile.Profile, base string) plugin.State { + live := LiveProfiles(cfg) + return plugin.State{ Config: cfg, - Name: p.Name, + Name: name, Profile: p, - BaseURL: BaseURL(cfg), - }) + BaseURL: base, + Live: live, + Default: DefaultInstance(cfg, live), + } +} + +// NotifyAndRestart tells the plugins an instance is starting, then calls +// restart. The instance file must already be written. Plugins run first so +// Coder's entry describes the model by the time it is up. +func NotifyAndRestart(ctx context.Context, cfg *config.Config, p *profile.Profile, inst *Instance, out io.Writer, restart func() error) error { + plugin.Dispatch(ctx, out, plugin.EventStarting, state(cfg, p.Name, p, inst.BaseURL())) if err := restart(); err != nil { return fmt.Errorf("restarting service: %w", err) } return nil } +// NotifyStopped tells the plugins an instance is gone. Call it after the +// instance file is removed. +func NotifyStopped(ctx context.Context, cfg *config.Config, name string, out io.Writer) { + p, _ := profile.Load(cfg.Dirs, name) + plugin.Dispatch(ctx, out, plugin.EventStopped, state(cfg, name, p, BaseURL(cfg))) +} + +// NotifyDefault tells the plugins the default instance changed. +func NotifyDefault(ctx context.Context, cfg *config.Config, out io.Writer) { + st := state(cfg, "", nil, BaseURL(cfg)) + if st.Default == "" { + return + } + st.Name = st.Default + st.Profile, _ = profile.Load(cfg.Dirs, st.Default) + plugin.Dispatch(ctx, out, plugin.EventDefault, st) +} + // NotifyRemoved tells the plugins a profile is going away, so integrations can // drop their copy of it. Called before the file is deleted, since plugins key // off the name. func NotifyRemoved(ctx context.Context, cfg *config.Config, name string, out io.Writer) { - plugin.Dispatch(ctx, out, plugin.EventRemoved, plugin.State{ - Config: cfg, - Name: name, - BaseURL: BaseURL(cfg), - }) + plugin.Dispatch(ctx, out, plugin.EventRemoved, state(cfg, name, nil, BaseURL(cfg))) } // ProbeHealth asks vLLM whether it is serving. Returns the HTTP status code @@ -167,29 +115,3 @@ func LastMeaningfulLine(logs string) string { } return "" } - -// ParseActiveEnv reads back the environment the service was last started with. -func ParseActiveEnv(cfg *config.Config) (*config.EnvFile, error) { - return config.ParseEnvFile(cfg.Dirs.ActiveEnvFile()) -} - -// ActiveHealthPath is the health path of the engine the service was last -// started with, so a local engine that answers somewhere other than /health -// is still seen to serve. It is empty, which probes /health, when that is -// unknown. -func ActiveHealthPath(cfg *config.Config) string { - active, err := ParseActiveEnv(cfg) - if err != nil { - return "" - } - e, err := engine.LookupEngine(active.Get(engine.Key)) - if err != nil { - return "" - } - return e.Health.Path -} - -// ProbeActive is ProbeHealth on the active engine's health path. -func ProbeActive(cfg *config.Config) (int, bool) { - return ProbeHealthPath("http://localhost:"+cfg.Port(), ActiveHealthPath(cfg)) -} diff --git a/internal/control/control_internal_test.go b/internal/control/control_internal_test.go index 1ee9c34..ae6cfa2 100644 --- a/internal/control/control_internal_test.go +++ b/internal/control/control_internal_test.go @@ -7,17 +7,12 @@ import ( "testing" "github.com/coder/llm-helper/internal/config" - "github.com/coder/llm-helper/internal/detect" "github.com/coder/llm-helper/internal/profile" ) -// active.env is sourced by the service, so what goes into it decides the argv -// vLLM runs with. This locks the three properties a second implementation had -// already broken -- it sorted the keys, dropped empty values, and re-quoted -// values that were fine as written -- because the symptom of getting any of -// them wrong is a server that serves something subtly different from what the -// profile says, with nothing in any log to say so. -func TestWriteActiveEnvCopiesLinesThrough(t *testing.T) { +// The instance file is sourced by the service, so it must keep file order, +// empty values and quoting exactly as written. +func TestWriteInstanceEnvCopiesLinesThrough(t *testing.T) { root := t.TempDir() dirs := config.Dirs{ Config: filepath.Join(root, "config"), @@ -54,19 +49,27 @@ func TestWriteActiveEnvCopiesLinesThrough(t *testing.T) { if err != nil { t.Fatal(err) } - if err := WriteActiveEnv(cfg, p); err != nil { + if err := WriteInstanceEnv(cfg, p, InstanceOpts{Port: "8002"}); err != nil { t.Fatal(err) } - b, err := os.ReadFile(dirs.ActiveEnvFile()) + b, err := os.ReadFile(dirs.InstanceEnvFile("demo")) if err != nil { t.Fatal(err) } got := string(b) + inst, err := LoadInstance(cfg, "demo") + if err != nil { + t.Fatal(err) + } + if inst.Port != "8002" || inst.Since.IsZero() { + t.Errorf("instance port %q since %v, want 8002 and a start time", inst.Port, inst.Since) + } want := "ZZZ_FIRST=1\nPORT=8001\nEMPTY_ON_PURPOSE=\nVLLM_API_KEY=sk-secret\n" + - "MODEL=org/demo\nEXTRA_ARGS=--tool-call-parser qwen3_coder\nAAA_LAST=z\n" + "MODEL=org/demo\nEXTRA_ARGS=--tool-call-parser qwen3_coder\nAAA_LAST=z\n" + + "INSTANCE_SINCE=" + inst.Env.Get("INSTANCE_SINCE") + "\nPORT=8002\n" if got != want { - t.Errorf("active.env mismatch\n got:\n%s\nwant:\n%s", got, want) + t.Errorf("instance env mismatch\n got:\n%s\nwant:\n%s", got, want) } // Said individually, so a failure names which property broke. @@ -84,25 +87,19 @@ func TestWriteActiveEnvCopiesLinesThrough(t *testing.T) { } // Secrets land here, so the file must not be readable by other users. - info, err := os.Stat(dirs.ActiveEnvFile()) + info, err := os.Stat(dirs.InstanceEnvFile("demo")) if err != nil { t.Fatal(err) } if perm := info.Mode().Perm(); perm != 0o600 { - t.Errorf("active.env mode = %o, want 600: it contains every secret on the machine", perm) - } - - if b, err := os.ReadFile(dirs.ActiveProfileFile()); err != nil { - t.Fatal(err) - } else if strings.TrimSpace(string(b)) != "demo" { - t.Errorf("active profile = %q, want demo", strings.TrimSpace(string(b))) + t.Errorf("instance file mode = %o, want 600: it contains every secret on the machine", perm) } } // config.env's PROVIDER is the default for new profiles. It must not reach an // existing profile through config.env's lines, or setting PROVIDER=mlx on a // Mac would hand every vLLM profile's flags to mlx_lm.server. -func TestWriteActiveEnvPinsTheProfilesProvider(t *testing.T) { +func TestWriteInstanceEnvPinsTheProfilesProvider(t *testing.T) { t.Parallel() tests := []struct { name string @@ -145,28 +142,28 @@ func TestWriteActiveEnvPinsTheProfilesProvider(t *testing.T) { t.Fatal(err) } - err = WriteActiveEnv(cfg, p) + err = WriteInstanceEnv(cfg, p, InstanceOpts{Port: "8001"}) if tt.wantErr { if err == nil { - t.Fatal("an unknown provider was written into active.env") + t.Fatal("an unknown provider was written into the instance file") } - if _, statErr := os.Stat(dirs.ActiveEnvFile()); statErr == nil { - t.Error("active.env was written for a profile that can't be served") + if _, statErr := os.Stat(dirs.InstanceEnvFile("p")); statErr == nil { + t.Error("an instance file was written for a profile that can't be served") } return } if err != nil { t.Fatal(err) } - active, err := config.ParseEnvFile(dirs.ActiveEnvFile()) + active, err := config.ParseEnvFile(dirs.InstanceEnvFile("p")) if err != nil { t.Fatal(err) } if got := active.Get("PROVIDER"); got != tt.want { - t.Errorf("PROVIDER in active.env = %q, want %q", got, tt.want) + t.Errorf("PROVIDER in the instance file = %q, want %q", got, tt.want) } if tt.want == "" && active.Has("PROVIDER") { - t.Error("an install that never set PROVIDER got one in active.env") + t.Error("an install that never set PROVIDER got one in its instance file") } }) } @@ -205,89 +202,19 @@ func TestLastMeaningfulLine(t *testing.T) { } } -// A profile can set CUDA_VISIBLE_DEVICES, so sizing from config.env alone -// described GPUs the server was not allowed to use. The panel and the CLI -// both read it from here, so they cannot disagree about it. -func TestServerCUDAVisibleReadsTheActiveProfile(t *testing.T) { +func TestServerCUDAVisibleReadsConfig(t *testing.T) { t.Setenv("CUDA_VISIBLE_DEVICES", "3") - root := t.TempDir() - dirs := config.Dirs{Config: filepath.Join(root, "config"), State: filepath.Join(root, "state"), Data: filepath.Join(root, "data"), Profiles: filepath.Join(root, "config", "models")} - for _, d := range []string{dirs.Config, dirs.State, dirs.Profiles} { - if err := os.MkdirAll(d, 0o755); err != nil { - t.Fatal(err) - } - } - if err := os.WriteFile(dirs.ConfigFile(), []byte("PORT=8001\nCUDA_VISIBLE_DEVICES=0,1\n"), 0o600); err != nil { - t.Fatal(err) + cfg := &config.Config{File: &config.EnvFile{}} + if got, set := ServerCUDAVisible(cfg); got != "3" || !set { + t.Errorf("without config: (%q, %v), want the environment's", got, set) } - cfgFile, err := config.ParseEnvFile(dirs.ConfigFile()) - if err != nil { - t.Fatal(err) - } - cfg := &config.Config{Dirs: dirs, File: cfgFile} - + cfg.File.Set("CUDA_VISIBLE_DEVICES", "0,1") if got, set := ServerCUDAVisible(cfg); got != "0,1" || !set { - t.Errorf("before any switch: (%q, %v), want config.env's", got, set) - } - if err := os.WriteFile(dirs.ProfileFile("one"), []byte("MODEL=org/m\nCUDA_VISIBLE_DEVICES=1\n"), 0o600); err != nil { - t.Fatal(err) - } - p, err := profile.Load(dirs, "one") - if err != nil { - t.Fatal(err) - } - if err := WriteActiveEnv(cfg, p); err != nil { - t.Fatal(err) - } - if got, set := ServerCUDAVisible(cfg); got != "1" || !set { - t.Errorf("after switching to a profile that sets it: (%q, %v), want the profile's", got, set) - } -} - -// The panel registers the server's GPUs once, at startup, and a switch made -// from the CLI rewrites active.env without telling it. Each probe has to read -// the GPU selection afresh, or the panel keeps sizing the old profile's GPU. -func TestUseServerGPUsFollowsActiveEnv(t *testing.T) { - bin := t.TempDir() - smi := "#!/bin/sh\ncase \"$1\" in\n--query-gpu=index,uuid,name,memory.total) printf '0, GPU-aaaa, NVIDIA H100 80GB HBM3, 81559\\n1, GPU-bbbb, NVIDIA RTX A6000, 49140\\n' ;;\n*) exit 1 ;;\nesac\n" - if err := os.WriteFile(filepath.Join(bin, "nvidia-smi"), []byte(smi), 0o755); err != nil { - t.Fatal(err) - } - t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) - t.Setenv("CUDA_VISIBLE_DEVICES", "0") - t.Cleanup(func() { detect.UseCUDAVisibleSource(nil) }) - - root := t.TempDir() - dirs := config.Dirs{Config: filepath.Join(root, "config"), State: filepath.Join(root, "state"), Data: filepath.Join(root, "data"), Profiles: filepath.Join(root, "config", "models")} - for _, d := range []string{dirs.Config, dirs.State, dirs.Profiles} { - if err := os.MkdirAll(d, 0o755); err != nil { - t.Fatal(err) - } - } - if err := os.WriteFile(dirs.ConfigFile(), []byte("PORT=8001\n"), 0o600); err != nil { - t.Fatal(err) - } - cfgFile, err := config.ParseEnvFile(dirs.ConfigFile()) - if err != nil { - t.Fatal(err) - } - cfg := &config.Config{Dirs: dirs, File: cfgFile} - - UseServerGPUs(cfg) - if got := detect.Probe().GPUName; got != "NVIDIA H100 80GB HBM3" { - t.Fatalf("before the switch the server's GPU is %q, want GPU 0", got) - } - if err := os.WriteFile(dirs.ProfileFile("other"), []byte("MODEL=org/m\nCUDA_VISIBLE_DEVICES=1\n"), 0o600); err != nil { - t.Fatal(err) - } - p, err := profile.Load(dirs, "other") - if err != nil { - t.Fatal(err) - } - if err := WriteActiveEnv(cfg, p); err != nil { - t.Fatal(err) + t.Errorf("with config: (%q, %v), want config.env's", got, set) } - if got := detect.Probe().GPUName; got != "NVIDIA RTX A6000" { - t.Errorf("after a switch nobody told this process about, the server's GPU is %q, want GPU 1", got) + p := &config.EnvFile{} + p.Set("CUDA_VISIBLE_DEVICES", "1") + if got, _ := CUDAVisible(cfg.File, p); got != "1" { + t.Errorf("a profile layered on top gives %q, want its own", got) } } diff --git a/internal/control/fit.go b/internal/control/fit.go new file mode 100644 index 0000000..69892da --- /dev/null +++ b/internal/control/fit.go @@ -0,0 +1,113 @@ +package control + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/detect" +) + +// MaxUnifiedMemUtilization is the most of a shared CPU/GPU pool all models +// together may ask for. +const MaxUnifiedMemUtilization = 0.85 + +// Share is the fraction of GPU memory one running instance asked for. +type Share struct { + Name string + Frac float64 +} + +// OtherInstances lists the running instances except name. +func OtherInstances(cfg *config.Config, name string) ([]*Instance, error) { + list, err := ListInstances(cfg) + if err != nil { + return nil, err + } + out := list[:0] + for _, inst := range list { + if inst.Name != name { + out = append(out, inst) + } + } + return out, nil +} + +// Shares is the memory share of each instance in list. +func Shares(list []*Instance) []Share { + out := make([]Share, 0, len(list)) + for _, inst := range list { + out = append(out, Share{Name: inst.Name, Frac: ParseFrac(MemUtil(inst.Env))}) + } + return out +} + +// MemUtil is the first GPU_MEMORY_UTILIZATION set among files. +func MemUtil(files ...*config.EnvFile) string { + for _, f := range files { + if f == nil { + continue + } + if v := f.Get("GPU_MEMORY_UTILIZATION"); v != "" { + return v + } + } + return "" +} + +// ParseFrac reads a positive fraction, or 0. +func ParseFrac(s string) float64 { + f, err := strconv.ParseFloat(strings.TrimSpace(s), 64) + if err != nil || f <= 0 { + return 0 + } + return f +} + +// CheckFit refuses to start name when its share of GPU memory plus what the +// running instances hold exceeds the pool. Unknown shares count as 0. +func CheckFit(hw detect.Hardware, running []Share, name string, want float64) error { + var limit float64 + switch { + case hw.Unified: + limit = MaxUnifiedMemUtilization + case hw.GPUName != "" || hw.GPUMemGB > 0: + limit = 1 + default: + return nil + } + if want == 0 { + return nil + } + total := want + var held []string + for _, r := range running { + if r.Frac == 0 { + continue + } + total += r.Frac + held = append(held, fmt.Sprintf("%s %.2f", r.Name, r.Frac)) + } + if total <= limit+1e-9 { + return nil + } + sort.Strings(held) + return fmt.Errorf("'%s' wants %.2f of GPU memory, but the running models already use %.2f of the %.2f available (%s); "+ + "stop one first, or give this one a smaller share (--mem-util or --kv-cache)", + name, want, total-want, limit, strings.Join(held, ", ")) +} + +// CheckFitFor runs CheckFit for p against every other running instance. want +// is the requested share; "" uses the profile's own. +func CheckFitFor(cfg *config.Config, name string, want string, profileFiles ...*config.EnvFile) error { + others, err := OtherInstances(cfg, name) + if err != nil { + return err + } + if want == "" { + want = MemUtil(profileFiles...) + } + return CheckFit(detect.Probe(), Shares(others), name, ParseFrac(want)) +} diff --git a/internal/control/fit_internal_test.go b/internal/control/fit_internal_test.go new file mode 100644 index 0000000..f2cce49 --- /dev/null +++ b/internal/control/fit_internal_test.go @@ -0,0 +1,40 @@ +package control + +import ( + "strings" + "testing" + + "github.com/coder/llm-helper/internal/detect" +) + +func TestCheckFit(t *testing.T) { + unified := detect.Hardware{Unified: true} + discrete := detect.Hardware{GPUName: "NVIDIA RTX 6000", GPUMemGB: 96} + cases := []struct { + name string + hw detect.Hardware + running []Share + want float64 + refuse bool + }{ + {"first model", unified, nil, 0.85, false}, + {"two halves", unified, []Share{{"a", 0.4}}, 0.4, false}, + {"over the unified cap", unified, []Share{{"a", 0.5}}, 0.4, true}, + {"discrete fits to 1", discrete, []Share{{"a", 0.5}}, 0.5, false}, + {"discrete over", discrete, []Share{{"a", 0.6}}, 0.5, true}, + {"unknown share ignored", unified, []Share{{"a", 0}}, 0.85, false}, + {"unknown request", unified, []Share{{"a", 0.85}}, 0, false}, + {"cpu only", detect.Hardware{}, []Share{{"a", 0.9}}, 0.9, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := CheckFit(tc.hw, tc.running, "new", tc.want) + if (err != nil) != tc.refuse { + t.Fatalf("refuse=%v, got %v", tc.refuse, err) + } + if err != nil && (!strings.Contains(err.Error(), "a 0.") || !strings.Contains(err.Error(), "stop one")) { + t.Errorf("message should name the running model and suggest stopping one: %v", err) + } + }) + } +} diff --git a/internal/control/gguf.go b/internal/control/gguf.go index ef522b7..76dd827 100644 --- a/internal/control/gguf.go +++ b/internal/control/gguf.go @@ -63,8 +63,7 @@ func GGUFInstallCommand(cfg *config.Config) string { // vLLM, writing pip's output to out. // // It does not restart anything. The plugin is read at engine start, so it -// takes effect on the next switch -- which means installing it can never -// disturb a model that is currently serving. +// takes effect when an instance next starts, and never disturbs one serving. func InstallGGUF(ctx context.Context, cfg *config.Config, out io.Writer) error { python := PythonForVLLM(cfg.VLLMBin()) if python == "" { diff --git a/internal/control/gpus.go b/internal/control/gpus.go index 267f632..b240ee0 100644 --- a/internal/control/gpus.go +++ b/internal/control/gpus.go @@ -22,22 +22,15 @@ func CUDAVisible(files ...*config.EnvFile) (string, bool) { return value, set } -// ServerCUDAVisible is the CUDA_VISIBLE_DEVICES the service runs with: the -// active profile's environment when one has been written, since a profile -// can set it, and config.env's otherwise. +// ServerCUDAVisible is the CUDA_VISIBLE_DEVICES config.env gives a server. +// Each instance's profile can narrow it; callers sizing one profile layer +// its file on top with CUDAVisible. func ServerCUDAVisible(cfg *config.Config) (string, bool) { - files := []*config.EnvFile{cfg.File, cfg.Secrets} - if active, err := ParseActiveEnv(cfg); err == nil { - files = append(files, active) - } - return CUDAVisible(files...) + return CUDAVisible(cfg.File, cfg.Secrets) } -// UseServerGPUs tells detect which GPUs the service may use, so memory is -// sized for the GPU it will load onto. The CLI and the panel both call it. -// active.env is read again at each probe rather than once here: the panel -// outlives a switch, and a switch made from the CLI changes active.env -// without the panel being told. +// UseServerGPUs tells detect which GPUs a server may use, so memory is sized +// for the GPU it will load onto. The CLI and the panel both call it. func UseServerGPUs(cfg *config.Config) { detect.UseCUDAVisibleSource(func() (string, bool) { return ServerCUDAVisible(cfg) }) } diff --git a/internal/control/instances.go b/internal/control/instances.go new file mode 100644 index 0000000..5934275 --- /dev/null +++ b/internal/control/instances.go @@ -0,0 +1,277 @@ +package control + +import ( + "errors" + "fmt" + "io/fs" + "net" + "os" + "sort" + "strconv" + "strings" + "time" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/engine" + "github.com/coder/llm-helper/internal/profile" +) + +// Keys llm-helper adds to an instance file. They follow the profile's lines, +// so they win when the file is sourced or parsed. +const ( + keySince = "INSTANCE_SINCE" + keyMemUtil = "INSTANCE_MEM_UTIL" + keyKVCache = "INSTANCE_KV_CACHE" +) + +// InstanceOpts are the per-instance settings that are not in the profile. +type InstanceOpts struct { + Port string + MemUtil string + KVCache string +} + +// Instance is one running model: a profile, its port and the environment its +// server starts with. +type Instance struct { + Name string + Port string + Since time.Time + Env *config.EnvFile +} + +func (i *Instance) Opts() InstanceOpts { + return InstanceOpts{Port: i.Port, MemUtil: i.Env.Get(keyMemUtil), KVCache: i.Env.Get(keyKVCache)} +} + +// BaseURL is where the instance's OpenAI-compatible API answers. +func (i *Instance) BaseURL() string { return "http://localhost:" + i.Port + "/v1" } + +// HealthPath is the engine's health path, "" (meaning /health) when unknown. +func (i *Instance) HealthPath() string { + e, err := engine.LookupEngine(i.Env.Get(engine.Key)) + if err != nil { + return "" + } + return e.Health.Path +} + +func (i *Instance) Probe() (int, bool) { + return ProbeHealthPath("http://localhost:"+i.Port, i.HealthPath()) +} + +// WriteInstanceEnv assembles the instance file from config.env, secrets.env +// and the profile, followed by the instance's own settings. Lines are copied +// as written; the service sources the file. +func WriteInstanceEnv(cfg *config.Config, p *profile.Profile, o InstanceOpts) error { + if err := profile.ValidInstanceName(p.Name); err != nil { + return err + } + if _, err := engine.Lookup(p.Provider()); err != nil { + return fmt.Errorf("profile '%s': %w", p.Name, err) + } + if o.Port == "" { + return fmt.Errorf("instance '%s': no port", p.Name) + } + if err := os.MkdirAll(cfg.Dirs.InstancesDir(), 0o755); err != nil { + return err + } + since := strconv.FormatInt(time.Now().Unix(), 10) + if old, err := LoadInstance(cfg, p.Name); err == nil && old.Env.Get(keySince) != "" { + since = old.Env.Get(keySince) + } + + var b strings.Builder + appendFile := func(f *config.EnvFile) { + if f == nil { + return + } + for _, line := range f.Lines { + trimmed := strings.TrimSpace(line) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + b.WriteString(line + "\n") + } + } + appendFile(cfg.File) + appendFile(cfg.Secrets) + appendFile(p.File) + // A profile that names no provider is vLLM's, whatever config.env's + // default for new profiles says. + if p.Provider() == "" && cfg.Provider() != "" { + b.WriteString(engine.Key + "=" + engine.VLLM + "\n") + } + line := func(k, v string) { b.WriteString(k + "=" + config.Quote(v) + "\n") } + line(keySince, since) + if o.MemUtil != "" { + line(keyMemUtil, o.MemUtil) + line("GPU_MEMORY_UTILIZATION", o.MemUtil) + } + if o.KVCache != "" { + line(keyKVCache, o.KVCache) + line("KV_CACHE_MEMORY_BYTES", o.KVCache) + } + line("PORT", o.Port) + + path := cfg.Dirs.InstanceEnvFile(p.Name) + tmp := path + ".tmp" + if err := os.WriteFile(tmp, []byte(b.String()), 0o600); err != nil { + return err + } + return os.Rename(tmp, path) +} + +// LoadInstance reads one instance file. A missing file is fs.ErrNotExist. +func LoadInstance(cfg *config.Config, name string) (*Instance, error) { + env, err := config.ParseEnvFile(cfg.Dirs.InstanceEnvFile(name)) + if err != nil { + return nil, err + } + inst := &Instance{Name: name, Port: env.Get("PORT"), Env: env} + if n, err := strconv.ParseInt(env.Get(keySince), 10, 64); err == nil { + inst.Since = time.Unix(n, 0) + } + return inst, nil +} + +// ListInstances returns every instance, oldest first. +func ListInstances(cfg *config.Config) ([]*Instance, error) { + entries, err := os.ReadDir(cfg.Dirs.InstancesDir()) + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + if err != nil { + return nil, err + } + var out []*Instance + for _, e := range entries { + name, ok := strings.CutSuffix(e.Name(), ".env") + if !ok || e.IsDir() { + continue + } + inst, err := LoadInstance(cfg, name) + if err != nil { + continue + } + out = append(out, inst) + } + sort.SliceStable(out, func(a, b int) bool { + if !out[a].Since.Equal(out[b].Since) { + return out[a].Since.Before(out[b].Since) + } + return out[a].Name < out[b].Name + }) + return out, nil +} + +func RemoveInstance(cfg *config.Config, name string) error { + err := os.Remove(cfg.Dirs.InstanceEnvFile(name)) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + return err +} + +// LiveProfiles names the profiles that have an instance, oldest first. +func LiveProfiles(cfg *config.Config) []string { + list, _ := ListInstances(cfg) + names := make([]string, len(list)) + for i, inst := range list { + names[i] = inst.Name + } + return names +} + +// DefaultInstance is the instance Coder should offer first: DEFAULT_PROFILE +// when it is running, the oldest instance otherwise. +func DefaultInstance(cfg *config.Config, live []string) string { + if d := cfg.DefaultProfile(); d != "" { + for _, n := range live { + if n == d { + return d + } + } + } + if len(live) > 0 { + return live[0] + } + return "" +} + +// portFree is swapped by tests. +var portFree = func(port string) bool { + l, err := net.Listen("tcp", net.JoinHostPort("", port)) + if err != nil { + return false + } + l.Close() + return true +} + +// AllocatePort picks the port for name: want when given, the port it already +// has, or the first port from PORT up that no other instance holds and +// nothing listens on. +func AllocatePort(cfg *config.Config, name, want string) (string, error) { + taken := map[string]string{} + list, err := ListInstances(cfg) + if err != nil { + return "", err + } + for _, inst := range list { + if inst.Name != name { + taken[inst.Port] = inst.Name + } + } + if rp := cfg.RouterPort(); rp != "" { + taken[rp] = "the router" + } + if want != "" { + if _, err := strconv.Atoi(want); err != nil { + return "", fmt.Errorf("port %q is not a number", want) + } + if owner, ok := taken[want]; ok { + return "", fmt.Errorf("port %s is used by %s", want, owner) + } + return want, nil + } + for _, inst := range list { + if inst.Name == name && inst.Port != "" { + return inst.Port, nil + } + } + start, err := strconv.Atoi(cfg.Port()) + if err != nil { + return "", fmt.Errorf("PORT %q is not a number", cfg.Port()) + } + for p := start; p < start+100 && p <= 65535; p++ { + s := strconv.Itoa(p) + if _, ok := taken[s]; ok { + continue + } + if portFree(s) { + return s, nil + } + } + return "", fmt.Errorf("no free port in %d-%d", start, start+99) +} + +// CheckRouterPort reports why port can't take the router, nil when it can. +func CheckRouterPort(cfg *config.Config, port string) error { + if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 { + return fmt.Errorf("port %q is not a valid port", port) + } + list, err := ListInstances(cfg) + if err != nil { + return err + } + for _, inst := range list { + if inst.Port == port { + return fmt.Errorf("port %s is used by %s", port, inst.Name) + } + } + if port != cfg.RouterPort() && !portFree(port) { + return fmt.Errorf("port %s is already in use", port) + } + return nil +} diff --git a/internal/control/instances_internal_test.go b/internal/control/instances_internal_test.go new file mode 100644 index 0000000..2633470 --- /dev/null +++ b/internal/control/instances_internal_test.go @@ -0,0 +1,167 @@ +package control + +import ( + "os" + "path/filepath" + "testing" + + "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/profile" +) + +func instanceTestConfig(t *testing.T, conf string) *config.Config { + t.Helper() + root := t.TempDir() + dirs := config.Dirs{ + Config: filepath.Join(root, "config"), + State: filepath.Join(root, "state"), + Data: filepath.Join(root, "data"), + Profiles: filepath.Join(root, "config", "models"), + } + if err := os.MkdirAll(dirs.Profiles, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(dirs.ConfigFile(), []byte(conf), 0o600); err != nil { + t.Fatal(err) + } + f, err := config.ParseEnvFile(dirs.ConfigFile()) + if err != nil { + t.Fatal(err) + } + return &config.Config{Dirs: dirs, File: f} +} + +func writeInstance(t *testing.T, cfg *config.Config, name, port, since string) { + t.Helper() + if err := os.WriteFile(cfg.Dirs.ProfileFile(name), []byte("MODEL=org/"+name+"\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(cfg.Dirs.InstancesDir(), 0o755); err != nil { + t.Fatal(err) + } + body := "MODEL=org/" + name + "\nINSTANCE_SINCE=" + since + "\nPORT=" + port + "\n" + if err := os.WriteFile(cfg.Dirs.InstanceEnvFile(name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} + +func TestListInstancesOldestFirst(t *testing.T) { + cfg := instanceTestConfig(t, "PORT=8001\n") + if got, err := ListInstances(cfg); err != nil || got != nil { + t.Fatalf("without an instances dir: %v, %v", got, err) + } + writeInstance(t, cfg, "b", "8002", "200") + writeInstance(t, cfg, "a", "8001", "100") + writeInstance(t, cfg, "c", "8003", "200") + if got := LiveProfiles(cfg); len(got) != 3 || got[0] != "a" || got[1] != "b" || got[2] != "c" { + t.Errorf("LiveProfiles = %v, want [a b c]", got) + } + if err := RemoveInstance(cfg, "b"); err != nil { + t.Fatal(err) + } + if err := RemoveInstance(cfg, "b"); err != nil { + t.Errorf("removing a missing instance: %v", err) + } + if got := LiveProfiles(cfg); len(got) != 2 { + t.Errorf("after remove: %v", got) + } +} + +func TestWriteInstanceEnvKeepsStartTimeAndOptions(t *testing.T) { + cfg := instanceTestConfig(t, "PORT=8001\n") + writeInstance(t, cfg, "m", "8001", "100") + p, err := profile.Load(cfg.Dirs, "m") + if err != nil { + t.Fatal(err) + } + if err := WriteInstanceEnv(cfg, p, InstanceOpts{Port: "8004", KVCache: "8G"}); err != nil { + t.Fatal(err) + } + inst, err := LoadInstance(cfg, "m") + if err != nil { + t.Fatal(err) + } + if inst.Since.Unix() != 100 { + t.Errorf("restart reset the start time to %v", inst.Since) + } + if o := inst.Opts(); o.Port != "8004" || o.KVCache != "8G" || o.MemUtil != "" { + t.Errorf("opts = %+v", o) + } + if inst.Env.Get("KV_CACHE_MEMORY_BYTES") != "8G" { + t.Error("the KV cache override does not reach the server") + } + if inst.BaseURL() != "http://localhost:8004/v1" { + t.Errorf("BaseURL = %s", inst.BaseURL()) + } + + bad := &profile.Profile{Name: "Bad Name", File: &config.EnvFile{}} + if err := WriteInstanceEnv(cfg, bad, InstanceOpts{Port: "8005"}); err == nil { + t.Error("an invalid instance name was accepted") + } + if err := WriteInstanceEnv(cfg, p, InstanceOpts{}); err == nil { + t.Error("an instance without a port was accepted") + } +} + +func TestDefaultInstance(t *testing.T) { + cfg := instanceTestConfig(t, "PORT=8001\n") + if got := DefaultInstance(cfg, nil); got != "" { + t.Errorf("no instances: %q", got) + } + live := []string{"a", "b"} + if got := DefaultInstance(cfg, live); got != "a" { + t.Errorf("unset: %q, want the oldest", got) + } + cfg.File.Set("DEFAULT_PROFILE", "b") + if got := DefaultInstance(cfg, live); got != "b" { + t.Errorf("set and running: %q", got) + } + cfg.File.Set("DEFAULT_PROFILE", "gone") + if got := DefaultInstance(cfg, live); got != "a" { + t.Errorf("set but not running: %q, want the oldest", got) + } +} + +func TestAllocatePort(t *testing.T) { + busy := map[string]bool{"8002": true} + old := portFree + portFree = func(p string) bool { return !busy[p] } + t.Cleanup(func() { portFree = old }) + + cfg := instanceTestConfig(t, "PORT=8001\nROUTER_PORT=8003\n") + got, err := AllocatePort(cfg, "a", "") + if err != nil || got != "8001" { + t.Fatalf("first instance: %q, %v; want PORT", got, err) + } + writeInstance(t, cfg, "a", "8001", "100") + if got, _ := AllocatePort(cfg, "a", ""); got != "8001" { + t.Errorf("an instance keeps its port, got %q", got) + } + if got, _ := AllocatePort(cfg, "b", ""); got != "8004" { + t.Errorf("second instance got %q, want 8004 (8001 taken, 8002 listening, 8003 router)", got) + } + if _, err := AllocatePort(cfg, "b", "8001"); err == nil { + t.Error("--port on another instance's port was accepted") + } + if _, err := AllocatePort(cfg, "b", "nope"); err == nil { + t.Error("a non-numeric port was accepted") + } + if got, err := AllocatePort(cfg, "b", "9000"); err != nil || got != "9000" { + t.Errorf("--port 9000: %q, %v", got, err) + } +} + +func TestCheckRouterPort(t *testing.T) { + busy := map[string]bool{"8000": true, "8003": true} + old := portFree + portFree = func(p string) bool { return !busy[p] } + t.Cleanup(func() { portFree = old }) + + cfg := instanceTestConfig(t, "ROUTER_PORT=8003\n") + writeInstance(t, cfg, "a", "8001", "100") + for port, ok := range map[string]bool{"8001": false, "8000": false, "x": false, "8003": true, "8010": true} { + if err := CheckRouterPort(cfg, port); (err == nil) != ok { + t.Errorf("%s: err %v, want ok=%v", port, err, ok) + } + } +} diff --git a/internal/engine/registry.go b/internal/engine/registry.go index 5506a2b..3638443 100644 --- a/internal/engine/registry.go +++ b/internal/engine/registry.go @@ -21,7 +21,7 @@ import ( ) // Key is the config.env and profile setting that picks a provider. A profile's -// value wins over config.env's, because active.env lists the profile last. +// value wins over config.env's, because an instance's env lists the profile last. const Key = "PROVIDER" // Ids of shipped engines that Go code names: vLLM is the default, and the diff --git a/internal/plugin/coder/sync.go b/internal/plugin/coder/sync.go index 5edfc34..5c0ad17 100644 --- a/internal/plugin/coder/sync.go +++ b/internal/plugin/coder/sync.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "os" + "slices" "strconv" "strings" "time" @@ -13,6 +14,7 @@ import ( "github.com/spf13/cobra" "github.com/coder/llm-helper/internal/config" + "github.com/coder/llm-helper/internal/control" "github.com/coder/llm-helper/internal/engine" "github.com/coder/llm-helper/internal/plugin" "github.com/coder/llm-helper/internal/profile" @@ -48,17 +50,21 @@ func (Plugin) Configured(cfg *config.Config) bool { return cfg != nil && cfg.HasAll(keyURL, keyToken, keyProviderID) } -// Events: Coder needs to know which model is selectable, so it reacts to a -// switch and to a profile being deleted. It deliberately does not use -// EventReady -- see the comment on activate. +// Events: Coder needs to know which models are selectable. It does not use +// EventReady, which nothing raises under --no-follow. func (Plugin) Events() []plugin.Event { - return []plugin.Event{plugin.EventActivating, plugin.EventRemoved} + return []plugin.Event{plugin.EventStarting, plugin.EventStopped, plugin.EventDefault, plugin.EventRemoved} } func (p Plugin) Handle(_ context.Context, e plugin.Event, s plugin.State) error { switch e { - case plugin.EventActivating: - activate(s.Config, s.Profile) + case plugin.EventStarting, plugin.EventStopped, plugin.EventDefault: + on, off := reachable(s.Config, s.Live) + mirror(s.Config, on, s.Default) + if len(off) > 0 { + fmt.Printf("Coder sync: %s not offered: the Coder provider reaches only port %s. Run 'llm-helper router install-service' to offer every instance.\n", + strings.Join(off, ", "), s.Config.Port()) + } case plugin.EventRemoved: remove(s.Config, s.Name) default: @@ -282,7 +288,11 @@ func run(args []string, yes bool) error { } switch op { case "activate": - activate(cfg, p) + on, _ := reachable(cfg, control.LiveProfiles(cfg)) + if !slices.Contains(on, p.Name) { + on = append(on, p.Name) + } + mirror(cfg, on, p.Name) case "register": register(cfg, p) case "remove": @@ -396,40 +406,80 @@ func outputOptions(cfg *config.Config) map[string]any { return map[string]any{"max_output_tokens": n} } -// activate makes this profile the single enabled/default model in Coder and -// disables every other known profile's entry. Failures warn and continue -- -// Coder sync never blocks a local switch. -// -// This runs before the service restarts, so Coder briefly advertises a model -// that is still loading. Deferring it until the endpoint is healthy would -// need a hook that nothing raises under --no-follow, and Coder retries an -// unreachable provider for long enough to cover a normal load, so the entry -// goes in early and stays put. -// -// Disabling the others is not about is_default (Coder enforces a single -// default itself); it is because vLLM serves one model at a time, so any -// other entry names something this machine cannot answer for right now. -func activate(cfg *config.Config, p *profile.Profile) { +// reachable splits live instances into those the provider's one URL reaches +// (all of them through the router, otherwise the one on PORT) and the rest. +func reachable(cfg *config.Config, live []string) (on, off []string) { + if cfg.RouterPort() != "" { + return live, nil + } + for _, n := range live { + if inst, err := control.LoadInstance(cfg, n); err == nil && inst.Port == cfg.Port() { + on = append(on, n) + } else { + off = append(off, n) + } + } + return on, off +} + +// mirror copies the running instances into Coder: entries for on are enabled, +// def is the default, every other profile's entry is disabled. It runs +// before a starting server is up; Coder retries an unreachable provider for +// longer than a normal load. Failures warn and continue. +func mirror(cfg *config.Config, on []string, def string) { client := newClient(cfg) if client == nil { fmt.Println(notConfigured) return } + if !slices.Contains(on, def) { + def = "" + if len(on) > 0 { + def = on[0] + } + } + order := slices.DeleteFunc(slices.Clone(on), func(n string) bool { return n == def }) + if def != "" { + order = append(order, def) + } + state := ids(cfg) + keep := map[string]bool{} + var enabled []string + for _, name := range order { + p, err := profile.Load(cfg.Dirs, name) + if err != nil { + fmt.Fprintf(os.Stderr, "Coder sync: '%s': %v\n", name, err) + continue + } + if id := upsert(cfg, client, state, p, name == def); id != "" { + keep[id] = true + enabled = append(enabled, name) + } + } + disableExcept(cfg, client, state, keep) + switch { + case len(enabled) == 0: + fmt.Println("Coder sync: no model is enabled in Coder Agents.") + case len(enabled) == 1: + fmt.Printf("Coder sync: '%s' is the enabled/default model in Coder Agents.\n", enabled[0]) + default: + fmt.Printf("Coder sync: enabled %s in Coder Agents, '%s' is the default.\n", strings.Join(enabled, ", "), def) + } +} + +// upsert creates or updates p's entry and returns its ID, "" on failure. +func upsert(cfg *config.Config, client *Client, state *config.EnvFile, p *profile.Profile, isDefault bool) string { mc, err := modelConfig(cfg, p, true) if err != nil { fmt.Fprintln(os.Stderr, "Coder sync:", err) - return + return "" } - state := ids(cfg) + mc.IsDefault = isDefault - // A saved ID can name an entry that no longer exists. Recreating is what - // was wanted; refusing would leave the picker permanently stale. + // Nothing saved does not mean nothing exists: the picker is per model, + // so a renamed profile or a second machine would otherwise duplicate it. id := state.Get(p.Name) if id == "" { - // Nothing saved does not mean nothing exists. State is per profile - // name, but the picker is per model, so a renamed profile or a second - // machine syncing the same model would otherwise post a duplicate row - // and then fight with it over enabled/is_default. if adopted, err := client.FindByModel(mc.Model); err != nil { fmt.Fprintln(os.Stderr, "Coder sync: could not look for an existing entry:", err) } else if adopted != "" { @@ -440,16 +490,13 @@ func activate(cfg *config.Config, p *profile.Profile) { } } if id != "" { - fmt.Printf("Coder sync: updating model-config for '%s'...\n", p.Name) err := client.Update(id, mc) if err == nil { - state = disableOthers(cfg, client, state, p.Name) - fmt.Printf("Coder sync: '%s' is now the only enabled/default model in Coder Agents.\n", p.Name) - return + return id } if !errors.Is(err, ErrNotFound) { fmt.Fprintln(os.Stderr, "Coder sync: update failed, continuing without it:", err) - return + return "" } fmt.Printf("Coder sync: '%s' had no entry on this deployment, creating one...\n", p.Name) } else { @@ -458,32 +505,22 @@ func activate(cfg *config.Config, p *profile.Profile) { newID, err := client.Create(mc) if err != nil { fmt.Fprintln(os.Stderr, "Coder sync: create failed, continuing without it:", err) - return + return "" } state.Set(p.Name, newID) saveIDs(cfg, state) - - state = disableOthers(cfg, client, state, p.Name) - fmt.Printf("Coder sync: '%s' is now the only enabled/default model in Coder Agents.\n", p.Name) + return newID } -// disableOthers turns off every other profile's entry: vLLM serves one model -// at a time, so any other entry names something this machine cannot answer -// for. An entry that has already gone is forgotten rather than reported. -// -// Entries are owned by ID, not by profile name: adoption maps a second -// profile for the same model onto the existing row, and disabling that row on -// behalf of the other name turned off the entry this switch just enabled. -func disableOthers(cfg *config.Config, client *Client, state *config.EnvFile, keep string) *config.EnvFile { +// disableExcept turns off every profile's entry whose ID is not in keep. +// Entries are owned by ID, since adoption maps two profiles onto one row. +// An entry that has already gone is forgotten rather than reported. +func disableExcept(cfg *config.Config, client *Client, state *config.EnvFile, keep map[string]bool) { profiles, _ := profile.List(cfg.Dirs) - keepID := state.Get(keep) dropped := false for _, other := range profiles { - if other.Name == keep { - continue - } id := state.Get(other.Name) - if id == "" || id == keepID { + if id == "" || keep[id] { continue } switch err := client.Disable(id); { @@ -498,7 +535,6 @@ func disableOthers(cfg *config.Config, client *Client, state *config.EnvFile, ke if dropped { saveIDs(cfg, state) } - return state } // register creates a disabled entry for a profile that isn't active diff --git a/internal/plugin/coder/sync_internal_test.go b/internal/plugin/coder/sync_internal_test.go index 733ff00..6ee833f 100644 --- a/internal/plugin/coder/sync_internal_test.go +++ b/internal/plugin/coder/sync_internal_test.go @@ -154,7 +154,7 @@ func TestEntriesAreOwnedByID(t *testing.T) { state.Set("alias-b", "shared-id") state.Set("other", "other-id") - disableOthers(cfg, client, state, "alias-b") + disableExcept(cfg, client, state, map[string]bool{"shared-id": true}) if strings.Join(patched, ",") != "other-id" { t.Errorf("disabled %v, want only other-id: the shared entry is the one just enabled", patched) } diff --git a/internal/plugin/plugin.go b/internal/plugin/plugin.go index 12ee993..4d22baf 100644 --- a/internal/plugin/plugin.go +++ b/internal/plugin/plugin.go @@ -26,10 +26,15 @@ import ( type Event string const ( - // EventActivating fires after the active profile is recorded but before - // the service restarts, so a plugin can point elsewhere at the model - // that is about to load. - EventActivating Event = "activating" + // EventStarting fires after an instance is recorded but before its + // service (re)starts. + EventStarting Event = "starting" + + // EventStopped fires after an instance is stopped and its record removed. + EventStopped Event = "stopped" + + // EventDefault fires when the default instance changes. + EventDefault Event = "default" // EventReady fires once the endpoint answers /health. Only commands // that wait for the boot can raise it -- with --no-follow nothing is @@ -53,6 +58,11 @@ type State struct { // vllm-helper understands it. Plugins that register an endpoint (rather // than being told about one out of band, as Coder is) need it. BaseURL string + + // Live names every profile with an instance, oldest first, including + // one that is starting. Default is the one to offer first. + Live []string + Default string } // Plugin is one integration. diff --git a/internal/plugin/registry_internal_test.go b/internal/plugin/registry_internal_test.go index d640227..8a91fa2 100644 --- a/internal/plugin/registry_internal_test.go +++ b/internal/plugin/registry_internal_test.go @@ -44,17 +44,17 @@ func withRegistry(t *testing.T, plugins ...Plugin) { } func TestDispatchOnlyReachesPluginsThatOptedIn(t *testing.T) { - interested := &fake{name: "interested", configured: true, events: []Event{EventActivating, EventRemoved}} + interested := &fake{name: "interested", configured: true, events: []Event{EventStarting, EventRemoved}} // An integration with nothing to do on a switch shouldn't be called at // all -- that's what keeps an unused plugin off the critical path. uninterested := &fake{name: "uninterested", configured: true, events: []Event{EventRemoved}} - unconfigured := &fake{name: "unconfigured", configured: false, events: []Event{EventActivating}} + unconfigured := &fake{name: "unconfigured", configured: false, events: []Event{EventStarting}} withRegistry(t, interested, uninterested, unconfigured) var out strings.Builder - Dispatch(context.Background(), &out, EventActivating, State{}) + Dispatch(context.Background(), &out, EventStarting, State{}) - if len(interested.got) != 1 || interested.got[0] != EventActivating { + if len(interested.got) != 1 || interested.got[0] != EventStarting { t.Errorf("interested plugin saw %v, want one activating event", interested.got) } if len(uninterested.got) != 0 { @@ -71,12 +71,12 @@ func TestDispatchOnlyReachesPluginsThatOptedIn(t *testing.T) { // A failing integration must not stop the local operation, and the others // still have to run: switching models works with the network unplugged. func TestDispatchReportsFailuresAndKeepsGoing(t *testing.T) { - broken := &fake{name: "broken", configured: true, events: []Event{EventActivating}, err: errors.New("deployment unreachable")} - after := &fake{name: "after", configured: true, events: []Event{EventActivating}} + broken := &fake{name: "broken", configured: true, events: []Event{EventStarting}, err: errors.New("deployment unreachable")} + after := &fake{name: "after", configured: true, events: []Event{EventStarting}} withRegistry(t, broken, after) var out strings.Builder - Dispatch(context.Background(), &out, EventActivating, State{}) + Dispatch(context.Background(), &out, EventStarting, State{}) if len(after.got) != 1 { t.Error("a plugin registered after a failing one was skipped") @@ -102,14 +102,14 @@ func TestConfiguredFiltersTheRegistry(t *testing.T) { // still must not escape: activating dispatches before the service restarts, // so letting one through would stop the model from loading. func TestDispatchSurvivesAPanickingPlugin(t *testing.T) { - after := &fake{name: "after", configured: true, events: []Event{EventActivating}} + after := &fake{name: "after", configured: true, events: []Event{EventStarting}} withRegistry(t, - &fake{name: "panicky", configured: true, events: []Event{EventActivating}, panics: true}, + &fake{name: "panicky", configured: true, events: []Event{EventStarting}, panics: true}, after, ) var out strings.Builder - Dispatch(context.Background(), &out, EventActivating, State{}) + Dispatch(context.Background(), &out, EventStarting, State{}) if len(after.got) != 1 { t.Error("a plugin registered after a panicking one was skipped") @@ -126,9 +126,9 @@ func TestDispatchSurvivesAPanickingPlugin(t *testing.T) { // Warnings have nowhere to go in a caller that doesn't want them, and that // must not be a crash. func TestDispatchToleratesANilWriter(t *testing.T) { - p := &fake{name: "broken", configured: true, events: []Event{EventActivating}, err: errors.New("nope")} + p := &fake{name: "broken", configured: true, events: []Event{EventStarting}, err: errors.New("nope")} withRegistry(t, p) - Dispatch(context.Background(), nil, EventActivating, State{}) + Dispatch(context.Background(), nil, EventStarting, State{}) if len(p.got) != 1 { t.Error("the plugin was not called when the writer was nil") } diff --git a/internal/profile/profile.go b/internal/profile/profile.go index c871c65..77f856c 100644 --- a/internal/profile/profile.go +++ b/internal/profile/profile.go @@ -257,6 +257,23 @@ func kvCacheValue(n int64) string { return strconv.FormatInt(n, 10) } +// ValidInstanceName reports whether name can be run as an instance: it ends up +// in a systemd unit or launchd label, so it must need no escaping. +func ValidInstanceName(name string) error { + if name == "" { + return fmt.Errorf("empty profile name") + } + for i, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + case i > 0 && (r == '.' || r == '_' || r == '-'): + default: + return fmt.Errorf("profile name %q can't run as an instance: use lowercase letters, digits, '.', '_' and '-', starting with a letter or digit", name) + } + } + return nil +} + func Remove(dirs config.Dirs, name string) error { return os.Remove(dirs.ProfileFile(name)) } diff --git a/internal/profile/profile_test.go b/internal/profile/profile_test.go index 71ea0f4..04d0e0f 100644 --- a/internal/profile/profile_test.go +++ b/internal/profile/profile_test.go @@ -396,3 +396,17 @@ func TestCreateDefaultsForVLLMMetal(t *testing.T) { t.Errorf("GPU_MEMORY_UTILIZATION = %q, want %q", got, want) } } + +func TestValidInstanceName(t *testing.T) { + t.Parallel() + for _, ok := range []string{"qwen3.5-27b", "a", "0x_y"} { + if err := profile.ValidInstanceName(ok); err != nil { + t.Errorf("%q rejected: %v", ok, err) + } + } + for _, bad := range []string{"", "Qwen", "-a", ".a", "a b", "a/b", "a@b"} { + if profile.ValidInstanceName(bad) == nil { + t.Errorf("%q accepted", bad) + } + } +} diff --git a/internal/router/models.go b/internal/router/models.go new file mode 100644 index 0000000..a2b8999 --- /dev/null +++ b/internal/router/models.go @@ -0,0 +1,207 @@ +package router + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "sync" + "time" +) + +// Models sends each request to the backend that serves the model it names. +// It learns which backend serves what from each backend's own /v1/models. +type Models struct { + backends func() []string + client *http.Client + + mu sync.RWMutex + routes map[string]*url.URL + entries []json.RawMessage + urls []*url.URL + tried time.Time +} + +// NewModels returns a router over the base URLs (scheme://host:port) that +// backends returns. backends is called on every refresh. +func NewModels(backends func() []string) *Models { + return &Models{ + backends: backends, + client: &http.Client{Timeout: 5 * time.Second}, + routes: map[string]*url.URL{}, + } +} + +type modelList struct { + Data []json.RawMessage `json:"data"` +} + +type modelID struct { + ID string `json:"id"` +} + +// Refresh asks every backend for its models. When two backends serve the +// same id, the earlier one keeps it. +func (m *Models) Refresh(ctx context.Context) { + routes := map[string]*url.URL{} + var entries []json.RawMessage + var urls []*url.URL + for _, b := range m.backends() { + u, err := url.Parse(strings.TrimRight(b, "/")) + if err != nil { + continue + } + urls = append(urls, u) + for _, e := range m.fetch(ctx, u) { + var id modelID + if json.Unmarshal(e, &id) != nil || id.ID == "" { + continue + } + if _, dup := routes[id.ID]; dup { + continue + } + routes[id.ID] = u + entries = append(entries, e) + } + } + m.mu.Lock() + m.routes, m.entries, m.urls, m.tried = routes, entries, urls, time.Now() + m.mu.Unlock() +} + +func (m *Models) fetch(ctx context.Context, u *url.URL) []json.RawMessage { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String()+"/v1/models", nil) + if err != nil { + return nil + } + resp, err := m.client.Do(req) + if err != nil { + return nil + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil + } + var l modelList + if json.NewDecoder(resp.Body).Decode(&l) != nil { + return nil + } + return l.Data +} + +// StartRefresh refreshes now and then every interval until ctx is done. +func (m *Models) StartRefresh(ctx context.Context, every time.Duration) { + m.Refresh(ctx) + go func() { + t := time.NewTicker(every) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + m.Refresh(ctx) + } + } + }() +} + +// lookup finds the backend for model, refreshing once on a miss unless a +// refresh ran in the last second. +func (m *Models) lookup(ctx context.Context, model string) *url.URL { + m.mu.RLock() + u, recent := m.routes[model], time.Since(m.tried) < time.Second + only := len(m.urls) == 1 + var single *url.URL + if only { + single = m.urls[0] + } + m.mu.RUnlock() + if model == "" { + return single + } + if u != nil || recent { + return u + } + m.Refresh(ctx) + m.mu.RLock() + defer m.mu.RUnlock() + return m.routes[model] +} + +// Routes reports which backend serves each model. +func (m *Models) Routes() map[string]string { + m.mu.RLock() + defer m.mu.RUnlock() + out := make(map[string]string, len(m.routes)) + for id, u := range m.routes { + out[id] = u.String() + } + return out +} + +func (m *Models) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/models" && r.Method == http.MethodGet { + m.Refresh(r.Context()) + m.mu.RLock() + data := m.entries + m.mu.RUnlock() + if data == nil { + data = []json.RawMessage{} + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"object": "list", "data": data}) + return + } + + model := strings.TrimPrefix(r.URL.Path, "/v1/models/") + if model == r.URL.Path { + model = "" + if r.Body != nil { + body, err := io.ReadAll(r.Body) + if err != nil { + apiError(w, http.StatusBadRequest, "reading the request body: "+err.Error(), "") + return + } + r.Body.Close() + var req struct { + Model string `json:"model"` + } + _ = json.Unmarshal(body, &req) + model = req.Model + r.Body = io.NopCloser(bytes.NewReader(body)) + r.ContentLength = int64(len(body)) + } + } + + u := m.lookup(r.Context(), model) + if u == nil { + if model == "" { + apiError(w, http.StatusBadRequest, "the request names no model, and more than one is running", "") + return + } + apiError(w, http.StatusNotFound, fmt.Sprintf("no running instance serves the model %q", model), "model_not_found") + return + } + rx := httputil.NewSingleHostReverseProxy(u) + rx.FlushInterval = -1 + rx.ErrorHandler = func(w http.ResponseWriter, _ *http.Request, err error) { + apiError(w, http.StatusBadGateway, fmt.Sprintf("the instance serving %q did not answer: %v", model, err), "") + } + rx.ServeHTTP(w, r) +} + +func apiError(w http.ResponseWriter, status int, msg, code string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + e := map[string]any{"message": msg, "type": "invalid_request_error"} + if code != "" { + e["code"] = code + } + _ = json.NewEncoder(w).Encode(map[string]any{"error": e}) +} diff --git a/internal/router/models_internal_test.go b/internal/router/models_internal_test.go new file mode 100644 index 0000000..7f1ecbf --- /dev/null +++ b/internal/router/models_internal_test.go @@ -0,0 +1,121 @@ +package router + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +func fakeInstance(t *testing.T, name string, models ...string) *httptest.Server { + t.Helper() + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/models" { + var data []map[string]string + for _, m := range models { + data = append(data, map[string]string{"id": m, "object": "model"}) + } + _ = json.NewEncoder(w).Encode(map[string]any{"object": "list", "data": data}) + return + } + body, _ := io.ReadAll(r.Body) + _, _ = io.WriteString(w, name+" "+r.URL.Path+" "+string(body)) + })) + t.Cleanup(s.Close) + return s +} + +func do(t *testing.T, h http.Handler, method, path, body string) (int, string) { + t.Helper() + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(method, path, strings.NewReader(body))) + return rec.Code, rec.Body.String() +} + +func TestModelsDispatchesByModel(t *testing.T) { + a := fakeInstance(t, "a", "qwen") + b := fakeInstance(t, "b", "glm", "qwen") + m := NewModels(func() []string { return []string{a.URL, b.URL} }) + m.Refresh(context.Background()) + + body := `{"model":"glm","messages":[]}` + if code, got := do(t, m, http.MethodPost, "/v1/chat/completions", body); code != 200 || got != "b /v1/chat/completions "+body { + t.Errorf("glm: %d %q", code, got) + } + if _, got := do(t, m, http.MethodPost, "/v1/completions", `{"model":"qwen"}`); !strings.HasPrefix(got, "a ") { + t.Errorf("qwen should stay on the first instance: %q", got) + } + if _, got := do(t, m, http.MethodGet, "/v1/models/glm", ""); !strings.HasPrefix(got, "b ") { + t.Errorf("model path: %q", got) + } +} + +func TestModelsMergesList(t *testing.T) { + a := fakeInstance(t, "a", "qwen") + b := fakeInstance(t, "b", "glm", "qwen") + m := NewModels(func() []string { return []string{a.URL, b.URL} }) + + code, got := do(t, m, http.MethodGet, "/v1/models", "") + var l struct { + Data []struct { + ID string `json:"id"` + } `json:"data"` + } + if err := json.Unmarshal([]byte(got), &l); code != 200 || err != nil { + t.Fatalf("%d %q %v", code, got, err) + } + var ids []string + for _, d := range l.Data { + ids = append(ids, d.ID) + } + if strings.Join(ids, ",") != "qwen,glm" { + t.Errorf("ids = %v", ids) + } +} + +func TestModelsUnknownAndMissingModel(t *testing.T) { + a := fakeInstance(t, "a", "qwen") + b := fakeInstance(t, "b", "glm") + m := NewModels(func() []string { return []string{a.URL, b.URL} }) + m.Refresh(context.Background()) + + if code, got := do(t, m, http.MethodPost, "/v1/chat/completions", `{"model":"nope"}`); code != 404 || !strings.Contains(got, "model_not_found") { + t.Errorf("unknown model: %d %q", code, got) + } + if code, _ := do(t, m, http.MethodPost, "/v1/chat/completions", `{}`); code != 400 { + t.Errorf("no model with two instances: %d", code) + } + + one := NewModels(func() []string { return []string{a.URL} }) + one.Refresh(context.Background()) + if code, got := do(t, one, http.MethodPost, "/v1/chat/completions", `{}`); code != 200 || !strings.HasPrefix(got, "a ") { + t.Errorf("no model with one instance: %d %q", code, got) + } +} + +func TestModelsLearnsNewInstanceOnMiss(t *testing.T) { + a := fakeInstance(t, "a", "qwen") + backends := []string{a.URL} + m := NewModels(func() []string { return backends }) + m.Refresh(context.Background()) + m.tried = m.tried.Add(-2e9) + + b := fakeInstance(t, "b", "glm") + backends = append(backends, b.URL) + if _, got := do(t, m, http.MethodPost, "/v1/chat/completions", `{"model":"glm"}`); !strings.HasPrefix(got, "b ") { + t.Errorf("new instance: %q", got) + } +} + +func TestModelsBackendDown(t *testing.T) { + a := fakeInstance(t, "a", "qwen") + m := NewModels(func() []string { return []string{a.URL} }) + m.Refresh(context.Background()) + a.Close() + if code, _ := do(t, m, http.MethodPost, "/v1/chat/completions", `{"model":"qwen"}`); code != http.StatusBadGateway { + t.Errorf("down instance: %d", code) + } +} diff --git a/internal/service/service.go b/internal/service/service.go index de59a9c..2af783b 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -1,4 +1,4 @@ -// Package service manages the long-running vLLM server through whatever init +// Package service manages the long-running model servers through whatever init // system the host provides: systemd on Linux, launchd on macOS. Every // implementation honors the configured scope, "system" or "user". package service @@ -8,11 +8,19 @@ import ( "os/exec" ) +// ExitNoFit is the exit code of an instance whose model does not fit in the +// memory left. The init system does not restart it. +const ExitNoFit = 78 + // Service identifies the unit (systemd) or job (launchd) running the server. type Service struct { Name string // systemd unit name, or launchd label Scope string // "system" or "user" + // Args, when set, is what the unit runs after the binary instead of + // "start --instance ". + Args []string + // NonInteractive makes privileged calls fail fast (sudo -n) instead of // prompting for a password nothing can answer (scripts, the UI daemon). NonInteractive bool diff --git a/internal/service/service_darwin.go b/internal/service/service_darwin.go index 01c97f0..78a5706 100644 --- a/internal/service/service_darwin.go +++ b/internal/service/service_darwin.go @@ -12,6 +12,14 @@ import ( "strings" ) +const instancePrefix = "com.coder.llm-helper." + +// RouterName is the launchd label that runs the model router. +const RouterName = "com.coder.llm-helper-router" + +// InstanceName is the launchd label that runs profile's instance. +func InstanceName(profile string) string { return instancePrefix + profile } + // domain is the launchd domain the job lives in: the calling user's GUI // session for user scope, the system domain for system scope. func (s Service) domain() string { @@ -257,8 +265,8 @@ func (s Service) Installed() bool { return err == nil } -// Install writes the launchd plist pointing at " start", bootstraps it, -// and returns the path it wrote. +// Install writes the job's plist (an instance unless Args is set), +// bootstraps it, and returns the path it wrote. func (s Service) Install(self string) (string, error) { path := s.plistPath() plist, err := s.renderPlist(self) @@ -276,7 +284,7 @@ func (s Service) Install(self string) (string, error) { return "", err } } else { - tmp, err := os.CreateTemp("", "vllm-*.plist") + tmp, err := os.CreateTemp("", "llm-helper-*.plist") if err != nil { return "", err } @@ -329,6 +337,21 @@ func (s Service) Install(self string) (string, error) { return path, nil } +// Uninstall unloads the job and removes its plist. +func (s Service) Uninstall() error { + if err := s.Stop(); err != nil { + return err + } + if !s.Installed() { + return nil + } + if s.userScope() { + return os.Remove(s.plistPath()) + } + bin, args := s.sudo("rm", "-f", s.plistPath()) + return run(nil, bin, args...) +} + // explainBootstrap names the usual reason a LaunchAgent will not load: the // user has no GUI login session, which is the normal state of a headless Mac // reached over SSH. launchd reports it only as error 125. @@ -342,6 +365,14 @@ func (s Service) explainBootstrap(err error) error { } func (s Service) renderPlist(self string) ([]byte, error) { + cmd := s.Args + if len(cmd) == 0 { + profile, ok := strings.CutPrefix(s.Name, instancePrefix) + if !ok || profile == "" { + return nil, fmt.Errorf("%s is not an llm-helper instance label", s.Name) + } + cmd = []string{"start", "--instance", profile} + } esc := func(v string) (string, error) { var b bytes.Buffer if err := xml.EscapeText(&b, []byte(v)); err != nil { @@ -357,6 +388,14 @@ func (s Service) renderPlist(self string) ([]byte, error) { if err != nil { return nil, err } + var args strings.Builder + for _, a := range cmd { + v, err := esc(a) + if err != nil { + return nil, err + } + fmt.Fprintf(&args, "\t\t%s\n", v) + } log, err := esc(s.logPath()) if err != nil { return nil, err @@ -397,8 +436,7 @@ func (s Service) renderPlist(self string) ([]byte, error) { ProgramArguments %s - start - +%s RunAtLoad KeepAlive @@ -419,7 +457,7 @@ func (s Service) renderPlist(self string) ([]byte, error) { -`, label, prog, daemonUser, log, log, daemonHome) +`, label, prog, args.String(), daemonUser, log, log, daemonHome) return []byte(body), nil } diff --git a/internal/service/service_darwin_internal_test.go b/internal/service/service_darwin_internal_test.go index 55879b1..3cf8a78 100644 --- a/internal/service/service_darwin_internal_test.go +++ b/internal/service/service_darwin_internal_test.go @@ -39,17 +39,17 @@ func TestRenderPlist(t *testing.T) { home := t.TempDir() t.Setenv("HOME", home) - s := Service{Name: "com.coder.vllm", Scope: "user"} - raw, err := s.renderPlist("/opt/homebrew/bin/vllm-helper") + s := Service{Name: InstanceName("qwen"), Scope: "user"} + raw, err := s.renderPlist("/opt/homebrew/bin/llm-helper") if err != nil { t.Fatalf("renderPlist: %v", err) } p := parsePlist(t, raw) - if got := p.Dict.Strings[0]; got != "com.coder.vllm" { + if got := p.Dict.Strings[0]; got != "com.coder.llm-helper.qwen" { t.Errorf("Label = %q, want the service name", got) } - wantArgs := []string{"/opt/homebrew/bin/vllm-helper", "start"} + wantArgs := []string{"/opt/homebrew/bin/llm-helper", "start", "--instance", "qwen"} if strings.Join(p.Dict.Array.Strings, " ") != strings.Join(wantArgs, " ") { t.Errorf("ProgramArguments = %v, want %v", p.Dict.Array.Strings, wantArgs) } @@ -58,7 +58,7 @@ func TestRenderPlist(t *testing.T) { t.Errorf("plist is missing the %s key: %v", key, p.Dict.Keys) } } - log := filepath.Join(home, "Library", "Logs", "vllm-helper", "com.coder.vllm.log") + log := filepath.Join(home, "Library", "Logs", "vllm-helper", "com.coder.llm-helper.qwen.log") if !strings.Contains(string(raw), log) { t.Errorf("plist should redirect output to %s:\n%s", log, raw) } @@ -72,7 +72,7 @@ func TestRenderPlistRunsDaemonAsInstallingUser(t *testing.T) { if err != nil { t.Skipf("no current user: %v", err) } - raw, err := (Service{Name: "com.coder.vllm", Scope: "system"}).renderPlist("/usr/local/bin/vllm-helper") + raw, err := (Service{Name: InstanceName("qwen"), Scope: "system"}).renderPlist("/usr/local/bin/llm-helper") if err != nil { t.Fatalf("renderPlist: %v", err) } @@ -89,7 +89,7 @@ func TestRenderPlistRunsDaemonAsInstallingUser(t *testing.T) { // A LaunchAgent already runs as its user, so it names none. t.Setenv("HOME", t.TempDir()) - raw, err = (Service{Name: "com.coder.vllm", Scope: "user"}).renderPlist("/usr/local/bin/vllm-helper") + raw, err = (Service{Name: InstanceName("qwen"), Scope: "user"}).renderPlist("/usr/local/bin/llm-helper") if err != nil { t.Fatalf("renderPlist: %v", err) } @@ -104,8 +104,8 @@ func TestRenderPlistEscapesXML(t *testing.T) { t.Setenv("HOME", t.TempDir()) const ( - label = `a&b-"helper"` + label = instancePrefix + `a&b-"helper"` ) raw, err := (Service{Name: label, Scope: "user"}).renderPlist(self) if err != nil { @@ -121,6 +121,13 @@ func TestRenderPlistEscapesXML(t *testing.T) { } } +func TestRenderPlistRefusesNonInstanceLabels(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + if _, err := (Service{Name: "com.coder.vllm", Scope: "user"}).renderPlist("/usr/local/bin/llm-helper"); err == nil { + t.Error("renderPlist should refuse a label that names no instance") + } +} + func TestLastExitStatus(t *testing.T) { t.Parallel() @@ -173,12 +180,12 @@ func TestInstallUserScope(t *testing.T) { t.Setenv("HOME", home) t.Setenv("PATH", t.TempDir()) - s := Service{Name: "com.coder.vllm-test", Scope: "user"} + s := Service{Name: InstanceName("test"), Scope: "user"} if s.Installed() { t.Fatal("Installed() should be false before Install") } - _, err := s.Install("/opt/homebrew/bin/vllm-helper") + _, err := s.Install("/opt/homebrew/bin/llm-helper") if err == nil { t.Fatal("Install should report the failure to bootstrap when launchctl is absent") } @@ -197,7 +204,7 @@ func TestInstallUserScope(t *testing.T) { t.Fatalf("Install should have written %s: %v", s.plistPath(), err) } p := parsePlist(t, raw) - wantArgs := []string{"/opt/homebrew/bin/vllm-helper", "start"} + wantArgs := []string{"/opt/homebrew/bin/llm-helper", "start", "--instance", "test"} if strings.Join(p.Dict.Array.Strings, " ") != strings.Join(wantArgs, " ") { t.Errorf("ProgramArguments = %v, want %v", p.Dict.Array.Strings, wantArgs) } @@ -277,3 +284,14 @@ func TestLaunchctlArgs(t *testing.T) { }) } } + +func TestRenderPlistWithArgs(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + raw, err := (Service{Name: RouterName, Scope: "user", Args: []string{"router", "serve", "--port", "8000"}}).renderPlist("/usr/local/bin/llm-helper") + if err != nil { + t.Fatalf("renderPlist: %v", err) + } + if !strings.Contains(string(raw), "/usr/local/bin/llm-helper\n\t\trouter\n\t\tserve\n\t\t--port\n\t\t8000\n\t") { + t.Errorf("ProgramArguments:\n%s", raw) + } +} diff --git a/internal/service/service_linux.go b/internal/service/service_linux.go index 9a05169..04786c2 100644 --- a/internal/service/service_linux.go +++ b/internal/service/service_linux.go @@ -104,7 +104,7 @@ func healthFromProperties(props map[string]string) Health { } // Only unambiguous signals count. Result alone does not: it still holds // the previous run's verdict while a freshly restarted unit is - // activating, which would flag every normal switch as a crash. + // activating, which would flag every normal restart as a crash. switch { case h.State == "failed": // Gave up: hit the start limit, or a one-shot failure. @@ -180,16 +180,29 @@ func (s Service) LogHint() string { return fmt.Sprintf("journalctl -u %s -n 100", s.Name) } +// templateName is the unit file behind an instance unit: llm-helper@x.service +// is served by llm-helper@.service. +func templateName(name string) string { + if at := strings.Index(name, "@"); at >= 0 { + return name[:at] + "@.service" + } + return name +} + +// InstanceName is the unit that runs profile's instance. +func InstanceName(profile string) string { return "llm-helper@" + profile + ".service" } + func (s Service) unitPath() string { + name := templateName(s.Name) if s.userScope() { home, _ := os.UserHomeDir() base := os.Getenv("XDG_CONFIG_HOME") if base == "" { base = filepath.Join(home, ".config") } - return filepath.Join(base, "systemd", "user", s.Name) + return filepath.Join(base, "systemd", "user", name) } - return filepath.Join("/etc/systemd/system", s.Name) + return filepath.Join("/etc/systemd/system", name) } // Installed reports whether the unit file already exists. @@ -198,30 +211,54 @@ func (s Service) Installed() bool { return err == nil } -// Install renders and installs the unit, pointing it at " start", and -// returns the path it wrote. -func (s Service) Install(self string) (string, error) { +// RouterName is the unit that runs the model router. +const RouterName = "llm-helper-router.service" + +func (s Service) renderUnit(self string) string { userLine, wantedBy := "User="+os.Getenv("USER"), "multi-user.target" if s.userScope() { userLine, wantedBy = "", "default.target" } - unit := fmt.Sprintf(`[Unit] -Description=vLLM OpenAI-compatible server (active model set via llm-helper) + if len(s.Args) > 0 { + return fmt.Sprintf(`[Unit] +Description=llm-helper %s +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +%s +ExecStart=%s %s +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=%s +`, s.Args[0], userLine, self, strings.Join(s.Args, " "), wantedBy) + } + return fmt.Sprintf(`[Unit] +Description=llm-helper model instance %%i After=network-online.target Wants=network-online.target [Service] Type=simple %s -ExecStart=%s start +ExecStart=%s start --instance %%i Restart=on-failure RestartSec=5 +RestartPreventExitStatus=%d TimeoutStartSec=600 [Install] WantedBy=%s -`, userLine, self, wantedBy) +`, userLine, self, ExitNoFit, wantedBy) +} +// Install writes the unit (the instance template unless Args is set), +// enables s.Name, and returns the path it wrote. +func (s Service) Install(self string) (string, error) { + unit := s.renderUnit(self) path := s.unitPath() if s.userScope() { if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { @@ -238,22 +275,55 @@ WantedBy=%s return path, nil } - tmp, err := os.CreateTemp("", "vllm-unit-*.service") - if err != nil { - return "", err - } - defer os.Remove(tmp.Name()) - if _, err := tmp.WriteString(unit); err != nil { - return "", err + if old, err := os.ReadFile(path); err != nil || string(old) != unit { + tmp, err := os.CreateTemp("", "llm-helper-unit-*.service") + if err != nil { + return "", err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.WriteString(unit); err != nil { + return "", err + } + tmp.Close() + bin, args := s.sudo("cp", tmp.Name(), path) + if err := run(os.Stdout, bin, args...); err != nil { + return "", fmt.Errorf("installing unit (needs sudo): %w", err) + } + bin, args = s.sudo("chmod", "644", path) + run(nil, bin, args...) + bin, args = s.systemctlArgs("daemon-reload") + run(nil, bin, args...) } - tmp.Close() - if err := run(os.Stdout, "sudo", "cp", tmp.Name(), path); err != nil { - return "", fmt.Errorf("installing unit (needs sudo): %w", err) + bin, args := s.systemctlArgs("enable", s.Name) + if err := run(nil, bin, args...); err != nil { + return "", fmt.Errorf("enabling %s: %w", s.Name, err) } - run(nil, "sudo", "chmod", "644", path) - run(nil, "sudo", "systemctl", "daemon-reload") - run(nil, "sudo", "systemctl", "enable", s.Name) return path, nil } +// Uninstall stops and disables the unit. An instance unit leaves the shared +// template in place; any other unit file is removed. +func (s Service) Uninstall() error { + bin, args := s.systemctlArgs("disable", "--now", s.Name) + if err := run(nil, bin, args...); err != nil { + return fmt.Errorf("disabling %s: %w", s.Name, err) + } + if strings.Contains(s.Name, "@") || !s.Installed() { + return nil + } + if s.userScope() { + if err := os.Remove(s.unitPath()); err != nil { + return err + } + } else { + bin, args := s.sudo("rm", "-f", s.unitPath()) + if err := run(nil, bin, args...); err != nil { + return err + } + } + bin, args = s.systemctlArgs("daemon-reload") + run(nil, bin, args...) + return nil +} + func (s Service) Label() string { return "systemd" } diff --git a/internal/service/service_linux_internal_test.go b/internal/service/service_linux_internal_test.go index 888afa5..db0ac1a 100644 --- a/internal/service/service_linux_internal_test.go +++ b/internal/service/service_linux_internal_test.go @@ -33,6 +33,13 @@ func TestUnitPath(t *testing.T) { } }) + t.Run("an instance unit lives in its template", func(t *testing.T) { + got := (Service{Name: InstanceName("qwen"), Scope: "system"}).unitPath() + if want := "/etc/systemd/system/llm-helper@.service"; got != want { + t.Errorf("unitPath() = %q, want %q", got, want) + } + }) + t.Run("system scope", func(t *testing.T) { got := (Service{Name: "vllm.service", Scope: "system"}).unitPath() if want := "/etc/systemd/system/vllm.service"; got != want { @@ -130,12 +137,12 @@ func TestInstallUserScope(t *testing.T) { t.Setenv("PATH", t.TempDir()) t.Setenv("USER", "") - s := Service{Name: "vllm-test.service", Scope: "user"} + s := Service{Name: InstanceName("qwen"), Scope: "user"} if s.Installed() { t.Fatal("Installed() should be false before Install") } - path, err := s.Install("/opt/bin/vllm-helper") + path, err := s.Install("/opt/bin/llm-helper") if err != nil { t.Fatalf("Install: %v", err) } @@ -145,6 +152,9 @@ func TestInstallUserScope(t *testing.T) { if !s.Installed() { t.Error("Installed() should be true after Install") } + if !(Service{Name: InstanceName("other"), Scope: "user"}).Installed() { + t.Error("every instance shares the template unit") + } raw, err := os.ReadFile(path) if err != nil { @@ -152,8 +162,9 @@ func TestInstallUserScope(t *testing.T) { } unit := string(raw) for _, want := range []string{ - "ExecStart=/opt/bin/vllm-helper start", + "ExecStart=/opt/bin/llm-helper start --instance %i", "Restart=on-failure", + "RestartPreventExitStatus=78", "WantedBy=default.target", "Type=simple", } { @@ -165,3 +176,24 @@ func TestInstallUserScope(t *testing.T) { t.Errorf("a user-scope unit already runs as the user:\n%s", unit) } } + +func TestRenderUnitWithArgs(t *testing.T) { + t.Setenv("USER", "dev") + unit := (Service{Name: RouterName, Scope: "system", Args: []string{"router", "serve", "--port", "8000"}}).renderUnit("/opt/bin/llm-helper") + for _, want := range []string{ + "ExecStart=/opt/bin/llm-helper router serve --port 8000", + "Restart=on-failure", + "User=dev", + "WantedBy=multi-user.target", + } { + if !strings.Contains(unit, want) { + t.Errorf("unit is missing %q:\n%s", want, unit) + } + } + if strings.Contains(unit, "%i") || strings.Contains(unit, "RestartPreventExitStatus") { + t.Errorf("router unit should not be an instance template:\n%s", unit) + } + if (Service{Name: RouterName}).unitPath() != "/etc/systemd/system/"+RouterName { + t.Error("router unit is not a template") + } +} diff --git a/internal/service/service_other.go b/internal/service/service_other.go index 3a84e7c..9cda784 100644 --- a/internal/service/service_other.go +++ b/internal/service/service_other.go @@ -34,3 +34,10 @@ func (s Service) LogHint() string { return "" } func (s Service) Installed() bool { return false } func (s Service) Install(string) (string, error) { return "", errUnsupported } + +func (s Service) Uninstall() error { return errUnsupported } + +func InstanceName(profile string) string { return profile } + +// RouterName is the name the model router would run under. +const RouterName = "llm-helper-router" diff --git a/internal/web/activate.go b/internal/web/activate.go index 3f3bc45..b629cf6 100644 --- a/internal/web/activate.go +++ b/internal/web/activate.go @@ -6,35 +6,75 @@ package web import ( "bytes" "context" + "errors" + "fmt" + "net/http" "os" "path/filepath" + "strconv" "strings" "github.com/coder/llm-helper/internal/control" "github.com/coder/llm-helper/internal/profile" ) -// activate switches this machine to a saved profile. +// up starts a saved profile as an instance beside whatever already runs. +// Starting one that already runs restarts it with the new settings. // // The work itself is control's: the panel must not have its own idea of what -// switching means, or the same profile would serve differently depending on -// whether a browser or a terminal asked for it. -func (s *Server) activate(ctx context.Context, name string) ([]string, error) { +// running a profile means, or the same profile would serve differently +// depending on whether a browser or a terminal asked for it. +func (s *Server) up(ctx context.Context, name string, opts control.InstanceOpts) ([]string, int, error) { + if opts.MemUtil != "" { + if f, err := strconv.ParseFloat(opts.MemUtil, 64); err != nil || f <= 0 || f > 1 { + return nil, http.StatusBadRequest, fmt.Errorf("mem_util %q: want a fraction between 0 and 1", opts.MemUtil) + } + } + if _, err := os.Stat(s.cfg.Dirs.LegacyActiveEnvFile()); err == nil { + return nil, http.StatusConflict, errors.New("this machine still runs a model the old single-model way; run 'llm-helper migrate' first") + } p, err := profile.Load(s.cfg.Dirs, name) if err != nil { - return nil, err + return nil, http.StatusNotFound, err + } + if p.Generated() { + return nil, http.StatusConflict, errors.New(clusterControlReason) + } + svc := s.service(name) + if !svc.Installed() && s.cfg.ServiceScope() == "system" && os.Geteuid() != 0 { + return nil, http.StatusConflict, fmt.Errorf("the model service isn't installed yet; run 'llm-helper up %s' once from a terminal", name) + } + if err := control.CheckFitFor(s.cfg, name, opts.MemUtil, p.File, s.cfg.File); err != nil { + return nil, http.StatusConflict, err + } + port, err := control.AllocatePort(s.cfg, name, opts.Port) + if err != nil { + return nil, http.StatusConflict, err + } + opts.Port = port + if err := control.WriteInstanceEnv(s.cfg, p, opts); err != nil { + return nil, 0, err } - if err := control.WriteActiveEnv(s.cfg, p); err != nil { - return nil, err + inst, err := control.LoadInstance(s.cfg, name) + if err != nil { + return nil, 0, err + } + self, err := os.Executable() + if err != nil { + return nil, 0, err + } + install := func() error { _, err := svc.Install(self); return err } + if err := s.privilegedCall(ctx, name, install); err != nil { + _ = control.RemoveInstance(s.cfg, name) + return nil, 0, err } // Plugins write to an io.Writer rather than assuming a terminal, so // their warnings can be captured and shown in the panel. var pluginOut bytes.Buffer - restart := func() error { return s.privilegedCall(ctx, s.svc.Restart) } - err = control.NotifyAndRestart(ctx, s.cfg, p, &pluginOut, restart) - - return warningsFrom(&pluginOut), err + restart := func() error { return s.privilegedCall(ctx, name, svc.Restart) } + err = control.NotifyAndRestart(ctx, s.cfg, p, inst, &pluginOut, restart) + return warningsFrom(&pluginOut), 0, err } // warningsFrom turns captured plugin output into lines a page can show. diff --git a/internal/web/api.go b/internal/web/api.go index 80db119..0321f93 100644 --- a/internal/web/api.go +++ b/internal/web/api.go @@ -1,6 +1,7 @@ package web import ( + "bytes" "context" "encoding/json" "errors" @@ -14,6 +15,7 @@ import ( "github.com/coder/llm-helper/internal/control" "github.com/coder/llm-helper/internal/detect" "github.com/coder/llm-helper/internal/profile" + "github.com/coder/llm-helper/internal/service" ) // run executes a command and reports only whether it succeeded. Used for @@ -42,11 +44,23 @@ func writeJSON(w http.ResponseWriter, code int, v any) { // carry credentials, and an explicit DTO is the only shape where adding a // secret has to be a conscious act. type statusResponse struct { - Service string `json:"service"` - State string `json:"state"` + Instances []instanceStatus `json:"instances"` + // Default is the instance Coder offers first. + Default string `json:"default,omitempty"` + + Privileged bool `json:"privileged"` + PrivReason string `json:"priv_reason,omitempty"` + + Build string `json:"build"` + Host string `json:"host"` +} + +// instanceStatus is one running model. +type instanceStatus struct { Profile string `json:"profile"` Model string `json:"model"` Port string `json:"port"` + State string `json:"state"` Healthy bool `json:"healthy"` HTTPStatus int `json:"http_status"` Crashed bool `json:"crashed"` @@ -60,23 +74,23 @@ type statusResponse struct { // that is crash-looping wastes their afternoon. Phase string `json:"phase"` - Privileged bool `json:"privileged"` - PrivReason string `json:"priv_reason,omitempty"` - - // Cluster is set while a recipe cluster serves the active profile. The - // panel reports it but cannot drive it, so switch, restart and stop are - // refused and ControlReason says what to use instead. + // Cluster is set while a recipe cluster serves the instance. The panel + // reports it but cannot drive it, so its actions are refused and + // ControlReason says what to use instead. Cluster bool `json:"cluster"` ControlReason string `json:"control_reason,omitempty"` - Build string `json:"build"` - Host string `json:"host"` + Default bool `json:"default"` } func (s *Server) handleStatus(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusOK, s.status()) } +func (s *Server) handleInstances(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, s.instances()) +} + func (s *Server) status() statusResponse { // Re-probe only while we believe privilege is missing. Probing when we // have it would refresh the sudo timestamp on every poll and keep it @@ -86,55 +100,76 @@ func (s *Server) status() statusResponse { privileged, privReason = checkPrivilege(s.cfg) s.setPrivilege(privileged, privReason) } - cluster := s.activeCluster() - health, lastLogs, unit := s.svc.Health, s.svc.LastLogs, s.cfg.ServiceName() + r := statusResponse{ + Instances: s.instances(), + Privileged: privileged, + PrivReason: privReason, + Build: s.build, + Host: hostname(), + } + for _, i := range r.Instances { + if i.Default { + r.Default = i.Profile + } + } + return r +} + +func (s *Server) instances() []instanceStatus { + list, _ := control.ListInstances(s.cfg) + names := make([]string, 0, len(list)) + for _, inst := range list { + names = append(names, inst.Name) + } + def := control.DefaultInstance(s.cfg, names) + out := make([]instanceStatus, 0, len(list)) + for _, inst := range list { + out = append(out, s.instanceStatus(inst, inst.Name == def)) + } + return out +} + +func (s *Server) instanceStatus(inst *control.Instance, isDefault bool) instanceStatus { + cluster := s.cluster(inst.Name) + var health func() service.Health + var lastLogs func(int) string if cluster != nil { - health, lastLogs, unit = cluster.Health, cluster.LastLogs, "cluster" + health, lastLogs = cluster.Health, cluster.LastLogs + } else { + svc := s.service(inst.Name) + health, lastLogs = svc.Health, svc.LastLogs } h := health() - r := statusResponse{ - Service: unit, + r := instanceStatus{ + Profile: inst.Name, + Model: "unknown", + Port: inst.Port, State: h.State, Crashed: h.Crashed, Restarts: h.NRestarts, FailReason: h.Result, - Profile: s.cfg.ActiveProfile(), - Port: s.cfg.Port(), - Model: "unknown", - Privileged: privileged, - PrivReason: privReason, - Build: s.build, - Host: hostname(), + Default: isDefault, + } + if m := inst.Env.Get("MODEL"); m != "" { + r.Model = m } if cluster != nil { r.Cluster, r.ControlReason = true, clusterControlReason } - if active, err := control.ParseActiveEnv(s.cfg); err == nil { - if m := active.Get("MODEL"); m != "" { - r.Model = m - } - } - // Probing a unit that isn't running can only fail: vLLM binds the port - // after the weights are in memory, so a stopped unit means a closed - // port. Skipping it means a machine with the server stopped produces no - // probe traffic at all. - if h.State == "active" || h.State == "activating" { - r.HTTPStatus, r.Healthy = s.health() + // Probing a unit that isn't running can only fail: engines bind the + // port after the weights are in memory. + running := h.State == "active" || h.State == "activating" + if running { + r.HTTPStatus, r.Healthy = s.probe(inst) } - switch { case r.Healthy: r.Phase = "serving" case h.Crashed: r.Phase = "failing" r.LastLog = control.LastMeaningfulLine(lastLogs(40)) - case h.State == "active" || h.State == "activating": - // The port stays closed for the whole load -- vLLM binds only - // after the weights are in memory -- so an unresponsive endpoint - // on a running unit is the normal shape of a load in progress. + case running: r.Phase = "loading" - case r.Profile == "": - r.Phase = "idle" default: r.Phase = "stopped" } @@ -152,7 +187,7 @@ type profileResponse struct { MemUtil string `json:"gpu_memory_utilization"` ExtraArgs string `json:"extra_args"` Status string `json:"status"` - Active bool `json:"active"` + Running bool `json:"running"` } func (s *Server) handleProfiles(w http.ResponseWriter, _ *http.Request) { @@ -161,7 +196,10 @@ func (s *Server) handleProfiles(w http.ResponseWriter, _ *http.Request) { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()}) return } - active := s.cfg.ActiveProfile() + running := map[string]bool{} + for _, n := range control.LiveProfiles(s.cfg) { + running[n] = true + } out := make([]profileResponse, 0, len(profiles)) for _, p := range profiles { out = append(out, profileResponse{ @@ -172,7 +210,7 @@ func (s *Server) handleProfiles(w http.ResponseWriter, _ *http.Request) { MemUtil: p.File.Get("GPU_MEMORY_UTILIZATION"), ExtraArgs: p.ExtraArgs(), Status: p.Status(), - Active: p.Name == active, + Running: running[p.Name], }) } writeJSON(w, http.StatusOK, out) @@ -222,14 +260,25 @@ func (s *Server) handleLogs(w http.ResponseWriter, r *http.Request) { n = parsed } } - if c := s.activeCluster(); c != nil { + name := r.URL.Query().Get("instance") + if name == "" { + list, _ := control.ListInstances(s.cfg) + if len(list) != 1 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "name an instance with ?instance="}) + return + } + name = list[0].Name + } + if profile.ValidInstanceName(name) != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid instance name"}) + return + } + if c := s.cluster(name); c != nil { writeJSON(w, http.StatusOK, map[string]string{"logs": c.LastLogs(n), "hint": c.LogHint()}) return } - writeJSON(w, http.StatusOK, map[string]string{ - "logs": s.svc.LastLogs(n), - "hint": s.svc.LogHint(), - }) + svc := s.service(name) + writeJSON(w, http.StatusOK, map[string]string{"logs": svc.LastLogs(n), "hint": svc.LogHint()}) } // actionResponse is the result of something that changes state. @@ -239,48 +288,84 @@ type actionResponse struct { Warnings []string `json:"warnings,omitempty"` } -func (s *Server) handleSwitch(w http.ResponseWriter, r *http.Request) { +// action runs fn for one instance with the checks every action shares. +func (s *Server) action(w http.ResponseWriter, r *http.Request, fn func(ctx context.Context, name string) ([]string, int, error)) { name := r.PathValue("name") - if !s.requirePrivilege(w) { + if err := profile.ValidInstanceName(name); err != nil { + writeJSON(w, http.StatusBadRequest, actionResponse{Error: err.Error()}) + return + } + if !s.requirePrivilege(w, name) { return } - s.actionMu.Lock() - defer s.actionMu.Unlock() - defer s.forgetHealth() - warnings, err := s.activate(r.Context(), name) + defer s.lock(name)() + defer s.forgetHealth(name) + warnings, code, err := fn(r.Context(), name) if err != nil { - writeJSON(w, http.StatusInternalServerError, actionResponse{Error: err.Error(), Warnings: warnings}) + if code == 0 { + code = http.StatusInternalServerError + } + writeJSON(w, code, actionResponse{Error: err.Error(), Warnings: warnings}) return } writeJSON(w, http.StatusOK, actionResponse{OK: true, Warnings: warnings}) } +func (s *Server) handleUp(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + opts := control.InstanceOpts{Port: q.Get("port"), MemUtil: q.Get("mem_util")} + s.action(w, r, func(ctx context.Context, name string) ([]string, int, error) { + return s.up(ctx, name, opts) + }) +} + +func (s *Server) handleDown(w http.ResponseWriter, r *http.Request) { + s.action(w, r, func(ctx context.Context, name string) ([]string, int, error) { + if _, err := control.LoadInstance(s.cfg, name); err != nil { + return nil, http.StatusNotFound, fmt.Errorf("'%s' is not running", name) + } + if err := s.privilegedCall(ctx, name, s.service(name).Uninstall); err != nil { + return nil, 0, err + } + if err := control.RemoveInstance(s.cfg, name); err != nil { + return nil, 0, err + } + var out bytes.Buffer + control.NotifyStopped(ctx, s.cfg, name, &out) + return warningsFrom(&out), 0, nil + }) +} + func (s *Server) handleRestart(w http.ResponseWriter, r *http.Request) { - if !s.requirePrivilege(w) { - return - } - s.actionMu.Lock() - defer s.actionMu.Unlock() - defer s.forgetHealth() - if err := s.privilegedCall(r.Context(), s.svc.Restart); err != nil { - writeJSON(w, http.StatusInternalServerError, actionResponse{Error: err.Error()}) - return - } - writeJSON(w, http.StatusOK, actionResponse{OK: true}) + s.action(w, r, func(ctx context.Context, name string) ([]string, int, error) { + inst, err := control.LoadInstance(s.cfg, name) + if err != nil { + return nil, http.StatusNotFound, fmt.Errorf("'%s' is not running", name) + } + p, err := profile.Load(s.cfg.Dirs, name) + if err != nil { + return nil, 0, err + } + if err := control.WriteInstanceEnv(s.cfg, p, inst.Opts()); err != nil { + return nil, 0, err + } + return nil, 0, s.privilegedCall(ctx, name, s.service(name).Restart) + }) } -func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { - if !s.requirePrivilege(w) { - return - } - s.actionMu.Lock() - defer s.actionMu.Unlock() - defer s.forgetHealth() - if err := s.privilegedCall(r.Context(), s.svc.Stop); err != nil { - writeJSON(w, http.StatusInternalServerError, actionResponse{Error: err.Error()}) - return - } - writeJSON(w, http.StatusOK, actionResponse{OK: true}) +func (s *Server) handleDefault(w http.ResponseWriter, r *http.Request) { + s.action(w, r, func(ctx context.Context, name string) ([]string, int, error) { + if _, err := control.LoadInstance(s.cfg, name); err != nil { + return nil, http.StatusNotFound, fmt.Errorf("'%s' is not running", name) + } + s.cfg.File.Set("DEFAULT_PROFILE", name) + if err := s.cfg.File.Write(s.cfg.Dirs.ConfigFile(), 0o644); err != nil { + return nil, 0, err + } + var out bytes.Buffer + control.NotifyDefault(ctx, s.cfg, &out) + return warningsFrom(&out), 0, nil + }) } // explain turns a failed systemctl call into something actionable. @@ -291,23 +376,26 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { // buttons were enabled, and by the time one was pressed the credential was // gone. Re-checking on failure costs one fork and turns a meaningless error // into the sudoers line that fixes it. -func (s *Server) explain(err error) error { +func (s *Server) explain(name string, err error) error { if ok, reason := checkPrivilege(s.cfg); !ok { s.setPrivilege(false, reason) return errors.New(reason) } - return fmt.Errorf("%w -- see 'journalctl -u %s'", err, s.cfg.ServiceName()) + if hint := s.service(name).LogHint(); hint != "" { + return fmt.Errorf("%w -- see '%s'", err, hint) + } + return err } -func (s *Server) requirePrivilege(w http.ResponseWriter) bool { - return s.requireNative(w) && s.requireSudo(w) +func (s *Server) requirePrivilege(w http.ResponseWriter, name string) bool { + return s.requireNative(w, name) && s.requireSudo(w) } -// requireNative refuses a service action while a recipe cluster serves: the -// panel's verbs drive the native unit, so a restart would start vLLM beside -// the cluster and a stop would report success while the cluster kept serving. -func (s *Server) requireNative(w http.ResponseWriter) bool { - if s.activeCluster() == nil { +// requireNative refuses an action on an instance a recipe cluster serves: the +// panel's verbs drive the instance's own unit, so a restart would start a +// second server beside the cluster. +func (s *Server) requireNative(w http.ResponseWriter, name string) bool { + if s.cluster(name) == nil { return true } writeJSON(w, http.StatusConflict, actionResponse{Error: clusterControlReason}) @@ -326,13 +414,13 @@ func (s *Server) requireSudo(w http.ResponseWriter) bool { // privilegedCall bounds a systemctl call in time. Nothing in the service // package takes a context, and a systemctl that blocks would otherwise hold // an HTTP request open indefinitely. -func (s *Server) privilegedCall(ctx context.Context, fn func() error) error { +func (s *Server) privilegedCall(ctx context.Context, name string, fn func() error) error { done := make(chan error, 1) go func() { done <- fn() }() select { case err := <-done: if err != nil { - return s.explain(err) + return s.explain(name, err) } return nil case <-ctx.Done(): diff --git a/internal/web/docssite.go b/internal/web/docssite.go index fc81956..8c2abe4 100644 --- a/internal/web/docssite.go +++ b/internal/web/docssite.go @@ -19,7 +19,7 @@ import ( // docsSiteSummary is the one-line description llms.txt leads with. It is the // README's tagline. -const docsSiteSummary = "Serve large models with vLLM, mlx-lm or llama.cpp on one GPU machine or several, and switch models with one command." +const docsSiteSummary = "Serve large models with vLLM, mlx-lm or llama.cpp on one GPU machine or several, and run several models at once." // DocsSiteOptions configures NewDocsSite. type DocsSiteOptions struct { diff --git a/internal/web/models.go b/internal/web/models.go index eeaefe7..d5fce49 100644 --- a/internal/web/models.go +++ b/internal/web/models.go @@ -178,7 +178,8 @@ type pullResponse struct { Warnings []string `json:"warnings,omitempty"` } -// handlePull saves a profile for a model and switches to it. +// handlePull saves a profile for a model and starts it beside whatever +// already runs. // // No weights are downloaded here. vLLM fetches them itself when it starts, // which is why this returns as soon as the service has been told to restart: @@ -190,10 +191,6 @@ func (s *Server) handlePull(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) return } - if c := s.activeCluster(); c != nil { - writeJSON(w, http.StatusConflict, map[string]string{"error": clusterControlReason}) - return - } if privileged, reason := s.privilege(); !privileged { writeJSON(w, http.StatusForbidden, map[string]string{"error": reason}) return @@ -261,9 +258,12 @@ func (s *Server) handlePull(w http.ResponseWriter, r *http.Request) { } out := pullResponse{OK: true, Profile: name} - s.actionMu.Lock() - defer s.actionMu.Unlock() - defer s.forgetHealth() + if err := profile.ValidInstanceName(name); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) + return + } + defer s.lock(name)() + defer s.forgetHealth(name) // An existing profile is reused rather than overwritten. Someone may // have edited it into something that works, and silently replacing that @@ -290,7 +290,7 @@ func (s *Server) handlePull(w http.ResponseWriter, r *http.Request) { } out.Existing = true out.Notes = append(out.Notes, - "Profile '"+name+"' already existed -- switching to it with its saved settings.") + "Profile '"+name+"' already existed -- running it with its saved settings.") default: var caps vllm.Caps if prov.Capabilities().VLLMArgs { @@ -308,12 +308,15 @@ func (s *Server) handlePull(w http.ResponseWriter, r *http.Request) { } } - warnings, err := s.activate(r.Context(), name) + warnings, code, err := s.up(r.Context(), name, control.InstanceOpts{}) if err != nil { - // The profile is saved either way: a failed restart is worth + // The profile is saved either way: a failed start is worth // reporting, but it does not make the profile wrong. - writeJSON(w, http.StatusInternalServerError, map[string]string{ - "error": "saved profile '" + name + "' but could not switch to it: " + err.Error(), + if code == 0 { + code = http.StatusInternalServerError + } + writeJSON(w, code, map[string]string{ + "error": "saved profile '" + name + "' but could not start it: " + err.Error(), }) return } @@ -325,17 +328,16 @@ func (s *Server) handlePull(w http.ResponseWriter, r *http.Request) { // configured. func (s *Server) handleRemoveProfile(w http.ResponseWriter, r *http.Request) { name := r.PathValue("name") - s.actionMu.Lock() - defer s.actionMu.Unlock() + defer s.lock(name)() if _, err := profile.Load(s.cfg.Dirs, name); err != nil { // Load resolves the name against the profiles directory, so this // also rejects anything that isn't a profile that exists. writeJSON(w, http.StatusNotFound, map[string]string{"error": err.Error()}) return } - if name == s.cfg.ActiveProfile() { + if _, err := control.LoadInstance(s.cfg, name); err == nil { writeJSON(w, http.StatusConflict, map[string]string{ - "error": "'" + name + "' is the active model -- switch to something else first", + "error": "'" + name + "' is running -- stop it first", }) return } @@ -413,8 +415,7 @@ func (s *Server) handleGGUF(w http.ResponseWriter, _ *http.Request) { // people towards running the whole panel as root, which is worse. // // It also cannot disturb anything running. The plugin is read when an engine -// starts, so it takes effect on the next switch -- unlike every other mutation -// here, this one leaves the current model alone. +// starts, so it leaves every running instance alone. func (s *Server) handleInstallGGUF(w http.ResponseWriter, _ *http.Request) { if s.caps().SupportsGGUF() { writeJSON(w, http.StatusOK, map[string]any{ diff --git a/internal/web/server.go b/internal/web/server.go index 951e5db..72ac7e8 100644 --- a/internal/web/server.go +++ b/internal/web/server.go @@ -34,14 +34,14 @@ import ( // Server holds everything the handlers need. type Server struct { cfg *config.Config - svc service.Service token string - // cluster reports the recipe cluster serving the active profile, or nil - // when the native unit owns it. The CLI owns that lookup, so it is passed - // in; asked per request because a switch from the terminal can change the - // answer under a running panel. - cluster func() Cluster + // clusterFor reports the recipe cluster serving an instance, or nil when + // its own unit does. The CLI owns that lookup, so it is passed in. + clusterFor func(name string) Cluster + + // svcFor builds the unit for an instance; tests replace it. + svcFor func(name string) service.Service // build identifies this binary in the UI, passed in because the CLI // package owns the ldflag-stamped values. @@ -50,29 +50,19 @@ type Server struct { // privileged reports whether service control can actually work. On a // system-scope unit the daemon needs passwordless sudo; without it the // UI shows the controls as unavailable instead of offering buttons - // that fail with "exit status 1". - // - // It is re-evaluated while it is false, because a sudo timestamp can - // appear or expire under a running panel, so it is guarded: handlers - // run concurrently. + // that fail with "exit status 1". Re-evaluated while it is false. mu sync.RWMutex privileged bool privReason string - // actionMu serializes everything that changes the service or the set of - // profiles. Two tabs, or one impatient double click, otherwise write the - // same active.env.tmp at once and race its rename, or restart a profile - // another request has just removed. - actionMu sync.Mutex - - // The last health probe, reused for healthProbeTTL. Every probe is a - // request vLLM writes to its own access log, so the panel's polling is - // visible to anyone reading it -- and several open tabs must not turn - // into several streams of it. - healthMu sync.Mutex - healthAt time.Time - healthCode int - healthOK bool + // actionMu holds one mutex per profile name, so two requests never + // change the same instance at once while different instances proceed. + actionMu sync.Map + + // Health probes, reused for healthProbeTTL per instance: every probe + // lands in the engine's access log. + healthMu sync.Mutex + health map[string]healthEntry // install tracks a GGUF plugin install, which builds from source and runs // for minutes. It is deliberately not a blocking request: a POST held open @@ -123,36 +113,60 @@ func (s *Server) finishInstall(err error, log string) { } } -// healthProbeTTL bounds how often the vLLM endpoint is asked how it is. The -// panel polls in seconds and a model loads in minutes, so a result this stale -// is indistinguishable from a fresh one. +type healthEntry struct { + at time.Time + code int + ok bool +} + +// healthProbeTTL bounds how often an endpoint is asked how it is. The panel +// polls in seconds and a model loads in minutes. const healthProbeTTL = 5 * time.Second -// health probes the endpoint, or returns a recent answer instead. -// -// The lock is held across the request deliberately: concurrent callers should -// queue behind one probe and share its result, which is the whole point. -func (s *Server) health() (int, bool) { +// probe asks an instance's endpoint, or returns a recent answer instead. The +// lock is held across the request so concurrent callers share one probe. +func (s *Server) probe(inst *control.Instance) (int, bool) { s.healthMu.Lock() defer s.healthMu.Unlock() - if !s.healthAt.IsZero() && time.Since(s.healthAt) < healthProbeTTL { - return s.healthCode, s.healthOK + if e, ok := s.health[inst.Name]; ok && time.Since(e.at) < healthProbeTTL { + return e.code, e.ok + } + code, ok := inst.Probe() + if s.health == nil { + s.health = map[string]healthEntry{} } - code, ok := control.ProbeActive(s.cfg) - s.healthAt, s.healthCode, s.healthOK = time.Now(), code, ok + s.health[inst.Name] = healthEntry{at: time.Now(), code: code, ok: ok} return code, ok } -// forgetHealth drops the cached probe, so the next status request asks the -// process that is actually running now. -// -// Without this a refresh straight after a switch can report the new profile as -// serving on the old process's cached 200, pick the slow polling interval on -// the strength of it, and hide the first seconds of a load that takes minutes. -func (s *Server) forgetHealth() { +// forgetHealth drops an instance's cached probe, so the next status request +// asks the process that is actually running now. +func (s *Server) forgetHealth(name string) { s.healthMu.Lock() defer s.healthMu.Unlock() - s.healthAt = time.Time{} + delete(s.health, name) +} + +// lock serializes changes to one instance. +func (s *Server) lock(name string) func() { + m, _ := s.actionMu.LoadOrStore(name, &sync.Mutex{}) + mu := m.(*sync.Mutex) + mu.Lock() + return mu.Unlock +} + +// service is the unit that runs an instance. +func (s *Server) service(name string) service.Service { + if s.svcFor != nil { + return s.svcFor(name) + } + // NonInteractive always: a server never has anyone to answer a sudo + // prompt, and a bare sudo would block with a request still open. + return service.Service{ + Name: service.InstanceName(name), + Scope: s.cfg.ServiceScope(), + NonInteractive: true, + } } // caps describes what the installed vLLM will accept, so a model card's flags @@ -187,17 +201,6 @@ func New(cfg *config.Config, build string) (*Server, error) { cfg: cfg, build: build, token: token, - // Deliberately not cli.svc(): that derives NonInteractive from - // whether stdin is a terminal, and a daemon started from a shell - // would inherit NonInteractive=false. Its first restart would run - // bare `sudo`, inherit the terminal's stdin, and block forever - // with an HTTP request still open. A server never has anyone to - // answer a password prompt, so it always fails fast instead. - svc: service.Service{ - Name: cfg.ServiceName(), - Scope: cfg.ServiceScope(), - NonInteractive: true, - }, } ok, reason := checkPrivilege(cfg) s.setPrivilege(ok, reason) @@ -215,18 +218,17 @@ type Cluster interface { // clusterControlReason is why the controls are off while a cluster serves. const clusterControlReason = "A recipe cluster is serving this profile. The panel shows its status " + - "but can't control it yet: use 'llm-helper restart', 'stop' or 'switch' on the command line." + "but can't control it yet: use 'llm-helper restart' or 'down' on the command line." -// SetCluster tells the panel how to find a recipe cluster serving the active -// profile. fn returns nil when the native unit owns it. -func (s *Server) SetCluster(fn func() Cluster) { s.cluster = fn } +// SetCluster tells the panel how to find the recipe cluster serving an +// instance. fn returns nil when the instance's own unit serves it. +func (s *Server) SetCluster(fn func(name string) Cluster) { s.clusterFor = fn } -// activeCluster is the cluster serving the active profile, or nil. -func (s *Server) activeCluster() Cluster { - if s.cluster == nil { +func (s *Server) cluster(name string) Cluster { + if s.clusterFor == nil { return nil } - return s.cluster() + return s.clusterFor(name) } // checkPrivilege works out up front whether restart/stop can succeed, so the @@ -240,29 +242,23 @@ func checkPrivilege(cfg *config.Config) (bool, string) { } if runtime.GOOS != "linux" { // launchd's system domain is root's, and there is no per-command - // rule to probe for the way there is with sudoers. Saying so beats - // asking systemctl about a machine that has none. - return false, fmt.Sprintf( - "%s runs in system scope, which belongs to root. Start the panel with sudo, "+ - "or run the service in user scope.", cfg.ServiceName()) + // rule to probe for the way there is with sudoers. + return false, "The model services run in system scope, which belongs to root. " + + "Start the panel with sudo, or run the services in user scope." } - // Ask sudo whether these exact commands are permitted, rather than whether - // sudo works at all. A sensible rule grants only the systemctl verbs the - // panel needs, and under such a rule `sudo -n true` fails -- which would - // report the panel as powerless while its buttons would in fact have - // worked. - // - // Both verbs the panel offers are checked, because one answer drives both - // buttons: a rule granting restart alone would light up a Stop button that - // then fails with a bare exit status, which is the thing this check exists - // to prevent. + // Ask sudo whether the exact commands are permitted rather than whether + // sudo works at all: a rule granting only these verbs makes `sudo -n + // true` fail. Every verb the panel uses is checked, against a sample + // instance unit, so a wildcard rule is what passes. systemctl, err := exec.LookPath("systemctl") if err != nil { systemctl = "/usr/bin/systemctl" } + unit := service.InstanceName("probe") allowed := true - for _, verb := range []string{"restart", "stop"} { - if err := run("sudo", "-n", "-l", systemctl, verb, cfg.ServiceName()); err != nil { + for _, verb := range [][]string{{"restart"}, {"enable"}, {"disable", "--now"}} { + args := append(append([]string{"-n", "-l", systemctl}, verb...), unit) + if err := run("sudo", args...); err != nil { allowed = false break } @@ -270,11 +266,13 @@ func checkPrivilege(cfg *config.Config) (bool, string) { if allowed { return true, "" } + pattern := service.InstanceName("*") return false, fmt.Sprintf( - "%s runs in system scope, and this user can't run systemctl without a password. "+ - "Start the server with sudo, or allow it for this one unit:\n"+ - " %s ALL=(root) NOPASSWD: /usr/bin/systemctl restart %s, /usr/bin/systemctl stop %s, /usr/bin/systemctl start %s", - cfg.ServiceName(), currentUser(), cfg.ServiceName(), cfg.ServiceName(), cfg.ServiceName()) + "The model services run in system scope, and this user can't run systemctl without a password. "+ + "Start the server with sudo, or allow it for the model units:\n"+ + " %s ALL=(root) NOPASSWD: /usr/bin/systemctl restart %s, /usr/bin/systemctl enable %s, "+ + "/usr/bin/systemctl disable --now %s", + currentUser(), pattern, pattern, pattern) } // Handler builds the routes. @@ -292,9 +290,11 @@ func (s *Server) Handler(static fs.FS) http.Handler { mux.HandleFunc("POST /api/gguf/install", s.guard(s.handleInstallGGUF)) mux.HandleFunc("POST /api/pull", s.guard(s.handlePull)) mux.HandleFunc("POST /api/profiles/{name}/remove", s.guard(s.handleRemoveProfile)) - mux.HandleFunc("POST /api/switch/{name}", s.guard(s.handleSwitch)) - mux.HandleFunc("POST /api/restart", s.guard(s.handleRestart)) - mux.HandleFunc("POST /api/stop", s.guard(s.handleStop)) + mux.HandleFunc("GET /api/instances", s.guard(s.handleInstances)) + mux.HandleFunc("POST /api/instances/{name}/up", s.guard(s.handleUp)) + mux.HandleFunc("POST /api/instances/{name}/down", s.guard(s.handleDown)) + mux.HandleFunc("POST /api/instances/{name}/restart", s.guard(s.handleRestart)) + mux.HandleFunc("POST /api/instances/{name}/default", s.guard(s.handleDefault)) // Real pages rather than one document that hides half of itself: // watching a machine and choosing a model are separate jobs, and each @@ -415,7 +415,7 @@ func (s *Server) Token() string { return s.token } // loadOrCreateToken keeps the token in the state dir rather than // secrets.env, because everything in secrets.env is inlined into the -// generated active.env and handed to vLLM's environment. A credential for +// generated instance env and handed to vLLM's environment. A credential for // this panel has no business being in the model server's environment. func loadOrCreateToken(stateDir string) (string, error) { path := filepath.Join(stateDir, "web-token") diff --git a/internal/web/server_internal_test.go b/internal/web/server_internal_test.go index 892bd6d..71ed85e 100644 --- a/internal/web/server_internal_test.go +++ b/internal/web/server_internal_test.go @@ -40,7 +40,7 @@ func newTestServer(t *testing.T) *Server { t.Fatal(err) } } - write(dirs.ConfigFile(), "PORT=8001\nSERVICE_NAME=vllm.service\nSERVICE_SCOPE=user\n") + write(dirs.ConfigFile(), "PORT=8001\nSERVICE_SCOPE=user\n") write(dirs.SecretsFile(), "VLLM_API_KEY=sk-do-not-leak-me\nCODER_API_TOKEN=coder-do-not-leak\nHF_TOKEN=hf_do_not_leak\n") write(dirs.ProfileFile("demo"), "MODEL=org/demo\nMAX_MODEL_LEN=4096\n") @@ -59,15 +59,29 @@ func newTestServer(t *testing.T) *Server { t.Fatal(err) } s := &Server{ - cfg: cfg, - build: "test", - token: token, - svc: service.Service{Name: "vllm.service", Scope: "user", NonInteractive: true}, + cfg: cfg, + build: "test", + token: token, + svcFor: func(name string) service.Service { + return service.Service{Name: service.InstanceName(name), Scope: "user", NonInteractive: true} + }, privileged: true, } return s } +// writeInstance records name as running on port, the way 'up' leaves it. +func writeInstance(t *testing.T, s *Server, name, port, extra string) { + t.Helper() + if err := os.MkdirAll(s.cfg.Dirs.InstancesDir(), 0o755); err != nil { + t.Fatal(err) + } + body := "MODEL=org/" + name + "\nINSTANCE_SINCE=100\nPORT=" + port + "\n" + extra + if err := os.WriteFile(s.cfg.Dirs.InstanceEnvFile(name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } +} + // The panel can stop a server and rewrite profiles, so an unauthenticated // request must not reach a handler even on loopback. func TestUnauthenticatedRequestsAreRejected(t *testing.T) { @@ -118,7 +132,7 @@ func TestMutationsNeedTheTokenInAHeader(t *testing.T) { s := newTestServer(t) h := s.Handler(Static()) - cookieOnly := httptest.NewRequest(http.MethodPost, "/api/restart", nil) + cookieOnly := httptest.NewRequest(http.MethodPost, "/api/instances/demo/restart", nil) cookieOnly.AddCookie(&http.Cookie{Name: tokenCookie, Value: s.token}) rec := httptest.NewRecorder() h.ServeHTTP(rec, cookieOnly) @@ -126,7 +140,7 @@ func TestMutationsNeedTheTokenInAHeader(t *testing.T) { t.Errorf("POST with only a cookie = %d, want 403", rec.Code) } - queryOnly := httptest.NewRequest(http.MethodPost, "/api/restart?token="+s.token, nil) + queryOnly := httptest.NewRequest(http.MethodPost, "/api/instances/demo/restart?token="+s.token, nil) rec = httptest.NewRecorder() h.ServeHTTP(rec, queryOnly) if rec.Code != http.StatusForbidden { @@ -187,10 +201,11 @@ func TestProfileHoldsDoesNotHandOneOwnersProfileToAnother(t *testing.T) { // really does hold secrets. func TestNoResponseLeaksACredential(t *testing.T) { s := newTestServer(t) + writeInstance(t, s, "demo", "1", "VLLM_API_KEY=sk-do-not-leak-me\n") h := s.Handler(Static()) secrets := []string{"sk-do-not-leak-me", "coder-do-not-leak", "hf_do_not_leak"} - for _, path := range []string{"/api/status", "/api/profiles", "/api/machine", "/api/logs"} { + for _, path := range []string{"/api/status", "/api/instances", "/api/profiles", "/api/machine", "/api/logs?instance=demo"} { r := httptest.NewRequest(http.MethodGet, path, nil) r.Header.Set(tokenHeader, s.token) rec := httptest.NewRecorder() @@ -206,7 +221,7 @@ func TestNoResponseLeaksACredential(t *testing.T) { } // The token lives in the state dir rather than secrets.env because -// everything in secrets.env is inlined into active.env and handed to vLLM's +// everything in secrets.env is inlined into each instance env and handed to vLLM's // environment. A panel credential has no business being there. func TestTokenIsStoredPrivatelyAndReused(t *testing.T) { dir := t.TempDir() @@ -240,7 +255,7 @@ func TestActionsRefuseWithoutPrivilege(t *testing.T) { s.setPrivilege(false, "needs passwordless sudo") h := s.Handler(Static()) - r := httptest.NewRequest(http.MethodPost, "/api/restart", nil) + r := httptest.NewRequest(http.MethodPost, "/api/instances/demo/restart", nil) r.Header.Set(tokenHeader, s.token) rec := httptest.NewRecorder() h.ServeHTTP(rec, r) @@ -297,7 +312,11 @@ func TestHealthProbeIsReusedBetweenCallers(t *testing.T) { defer vllm.Close() s := newTestServer(t) - s.cfg.File.Set("PORT", portOf(t, vllm.URL)) + writeInstance(t, s, "demo", portOf(t, vllm.URL), "") + inst, err := control.LoadInstance(s.cfg, "demo") + if err != nil { + t.Fatal(err) + } // Ten callers, as ten open tabs would be. var wg sync.WaitGroup @@ -305,8 +324,8 @@ func TestHealthProbeIsReusedBetweenCallers(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - if _, ok := s.health(); !ok { - t.Error("health() = not ok, want the endpoint reported healthy") + if _, ok := s.probe(inst); !ok { + t.Error("probe() = not ok, want the endpoint reported healthy") } }() } @@ -319,10 +338,12 @@ func TestHealthProbeIsReusedBetweenCallers(t *testing.T) { // And an answer older than the TTL is not reused, or the panel would // never notice the server going down. s.healthMu.Lock() - s.healthAt = time.Now().Add(-2 * healthProbeTTL) + e := s.health["demo"] + e.at = time.Now().Add(-2 * healthProbeTTL) + s.health["demo"] = e s.healthMu.Unlock() - if _, ok := s.health(); !ok { - t.Error("health() = not ok after expiry") + if _, ok := s.probe(inst); !ok { + t.Error("probe() = not ok after expiry") } if got := atomic.LoadInt32(&probes); got != 2 { t.Errorf("probes = %d, want 2: a stale answer should be replaced", got) @@ -369,13 +390,13 @@ func TestReloadedTabCanRecoverTheTokenAndMutate(t *testing.T) { // The recovered token must satisfy the mutation check -- the actual // symptom was a restart button that refused with "no token". - req = httptest.NewRequest("POST", "/api/restart", nil) + req = httptest.NewRequest("POST", "/api/instances/demo/restart", nil) req.AddCookie(cookie) req.Header.Set(tokenHeader, body.Token) rec = httptest.NewRecorder() h.ServeHTTP(rec, req) if rec.Code == http.StatusForbidden || rec.Code == http.StatusUnauthorized { - t.Errorf("POST /api/restart = %d, want the recovered token to be accepted", rec.Code) + t.Errorf("POST restart = %d, want the recovered token to be accepted", rec.Code) } // And the endpoint is not a way in: without the cookie it says nothing. @@ -463,16 +484,11 @@ func TestMalformedRepoIsRejectedBeforeAnyWork(t *testing.T) { } } -// Removing the model currently being served would leave the running service -// with no profile behind it, so it is refused rather than allowed to happen -// and be discovered at the next restart. -func TestRemoveRefusesTheActiveProfile(t *testing.T) { +// Removing a model that is being served would leave its service with no +// profile behind it, so it is refused. +func TestRemoveRefusesARunningProfile(t *testing.T) { s := newTestServer(t) - // The active profile is state, not config: it lives in its own file so a - // switch does not have to rewrite config.env. - if err := os.WriteFile(s.cfg.Dirs.ActiveProfileFile(), []byte("demo\n"), 0o600); err != nil { - t.Fatal(err) - } + writeInstance(t, s, "demo", "8001", "") h := s.Handler(nil) req := httptest.NewRequest("POST", "/api/profiles/demo/remove", nil) @@ -480,10 +496,10 @@ func TestRemoveRefusesTheActiveProfile(t *testing.T) { rec := httptest.NewRecorder() h.ServeHTTP(rec, req) if rec.Code != http.StatusConflict { - t.Errorf("removing the active profile = %d, want 409", rec.Code) + t.Errorf("removing a running profile = %d, want 409", rec.Code) } if _, err := os.Stat(s.cfg.Dirs.ProfileFile("demo")); err != nil { - t.Error("the active profile was deleted despite the refusal") + t.Error("a running profile was deleted despite the refusal") } // A profile that does not exist is a 404, not a 500 and not a stack @@ -715,9 +731,10 @@ func (f fakeCluster) Health() service.Health { return f.health } func (f fakeCluster) LastLogs(int) string { return "cluster log line" } func (f fakeCluster) LogHint() string { return "docker logs vllm-node on the head node" } -// While a recipe cluster serves, the native unit is idle. Reading the unit -// reported a working model as stopped, and its buttons would have started a -// second vLLM beside the cluster or claimed a stop that changed nothing. +// While a recipe cluster serves an instance, its native unit is idle. Reading +// the unit reported a working model as stopped, and its buttons would have +// started a second server beside the cluster or claimed a stop that changed +// nothing. func TestClusterIsReportedAndNotDriven(t *testing.T) { vllm := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -725,15 +742,24 @@ func TestClusterIsReportedAndNotDriven(t *testing.T) { defer vllm.Close() s := newTestServer(t) - s.cfg.File.Set("PORT", portOf(t, vllm.URL)) - s.SetCluster(func() Cluster { return fakeCluster{health: service.Health{State: "active", SubState: "running"}} }) + writeInstance(t, s, "demo", portOf(t, vllm.URL), "") + s.SetCluster(func(name string) Cluster { + if name != "demo" { + return nil + } + return fakeCluster{health: service.Health{State: "active", SubState: "running"}} + }) got := s.status() - if got.Phase != "serving" || !got.Healthy { - t.Errorf("phase = %q healthy = %v, want serving and healthy: the cluster is up", got.Phase, got.Healthy) + if len(got.Instances) != 1 { + t.Fatalf("instances = %+v, want demo", got.Instances) } - if got.Service != "cluster" || !got.Cluster || got.ControlReason == "" { - t.Errorf("service = %q cluster = %v reason = %q, want the cluster named and the controls explained", got.Service, got.Cluster, got.ControlReason) + inst := got.Instances[0] + if inst.Phase != "serving" || !inst.Healthy { + t.Errorf("phase = %q healthy = %v, want serving and healthy: the cluster is up", inst.Phase, inst.Healthy) + } + if !inst.Cluster || inst.ControlReason == "" { + t.Errorf("cluster = %v reason = %q, want the cluster reported and the controls explained", inst.Cluster, inst.ControlReason) } h := s.Handler(Static()) @@ -744,7 +770,8 @@ func TestClusterIsReportedAndNotDriven(t *testing.T) { h.ServeHTTP(rec, r) return rec } - for _, target := range []string{"/api/restart", "/api/stop", "/api/switch/demo", "/api/pull?repo=org/other"} { + for _, verb := range []string{"restart", "down", "up"} { + target := "/api/instances/demo/" + verb rec := do(http.MethodPost, target) if rec.Code != http.StatusConflict { t.Errorf("POST %s with a cluster serving = %d, want 409", target, rec.Code) @@ -754,16 +781,94 @@ func TestClusterIsReportedAndNotDriven(t *testing.T) { } } - rec := do(http.MethodGet, "/api/logs") + rec := do(http.MethodGet, "/api/logs?instance=demo") if !strings.Contains(rec.Body.String(), "cluster log line") { t.Errorf("logs = %q, want the cluster's log, not the idle unit's", rec.Body.String()) } - // No cluster: the native unit is back in charge and nothing is refused - // on the cluster's account. - s.SetCluster(func() Cluster { return nil }) - if got := s.status(); got.Cluster || got.Service != "vllm.service" { - t.Errorf("without a cluster: cluster = %v service = %q, want the native unit", got.Cluster, got.Service) + s.SetCluster(func(string) Cluster { return nil }) + if got := s.status(); len(got.Instances) != 1 || got.Instances[0].Cluster { + t.Errorf("without a cluster: %+v, want the native unit", got.Instances) + } +} + +// Every running model is listed, and exactly one is the default Coder offers. +func TestInstancesListsEveryRunningModel(t *testing.T) { + s := newTestServer(t) + writeInstance(t, s, "demo", "8001", "") + writeInstance(t, s, "other", "8002", "") + s.cfg.File.Set("DEFAULT_PROFILE", "other") + + r := httptest.NewRequest(http.MethodGet, "/api/instances", nil) + r.Header.Set(tokenHeader, s.token) + rec := httptest.NewRecorder() + s.Handler(nil).ServeHTTP(rec, r) + var got []instanceStatus + if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil { + t.Fatalf("GET /api/instances = %d %s: %v", rec.Code, rec.Body, err) + } + if len(got) != 2 { + t.Fatalf("instances = %+v, want demo and other", got) + } + for _, i := range got { + if want := i.Profile == "other"; i.Default != want { + t.Errorf("%s default = %v, want %v", i.Profile, i.Default, want) + } + if i.Model != "org/"+i.Profile { + t.Errorf("%s model = %q", i.Profile, i.Model) + } + } + if st := s.status(); st.Default != "other" { + t.Errorf("status default = %q, want other", st.Default) + } +} + +// An action on a model that is not running says so rather than failing deeper. +func TestInstanceActionsRefuseCleanly(t *testing.T) { + s := newTestServer(t) + h := s.Handler(nil) + for _, c := range []struct { + target string + want int + }{ + {"/api/instances/demo/down", http.StatusNotFound}, + {"/api/instances/demo/restart", http.StatusNotFound}, + {"/api/instances/demo/default", http.StatusNotFound}, + {"/api/instances/nope/up", http.StatusNotFound}, + {"/api/instances/demo/up?mem_util=2", http.StatusBadRequest}, + {"/api/instances/Bad..Name/up", http.StatusBadRequest}, + } { + r := httptest.NewRequest(http.MethodPost, c.target, nil) + r.Header.Set(tokenHeader, s.token) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, r) + if rec.Code != c.want { + t.Errorf("POST %s = %d, want %d: %s", c.target, rec.Code, c.want, rec.Body) + } + } + if _, err := os.Stat(s.cfg.Dirs.InstanceEnvFile("demo")); err == nil { + t.Error("a refused action left an instance file behind") + } +} + +// Making a model the default is a config change, so it survives a restart of +// the panel and is what the CLI reads too. +func TestDefaultIsWrittenToConfig(t *testing.T) { + s := newTestServer(t) + writeInstance(t, s, "demo", "8001", "") + r := httptest.NewRequest(http.MethodPost, "/api/instances/demo/default", nil) + r.Header.Set(tokenHeader, s.token) + rec := httptest.NewRecorder() + s.Handler(nil).ServeHTTP(rec, r) + if rec.Code != http.StatusOK { + t.Fatalf("POST default = %d: %s", rec.Code, rec.Body) + } + b, err := os.ReadFile(s.cfg.Dirs.ConfigFile()) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), "DEFAULT_PROFILE=demo") { + t.Errorf("config.env = %q, want DEFAULT_PROFILE=demo", b) } } diff --git a/internal/web/static/app.css b/internal/web/static/app.css index 8bc2a40..b96a8ec 100644 --- a/internal/web/static/app.css +++ b/internal/web/static/app.css @@ -173,8 +173,8 @@ main.discover { /* ---------- the memory rail ---------- */ /* The machine's whole memory pool, drawn to scale. This is the constraint - the tool exists to manage: one model at a time because one model is what - fits. During a load you can watch it fill. */ + the tool exists to manage: the models running at once have to fit in it. + During a load you can watch it fill. */ .rail { margin-top: 1.5rem; @@ -333,14 +333,48 @@ main.discover { outline-offset: 1px; } -/* ---------- buttons ---------- */ +/* ---------- running instances ---------- */ + +.instances { + list-style: none; + margin: 1.5rem 0 0; + padding: 0; +} -.now-actions { +.instances:empty { + display: none; +} + +.instances li { + padding: 0.875rem 1rem; + border: 1px solid var(--border-default); + border-radius: var(--radius); + background: var(--surface-secondary); +} + +.instances li + li { + margin-top: 0.5rem; +} + +.i-head { + display: flex; + align-items: center; + gap: 0.75rem; +} + +.i-head .m-name { + flex: 1; + min-width: 0; +} + +.i-actions { display: flex; gap: 0.5rem; - margin-top: 1.5rem; + margin-top: 0.75rem; } +/* ---------- buttons ---------- */ + .btn { font: inherit; font-size: 0.8125rem; @@ -398,6 +432,19 @@ input:focus-visible { letter-spacing: 0; } +.log-pick { + font: inherit; + font-family: var(--font-mono); + font-size: 0.6875rem; + letter-spacing: 0; + text-transform: none; + padding: 0.125rem 0.25rem; + border: 1px solid var(--border-secondary); + border-radius: calc(var(--radius) - 2px); + background: var(--surface-tertiary); + color: var(--content-primary); +} + .follow { margin-left: auto; display: flex; diff --git a/internal/web/static/index.html b/internal/web/static/index.html index 170786c..0602da6 100644 --- a/internal/web/static/index.html +++ b/internal/web/static/index.html @@ -24,8 +24,7 @@ costs it. Everything else on the page is secondary to this. -->

Connecting

-

 

-

+

 

- -
- - -
@@ -89,6 +82,7 @@

Log +