diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a95193..c652de9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,9 @@ the fact. for saved profiles that are not running. New API: `GET /api/instances` and `POST /api/instances/{name}/up|down|restart|default`; `/api/logs` takes `?instance=`. +- `recipes install-service` follows `SERVICE_SCOPE`. With `user` it installs + the recipe unit and its watchdog in `~/.config/systemd/user`, enables + lingering so they start at boot, and needs no sudo. ### Changed diff --git a/docs/multi-node.md b/docs/multi-node.md index 64d8640..038352a 100644 --- a/docs/multi-node.md +++ b/docs/multi-node.md @@ -63,6 +63,11 @@ To keep more than one recipe serving, install one unit per profile, each with it A watchdog timer relaunches the cluster when the endpoint stops answering. A deliberate stop (`llm-helper down`, `recipes stop`, or stopping the unit) is remembered, so the watchdog leaves it stopped until the next launch. +By default the units go to `/etc/systemd/system` and installing them needs sudo. +Without sudo, set `SERVICE_SCOPE=user` in `config.env` first. +The units then go to `~/.config/systemd/user`, run as you, and are controlled with `systemctl --user`. +The install enables lingering with `loginctl enable-linger` so they start at boot without a login; if that fails, it prints the command to run. + ## A pool is replicas Every node serves the same model, and a router in front of them spreads requests across the healthy ones: diff --git a/docs/reference/cli/recipes/install-service.md b/docs/reference/cli/recipes/install-service.md index 0ce0eb0..4079fcc 100644 --- a/docs/reference/cli/recipes/install-service.md +++ b/docs/reference/cli/recipes/install-service.md @@ -9,6 +9,7 @@ description: "Install a systemd service that keeps a recipe profile serving" Installs a unit that brings a recipe profile back at boot and stops its containers on shutdown. --profile names it; without it, the only recipe profile running is used. Install one unit per recipe profile, each with its own --name. Linux hosts only. +With SERVICE_SCOPE=user the units go to the user's systemd manager and need no sudo. ## Usage @@ -67,7 +68,7 @@ seconds to allow for the model to load | ---- | -------- | | Type | `string` | -user the launcher runs as (default: $USER) +user the launcher runs as (default: $USER; system scope only) ### --watchdog diff --git a/internal/cli/recipehealth.go b/internal/cli/recipehealth.go index 3259d9e..101be7d 100644 --- a/internal/cli/recipehealth.go +++ b/internal/cli/recipehealth.go @@ -32,6 +32,7 @@ func newRecipesHealthCheckCmd() *cobra.Command { threshold int statePath string interval int + userScope bool ) c := &cobra.Command{ Use: "health-check", @@ -45,8 +46,11 @@ func newRecipesHealthCheckCmd() *cobra.Command { } if statePath == "" { statePath = healthStatePath(unit) + if userScope { + statePath = userHealthStatePath(unit) + } } - return runHealthCheck(unit, prof, threshold, statePath, time.Duration(interval)*time.Minute) + return runHealthCheck(unit, prof, threshold, statePath, time.Duration(interval)*time.Minute, userScope) }, } c.Flags().StringVar(&unit, "unit", defaultRecipeServiceName, "unit to relaunch") @@ -54,6 +58,7 @@ func newRecipesHealthCheckCmd() *cobra.Command { c.Flags().IntVar(&threshold, "misses", defaultHealthMisses, "consecutive failures before relaunching") c.Flags().StringVar(&statePath, "state", "", "where the miss count is kept") c.Flags().IntVar(&interval, "interval", defaultHealthInterval, "minutes between checks, which is also how long a fresh launch marker is trusted") + c.Flags().BoolVar(&userScope, "user-scope", false, "the unit runs in the user's systemd manager") return c } @@ -63,6 +68,24 @@ func healthStatePath(unit string) string { return "/run/" + strings.TrimSuffix(unit, ".service") + "-misses" } +// userHealthStatePath is the user-scope counterpart: the user's runtime dir is +// also cleared by a reboot. +func userHealthStatePath(unit string) string { + dir := os.Getenv("XDG_RUNTIME_DIR") + if dir == "" { + dir = os.TempDir() + } + return filepath.Join(dir, strings.TrimSuffix(unit, ".service")+"-misses") +} + +// healthSystemctl returns the args for unit's manager. +func healthSystemctl(userScope bool, args ...string) []string { + if userScope { + return append([]string{"--user"}, args...) + } + return args +} + // launchMarkerName is the file a launch leaves behind while it is working. // It lives in the state dir so the launcher (the serving user) writes it and // the watchdog (root, with HOME set to that user) reads it. @@ -212,9 +235,9 @@ func healthDecision(unitState string, launching, stopped, healthy bool, misses, return healthCount, misses } -func runHealthCheck(unit, prof string, threshold int, statePath string, interval time.Duration) error { +func runHealthCheck(unit, prof string, threshold int, statePath string, interval time.Duration, userScope bool) error { cfg, _ := loadConfig() - state := strings.TrimSpace(systemctlOutput("is-active", unit)) + state := strings.TrimSpace(systemctlOutput(healthSystemctl(userScope, "is-active", unit)...)) healthy := endpointHealthy(cfg, prof) misses := readMisses(statePath) @@ -235,7 +258,7 @@ func runHealthCheck(unit, prof string, threshold int, statePath string, interval default: _ = os.Remove(statePath) fmt.Fprintf(os.Stderr, "%s: endpoint unhealthy %d times; relaunching\n", unit, threshold) - return runPoolCommand(os.Stdout, "systemctl", "restart", unit) + return runPoolCommand(os.Stdout, "systemctl", healthSystemctl(userScope, "restart", unit)...) } } @@ -300,16 +323,21 @@ func writeMisses(path string, n int) { // renderHealthUnits builds the watchdog pair. // -// The service runs as root so it can restart the unit without a sudo rule, -// and carries HOME so it reads the same config as the user who installed it. +// A system-scope service runs as root so it can restart the unit without a +// sudo rule, and carries HOME so it reads the same config as the user who +// installed it. A user-scope one runs in that user's manager. func renderHealthUnits(u recipeSystemdUnit, unitName string, intervalMin int) (service, timer string) { + scope := "" + if u.UserScope { + scope = " --user-scope" + } service = fmt.Sprintf("[Unit]\n"+ "Description=Health check for %s\n\n"+ "[Service]\n"+ "Type=oneshot\n"+ "Environment=HOME=%s\n"+ - "ExecStart=%s recipes health-check --unit %s --profile %s --interval %d\n", - unitName, u.Home, u.Self, unitName, u.Profile, intervalMin) + "ExecStart=%s recipes health-check --unit %s --profile %s --interval %d%s\n", + unitName, u.Home, u.Self, unitName, u.Profile, intervalMin, scope) timer = fmt.Sprintf("[Unit]\n"+ "Description=Run the health check for %s every %d minutes\n\n"+ "[Timer]\n"+ diff --git a/internal/cli/recipehealth_internal_test.go b/internal/cli/recipehealth_internal_test.go index b324899..8b1af60 100644 --- a/internal/cli/recipehealth_internal_test.go +++ b/internal/cli/recipehealth_internal_test.go @@ -1,6 +1,8 @@ package cli import ( + "os" + "path/filepath" "strings" "testing" "time" @@ -140,3 +142,31 @@ func TestHealthUnitsRunTheCheckerAgainstTheirUnit(t *testing.T) { t.Errorf("health service pins a user, so it could not restart the unit:\n%s", svc) } } + +func TestHealthUnitsInUserScope(t *testing.T) { + svc, _ := renderHealthUnits( + recipeSystemdUnit{Home: "/home/serve", Self: "/usr/local/bin/vllm-helper", Profile: "x", UserScope: true}, + "vllm-cluster.service", 2) + want := "ExecStart=/usr/local/bin/vllm-helper recipes health-check --unit vllm-cluster.service --profile x --interval 2 --user-scope\n" + if !strings.Contains(svc, want) { + t.Errorf("health service missing %q:\n%s", want, svc) + } + if got := healthSystemctl(true, "restart", "u"); strings.Join(got, " ") != "--user restart u" { + t.Errorf("user args = %v", got) + } + if got := healthSystemctl(false, "restart", "u"); strings.Join(got, " ") != "restart u" { + t.Errorf("system args = %v", got) + } +} + +// A user cannot write /run, so the count goes to the user's runtime dir. +func TestUserHealthStatePath(t *testing.T) { + t.Setenv("XDG_RUNTIME_DIR", "/run/user/1000") + if got := userHealthStatePath("vllm-cluster.service"); got != "/run/user/1000/vllm-cluster-misses" { + t.Errorf("state path = %q", got) + } + t.Setenv("XDG_RUNTIME_DIR", "") + if got, want := userHealthStatePath("vllm-cluster.service"), filepath.Join(os.TempDir(), "vllm-cluster-misses"); got != want { + t.Errorf("fallback state path = %q, want %q", got, want) + } +} diff --git a/internal/cli/recipeservice.go b/internal/cli/recipeservice.go index b19eadc..ad80c39 100644 --- a/internal/cli/recipeservice.go +++ b/internal/cli/recipeservice.go @@ -5,6 +5,7 @@ import ( "fmt" "io/fs" "os" + "os/exec" "path/filepath" "runtime" "strconv" @@ -40,7 +41,8 @@ func newRecipesInstallServiceCmd() *cobra.Command { Short: "Install a systemd service that keeps a recipe profile serving", Long: "Installs a unit that brings a recipe profile back at boot and stops its containers\n" + "on shutdown. --profile names it; without it, the only recipe profile running is used.\n" + - "Install one unit per recipe profile, each with its own --name. Linux hosts only.", + "Install one unit per recipe profile, each with its own --name. Linux hosts only.\n" + + "With SERVICE_SCOPE=user the units go to the user's systemd manager and need no sudo.", Args: cobra.NoArgs, RunE: func(*cobra.Command, []string) error { cfg, err := loadConfig() @@ -51,6 +53,7 @@ func newRecipesInstallServiceCmd() *cobra.Command { if err != nil { return err } + userScope := cfg.ServiceScope() == "user" return installRecipeService(recipeServiceOptions{ Name: name, Profile: prof, @@ -61,13 +64,14 @@ func newRecipesInstallServiceCmd() *cobra.Command { Native: cfg.ServiceName(), Watchdog: watchdog, Interval: interval, - Service: defaultRecipeServiceRunner{}, + UserScope: userScope, + Service: defaultRecipeServiceRunner{UserScope: userScope}, }) }, } c.Flags().StringVar(&name, "name", defaultRecipeServiceName, "systemd unit name") c.Flags().StringVar(&prof, "profile", "", "recipe profile the unit serves (default: the only one running)") - c.Flags().StringVar(&user, "user", "", "user the launcher runs as (default: $USER)") + c.Flags().StringVar(&user, "user", "", "user the launcher runs as (default: $USER; system scope only)") c.Flags().IntVar(&timeout, "timeout", defaultRecipeServiceTimeout, "seconds to allow for the model to load") c.Flags().BoolVar(&now, "now", false, "enable and start the service immediately") c.Flags().BoolVar(&supersede, "supersede-native", true, "stop and disable the single-node unit this replaces") @@ -117,6 +121,7 @@ type recipeServiceOptions struct { Native string Watchdog bool Interval int + UserScope bool Service recipeServiceRunner } @@ -128,6 +133,7 @@ type recipeServiceRunner interface { InstallSystemd(name, unit string) (string, error) EnableStart(name string) error Supersede(name string) error + Linger(user string) error } func installRecipeService(o recipeServiceOptions) error { @@ -146,6 +152,9 @@ func installRecipeService(o recipeServiceOptions) error { if o.Timeout <= 0 { return errors.New("--timeout must be a number of seconds") } + if o.UserScope && o.User != os.Getenv("USER") { + return fmt.Errorf("a user-scope unit runs as %s; --user needs SERVICE_SCOPE=system", os.Getenv("USER")) + } self, err := o.Service.Self() if err != nil { return err @@ -155,17 +164,24 @@ func installRecipeService(o recipeServiceOptions) error { return err } unit := renderRecipeSystemdUnit(recipeSystemdUnit{ - User: o.User, - Home: home, - Self: self, - Profile: o.Profile, - Timeout: o.Timeout, + User: o.User, + Home: home, + Self: self, + Profile: o.Profile, + Timeout: o.Timeout, + UserScope: o.UserScope, }) path, err := o.Service.InstallSystemd(o.Name, unit) if err != nil { return err } fmt.Printf("Installed cluster unit at %s\n", path) + // A user manager stops at logout and does not start at boot without it. + if o.UserScope { + if err := o.Service.Linger(o.User); err != nil { + fmt.Printf("note: couldn't enable lingering -- the cluster stops at logout (fix: sudo loginctl enable-linger %s)\n", o.User) + } + } // The single-node unit cannot serve a container recipe, and left enabled // it races this one for the port at boot. A unit that failed months ago @@ -188,7 +204,7 @@ func installRecipeService(o recipeServiceOptions) error { } healthService, timer := healthUnitNames(o.Name) healthTimer = timer - svcUnit, timerUnit := renderHealthUnits(recipeSystemdUnit{Home: home, Self: self, Profile: o.Profile}, o.Name, interval) + svcUnit, timerUnit := renderHealthUnits(recipeSystemdUnit{Home: home, Self: self, Profile: o.Profile, UserScope: o.UserScope}, o.Name, interval) if _, err := o.Service.InstallSystemd(healthService, svcUnit); err != nil { return err } @@ -217,16 +233,21 @@ func installRecipeService(o recipeServiceOptions) error { if healthTimer != "" { units += " " + healthTimer } - fmt.Printf("Not started. Launch a recipe first so the profile records its nodes, then: sudo systemctl enable --now %s\n", units) + systemctl := "sudo systemctl" + if o.UserScope { + systemctl = "systemctl --user" + } + fmt.Printf("Not started. Launch a recipe first so the profile records its nodes, then: %s enable --now %s\n", systemctl, units) return nil } type recipeSystemdUnit struct { - User string - Home string - Self string - Profile string - Timeout int + User string + Home string + Self string + Profile string + Timeout int + UserScope bool } // renderRecipeSystemdUnit builds the unit. @@ -235,23 +256,29 @@ type recipeSystemdUnit struct { // that returns once the endpoint answers; the containers it starts outlive // the command. ExecStop hands the same binary the teardown so both nodes are // cleared, not just the head. +// +// A user manager cannot order against docker.service or set User=. func renderRecipeSystemdUnit(u recipeSystemdUnit) string { + after, userLine, wantedBy := "network-online.target docker.service", "User="+u.User+"\n", "multi-user.target" + if u.UserScope { + after, userLine, wantedBy = "network-online.target", "", "default.target" + } return fmt.Sprintf("[Unit]\n"+ "Description=LLM Helper cluster (%s)\n"+ - "After=network-online.target docker.service\n"+ + "After=%s\n"+ "Wants=network-online.target\n\n"+ "[Service]\n"+ "Type=oneshot\n"+ "RemainAfterExit=yes\n"+ - "User=%s\n"+ + "%s"+ "Environment=HOME=%s\n"+ "WorkingDirectory=%s\n"+ "ExecStart=%s recipes service-start %s\n"+ "ExecStop=%s recipes service-stop %s\n"+ "TimeoutStartSec=%d\n\n"+ "[Install]\n"+ - "WantedBy=multi-user.target\n", - u.Profile, u.User, u.Home, u.Home, u.Self, u.Profile, u.Self, u.Profile, u.Timeout) + "WantedBy=%s\n", + u.Profile, after, userLine, u.Home, u.Home, u.Self, u.Profile, u.Self, u.Profile, u.Timeout, wantedBy) } // newRecipesServiceStartCmd is the unit's ExecStart. A profile taken down on @@ -386,7 +413,9 @@ func unitField(body, key string) string { return "" } -type defaultRecipeServiceRunner struct{} +type defaultRecipeServiceRunner struct { + UserScope bool +} func (defaultRecipeServiceRunner) Self() (string, error) { return defaultPoolServiceRunner{}.Self() } @@ -394,23 +423,59 @@ func (defaultRecipeServiceRunner) Home(user string) (string, error) { return defaultPoolServiceRunner{}.Home(user) } -func (defaultRecipeServiceRunner) InstallSystemd(name, unit string) (string, error) { - return defaultPoolServiceRunner{}.InstallSystemd(name, unit) +func (r defaultRecipeServiceRunner) InstallSystemd(name, unit string) (string, error) { + if !r.UserScope { + return defaultPoolServiceRunner{}.InstallSystemd(name, unit) + } + path := filepath.Join(userUnitDir(), name) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return "", err + } + if err := os.WriteFile(path, []byte(unit), 0o644); err != nil { + return "", err + } + return path, runPoolCommand(nil, "systemctl", "--user", "daemon-reload") +} + +// systemctl returns the command and leading args for this runner's manager. +func (r defaultRecipeServiceRunner) systemctl(args ...string) (string, []string) { + if r.UserScope { + return "systemctl", append([]string{"--user"}, args...) + } + return "sudo", append([]string{"systemctl"}, args...) } -func (defaultRecipeServiceRunner) EnableStart(name string) error { - return runPoolCommand(os.Stdout, "sudo", "systemctl", "enable", "--now", name) +func (r defaultRecipeServiceRunner) EnableStart(name string) error { + bin, args := r.systemctl("enable", "--now", name) + return runPoolCommand(os.Stdout, bin, args...) +} + +func (defaultRecipeServiceRunner) Linger(user string) error { + return exec.Command("loginctl", "enable-linger", user).Run() +} + +// userUnitDir is where the user's systemd manager reads units from. +func userUnitDir() string { + base := os.Getenv("XDG_CONFIG_HOME") + if base == "" { + home, _ := os.UserHomeDir() + base = filepath.Join(home, ".config") + } + return filepath.Join(base, "systemd", "user") } // Supersede stands the old unit down without deleting it: native mode is // still the only way most machines serve anything, so the unit file stays as // the way back. reset-failed clears a stale failure, which otherwise keeps // systemctl reporting the whole host as degraded. -func (defaultRecipeServiceRunner) Supersede(name string) error { - _ = runPoolCommand(nil, "sudo", "systemctl", "stop", name) - if err := runPoolCommand(nil, "sudo", "systemctl", "disable", name); err != nil { +func (r defaultRecipeServiceRunner) Supersede(name string) error { + bin, args := r.systemctl("stop", name) + _ = runPoolCommand(nil, bin, args...) + bin, args = r.systemctl("disable", name) + if err := runPoolCommand(nil, bin, args...); err != nil { return err } - _ = runPoolCommand(nil, "sudo", "systemctl", "reset-failed", name) + bin, args = r.systemctl("reset-failed", name) + _ = runPoolCommand(nil, bin, args...) return nil } diff --git a/internal/cli/recipeservice_internal_test.go b/internal/cli/recipeservice_internal_test.go index 535b766..5f19605 100644 --- a/internal/cli/recipeservice_internal_test.go +++ b/internal/cli/recipeservice_internal_test.go @@ -14,6 +14,7 @@ type fakeRecipeService struct { enabled []string superseded string failStop bool + lingered string } func (f *fakeRecipeService) installed(name string) bool { @@ -38,6 +39,7 @@ func (f *fakeRecipeService) Supersede(n string) error { f.superseded = n return nil } +func (f *fakeRecipeService) Linger(u string) error { f.lingered = u; return nil } // The unit names the profile it serves, not the recipe behind it. func TestRecipeUnitNamesNoRecipe(t *testing.T) { @@ -61,6 +63,30 @@ func TestRecipeUnitNamesNoRecipe(t *testing.T) { } } +// A user manager has no docker.service to order against and rejects User=. +func TestRecipeUnitInUserScope(t *testing.T) { + unit := renderRecipeSystemdUnit(recipeSystemdUnit{ + User: "serve", Home: "/home/serve", Self: "/usr/local/bin/vllm-helper", Timeout: 1800, Profile: "x", UserScope: true, + }) + for _, want := range []string{"After=network-online.target\n", "WantedBy=default.target", "ExecStart=/usr/local/bin/vllm-helper recipes service-start x"} { + if !strings.Contains(unit, want) { + t.Errorf("unit is missing %q:\n%s", want, unit) + } + } + for _, bad := range []string{"User=", "docker.service", "multi-user.target"} { + if strings.Contains(unit, bad) { + t.Errorf("user unit contains %q:\n%s", bad, unit) + } + } +} + +func TestUserUnitDirFollowsXDG(t *testing.T) { + t.Setenv("XDG_CONFIG_HOME", "/tmp/cfg") + if got := userUnitDir(); got != "/tmp/cfg/systemd/user" { + t.Errorf("userUnitDir() = %q", got) + } +} + // A load takes minutes, so the default systemd start timeout would kill a // healthy launch. func TestRecipeUnitAllowsALongLoad(t *testing.T) { @@ -174,6 +200,76 @@ func TestInstallRecipeServiceHintStartsTheWatchdogToo(t *testing.T) { } } +func TestInstallRecipeServiceInUserScope(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("install is implemented for systemd hosts") + } + t.Setenv("USER", "serve") + fake := &fakeRecipeService{} + out, err := captureStdout(t, func() error { + return installRecipeService(recipeServiceOptions{ + Name: "vllm-cluster.service", Profile: "x", User: "serve", Timeout: 60, + Supersede: true, Native: "vllm.service", Watchdog: true, Interval: 2, + UserScope: true, Service: fake, + }) + }) + if err != nil { + t.Fatal(err) + } + if fake.lingered != "serve" { + t.Errorf("lingered %q, want serve", fake.lingered) + } + if fake.superseded != "vllm.service" { + t.Errorf("superseded %q, want vllm.service", fake.superseded) + } + if strings.Contains(fake.units["vllm-cluster.service"], "User=") { + t.Errorf("user unit sets User=:\n%s", fake.units["vllm-cluster.service"]) + } + if !strings.Contains(fake.units["vllm-cluster-health.service"], "--interval 2 --user-scope\n") { + t.Errorf("watchdog does not check the user manager:\n%s", fake.units["vllm-cluster-health.service"]) + } + want := "systemctl --user enable --now vllm-cluster.service vllm-cluster-health.timer\n" + if !strings.HasSuffix(out, want) { + t.Errorf("hint does not end with %q:\n%s", want, out) + } +} + +func TestInstallRecipeServiceLingersOnlyInUserScope(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("install is implemented for systemd hosts") + } + fake := &fakeRecipeService{} + if _, err := captureStdout(t, func() error { + return installRecipeService(recipeServiceOptions{ + Name: "vllm-cluster.service", Profile: "x", User: "serve", Timeout: 60, Service: fake, + }) + }); err != nil { + t.Fatal(err) + } + if fake.lingered != "" { + t.Errorf("system scope enabled lingering for %q", fake.lingered) + } +} + +// A user manager can only run units as its own user. +func TestInstallRecipeServiceRejectsAnotherUserInUserScope(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("install is implemented for systemd hosts") + } + t.Setenv("USER", "me") + fake := &fakeRecipeService{} + err := installRecipeService(recipeServiceOptions{ + Name: "vllm-cluster.service", Profile: "x", User: "serve", Timeout: 60, + UserScope: true, Service: fake, + }) + if err == nil || !strings.Contains(err.Error(), "SERVICE_SCOPE=system") { + t.Fatalf("err = %v", err) + } + if len(fake.units) != 0 { + t.Errorf("installed %v anyway", fake.units) + } +} + // A unit from before instances ran a bare 'stop', which now forgets the // instance; migrate rewrites it and its watchdog to name the profile. func TestUpgradeRecipeUnits(t *testing.T) {