Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ the fact.
for saved profiles that are not running. New API: `GET /api/instances` and
`POST /api/instances/{name}/up|down|restart|default`; `/api/logs` takes
`?instance=`.
- `recipes install-service` follows `SERVICE_SCOPE`. With `user` it installs
the recipe unit and its watchdog in `~/.config/systemd/user`, enables
lingering so they start at boot, and needs no sudo.

### Changed

Expand Down
5 changes: 5 additions & 0 deletions docs/multi-node.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,11 @@ To keep more than one recipe serving, install one unit per profile, each with it
A watchdog timer relaunches the cluster when the endpoint stops answering.
A deliberate stop (`llm-helper down`, `recipes stop`, or stopping the unit) is remembered, so the watchdog leaves it stopped until the next launch.

By default the units go to `/etc/systemd/system` and installing them needs sudo.
Without sudo, set `SERVICE_SCOPE=user` in `config.env` first.
The units then go to `~/.config/systemd/user`, run as you, and are controlled with `systemctl --user`.
The install enables lingering with `loginctl enable-linger` so they start at boot without a login; if that fails, it prints the command to run.

## A pool is replicas

Every node serves the same model, and a router in front of them spreads requests across the healthy ones:
Expand Down
3 changes: 2 additions & 1 deletion docs/reference/cli/recipes/install-service.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ description: "Install a systemd service that keeps a recipe profile serving"
Installs a unit that brings a recipe profile back at boot and stops its containers
on shutdown. --profile names it; without it, the only recipe profile running is used.
Install one unit per recipe profile, each with its own --name. Linux hosts only.
With SERVICE_SCOPE=user the units go to the user's systemd manager and need no sudo.

## Usage

Expand Down Expand Up @@ -67,7 +68,7 @@ seconds to allow for the model to load
| ---- | -------- |
| Type | `string` |

user the launcher runs as (default: $USER)
user the launcher runs as (default: $USER; system scope only)

### --watchdog

Expand Down
44 changes: 36 additions & 8 deletions internal/cli/recipehealth.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ func newRecipesHealthCheckCmd() *cobra.Command {
threshold int
statePath string
interval int
userScope bool
)
c := &cobra.Command{
Use: "health-check",
Expand All @@ -45,15 +46,19 @@ func newRecipesHealthCheckCmd() *cobra.Command {
}
if statePath == "" {
statePath = healthStatePath(unit)
if userScope {
statePath = userHealthStatePath(unit)
}
}
return runHealthCheck(unit, prof, threshold, statePath, time.Duration(interval)*time.Minute)
return runHealthCheck(unit, prof, threshold, statePath, time.Duration(interval)*time.Minute, userScope)
},
}
c.Flags().StringVar(&unit, "unit", defaultRecipeServiceName, "unit to relaunch")
c.Flags().StringVar(&prof, "profile", "", "recipe profile whose endpoint to check (default: any running)")
c.Flags().IntVar(&threshold, "misses", defaultHealthMisses, "consecutive failures before relaunching")
c.Flags().StringVar(&statePath, "state", "", "where the miss count is kept")
c.Flags().IntVar(&interval, "interval", defaultHealthInterval, "minutes between checks, which is also how long a fresh launch marker is trusted")
c.Flags().BoolVar(&userScope, "user-scope", false, "the unit runs in the user's systemd manager")
return c
}

Expand All @@ -63,6 +68,24 @@ func healthStatePath(unit string) string {
return "/run/" + strings.TrimSuffix(unit, ".service") + "-misses"
}

// userHealthStatePath is the user-scope counterpart: the user's runtime dir is
// also cleared by a reboot.
func userHealthStatePath(unit string) string {
dir := os.Getenv("XDG_RUNTIME_DIR")
if dir == "" {
dir = os.TempDir()
}
return filepath.Join(dir, strings.TrimSuffix(unit, ".service")+"-misses")
Comment on lines +75 to +78
}

// healthSystemctl returns the args for unit's manager.
func healthSystemctl(userScope bool, args ...string) []string {
if userScope {
return append([]string{"--user"}, args...)
}
return args
}

// launchMarkerName is the file a launch leaves behind while it is working.
// It lives in the state dir so the launcher (the serving user) writes it and
// the watchdog (root, with HOME set to that user) reads it.
Expand Down Expand Up @@ -212,9 +235,9 @@ func healthDecision(unitState string, launching, stopped, healthy bool, misses,
return healthCount, misses
}

func runHealthCheck(unit, prof string, threshold int, statePath string, interval time.Duration) error {
func runHealthCheck(unit, prof string, threshold int, statePath string, interval time.Duration, userScope bool) error {
cfg, _ := loadConfig()
state := strings.TrimSpace(systemctlOutput("is-active", unit))
state := strings.TrimSpace(systemctlOutput(healthSystemctl(userScope, "is-active", unit)...))
healthy := endpointHealthy(cfg, prof)
misses := readMisses(statePath)

Expand All @@ -235,7 +258,7 @@ func runHealthCheck(unit, prof string, threshold int, statePath string, interval
default:
_ = os.Remove(statePath)
fmt.Fprintf(os.Stderr, "%s: endpoint unhealthy %d times; relaunching\n", unit, threshold)
return runPoolCommand(os.Stdout, "systemctl", "restart", unit)
return runPoolCommand(os.Stdout, "systemctl", healthSystemctl(userScope, "restart", unit)...)
}
}

Expand Down Expand Up @@ -300,16 +323,21 @@ func writeMisses(path string, n int) {

// renderHealthUnits builds the watchdog pair.
//
// The service runs as root so it can restart the unit without a sudo rule,
// and carries HOME so it reads the same config as the user who installed it.
// A system-scope service runs as root so it can restart the unit without a
// sudo rule, and carries HOME so it reads the same config as the user who
// installed it. A user-scope one runs in that user's manager.
func renderHealthUnits(u recipeSystemdUnit, unitName string, intervalMin int) (service, timer string) {
scope := ""
if u.UserScope {
scope = " --user-scope"
}
service = fmt.Sprintf("[Unit]\n"+
"Description=Health check for %s\n\n"+
"[Service]\n"+
"Type=oneshot\n"+
"Environment=HOME=%s\n"+
"ExecStart=%s recipes health-check --unit %s --profile %s --interval %d\n",
unitName, u.Home, u.Self, unitName, u.Profile, intervalMin)
"ExecStart=%s recipes health-check --unit %s --profile %s --interval %d%s\n",
unitName, u.Home, u.Self, unitName, u.Profile, intervalMin, scope)
timer = fmt.Sprintf("[Unit]\n"+
"Description=Run the health check for %s every %d minutes\n\n"+
"[Timer]\n"+
Expand Down
30 changes: 30 additions & 0 deletions internal/cli/recipehealth_internal_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package cli

import (
"os"
"path/filepath"
"strings"
"testing"
"time"
Expand Down Expand Up @@ -140,3 +142,31 @@ func TestHealthUnitsRunTheCheckerAgainstTheirUnit(t *testing.T) {
t.Errorf("health service pins a user, so it could not restart the unit:\n%s", svc)
}
}

func TestHealthUnitsInUserScope(t *testing.T) {
svc, _ := renderHealthUnits(
recipeSystemdUnit{Home: "/home/serve", Self: "/usr/local/bin/vllm-helper", Profile: "x", UserScope: true},
"vllm-cluster.service", 2)
want := "ExecStart=/usr/local/bin/vllm-helper recipes health-check --unit vllm-cluster.service --profile x --interval 2 --user-scope\n"
if !strings.Contains(svc, want) {
t.Errorf("health service missing %q:\n%s", want, svc)
}
if got := healthSystemctl(true, "restart", "u"); strings.Join(got, " ") != "--user restart u" {
t.Errorf("user args = %v", got)
}
if got := healthSystemctl(false, "restart", "u"); strings.Join(got, " ") != "restart u" {
t.Errorf("system args = %v", got)
}
}

// A user cannot write /run, so the count goes to the user's runtime dir.
func TestUserHealthStatePath(t *testing.T) {
t.Setenv("XDG_RUNTIME_DIR", "/run/user/1000")
if got := userHealthStatePath("vllm-cluster.service"); got != "/run/user/1000/vllm-cluster-misses" {
t.Errorf("state path = %q", got)
}
t.Setenv("XDG_RUNTIME_DIR", "")
if got, want := userHealthStatePath("vllm-cluster.service"), filepath.Join(os.TempDir(), "vllm-cluster-misses"); got != want {
t.Errorf("fallback state path = %q, want %q", got, want)
}
}
Loading
Loading