Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,12 @@ the fact.
- The recipe health watchdog keeps its launch and stopped markers per
profile, so stopping or launching one recipe no longer pauses another's
watchdog. Markers left by an earlier release are ignored.
- The router answers a request for a model whose instance is still loading
with 503 and `Retry-After`, as llama.cpp does, instead of 404
`model_not_found`. OpenAI-compatible clients, Coder among them, retry it
instead of failing the chat. A request to an instance that does not answer
also gets 503 instead of 502, with an OpenAI-style JSON error body; so does
a pool backend.

### Removed

Expand Down
3 changes: 3 additions & 0 deletions docs/multi-instance.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,9 @@ llm-helper router install-service
- `llm-helper router serve` runs it in the foreground instead; `--bind` there sets the address and overrides `--port`.
- `llm-helper router uninstall-service` removes it and clears `ROUTER_PORT`.

A request for a model whose instance is still loading gets 503 with `Retry-After`, so OpenAI-compatible clients, Coder among them, wait and retry instead of failing.
A model no instance serves gets 404 `model_not_found`.

The instances keep their own ports, so clients that already talk to one directly are unaffected.

## Upgrading from a single-model release
Expand Down
11 changes: 6 additions & 5 deletions internal/cli/router.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,22 +53,23 @@ func routerBind(cfg *config.Config, port string) string {
return net.JoinHostPort("127.0.0.1", port)
}

func instanceBackends(cfg *config.Config) func() []string {
return func() []string {
func instanceBackends(cfg *config.Config) func() []router.Backend {
return func() []router.Backend {
list, err := control.ListInstances(cfg)
if err != nil {
return nil
}
out := make([]string, 0, len(list))
out := make([]router.Backend, 0, len(list))
for _, inst := range list {
model := inst.Env.Get("MODEL")
if c := clusterRunnerForName(cfg, inst.Name); c != nil {
if u, err := c.endpointURL(); err == nil {
out = append(out, u)
out = append(out, router.Backend{URL: u, Model: model})
continue
}
}
if inst.Port != "" {
out = append(out, "http://localhost:"+inst.Port)
out = append(out, router.Backend{URL: "http://localhost:" + inst.Port, Model: model})
}
}
return out
Expand Down
15 changes: 14 additions & 1 deletion internal/cli/router_internal_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"

Expand Down Expand Up @@ -61,8 +62,20 @@ func TestRouterBind(t *testing.T) {
}
}

func TestInstanceBackendsUseRunningModel(t *testing.T) {
cfg := routerTestConfig(t, "PORT=8001\n")
if err := os.WriteFile(cfg.Dirs.ProfileFile("x"), []byte("MODEL=org/edited\n"), 0o600); err != nil {
t.Fatal(err)
}
writeInstance(t, cfg.Dirs, "x", "PORT=9123\nMODEL=org/running\n")
got := instanceBackends(cfg)()
if len(got) != 1 || got[0].URL != "http://localhost:9123" || got[0].Model != "org/running" {
t.Errorf("backends: %+v", got)
}
}

func TestRouterMuxRequiresKey(t *testing.T) {
mux := routerMux(router.NewModels(func() []string { return nil }, "k"), "k")
mux := routerMux(router.NewModels(func() []router.Backend { return nil }, "k"), "k")
for _, path := range []string{"/v1/models", "/router/status"} {
for auth, want := range map[string]int{"": http.StatusUnauthorized, "Bearer bad": http.StatusUnauthorized, "Bearer k": http.StatusOK} {
req := httptest.NewRequest(http.MethodGet, path, nil)
Expand Down
100 changes: 77 additions & 23 deletions internal/router/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"net/http"
"net/http/httputil"
"net/url"
"strconv"
"strings"
"sync"
"time"
Expand All @@ -18,21 +19,33 @@ import (
// Models sends each request to the backend that serves the model it names.
// It learns which backend serves what from each backend's own /v1/models.
type Models struct {
backends func() []string
backends func() []Backend
key string
client *http.Client

mu sync.RWMutex
routes map[string]*url.URL
entries []json.RawMessage
urls []*url.URL
tried time.Time
mu sync.RWMutex
routes map[string]*url.URL
starting map[string]bool
learned map[string][]string
entries []json.RawMessage
urls []*url.URL
tried time.Time
}

// NewModels returns a router over the base URLs (scheme://host:port) that
// backends returns. backends is called on every refresh. A non-empty key is
// sent as the bearer token to every backend.
func NewModels(backends func() []string, key string) *Models {
// Backend is one instance: its base URL (scheme://host:port) and the model
// it is configured to serve, "" when unknown.
type Backend struct {
URL string
Model string
}

// RetryAfter is how long a client is told to wait for a starting instance.
const RetryAfter = 10 * time.Second

// NewModels returns a router over the instances backends returns. backends is
// called on every refresh. A non-empty key is sent as the bearer token to
// every backend.
func NewModels(backends func() []Backend, key string) *Models {
return &Models{
backends: backends,
key: key,
Expand All @@ -50,53 +63,73 @@ type modelID struct {
}

// Refresh asks every backend for its models. When two backends serve the
// same id, the earlier one keeps it.
// same id, the earlier one keeps it. The model of a backend whose /v1/models
// does not answer yet is starting, as are the ids it last advertised.
func (m *Models) Refresh(ctx context.Context) {
m.mu.RLock()
prev := m.learned
m.mu.RUnlock()
learned := map[string][]string{}
routes := map[string]*url.URL{}
var entries []json.RawMessage
var urls []*url.URL
var waiting []string
for _, b := range m.backends() {
u, err := url.Parse(strings.TrimRight(b, "/"))
u, err := url.Parse(strings.TrimRight(b.URL, "/"))
if err != nil {
continue
}
urls = append(urls, u)
for _, e := range m.fetch(ctx, u) {
key := u.String() + "\x00" + b.Model
list, ok := m.fetch(ctx, u)
if !ok && b.Model != "" {
waiting = append(waiting, b.Model)
Comment thread
DevelopmentCats marked this conversation as resolved.
Resolved
waiting = append(waiting, prev[key]...)
learned[key] = prev[key]
}
for _, e := range list {
var id modelID
if json.Unmarshal(e, &id) != nil || id.ID == "" {
continue
}
learned[key] = append(learned[key], id.ID)
if _, dup := routes[id.ID]; dup {
continue
}
routes[id.ID] = u
entries = append(entries, e)
}
}
starting := map[string]bool{}
for _, id := range waiting {
if routes[id] == nil {
starting[id] = true
}
}
m.mu.Lock()
m.routes, m.entries, m.urls, m.tried = routes, entries, urls, time.Now()
m.routes, m.starting, m.learned, m.entries, m.urls, m.tried = routes, starting, learned, entries, urls, time.Now()
m.mu.Unlock()
}

func (m *Models) fetch(ctx context.Context, u *url.URL) []json.RawMessage {
func (m *Models) fetch(ctx context.Context, u *url.URL) ([]json.RawMessage, bool) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String()+"/v1/models", nil)
if err != nil {
return nil
return nil, false
}
m.authorize(req)
resp, err := m.client.Do(req)
if err != nil {
return nil
return nil, false
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil
return nil, false
}
var l modelList
if json.NewDecoder(resp.Body).Decode(&l) != nil {
return nil
return nil, false
}
return l.Data
return l.Data, len(l.Data) > 0
}

// StartRefresh refreshes now and then every interval until ctx is done.
Expand Down Expand Up @@ -190,8 +223,18 @@ func (m *Models) ServeHTTP(w http.ResponseWriter, r *http.Request) {
apiError(w, http.StatusBadRequest, "the request names no model, and more than one is running", "")
return
}
apiError(w, http.StatusNotFound, fmt.Sprintf("no running instance serves the model %q", model), "model_not_found")
return
m.mu.RLock()
u = m.routes[model]
starting := m.starting[model]
m.mu.RUnlock()
if u == nil && starting {
unavailable(w, fmt.Sprintf("%s is still loading; try again shortly", model))
return
}
if u == nil {
apiError(w, http.StatusNotFound, fmt.Sprintf("no running instance serves the model %q", model), "model_not_found")
return
}
}
rx := httputil.NewSingleHostReverseProxy(u)
direct := rx.Director
Expand All @@ -201,7 +244,7 @@ func (m *Models) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
rx.FlushInterval = -1
rx.ErrorHandler = func(w http.ResponseWriter, _ *http.Request, err error) {
apiError(w, http.StatusBadGateway, fmt.Sprintf("the instance serving %q did not answer: %v", model, err), "")
unavailable(w, fmt.Sprintf("the instance serving %s is not answering; it may be restarting: %v", model, err))
}
rx.ServeHTTP(w, r)
}
Expand Down Expand Up @@ -237,3 +280,14 @@ func apiError(w http.ResponseWriter, status int, msg, code string) {
}
_ = json.NewEncoder(w).Encode(map[string]any{"error": e})
}

// unavailable answers 503 the way llama.cpp does while a model loads, with
// Retry-After so OpenAI-compatible clients retry.
func unavailable(w http.ResponseWriter, msg string) {
w.Header().Set("Content-Type", "application/json")
w.Header().Set("Retry-After", strconv.Itoa(int(RetryAfter.Seconds())))
w.Header().Set("Retry-After-Ms", strconv.FormatInt(RetryAfter.Milliseconds(), 10))
w.WriteHeader(http.StatusServiceUnavailable)
e := map[string]any{"message": msg, "type": "unavailable_error", "code": http.StatusServiceUnavailable}
_ = json.NewEncoder(w).Encode(map[string]any{"error": e})
}
Loading
Loading