From 067c6957df64bcc23924fc15fa45239a210919fb Mon Sep 17 00:00:00 2001 From: DevelopmentCats Date: Tue, 10 Nov 2026 21:10:50 +0000 Subject: [PATCH 1/2] router: answer 503 with retry hints while an instance loads --- CHANGELOG.md | 6 + docs/multi-instance.md | 3 + internal/cli/router.go | 15 ++- internal/cli/router_internal_test.go | 2 +- internal/router/models.go | 84 ++++++++++---- internal/router/models_internal_test.go | 121 +++++++++++++++++--- internal/router/poolrouter.go | 7 +- internal/router/poolrouter_internal_test.go | 12 ++ 8 files changed, 204 insertions(+), 46 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2f604f0..f8fcdf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,12 @@ the fact. - The recipe health watchdog keeps its launch and stopped markers per profile, so stopping or launching one recipe no longer pauses another's watchdog. Markers left by an earlier release are ignored. +- The router answers a request for a model whose instance is still loading + with 503 and `Retry-After`, as llama.cpp does, instead of 404 + `model_not_found`. OpenAI-compatible clients, Coder among them, retry it + instead of failing the chat. A request to an instance that does not answer + also gets 503 instead of 502, with an OpenAI-style JSON error body; so does + a pool backend. ### Removed diff --git a/docs/multi-instance.md b/docs/multi-instance.md index d842300..eb94cf9 100644 --- a/docs/multi-instance.md +++ b/docs/multi-instance.md @@ -93,6 +93,9 @@ llm-helper router install-service - `llm-helper router serve` runs it in the foreground instead; `--bind` there sets the address and overrides `--port`. - `llm-helper router uninstall-service` removes it and clears `ROUTER_PORT`. +A request for a model whose instance is still loading gets 503 with `Retry-After`, so OpenAI-compatible clients, Coder among them, wait and retry instead of failing. +A model no instance serves gets 404 `model_not_found`. + The instances keep their own ports, so clients that already talk to one directly are unaffected. ## Upgrading from a single-model release diff --git a/internal/cli/router.go b/internal/cli/router.go index 5349873..b20fbaf 100644 --- a/internal/cli/router.go +++ b/internal/cli/router.go @@ -15,6 +15,7 @@ import ( "github.com/coder/llm-helper/internal/config" "github.com/coder/llm-helper/internal/control" + "github.com/coder/llm-helper/internal/profile" "github.com/coder/llm-helper/internal/router" "github.com/coder/llm-helper/internal/service" ) @@ -53,22 +54,26 @@ func routerBind(cfg *config.Config, port string) string { return net.JoinHostPort("127.0.0.1", port) } -func instanceBackends(cfg *config.Config) func() []string { - return func() []string { +func instanceBackends(cfg *config.Config) func() []router.Backend { + return func() []router.Backend { list, err := control.ListInstances(cfg) if err != nil { return nil } - out := make([]string, 0, len(list)) + out := make([]router.Backend, 0, len(list)) for _, inst := range list { + var model string + if p, err := profile.Load(cfg.Dirs, inst.Name); err == nil { + model = p.Model() + } if c := clusterRunnerForName(cfg, inst.Name); c != nil { if u, err := c.endpointURL(); err == nil { - out = append(out, u) + out = append(out, router.Backend{URL: u, Model: model}) continue } } if inst.Port != "" { - out = append(out, "http://localhost:"+inst.Port) + out = append(out, router.Backend{URL: "http://localhost:" + inst.Port, Model: model}) } } return out diff --git a/internal/cli/router_internal_test.go b/internal/cli/router_internal_test.go index b006818..2851473 100644 --- a/internal/cli/router_internal_test.go +++ b/internal/cli/router_internal_test.go @@ -62,7 +62,7 @@ func TestRouterBind(t *testing.T) { } func TestRouterMuxRequiresKey(t *testing.T) { - mux := routerMux(router.NewModels(func() []string { return nil }, "k"), "k") + mux := routerMux(router.NewModels(func() []router.Backend { return nil }, "k"), "k") for _, path := range []string{"/v1/models", "/router/status"} { for auth, want := range map[string]int{"": http.StatusUnauthorized, "Bearer bad": http.StatusUnauthorized, "Bearer k": http.StatusOK} { req := httptest.NewRequest(http.MethodGet, path, nil) diff --git a/internal/router/models.go b/internal/router/models.go index 1a394f2..83edadd 100644 --- a/internal/router/models.go +++ b/internal/router/models.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httputil" "net/url" + "strconv" "strings" "sync" "time" @@ -18,21 +19,32 @@ import ( // Models sends each request to the backend that serves the model it names. // It learns which backend serves what from each backend's own /v1/models. type Models struct { - backends func() []string + backends func() []Backend key string client *http.Client - mu sync.RWMutex - routes map[string]*url.URL - entries []json.RawMessage - urls []*url.URL - tried time.Time + mu sync.RWMutex + routes map[string]*url.URL + starting map[string]bool + entries []json.RawMessage + urls []*url.URL + tried time.Time } -// NewModels returns a router over the base URLs (scheme://host:port) that -// backends returns. backends is called on every refresh. A non-empty key is -// sent as the bearer token to every backend. -func NewModels(backends func() []string, key string) *Models { +// Backend is one instance: its base URL (scheme://host:port) and the model +// it is configured to serve, "" when unknown. +type Backend struct { + URL string + Model string +} + +// RetryAfter is how long a client is told to wait for a starting instance. +const RetryAfter = 10 * time.Second + +// NewModels returns a router over the instances backends returns. backends is +// called on every refresh. A non-empty key is sent as the bearer token to +// every backend. +func NewModels(backends func() []Backend, key string) *Models { return &Models{ backends: backends, key: key, @@ -50,18 +62,24 @@ type modelID struct { } // Refresh asks every backend for its models. When two backends serve the -// same id, the earlier one keeps it. +// same id, the earlier one keeps it. The model of a backend whose /v1/models +// does not answer yet is starting. func (m *Models) Refresh(ctx context.Context) { routes := map[string]*url.URL{} var entries []json.RawMessage var urls []*url.URL + var waiting []string for _, b := range m.backends() { - u, err := url.Parse(strings.TrimRight(b, "/")) + u, err := url.Parse(strings.TrimRight(b.URL, "/")) if err != nil { continue } urls = append(urls, u) - for _, e := range m.fetch(ctx, u) { + list, ok := m.fetch(ctx, u) + if !ok && b.Model != "" { + waiting = append(waiting, b.Model) + } + for _, e := range list { var id modelID if json.Unmarshal(e, &id) != nil || id.ID == "" { continue @@ -73,30 +91,36 @@ func (m *Models) Refresh(ctx context.Context) { entries = append(entries, e) } } + starting := map[string]bool{} + for _, id := range waiting { + if routes[id] == nil { + starting[id] = true + } + } m.mu.Lock() - m.routes, m.entries, m.urls, m.tried = routes, entries, urls, time.Now() + m.routes, m.starting, m.entries, m.urls, m.tried = routes, starting, entries, urls, time.Now() m.mu.Unlock() } -func (m *Models) fetch(ctx context.Context, u *url.URL) []json.RawMessage { +func (m *Models) fetch(ctx context.Context, u *url.URL) ([]json.RawMessage, bool) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String()+"/v1/models", nil) if err != nil { - return nil + return nil, false } m.authorize(req) resp, err := m.client.Do(req) if err != nil { - return nil + return nil, false } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { - return nil + return nil, false } var l modelList if json.NewDecoder(resp.Body).Decode(&l) != nil { - return nil + return nil, false } - return l.Data + return l.Data, len(l.Data) > 0 } // StartRefresh refreshes now and then every interval until ctx is done. @@ -190,6 +214,13 @@ func (m *Models) ServeHTTP(w http.ResponseWriter, r *http.Request) { apiError(w, http.StatusBadRequest, "the request names no model, and more than one is running", "") return } + m.mu.RLock() + starting := m.starting[model] + m.mu.RUnlock() + if starting { + unavailable(w, fmt.Sprintf("%s is still loading; try again shortly", model)) + return + } apiError(w, http.StatusNotFound, fmt.Sprintf("no running instance serves the model %q", model), "model_not_found") return } @@ -201,7 +232,7 @@ func (m *Models) ServeHTTP(w http.ResponseWriter, r *http.Request) { } rx.FlushInterval = -1 rx.ErrorHandler = func(w http.ResponseWriter, _ *http.Request, err error) { - apiError(w, http.StatusBadGateway, fmt.Sprintf("the instance serving %q did not answer: %v", model, err), "") + unavailable(w, fmt.Sprintf("the instance serving %s is not answering; it may be restarting: %v", model, err)) } rx.ServeHTTP(w, r) } @@ -237,3 +268,14 @@ func apiError(w http.ResponseWriter, status int, msg, code string) { } _ = json.NewEncoder(w).Encode(map[string]any{"error": e}) } + +// unavailable answers 503 the way llama.cpp does while a model loads, with +// Retry-After so OpenAI-compatible clients retry. +func unavailable(w http.ResponseWriter, msg string) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Retry-After", strconv.Itoa(int(RetryAfter.Seconds()))) + w.Header().Set("Retry-After-Ms", strconv.FormatInt(RetryAfter.Milliseconds(), 10)) + w.WriteHeader(http.StatusServiceUnavailable) + e := map[string]any{"message": msg, "type": "unavailable_error", "code": http.StatusServiceUnavailable} + _ = json.NewEncoder(w).Encode(map[string]any{"error": e}) +} diff --git a/internal/router/models_internal_test.go b/internal/router/models_internal_test.go index ded0b71..e783ca6 100644 --- a/internal/router/models_internal_test.go +++ b/internal/router/models_internal_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync/atomic" "testing" ) @@ -28,6 +29,16 @@ func fakeInstance(t *testing.T, name string, models ...string) *httptest.Server return s } +func urls(us ...string) func() []Backend { + return func() []Backend { + out := make([]Backend, len(us)) + for i, u := range us { + out[i] = Backend{URL: u} + } + return out + } +} + func do(t *testing.T, h http.Handler, method, path, body string) (int, string) { t.Helper() rec := httptest.NewRecorder() @@ -38,7 +49,7 @@ func do(t *testing.T, h http.Handler, method, path, body string) (int, string) { func TestModelsDispatchesByModel(t *testing.T) { a := fakeInstance(t, "a", "qwen") b := fakeInstance(t, "b", "glm", "qwen") - m := NewModels(func() []string { return []string{a.URL, b.URL} }, "") + m := NewModels(urls(a.URL, b.URL), "") m.Refresh(context.Background()) body := `{"model":"glm","messages":[]}` @@ -56,7 +67,7 @@ func TestModelsDispatchesByModel(t *testing.T) { func TestModelsMergesList(t *testing.T) { a := fakeInstance(t, "a", "qwen") b := fakeInstance(t, "b", "glm", "qwen") - m := NewModels(func() []string { return []string{a.URL, b.URL} }, "") + m := NewModels(urls(a.URL, b.URL), "") code, got := do(t, m, http.MethodGet, "/v1/models", "") var l struct { @@ -79,7 +90,7 @@ func TestModelsMergesList(t *testing.T) { func TestModelsUnknownAndMissingModel(t *testing.T) { a := fakeInstance(t, "a", "qwen") b := fakeInstance(t, "b", "glm") - m := NewModels(func() []string { return []string{a.URL, b.URL} }, "") + m := NewModels(urls(a.URL, b.URL), "") m.Refresh(context.Background()) if code, got := do(t, m, http.MethodPost, "/v1/chat/completions", `{"model":"nope"}`); code != 404 || !strings.Contains(got, "model_not_found") { @@ -89,7 +100,7 @@ func TestModelsUnknownAndMissingModel(t *testing.T) { t.Errorf("no model with two instances: %d", code) } - one := NewModels(func() []string { return []string{a.URL} }, "") + one := NewModels(urls(a.URL), "") one.Refresh(context.Background()) if code, got := do(t, one, http.MethodPost, "/v1/chat/completions", `{}`); code != 200 || !strings.HasPrefix(got, "a ") { t.Errorf("no model with one instance: %d %q", code, got) @@ -99,7 +110,7 @@ func TestModelsUnknownAndMissingModel(t *testing.T) { func TestModelsLearnsNewInstanceOnMiss(t *testing.T) { a := fakeInstance(t, "a", "qwen") backends := []string{a.URL} - m := NewModels(func() []string { return backends }, "") + m := NewModels(func() []Backend { return urls(backends...)() }, "") m.Refresh(context.Background()) m.tried = m.tried.Add(-2e9) @@ -110,16 +121,6 @@ func TestModelsLearnsNewInstanceOnMiss(t *testing.T) { } } -func TestModelsBackendDown(t *testing.T) { - a := fakeInstance(t, "a", "qwen") - m := NewModels(func() []string { return []string{a.URL} }, "") - m.Refresh(context.Background()) - a.Close() - if code, _ := do(t, m, http.MethodPost, "/v1/chat/completions", `{"model":"qwen"}`); code != http.StatusBadGateway { - t.Errorf("down instance: %d", code) - } -} - func TestModelsSendsKeyToBackends(t *testing.T) { a := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("Authorization") != "Bearer sekrit" { @@ -133,7 +134,7 @@ func TestModelsSendsKeyToBackends(t *testing.T) { _, _ = io.WriteString(w, "a") })) t.Cleanup(a.Close) - m := NewModels(func() []string { return []string{a.URL} }, "sekrit") + m := NewModels(urls(a.URL), "sekrit") if _, got := do(t, m, http.MethodGet, "/v1/models", ""); !strings.Contains(got, "qwen") { t.Errorf("list: %q", got) } @@ -142,6 +143,94 @@ func TestModelsSendsKeyToBackends(t *testing.T) { } } +func assertUnavailable(t *testing.T, rec *httptest.ResponseRecorder) { + t.Helper() + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("code = %d, body %q", rec.Code, rec.Body.String()) + } + if got := rec.Header().Get("Retry-After"); got != "10" { + t.Errorf("Retry-After = %q", got) + } + if got := rec.Header().Get("Retry-After-Ms"); got != "10000" { + t.Errorf("Retry-After-Ms = %q", got) + } + var e struct { + Error struct { + Message string `json:"message"` + Type string `json:"type"` + Code int `json:"code"` + } `json:"error"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &e); err != nil { + t.Fatalf("body %q: %v", rec.Body.String(), err) + } + if e.Error.Type != "unavailable_error" || e.Error.Code != 503 || e.Error.Message == "" { + t.Errorf("error = %+v", e.Error) + } + // Clients such as Coder give up at once on these phrases. + for _, p := range []string{"model not found", "invalid model", "unsupported model"} { + if strings.Contains(strings.ToLower(e.Error.Message), p) { + t.Errorf("message %q contains %q", e.Error.Message, p) + } + } +} + +func TestModelsStartingInstance(t *testing.T) { + var ready atomic.Bool + a := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !ready.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + if r.URL.Path == "/v1/models" { + _, _ = io.WriteString(w, `{"data":[{"id":"glm"}]}`) + return + } + _, _ = io.WriteString(w, "a") + })) + t.Cleanup(a.Close) + b := fakeInstance(t, "b", "qwen") + m := NewModels(func() []Backend { + return []Backend{{URL: a.URL, Model: "glm"}, {URL: b.URL, Model: "qwen"}} + }, "") + m.Refresh(context.Background()) + + rec := httptest.NewRecorder() + m.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"glm"}`))) + assertUnavailable(t, rec) + if code, got := do(t, m, http.MethodPost, "/v1/chat/completions", `{"model":"nope"}`); code != 404 || !strings.Contains(got, "model_not_found") { + t.Errorf("unknown model while another starts: %d %q", code, got) + } + if _, got := do(t, m, http.MethodPost, "/v1/chat/completions", `{"model":"qwen"}`); !strings.HasPrefix(got, "b ") { + t.Errorf("ready instance: %q", got) + } + + ready.Store(true) + m.Refresh(context.Background()) + if code, got := do(t, m, http.MethodPost, "/v1/chat/completions", `{"model":"glm"}`); code != 200 || got != "a" { + t.Errorf("after loading: %d %q", code, got) + } +} + +func TestModelsStartingNotStuckOnRename(t *testing.T) { + a := fakeInstance(t, "a", "served-name") + m := NewModels(func() []Backend { return []Backend{{URL: a.URL, Model: "org/model"}} }, "") + m.Refresh(context.Background()) + if code, _ := do(t, m, http.MethodPost, "/v1/chat/completions", `{"model":"org/model"}`); code != 404 { + t.Errorf("an answering instance under another name is not starting: %d", code) + } +} + +func TestModelsBackendDownBody(t *testing.T) { + a := fakeInstance(t, "a", "qwen") + m := NewModels(urls(a.URL), "") + m.Refresh(context.Background()) + a.Close() + rec := httptest.NewRecorder() + m.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{"model":"qwen"}`))) + assertUnavailable(t, rec) +} + func TestRequireKey(t *testing.T) { ok := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, "ok") }) h := RequireKey("sekrit", ok) diff --git a/internal/router/poolrouter.go b/internal/router/poolrouter.go index 772e726..588f83b 100644 --- a/internal/router/poolrouter.go +++ b/internal/router/poolrouter.go @@ -2,6 +2,7 @@ package router import ( "context" + "fmt" "net/http" "net/http/httputil" "net/url" @@ -126,12 +127,12 @@ func (p *Pool) ServeHTTP(w http.ResponseWriter, r *http.Request) { key := p.sticky(r) b, ok := p.pick(key) if !ok { - http.Error(w, "no backends", http.StatusBadGateway) + unavailable(w, "the pool has no backends") return } rx := httputil.NewSingleHostReverseProxy(b.url) - rx.ErrorHandler = func(w http.ResponseWriter, _ *http.Request, _ error) { - http.Error(w, "backend error", http.StatusBadGateway) + rx.ErrorHandler = func(w http.ResponseWriter, _ *http.Request, err error) { + unavailable(w, fmt.Sprintf("the backend is not answering; it may be starting: %v", err)) } rx.ServeHTTP(w, r) } diff --git a/internal/router/poolrouter_internal_test.go b/internal/router/poolrouter_internal_test.go index 0da7120..61fe64a 100644 --- a/internal/router/poolrouter_internal_test.go +++ b/internal/router/poolrouter_internal_test.go @@ -84,3 +84,15 @@ func TestStartHealthUpdatesBackendStatus(t *testing.T) { } t.Fatalf("health did not update: %+v", p.Status()) } + +func TestPoolBackendDownIsUnavailable(t *testing.T) { + backend := httptest.NewServer(http.NotFoundHandler()) + backend.Close() + p, err := NewPool([]string{backend.URL}, func(*http.Request) string { return "" }) + if err != nil { + t.Fatal(err) + } + rec := httptest.NewRecorder() + p.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader(`{}`))) + assertUnavailable(t, rec) +} From 9c0b84cc4dab8340fdecc07e50fd8c98702d318a Mon Sep 17 00:00:00 2001 From: DevelopmentCats Date: Tue, 10 Nov 2026 21:32:34 +0000 Subject: [PATCH 2/2] router: address review on loading 503 --- internal/cli/router.go | 6 +----- internal/cli/router_internal_test.go | 13 ++++++++++++ internal/router/models.go | 22 ++++++++++++++----- internal/router/models_internal_test.go | 28 ++++++++++++++++++++++++- 4 files changed, 58 insertions(+), 11 deletions(-) diff --git a/internal/cli/router.go b/internal/cli/router.go index b20fbaf..c047411 100644 --- a/internal/cli/router.go +++ b/internal/cli/router.go @@ -15,7 +15,6 @@ import ( "github.com/coder/llm-helper/internal/config" "github.com/coder/llm-helper/internal/control" - "github.com/coder/llm-helper/internal/profile" "github.com/coder/llm-helper/internal/router" "github.com/coder/llm-helper/internal/service" ) @@ -62,10 +61,7 @@ func instanceBackends(cfg *config.Config) func() []router.Backend { } out := make([]router.Backend, 0, len(list)) for _, inst := range list { - var model string - if p, err := profile.Load(cfg.Dirs, inst.Name); err == nil { - model = p.Model() - } + model := inst.Env.Get("MODEL") if c := clusterRunnerForName(cfg, inst.Name); c != nil { if u, err := c.endpointURL(); err == nil { out = append(out, router.Backend{URL: u, Model: model}) diff --git a/internal/cli/router_internal_test.go b/internal/cli/router_internal_test.go index 2851473..2884639 100644 --- a/internal/cli/router_internal_test.go +++ b/internal/cli/router_internal_test.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "net/http/httptest" + "os" "strings" "testing" @@ -61,6 +62,18 @@ func TestRouterBind(t *testing.T) { } } +func TestInstanceBackendsUseRunningModel(t *testing.T) { + cfg := routerTestConfig(t, "PORT=8001\n") + if err := os.WriteFile(cfg.Dirs.ProfileFile("x"), []byte("MODEL=org/edited\n"), 0o600); err != nil { + t.Fatal(err) + } + writeInstance(t, cfg.Dirs, "x", "PORT=9123\nMODEL=org/running\n") + got := instanceBackends(cfg)() + if len(got) != 1 || got[0].URL != "http://localhost:9123" || got[0].Model != "org/running" { + t.Errorf("backends: %+v", got) + } +} + func TestRouterMuxRequiresKey(t *testing.T) { mux := routerMux(router.NewModels(func() []router.Backend { return nil }, "k"), "k") for _, path := range []string{"/v1/models", "/router/status"} { diff --git a/internal/router/models.go b/internal/router/models.go index 83edadd..d9dd8dd 100644 --- a/internal/router/models.go +++ b/internal/router/models.go @@ -26,6 +26,7 @@ type Models struct { mu sync.RWMutex routes map[string]*url.URL starting map[string]bool + learned map[string][]string entries []json.RawMessage urls []*url.URL tried time.Time @@ -63,8 +64,12 @@ type modelID struct { // Refresh asks every backend for its models. When two backends serve the // same id, the earlier one keeps it. The model of a backend whose /v1/models -// does not answer yet is starting. +// does not answer yet is starting, as are the ids it last advertised. func (m *Models) Refresh(ctx context.Context) { + m.mu.RLock() + prev := m.learned + m.mu.RUnlock() + learned := map[string][]string{} routes := map[string]*url.URL{} var entries []json.RawMessage var urls []*url.URL @@ -75,15 +80,19 @@ func (m *Models) Refresh(ctx context.Context) { continue } urls = append(urls, u) + key := u.String() + "\x00" + b.Model list, ok := m.fetch(ctx, u) if !ok && b.Model != "" { waiting = append(waiting, b.Model) + waiting = append(waiting, prev[key]...) + learned[key] = prev[key] } for _, e := range list { var id modelID if json.Unmarshal(e, &id) != nil || id.ID == "" { continue } + learned[key] = append(learned[key], id.ID) if _, dup := routes[id.ID]; dup { continue } @@ -98,7 +107,7 @@ func (m *Models) Refresh(ctx context.Context) { } } m.mu.Lock() - m.routes, m.starting, m.entries, m.urls, m.tried = routes, starting, entries, urls, time.Now() + m.routes, m.starting, m.learned, m.entries, m.urls, m.tried = routes, starting, learned, entries, urls, time.Now() m.mu.Unlock() } @@ -215,14 +224,17 @@ func (m *Models) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } m.mu.RLock() + u = m.routes[model] starting := m.starting[model] m.mu.RUnlock() - if starting { + if u == nil && starting { unavailable(w, fmt.Sprintf("%s is still loading; try again shortly", model)) return } - apiError(w, http.StatusNotFound, fmt.Sprintf("no running instance serves the model %q", model), "model_not_found") - return + if u == nil { + apiError(w, http.StatusNotFound, fmt.Sprintf("no running instance serves the model %q", model), "model_not_found") + return + } } rx := httputil.NewSingleHostReverseProxy(u) direct := rx.Director diff --git a/internal/router/models_internal_test.go b/internal/router/models_internal_test.go index e783ca6..f4aee23 100644 --- a/internal/router/models_internal_test.go +++ b/internal/router/models_internal_test.go @@ -213,12 +213,38 @@ func TestModelsStartingInstance(t *testing.T) { } func TestModelsStartingNotStuckOnRename(t *testing.T) { - a := fakeInstance(t, "a", "served-name") + var down atomic.Bool + a := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if down.Load() { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + if r.URL.Path == "/v1/models" { + _, _ = io.WriteString(w, `{"data":[{"id":"served-name"}]}`) + return + } + _, _ = io.WriteString(w, "a") + })) + t.Cleanup(a.Close) m := NewModels(func() []Backend { return []Backend{{URL: a.URL, Model: "org/model"}} }, "") m.Refresh(context.Background()) if code, _ := do(t, m, http.MethodPost, "/v1/chat/completions", `{"model":"org/model"}`); code != 404 { t.Errorf("an answering instance under another name is not starting: %d", code) } + + down.Store(true) + m.Refresh(context.Background()) + for _, id := range []string{"served-name", "org/model"} { + if code, _ := do(t, m, http.MethodPost, "/v1/chat/completions", `{"model":"`+id+`"}`); code != 503 { + t.Errorf("%s while restarting: %d", id, code) + } + } + + down.Store(false) + m.Refresh(context.Background()) + if code, got := do(t, m, http.MethodPost, "/v1/chat/completions", `{"model":"served-name"}`); code != 200 || got != "a" { + t.Errorf("after restart: %d %q", code, got) + } } func TestModelsBackendDownBody(t *testing.T) {