fix: preserve models during catalog outages

CHANGELOG.md

@@ -7,12 +7,14 @@ Write each change in both `### English` and `### 中文` under `## Unreleased`.
 
 ### English
 
+- Preserve cached model routes when the dynamic model catalog is temporarily unavailable, while refreshing genuine model misses
 - Let WorkBuddy accounts choose their automatic daily check-in time during creation, with a 21:00 retry for failed attempts
 - Enlarge the account-name input in the create-account wizard
 - Open request-history details immediately with a loading skeleton while the full record is fetched
 
 ### 中文
 
+- 动态模型目录暂时不可用时保留已有模型路由，真实模型缺失时立即刷新目录
 - WorkBuddy 账号创建时可选择每日自动签到时间，失败会在 21:00 重试
 - 放大创建账号向导中的账号名称输入框
 - 点击请求历史后立即打开详情弹窗，并在完整记录获取期间显示骨架屏

internal/accounts/classify.go

@@ -200,6 +200,9 @@ func Classify(status int, body, retryAfter, kindHint, failoverHint string) Class
 	msg, code, typ, nestedKind := extractError(body)
 	kind := strings.TrimSpace(firstNonEmpty(kindHint, nestedKind))
 	lower := strings.ToLower(msg + " " + code + " " + typ)
+	catalogUnavailable := strings.Contains(lower, "model_catalog_unavailable") ||
+		strings.Contains(lower, "dynamic model catalog is unavailable") ||
+		strings.Contains(lower, "model catalog unavailable")
 	if kind == "" {
 		switch {
 		case quotaLike(lower, code, typ):
@@ -282,6 +285,11 @@ func Classify(status int, body, retryAfter, kindHint, failoverHint string) Class
 		out.Cooldown = 0
 		out.Type = firstNonEmpty(typ, "invalid_request_error")
 		out.Code = firstNonEmpty(code, "model_not_available")
+		if catalogUnavailable {
+			out.Status = 503
+			out.Type = "api_error"
+			out.Code = "model_catalog_unavailable"
+		}
 	default:
 		if status >= 500 {
 			out.Status = status

internal/accounts/classify_test.go

@@ -69,6 +69,16 @@ func TestClassifyModelNotAvailableDoesNotCooldown(t *testing.T) {
 	}
 }
 
+func TestClassifyModelCatalogUnavailablePreservesDistinctCode(t *testing.T) {
+	got := Classify(400, `{"error":{"message":"model_catalog_unavailable: dynamic model catalog is unavailable","code":"model_not_available"}}`, "", "", "")
+	if got.Kind != KindModelNotAvailable || !got.Failover || got.Cooldown != 0 {
+		t.Fatalf("got %#v", got)
+	}
+	if got.Status != 503 || got.Code != "model_catalog_unavailable" || got.Type != "api_error" {
+		t.Fatalf("catalog outage must remain distinguishable, got %#v", got)
+	}
+}
+
 func TestClassifyTraePlanLimitIsQuota(t *testing.T) {
 	got := Classify(0, `{"code":1005,"message":""}`, "", "", "")
 	if got.Kind != KindQuota || got.Failover || got.Status != 429 {

internal/accounts/pool.go

@@ -1148,6 +1148,9 @@ func (p *Pool) RemoveModel(id, model string) {
 			p.items[i].Models = next
 		}
 		dropProvenModel(&p.items[i], want)
+		// Force the manager to refresh this account's catalog on the next
+		// request instead of trusting the now-mutated snapshot for the TTL.
+		p.items[i].ModelsAt = time.Time{}
 		return
 	}
 }
@@ -1387,7 +1390,9 @@ func (p *Pool) Items() []Item {
 	p.mu.Lock()
 	defer p.mu.Unlock()
 	items := make([]Item, len(p.items))
-	copy(items, p.items)
+	for i := range p.items {
+		items[i] = p.items[i].clone()
+	}
 	return items
 }
 

internal/executor/chat.go

@@ -7,7 +7,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
+	"sort"
 	"strings"
 	"time"
 
@@ -300,7 +302,7 @@ func (e ChatExecutor) routeQuery(prefer, providerFilter, regionFilter, publicMod
 	}
 }
 
-func (e ChatExecutor) pick(prefer, providerFilter, regionFilter, publicModel string, allowed []string, excluded map[string]struct{}) (accounts.Item, error) {
+func (e ChatExecutor) pick(requestID, prefer, providerFilter, regionFilter, publicModel string, allowed []string, excluded map[string]struct{}) (accounts.Item, error) {
 	query := e.routeQuery(prefer, providerFilter, regionFilter, publicModel, allowed, excluded)
 	if e.Pool != nil {
 		if item, ok := e.Pool.PickRoute(query); ok {
@@ -328,7 +330,8 @@ func (e ChatExecutor) pick(prefer, providerFilter, regionFilter, publicModel str
 			unfiltered := query
 			unfiltered.PublicModel = ""
 			if e.Pool.LenRoute(unfiltered) > 0 {
-				return accounts.Item{}, fmt.Errorf("model_not_available: %s is not available for this Qoder account", publicModel)
+				e.logModelRouteMiss(requestID, query)
+				return accounts.Item{}, fmt.Errorf("model_not_available: %s is not available for the selected accounts", publicModel)
 			}
 		}
 	}
@@ -347,6 +350,77 @@ func (e ChatExecutor) pick(prefer, providerFilter, regionFilter, publicModel str
 	return accounts.Item{}, fmt.Errorf("no worker accounts configured")
 }
 
+func (e ChatExecutor) logModelRouteMiss(requestID string, query accounts.RouteQuery) {
+	if e.Pool == nil {
+		return
+	}
+	allowed := append([]string(nil), query.AllowedProviders...)
+	sort.Strings(allowed)
+	excluded := make([]string, 0, len(query.Excluded))
+	for id := range query.Excluded {
+		excluded = append(excluded, id)
+	}
+	sort.Strings(excluded)
+	summaries := make([]string, 0, e.Pool.Len())
+	for _, item := range e.Pool.Items() {
+		summaries = append(summaries, modelRouteAccountSummary(item, query.PublicModel, query.Excluded))
+	}
+	log.Printf("model route unavailable request_id=%q model=%q provider=%q region=%q prefer=%q allowed=%q excluded=%q accounts=[%s]",
+		requestID, query.PublicModel, query.ProviderFilter, query.RegionFilter, query.PreferAccount,
+		strings.Join(allowed, ","), strings.Join(excluded, ","), strings.Join(summaries, " "))
+}
+
+func modelRouteAccountSummary(item accounts.Item, publicModel string, excluded map[string]struct{}) string {
+	ready := item.Ready == nil || *item.Ready
+	hot := item.Hot != nil && *item.Hot
+	quotaExceeded := item.Quota != nil && item.Quota.Exceeded
+	_, isExcluded := excluded[item.ID]
+	want := accounts.CanonicalModelID(publicModel)
+	catalogHas, provenHas := false, false
+	for _, model := range item.Models {
+		catalogHas = catalogHas || accounts.CanonicalModelID(model) == want
+	}
+	for _, model := range item.ProvenModels {
+		provenHas = provenHas || accounts.CanonicalModelID(model) == want
+	}
+	catalog := "unknown"
+	if item.Models != nil {
+		catalog = fmt.Sprintf("count:%d models:%s", len(item.Models), compactModelList(item.Models, 12))
+	}
+	catalogAge := "unknown"
+	if !item.ModelsAt.IsZero() {
+		catalogAge = time.Since(item.ModelsAt).Round(time.Second).String()
+	}
+	modelDownUntil := time.Time{}
+	if item.ModelDownUntil != nil {
+		modelDownUntil = item.ModelDownUntil[want]
+	}
+	return fmt.Sprintf("{id:%q provider:%q region:%q ready:%t hot:%t quota_exceeded:%t down_until:%q model_down_until:%q in_flight:%d/%d excluded:%t catalog_has:%t proven_has:%t catalog_age:%q catalog:%q}",
+		item.ID, item.Provider, item.Region, ready, hot, quotaExceeded, logTime(item.DownUntil), logTime(modelDownUntil),
+		item.InFlight, item.MaxInFlight, isExcluded, catalogHas, provenHas, catalogAge, catalog)
+}
+
+func compactModelList(models []string, limit int) string {
+	if len(models) == 0 {
+		return "[]"
+	}
+	if limit <= 0 || limit > len(models) {
+		limit = len(models)
+	}
+	shown := append([]string(nil), models[:limit]...)
+	if limit < len(models) {
+		return fmt.Sprintf("[%s,+%d]", strings.Join(shown, ","), len(models)-limit)
+	}
+	return "[" + strings.Join(shown, ",") + "]"
+}
+
+func logTime(value time.Time) string {
+	if value.IsZero() {
+		return ""
+	}
+	return value.UTC().Format(time.RFC3339)
+}
+
 func coolingPickError(item accounts.Item, publicModel string, retryAfter time.Duration) error {
 	failover := true
 	kind := accounts.KindRateLimit
@@ -429,10 +503,7 @@ func (e ChatExecutor) ObserveStreamFailure(accountID string, err error, model st
 		return
 	}
 	if classified.Kind == accounts.KindModelNotAvailable {
-		// Streaming 200 headers already recorded this model as proven.
-		// MarkClassified ignores catalog misses, so drop the stale ID
-		// here or the next pick will keep sending it.
-		e.Pool.RemoveModel(accountID, model)
+		e.handleModelAvailabilityFailure("", "stream_body", accountID, model, classified)
 		return
 	}
 	if classified.Kind == accounts.KindQuota {
@@ -449,6 +520,39 @@ func (e ChatExecutor) ObserveStreamFailure(accountID string, err error, model st
 	e.markClassified(accountID, classified, model)
 }
 
+func (e ChatExecutor) handleModelAvailabilityFailure(requestID, source, accountID, model string, classified accounts.Classified) {
+	if e.Pool == nil || accountID == "" {
+		return
+	}
+	item, _ := e.Pool.ByID(accountID)
+	action := "preserve_catalog"
+	if shouldEvictUnavailableModel(classified) {
+		e.Pool.RemoveModel(accountID, model)
+		action = "evict_model"
+	}
+	log.Printf("model route account failure request_id=%q source=%q account=%q provider=%q region=%q model=%q kind=%q code=%q status=%d action=%q message=%q account_state=%s",
+		requestID, source, accountID, item.Provider, item.Region, model, classified.Kind, classified.Code,
+		classified.Status, action, truncateLogValue(classified.Message, 300), modelRouteAccountSummary(item, model, nil))
+}
+
+func shouldEvictUnavailableModel(classified accounts.Classified) bool {
+	if classified.Kind != accounts.KindModelNotAvailable {
+		return false
+	}
+	searchable := strings.ToLower(strings.Join([]string{classified.Code, classified.Message}, " "))
+	return !strings.Contains(searchable, "model_catalog_unavailable") &&
+		!strings.Contains(searchable, "dynamic model catalog is unavailable") &&
+		!strings.Contains(searchable, "model catalog unavailable")
+}
+
+func truncateLogValue(value string, limit int) string {
+	value = strings.TrimSpace(value)
+	if limit <= 0 || len(value) <= limit {
+		return value
+	}
+	return value[:limit] + "..."
+}
+
 // markClassified records a classified failure. model scopes the cooldown to
 // the requested public model so one rate-limited model does not take the
 // whole account offline; pass "" for an account-wide cooldown.
@@ -525,6 +629,7 @@ func classifyWorkerErr(resp *http.Response, body string) accounts.Classified {
 }
 
 type routeLoop struct {
+	requestID      string
 	prefer         string
 	providerFilter string
 	regionFilter   string
@@ -546,6 +651,7 @@ func (e ChatExecutor) newRouteLoop(ctx context.Context, prefer, providerFilter s
 	}
 	allowed := allowedProvidersFrom(ctx)
 	loop := routeLoop{
+		requestID:      RequestIDFromContext(ctx),
 		prefer:         prefer,
 		providerFilter: providerFilter,
 		regionFilter:   regionFilter,
@@ -561,7 +667,7 @@ func (e ChatExecutor) newRouteLoop(ctx context.Context, prefer, providerFilter s
 }
 
 func (l *routeLoop) pickNext(e ChatExecutor, publicModel string) (accounts.Item, int, error) {
-	item, err := e.pick(l.prefer, l.providerFilter, l.regionFilter, publicModel, l.allowed, l.excluded)
+	item, err := e.pick(l.requestID, l.prefer, l.providerFilter, l.regionFilter, publicModel, l.allowed, l.excluded)
 	if err != nil {
 		return accounts.Item{}, l.index, err
 	}
@@ -675,7 +781,7 @@ func (e ChatExecutor) ChatNonStream(ctx context.Context, req translate.ChatReque
 			classified := classifyWorkerErr(resp, msg)
 			loop.lastErr = providerErrorFromClassified(classified)
 			if classified.Kind == accounts.KindModelNotAvailable {
-				e.Pool.RemoveModel(item.ID, req.Model)
+				e.handleModelAvailabilityFailure(RequestIDFromContext(ctx), "worker_non_stream", item.ID, req.Model, classified)
 			}
 			e.markClassified(item.ID, classified, req.Model)
 			status := accounts.AttemptStatusError
@@ -755,7 +861,7 @@ func (e ChatExecutor) chatInProcessNonStreamAttempt(ctx context.Context, item ac
 		}
 		classified := e.classifyInProcessError(err)
 		if classified.Kind == accounts.KindModelNotAvailable {
-			e.Pool.RemoveModel(item.ID, req.Model)
+			e.handleModelAvailabilityFailure(RequestIDFromContext(ctx), "provider_non_stream", item.ID, req.Model, classified)
 		}
 		e.markClassified(item.ID, classified, req.Model)
 		status := accounts.AttemptStatusError
@@ -804,7 +910,7 @@ func (e ChatExecutor) chatInProcessStreamAttempt(ctx context.Context, item accou
 		}
 		classified := e.classifyInProcessError(err)
 		if classified.Kind == accounts.KindModelNotAvailable {
-			e.Pool.RemoveModel(item.ID, req.Model)
+			e.handleModelAvailabilityFailure(RequestIDFromContext(ctx), "provider_stream", item.ID, req.Model, classified)
 		}
 		e.markClassified(item.ID, classified, req.Model)
 		status := accounts.AttemptStatusError
@@ -1062,7 +1168,7 @@ func (e ChatExecutor) ChatStreamProxy(ctx context.Context, req translate.ChatReq
 			classified := classifyWorkerErr(resp, msg)
 			loop.lastErr = providerErrorFromClassified(classified)
 			if classified.Kind == accounts.KindModelNotAvailable {
-				e.Pool.RemoveModel(item.ID, req.Model)
+				e.handleModelAvailabilityFailure(RequestIDFromContext(ctx), "worker_stream", item.ID, req.Model, classified)
 			}
 			e.markClassified(item.ID, classified, req.Model)
 			finished := time.Now().UTC()

internal/executor/chat_test.go

@@ -625,11 +625,39 @@ func TestObserveStreamFailureDropsProvenModel(t *testing.T) {
 	if len(item.ProvenModels) != 0 {
 		t.Fatalf("stream catalog miss must drop proven model, got %v", item.ProvenModels)
 	}
+	if !item.ModelsAt.IsZero() {
+		t.Fatalf("explicit model miss must invalidate catalog freshness, got %s", item.ModelsAt)
+	}
 	if _, ok := pool.PickRoute(accounts.RouteQuery{PublicModel: "deepseek-v4-flash", ProviderFilter: "workbuddy"}); ok {
 		t.Fatal("account must leave the omitted-model route after a stream catalog miss")
 	}
 }
 
+func TestObserveStreamCatalogUnavailablePreservesModel(t *testing.T) {
+	pool := accounts.NewPool(nil, nil)
+	pool.Upsert(accounts.Item{ID: "ready", Provider: "workbuddy", Region: "cn", Runtime: "child_process"})
+	pool.MergeModels("ready", []string{"deepseek-v4-flash"})
+
+	ex := NewChatExecutor(pool, "")
+	ex.ObserveStreamFailure("ready", &providers.Error{
+		Kind:    accounts.KindModelNotAvailable,
+		Status:  503,
+		Code:    "model_catalog_unavailable",
+		Message: "model_catalog_unavailable: dynamic model catalog is unavailable",
+	}, "deepseek-v4-flash")
+
+	item, _ := pool.ByID("ready")
+	if len(item.Models) != 1 || item.Models[0] != "deepseek-v4-flash" {
+		t.Fatalf("catalog outage must preserve cached model, got %v", item.Models)
+	}
+	if item.ModelsAt.IsZero() {
+		t.Fatal("catalog outage must preserve the last successful snapshot timestamp")
+	}
+	if _, ok := pool.PickRoute(accounts.RouteQuery{PublicModel: "deepseek-v4-flash", ProviderFilter: "workbuddy"}); !ok {
+		t.Fatal("catalog outage must not remove a healthy account from the model route")
+	}
+}
+
 func TestObserveStreamFailureWithoutModelTakesAccountDown(t *testing.T) {
 	pool := accounts.NewPool([]string{"http://127.0.0.1:1"}, []string{"acc-quota"})
 	pool.Upsert(accounts.Item{ID: "acc-quota"})

worker/src/errors.mjs

@@ -128,6 +128,7 @@ export function classifyError(input = {}) {
   const payloadCode = String(payload.code ?? payload.msgCode ?? nestedPayload.code ?? nestedPayload.msgCode ?? "").trim();
   const payloadType = String(payload.type || nestedPayload.type || "").trim();
   const searchable = `${message} ${payloadCode} ${payloadType} ${payload.msg || nestedPayload.msg || ""}`;
+  const catalogUnavailable = /model_catalog_unavailable|dynamic model catalog is unavailable|model catalog unavailable/i.test(searchable);
   const retryRaw = input.retryAfter ?? input.retry_after ?? payload.retry_after ?? payload.retryAfter ?? nestedPayload.retry_after ?? nestedPayload.retryAfter;
 
   let kind = kindHint;
@@ -175,12 +176,12 @@ export function classifyError(input = {}) {
   }
   return {
     kind,
-    status: kind === KIND_QUOTA ? 429 : status || conf.status,
+    status: kind === KIND_QUOTA ? 429 : catalogUnavailable ? 503 : status || conf.status,
     failover: typeof payload.failover === "boolean" ? payload.failover : conf.failover,
     cooldownSec: retryAfterSec,
     retryAfterSec,
-    code: payloadCode || conf.code,
-    type: payloadType || conf.type,
+    code: catalogUnavailable ? "model_catalog_unavailable" : payloadCode || conf.code,
+    type: catalogUnavailable ? "api_error" : payloadType || conf.type,
     message: message || payloadCode || conf.code,
   };
 }

worker/test/errors.test.mjs

@@ -64,6 +64,18 @@ test("model_not_available failovers without cooldown", () => {
   assert.equal(got.cooldownSec, 0);
 });
 
+test("model catalog outage keeps a distinct retryable code", () => {
+  const got = classifyError({
+    message: "model_catalog_unavailable: Qoder dynamic model catalog is unavailable",
+  });
+  assert.equal(got.kind, "model_not_available");
+  assert.equal(got.status, 503);
+  assert.equal(got.failover, true);
+  assert.equal(got.cooldownSec, 0);
+  assert.equal(got.code, "model_catalog_unavailable");
+  assert.equal(got.type, "api_error");
+});
+
 test("shouldFailover reads nested JSON", () => {
   assert.equal(
     shouldFailover(429, JSON.stringify({ error: { message: "insufficient_quota", code: "insufficient_quota" } })),