diff --git a/CHANGELOG.md b/CHANGELOG.md index ed05d4e..9edbe18 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,12 +7,14 @@ Write each change in both `### English` and `### 中文` under `## Unreleased`. ### English +- Preserve cached model routes when the dynamic model catalog is temporarily unavailable, while refreshing genuine model misses - Let WorkBuddy accounts choose their automatic daily check-in time during creation, with a 21:00 retry for failed attempts - Enlarge the account-name input in the create-account wizard - Open request-history details immediately with a loading skeleton while the full record is fetched ### 中文 +- 动态模型目录暂时不可用时保留已有模型路由，真实模型缺失时立即刷新目录 - WorkBuddy 账号创建时可选择每日自动签到时间，失败会在 21:00 重试 - 放大创建账号向导中的账号名称输入框 - 点击请求历史后立即打开详情弹窗，并在完整记录获取期间显示骨架屏 diff --git a/internal/accounts/classify.go b/internal/accounts/classify.go index a35fbd2..30c957c 100644 --- a/internal/accounts/classify.go +++ b/internal/accounts/classify.go @@ -200,6 +200,9 @@ func Classify(status int, body, retryAfter, kindHint, failoverHint string) Class msg, code, typ, nestedKind := extractError(body) kind := strings.TrimSpace(firstNonEmpty(kindHint, nestedKind)) lower := strings.ToLower(msg + " " + code + " " + typ) + catalogUnavailable := strings.Contains(lower, "model_catalog_unavailable") || + strings.Contains(lower, "dynamic model catalog is unavailable") || + strings.Contains(lower, "model catalog unavailable") if kind == "" { switch { case quotaLike(lower, code, typ): @@ -282,6 +285,11 @@ func Classify(status int, body, retryAfter, kindHint, failoverHint string) Class out.Cooldown = 0 out.Type = firstNonEmpty(typ, "invalid_request_error") out.Code = firstNonEmpty(code, "model_not_available") + if catalogUnavailable { + out.Status = 503 + out.Type = "api_error" + out.Code = "model_catalog_unavailable" + } default: if status >= 500 { out.Status = status diff --git a/internal/accounts/classify_test.go b/internal/accounts/classify_test.go index ee5e83e..f9593bf 100644 --- a/internal/accounts/classify_test.go +++ b/internal/accounts/classify_test.go @@ -69,6 +69,16 @@ func TestClassifyModelNotAvailableDoesNotCooldown(t *testing.T) { } } +func TestClassifyModelCatalogUnavailablePreservesDistinctCode(t *testing.T) { + got := Classify(400, `{"error":{"message":"model_catalog_unavailable: dynamic model catalog is unavailable","code":"model_not_available"}}`, "", "", "") + if got.Kind != KindModelNotAvailable || !got.Failover || got.Cooldown != 0 { + t.Fatalf("got %#v", got) + } + if got.Status != 503 || got.Code != "model_catalog_unavailable" || got.Type != "api_error" { + t.Fatalf("catalog outage must remain distinguishable, got %#v", got) + } +} + func TestClassifyTraePlanLimitIsQuota(t *testing.T) { got := Classify(0, `{"code":1005,"message":""}`, "", "", "") if got.Kind != KindQuota || got.Failover || got.Status != 429 { diff --git a/internal/accounts/pool.go b/internal/accounts/pool.go index 1375391..050db84 100644 --- a/internal/accounts/pool.go +++ b/internal/accounts/pool.go @@ -1148,6 +1148,9 @@ func (p *Pool) RemoveModel(id, model string) { p.items[i].Models = next } dropProvenModel(&p.items[i], want) + // Force the manager to refresh this account's catalog on the next + // request instead of trusting the now-mutated snapshot for the TTL. + p.items[i].ModelsAt = time.Time{} return } } @@ -1387,7 +1390,9 @@ func (p *Pool) Items() []Item { p.mu.Lock() defer p.mu.Unlock() items := make([]Item, len(p.items)) - copy(items, p.items) + for i := range p.items { + items[i] = p.items[i].clone() + } return items } diff --git a/internal/executor/chat.go b/internal/executor/chat.go index a8df010..7f47666 100644 --- a/internal/executor/chat.go +++ b/internal/executor/chat.go @@ -7,7 +7,9 @@ import ( "errors" "fmt" "io" + "log" "net/http" + "sort" "strings" "time" @@ -300,7 +302,7 @@ func (e ChatExecutor) routeQuery(prefer, providerFilter, regionFilter, publicMod } } -func (e ChatExecutor) pick(prefer, providerFilter, regionFilter, publicModel string, allowed []string, excluded map[string]struct{}) (accounts.Item, error) { +func (e ChatExecutor) pick(requestID, prefer, providerFilter, regionFilter, publicModel string, allowed []string, excluded map[string]struct{}) (accounts.Item, error) { query := e.routeQuery(prefer, providerFilter, regionFilter, publicModel, allowed, excluded) if e.Pool != nil { if item, ok := e.Pool.PickRoute(query); ok { @@ -328,7 +330,8 @@ func (e ChatExecutor) pick(prefer, providerFilter, regionFilter, publicModel str unfiltered := query unfiltered.PublicModel = "" if e.Pool.LenRoute(unfiltered) > 0 { - return accounts.Item{}, fmt.Errorf("model_not_available: %s is not available for this Qoder account", publicModel) + e.logModelRouteMiss(requestID, query) + return accounts.Item{}, fmt.Errorf("model_not_available: %s is not available for the selected accounts", publicModel) } } } @@ -347,6 +350,77 @@ func (e ChatExecutor) pick(prefer, providerFilter, regionFilter, publicModel str return accounts.Item{}, fmt.Errorf("no worker accounts configured") } +func (e ChatExecutor) logModelRouteMiss(requestID string, query accounts.RouteQuery) { + if e.Pool == nil { + return + } + allowed := append([]string(nil), query.AllowedProviders...) + sort.Strings(allowed) + excluded := make([]string, 0, len(query.Excluded)) + for id := range query.Excluded { + excluded = append(excluded, id) + } + sort.Strings(excluded) + summaries := make([]string, 0, e.Pool.Len()) + for _, item := range e.Pool.Items() { + summaries = append(summaries, modelRouteAccountSummary(item, query.PublicModel, query.Excluded)) + } + log.Printf("model route unavailable request_id=%q model=%q provider=%q region=%q prefer=%q allowed=%q excluded=%q accounts=[%s]", + requestID, query.PublicModel, query.ProviderFilter, query.RegionFilter, query.PreferAccount, + strings.Join(allowed, ","), strings.Join(excluded, ","), strings.Join(summaries, " ")) +} + +func modelRouteAccountSummary(item accounts.Item, publicModel string, excluded map[string]struct{}) string { + ready := item.Ready == nil || *item.Ready + hot := item.Hot != nil && *item.Hot + quotaExceeded := item.Quota != nil && item.Quota.Exceeded + _, isExcluded := excluded[item.ID] + want := accounts.CanonicalModelID(publicModel) + catalogHas, provenHas := false, false + for _, model := range item.Models { + catalogHas = catalogHas || accounts.CanonicalModelID(model) == want + } + for _, model := range item.ProvenModels { + provenHas = provenHas || accounts.CanonicalModelID(model) == want + } + catalog := "unknown" + if item.Models != nil { + catalog = fmt.Sprintf("count:%d models:%s", len(item.Models), compactModelList(item.Models, 12)) + } + catalogAge := "unknown" + if !item.ModelsAt.IsZero() { + catalogAge = time.Since(item.ModelsAt).Round(time.Second).String() + } + modelDownUntil := time.Time{} + if item.ModelDownUntil != nil { + modelDownUntil = item.ModelDownUntil[want] + } + return fmt.Sprintf("{id:%q provider:%q region:%q ready:%t hot:%t quota_exceeded:%t down_until:%q model_down_until:%q in_flight:%d/%d excluded:%t catalog_has:%t proven_has:%t catalog_age:%q catalog:%q}", + item.ID, item.Provider, item.Region, ready, hot, quotaExceeded, logTime(item.DownUntil), logTime(modelDownUntil), + item.InFlight, item.MaxInFlight, isExcluded, catalogHas, provenHas, catalogAge, catalog) +} + +func compactModelList(models []string, limit int) string { + if len(models) == 0 { + return "[]" + } + if limit <= 0 || limit > len(models) { + limit = len(models) + } + shown := append([]string(nil), models[:limit]...) + if limit < len(models) { + return fmt.Sprintf("[%s,+%d]", strings.Join(shown, ","), len(models)-limit) + } + return "[" + strings.Join(shown, ",") + "]" +} + +func logTime(value time.Time) string { + if value.IsZero() { + return "" + } + return value.UTC().Format(time.RFC3339) +} + func coolingPickError(item accounts.Item, publicModel string, retryAfter time.Duration) error { failover := true kind := accounts.KindRateLimit @@ -429,10 +503,7 @@ func (e ChatExecutor) ObserveStreamFailure(accountID string, err error, model st return } if classified.Kind == accounts.KindModelNotAvailable { - // Streaming 200 headers already recorded this model as proven. - // MarkClassified ignores catalog misses, so drop the stale ID - // here or the next pick will keep sending it. - e.Pool.RemoveModel(accountID, model) + e.handleModelAvailabilityFailure("", "stream_body", accountID, model, classified) return } if classified.Kind == accounts.KindQuota { @@ -449,6 +520,39 @@ func (e ChatExecutor) ObserveStreamFailure(accountID string, err error, model st e.markClassified(accountID, classified, model) } +func (e ChatExecutor) handleModelAvailabilityFailure(requestID, source, accountID, model string, classified accounts.Classified) { + if e.Pool == nil || accountID == "" { + return + } + item, _ := e.Pool.ByID(accountID) + action := "preserve_catalog" + if shouldEvictUnavailableModel(classified) { + e.Pool.RemoveModel(accountID, model) + action = "evict_model" + } + log.Printf("model route account failure request_id=%q source=%q account=%q provider=%q region=%q model=%q kind=%q code=%q status=%d action=%q message=%q account_state=%s", + requestID, source, accountID, item.Provider, item.Region, model, classified.Kind, classified.Code, + classified.Status, action, truncateLogValue(classified.Message, 300), modelRouteAccountSummary(item, model, nil)) +} + +func shouldEvictUnavailableModel(classified accounts.Classified) bool { + if classified.Kind != accounts.KindModelNotAvailable { + return false + } + searchable := strings.ToLower(strings.Join([]string{classified.Code, classified.Message}, " ")) + return !strings.Contains(searchable, "model_catalog_unavailable") && + !strings.Contains(searchable, "dynamic model catalog is unavailable") && + !strings.Contains(searchable, "model catalog unavailable") +} + +func truncateLogValue(value string, limit int) string { + value = strings.TrimSpace(value) + if limit <= 0 || len(value) <= limit { + return value + } + return value[:limit] + "..." +} + // markClassified records a classified failure. model scopes the cooldown to // the requested public model so one rate-limited model does not take the // whole account offline; pass "" for an account-wide cooldown. @@ -525,6 +629,7 @@ func classifyWorkerErr(resp *http.Response, body string) accounts.Classified { } type routeLoop struct { + requestID string prefer string providerFilter string regionFilter string @@ -546,6 +651,7 @@ func (e ChatExecutor) newRouteLoop(ctx context.Context, prefer, providerFilter s } allowed := allowedProvidersFrom(ctx) loop := routeLoop{ + requestID: RequestIDFromContext(ctx), prefer: prefer, providerFilter: providerFilter, regionFilter: regionFilter, @@ -561,7 +667,7 @@ func (e ChatExecutor) newRouteLoop(ctx context.Context, prefer, providerFilter s } func (l *routeLoop) pickNext(e ChatExecutor, publicModel string) (accounts.Item, int, error) { - item, err := e.pick(l.prefer, l.providerFilter, l.regionFilter, publicModel, l.allowed, l.excluded) + item, err := e.pick(l.requestID, l.prefer, l.providerFilter, l.regionFilter, publicModel, l.allowed, l.excluded) if err != nil { return accounts.Item{}, l.index, err } @@ -675,7 +781,7 @@ func (e ChatExecutor) ChatNonStream(ctx context.Context, req translate.ChatReque classified := classifyWorkerErr(resp, msg) loop.lastErr = providerErrorFromClassified(classified) if classified.Kind == accounts.KindModelNotAvailable { - e.Pool.RemoveModel(item.ID, req.Model) + e.handleModelAvailabilityFailure(RequestIDFromContext(ctx), "worker_non_stream", item.ID, req.Model, classified) } e.markClassified(item.ID, classified, req.Model) status := accounts.AttemptStatusError @@ -755,7 +861,7 @@ func (e ChatExecutor) chatInProcessNonStreamAttempt(ctx context.Context, item ac } classified := e.classifyInProcessError(err) if classified.Kind == accounts.KindModelNotAvailable { - e.Pool.RemoveModel(item.ID, req.Model) + e.handleModelAvailabilityFailure(RequestIDFromContext(ctx), "provider_non_stream", item.ID, req.Model, classified) } e.markClassified(item.ID, classified, req.Model) status := accounts.AttemptStatusError @@ -804,7 +910,7 @@ func (e ChatExecutor) chatInProcessStreamAttempt(ctx context.Context, item accou } classified := e.classifyInProcessError(err) if classified.Kind == accounts.KindModelNotAvailable { - e.Pool.RemoveModel(item.ID, req.Model) + e.handleModelAvailabilityFailure(RequestIDFromContext(ctx), "provider_stream", item.ID, req.Model, classified) } e.markClassified(item.ID, classified, req.Model) status := accounts.AttemptStatusError @@ -1062,7 +1168,7 @@ func (e ChatExecutor) ChatStreamProxy(ctx context.Context, req translate.ChatReq classified := classifyWorkerErr(resp, msg) loop.lastErr = providerErrorFromClassified(classified) if classified.Kind == accounts.KindModelNotAvailable { - e.Pool.RemoveModel(item.ID, req.Model) + e.handleModelAvailabilityFailure(RequestIDFromContext(ctx), "worker_stream", item.ID, req.Model, classified) } e.markClassified(item.ID, classified, req.Model) finished := time.Now().UTC() diff --git a/internal/executor/chat_test.go b/internal/executor/chat_test.go index 9dc510b..1affca0 100644 --- a/internal/executor/chat_test.go +++ b/internal/executor/chat_test.go @@ -625,11 +625,39 @@ func TestObserveStreamFailureDropsProvenModel(t *testing.T) { if len(item.ProvenModels) != 0 { t.Fatalf("stream catalog miss must drop proven model, got %v", item.ProvenModels) } + if !item.ModelsAt.IsZero() { + t.Fatalf("explicit model miss must invalidate catalog freshness, got %s", item.ModelsAt) + } if _, ok := pool.PickRoute(accounts.RouteQuery{PublicModel: "deepseek-v4-flash", ProviderFilter: "workbuddy"}); ok { t.Fatal("account must leave the omitted-model route after a stream catalog miss") } } +func TestObserveStreamCatalogUnavailablePreservesModel(t *testing.T) { + pool := accounts.NewPool(nil, nil) + pool.Upsert(accounts.Item{ID: "ready", Provider: "workbuddy", Region: "cn", Runtime: "child_process"}) + pool.MergeModels("ready", []string{"deepseek-v4-flash"}) + + ex := NewChatExecutor(pool, "") + ex.ObserveStreamFailure("ready", &providers.Error{ + Kind: accounts.KindModelNotAvailable, + Status: 503, + Code: "model_catalog_unavailable", + Message: "model_catalog_unavailable: dynamic model catalog is unavailable", + }, "deepseek-v4-flash") + + item, _ := pool.ByID("ready") + if len(item.Models) != 1 || item.Models[0] != "deepseek-v4-flash" { + t.Fatalf("catalog outage must preserve cached model, got %v", item.Models) + } + if item.ModelsAt.IsZero() { + t.Fatal("catalog outage must preserve the last successful snapshot timestamp") + } + if _, ok := pool.PickRoute(accounts.RouteQuery{PublicModel: "deepseek-v4-flash", ProviderFilter: "workbuddy"}); !ok { + t.Fatal("catalog outage must not remove a healthy account from the model route") + } +} + func TestObserveStreamFailureWithoutModelTakesAccountDown(t *testing.T) { pool := accounts.NewPool([]string{"http://127.0.0.1:1"}, []string{"acc-quota"}) pool.Upsert(accounts.Item{ID: "acc-quota"}) diff --git a/worker/src/errors.mjs b/worker/src/errors.mjs index c660ce3..6f198e6 100644 --- a/worker/src/errors.mjs +++ b/worker/src/errors.mjs @@ -128,6 +128,7 @@ export function classifyError(input = {}) { const payloadCode = String(payload.code ?? payload.msgCode ?? nestedPayload.code ?? nestedPayload.msgCode ?? "").trim(); const payloadType = String(payload.type || nestedPayload.type || "").trim(); const searchable = `${message} ${payloadCode} ${payloadType} ${payload.msg || nestedPayload.msg || ""}`; + const catalogUnavailable = /model_catalog_unavailable|dynamic model catalog is unavailable|model catalog unavailable/i.test(searchable); const retryRaw = input.retryAfter ?? input.retry_after ?? payload.retry_after ?? payload.retryAfter ?? nestedPayload.retry_after ?? nestedPayload.retryAfter; let kind = kindHint; @@ -175,12 +176,12 @@ export function classifyError(input = {}) { } return { kind, - status: kind === KIND_QUOTA ? 429 : status || conf.status, + status: kind === KIND_QUOTA ? 429 : catalogUnavailable ? 503 : status || conf.status, failover: typeof payload.failover === "boolean" ? payload.failover : conf.failover, cooldownSec: retryAfterSec, retryAfterSec, - code: payloadCode || conf.code, - type: payloadType || conf.type, + code: catalogUnavailable ? "model_catalog_unavailable" : payloadCode || conf.code, + type: catalogUnavailable ? "api_error" : payloadType || conf.type, message: message || payloadCode || conf.code, }; } diff --git a/worker/test/errors.test.mjs b/worker/test/errors.test.mjs index 625845b..e4eb985 100644 --- a/worker/test/errors.test.mjs +++ b/worker/test/errors.test.mjs @@ -64,6 +64,18 @@ test("model_not_available failovers without cooldown", () => { assert.equal(got.cooldownSec, 0); }); +test("model catalog outage keeps a distinct retryable code", () => { + const got = classifyError({ + message: "model_catalog_unavailable: Qoder dynamic model catalog is unavailable", + }); + assert.equal(got.kind, "model_not_available"); + assert.equal(got.status, 503); + assert.equal(got.failover, true); + assert.equal(got.cooldownSec, 0); + assert.equal(got.code, "model_catalog_unavailable"); + assert.equal(got.type, "api_error"); +}); + test("shouldFailover reads nested JSON", () => { assert.equal( shouldFailover(429, JSON.stringify({ error: { message: "insufficient_quota", code: "insufficient_quota" } })),