From 88515f876f7ae8d499d4c4f70356bac1c55c9619 Mon Sep 17 00:00:00 2001 From: N Rohith Reddy Date: Fri, 17 Jul 2026 13:05:49 -0400 Subject: [PATCH 1/5] test(proxy): add failing OpenAI baseline-failover coverage Lock in ProxyOpenAIChatCompletion parity with Messages for OSS-outage baseline failover (happy path + negative cases) before the port. Signed-off-by: N Rohith Reddy Co-authored-by: Cursor --- .../proxy/openai_baseline_failover_test.go | 221 ++++++++++++++++++ 1 file changed, 221 insertions(+) create mode 100644 internal/proxy/openai_baseline_failover_test.go diff --git a/internal/proxy/openai_baseline_failover_test.go b/internal/proxy/openai_baseline_failover_test.go new file mode 100644 index 00000000..5eedba06 --- /dev/null +++ b/internal/proxy/openai_baseline_failover_test.go @@ -0,0 +1,221 @@ +package proxy_test + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + + "workweave/router/internal/providers" + "workweave/router/internal/providers/anthropic" + "workweave/router/internal/providers/openaicompat" + "workweave/router/internal/proxy" + "workweave/router/internal/router" + "workweave/router/internal/translate" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +// anthropicMessageJSON is a minimal non-streaming Anthropic Messages body the +// OpenAI→Anthropic translator can turn into a chat.completion response. +const anthropicMessageJSON = `{"id":"msg_1","type":"message","role":"assistant","model":"claude-opus-4-8","content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn","usage":{"input_tokens":5,"output_tokens":1}}` + +// TestProxyOpenAI_OSSOutageFailsOverToBaselineAnthropic mirrors +// TestProxyMessages_OSSOutageFailsOverToBaselineAnthropic on the OpenAI wire: +// cost-routed OSS bindings all fail, then the requested Anthropic model serves. +func TestProxyOpenAI_OSSOutageFailsOverToBaselineAnthropic(t *testing.T) { + var ( + mu sync.Mutex + fireworksCount int + openRouterCount int + anthropicCount int + anthropicReceivedModel string + ) + + fail503 := func(counter *int) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + *counter++ + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":{"message":"provider unavailable"}}`)) + } + } + fireworks := httptest.NewServer(fail503(&fireworksCount)) + defer fireworks.Close() + openrouter := httptest.NewServer(fail503(&openRouterCount)) + defer openrouter.Close() + + anthropicUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body, _ := io.ReadAll(r.Body) + mu.Lock() + anthropicCount++ + anthropicReceivedModel = gjson.GetBytes(body, "model").String() + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(anthropicMessageJSON)) + })) + defer anthropicUpstream.Close() + + store := newFakePinStore() + tel := newCaptureTelemetry() + svc := proxy.NewService( + &fakeRouter{decision: router.Decision{Provider: "fireworks", Model: "deepseek/deepseek-v4-pro"}}, + map[string]providers.Client{ + "fireworks": openaicompat.NewClient("test-fw-key", fireworks.URL), + "openrouter": openaicompat.NewClient("test-or-key", openrouter.URL), + "anthropic": anthropic.NewClient("test-anthropic-key", anthropicUpstream.URL), + }, + nil, false, nil, store, false, providers.ProviderAnthropic, "claude-haiku-4-5", tel, + ).WithDeploymentKeyedProviders(map[string]struct{}{ + "fireworks": {}, + "openrouter": {}, + "anthropic": {}, + }) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader("")) + body := []byte(`{"model":"claude-opus-4-8","messages":[{"role":"user","content":"hi"}]}`) + + err := svc.ProxyOpenAIChatCompletion(authedCtx("11111111-1111-1111-1111-111111111111"), body, rec, req) + require.NoError(t, err, "ProxyOpenAIChatCompletion should succeed via baseline failover to Anthropic") + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, 1, fireworksCount, "Fireworks (primary OSS binding) tried once") + assert.Equal(t, 1, openRouterCount, "OpenRouter (OSS fallback binding) tried once") + assert.Equal(t, 1, anthropicCount, "Anthropic baseline failover dispatched once") + assert.Equal(t, "claude-opus-4-8", anthropicReceivedModel, "baseline failover must request the caller's model on Anthropic") + + assert.Equal(t, http.StatusOK, rec.Code) + assert.Equal(t, "anthropic", rec.Header().Get(proxy.HeaderRouterProvider), "served provider header reflects the baseline failover") + assert.Equal(t, "claude-opus-4-8", rec.Header().Get(proxy.HeaderRouterModel), "x-router-model reflects the baseline model that served") + assert.Contains(t, rec.Body.String(), `"object":"chat.completion"`) + assert.NotContains(t, rec.Body.String(), "deepseek/deepseek-v4-pro") + + require.NotEmpty(t, store.usages, "baseline failover must write pin usage") + assert.Equal(t, "claude-opus-4-8", store.usages[len(store.usages)-1].ServedModel, "pin usage records the served baseline model") +} + +// TestProxyOpenAI_ForcedModelUnavailableDoesNotSubstituteAnthropic ensures a +// forced-model OpenAI request hard-fails instead of silently substituting Anthropic. +func TestProxyOpenAI_ForcedModelUnavailableDoesNotSubstituteAnthropic(t *testing.T) { + var anthropicCount int + var googleCount int + anthropicProv := &fakeProvider{proxyResponse: func(w http.ResponseWriter) { + anthropicCount++ + w.WriteHeader(http.StatusOK) + }} + google := &fakeProvider{proxyResponse: func(w http.ResponseWriter) { + googleCount++ + w.WriteHeader(http.StatusOK) + }} + svc := makeProxyService( + router.Decision{ + Provider: providers.ProviderGoogle, + Model: "gemini-3.1-pro-preview", + Reason: translate.ReasonUserForceModel, + }, + map[string]providers.Client{ + providers.ProviderAnthropic: anthropicProv, + providers.ProviderGoogle: google, + }, + ).WithDeploymentKeyedProviders(map[string]struct{}{providers.ProviderAnthropic: {}}) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader("")) + body := []byte(`{"model":"claude-opus-4-8","messages":[{"role":"user","content":"hi"}]}`) + + err := svc.ProxyOpenAIChatCompletion(context.Background(), body, rec, req) + require.Error(t, err) + assert.Equal(t, 0, anthropicCount, "forced model requests must never substitute Anthropic") + assert.Equal(t, 0, googleCount, "unwired forced provider must not be dispatched") +} + +// TestProxyOpenAI_OSSOutageNoBaselineWhenRequestedModelIsOSS: when the caller +// requested the OSS model directly, exhaustion surfaces the upstream error. +func TestProxyOpenAI_OSSOutageNoBaselineWhenRequestedModelIsOSS(t *testing.T) { + var anthropicCount int + anthropicUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + anthropicCount++ + w.WriteHeader(http.StatusOK) + })) + defer anthropicUpstream.Close() + + fail := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":{"message":"down"}}`)) + } + fireworks := httptest.NewServer(http.HandlerFunc(fail)) + defer fireworks.Close() + openrouter := httptest.NewServer(http.HandlerFunc(fail)) + defer openrouter.Close() + + svc := proxy.NewService( + &fakeRouter{decision: router.Decision{Provider: "fireworks", Model: "deepseek/deepseek-v4-pro"}}, + map[string]providers.Client{ + "fireworks": openaicompat.NewClient("k", fireworks.URL), + "openrouter": openaicompat.NewClient("k", openrouter.URL), + "anthropic": anthropic.NewClient("k", anthropicUpstream.URL), + }, + nil, false, nil, nil, false, providers.ProviderAnthropic, "claude-haiku-4-5", nil, + ).WithDeploymentKeyedProviders(map[string]struct{}{ + "fireworks": {}, "openrouter": {}, "anthropic": {}, + }) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader("")) + body := []byte(`{"model":"deepseek/deepseek-v4-pro","messages":[{"role":"user","content":"hi"}]}`) + + _ = svc.ProxyOpenAIChatCompletion(context.Background(), body, rec, req) + assert.Equal(t, 0, anthropicCount, "baseline failover must not fire when the caller requested the OSS model") +} + +// TestProxyOpenAI_OSSOutageNoBaselineWhenAnthropicExcluded: when Anthropic is +// excluded, baseline failover must not retry against it. +func TestProxyOpenAI_OSSOutageNoBaselineWhenAnthropicExcluded(t *testing.T) { + var anthropicCount int + anthropicUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + anthropicCount++ + w.WriteHeader(http.StatusOK) + })) + defer anthropicUpstream.Close() + + fail := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"error":{"message":"provider unavailable"}}`)) + } + fireworks := httptest.NewServer(http.HandlerFunc(fail)) + defer fireworks.Close() + openrouter := httptest.NewServer(http.HandlerFunc(fail)) + defer openrouter.Close() + + svc := proxy.NewService( + &fakeRouter{decision: router.Decision{Provider: "fireworks", Model: "deepseek/deepseek-v4-pro"}}, + map[string]providers.Client{ + "fireworks": openaicompat.NewClient("k", fireworks.URL), + "openrouter": openaicompat.NewClient("k", openrouter.URL), + "anthropic": anthropic.NewClient("k", anthropicUpstream.URL), + }, + nil, false, nil, nil, false, providers.ProviderAnthropic, "claude-haiku-4-5", nil, + ).WithDeploymentKeyedProviders(map[string]struct{}{ + "fireworks": {}, "openrouter": {}, "anthropic": {}, + }).WithExcludedProvidersOverride([]string{providers.ProviderAnthropic}) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader("")) + body := []byte(`{"model":"claude-opus-4-8","messages":[{"role":"user","content":"hi"}]}`) + + _ = svc.ProxyOpenAIChatCompletion(context.Background(), body, rec, req) + assert.Equal(t, 0, anthropicCount, "baseline failover must not hit Anthropic when it is excluded") + assert.NotEqual(t, providers.ProviderAnthropic, rec.Header().Get(proxy.HeaderRouterProvider), "served provider must not be the excluded Anthropic") +} From 0578adafb7b2cd767a3d7505d751b1ae9b4ca2af Mon Sep 17 00:00:00 2001 From: N Rohith Reddy Date: Fri, 17 Jul 2026 13:06:58 -0400 Subject: [PATCH 2/5] fix(proxy): port baseline failover to OpenAI chat completions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a cost-routed OSS binding fails on the OpenAI wire, retry the caller-requested Anthropic model pre-commit — same eligibility and commit gates as ProxyMessages. Signed-off-by: N Rohith Reddy Co-authored-by: Cursor --- internal/proxy/service.go | 116 +++++++++++++++++++++++++++++++++++--- 1 file changed, 108 insertions(+), 8 deletions(-) diff --git a/internal/proxy/service.go b/internal/proxy/service.go index 993c0039..9b4c6975 100644 --- a/internal/proxy/service.go +++ b/internal/proxy/service.go @@ -4541,20 +4541,119 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w return fmt.Errorf("%w: %s (no translation path defined)", ErrProviderNotConfigured, decision.Provider) } + // openaiAnthropicAttempt builds the OpenAI-wire → Anthropic upstream closure + // used by baseline failover (and subscription-credit retry). Mirrors the + // FamilyAnthropic primary attempt, rebuilt per retry with a fresh prep. + openaiAnthropicAttempt := func(prep providers.PreparedRequest, markerText string) dispatchAttempt { + return func(actx context.Context, d router.Decision, p providers.Client) error { + var usageSink otel.UsageSink + if s.usageRequired() { + extractor = otel.NewUsageExtractor(nil, providers.ProviderAnthropic) + usageSink = extractor + } + attemptSink := sink + if !isResponses && !verbatimPassthrough { + mw := translate.NewOpenAIRoutingMarkerWriter(sink, d.Model, markerText) + if err := mw.Prelude(env.Stream()); err != nil { + log.Error("OpenAI routing-marker prelude failed", "err", err) + } + attemptSink = mw + } + translator := translate.NewSSETranslator(attemptSink, d.Model, usageSink) + if preludeBuf != nil { + preludeBuf.Seal() + } + err := p.Proxy(actx, d, prep, translator, r) + if err != nil && env.Stream() && preludeBuf.Committed() { + err = emitOpenAISSEErrorEvent(sink, err) + } + return finalizeAfterProxy(err, translator.Finalize) + } + } + + // In-turn baseline failover eligibility — same predicates as ProxyMessages. + baselineModel := s.baselineFor(feats.Model) + baselineCatalog, baselineKnown := catalog.ByID(baselineModel) + _, anthropicExcluded := s.excludedProvidersForRequest(ctx)[providers.ProviderAnthropic] + baselineEligible := decision.Reason != translate.ReasonUserForceModel && + s.shouldFailover(ctx) && + !anthropicExcluded && + decision.Provider != providers.ProviderAnthropic && + baselineModel != decision.Model && + baselineKnown && baselineCatalog.PrimaryProvider() == providers.ProviderAnthropic + primaryProvider := decision.Provider var winnerIdx int winnerIdx, proxyErr = s.dispatchWithFallback(ctx, failoverInputs{ // contentSink is the raw w when capture is off. - w: contentSink, - buf: preludeBuf, - initialDecision: decision, - bindings: bindings, - attempt: attempt, - flushErr: flushBufferedIfPresent, + w: contentSink, + buf: preludeBuf, + initialDecision: decision, + bindings: bindings, + attempt: attempt, + flushErr: flushBufferedIfPresent, + deferFlushOnExhaustion: baselineEligible, }) + + baselineFailoverUsed := false + baselineAttempted := false + if baselineEligible && proxyErr != nil && !preludeBuf.Committed() && + (providers.IsRetryable(proxyErr) || providers.IsUpstreamModelNotFound(proxyErr)) { + baselineDecision := decision + baselineDecision.Model = baselineModel + baselineDecision.Provider = providers.ProviderAnthropic + baselineOpts := opts + baselineOpts.TargetModel = baselineModel + baselineOpts.TargetProvider = providers.ProviderAnthropic + baselineOpts.Capabilities = router.Lookup(baselineModel) + baselineOpts.ModelSwitched = routeRes.PriorServedModel != baselineModel || routeRes.SessionEverSwitched + if s.effortEscalation { + baselineOpts.ForceReasoningEffort = forcedReasoningEffort(baselineModel, routeRes.EscalateEffort) + } + baselinePrep, baselineEmitErr := env.PrepareAnthropic(r.Header, baselineOpts) + if baselineEmitErr != nil { + log.Error("Baseline failover: emit Anthropic body failed; surfacing original error", "err", baselineEmitErr, "baseline_model", baselineModel) + flushBufferedIfPresent(contentSink, proxyErr) + } else { + log.Warn("Baseline failover: routed model exhausted, retrying requested model on Anthropic", + "failed_model", decision.Model, + "failed_provider", primaryProvider, + "baseline_model", baselineModel, + "err", proxyErr) + baselineCtx := ctx + if s.claudeSubscriptionExhausted(ctx, r.Header) { + baselineCtx = withSuppressedClaudeSubscription(baselineCtx) + } + baselineCtx = resolveAndInjectCredentials(baselineCtx, providers.ProviderAnthropic, r.Header) + baselineBindings := s.resolveBindingsForDispatch(baselineCtx, baselineDecision) + baselineMarker := suppressMarkerIfRequested(r.Header, baselineRoutingMarkerFor(routeRes, baselineModel)) + if billing.SubscriptionOnlyFromContext(ctx) { + baselineMarker = subscriptionOnlyWarningMarkerCodex + } + crossFormat = true + logUpstreamBody(log, routeRes.SessionKey, baselineDecision, feats, baselinePrep.Body) + winnerIdx, proxyErr = s.dispatchWithFallback(baselineCtx, failoverInputs{ + w: contentSink, + buf: preludeBuf, + initialDecision: baselineDecision, + bindings: baselineBindings, + attempt: openaiAnthropicAttempt(baselinePrep, baselineMarker), + flushErr: flushBufferedIfPresent, + }) + decision = baselineDecision + bindings = baselineBindings + baselineAttempted = true + baselineFailoverUsed = proxyErr == nil + } + } else if baselineEligible && proxyErr != nil { + flushBufferedIfPresent(contentSink, proxyErr) + } + finalProvider := primaryProvider if winnerIdx >= 0 && winnerIdx < len(bindings) { finalProvider = bindings[winnerIdx].Provider + } else if baselineAttempted { + finalProvider = providers.ProviderAnthropic } decision.Provider = finalProvider @@ -4612,7 +4711,8 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w String("dispatch.primary_provider", primaryProvider). String("dispatch.final_provider", finalProvider). Int64("dispatch.fallback_attempts", int64(winnerIdx)). - Bool("dispatch.failover_used", finalProvider != primaryProvider) + Bool("dispatch.failover_used", finalProvider != primaryProvider). + Bool("dispatch.baseline_failover", baselineFailoverUsed) applyPlannerAttrs(openaiUpstreamBuilder, routeRes) applyRoutingStateAttrs(openaiUpstreamBuilder, routeRes, decision.Model, sessionKey) addTimingAttrs(ctx, openaiUpstreamBuilder) @@ -4729,7 +4829,7 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w }) } - log.Info("ProxyOpenAIChatCompletion complete", append([]any{"requested_model", feats.Model, "baseline_model", s.baselineFor(feats.Model), "decision_model", decision.Model, "decision_provider", decision.Provider, "primary_provider", primaryProvider, "fallback_attempts", winnerIdx, "failover_used", finalProvider != primaryProvider, "decision_reason", decision.Reason, "requested_tier", routeRes.RequestedTier.String(), "decision_tier", catalog.TierFor(decision.Model).String(), "embedded_tokens", len(promptText) / 4, "total_input_tokens", feats.Tokens, "has_tools", feats.HasTools, "embed_input", embedInput, "cross_format", crossFormat, "sticky_hit", stickyHit, "pin_tier", pinTier, "turn_type", string(tt), "route_ms", routeMs, "proxy_ms", proxyMs, "proxy_err", proxyErr, "upstream_status", upstreamStatus(proxyErr)}, plannerLogFields(routeRes)...)...) + log.Info("ProxyOpenAIChatCompletion complete", append([]any{"requested_model", feats.Model, "baseline_model", s.baselineFor(feats.Model), "decision_model", decision.Model, "decision_provider", decision.Provider, "primary_provider", primaryProvider, "fallback_attempts", winnerIdx, "failover_used", finalProvider != primaryProvider, "baseline_failover", baselineFailoverUsed, "decision_reason", decision.Reason, "requested_tier", routeRes.RequestedTier.String(), "decision_tier", catalog.TierFor(decision.Model).String(), "embedded_tokens", len(promptText) / 4, "total_input_tokens", feats.Tokens, "has_tools", feats.HasTools, "embed_input", embedInput, "cross_format", crossFormat, "sticky_hit", stickyHit, "pin_tier", pinTier, "turn_type", string(tt), "route_ms", routeMs, "proxy_ms", proxyMs, "proxy_err", proxyErr, "upstream_status", upstreamStatus(proxyErr)}, plannerLogFields(routeRes)...)...) s.reportPolicyOutcome(ctx, routeRes, decision, finalProvider, feats.Tokens, in, out, cacheCreation, cacheRead, routeMs, proxyMs, proxyErr, nil) // Subscription-only mode disables paid failover by pinning dispatch to the From 14fbdffdeae272bc444da5786b9ef105c36588c0 Mon Sep 17 00:00:00 2001 From: N Rohith Reddy Date: Fri, 17 Jul 2026 13:07:28 -0400 Subject: [PATCH 3/5] test(proxy): add failing OpenAI subscription-credit retry coverage Cover live 429 and OAuth 401/403 arms on ProxyOpenAIChatCompletion before porting the Messages subscriptionRetryEligible path. Signed-off-by: N Rohith Reddy Co-authored-by: Cursor --- .../openai_subscription_failover_test.go | 169 ++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 internal/proxy/openai_subscription_failover_test.go diff --git a/internal/proxy/openai_subscription_failover_test.go b/internal/proxy/openai_subscription_failover_test.go new file mode 100644 index 00000000..b3d3ff0f --- /dev/null +++ b/internal/proxy/openai_subscription_failover_test.go @@ -0,0 +1,169 @@ +package proxy_test + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" + + "workweave/router/internal/providers" + "workweave/router/internal/proxy" + "workweave/router/internal/proxy/usage" + "workweave/router/internal/router" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + oaiSubFailoverToken = "sk-ant-oat01-openai-sub-failover-token" + oaiSubFailoverModel = "claude-haiku-4-5" + oaiSubFailoverOK = `{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"hi"}],"model":"claude-haiku-4-5","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}` +) + +// oauthRejectThenDeployOK returns a buffered error while an OAuth subscription +// credential is present, then succeeds with no credential (deploy-key path). +type oauthRejectThenDeployOK struct { + mu sync.Mutex + onOAuth error + calls []oauthCallSnap +} + +type oauthCallSnap struct { + nilCreds bool + oauth bool + source string + key string + err error +} + +func (p *oauthRejectThenDeployOK) Proxy(ctx context.Context, decision router.Decision, prep providers.PreparedRequest, w http.ResponseWriter, r *http.Request) error { + creds := proxy.CredentialsFromContext(ctx) + snap := oauthCallSnap{nilCreds: creds == nil} + if creds != nil { + snap.oauth = creds.OAuth + snap.source = creds.Source + snap.key = string(creds.APIKey) + } + p.mu.Lock() + defer p.mu.Unlock() + if creds != nil && creds.OAuth { + snap.err = p.onOAuth + p.calls = append(p.calls, snap) + return p.onOAuth + } + p.calls = append(p.calls, snap) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(oaiSubFailoverOK)) + return nil +} + +func (p *oauthRejectThenDeployOK) Passthrough(context.Context, providers.PreparedRequest, http.ResponseWriter, *http.Request) error { + return fmt.Errorf("unused") +} + +func oaiSubSlackObs(t *testing.T) *usage.Observer { + t.Helper() + obs := usage.NewObserver([]byte("salt"), 10*time.Minute, time.Now) + // Slack (not exhausted): preemptive suppress must not fire; only the + // reactive subscription-credit retry can recover. + obs.Record(obs.Key([]byte(oaiSubFailoverToken)), usage.Snapshot{ + Primary: usage.Window{UsedPercent: 0.50, WindowMinutes: 300}, + }) + return obs +} + +func oaiSubCtx() context.Context { + return context.WithValue(context.Background(), proxy.AnthropicSubscriptionContextKey{}, oaiSubFailoverToken) +} + +func oaiSubMainLoopBody() []byte { + return []byte(`{"model":"` + oaiSubFailoverModel + `","max_tokens":4096,"messages":[{"role":"user","content":"Refactor the auth middleware and add tests."}],"tools":[{"type":"function","function":{"name":"edit_file","parameters":{"type":"object","properties":{}}}}]}`) +} + +func oaiSubFailoverSvc(t *testing.T, p providers.Client) *proxy.Service { + t.Helper() + fr := &fakeRouter{decision: router.Decision{Provider: providers.ProviderAnthropic, Model: oaiSubFailoverModel, Reason: "cluster:test"}} + return proxy.NewService(fr, map[string]providers.Client{providers.ProviderAnthropic: p}, nil, false, nil, nil, false, providers.ProviderAnthropic, oaiSubFailoverModel, nil). + WithSubscriptionAwareRouting(oaiSubSlackObs(t), 0.05, 2.0). + WithDeploymentKeyedProviders(map[string]struct{}{providers.ProviderAnthropic: {}}) +} + +// TestProxyOpenAI_SubscriptionRetry_Live429FailsOverToDeployKey: a live 429 on +// the Claude OAuth token (observer still has slack) must suppress the +// subscription and retry once on the Weave/BYOK key — Messages parity. +func TestProxyOpenAI_SubscriptionRetry_Live429FailsOverToDeployKey(t *testing.T) { + reject := &providers.UpstreamErrorResponse{ + Status: http.StatusTooManyRequests, + Body: []byte(`{"type":"error","error":{"type":"rate_limit_error","message":"weekly limit exceeded"}}`), + } + p := &oauthRejectThenDeployOK{onOAuth: reject} + svc := oaiSubFailoverSvc(t, p) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader("")) + err := svc.ProxyOpenAIChatCompletion(oaiSubCtx(), oaiSubMainLoopBody(), rec, req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, rec.Code) + + p.mu.Lock() + defer p.mu.Unlock() + require.GreaterOrEqual(t, len(p.calls), 2, "must retry after the subscription 429") + assert.True(t, p.calls[0].oauth && p.calls[0].source == "subscription") + last := p.calls[len(p.calls)-1] + assert.True(t, last.nilCreds || !last.oauth, "final dispatch must use the deploy key, not the spent OAuth token") + assert.Contains(t, rec.Body.String(), `"object":"chat.completion"`) +} + +// TestProxyOpenAI_SubscriptionRetry_OAuth401FailsOverToDeployKey covers the +// authentication_error arm of anthropicOAuthCredentialRejected. +func TestProxyOpenAI_SubscriptionRetry_OAuth401FailsOverToDeployKey(t *testing.T) { + reject := &providers.UpstreamErrorResponse{ + Status: http.StatusUnauthorized, + Body: []byte(`{"type":"error","error":{"type":"authentication_error","message":"Invalid authentication credentials"}}`), + } + p := &oauthRejectThenDeployOK{onOAuth: reject} + svc := oaiSubFailoverSvc(t, p) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader("")) + err := svc.ProxyOpenAIChatCompletion(oaiSubCtx(), oaiSubMainLoopBody(), rec, req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, rec.Code) + + p.mu.Lock() + defer p.mu.Unlock() + require.GreaterOrEqual(t, len(p.calls), 2) + assert.True(t, p.calls[0].oauth) + last := p.calls[len(p.calls)-1] + assert.True(t, last.nilCreds || !last.oauth) +} + +// TestProxyOpenAI_SubscriptionRetry_OAuth403FailsOverToDeployKey covers the +// permission_error arm of anthropicOAuthCredentialRejected. +func TestProxyOpenAI_SubscriptionRetry_OAuth403FailsOverToDeployKey(t *testing.T) { + reject := &providers.UpstreamErrorResponse{ + Status: http.StatusForbidden, + Body: []byte(`{"type":"error","error":{"type":"permission_error","message":"OAuth authentication is currently not allowed for this organization."}}`), + } + p := &oauthRejectThenDeployOK{onOAuth: reject} + svc := oaiSubFailoverSvc(t, p) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/v1/chat/completions", strings.NewReader("")) + err := svc.ProxyOpenAIChatCompletion(oaiSubCtx(), oaiSubMainLoopBody(), rec, req) + require.NoError(t, err) + assert.Equal(t, http.StatusOK, rec.Code) + + p.mu.Lock() + defer p.mu.Unlock() + require.GreaterOrEqual(t, len(p.calls), 2) + assert.True(t, p.calls[0].oauth) + last := p.calls[len(p.calls)-1] + assert.True(t, last.nilCreds || !last.oauth) +} From 963314a475079bf8098921ac23e9622aaf875b6e Mon Sep 17 00:00:00 2001 From: N Rohith Reddy Date: Fri, 17 Jul 2026 13:10:49 -0400 Subject: [PATCH 4/5] fix(proxy): port subscription-credit retry to OpenAI chat completions Retry Anthropic-routed OpenAI turns on the Weave/BYOK key after subscription 429 or OAuth 401/403, matching ProxyMessages. Signed-off-by: N Rohith Reddy Co-authored-by: Cursor --- .../openai_subscription_failover_test.go | 6 +- internal/proxy/service.go | 76 +++++++++++++++++-- 2 files changed, 73 insertions(+), 9 deletions(-) diff --git a/internal/proxy/openai_subscription_failover_test.go b/internal/proxy/openai_subscription_failover_test.go index b3d3ff0f..a58b2107 100644 --- a/internal/proxy/openai_subscription_failover_test.go +++ b/internal/proxy/openai_subscription_failover_test.go @@ -28,9 +28,9 @@ const ( // oauthRejectThenDeployOK returns a buffered error while an OAuth subscription // credential is present, then succeeds with no credential (deploy-key path). type oauthRejectThenDeployOK struct { - mu sync.Mutex - onOAuth error - calls []oauthCallSnap + mu sync.Mutex + onOAuth error + calls []oauthCallSnap } type oauthCallSnap struct { diff --git a/internal/proxy/service.go b/internal/proxy/service.go index 9b4c6975..c6addc4f 100644 --- a/internal/proxy/service.go +++ b/internal/proxy/service.go @@ -4582,6 +4582,12 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w baselineModel != decision.Model && baselineKnown && baselineCatalog.PrimaryProvider() == providers.ProviderAnthropic + // Subscription-credit failover eligibility — same predicates as ProxyMessages. + subscriptionRetryEligible := decision.Provider == providers.ProviderAnthropic && + servedOnSubscription(ctx) && + !billing.SubscriptionOnlyFromContext(ctx) && + s.anthropicFallbackKeyAvailable(ctx) + primaryProvider := decision.Provider var winnerIdx int winnerIdx, proxyErr = s.dispatchWithFallback(ctx, failoverInputs{ @@ -4592,7 +4598,7 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w bindings: bindings, attempt: attempt, flushErr: flushBufferedIfPresent, - deferFlushOnExhaustion: baselineEligible, + deferFlushOnExhaustion: baselineEligible || subscriptionRetryEligible, }) baselineFailoverUsed := false @@ -4649,6 +4655,59 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w flushBufferedIfPresent(contentSink, proxyErr) } + // Subscription-credit failover: suppress the OAuth token and retry the SAME + // model once on the Weave/BYOK key when a subscription-served Anthropic turn + // hit a transient fault (429/timeout) or an OAuth rejection (401/403), + // pre-commit. Skipped when baseline failover already ran (non-Anthropic). + subscriptionFailoverUsed := false + subscriptionRetryRan := false + if subscriptionRetryEligible && !baselineAttempted && proxyErr != nil && + !preludeBuf.Committed() && + (providers.IsRetryable(proxyErr) || anthropicOAuthCredentialRejected(proxyErr)) { + subscriptionRetryRan = true + subCtx := withSuppressedClaudeSubscription(ctx) + subCtx = resolveAndInjectCredentials(subCtx, providers.ProviderAnthropic, r.Header) + subOpts := opts + subOpts.TargetProvider = providers.ProviderAnthropic + subOpts.TargetModel = decision.Model + subOpts.Capabilities = router.Lookup(decision.Model) + subPrep, subEmitErr := env.PrepareAnthropic(r.Header, subOpts) + if subEmitErr != nil { + log.Error("Subscription failover: emit Anthropic body failed; surfacing original error", "err", subEmitErr, "model", decision.Model) + flushBufferedIfPresent(contentSink, proxyErr) + } else if subBindings := s.resolveBindingsForDispatch(subCtx, decision); len(subBindings) == 0 { + log.Warn("Subscription failover: no fallback Anthropic binding available; surfacing original error", + "model", decision.Model, + "err", proxyErr, + "upstream_status", upstreamStatus(proxyErr)) + flushBufferedIfPresent(contentSink, proxyErr) + } else { + log.Warn("Subscription failover: subscription throttled/timed out, retrying requested model on Weave key", + "model", decision.Model, + "err", proxyErr, + "upstream_status", upstreamStatus(proxyErr)) + subMarker := marker + if billing.SubscriptionOnlyFromContext(ctx) { + subMarker = subscriptionOnlyWarningMarkerCodex + } + crossFormat = true + logUpstreamBody(log, routeRes.SessionKey, decision, feats, subPrep.Body) + winnerIdx, proxyErr = s.dispatchWithFallback(subCtx, failoverInputs{ + w: contentSink, + buf: preludeBuf, + initialDecision: decision, + bindings: subBindings, + attempt: openaiAnthropicAttempt(subPrep, subMarker), + flushErr: flushBufferedIfPresent, + }) + bindings = subBindings + subscriptionFailoverUsed = proxyErr == nil + } + } + if subscriptionRetryEligible && !baselineAttempted && !subscriptionRetryRan && proxyErr != nil && !preludeBuf.Committed() { + flushBufferedIfPresent(contentSink, proxyErr) + } + finalProvider := primaryProvider if winnerIdx >= 0 && winnerIdx < len(bindings) { finalProvider = bindings[winnerIdx].Provider @@ -4658,7 +4717,11 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w decision.Provider = finalProvider // Re-resolve credentials for the binding that actually served — each - // failover attempt gets its own context with potentially different creds. + // failover attempt gets its own context. Carry suppression forward only + // when the Weave retry actually succeeded. + if subscriptionFailoverUsed { + ctx = withSuppressedClaudeSubscription(ctx) + } ctx = resolveAndInjectCredentials(ctx, finalProvider, r.Header) // Re-resolve pricing for the binding that actually served (see ProxyMessages). @@ -4711,8 +4774,9 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w String("dispatch.primary_provider", primaryProvider). String("dispatch.final_provider", finalProvider). Int64("dispatch.fallback_attempts", int64(winnerIdx)). - Bool("dispatch.failover_used", finalProvider != primaryProvider). - Bool("dispatch.baseline_failover", baselineFailoverUsed) + Bool("dispatch.failover_used", finalProvider != primaryProvider || subscriptionFailoverUsed). + Bool("dispatch.baseline_failover", baselineFailoverUsed). + Bool("dispatch.subscription_failover", subscriptionFailoverUsed) applyPlannerAttrs(openaiUpstreamBuilder, routeRes) applyRoutingStateAttrs(openaiUpstreamBuilder, routeRes, decision.Model, sessionKey) addTimingAttrs(ctx, openaiUpstreamBuilder) @@ -4811,7 +4875,7 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w ClientApp: clientID.ClientApp, TurnType: string(routeRes.TurnType), RolloutID: openaiObs.RolloutID, - FailoverUsed: boolPtrTrue(finalProvider != primaryProvider), + FailoverUsed: boolPtrTrue(finalProvider != primaryProvider || subscriptionFailoverUsed), // (session_key, role) join key — see the Anthropic-path write site. SessionKey: sessionKey[:], Role: routeRes.PinRole, @@ -4829,7 +4893,7 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w }) } - log.Info("ProxyOpenAIChatCompletion complete", append([]any{"requested_model", feats.Model, "baseline_model", s.baselineFor(feats.Model), "decision_model", decision.Model, "decision_provider", decision.Provider, "primary_provider", primaryProvider, "fallback_attempts", winnerIdx, "failover_used", finalProvider != primaryProvider, "baseline_failover", baselineFailoverUsed, "decision_reason", decision.Reason, "requested_tier", routeRes.RequestedTier.String(), "decision_tier", catalog.TierFor(decision.Model).String(), "embedded_tokens", len(promptText) / 4, "total_input_tokens", feats.Tokens, "has_tools", feats.HasTools, "embed_input", embedInput, "cross_format", crossFormat, "sticky_hit", stickyHit, "pin_tier", pinTier, "turn_type", string(tt), "route_ms", routeMs, "proxy_ms", proxyMs, "proxy_err", proxyErr, "upstream_status", upstreamStatus(proxyErr)}, plannerLogFields(routeRes)...)...) + log.Info("ProxyOpenAIChatCompletion complete", append([]any{"requested_model", feats.Model, "baseline_model", s.baselineFor(feats.Model), "decision_model", decision.Model, "decision_provider", decision.Provider, "primary_provider", primaryProvider, "fallback_attempts", winnerIdx, "failover_used", finalProvider != primaryProvider || subscriptionFailoverUsed, "baseline_failover", baselineFailoverUsed, "subscription_failover", subscriptionFailoverUsed, "decision_reason", decision.Reason, "requested_tier", routeRes.RequestedTier.String(), "decision_tier", catalog.TierFor(decision.Model).String(), "embedded_tokens", len(promptText) / 4, "total_input_tokens", feats.Tokens, "has_tools", feats.HasTools, "embed_input", embedInput, "cross_format", crossFormat, "sticky_hit", stickyHit, "pin_tier", pinTier, "turn_type", string(tt), "route_ms", routeMs, "proxy_ms", proxyMs, "proxy_err", proxyErr, "upstream_status", upstreamStatus(proxyErr)}, plannerLogFields(routeRes)...)...) s.reportPolicyOutcome(ctx, routeRes, decision, finalProvider, feats.Tokens, in, out, cacheCreation, cacheRead, routeMs, proxyMs, proxyErr, nil) // Subscription-only mode disables paid failover by pinning dispatch to the From ea24e945a3e7db943140250e39d9bf53ec73b192 Mon Sep 17 00:00:00 2001 From: N Rohith Reddy Date: Fri, 17 Jul 2026 13:56:28 -0400 Subject: [PATCH 5/5] chore(proxy): tighten OpenAI failover comments Keep new comment blocks under the comment-length review threshold. Signed-off-by: N Rohith Reddy Co-authored-by: Cursor --- internal/proxy/openai_baseline_failover_test.go | 16 +++++----------- .../proxy/openai_subscription_failover_test.go | 16 +++++----------- internal/proxy/service.go | 13 +++---------- 3 files changed, 13 insertions(+), 32 deletions(-) diff --git a/internal/proxy/openai_baseline_failover_test.go b/internal/proxy/openai_baseline_failover_test.go index 5eedba06..bbd0a70a 100644 --- a/internal/proxy/openai_baseline_failover_test.go +++ b/internal/proxy/openai_baseline_failover_test.go @@ -21,13 +21,10 @@ import ( "github.com/tidwall/gjson" ) -// anthropicMessageJSON is a minimal non-streaming Anthropic Messages body the -// OpenAI→Anthropic translator can turn into a chat.completion response. +// anthropicMessageJSON is a minimal Anthropic Messages body for OpenAI→Anthropic translation. const anthropicMessageJSON = `{"id":"msg_1","type":"message","role":"assistant","model":"claude-opus-4-8","content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn","usage":{"input_tokens":5,"output_tokens":1}}` -// TestProxyOpenAI_OSSOutageFailsOverToBaselineAnthropic mirrors -// TestProxyMessages_OSSOutageFailsOverToBaselineAnthropic on the OpenAI wire: -// cost-routed OSS bindings all fail, then the requested Anthropic model serves. +// TestProxyOpenAI_OSSOutageFailsOverToBaselineAnthropic: OSS outage → Anthropic baseline on the OpenAI wire. func TestProxyOpenAI_OSSOutageFailsOverToBaselineAnthropic(t *testing.T) { var ( mu sync.Mutex @@ -104,8 +101,7 @@ func TestProxyOpenAI_OSSOutageFailsOverToBaselineAnthropic(t *testing.T) { assert.Equal(t, "claude-opus-4-8", store.usages[len(store.usages)-1].ServedModel, "pin usage records the served baseline model") } -// TestProxyOpenAI_ForcedModelUnavailableDoesNotSubstituteAnthropic ensures a -// forced-model OpenAI request hard-fails instead of silently substituting Anthropic. +// TestProxyOpenAI_ForcedModelUnavailableDoesNotSubstituteAnthropic: forced-model must not substitute Anthropic. func TestProxyOpenAI_ForcedModelUnavailableDoesNotSubstituteAnthropic(t *testing.T) { var anthropicCount int var googleCount int @@ -139,8 +135,7 @@ func TestProxyOpenAI_ForcedModelUnavailableDoesNotSubstituteAnthropic(t *testing assert.Equal(t, 0, googleCount, "unwired forced provider must not be dispatched") } -// TestProxyOpenAI_OSSOutageNoBaselineWhenRequestedModelIsOSS: when the caller -// requested the OSS model directly, exhaustion surfaces the upstream error. +// TestProxyOpenAI_OSSOutageNoBaselineWhenRequestedModelIsOSS: no baseline when the caller requested OSS. func TestProxyOpenAI_OSSOutageNoBaselineWhenRequestedModelIsOSS(t *testing.T) { var anthropicCount int anthropicUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -179,8 +174,7 @@ func TestProxyOpenAI_OSSOutageNoBaselineWhenRequestedModelIsOSS(t *testing.T) { assert.Equal(t, 0, anthropicCount, "baseline failover must not fire when the caller requested the OSS model") } -// TestProxyOpenAI_OSSOutageNoBaselineWhenAnthropicExcluded: when Anthropic is -// excluded, baseline failover must not retry against it. +// TestProxyOpenAI_OSSOutageNoBaselineWhenAnthropicExcluded: no baseline when Anthropic is excluded. func TestProxyOpenAI_OSSOutageNoBaselineWhenAnthropicExcluded(t *testing.T) { var anthropicCount int anthropicUpstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/proxy/openai_subscription_failover_test.go b/internal/proxy/openai_subscription_failover_test.go index a58b2107..f015d773 100644 --- a/internal/proxy/openai_subscription_failover_test.go +++ b/internal/proxy/openai_subscription_failover_test.go @@ -25,8 +25,7 @@ const ( oaiSubFailoverOK = `{"id":"msg_1","type":"message","role":"assistant","content":[{"type":"text","text":"hi"}],"model":"claude-haiku-4-5","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}` ) -// oauthRejectThenDeployOK returns a buffered error while an OAuth subscription -// credential is present, then succeeds with no credential (deploy-key path). +// oauthRejectThenDeployOK fails under OAuth subscription creds, then succeeds on the deploy-key path. type oauthRejectThenDeployOK struct { mu sync.Mutex onOAuth error @@ -70,8 +69,7 @@ func (p *oauthRejectThenDeployOK) Passthrough(context.Context, providers.Prepare func oaiSubSlackObs(t *testing.T) *usage.Observer { t.Helper() obs := usage.NewObserver([]byte("salt"), 10*time.Minute, time.Now) - // Slack (not exhausted): preemptive suppress must not fire; only the - // reactive subscription-credit retry can recover. + // Slack: preemptive suppress must not fire; only reactive subscription retry recovers. obs.Record(obs.Key([]byte(oaiSubFailoverToken)), usage.Snapshot{ Primary: usage.Window{UsedPercent: 0.50, WindowMinutes: 300}, }) @@ -94,9 +92,7 @@ func oaiSubFailoverSvc(t *testing.T, p providers.Client) *proxy.Service { WithDeploymentKeyedProviders(map[string]struct{}{providers.ProviderAnthropic: {}}) } -// TestProxyOpenAI_SubscriptionRetry_Live429FailsOverToDeployKey: a live 429 on -// the Claude OAuth token (observer still has slack) must suppress the -// subscription and retry once on the Weave/BYOK key — Messages parity. +// TestProxyOpenAI_SubscriptionRetry_Live429FailsOverToDeployKey: live 429 on OAuth → Weave/BYOK retry. func TestProxyOpenAI_SubscriptionRetry_Live429FailsOverToDeployKey(t *testing.T) { reject := &providers.UpstreamErrorResponse{ Status: http.StatusTooManyRequests, @@ -120,8 +116,7 @@ func TestProxyOpenAI_SubscriptionRetry_Live429FailsOverToDeployKey(t *testing.T) assert.Contains(t, rec.Body.String(), `"object":"chat.completion"`) } -// TestProxyOpenAI_SubscriptionRetry_OAuth401FailsOverToDeployKey covers the -// authentication_error arm of anthropicOAuthCredentialRejected. +// TestProxyOpenAI_SubscriptionRetry_OAuth401FailsOverToDeployKey: OAuth authentication_error → deploy key. func TestProxyOpenAI_SubscriptionRetry_OAuth401FailsOverToDeployKey(t *testing.T) { reject := &providers.UpstreamErrorResponse{ Status: http.StatusUnauthorized, @@ -144,8 +139,7 @@ func TestProxyOpenAI_SubscriptionRetry_OAuth401FailsOverToDeployKey(t *testing.T assert.True(t, last.nilCreds || !last.oauth) } -// TestProxyOpenAI_SubscriptionRetry_OAuth403FailsOverToDeployKey covers the -// permission_error arm of anthropicOAuthCredentialRejected. +// TestProxyOpenAI_SubscriptionRetry_OAuth403FailsOverToDeployKey: OAuth permission_error → deploy key. func TestProxyOpenAI_SubscriptionRetry_OAuth403FailsOverToDeployKey(t *testing.T) { reject := &providers.UpstreamErrorResponse{ Status: http.StatusForbidden, diff --git a/internal/proxy/service.go b/internal/proxy/service.go index c6addc4f..8c373809 100644 --- a/internal/proxy/service.go +++ b/internal/proxy/service.go @@ -4541,9 +4541,7 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w return fmt.Errorf("%w: %s (no translation path defined)", ErrProviderNotConfigured, decision.Provider) } - // openaiAnthropicAttempt builds the OpenAI-wire → Anthropic upstream closure - // used by baseline failover (and subscription-credit retry). Mirrors the - // FamilyAnthropic primary attempt, rebuilt per retry with a fresh prep. + // openaiAnthropicAttempt rebuilds the OpenAI→Anthropic dispatch closure for failover retries. openaiAnthropicAttempt := func(prep providers.PreparedRequest, markerText string) dispatchAttempt { return func(actx context.Context, d router.Decision, p providers.Client) error { var usageSink otel.UsageSink @@ -4655,10 +4653,7 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w flushBufferedIfPresent(contentSink, proxyErr) } - // Subscription-credit failover: suppress the OAuth token and retry the SAME - // model once on the Weave/BYOK key when a subscription-served Anthropic turn - // hit a transient fault (429/timeout) or an OAuth rejection (401/403), - // pre-commit. Skipped when baseline failover already ran (non-Anthropic). + // Subscription-credit failover: drop OAuth and retry once on Weave/BYOK (429/timeout or OAuth 401/403), pre-commit. subscriptionFailoverUsed := false subscriptionRetryRan := false if subscriptionRetryEligible && !baselineAttempted && proxyErr != nil && @@ -4716,9 +4711,7 @@ func (s *Service) ProxyOpenAIChatCompletion(ctx context.Context, body []byte, w } decision.Provider = finalProvider - // Re-resolve credentials for the binding that actually served — each - // failover attempt gets its own context. Carry suppression forward only - // when the Weave retry actually succeeded. + // Re-resolve credentials for the served binding; keep suppression only if Weave retry succeeded. if subscriptionFailoverUsed { ctx = withSuppressedClaudeSubscription(ctx) }