From 0f1085b8b88674ce0b07b8579922c059485860d8 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Tue, 20 Oct 2026 21:53:13 +0200 Subject: [PATCH 1/3] fix(task): stop retrying a persistently failing stream forever A mid-stream failure (e.g. OpenRouter 'Upstream idle timeout exceeded') re-queued the same request with no limit. Without auto-approval it did so immediately and silently, and with auto-approval it backed off indefinitely. Retry automatically at most 5 times, then ask the user via api_req_failed. Fixes #2107 Co-Authored-By: Claude Sonnet 5.5 --- src/core/task/Task.ts | 51 ++++++++++++++++++++-------- src/core/task/__tests__/Task.spec.ts | 51 ++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 14 deletions(-) diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index e573a7f107..d14a1d2e2e 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -153,6 +153,8 @@ import { } from "./run-state/runStateDivergence" const MAX_EXPONENTIAL_BACKOFF_SECONDS = 600 // 10 minutes +// Consecutive mid-stream failures retried automatically before the user is asked. +const MAX_MID_STREAM_AUTO_RETRIES = 5 const DEFAULT_USAGE_COLLECTION_TIMEOUT_MS = 5000 // 5 seconds // Upper bound on awaiting lazily loaded model metadata. Some model-catalog // fetchers (e.g. OpenRouter's bare axios GET) have no request timeout, so a @@ -4671,26 +4673,47 @@ export class Task extends EventEmitter implements TaskLike { }) continue } else { + const stateForBackoff = await this.providerRef.deref()?.getState() + const midStreamRetryAttempt = currentItem.retryAttempt ?? 0 + + // Without auto-approval, or once automatic retries are exhausted, a + // persistently failing stream (e.g. an upstream idle timeout) would + // otherwise be re-requested forever. Let the user decide instead. + if ( + !stateForBackoff?.autoApprovalEnabled || + midStreamRetryAttempt >= MAX_MID_STREAM_AUTO_RETRIES + ) { + const { response } = await this.ask("api_req_failed", rawErrorMessage) + + if (response !== "yesButtonClicked") { + throw new Error("API request failed") + } + + await this.say("api_req_retried") + stack.push({ + userContent: currentUserContent, + includeFileDetails: false, + retryAttempt: 0, + }) + continue + } + // Stream failed - log the error and retry with the same content - // The existing rate limiting will prevent rapid retries console.error( `[Task#${this.taskId}.${this.instanceId}] Stream failed, will retry: ${streamingFailedMessage}`, ) - // Apply exponential backoff similar to first-chunk errors when auto-resubmit is enabled - const stateForBackoff = await this.providerRef.deref()?.getState() - if (stateForBackoff?.autoApprovalEnabled) { - await this.backoffAndAnnounce(currentItem.retryAttempt ?? 0, error) + // Apply exponential backoff similar to first-chunk errors + await this.backoffAndAnnounce(currentItem.retryAttempt ?? 0, error) - // Check if task was aborted during the backoff - if (this.abort) { - console.log( - `[Task#${this.taskId}.${this.instanceId}] Task aborted during mid-stream retry backoff`, - ) - this.setAbortReason("user_cancelled") - await this.abortTask() - break - } + // Check if task was aborted during the backoff + if (this.abort) { + console.log( + `[Task#${this.taskId}.${this.instanceId}] Task aborted during mid-stream retry backoff`, + ) + this.setAbortReason("user_cancelled") + await this.abortTask() + break } // Push the same content back onto the stack to retry, incrementing the retry attempt counter diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index ad07230204..a172a9528b 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -475,6 +475,8 @@ describe("Cline", () => { apiConfiguration: mockApiConfig, autoApprovalEnabled: false, }) + // A persistent mid-stream failure asks the user; these tests confirm the retry. + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) return task } @@ -621,6 +623,49 @@ describe("Cline", () => { }) }) + describe("persistent mid-stream failure", () => { + async function* idleTimeoutStream(): AsyncGenerator { + yield { type: "text", text: "partial answer" } + throw new Error("Provider ended the request: Upstream idle timeout exceeded.") + } + + // 5 is MAX_MID_STREAM_AUTO_RETRIES: the first attempt plus 5 automatic retries. + it.each([ + { autoApprovalEnabled: true, expectedAttempts: 6, expectedBackoffs: 5 }, + { autoApprovalEnabled: false, expectedAttempts: 1, expectedBackoffs: 0 }, + ])( + "asks the user instead of retrying forever (autoApprovalEnabled=$autoApprovalEnabled)", + async ({ autoApprovalEnabled, expectedAttempts, expectedBackoffs }) => { + vi.spyOn(mockProvider, "getState").mockResolvedValue( + providerStateWith({ autoApprovalEnabled, requestDelaySeconds: 0 }), + ) + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + task: "test task", + startTask: false, + }) + vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) + vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) + vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) + const backoffSpy = vi.spyOn(getTaskTestAccess(task), "backoffAndAnnounce").mockResolvedValue(undefined) + const askSpy = vi + .spyOn(task, "ask") + .mockResolvedValue({ response: "noButtonClicked" } satisfies TaskAskResult) + const attemptApiRequestSpy = vi + .spyOn(task, "attemptApiRequest") + .mockImplementation(() => idleTimeoutStream()) + + await task.recursivelyMakeClineRequests([{ type: "text", text: "long request" }]).catch(() => {}) + + expect(attemptApiRequestSpy).toHaveBeenCalledTimes(expectedAttempts) + expect(backoffSpy).toHaveBeenCalledTimes(expectedBackoffs) + expect(askSpy).toHaveBeenCalledOnce() + expect(askSpy).toHaveBeenCalledWith("api_req_failed", expect.stringContaining("idle timeout")) + }, + ) + }) + describe("native tool-call request isolation", () => { it("derives readiness only from a closed, idle, fully paired tool turn", () => { const task = new Task({ @@ -1163,6 +1208,8 @@ describe("Cline", () => { startTask: false, }) + // Mid-stream failures ask the user; confirm the retry. + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) @@ -1211,6 +1258,8 @@ describe("Cline", () => { startTask: false, }) + // Mid-stream failures ask the user; confirm the retry. + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) @@ -1303,6 +1352,8 @@ describe("Cline", () => { startTask: false, }) + // Mid-stream failures ask the user; confirm the retry. + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) From ba7c3d9dfe90f617201f38a6b10222cb86bc4d03 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Sun, 25 Oct 2026 11:01:53 +0100 Subject: [PATCH 2/3] fix(task): keep history attempt separate from the mid-stream auto-retry count Co-Authored-By: Claude Sonnet 5.5 --- src/core/task/Task.ts | 13 ++++++-- src/core/task/__tests__/Task.spec.ts | 50 +++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 8 deletions(-) diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index d14a1d2e2e..dc5e62e9b3 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -3866,6 +3866,9 @@ export class Task extends EventEmitter implements TaskLike { userContent: Anthropic.Messages.ContentBlockParam[] includeFileDetails: boolean retryAttempt?: number + // Automatic retries of a mid-stream failure. Kept apart from `retryAttempt`, which also + // tells shouldAddUserMessageToHistory that the user message is already in history. + midStreamAutoRetries?: number userMessageWasRemoved?: boolean // Track if user message was removed due to empty response } @@ -4674,7 +4677,7 @@ export class Task extends EventEmitter implements TaskLike { continue } else { const stateForBackoff = await this.providerRef.deref()?.getState() - const midStreamRetryAttempt = currentItem.retryAttempt ?? 0 + const midStreamRetryAttempt = currentItem.midStreamAutoRetries ?? 0 // Without auto-approval, or once automatic retries are exhausted, a // persistently failing stream (e.g. an upstream idle timeout) would @@ -4690,10 +4693,13 @@ export class Task extends EventEmitter implements TaskLike { } await this.say("api_req_retried") + // The user message is already in history, so keep the history attempt positive + // and only reset the automatic-retry count. stack.push({ userContent: currentUserContent, includeFileDetails: false, - retryAttempt: 0, + retryAttempt: (currentItem.retryAttempt ?? 0) + 1, + midStreamAutoRetries: 0, }) continue } @@ -4704,7 +4710,7 @@ export class Task extends EventEmitter implements TaskLike { ) // Apply exponential backoff similar to first-chunk errors - await this.backoffAndAnnounce(currentItem.retryAttempt ?? 0, error) + await this.backoffAndAnnounce(midStreamRetryAttempt, error) // Check if task was aborted during the backoff if (this.abort) { @@ -4721,6 +4727,7 @@ export class Task extends EventEmitter implements TaskLike { userContent: currentUserContent, includeFileDetails: false, retryAttempt: (currentItem.retryAttempt ?? 0) + 1, + midStreamAutoRetries: midStreamRetryAttempt + 1, }) // Continue to retry the request diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index a172a9528b..a3923a4656 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -631,11 +631,11 @@ describe("Cline", () => { // 5 is MAX_MID_STREAM_AUTO_RETRIES: the first attempt plus 5 automatic retries. it.each([ - { autoApprovalEnabled: true, expectedAttempts: 6, expectedBackoffs: 5 }, - { autoApprovalEnabled: false, expectedAttempts: 1, expectedBackoffs: 0 }, + { autoApprovalEnabled: true, expectedAttempts: 6, expectedBackoffAttempts: [0, 1, 2, 3, 4] }, + { autoApprovalEnabled: false, expectedAttempts: 1, expectedBackoffAttempts: [] }, ])( "asks the user instead of retrying forever (autoApprovalEnabled=$autoApprovalEnabled)", - async ({ autoApprovalEnabled, expectedAttempts, expectedBackoffs }) => { + async ({ autoApprovalEnabled, expectedAttempts, expectedBackoffAttempts }) => { vi.spyOn(mockProvider, "getState").mockResolvedValue( providerStateWith({ autoApprovalEnabled, requestDelaySeconds: 0 }), ) @@ -656,14 +656,54 @@ describe("Cline", () => { .spyOn(task, "attemptApiRequest") .mockImplementation(() => idleTimeoutStream()) - await task.recursivelyMakeClineRequests([{ type: "text", text: "long request" }]).catch(() => {}) + // Declining ends the turn: the "API request failed" error is handled inside the loop, + // so the call resolves rather than rejecting. + await expect(task.recursivelyMakeClineRequests([{ type: "text", text: "long request" }])).resolves.toBe( + true, + ) expect(attemptApiRequestSpy).toHaveBeenCalledTimes(expectedAttempts) - expect(backoffSpy).toHaveBeenCalledTimes(expectedBackoffs) + expect(backoffSpy.mock.calls.map(([attempt]) => attempt)).toEqual(expectedBackoffAttempts) expect(askSpy).toHaveBeenCalledOnce() expect(askSpy).toHaveBeenCalledWith("api_req_failed", expect.stringContaining("idle timeout")) }, ) + + it("does not add the user message to history again when the user confirms a retry", async () => { + vi.spyOn(mockProvider, "getState").mockResolvedValue( + providerStateWith({ autoApprovalEnabled: false, requestDelaySeconds: 0 }), + ) + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + task: "test task", + startTask: false, + }) + vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) + vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) + vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) + const historySpy = vi + .spyOn(getTaskTestAccess(task), "addToApiConversationHistory") + .mockResolvedValue(undefined) + const askSpy = vi + .spyOn(task, "ask") + .mockResolvedValueOnce({ response: "yesButtonClicked" } satisfies TaskAskResult) + .mockResolvedValue({ response: "noButtonClicked" } satisfies TaskAskResult) + const attemptApiRequestSpy = vi + .spyOn(task, "attemptApiRequest") + .mockImplementation(() => idleTimeoutStream()) + + await expect(task.recursivelyMakeClineRequests([{ type: "text", text: "long request" }])).resolves.toBe( + true, + ) + + expect(askSpy).toHaveBeenCalledTimes(2) + expect(attemptApiRequestSpy).toHaveBeenCalledTimes(2) + const userMessages = historySpy.mock.calls.filter( + ([message]) => (message as { role: string }).role === "user", + ) + expect(userMessages).toHaveLength(1) + }) }) describe("native tool-call request isolation", () => { From 5e0a633dcf51e850b3f42fa43963b787f87f3644 Mon Sep 17 00:00:00 2001 From: PierrunoYT Date: Wed, 4 Nov 2026 15:41:25 +0100 Subject: [PATCH 3/3] fix(task): count context-window truncation retries apart from the history retry attempt --- src/core/task/Task.ts | 14 +++-- src/core/task/__tests__/Task.spec.ts | 90 ++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 83066ef29b..138c852ed2 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -5637,13 +5637,16 @@ export class Task extends EventEmitter implements TaskLike { public async *attemptApiRequest( retryAttempt: number = 0, options: { skipProviderRateLimit?: boolean; requestModelInfo?: ModelInfo } = {}, + // Truncation retries already made for this request. Counted apart from `retryAttempt`, + // which callers also advance for mid-stream and empty-response retries. + contextWindowRetries: number = 0, ): ApiStream { // Hold replaced handlers from request setup until the stream is fully // consumed (including the background usage drain) or closed early. // Retries recurse through this method and nest their own unit of work. const endApiWork = this.beginApiWork() try { - yield* this.attemptApiRequestWithActiveApi(retryAttempt, options) + yield* this.attemptApiRequestWithActiveApi(retryAttempt, options, contextWindowRetries) } finally { endApiWork() } @@ -5652,6 +5655,7 @@ export class Task extends EventEmitter implements TaskLike { private async *attemptApiRequestWithActiveApi( retryAttempt: number, options: { skipProviderRateLimit?: boolean; requestModelInfo?: ModelInfo }, + contextWindowRetries: number, ): ApiStream { const state = await this.providerRef.deref()?.getState() @@ -6043,15 +6047,15 @@ export class Task extends EventEmitter implements TaskLike { } // If it's a context window error and we haven't exceeded max retries for this error type - if (isContextWindowExceededError && retryAttempt < MAX_CONTEXT_WINDOW_RETRIES) { + if (isContextWindowExceededError && contextWindowRetries < MAX_CONTEXT_WINDOW_RETRIES) { console.warn( `[Task#${this.taskId}] Context window exceeded for model ${this.api.getModel().id}. ` + - `Retry attempt ${retryAttempt + 1}/${MAX_CONTEXT_WINDOW_RETRIES}. ` + + `Retry attempt ${contextWindowRetries + 1}/${MAX_CONTEXT_WINDOW_RETRIES}. ` + `Attempting automatic truncation...`, ) await this.handleContextWindowExceededError(requestModelInfo) // Retry the request after handling the context window error - yield* this.attemptApiRequest(retryAttempt + 1, retryOptions) + yield* this.attemptApiRequest(retryAttempt + 1, retryOptions, contextWindowRetries + 1) return } @@ -6071,7 +6075,7 @@ export class Task extends EventEmitter implements TaskLike { // Delegate generator output from the recursive call with // incremented retry count. - yield* this.attemptApiRequest(retryAttempt + 1, retryOptions) + yield* this.attemptApiRequest(retryAttempt + 1, retryOptions, contextWindowRetries) return } else { diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index aa8c077c2a..e987de8b14 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -723,6 +723,38 @@ describe("Cline", () => { }, ) + it("restarts automatic retries from attempt 0 after the user approves the capped prompt", async () => { + vi.spyOn(mockProvider, "getState").mockResolvedValue( + providerStateWith({ autoApprovalEnabled: true, requestDelaySeconds: 0 }), + ) + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + task: "test task", + startTask: false, + }) + vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) + vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) + vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) + const backoffSpy = vi.spyOn(getTaskTestAccess(task), "backoffAndAnnounce").mockResolvedValue(undefined) + const askSpy = vi + .spyOn(task, "ask") + .mockResolvedValueOnce({ response: "yesButtonClicked" } satisfies TaskAskResult) + .mockResolvedValue({ response: "noButtonClicked" } satisfies TaskAskResult) + const attemptApiRequestSpy = vi + .spyOn(task, "attemptApiRequest") + .mockImplementation(() => idleTimeoutStream()) + + await expect(task.recursivelyMakeClineRequests([{ type: "text", text: "long request" }])).resolves.toBe( + true, + ) + + // Two rounds of the first attempt plus 5 automatic retries, each ending in a prompt. + expect(attemptApiRequestSpy).toHaveBeenCalledTimes(12) + expect(backoffSpy.mock.calls.map(([attempt]) => attempt)).toEqual([0, 1, 2, 3, 4, 0, 1, 2, 3, 4]) + expect(askSpy).toHaveBeenCalledTimes(2) + }) + it("does not add the user message to history again when the user confirms a retry", async () => { vi.spyOn(mockProvider, "getState").mockResolvedValue( providerStateWith({ autoApprovalEnabled: false, requestDelaySeconds: 0 }), @@ -5620,6 +5652,64 @@ describe("Cline", () => { expect(attemptApiRequestSpy.mock.calls[1]?.[1]).toBe(options) }) + it("truncates on a context-window overflow even when the history retry attempt is past the truncation cap", async () => { + // A request re-sent after mid-stream retries carries a high history retry attempt. + const task = await createRetryForwardingTask() + const truncateSpy = vi + .spyOn(getTaskTestAccess(task), "handleContextWindowExceededError") + .mockResolvedValue(undefined) + vi.spyOn(task, "ask").mockRejectedValue( + new Error("a context-window overflow must truncate, not prompt"), + ) + vi.spyOn(task.api, "createMessage") + .mockImplementationOnce(() => failingStream({ status: 400, message: "context length exceeded" })) + .mockImplementationOnce(() => + asyncStreamFrom([{ type: "text", text: "retry response" }]), + ) + + const attemptApiRequestSpy = vi.spyOn(task, "attemptApiRequest") + const options = retryForwardingOptions() + const iterator = task.attemptApiRequest(6, options) + + await expect(iterator.next()).resolves.toMatchObject({ + done: false, + value: { type: "text", text: "retry response" }, + }) + + expect(truncateSpy).toHaveBeenCalledOnce() + expect(attemptApiRequestSpy.mock.calls[1]).toEqual([7, options, 1]) + }) + + it("stops truncating once the context-window retries for the request are exhausted", async () => { + const task = await createRetryForwardingTask() + const truncateSpy = vi + .spyOn(getTaskTestAccess(task), "handleContextWindowExceededError") + .mockResolvedValue(undefined) + const askSpy = vi + .spyOn(task, "ask") + .mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) + vi.spyOn(task.api, "createMessage") + .mockImplementationOnce(() => failingStream({ status: 400, message: "context length exceeded" })) + .mockImplementationOnce(() => + asyncStreamFrom([{ type: "text", text: "retry response" }]), + ) + + const attemptApiRequestSpy = vi.spyOn(task, "attemptApiRequest") + const options = retryForwardingOptions() + // 3 is MAX_CONTEXT_WINDOW_RETRIES. + const iterator = task.attemptApiRequest(0, options, 3) + + await expect(iterator.next()).resolves.toMatchObject({ + done: false, + value: { type: "text", text: "retry response" }, + }) + + expect(truncateSpy).not.toHaveBeenCalled() + expect(askSpy).toHaveBeenCalledWith("api_req_failed", "context length exceeded") + // The user-confirmed retry is a fresh request with a fresh truncation budget. + expect(attemptApiRequestSpy.mock.calls[1]).toEqual([0, options]) + }) + it("carries the derived model snapshot into the retry recursion when the caller omitted one", async () => { const task = await createRetryForwardingTask() vi.spyOn(getTaskTestAccess(task), "handleContextWindowExceededError").mockResolvedValue(undefined)