diff --git a/src/core/task/Task.ts b/src/core/task/Task.ts index 90cf7bf7fb..138c852ed2 100644 --- a/src/core/task/Task.ts +++ b/src/core/task/Task.ts @@ -154,6 +154,8 @@ import { } from "./run-state/runStateDivergence" const MAX_EXPONENTIAL_BACKOFF_SECONDS = 600 // 10 minutes +// Consecutive mid-stream failures retried automatically before the user is asked. +const MAX_MID_STREAM_AUTO_RETRIES = 5 const DEFAULT_USAGE_COLLECTION_TIMEOUT_MS = 5000 // 5 seconds // Upper bound on awaiting lazily loaded model metadata. Some model-catalog // fetchers (e.g. OpenRouter's bare axios GET) have no request timeout, so a @@ -3882,6 +3884,9 @@ export class Task extends EventEmitter implements TaskLike { userContent: Anthropic.Messages.ContentBlockParam[] includeFileDetails: boolean retryAttempt?: number + // Automatic retries of a mid-stream failure. Kept apart from `retryAttempt`, which also + // tells shouldAddUserMessageToHistory that the user message is already in history. + midStreamAutoRetries?: number userMessageWasRemoved?: boolean // Track if user message was removed due to empty response } @@ -4720,26 +4725,50 @@ export class Task extends EventEmitter implements TaskLike { }) continue } else { + const stateForBackoff = await this.providerRef.deref()?.getState() + const midStreamRetryAttempt = currentItem.midStreamAutoRetries ?? 0 + + // Without auto-approval, or once automatic retries are exhausted, a + // persistently failing stream (e.g. an upstream idle timeout) would + // otherwise be re-requested forever. Let the user decide instead. + if ( + !stateForBackoff?.autoApprovalEnabled || + midStreamRetryAttempt >= MAX_MID_STREAM_AUTO_RETRIES + ) { + const { response } = await this.ask("api_req_failed", rawErrorMessage) + + if (response !== "yesButtonClicked") { + throw new Error("API request failed") + } + + await this.say("api_req_retried") + // The user message is already in history, so keep the history attempt positive + // and only reset the automatic-retry count. + stack.push({ + userContent: currentUserContent, + includeFileDetails: false, + retryAttempt: (currentItem.retryAttempt ?? 0) + 1, + midStreamAutoRetries: 0, + }) + continue + } + // Stream failed - log the error and retry with the same content - // The existing rate limiting will prevent rapid retries console.error( `[Task#${this.taskId}.${this.instanceId}] Stream failed, will retry: ${streamingFailedMessage}`, ) - // Apply exponential backoff similar to first-chunk errors when auto-resubmit is enabled - const stateForBackoff = await this.providerRef.deref()?.getState() - if (stateForBackoff?.autoApprovalEnabled) { - await this.backoffAndAnnounce(currentItem.retryAttempt ?? 0, error) + // Apply exponential backoff similar to first-chunk errors + await this.backoffAndAnnounce(midStreamRetryAttempt, error) - // Check if task was aborted during the backoff - if (this.abort) { - console.log( - `[Task#${this.taskId}.${this.instanceId}] Task aborted during mid-stream retry backoff`, - ) - this.setAbortReason("user_cancelled") - await this.abortTask() - break - } + // Check if task was aborted during the backoff + if (this.abort) { + console.log( + `[Task#${this.taskId}.${this.instanceId}] Task aborted during mid-stream retry backoff`, + ) + this.setAbortReason("user_cancelled") + await this.abortTask() + break } // Push the same content back onto the stack to retry, incrementing the retry attempt counter @@ -4747,6 +4776,7 @@ export class Task extends EventEmitter implements TaskLike { userContent: currentUserContent, includeFileDetails: false, retryAttempt: (currentItem.retryAttempt ?? 0) + 1, + midStreamAutoRetries: midStreamRetryAttempt + 1, }) // Continue to retry the request @@ -5607,13 +5637,16 @@ export class Task extends EventEmitter implements TaskLike { public async *attemptApiRequest( retryAttempt: number = 0, options: { skipProviderRateLimit?: boolean; requestModelInfo?: ModelInfo } = {}, + // Truncation retries already made for this request. Counted apart from `retryAttempt`, + // which callers also advance for mid-stream and empty-response retries. + contextWindowRetries: number = 0, ): ApiStream { // Hold replaced handlers from request setup until the stream is fully // consumed (including the background usage drain) or closed early. // Retries recurse through this method and nest their own unit of work. const endApiWork = this.beginApiWork() try { - yield* this.attemptApiRequestWithActiveApi(retryAttempt, options) + yield* this.attemptApiRequestWithActiveApi(retryAttempt, options, contextWindowRetries) } finally { endApiWork() } @@ -5622,6 +5655,7 @@ export class Task extends EventEmitter implements TaskLike { private async *attemptApiRequestWithActiveApi( retryAttempt: number, options: { skipProviderRateLimit?: boolean; requestModelInfo?: ModelInfo }, + contextWindowRetries: number, ): ApiStream { const state = await this.providerRef.deref()?.getState() @@ -6013,15 +6047,15 @@ export class Task extends EventEmitter implements TaskLike { } // If it's a context window error and we haven't exceeded max retries for this error type - if (isContextWindowExceededError && retryAttempt < MAX_CONTEXT_WINDOW_RETRIES) { + if (isContextWindowExceededError && contextWindowRetries < MAX_CONTEXT_WINDOW_RETRIES) { console.warn( `[Task#${this.taskId}] Context window exceeded for model ${this.api.getModel().id}. ` + - `Retry attempt ${retryAttempt + 1}/${MAX_CONTEXT_WINDOW_RETRIES}. ` + + `Retry attempt ${contextWindowRetries + 1}/${MAX_CONTEXT_WINDOW_RETRIES}. ` + `Attempting automatic truncation...`, ) await this.handleContextWindowExceededError(requestModelInfo) // Retry the request after handling the context window error - yield* this.attemptApiRequest(retryAttempt + 1, retryOptions) + yield* this.attemptApiRequest(retryAttempt + 1, retryOptions, contextWindowRetries + 1) return } @@ -6041,7 +6075,7 @@ export class Task extends EventEmitter implements TaskLike { // Delegate generator output from the recursive call with // incremented retry count. - yield* this.attemptApiRequest(retryAttempt + 1, retryOptions) + yield* this.attemptApiRequest(retryAttempt + 1, retryOptions, contextWindowRetries) return } else { diff --git a/src/core/task/__tests__/Task.spec.ts b/src/core/task/__tests__/Task.spec.ts index b73b9afea7..e987de8b14 100644 --- a/src/core/task/__tests__/Task.spec.ts +++ b/src/core/task/__tests__/Task.spec.ts @@ -476,6 +476,8 @@ describe("Cline", () => { apiConfiguration: mockApiConfig, autoApprovalEnabled: false, }) + // A persistent mid-stream failure asks the user; these tests confirm the retry. + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) return task } @@ -675,6 +677,121 @@ describe("Cline", () => { }) }) + describe("persistent mid-stream failure", () => { + async function* idleTimeoutStream(): AsyncGenerator { + yield { type: "text", text: "partial answer" } + throw new Error("Provider ended the request: Upstream idle timeout exceeded.") + } + + // 5 is MAX_MID_STREAM_AUTO_RETRIES: the first attempt plus 5 automatic retries. + it.each([ + { autoApprovalEnabled: true, expectedAttempts: 6, expectedBackoffAttempts: [0, 1, 2, 3, 4] }, + { autoApprovalEnabled: false, expectedAttempts: 1, expectedBackoffAttempts: [] }, + ])( + "asks the user instead of retrying forever (autoApprovalEnabled=$autoApprovalEnabled)", + async ({ autoApprovalEnabled, expectedAttempts, expectedBackoffAttempts }) => { + vi.spyOn(mockProvider, "getState").mockResolvedValue( + providerStateWith({ autoApprovalEnabled, requestDelaySeconds: 0 }), + ) + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + task: "test task", + startTask: false, + }) + vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) + vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) + vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) + const backoffSpy = vi.spyOn(getTaskTestAccess(task), "backoffAndAnnounce").mockResolvedValue(undefined) + const askSpy = vi + .spyOn(task, "ask") + .mockResolvedValue({ response: "noButtonClicked" } satisfies TaskAskResult) + const attemptApiRequestSpy = vi + .spyOn(task, "attemptApiRequest") + .mockImplementation(() => idleTimeoutStream()) + + // Declining ends the turn: the "API request failed" error is handled inside the loop, + // so the call resolves rather than rejecting. + await expect(task.recursivelyMakeClineRequests([{ type: "text", text: "long request" }])).resolves.toBe( + true, + ) + + expect(attemptApiRequestSpy).toHaveBeenCalledTimes(expectedAttempts) + expect(backoffSpy.mock.calls.map(([attempt]) => attempt)).toEqual(expectedBackoffAttempts) + expect(askSpy).toHaveBeenCalledOnce() + expect(askSpy).toHaveBeenCalledWith("api_req_failed", expect.stringContaining("idle timeout")) + }, + ) + + it("restarts automatic retries from attempt 0 after the user approves the capped prompt", async () => { + vi.spyOn(mockProvider, "getState").mockResolvedValue( + providerStateWith({ autoApprovalEnabled: true, requestDelaySeconds: 0 }), + ) + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + task: "test task", + startTask: false, + }) + vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) + vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) + vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) + const backoffSpy = vi.spyOn(getTaskTestAccess(task), "backoffAndAnnounce").mockResolvedValue(undefined) + const askSpy = vi + .spyOn(task, "ask") + .mockResolvedValueOnce({ response: "yesButtonClicked" } satisfies TaskAskResult) + .mockResolvedValue({ response: "noButtonClicked" } satisfies TaskAskResult) + const attemptApiRequestSpy = vi + .spyOn(task, "attemptApiRequest") + .mockImplementation(() => idleTimeoutStream()) + + await expect(task.recursivelyMakeClineRequests([{ type: "text", text: "long request" }])).resolves.toBe( + true, + ) + + // Two rounds of the first attempt plus 5 automatic retries, each ending in a prompt. + expect(attemptApiRequestSpy).toHaveBeenCalledTimes(12) + expect(backoffSpy.mock.calls.map(([attempt]) => attempt)).toEqual([0, 1, 2, 3, 4, 0, 1, 2, 3, 4]) + expect(askSpy).toHaveBeenCalledTimes(2) + }) + + it("does not add the user message to history again when the user confirms a retry", async () => { + vi.spyOn(mockProvider, "getState").mockResolvedValue( + providerStateWith({ autoApprovalEnabled: false, requestDelaySeconds: 0 }), + ) + const task = new Task({ + provider: mockProvider, + apiConfiguration: mockApiConfig, + task: "test task", + startTask: false, + }) + vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) + vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) + vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) + const historySpy = vi + .spyOn(getTaskTestAccess(task), "addToApiConversationHistory") + .mockResolvedValue(undefined) + const askSpy = vi + .spyOn(task, "ask") + .mockResolvedValueOnce({ response: "yesButtonClicked" } satisfies TaskAskResult) + .mockResolvedValue({ response: "noButtonClicked" } satisfies TaskAskResult) + const attemptApiRequestSpy = vi + .spyOn(task, "attemptApiRequest") + .mockImplementation(() => idleTimeoutStream()) + + await expect(task.recursivelyMakeClineRequests([{ type: "text", text: "long request" }])).resolves.toBe( + true, + ) + + expect(askSpy).toHaveBeenCalledTimes(2) + expect(attemptApiRequestSpy).toHaveBeenCalledTimes(2) + const userMessages = historySpy.mock.calls.filter( + ([message]) => (message as { role: string }).role === "user", + ) + expect(userMessages).toHaveLength(1) + }) + }) + describe("native tool-call request isolation", () => { it("derives readiness only from a closed, idle, fully paired tool turn", () => { const task = new Task({ @@ -1219,6 +1336,8 @@ describe("Cline", () => { startTask: false, }) + // Mid-stream failures ask the user; confirm the retry. + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) @@ -1267,6 +1386,8 @@ describe("Cline", () => { startTask: false, }) + // Mid-stream failures ask the user; confirm the retry. + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) @@ -1359,6 +1480,8 @@ describe("Cline", () => { startTask: false, }) + // Mid-stream failures ask the user; confirm the retry. + vi.spyOn(task, "ask").mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) vi.spyOn(task.diffViewProvider, "reset").mockResolvedValue(undefined) vi.spyOn(getTaskTestAccess(task), "safeEnsureModelFetched").mockResolvedValue(stubModelInfo) vi.spyOn(getTaskTestAccess(task), "presentAssistantMessageSafe").mockImplementation(() => {}) @@ -5529,6 +5652,64 @@ describe("Cline", () => { expect(attemptApiRequestSpy.mock.calls[1]?.[1]).toBe(options) }) + it("truncates on a context-window overflow even when the history retry attempt is past the truncation cap", async () => { + // A request re-sent after mid-stream retries carries a high history retry attempt. + const task = await createRetryForwardingTask() + const truncateSpy = vi + .spyOn(getTaskTestAccess(task), "handleContextWindowExceededError") + .mockResolvedValue(undefined) + vi.spyOn(task, "ask").mockRejectedValue( + new Error("a context-window overflow must truncate, not prompt"), + ) + vi.spyOn(task.api, "createMessage") + .mockImplementationOnce(() => failingStream({ status: 400, message: "context length exceeded" })) + .mockImplementationOnce(() => + asyncStreamFrom([{ type: "text", text: "retry response" }]), + ) + + const attemptApiRequestSpy = vi.spyOn(task, "attemptApiRequest") + const options = retryForwardingOptions() + const iterator = task.attemptApiRequest(6, options) + + await expect(iterator.next()).resolves.toMatchObject({ + done: false, + value: { type: "text", text: "retry response" }, + }) + + expect(truncateSpy).toHaveBeenCalledOnce() + expect(attemptApiRequestSpy.mock.calls[1]).toEqual([7, options, 1]) + }) + + it("stops truncating once the context-window retries for the request are exhausted", async () => { + const task = await createRetryForwardingTask() + const truncateSpy = vi + .spyOn(getTaskTestAccess(task), "handleContextWindowExceededError") + .mockResolvedValue(undefined) + const askSpy = vi + .spyOn(task, "ask") + .mockResolvedValue({ response: "yesButtonClicked" } satisfies TaskAskResult) + vi.spyOn(task.api, "createMessage") + .mockImplementationOnce(() => failingStream({ status: 400, message: "context length exceeded" })) + .mockImplementationOnce(() => + asyncStreamFrom([{ type: "text", text: "retry response" }]), + ) + + const attemptApiRequestSpy = vi.spyOn(task, "attemptApiRequest") + const options = retryForwardingOptions() + // 3 is MAX_CONTEXT_WINDOW_RETRIES. + const iterator = task.attemptApiRequest(0, options, 3) + + await expect(iterator.next()).resolves.toMatchObject({ + done: false, + value: { type: "text", text: "retry response" }, + }) + + expect(truncateSpy).not.toHaveBeenCalled() + expect(askSpy).toHaveBeenCalledWith("api_req_failed", "context length exceeded") + // The user-confirmed retry is a fresh request with a fresh truncation budget. + expect(attemptApiRequestSpy.mock.calls[1]).toEqual([0, options]) + }) + it("carries the derived model snapshot into the retry recursion when the caller omitted one", async () => { const task = await createRetryForwardingTask() vi.spyOn(getTaskTestAccess(task), "handleContextWindowExceededError").mockResolvedValue(undefined)