Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/calm-swarms-retry.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@moonshot-ai/kimi-code": patch
---

web: Recover transient subagent rate limits without surfacing them as session errors.
1 change: 1 addition & 0 deletions apps/kimi-web/src/api/daemon/agentEventProjector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ const MAIN_AGENT_TRANSCRIPT_FRAMES = new Set<string>([
'tool.result',
'agent.status.updated',
'prompt.completed',
'error',
]);

// ---------------------------------------------------------------------------
Expand Down
40 changes: 40 additions & 0 deletions apps/kimi-web/test/agent-event-projector.test.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
/**
* Web daemon projector contract for transcript isolation, task progress, and
* client-visible error projection.
*/

import { describe, expect, it } from 'vitest';
import { classifyFrame, createAgentProjector, subagentProgressText } from '../src/api/daemon/agentEventProjector';

Expand Down Expand Up @@ -61,6 +66,41 @@ describe('subagent streaming text', () => {
});
});

describe('agent error projection', () => {
it('drops a subagent error instead of surfacing it as a session warning', () => {
const projector = createAgentProjector();

expect(
projector.project(
'error',
{ agentId: 'sub-1', code: 'provider.rate_limit', message: 'Rate limited' },
's1',
),
).toEqual([]);
});

it('keeps a main-agent error visible to the session', () => {
const projector = createAgentProjector();

expect(
projector.project(
'error',
{ agentId: 'main', code: 'provider.rate_limit', message: 'Rate limited' },
's1',
),
).toEqual([
{
type: 'unknown',
raw: {
_agentError: true,
code: 'provider.rate_limit',
message: 'Rate limited',
},
},
]);
});
});

describe('cron.fired', () => {
it('synthesizes a user message so the cron notice renders live', () => {
const projector = createAgentProjector();
Expand Down
27 changes: 17 additions & 10 deletions packages/agent-core-v2/src/_base/utils/retry.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* `_base` retry helpers — exponential backoff schedule, abortable retry
* `_base` retry helpers — exponential and server-directed backoff, abortable
* sleeps, and error-field extraction shared by retry policies (the loop's
* `stepRetry` plugin, full-compaction's self-managed resends).
*/
Expand All @@ -8,9 +8,10 @@ import { abortable } from '#/_base/utils/abort';

export const DEFAULT_MAX_RETRY_ATTEMPTS = 3;

const RETRY_MIN_TIMEOUT_MS = 300;
const RETRY_MAX_TIMEOUT_MS = 5000;
const BASE_DELAY_MS = 500;
const MAX_DELAY_MS = 32_000;
const RETRY_FACTOR = 2;
const JITTER_FACTOR = 0.25;

export interface RetryErrorFields {
readonly errorName: string;
Expand All @@ -19,13 +20,19 @@ export interface RetryErrorFields {
}

export function retryBackoffDelays(maxAttempts: number): number[] {
return Array.from({ length: Math.max(maxAttempts - 1, 0) }, (_unused, index) => {
const baseDelay = Math.min(
RETRY_MAX_TIMEOUT_MS,
RETRY_MIN_TIMEOUT_MS * RETRY_FACTOR ** index,
);
return Math.round(baseDelay * (1 + Math.random()));
});
const count = Math.max(maxAttempts - 1, 0);
const delays: number[] = [];
for (let i = 0; i < count; i += 1) {
const base = Math.min(BASE_DELAY_MS * Math.pow(RETRY_FACTOR, i), MAX_DELAY_MS);
delays.push(base + Math.random() * JITTER_FACTOR * base);
}
return delays;
}

export function readRetryAfterMs(error: unknown): number | null {
if (typeof error !== 'object' || error === null) return null;
const value = (error as { retryAfterMs?: unknown }).retryAfterMs;
return typeof value === 'number' && value > 0 ? value : null;
}

export async function sleepForRetry(delayMs: number, signal?: AbortSignal): Promise<void> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import { InstantiationType } from '#/_base/di/extensions';
import { LifecycleScope, registerScopedService } from '#/_base/di/scope';
import {
DEFAULT_MAX_RETRY_ATTEMPTS,
readRetryAfterMs,
retryBackoffDelays,
retryErrorFields,
sleepForRetry,
Expand Down Expand Up @@ -95,7 +96,9 @@ export class AgentStepRetryService extends Disposable implements IAgentStepRetry
return false;
}

const delayMs = retryBackoffDelays(maxAttempts)[this.failedAttempts - 1] ?? 0;
const error = unwrapErrorCause(context.error);
const delayMs =
readRetryAfterMs(error) ?? retryBackoffDelays(maxAttempts)[this.failedAttempts - 1] ?? 0;
this.eventBus.publish({
type: 'turn.step.retrying',
turnId: context.turnId,
Expand All @@ -105,7 +108,7 @@ export class AgentStepRetryService extends Disposable implements IAgentStepRetry
nextAttempt: this.failedAttempts + 1,
maxAttempts,
delayMs,
...retryErrorFields(unwrapErrorCause(context.error)),
...retryErrorFields(error),
});
await sleepForRetry(delayMs, context.signal);

Expand Down
71 changes: 54 additions & 17 deletions packages/agent-core-v2/src/app/llmProtocol/errors.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,12 +36,19 @@ export class APITimeoutError extends ChatProviderError {
export class APIStatusError extends ChatProviderError {
readonly statusCode: number;
readonly requestId: string | null;
readonly retryAfterMs: number | null;

constructor(statusCode: number, message: string, requestId?: string | null) {
constructor(
statusCode: number,
message: string,
requestId?: string | null,
retryAfterMs?: number | null,
) {
super(message);
this.name = 'APIStatusError';
this.statusCode = statusCode;
this.requestId = requestId ?? null;
this.retryAfterMs = retryAfterMs ?? null;
}
}

Expand All @@ -50,8 +57,13 @@ export class APIStatusError extends ChatProviderError {
* context window.
*/
export class APIContextOverflowError extends APIStatusError {
constructor(statusCode: number, message: string, requestId?: string | null) {
super(statusCode, message, requestId);
constructor(
statusCode: number,
message: string,
requestId?: string | null,
retryAfterMs?: number | null,
) {
super(statusCode, message, requestId, retryAfterMs);
this.name = 'APIContextOverflowError';
}
}
Expand All @@ -63,8 +75,13 @@ export class APIContextOverflowError extends APIStatusError {
* size rejection is not — it needs media to be dropped or shrunk.
*/
export class APIRequestTooLargeError extends APIStatusError {
constructor(statusCode: number, message: string, requestId?: string | null) {
super(statusCode, message, requestId);
constructor(
statusCode: number,
message: string,
requestId?: string | null,
retryAfterMs?: number | null,
) {
super(statusCode, message, requestId, retryAfterMs);
this.name = 'APIRequestTooLargeError';
}
}
Expand All @@ -74,8 +91,8 @@ export class APIRequestTooLargeError extends APIStatusError {
* request.
*/
export class APIProviderRateLimitError extends APIStatusError {
constructor(message: string, requestId?: string | null) {
super(429, message, requestId);
constructor(message: string, requestId?: string | null, retryAfterMs?: number | null) {
super(429, message, requestId, retryAfterMs);
this.name = 'APIProviderRateLimitError';
}
}
Expand All @@ -87,8 +104,13 @@ export class APIProviderRateLimitError extends APIStatusError {
* constraint, the provider is simply saturated — retry with backoff.
*/
export class APIProviderOverloadedError extends APIStatusError {
constructor(statusCode: number, message: string, requestId?: string | null) {
super(statusCode, message, requestId);
constructor(
statusCode: number,
message: string,
requestId?: string | null,
retryAfterMs?: number | null,
) {
super(statusCode, message, requestId, retryAfterMs);
this.name = 'APIProviderOverloadedError';
}
}
Expand Down Expand Up @@ -124,9 +146,10 @@ export function isRetryableGenerateError(error: unknown): boolean {
if (error instanceof APIProviderOverloadedError) {
return true;
}
return (
error instanceof APIStatusError && [429, 500, 502, 503, 504, 529].includes(error.statusCode)
);
if (error instanceof APIStatusError) {
return [408, 409, 429, 500, 502, 503, 504, 529].includes(error.statusCode);
}
return error instanceof ChatProviderError;

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Exclude deterministic provider validation errors from retry

When the Anthropic adapter rejects a malformed/unsupported data-URL image before sending any request, it throws a base ChatProviderError from imageUrlPartToAnthropic; this new catch-all now marks that deterministic validation failure as retryable, so stepRetry sleeps and re-enqueues the same step until maxRetriesPerStep instead of surfacing the bad input immediately. This is especially noticeable with higher retry configs and also emits misleading retry events; add the same kind of narrow exclusion for known deterministic ChatProviderErrors rather than retrying every base provider error.

Useful? React with 👍 / 👎.

}

const NETWORK_RE = /network|connection|connect|disconnect|terminated/i;
Expand Down Expand Up @@ -200,22 +223,36 @@ export function normalizeAPIStatusError(
statusCode: number,
message: string,
requestId?: string | null,
retryAfterMs?: number | null,
): APIStatusError {
if (statusCode === 429) {
return new APIProviderRateLimitError(message, requestId);
return new APIProviderRateLimitError(message, requestId, retryAfterMs);
}
// Context overflow first: Vertex returns prompt-too-long as a 413, and a
// token overflow must keep routing to compaction even on that status.
if (isContextOverflowStatusError(statusCode, message)) {
return new APIContextOverflowError(statusCode, message, requestId);
return new APIContextOverflowError(statusCode, message, requestId, retryAfterMs);
}
if (isRequestTooLargeStatusError(statusCode, message)) {
return new APIRequestTooLargeError(statusCode, message, requestId);
return new APIRequestTooLargeError(statusCode, message, requestId, retryAfterMs);
}
if (isProviderOverloadStatusError(statusCode, message)) {
return new APIProviderOverloadedError(statusCode, message, requestId);
return new APIProviderOverloadedError(statusCode, message, requestId, retryAfterMs);
}
return new APIStatusError(statusCode, message, requestId);
return new APIStatusError(statusCode, message, requestId, retryAfterMs);
}

export function parseRetryAfterMs(headers: unknown): number | null {
const raw =
headers !== null &&
typeof headers === 'object' &&
typeof (headers as { get?: unknown }).get === 'function'
? (headers as { get(name: string): string | null }).get('retry-after')
: null;
if (raw === null || raw === undefined) return null;
const seconds = Number.parseInt(raw, 10);
if (!Number.isFinite(seconds) || seconds < 0) return null;
return seconds * 1000;
}

export function isContextOverflowStatusError(statusCode: number, message: string): boolean {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
ChatProviderError,
classifyBaseApiError,
normalizeAPIStatusError,
parseRetryAfterMs,
} from '../errors';
import type { ContentPart, Message, StreamedMessagePart, ToolCall } from '../message';
import { isToolDeclarationOnlyMessage } from '../message';
Expand Down Expand Up @@ -675,7 +676,12 @@ export function convertAnthropicError(error: unknown): ChatProviderError {
// APIError with a status code => status error
if (error instanceof AnthropicAPIError && typeof error.status === 'number') {
const reqId = error.requestID ?? null;
return normalizeAPIStatusError(error.status, error.message, reqId);
return normalizeAPIStatusError(
error.status,
error.message,
reqId,
parseRetryAfterMs(error.headers),
);
}
if (error instanceof AnthropicError) {
return new ChatProviderError(`Anthropic error: ${error.message}`);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import {
ChatProviderError,
classifyBaseApiError,
normalizeAPIStatusError,
parseRetryAfterMs,
} from '../errors';
import { extractText } from '../message';
import type { ContentPart, Message } from '../message';
Expand Down Expand Up @@ -103,7 +104,12 @@ export function convertOpenAIError(error: unknown): ChatProviderError {
// APIError with a status code => status error
if (error instanceof OpenAIAPIError && typeof error.status === 'number') {
const reqId = error.requestID ?? null;
return normalizeAPIStatusError(error.status, error.message, reqId);
return normalizeAPIStatusError(
error.status,
error.message,
reqId,
parseRetryAfterMs(error.headers),
);
}
// Base APIError with no status and no body => transport-layer failure.
// When the error has a body (e.g. SSE error events from the server),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,13 @@ function formatResponsesErrorEvent(
return `${codeText}: ${message}${paramText}`;
}

const EMBEDDED_STATUS_CODE_RE = /\bstatus_code\s*[:=]\s*(\d{3})\b/;

function readEmbeddedStatusCode(message: string): number | undefined {
const match = EMBEDDED_STATUS_CODE_RE.exec(message);
return match === null ? undefined : Number(match[1]);
}

function errorFromOpenAIResponsesEvent(
prefix: string,
code: string | null,
Expand All @@ -246,7 +253,7 @@ function errorFromOpenAIResponsesEvent(
if (isContextOverflowErrorCode(code)) {
return new APIContextOverflowError(400, fullMessage);
}
if (code === 'rate_limit_exceeded') {
if (code === 'rate_limit_exceeded' || readEmbeddedStatusCode(message) === 429) {
return new APIProviderRateLimitError(fullMessage);
}
return new ChatProviderError(fullMessage);
Expand Down
18 changes: 10 additions & 8 deletions packages/agent-core-v2/src/session/swarm/sessionSwarmService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -189,14 +189,16 @@ export class SessionSwarmService implements ISessionSwarmService {
this.realignChildModel(caller, child);
const profileName =
child.accessor.get(IAgentProfileService).data().profileName ?? RESUMED_PROFILE_FALLBACK;
emitAgentRunSpawned(caller, agentId, {
profileName,
parentToolCallId: options.parentToolCallId,
parentToolCallUuid: options.parentToolCallUuid,
description: options.description,
swarmIndex: options.swarmIndex,
runInBackground: options.runInBackground,
});
if (!retryTurn) {
emitAgentRunSpawned(caller, agentId, {
profileName,
parentToolCallId: options.parentToolCallId,
parentToolCallUuid: options.parentToolCallUuid,
description: options.description,
swarmIndex: options.swarmIndex,
runInBackground: options.runInBackground,
});
}
const request = retryTurn
? ({ kind: 'retry' } as const)
: ({ kind: 'prompt', prompt: options.prompt } as const);
Expand Down
Loading
Loading