feat(agent-core-v2): add PYTHINKER_CODE_INFINITE_RETRY retry mode

Author: elkaix

docs/configuration/env-vars.md

@@ -143,6 +143,7 @@ Switches that control the behavior of subsystems such as telemetry, background t
 | `PYTHINKER_MCP_TOOL_TIMEOUT_MS` | Global default single tool-call timeout (ms) for all MCP servers; takes higher priority than `[mcp] tool_timeout_ms` in `config.toml`, but a per-server `toolTimeoutMs` in `mcp.json` still wins (default `60000`) | Integer from `1` to `2147483647`; invalid values are ignored |
 | `PYTHINKER_LOOP_MAX_STEPS_PER_TURN` | Maximum Agent steps per turn; takes higher priority than `[loop_control] max_steps_per_turn` in `config.toml` (unset or `0` means unlimited) | Non-negative integer; invalid values are ignored |
 | `PYTHINKER_LOOP_MAX_ATTEMPTS_PER_STEP` | Maximum total attempts for a failing step (including the initial attempt); takes higher priority than `[loop_control] max_attempts_per_step` in `config.toml` (default `10`). The deprecated `PYTHINKER_LOOP_MAX_RETRIES_PER_STEP` is still honored with a warning when this variable is unset | Non-negative integer; invalid values are ignored |
+| `PYTHINKER_CODE_INFINITE_RETRY` | Retry every failed LLM request indefinitely — turn steps and background operations such as compaction alike — instead of failing the task; waits use exponential backoff (capped at 32 s) and honor the server's `Retry-After` header, and aborting still cancels immediately. Intended for long-running unattended evaluations against endpoints that may fail temporarily | Truthy: `1`/`true`/`yes`/`on`; falsy: `0`/`false`/`no`/`off` |
 | `PYTHINKER_TOKEN_COUNTING_STRATEGY` | Which context token count is reported externally (the context-size display); takes higher priority than `[token_counting] strategy` in `config.toml` (default `measured+estimated`) | `measured+estimated`, `measured`, `estimated` (case-insensitive); invalid values are ignored |
 | `PYTHINKER_WEB_SEARCH_BASE_URL` | API URL of the web search (`WebSearch`) service; takes higher priority than `[services.pymodel_search] base_url` in `config.toml`, and enables the service without that config section. Persisted credentials and custom headers are not forwarded to an env-selected endpoint | Non-blank string; blank values are ignored |
 | `PYTHINKER_WEB_SEARCH_API_KEY` | API key of the web search (`WebSearch`) service; replaces both the configured API key and OAuth credential when set | Non-blank string; blank values are ignored |
@@ -159,7 +160,7 @@ Switches that control the behavior of subsystems such as telemetry, background t
 | `PYTHINKER_CODE_NO_AUTO_UPDATE` | Fully disable the update preflight — no check, background install, or prompt. Legacy alias `PYTHINKER_CLI_NO_AUTO_UPDATE` is also honored | Truthy: `1`/`true`/`yes`/`on` |
 | `PYTHINKER_DISABLE_CRON` | Disable the scheduled-task tool (`CronCreate` rejects new schedules; existing tasks do not fire) | `1` to disable |
 
-The three `PYTHINKER_CODE_IDENTITY_*` / `PYTHINKER_CODE_BUILTIN_PRODUCT_SKILLS` variables are read by the default `agent-core-v2` engine. The legacy `pythinker` / `pythinker -p` path selected with `PYTHINKER_CODE_LEGACY_FLAG=1` ignores them.
+The `PYTHINKER_CODE_INFINITE_RETRY`, `PYTHINKER_CODE_IDENTITY_*`, and `PYTHINKER_CODE_BUILTIN_PRODUCT_SKILLS` variables are read by the default `agent-core-v2` engine. The legacy `pythinker` / `pythinker -p` path selected with `PYTHINKER_CODE_LEGACY_FLAG=1` ignores them.
 
 ## Diagnostic logs
 

packages/agent-core-v2/src/_base/utils/retry.ts

@@ -13,12 +13,16 @@ export interface RetryErrorFields {
   readonly statusCode?: number;
 }
 
+export function retryBackoffDelay(attemptIndex: number): number {
+  const base = Math.min(BASE_DELAY_MS * Math.pow(RETRY_FACTOR, attemptIndex), MAX_DELAY_MS);
+  return base + Math.random() * JITTER_FACTOR * base;
+}
+
 export function retryBackoffDelays(maxAttempts: number): number[] {
   const count = Math.max(maxAttempts - 1, 0);
   const delays: number[] = [];
   for (let i = 0; i < count; i += 1) {
-    const base = Math.min(BASE_DELAY_MS * Math.pow(RETRY_FACTOR, i), MAX_DELAY_MS);
-    delays.push(base + Math.random() * JITTER_FACTOR * base);
+    delays.push(retryBackoffDelay(i));
   }
   return delays;
 }

packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts

@@ -17,6 +17,7 @@ import { IAgentMediaResolverService } from '#/agent/media/mediaResolver';
 import { ISessionUsageService } from '#/session/usage/sessionUsage';
 import { IConfigService } from '#/app/config/config';
 import {
+  APIContextOverflowError,
   APIRequestTooLargeError,
   APIStatusError,
   APITimeoutError,
@@ -73,8 +74,15 @@ import {
   type LlmRequestToolSchema,
 } from './llmRequestOps';
 import { isAbortError, linkAbortSignal } from '#/_base/utils/abort';
+import { parseBooleanEnv } from '#/_base/utils/env';
 import { ErrorCodes, Error2, unwrapErrorCause } from '#/errors';
-import { retryErrorFields } from '#/_base/utils/retry';
+import {
+  readRetryAfterMs,
+  retryBackoffDelay,
+  retryErrorFields,
+  sleepForRetry,
+} from '#/_base/utils/retry';
+import { IBootstrapService } from '#/app/bootstrap/bootstrap';
 
 const EMPTY_TOOL_PARAMETERS: Record<string, unknown> = {
   type: 'object',
@@ -86,6 +94,7 @@ const noopOnPart: AgentLLMRequestPartHandler = () => {};
 const DEFAULT_STREAM_IDLE_TIMEOUT_MS = 180_000;
 
 const STREAM_STALL_REASON = { reason: 'llm-stream-idle-timeout' };
+export const PYTHINKER_CODE_INFINITE_RETRY_ENV = 'PYTHINKER_CODE_INFINITE_RETRY';
 
 interface ResolvedLLMRequest {
   readonly requester: ModelRequester;
@@ -162,6 +171,7 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService {
     @IEventDispatcher private readonly dispatcher: IEventDispatcher,
     @IAgentScopeContext private readonly scopeContext: IAgentScopeContext,
     @IAgentStateService private readonly states: IAgentStateService,
+    @IBootstrapService private readonly bootstrap: IBootstrapService,
   ) {
     this.states.contributeState(llmRequestTraceKey);
     this.states.contributeState(llmRequesterLastConfigLogSignatureKey);
@@ -456,6 +466,7 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService {
       };
     };
 
+    let infiniteRetryAttempt = 0;
     for (;;) {
       try {
         return await run(policy);
@@ -467,12 +478,39 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService {
           signal,
           captureMediaStripPolicy,
         );
-        if (nextPolicy === undefined) throw error;
-        policy = nextPolicy;
+        if (nextPolicy !== undefined) {
+          policy = nextPolicy;
+          continue;
+        }
+        const raw = unwrapErrorCause(error);
+        if (
+          !this.infiniteRetryEnabled ||
+          isAbortError(error) ||
+          signal?.aborted === true ||
+          raw instanceof APIContextOverflowError
+        ) {
+          throw error;
+        }
+        infiniteRetryAttempt += 1;
+        const delayMs =
+          readRetryAfterMs(raw) ??
+          retryBackoffDelay(infiniteRetryAttempt - 1);
+        this.log.warn('llm request failed; retrying indefinitely (PYTHINKER_CODE_INFINITE_RETRY)', {
+          model: request.model.name,
+          ...request.logFields,
+          attempt: infiniteRetryAttempt,
+          delayMs,
+          ...retryErrorFields(error),
+        });
+        await sleepForRetry(delayMs, signal);
       }
     }
   }
 
+  private get infiniteRetryEnabled(): boolean {
+    return parseBooleanEnv(this.bootstrap.getEnv(PYTHINKER_CODE_INFINITE_RETRY_ENV)) === true;
+  }
+
   private nextProjectionPolicyForError(
     error: unknown,
     policy: ProjectionPolicy | undefined,

packages/agent-core-v2/test/agent/fullCompaction/fullCompaction.test.ts

@@ -666,6 +666,59 @@ describe('FullCompaction', () => {
     await ctx.expectResumeMatches();
   });
 
+  it('retries any compaction request error indefinitely when PYTHINKER_CODE_INFINITE_RETRY is set', async () => {
+    vi.stubEnv('PYTHINKER_CODE_INFINITE_RETRY', '1');
+    let attempts = 0;
+    const generate: GenerateFn = async () => {
+      attempts += 1;
+      if (attempts === 1) throw new APIStatusError(400, 'endpoint broken', null, 1);
+      if (attempts === 2) throw new APIStatusError(404, 'model not found', null, 1);
+      return textResult('Recovered compacted summary.');
+    };
+    const ctx = testAgent({ generate });
+    ctx.configure({
+      provider: CATALOGUED_PROVIDER,
+      modelCapabilities: CATALOGUED_MODEL_CAPABILITIES,
+    });
+    ctx.appendExchange(1, 'old user one', 'old assistant one', 20);
+    ctx.appendExchange(2, 'recent user two', 'recent assistant two', 80);
+    const compacted = ctx.once('full_compaction.complete');
+    const completed = ctx.once('compaction.completed');
+
+    await ctx.rpc.beginCompaction({});
+    await compacted;
+    await completed;
+
+    expect(attempts).toBe(3);
+    await ctx.expectResumeMatches();
+  });
+
+  it('lets context overflow reach compaction shrink instead of retrying when PYTHINKER_CODE_INFINITE_RETRY is set', async () => {
+    vi.stubEnv('PYTHINKER_CODE_INFINITE_RETRY', '1');
+    let attempts = 0;
+    const generate: GenerateFn = async () => {
+      attempts += 1;
+      if (attempts === 1) throw new APIContextOverflowError(400, 'context length exceeded');
+      return textResult('Recovered compacted summary.');
+    };
+    const ctx = testAgent({ generate });
+    ctx.configure({
+      provider: CATALOGUED_PROVIDER,
+      modelCapabilities: CATALOGUED_MODEL_CAPABILITIES,
+    });
+    ctx.appendExchange(1, 'old user one', 'old assistant one', 20);
+    ctx.appendExchange(2, 'recent user two', 'recent assistant two', 80);
+    const compacted = ctx.once('full_compaction.complete');
+    const completed = ctx.once('compaction.completed');
+
+    await ctx.rpc.beginCompaction({});
+    await compacted;
+    await completed;
+
+    expect(attempts).toBe(2);
+    await ctx.expectResumeMatches();
+  });
+
   it('recovers from an image-format rejection with a media-stripped resend', async () => {
     let attempts = 0;
     let sawMedia = false;

packages/agent-core-v2/test/agent/llmRequester/llmRequesterService.test.ts

@@ -13,8 +13,9 @@ import {
   type ProjectionPolicy,
 } from '#/agent/contextProjector/contextProjector';
 import { AgentContextProjectorService } from '#/agent/contextProjector/contextProjectorService';
-import { AgentLLMRequesterService } from '#/agent/llmRequester/llmRequesterService';
+import { AgentLLMRequesterService, PYTHINKER_CODE_INFINITE_RETRY_ENV } from '#/agent/llmRequester/llmRequesterService';
 import { IAgentLLMRequesterService } from '#/agent/llmRequester/llmRequester';
+import { IBootstrapService } from '#/app/bootstrap/bootstrap';
 import { ISessionTokenCountingService } from '#/session/tokenCounting/sessionTokenCounting';
 import { IAgentProfileService } from '#/agent/profile/profile';
 import { IAgentStateService } from '#/agent/state/agentState';
@@ -29,7 +30,10 @@ import type { Event2 } from '#/app/event/event2';
 import { IEventBus } from '#/app/event/eventBus';
 import {
   APIConnectionError,
+  APIContextOverflowError,
   APIEmptyResponseError,
+  APIProviderQuotaExhaustedError,
+  APIProviderRateLimitError,
   APIRequestTooLargeError,
   APIStatusError,
   APITimeoutError,
@@ -56,6 +60,7 @@ import { Error2, ErrorCodes } from '#/errors';
 import { IEventDispatcher } from '#/state/eventDispatcher';
 import type { WireRecord } from '#/wire/record';
 import { recordingTelemetry, type TelemetryRecord } from '../../app/telemetry/stubs';
+import { stubBootstrap } from '../../app/bootstrap/stubs';
 
 import {
   recordingWireLog,
@@ -163,9 +168,11 @@ function createService(
     readonly mediaResolver?: Partial<IAgentMediaResolverService>;
     readonly contextMessages?: Message[];
     readonly llmConfig?: LlmConfig;
+    readonly env?: Record<string, string>;
   } = {},
 ) {
   const ix = disposables.add(new TestInstantiationService());
+  ix.stub(IBootstrapService, stubBootstrap('/tmp/pythinker-code-llm-requester-test', options.env ?? {}));
   const thinkingLevel = options.thinkingLevel ?? 'off';
   const profile: Partial<IAgentProfileService> = {
     resolveModelContext: () => ({
@@ -357,6 +364,121 @@ describe('AgentLLMRequesterService strict resend', () => {
   });
 });
 
+describe('AgentLLMRequesterService infinite retry', () => {
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it('retries every request error while PYTHINKER_CODE_INFINITE_RETRY is set', async () => {
+    vi.useFakeTimers();
+    const calls = { value: 0 };
+    const requester = createRequester(calls, new APIStatusError(400, 'endpoint broken'), [
+      new APIStatusError(404, 'model not found'),
+      new APIConnectionError('socket hang up'),
+      new APIProviderQuotaExhaustedError('quota exhausted'),
+    ]);
+    const { service } = createService(requester, undefined, {
+      env: { [PYTHINKER_CODE_INFINITE_RETRY_ENV]: '1' },
+    });
+
+    const promise = service.request();
+    await vi.runAllTimersAsync();
+    const finish = await promise;
+
+    expect(calls.value).toBe(5);
+    expect(finish.message.content).toEqual([{ type: 'text', text: 'ok' }]);
+  });
+
+  it('honors the provider retry-after delay while retrying indefinitely', async () => {
+    const calls = { value: 0 };
+    const requester = createRequester(calls, new APIProviderRateLimitError('slow down', null, 1));
+    const { service } = createService(requester, undefined, {
+      env: { [PYTHINKER_CODE_INFINITE_RETRY_ENV]: '1' },
+    });
+
+    const startedAt = Date.now();
+    await service.request();
+
+    expect(calls.value).toBe(2);
+    expect(Date.now() - startedAt).toBeLessThan(500);
+  });
+
+  it('stops retrying when the caller aborts during the backoff wait', async () => {
+    vi.useFakeTimers();
+    const calls = { value: 0 };
+    const requester = createRequester(calls, new APIStatusError(400, 'endpoint broken'));
+    const { service } = createService(requester, undefined, {
+      env: { [PYTHINKER_CODE_INFINITE_RETRY_ENV]: '1' },
+    });
+    const controller = new AbortController();
+    setTimeout(() => controller.abort(new Error('stop')), 100);
+
+    const promise = service.request({}, undefined, controller.signal);
+    const assertion = expect(promise).rejects.toThrow('stop');
+    await vi.runAllTimersAsync();
+    await assertion;
+
+    expect(calls.value).toBe(1);
+  });
+
+  it('keeps deterministic projection recovery ahead of infinite retry', async () => {
+    vi.useFakeTimers();
+    const calls = { value: 0 };
+    const requester = createRequester(calls, new APIRequestTooLargeError(413, 'Request Entity Too Large'));
+    const { service } = createService(requester, undefined, {
+      env: { [PYTHINKER_CODE_INFINITE_RETRY_ENV]: '1' },
+    });
+
+    await service.request();
+
+    expect(calls.value).toBe(2);
+  });
+
+  it('lets context overflow reach deterministic recovery instead of retrying', async () => {
+    vi.useFakeTimers();
+    const calls = { value: 0 };
+    const requester = createRequester(
+      calls,
+      new APIContextOverflowError(400, 'context length exceeded'),
+    );
+    const { service } = createService(requester, undefined, {
+      env: { [PYTHINKER_CODE_INFINITE_RETRY_ENV]: '1' },
+    });
+
+    await expect(service.request()).rejects.toBeInstanceOf(APIContextOverflowError);
+    expect(calls.value).toBe(1);
+  });
+
+  it('retries operation requests indefinitely', async () => {
+    vi.useFakeTimers();
+    const calls = { value: 0 };
+    const requester = createRequester(calls, new APIStatusError(400, 'endpoint broken'), [
+      new APIStatusError(404, 'model not found'),
+    ]);
+    const { service } = createService(requester, undefined, {
+      env: { [PYTHINKER_CODE_INFINITE_RETRY_ENV]: '1' },
+    });
+
+    const promise = service.request({
+      source: { type: 'operation', requestKind: 'full_compaction' },
+    });
+    await vi.runAllTimersAsync();
+    await promise;
+
+    expect(calls.value).toBe(3);
+  });
+
+  it('does not retry when the switch is unset', async () => {
+    vi.useFakeTimers();
+    const calls = { value: 0 };
+    const requester = createRequester(calls, new APIStatusError(400, 'endpoint broken'));
+    const { service } = createService(requester, undefined);
+
+    await expect(service.request()).rejects.toMatchObject({ statusCode: 400 });
+    expect(calls.value).toBe(1);
+  });
+});
+
 describe('AgentLLMRequesterService media-stripped resend', () => {
   const IMAGE_FORMAT_400 = new APIStatusError(
     400,