diff --git a/.changeset/hook-stop-deadline.md b/.changeset/hook-stop-deadline.md new file mode 100644 index 0000000000..a3a18f0f46 --- /dev/null +++ b/.changeset/hook-stop-deadline.md @@ -0,0 +1,22 @@ +--- +id: hook-stop-deadline +tasks: [T13890] +kind: fix +summary: cleo hook stop waits at most 20 s for session end and 5 s for ask-enforce, so it returns under Claude Code's Stop timeout +--- + +The Stop dispatcher (`cleo hook stop`) could outlast Claude Code's 120 s Stop +timeout. The goal tick was already bounded: its process group is SIGKILLed at +the deadline, and its stdout stops being read even when an escaped +grandchild holds it. The two steps after the tick were not bounded. + +- `cleo session end` (claude-code) used to be a `spawnSync` with no timeout. + It is now waited for at most 20 s (`SESSION_END_WAIT_MS`), then left + running in the background. A failure or an overrun prints one stderr line + with the duration. In rel-qual's repro on a 1.38 GB store, session end + spent about 150 s writing three full backups (T13891). +- The ask-enforce step has its own 5 s deadline (`ASK_ENFORCE_DEADLINE_MS`). + Past that it counts as no block and prints one stderr line. + +A Stop now returns within the tick deadline (60 s at most) plus 5 s plus +20 s, under the 120 s harness timeout. diff --git a/packages/cleo/src/cli/__tests__/stop-dispatch-entry.test.ts b/packages/cleo/src/cli/__tests__/stop-dispatch-entry.test.ts index 60d59fbf38..915cb7ea81 100644 --- a/packages/cleo/src/cli/__tests__/stop-dispatch-entry.test.ts +++ b/packages/cleo/src/cli/__tests__/stop-dispatch-entry.test.ts @@ -10,12 +10,14 @@ import { GOAL_TICK_PROVIDER_ENV } from '@cleocode/contracts/goal-runtime.js'; import { describe, expect, it, vi } from 'vitest'; import { + ASK_ENFORCE_DEADLINE_MS, askEnforceOptedOut, hasGoalWorkspace, MAX_REASON_CHARS, MAX_TICK_TIMEOUT_MS, PROVIDER_ENDS_SESSION, processStopDispatchDeps, + runBounded, runTickKillable, type StopDispatchDeps, stopDispatch, @@ -226,6 +228,92 @@ describe('stopDispatch (T13796)', () => { } }); + it('returns within the deadline when the tick forks a grandchild that escapes the group and holds stdout (T13890)', async () => { + const marker = `cleo-t13890-${process.pid}-${Date.now()}`; + // The grandchild calls setsid (detached), so the group SIGKILL cannot reach it, + // and it inherits the tick's stdout, so that pipe never reaches EOF on its own. + const tick = `node -e "require('child_process').spawn('node',['-e','setTimeout(()=>{},30000)','${marker}'],{detached:true,stdio:['ignore','inherit','ignore']}).unref();setInterval(()=>{},1000)"`; + const real = processStopDispatchDeps('codex', { CLEO_ASK_ENFORCE: 'off' }); + const endSession = vi.fn(); + const started = Date.now(); + try { + const r = await stopDispatch( + STOP, + { CLEO_ASK_ENFORCE: 'off', CLEO_GOAL_TICK_COMMAND: tick, CLEO_GOAL_TICK_TIMEOUT_MS: '800' }, + { ...real, hasGoalWorkspace: () => true, endSession }, + true, + ); + expect(r).toEqual({ stdout: '', sessionEnded: true }); + expect(Date.now() - started).toBeLessThan(800 + 5000); + expect(endSession).toHaveBeenCalledTimes(1); + } finally { + const { execFileSync } = await import('node:child_process'); + try { + execFileSync('pkill', ['-f', marker]); + } catch { + // Already gone. + } + } + }); + + it('a real session end that outlasts its wait: the stop returns at the wait, with one stderr line (T13893)', async () => { + const marker = `cleo-t13893-${process.pid}-${Date.now()}`; + const warn = vi.fn(); + const d = deps({ status: 127, stdout: '' }); + const started = Date.now(); + try { + const r = await stopDispatch( + STOP, + {}, + { + ...d.deps, + warn, + endSession: () => + runBounded( + process.execPath, + ['-e', 'setTimeout(()=>{},30000)', marker], + 600, + 'session end', + warn, + ), + }, + ); + expect(r).toEqual({ stdout: '', sessionEnded: true }); + expect(Date.now() - started).toBeLessThan(600 + 3000); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]?.[0]).toMatch(/session end still running after \d+ ms/); + // A failing session end is reported with its duration and exit status. + const failed = vi.fn(); + await runBounded(process.execPath, ['-e', 'process.exit(3)'], 5000, 'session end', failed); + expect(failed.mock.calls[0]?.[0]).toMatch(/session end failed after \d+ ms \(exit 3\)/); + } finally { + const { execFileSync } = await import('node:child_process'); + try { + execFileSync('pkill', ['-f', marker]); + } catch { + // Already gone. + } + } + }); + + it('a hung ask-enforce cannot hold the stop past its own deadline (T13890)', async () => { + vi.useFakeTimers(); + try { + const d = deps({ status: 127, stdout: '' }, () => new Promise(() => {})); + const warn = vi.fn(); + const pending = stopDispatch(STOP, {}, { ...d.deps, warn }); + await vi.advanceTimersByTimeAsync(ASK_ENFORCE_DEADLINE_MS); + expect(await pending).toEqual({ stdout: '', sessionEnded: true }); + expect(d.endSession).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]?.[0]).toContain( + `ask-enforce gave no answer within ${ASK_ENFORCE_DEADLINE_MS} ms`, + ); + } finally { + vi.useRealTimers(); + } + }); + it('a tick that traps SIGTERM is SIGKILLed with its group at the deadline, and fails open', async () => { const marker = `cleo-t13796-${process.pid}-${Date.now()}`; const trap = `node -e "process.on('SIGTERM',()=>{});setInterval(()=>{},1000)" ${marker}-trap`; diff --git a/packages/cleo/src/cli/hook-entry.ts b/packages/cleo/src/cli/hook-entry.ts index 6a39dff8d3..ad47d52c07 100644 --- a/packages/cleo/src/cli/hook-entry.ts +++ b/packages/cleo/src/cli/hook-entry.ts @@ -1100,7 +1100,7 @@ async function runStopDispatch(rest: readonly string[], io: HookIo): Promise Promise; /** The ask-enforce hook's stdout for this Stop (empty when it does not answer). */ readonly askEnforce: (input: string) => Promise; - /** End the CLEO session, as the Stop hook always did. */ - readonly endSession: () => void; + /** End the CLEO session, as the Stop hook always did; resolves within {@link SESSION_END_WAIT_MS} (T13890). */ + readonly endSession: () => void | Promise; + /** One diagnostic line to stderr (a deadline hit, a failed or overrunning session end). */ + readonly warn?: (line: string) => void; } /** The goal runtime's verdict on one stop. */ @@ -132,6 +148,32 @@ export function parseTick(run: TickRun): TickDecision { return answer.final === false ? { kind: 'hold' } : { kind: 'final' }; } +/** `work`, or `fallback` once `ms` pass (calling `onTimeout`) or `work` rejects; the timer never holds the process. */ +function withDeadline( + work: Promise, + ms: number, + fallback: T, + onTimeout?: () => void, +): Promise { + return new Promise((done) => { + const timer = setTimeout(() => { + onTimeout?.(); + done(fallback); + }, ms); + timer.unref(); + work.then( + (value) => { + clearTimeout(timer); + done(value); + }, + () => { + clearTimeout(timer); + done(fallback); + }, + ); + }); +} + /** Whether a hook answer is a `{"decision":"block"}` (Claude Code and Codex). */ function isBlock(stdout: string): boolean { try { @@ -170,10 +212,14 @@ export async function stopDispatch( command === null ? { kind: 'final' } : parseTick(await deps.runTick(command, stdin, timeout)); if (decision.kind === 'block') return { stdout: decision.answer, sessionEnded: false }; if (decision.kind === 'hold') return { stdout: '', sessionEnded: false }; - const asked = await deps.askEnforce(stdin).catch(() => ''); + const asked = await withDeadline(deps.askEnforce(stdin), ASK_ENFORCE_DEADLINE_MS, '', () => + deps.warn?.( + `[cleo hook] ask-enforce gave no answer within ${ASK_ENFORCE_DEADLINE_MS} ms; treated as no block\n`, + ), + ); if ((asked !== '' && isBlock(asked)) || !endsSession) return { stdout: asked, sessionEnded: false }; - deps.endSession(); + await deps.endSession(); return { stdout: asked, sessionEnded: true }; } @@ -208,8 +254,8 @@ export async function askEnforceOptedOut( /** * The real side effects: the tick through the shell (the command is the * project's own configuration), ask-enforce in-process, and - * `cleo session end --quiet` through the running CLI, its envelope kept off - * the harness's stdout. + * `cleo session end --quiet` through the running CLI, waited for at most + * {@link SESSION_END_WAIT_MS}, its output kept off the harness's streams. * * @param provider - Harness protocol ask-enforce answers in. * @param env - Environment ask-enforce reads. @@ -219,6 +265,9 @@ export function processStopDispatchDeps( provider: StopDispatchProvider, env: Readonly>, ): StopDispatchDeps { + const warn = (line: string): void => { + process.stderr.write(line); + }; return { askEnforce: async (input) => { if (await askEnforceOptedOut(env, process.cwd())) return ''; @@ -233,17 +282,74 @@ export function processStopDispatchDeps( }, hasGoalWorkspace: (input) => hasGoalWorkspace(input, env), runTick: (command, input, timeoutMs) => runTickKillable(command, input, timeoutMs, provider), + // T13890: session end can outlast the harness's Stop timeout, so it is + // waited for at most SESSION_END_WAIT_MS, then left to finish on its own. endSession: () => { const cli = process.argv[1]; - spawnSync( + return runBounded( cli ? process.execPath : 'cleo', [...(cli ? [cli] : []), 'session', 'end', '--quiet'], - { stdio: ['ignore', 'ignore', 'inherit'] }, + SESSION_END_WAIT_MS, + 'session end', + warn, ); }, + warn, }; } +/** + * Run a detached process and wait for it at most `waitMs`. A non-zero exit or + * a spawn error gets one `warn` line with the duration. Past the wait the + * process keeps running, unref'd, with one `warn` line. Never rejects. + * + * @param command - Executable. + * @param args - Its arguments. + * @param waitMs - Longest wait. + * @param label - Name used in the warning. + * @param warn - Receives the diagnostic line. + * @returns Resolves on exit or at the deadline, whichever comes first. + */ +export function runBounded( + command: string, + args: readonly string[], + waitMs: number, + label: string, + warn: (line: string) => void, +): Promise { + return new Promise((done) => { + const started = Date.now(); + let settled = false; + const child = spawn(command, [...args], { detached: true, stdio: 'ignore' }); + const timer = setTimeout(() => { + settled = true; + child.unref(); + warn( + `[cleo hook] ${label} still running after ${Date.now() - started} ms; left to finish in the background\n`, + ); + done(); + }, waitMs); + child.on('error', (err) => { + if (settled) return; + settled = true; + clearTimeout(timer); + warn(`[cleo hook] ${label} failed after ${Date.now() - started} ms: ${err.message}\n`); + done(); + }); + child.on('exit', (code, signal) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (code !== 0) { + warn( + `[cleo hook] ${label} failed after ${Date.now() - started} ms (${signal ?? `exit ${code}`})\n`, + ); + } + done(); + }); + }); +} + /** * Whether `.opencode/goals/` exists at or above the Stop's project (the harness * project directory `CLAUDE_PROJECT_DIR`, else the payload's `cwd`, else the