Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions .changeset/hook-stop-deadline.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
---
id: hook-stop-deadline
tasks: [T13890]
kind: fix
summary: cleo hook stop waits at most 20 s for session end and 5 s for ask-enforce, so it returns under Claude Code's Stop timeout
---

The Stop dispatcher (`cleo hook stop`) could outlast Claude Code's 120 s Stop
timeout. The goal tick was already bounded: its process group is SIGKILLed at
the deadline, and its stdout stops being read even when an escaped
grandchild holds it. The two steps after the tick were not bounded.

- `cleo session end` (claude-code) used to be a `spawnSync` with no timeout.
It is now waited for at most 20 s (`SESSION_END_WAIT_MS`), then left
running in the background. A failure or an overrun prints one stderr line
with the duration. In rel-qual's repro on a 1.38 GB store, session end
spent about 150 s writing three full backups (T13891).
- The ask-enforce step has its own 5 s deadline (`ASK_ENFORCE_DEADLINE_MS`).
Past that it counts as no block and prints one stderr line.

A Stop now returns within the tick deadline (60 s at most) plus 5 s plus
20 s, under the 120 s harness timeout.
88 changes: 88 additions & 0 deletions packages/cleo/src/cli/__tests__/stop-dispatch-entry.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,14 @@
import { GOAL_TICK_PROVIDER_ENV } from '@cleocode/contracts/goal-runtime.js';
import { describe, expect, it, vi } from 'vitest';
import {
ASK_ENFORCE_DEADLINE_MS,
askEnforceOptedOut,
hasGoalWorkspace,
MAX_REASON_CHARS,
MAX_TICK_TIMEOUT_MS,
PROVIDER_ENDS_SESSION,
processStopDispatchDeps,
runBounded,
runTickKillable,
type StopDispatchDeps,
stopDispatch,
Expand Down Expand Up @@ -226,6 +228,92 @@ describe('stopDispatch (T13796)', () => {
}
});

it('returns within the deadline when the tick forks a grandchild that escapes the group and holds stdout (T13890)', async () => {
const marker = `cleo-t13890-${process.pid}-${Date.now()}`;
// The grandchild calls setsid (detached), so the group SIGKILL cannot reach it,
// and it inherits the tick's stdout, so that pipe never reaches EOF on its own.
const tick = `node -e "require('child_process').spawn('node',['-e','setTimeout(()=>{},30000)','${marker}'],{detached:true,stdio:['ignore','inherit','ignore']}).unref();setInterval(()=>{},1000)"`;
const real = processStopDispatchDeps('codex', { CLEO_ASK_ENFORCE: 'off' });
const endSession = vi.fn();
const started = Date.now();
try {
const r = await stopDispatch(
STOP,
{ CLEO_ASK_ENFORCE: 'off', CLEO_GOAL_TICK_COMMAND: tick, CLEO_GOAL_TICK_TIMEOUT_MS: '800' },
{ ...real, hasGoalWorkspace: () => true, endSession },
true,
);
expect(r).toEqual({ stdout: '', sessionEnded: true });
expect(Date.now() - started).toBeLessThan(800 + 5000);
expect(endSession).toHaveBeenCalledTimes(1);
} finally {
const { execFileSync } = await import('node:child_process');
try {
execFileSync('pkill', ['-f', marker]);
} catch {
// Already gone.
}
}
});

it('a real session end that outlasts its wait: the stop returns at the wait, with one stderr line (T13893)', async () => {
const marker = `cleo-t13893-${process.pid}-${Date.now()}`;
const warn = vi.fn();
const d = deps({ status: 127, stdout: '' });
const started = Date.now();
try {
const r = await stopDispatch(
STOP,
{},
{
...d.deps,
warn,
endSession: () =>
runBounded(
process.execPath,
['-e', 'setTimeout(()=>{},30000)', marker],
600,
'session end',
warn,
),
},
);
expect(r).toEqual({ stdout: '', sessionEnded: true });
expect(Date.now() - started).toBeLessThan(600 + 3000);
expect(warn).toHaveBeenCalledTimes(1);
expect(warn.mock.calls[0]?.[0]).toMatch(/session end still running after \d+ ms/);
// A failing session end is reported with its duration and exit status.
const failed = vi.fn();
await runBounded(process.execPath, ['-e', 'process.exit(3)'], 5000, 'session end', failed);
expect(failed.mock.calls[0]?.[0]).toMatch(/session end failed after \d+ ms \(exit 3\)/);
} finally {
const { execFileSync } = await import('node:child_process');
try {
execFileSync('pkill', ['-f', marker]);
} catch {
// Already gone.
}
}
});

it('a hung ask-enforce cannot hold the stop past its own deadline (T13890)', async () => {
vi.useFakeTimers();
try {
const d = deps({ status: 127, stdout: '' }, () => new Promise<string>(() => {}));
const warn = vi.fn();
const pending = stopDispatch(STOP, {}, { ...d.deps, warn });
await vi.advanceTimersByTimeAsync(ASK_ENFORCE_DEADLINE_MS);
expect(await pending).toEqual({ stdout: '', sessionEnded: true });
expect(d.endSession).toHaveBeenCalledTimes(1);
expect(warn).toHaveBeenCalledTimes(1);
expect(warn.mock.calls[0]?.[0]).toContain(
`ask-enforce gave no answer within ${ASK_ENFORCE_DEADLINE_MS} ms`,
);
} finally {
vi.useRealTimers();
}
});

it('a tick that traps SIGTERM is SIGKILLed with its group at the deadline, and fails open', async () => {
const marker = `cleo-t13796-${process.pid}-${Date.now()}`;
const trap = `node -e "process.on('SIGTERM',()=>{});setInterval(()=>{},1000)" ${marker}-trap`;
Expand Down
2 changes: 1 addition & 1 deletion packages/cleo/src/cli/hook-entry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1100,7 +1100,7 @@ async function runStopDispatch(rest: readonly string[], io: HookIo): Promise<num
`[cleo hook] stop dispatch failed: ${err instanceof Error ? err.message : String(err)}\n`,
);
// Fail open to what this provider's Stop did before the dispatcher.
if (entry.PROVIDER_ENDS_SESSION[provider]) deps.endSession();
if (entry.PROVIDER_ENDS_SESSION[provider]) await deps.endSession();
}
return 0;
}
Expand Down
124 changes: 115 additions & 9 deletions packages/cleo/src/cli/stop-dispatch-entry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
* @task T13796
*/

import { spawn, spawnSync } from 'node:child_process';
import { spawn } from 'node:child_process';
import { existsSync } from 'node:fs';
import { dirname, join, resolve } from 'node:path';
import type { HookJsonValue } from '@cleocode/contracts';
Expand Down Expand Up @@ -68,6 +68,20 @@ const DEFAULT_TICK_TIMEOUT_MS = 30_000;
/** Longest tick deadline, kept well under the Stop hook's harness timeout (120 s). */
export const MAX_TICK_TIMEOUT_MS = 60_000;

/**
* Longest the ask-enforce step may take once the tick is done (T13890): it is
* in-process and normally instant, but nothing after the tick may stretch the
* Stop toward the harness's own timeout.
*/
export const ASK_ENFORCE_DEADLINE_MS = 5_000;

/**
* Longest the Stop waits for `cleo session end` (T13890, T13893). Past it the
* session end is left to finish in the background. Tick (60 s at most) plus
* ask-enforce plus this stays under Claude Code's 120 s Stop timeout.
*/
export const SESSION_END_WAIT_MS = 20_000;

/** Longest block reason forwarded to the harness. */
export const MAX_REASON_CHARS = 4000;

Expand Down Expand Up @@ -96,8 +110,10 @@ export interface StopDispatchDeps {
readonly runTick: (command: string, input: string, timeoutMs: number) => Promise<TickRun>;
/** The ask-enforce hook's stdout for this Stop (empty when it does not answer). */
readonly askEnforce: (input: string) => Promise<string>;
/** End the CLEO session, as the Stop hook always did. */
readonly endSession: () => void;
/** End the CLEO session, as the Stop hook always did; resolves within {@link SESSION_END_WAIT_MS} (T13890). */
readonly endSession: () => void | Promise<void>;
/** One diagnostic line to stderr (a deadline hit, a failed or overrunning session end). */
readonly warn?: (line: string) => void;
}

/** The goal runtime's verdict on one stop. */
Expand Down Expand Up @@ -132,6 +148,32 @@ export function parseTick(run: TickRun): TickDecision {
return answer.final === false ? { kind: 'hold' } : { kind: 'final' };
}

/** `work`, or `fallback` once `ms` pass (calling `onTimeout`) or `work` rejects; the timer never holds the process. */
function withDeadline<T>(
work: Promise<T>,
ms: number,
fallback: T,
onTimeout?: () => void,
): Promise<T> {
return new Promise((done) => {
const timer = setTimeout(() => {
onTimeout?.();
done(fallback);
}, ms);
timer.unref();
work.then(
(value) => {
clearTimeout(timer);
done(value);
},
() => {
clearTimeout(timer);
done(fallback);
},
);
});
}

/** Whether a hook answer is a `{"decision":"block"}` (Claude Code and Codex). */
function isBlock(stdout: string): boolean {
try {
Expand Down Expand Up @@ -170,10 +212,14 @@ export async function stopDispatch(
command === null ? { kind: 'final' } : parseTick(await deps.runTick(command, stdin, timeout));
if (decision.kind === 'block') return { stdout: decision.answer, sessionEnded: false };
if (decision.kind === 'hold') return { stdout: '', sessionEnded: false };
const asked = await deps.askEnforce(stdin).catch(() => '');
const asked = await withDeadline(deps.askEnforce(stdin), ASK_ENFORCE_DEADLINE_MS, '', () =>
deps.warn?.(
`[cleo hook] ask-enforce gave no answer within ${ASK_ENFORCE_DEADLINE_MS} ms; treated as no block\n`,
),
);
if ((asked !== '' && isBlock(asked)) || !endsSession)
return { stdout: asked, sessionEnded: false };
deps.endSession();
await deps.endSession();
return { stdout: asked, sessionEnded: true };
}

Expand Down Expand Up @@ -208,8 +254,8 @@ export async function askEnforceOptedOut(
/**
* The real side effects: the tick through the shell (the command is the
* project's own configuration), ask-enforce in-process, and
* `cleo session end --quiet` through the running CLI, its envelope kept off
* the harness's stdout.
* `cleo session end --quiet` through the running CLI, waited for at most
* {@link SESSION_END_WAIT_MS}, its output kept off the harness's streams.
*
* @param provider - Harness protocol ask-enforce answers in.
* @param env - Environment ask-enforce reads.
Expand All @@ -219,6 +265,9 @@ export function processStopDispatchDeps(
provider: StopDispatchProvider,
env: Readonly<Record<string, string | undefined>>,
): StopDispatchDeps {
const warn = (line: string): void => {
process.stderr.write(line);
};
return {
askEnforce: async (input) => {
if (await askEnforceOptedOut(env, process.cwd())) return '';
Expand All @@ -233,17 +282,74 @@ export function processStopDispatchDeps(
},
hasGoalWorkspace: (input) => hasGoalWorkspace(input, env),
runTick: (command, input, timeoutMs) => runTickKillable(command, input, timeoutMs, provider),
// T13890: session end can outlast the harness's Stop timeout, so it is
// waited for at most SESSION_END_WAIT_MS, then left to finish on its own.
endSession: () => {
const cli = process.argv[1];
spawnSync(
return runBounded(
cli ? process.execPath : 'cleo',
[...(cli ? [cli] : []), 'session', 'end', '--quiet'],
{ stdio: ['ignore', 'ignore', 'inherit'] },
SESSION_END_WAIT_MS,
'session end',
warn,
);
},
warn,
};
}

/**
* Run a detached process and wait for it at most `waitMs`. A non-zero exit or
* a spawn error gets one `warn` line with the duration. Past the wait the
* process keeps running, unref'd, with one `warn` line. Never rejects.
*
* @param command - Executable.
* @param args - Its arguments.
* @param waitMs - Longest wait.
* @param label - Name used in the warning.
* @param warn - Receives the diagnostic line.
* @returns Resolves on exit or at the deadline, whichever comes first.
*/
export function runBounded(
command: string,
args: readonly string[],
waitMs: number,
label: string,
warn: (line: string) => void,
): Promise<void> {
return new Promise((done) => {
const started = Date.now();
let settled = false;
const child = spawn(command, [...args], { detached: true, stdio: 'ignore' });
const timer = setTimeout(() => {
settled = true;
child.unref();
warn(
`[cleo hook] ${label} still running after ${Date.now() - started} ms; left to finish in the background\n`,
);
done();
}, waitMs);
child.on('error', (err) => {
if (settled) return;
settled = true;
clearTimeout(timer);
warn(`[cleo hook] ${label} failed after ${Date.now() - started} ms: ${err.message}\n`);
done();
});
child.on('exit', (code, signal) => {
if (settled) return;
settled = true;
clearTimeout(timer);
if (code !== 0) {
warn(
`[cleo hook] ${label} failed after ${Date.now() - started} ms (${signal ?? `exit ${code}`})\n`,
);
}
done();
});
});
}

/**
* Whether `.opencode/goals/` exists at or above the Stop's project (the harness
* project directory `CLAUDE_PROJECT_DIR`, else the payload's `cwd`, else the
Expand Down
Loading