Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 48 additions & 9 deletions src/agent/claude/adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
} from './owner-only-settings';
import { cleanupSandboxPlaceholders, snapshotPlaceholders } from './sandbox-placeholders';
import { planRuntimeDirMask, wrapWithRuntimeDirMask } from './runtime-dir-mask';
import { killProcessTree } from '../../platform/kill-tree';
import { spawnProcess, type SpawnedProcessByStdio } from '../../platform/spawn';
import { buildBridgeSystemPrompt } from '../bridge-system-prompt';
import { buildLarkChannelEnv, type LarkChannelEnvContext } from '../lark-channel-env';
Expand Down Expand Up @@ -250,6 +251,7 @@ export class ClaudeAdapter implements AgentAdapter {

return {
runId: opts.runId,
pid: child.pid,
events: holdEmptyDone(createEventStream(child, stderrChunks, () => runtimeError), {
holdMs: this.emptyResultHoldMs,
}),
Expand All @@ -265,7 +267,10 @@ export class ClaudeAdapter implements AgentAdapter {
graceMs: stopGraceMs,
reason: 'grace-period-expired',
});
child.kill('SIGKILL');
// issue #9 (P2): child.kill() cannot reach past a .cmd shim to
// the real claude.exe or its grandchilds — the orphan that held
// a session file open for two hours was exactly that survivor.
void killProcessTree(child.pid, 'stop-grace-expired');
}
resolve();
}, stopGraceMs);
Expand All @@ -275,6 +280,11 @@ export class ClaudeAdapter implements AgentAdapter {
});
});
},
async forceKillTree() {
// Works even when the child already exited: the event stream can
// still be wedged by a grandchild holding the stdout write handle.
await killProcessTree(child.pid, 'force-kill');
},
waitForExit(timeoutMs: number): Promise<boolean> {
if (child.exitCode !== null || child.signalCode !== null) {
return Promise.resolve(true);
Expand Down Expand Up @@ -314,13 +324,30 @@ async function* createEventStream(

const rl = createInterface({ input: child.stdout, crlfDelay: Infinity });
let sawStdout = false;
let silentExitTimer: ReturnType<typeof setTimeout> | undefined;
const closeSilentStdout = (): void => {
silentExitTimer = setTimeout(() => {
if (!sawStdout && !child.stdout.readableEnded) child.stdout.destroy();
}, 50);
// issue #9: did the CLI emit its terminal `result` event? An exit without
// one means the run never finished — the caller must get an error, not
// silence.
let sawResultEvent = false;
// issue #9 (P0): stdout EOF is not a reliable end-of-stream. claude.exe can
// exit before emitting any terminal event while a grandchild that inherited
// the stdout write handle keeps it open — the for-await below then hangs
// forever (the 2026-11-08 two-hour freeze). When the child exits, give any
// final lines 2s to flush, then destroy stdout so readline ends regardless
// of who still holds the write handle.
const EXIT_FLUSH_GRACE_MS = 2_000;
const closeStdoutAfterExit = (sawStdout: boolean): void => {
setTimeout(() => {
if (!child.stdout.readableEnded) {
log.info('agent', 'stdout-force-close-after-exit', {
pid: child.pid ?? null,
sawStdout,
graceMs: EXIT_FLUSH_GRACE_MS,
});
child.stdout.destroy();
}
}, EXIT_FLUSH_GRACE_MS).unref();
};
child.once('exit', closeSilentStdout);
child.once('exit', () => closeStdoutAfterExit(sawStdout));
try {
for await (const line of rl) {
sawStdout = true;
Expand All @@ -333,10 +360,9 @@ async function* createEventStream(
continue;
}
yield* translateEvent(parsed);
if (parsedType(parsed) === 'result') sawResultEvent = true;
}
} finally {
if (silentExitTimer) clearTimeout(silentExitTimer);
child.removeListener('exit', closeSilentStdout);
rl.close();
}

Expand Down Expand Up @@ -376,6 +402,15 @@ async function* createEventStream(
message: `claude runtime error: ${runtimeError.message}`,
terminationReason: 'failed',
};
} else if (!sawResultEvent) {
// issue #9: a claude.exe that exits 0 without ever emitting its `result`
// event used to end the stream silently — no terminal for the caller, the
// run state stayed "running" and the slot froze. Synthesize the error.
yield {
type: 'error',
message: 'claude exited without emitting a result event',
terminationReason: 'failed',
};
}
}

Expand Down Expand Up @@ -409,6 +444,10 @@ function writeSystemPromptFile(
};
}

function parsedType(parsed: unknown): string | undefined {
return typeof parsed === 'object' && parsed !== null ? (parsed as { type?: string }).type : undefined;
}

function isWindowsCommandNotFoundLine(line: string): boolean {
return (
process.platform === 'win32' &&
Expand Down
37 changes: 28 additions & 9 deletions src/agent/codex/adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { join } from 'node:path';
import type { SandboxMode } from '../../config/profile-schema';
import { log } from '../../core/logger';
import { buildAgentEnv } from '../../platform/agent-env';
import { killProcessTree } from '../../platform/kill-tree';
import { spawnProcess, type SpawnedProcessByStdio } from '../../platform/spawn';
import { SpawnFailed } from '../../runtime/errors';
import { prefixBridgeSystemPrompt } from '../bridge-system-prompt';
Expand Down Expand Up @@ -183,6 +184,7 @@ export class CodexAdapter implements AgentAdapter {

return {
runId: opts.runId,
pid: child.pid,
events: createEventStream(child, stderrChunks, () => runtimeError, () => stopReason, codexHomeDir),
async stop() {
if (child.exitCode !== null || child.signalCode !== null) return;
Expand All @@ -197,7 +199,10 @@ export class CodexAdapter implements AgentAdapter {
graceMs: stopGraceMs,
reason: 'grace-period-expired',
});
child.kill('SIGKILL');
// issue #9 (P2): child.kill() cannot reach past a .cmd shim to
// the real codex.exe or its grandchilds — same orphan failure
// mode as the claude adapter.
void killProcessTree(child.pid, 'stop-grace-expired');
}
resolve();
}, stopGraceMs);
Expand All @@ -207,6 +212,11 @@ export class CodexAdapter implements AgentAdapter {
});
});
},
async forceKillTree() {
// Works even when the child already exited: the event stream can
// still be wedged by a grandchild holding the stdout write handle.
await killProcessTree(child.pid, 'force-kill');
},
waitForExit(timeoutMs: number): Promise<boolean> {
if (child.exitCode !== null || child.signalCode !== null) {
return Promise.resolve(true);
Expand Down Expand Up @@ -250,13 +260,24 @@ async function* createEventStream(
let lastUsageFetchAt = 0;
let threadId: string | undefined;
let lastKnownUsage: { inputTokens?: number; outputTokens?: number; cachedInputTokens?: number } | undefined;
let silentExitTimer: ReturnType<typeof setTimeout> | undefined;
const closeSilentStdout = (): void => {
silentExitTimer = setTimeout(() => {
if (!sawStdout && !child.stdout.readableEnded) child.stdout.destroy();
}, 50);
// issue #9 (P0): stdout EOF is not a reliable end-of-stream — same failure
// mode as the claude adapter. When the child exits, give any final lines 2s
// to flush, then destroy stdout so readline ends regardless of who still
// holds the write handle (a grandchild can keep it open indefinitely).
const EXIT_FLUSH_GRACE_MS = 2_000;
const closeStdoutAfterExit = (sawStdout: boolean): void => {
setTimeout(() => {
if (!child.stdout.readableEnded) {
log.info('agent', 'stdout-force-close-after-exit', {
pid: child.pid ?? null,
sawStdout,
graceMs: EXIT_FLUSH_GRACE_MS,
});
child.stdout.destroy();
}
}, EXIT_FLUSH_GRACE_MS).unref();
};
child.once('exit', closeSilentStdout);
child.once('exit', () => closeStdoutAfterExit(sawStdout));
try {
for await (const line of rl) {
sawStdout = true;
Expand Down Expand Up @@ -306,8 +327,6 @@ async function* createEventStream(
yield* translator.translate(parsed);
}
} finally {
if (silentExitTimer) clearTimeout(silentExitTimer);
child.removeListener('exit', closeSilentStdout);
rl.close();
// aiduCAT: final guaranteed refresh — run-state keeps whichever usage
// snapshot arrives last before/carrying the terminal event.
Expand Down
7 changes: 7 additions & 0 deletions src/agent/hermes/adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { createInterface } from 'node:readline';
import type { Readable, Writable } from 'node:stream';
import { log } from '../../core/logger';
import { buildAgentEnv } from '../../platform/agent-env';
import { killProcessTree } from '../../platform/kill-tree';
import { spawnProcess, type SpawnedProcessByStdio } from '../../platform/spawn';
import { SpawnFailed } from '../../runtime/errors';
import { prefixBridgeSystemPrompt } from '../bridge-system-prompt';
Expand Down Expand Up @@ -161,6 +162,9 @@ export class HermesAdapter implements AgentAdapter {

return {
runId: opts.runId,
get pid() {
return current?.pid;
},
events,
async stop() {
stopped = true;
Expand Down Expand Up @@ -198,6 +202,9 @@ export class HermesAdapter implements AgentAdapter {
child.once('exit', onExit);
});
},
async forceKillTree() {
await killProcessTree(current?.pid, 'force-kill');
},
};
}
}
Expand Down
10 changes: 10 additions & 0 deletions src/agent/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,8 @@ export interface AgentRunOptions {

export interface AgentRun {
readonly runId: string;
/** OS pid of the agent process, once spawned; undefined until/unless it is. */
readonly pid: number | undefined;
readonly events: AsyncIterable<AgentEvent>;
stop(): Promise<void>;
/**
Expand All @@ -92,6 +94,14 @@ export interface AgentRun {
* 143 instead of 0; waiting it out lets it exit cleanly.
*/
waitForExit(timeoutMs: number): Promise<boolean>;
/**
* issue #9 (P1): the watchdog's last-resort exit. Kills the agent process
* tree (Windows: `taskkill /T /F`, so grandchilds holding the stdout write
* handle die too) and must not throw even if the process is already gone.
* Unlike stop() this works after the child exited — the event stream can
* still be wedged on a grandchild-inherited stdout handle.
*/
forceKillTree(): Promise<void>;
}

/**
Expand Down
36 changes: 32 additions & 4 deletions src/bot/channel.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1611,7 +1611,7 @@ async function runAgentBatch(deps: RunBatchDeps): Promise<void> {
recordSession,
async () => {},
sessions,
capability.agentId);
capability.agentId, execution.forceClose);
await cotDone;
if (cotPublisher.degradedReason) {
await sendCotDegradedNotice({
Expand Down Expand Up @@ -1724,7 +1724,7 @@ async function runAgentBatch(deps: RunBatchDeps): Promise<void> {
},
sessions,
// (diagnostics)
capability.agentId);
capability.agentId, execution.forceClose);
// f0.3.4: claude runs open the card right away — the initial render is
// the "☤Thinking…" placeholder with the model logo in the panel header —
// instead of waiting for the first tool call / text (10–200s). Later
Expand Down Expand Up @@ -1851,7 +1851,7 @@ async function runAgentBatch(deps: RunBatchDeps): Promise<void> {
}
},
sessions,
capability.agentId);
capability.agentId, execution.forceClose);
let mdDelivered = false;
try {
await awaitRenderAwareStream({
Expand Down Expand Up @@ -1907,7 +1907,7 @@ async function runAgentBatch(deps: RunBatchDeps): Promise<void> {
recordSession,
async () => {},
sessions,
capability.agentId);
capability.agentId, execution.forceClose);
await sendFinalReply({
channel,
chatId,
Expand Down Expand Up @@ -2201,12 +2201,19 @@ async function processAgentStream(
flush: (state: RunState) => Promise<void>,
stats: Pick<SessionStore, 'recordUsage' | 'recordMessage' | 'recordModel' | 'recordTurnTps'>,
agentId: string,
// issue #9 (P1): the executor's hard exit, used when stop() cannot end the
// run (child already dead, event stream wedged on a grandchild stdout
// handle). Undefined in callers without an execution (tests).
forceClose: (reason: string) => Promise<void> | undefined,
): Promise<RunState> {
const runStart = Date.now();
let state: RunState = initialState;
// f0.3.5: this turn's TPS, timed here (model generation only, tool windows
// removed) — never from the CLI's session-cumulative duration fields.
const tpsMeter = new TurnTpsMeter();
// issue #9 (P1): true once this function reached a terminal state — the
// watchdog's delayed force-close must not fire after a normal end.
let settled = false;

// Idle watchdog: claude going silent for `idleTimeoutMs` is treated as
// "presumed hung", we stop() and surface a timeout marker on the card.
Expand Down Expand Up @@ -2235,6 +2242,26 @@ async function processAgentStream(
void handle.run.stop().catch(() => {
/* stop errors are non-fatal */
});
// issue #9 (P1): stop() is a no-op once the child already exited, and
// then nothing ever ends the event stream — the slot froze for two
// hours on 2026-11-08. Give the stream a short window to end on its
// own; if this function is still running (no terminal state reached),
// force-close: synthetic terminal + process-tree kill.
const FORCE_CLOSE_AFTER_MS = 10_000;
setTimeout(() => {
if (settled) return;
log.fail(
'agent',
new Error(`watchdog ${fire.kind} fire did not end the run`),
{ scope, action: 'force-close', afterMs: FORCE_CLOSE_AFTER_MS },
);
const close = forceClose(`watchdog-${fire.kind}-timeout`);
if (close) {
close.catch((err) => {
log.fail('agent', err, { scope, event: 'watchdog-force-close-failed' });
});
}
}, FORCE_CLOSE_AFTER_MS).unref();
},
});
watchdog.start();
Expand Down Expand Up @@ -2395,6 +2422,7 @@ async function processAgentStream(
log.info('card', 'final', { scope, terminal: state.terminal, interrupted: handle.interrupted });
reportMetric('run_e2e_ms', Date.now() - runStart, { terminal: state.terminal });
await flush(state);
settled = true;
if (handle.interrupted) {
await handle.run.stop();
}
Expand Down
49 changes: 49 additions & 0 deletions src/platform/kill-tree.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import { log } from '../core/logger';
import { spawnProcess } from './spawn';

/**
* issue #9 (P2): kill an agent process AND everything it spawned.
*
* On Windows, `child.kill()` only terminates the direct child — claude/codex
* resolve to a `.cmd` shim routed through `cmd.exe`, and the real CLI process
* (plus any grandchild that inherited the stdout write handle) survives. That
* is exactly how the 2026-11-08 orphan claude.exe hung around for two hours
* holding a session file. `taskkill /T /F` walks the whole tree instead.
*
* POSIX keeps the plain SIGKILL: killing a process group would hit our own
* group (the child is not spawned detached), and the bug this fixes is
* Windows-specific.
*
* Best-effort: a pid that already exited makes taskkill fail harmlessly, so
* this never throws.
*/
export async function killProcessTree(pid: number | undefined, why: string): Promise<void> {
if (pid === undefined) return;
if (process.platform !== 'win32') {
try {
process.kill(pid, 'SIGKILL');
} catch {
// already dead / already reaped — nothing to do
}
return;
}
await new Promise<void>((resolve) => {
// windowsHide: the daemon runs headless; a taskkill console flash on the
// desktop would be alarming noise.
const killer = spawnProcess('taskkill', ['/PID', String(pid), '/T', '/F'], {
stdio: 'ignore',
windowsHide: true,
});
const done = (): void => resolve();
// taskkill exits non-zero when the pid is already gone — that is success
// for us. Only a spawn failure (taskkill missing) is worth a log line.
killer.on('exit', done);
killer.on('error', () => {
log.warn('run', 'kill-tree-spawn-failed', { pid, why });
done();
});
// Belt and braces: taskkill finishes in milliseconds; never hang a caller
// on it no matter what.
setTimeout(done, 5_000).unref();
});
}
Loading
Loading