diff --git a/.changeset/event-runtime-standby.md b/.changeset/event-runtime-standby.md new file mode 100644 index 000000000..8d5ec69de --- /dev/null +++ b/.changeset/event-runtime-standby.md @@ -0,0 +1,5 @@ +--- +"agent-bundle": patch +--- + +Keep a generated MCP server running when another process from the same install already owns the event runtime socket: the server stands by and takes the socket over when the owner exits, instead of exiting with `Event runtime endpoint already has a live server`. `createEventRuntimeServer` gains `whenOwned: 'fail' | 'standby'` (default `'fail'`), and the returned server exposes `role()` and `onRoleChange()`; the standby start and the takeover are announced on stderr only (#561) diff --git a/packages/agent-bundle/src/events/ipc.ts b/packages/agent-bundle/src/events/ipc.ts index d05dcde5c..09f3fb1b2 100644 --- a/packages/agent-bundle/src/events/ipc.ts +++ b/packages/agent-bundle/src/events/ipc.ts @@ -4,7 +4,7 @@ import { createConnection, createServer, type Server, type Socket } from 'node:n import { dirname, join } from 'node:path'; import { StringDecoder } from 'node:string_decoder'; -import { Context, Duration, Effect, Exit, Layer } from 'effect'; +import { Context, Duration, Effect, Exit, Fiber, Layer, Random, Ref, type Scope } from 'effect'; import { z } from 'zod'; import { isAbortError, makeScopedEffectRuntime, runPromise, type ScopedEffectRuntime } from '../effect/boundary.ts'; @@ -14,6 +14,14 @@ const EVENT_RUNTIME_PROTOCOL_VERSION = 1 as const; const MAX_EVENT_MESSAGE_BYTES = 1024 * 1024; const ENDPOINT_CLAIM_RETRY_COUNT = 100; const ENDPOINT_CLAIM_RETRY_DELAY = Duration.millis(10); +/** + * How often a standby server asks whether the owner is still there. The + * interval bounds the hook-visible gap after an owner exits (hooks see + * `runtime-unavailable` until the takeover lands); the jitter keeps several + * standbys from probing — and claiming — in lockstep. + */ +const STANDBY_PROBE_INTERVAL = Duration.seconds(1); +const STANDBY_PROBE_JITTER_MS = 250; export type EventRuntimeTransportErrorCode = | 'epoch-mismatch' @@ -32,6 +40,25 @@ export class EventRuntimeTransportError extends Error { } } +/** + * The one `runtime-failed` outcome a standby server recovers from: a live + * server answered on the endpoint. It is the same `EventRuntimeTransportError` + * (code, message, and name) the fail-closed path has always thrown; the + * subclass exists only so `whenOwned: 'standby'` can tell "owned elsewhere" + * from a genuinely broken endpoint, which still fails. + */ +class EventRuntimeEndpointOwnedError extends EventRuntimeTransportError { + constructor() { + super('runtime-failed', 'Event runtime endpoint already has a live server.'); + } +} + +const endpointOwned = (): Effect.Effect => + Effect.fail(new EventRuntimeEndpointOwnedError()); + +const isEndpointOwned = (error: EventRuntimeTransportError): error is EventRuntimeEndpointOwnedError => + error instanceof EventRuntimeEndpointOwnedError; + const eventRequestSchema = z.object({ artifactEpoch: z.string().min(1), event: z.string().min(1), @@ -104,16 +131,52 @@ export interface EventRuntimeStatus { readonly startedAt?: string; } +/** + * `owner` binds the endpoint and answers hooks; `standby` binds nothing and + * waits for the owner to exit. A server only ever moves from `standby` to + * `owner`, never back. + */ +export type EventRuntimeServerRole = 'owner' | 'standby'; + +/** + * What `createEventRuntimeServer` does when a live server already owns the + * endpoint. `fail` (the default) rejects with `runtime-failed` + * (`Event runtime endpoint already has a live server.`); `standby` resolves a + * server in the `standby` role that takes the endpoint over once the owner + * exits. Every other startup failure fails closed under both policies. + */ +export type EventRuntimeOwnedEndpointPolicy = 'fail' | 'standby'; + export interface CreateEventRuntimeServerOptions { readonly artifactEpoch: string; readonly endpointId: string; readonly handle: (request: EventRuntimeRequest, signal: AbortSignal) => Promise; + /** + * Receives every failure the standby role recovers from by retrying, other + * than "still owned": a takeover attempt that failed for any other reason + * (a probe, claim, stale-path removal, listen, or release failure) and a + * role listener that threw (`Event runtime role listener failed.`). The + * standby keeps probing regardless — including when this callback itself + * throws, which is swallowed; without this callback those failures are + * invisible, and the process looks healthy while shared-only hooks stay + * unavailable. + */ + readonly onStandbyError?: (error: EventRuntimeTransportError) => void; readonly status?: () => EventRuntimeStatus; + readonly whenOwned?: EventRuntimeOwnedEndpointPolicy; } export interface EventRuntimeServer { readonly close: () => Promise; readonly endpoint: string; + /** + * Observes role changes — today only `standby` → `owner`, when this server + * takes the endpoint over. Returns the unsubscribe. Listeners run on the + * takeover's own turn, after the socket is bound and before any hook is + * answered through it. + */ + readonly onRoleChange: (listener: (role: EventRuntimeServerRole) => void) => () => void; + readonly role: () => EventRuntimeServerRole; } export interface RequestEventRuntimeOptions extends EventRuntimeRequest { @@ -287,14 +350,28 @@ const handleConnection = Effect.fnUntraced(function*( }); }); -interface EventSocketServiceShape { +/** The bound endpoint: what an owner holds and a standby is waiting to become. */ +interface EventSocketListener { readonly endpoint: string; readonly endpointIdentity?: Readonly<{ readonly device: number; readonly inode: number }>; readonly server: Server; readonly sockets: Set; } -class EventSocketService extends Context.Service()( +/** + * One server's lifetime state. `listener` is set exactly once — at open for + * an owner, at takeover for a standby — and only ever by the fiber that bound + * it; `standby` is the takeover loop, present only for a server that started + * behind a live owner, and interrupted by the scope or by `close()`. + */ +interface EventRuntimeServerState { + readonly endpoint: string; + readonly listener: Ref.Ref; + readonly roleListeners: Set<(role: EventRuntimeServerRole) => void>; + readonly standby: Fiber.Fiber | undefined; +} + +class EventSocketService extends Context.Service()( 'agent-bundle/events/EventSocketService', ) {} @@ -310,7 +387,11 @@ export interface EventRuntimeServerTestHooks { readonly beforeEndpointClaimRemoval?: () => Promise; /** Runs after the claim handle closed and before its lock file is removed; a rejection stands in for a failing release. */ readonly beforeEndpointClaimRelease?: () => Promise; - /** Runs at the start of the owned-endpoint removal on close; a rejection stands in for a failing removal. */ + /** + * Runs at the start of the owned-endpoint removal on close — the server has + * stopped listening and still holds the claim; a rejection stands in for a + * failing removal. + */ readonly beforeEndpointRemoval?: () => Promise; } @@ -651,12 +732,7 @@ const claimEndpoint = Effect.fnUntraced(function*( } const endpointState = yield* probeEndpoint(endpoint); - if (endpointState === 'live') { - return yield* Effect.fail(transportError( - 'runtime-failed', - 'Event runtime endpoint already has a live server.', - )); - } + if (endpointState === 'live') return yield* endpointOwned(); if (yield* reclaimOrphanedEndpointClaim(endpoint, testHooks)) continue; if (attempt + 1 < ENDPOINT_CLAIM_RETRY_COUNT) { yield* Effect.sleep(ENDPOINT_CLAIM_RETRY_DELAY); @@ -673,8 +749,8 @@ const claimEndpoint = Effect.fnUntraced(function*( const listenServer = ( options: CreateEventRuntimeServerOptions, endpoint: string, -): Effect.Effect => - Effect.callback((resume) => { +): Effect.Effect => + Effect.callback((resume) => { const sockets = new Set(); const server = createServer({ allowHalfOpen: true }, (socket) => { sockets.add(socket); @@ -694,7 +770,13 @@ const listenServer = ( socket.removeListener('error', interrupt); }); }); - const onError = (error: Error): void => { + const onError = (error: NodeJS.ErrnoException): void => { + // The path (or, on Windows, the pipe name — the only owner check that + // platform has) is bound by a server this process did not probe: owned. + if (error.code === 'EADDRINUSE') { + resume(endpointOwned()); + return; + } resume(Effect.fail(transportError('runtime-failed', 'Unable to listen on the event runtime endpoint.', error))); }; server.once('error', onError); @@ -723,10 +805,17 @@ const listenServer = ( }); }); +/** + * Probes, claims, and binds the endpoint — the whole owner path. A live + * server on the endpoint fails with the `EventRuntimeEndpointOwnedError` + * marker at every site that can see one (before the claim, inside the claim + * loop, and under the claim), so a standby retries exactly that and nothing + * else. + */ const openServer = Effect.fnUntraced(function*( options: CreateEventRuntimeServerOptions, testHooks?: EventRuntimeServerTestHooks, -): Effect.fn.Return { +): Effect.fn.Return { const endpoint = eventRuntimeEndpoint(options.endpointId); if (process.platform === 'win32') return yield* listenServer(options, endpoint); @@ -743,21 +832,11 @@ const openServer = Effect.fnUntraced(function*( Effect.mapError((error) => transportError('runtime-failed', 'Event runtime endpoint probe hook failed.', error)), ); } - if (endpointState === 'live') { - return yield* Effect.fail(transportError( - 'runtime-failed', - 'Event runtime endpoint already has a live server.', - )); - } + if (endpointState === 'live') return yield* endpointOwned(); const listenUnderClaim = Effect.gen(function*() { const claimedEndpointState = yield* probeEndpoint(endpoint); - if (claimedEndpointState === 'live') { - return yield* Effect.fail(transportError( - 'runtime-failed', - 'Event runtime endpoint already has a live server.', - )); - } + if (claimedEndpointState === 'live') return yield* endpointOwned(); if (claimedEndpointState === 'stale') { yield* liftPromise(() => rm(endpoint)).pipe( Effect.mapError((error) => transportError( @@ -769,13 +848,14 @@ const openServer = Effect.fnUntraced(function*( } return yield* listenServer(options, endpoint); }); - // The claim is held only across the probe and listen. Its release must - // *propagate* a failure, so this is an explicit exit sequence rather than a - // scope finalizer: capture the listen `Exit`, release the claim, and let the - // release failure win — after shutting down a server that did come up, so a - // failed claim release never leaks a listening socket. The mask keeps the - // acquire→release pairing intact under interruption, as `acquireUseRelease` - // would. + // The claim is held only across the probe and listen (and, on close, across + // the shutdown and identity-checked removal — `releaseOwnedEndpoint`). Its + // release must *propagate* a failure, so this is an explicit exit sequence + // rather than a scope finalizer: capture the listen `Exit`, release the + // claim, and let the release failure win — after shutting down a server that + // did come up, so a failed claim release never leaks a listening socket. The + // mask keeps the acquire→release pairing intact under interruption, as + // `acquireUseRelease` would. return yield* Effect.uninterruptibleMask((restore) => Effect.gen(function*() { const claim = yield* claimEndpoint(endpoint, testHooks); const listened = yield* Effect.exit(restore(listenUnderClaim)); @@ -788,82 +868,225 @@ const openServer = Effect.fnUntraced(function*( })); }); +/** Destroys every accepted socket and stops listening. Idempotent and infallible: the scope finalizer. */ +const shutdownServer = (listener: EventSocketListener): Effect.Effect => + Effect.callback((resume) => { + for (const socket of listener.sockets) socket.destroy(); + if (!listener.server.listening) { + resume(Effect.void); + return undefined; + } + listener.server.close(() => resume(Effect.void)); + return undefined; + }); + /** - * Removes the socket path only while it is still the inode this server bound - * (identity-matched `stat` + `rm`, kept raw like the claim lock). A removal - * failure is a typed transport error so `close()` can surface it instead of - * leaving a stale endpoint behind silently; a missing path is already gone. + * Stops listening and removes the socket path, holding the endpoint claim + * from before the first step until after the last. Closing the server already + * unlinks the path (libuv removes a listening pipe's name on close); the + * identity-matched `stat` + `rm` that follows, kept raw like the claim lock, + * only ever removes a path that still carries the inode this server bound. The + * claim is what makes that check trustworthy: a standby's takeover holds the + * same claim across its probe → unlink → listen, so no successor can bind a + * new socket at this path between the unlink and the identity read — and the + * inode the kernel just freed is exactly the one a new socket is likely to be + * handed, which would let the check match the successor's live socket and + * remove it. Standbys only claim an endpoint that is not live, so the claim is + * uncontended while this server listens; a claim that fails anyway is a + * foreign lock, and the server is shut down regardless before the failure + * propagates. A removal or release failure is a typed transport error so + * `close()` can surface it instead of leaving a stale endpoint behind + * silently; a missing path is already gone. */ -const removeOwnedEndpoint = ( - service: EventSocketServiceShape, +const releaseOwnedEndpoint = Effect.fnUntraced(function*( + listener: EventSocketListener, + identity: Readonly<{ readonly device: number; readonly inode: number }>, testHooks?: EventRuntimeServerTestHooks, -): Effect.Effect => - liftPromise(async () => { - if (service.endpointIdentity === undefined) return; +): Effect.fn.Return { + const removeIfStillOurs = liftPromise(async () => { await testHooks?.beforeEndpointRemoval?.(); let current; try { - current = await stat(service.endpoint); + current = await stat(listener.endpoint); } catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; throw error; } - if ( - current.dev === service.endpointIdentity.device - && current.ino === service.endpointIdentity.inode - ) { - await rm(service.endpoint, { force: true }); - } + if (current.dev !== identity.device || current.ino !== identity.inode) return; + await rm(listener.endpoint, { force: true }); }).pipe( Effect.mapError((error) => transportError('runtime-failed', 'Unable to remove the event runtime endpoint.', error)), ); - -/** Destroys every accepted socket and stops listening. Idempotent and infallible: the scope finalizer. */ -const shutdownServer = (service: EventSocketServiceShape): Effect.Effect => - Effect.callback((resume) => { - for (const socket of service.sockets) socket.destroy(); - if (!service.server.listening) { - resume(Effect.void); - return undefined; + // The same claim discipline as `openServer`: an explicit exit sequence so a + // release failure propagates (and wins), under a mask that keeps the + // acquire→release pairing intact if `close()` is interrupted. + return yield* Effect.uninterruptibleMask((restore) => Effect.gen(function*() { + const claimed = yield* Effect.exit(claimEndpoint(listener.endpoint, testHooks).pipe( + Effect.mapError((error) => isEndpointOwned(error) + ? transportError('runtime-failed', 'Event runtime endpoint claim is held by another process during teardown.', error) + : error), + )); + if (Exit.isFailure(claimed)) { + yield* shutdownServer(listener); + return yield* Effect.failCause(claimed.cause); } - service.server.close(() => resume(Effect.void)); - return undefined; - }); + yield* shutdownServer(listener); + const removed = yield* Effect.exit(restore(removeIfStillOurs)); + const released = yield* Effect.exit(releaseEndpointClaim(claimed.value, testHooks)); + if (Exit.isFailure(released)) return yield* Effect.failCause(released.cause); + return yield* removed; + })); +}); /** - * The explicit teardown `EventRuntimeServer.close()` runs: shut the server - * down, then remove the owned endpoint. The removal is fallible on purpose — - * the scope finalizer only ever repeats the (idempotent) shutdown, so the one - * teardown step that can fail reports to the caller rather than to a + * The explicit teardown `EventRuntimeServer.close()` runs: stop the standby + * loop, then shut the bound server down and remove the owned endpoint under + * the claim. The loop is interrupted — and awaited, so an in-flight takeover + * finishes its claim→listen→release before this reads the listener — first, + * or a takeover could bind a socket nobody removes. The removal is fallible on + * purpose: the scope finalizer only ever repeats the (idempotent) shutdown, so + * the one teardown step that can fail reports to the caller rather than to a * finalizer that would have to swallow it. */ -const closeServer = ( - service: EventSocketServiceShape, +const closeServer = Effect.fnUntraced(function*( + state: EventRuntimeServerState, testHooks?: EventRuntimeServerTestHooks, -): Effect.Effect => - shutdownServer(service).pipe( - Effect.andThen(process.platform === 'win32' ? Effect.void : removeOwnedEndpoint(service, testHooks)), - ); +): Effect.fn.Return { + if (state.standby !== undefined) yield* Fiber.interrupt(state.standby); + const listener = yield* Ref.get(state.listener); + if (listener === undefined) return; + // Windows binds a named pipe, not a path: nothing to claim or remove. + if (process.platform === 'win32' || listener.endpointIdentity === undefined) { + yield* shutdownServer(listener); + return; + } + yield* releaseOwnedEndpoint(listener, listener.endpointIdentity, testHooks); +}); + +type StandbyErrorReporter = (error: EventRuntimeTransportError) => void; + +/** + * The one way the standby path ever calls `onStandbyError`. The reporter is + * the caller's last line of diagnostics, so a reporter that itself throws has + * nowhere safe left to report to and is swallowed: it must never end the + * standby fiber (which would leave the process looking healthy and never + * taking over) or stop the role listeners after it. + */ +const guardedStandbyErrorReporter = (onStandbyError: StandbyErrorReporter | undefined): StandbyErrorReporter => + (error) => { + try { + onStandbyError?.(error); + } catch { + // Nothing safe is left to report to. + } + }; + +/** + * Delivers the one `standby` → `owner` notification to every listener. Each + * runs in its own guard: a listener that throws is reported through the + * (guarded) reporter and neither stops the listeners after it nor the role + * transition, which has already happened. + */ +const notifyRoleListeners = ( + roleListeners: ReadonlySet<(role: EventRuntimeServerRole) => void>, + role: EventRuntimeServerRole, + report: StandbyErrorReporter, +): Effect.Effect => + Effect.sync(() => { + for (const notify of roleListeners) { + try { + notify(role); + } catch (error) { + report(transportError('runtime-failed', 'Event runtime role listener failed.', error)); + } + } + }); + +/** + * The standby role's one job: notice the owner leaving and run the same + * `openServer` path an unowned start runs. Losing that race — another standby + * claimed first, or the endpoint is live again by the time this probe lands — + * is the expected outcome and leaves it standing by for the next tick in + * silence. Every other typed failure also leaves it standing by, because the + * MCP server this runtime serves is already up and there is no caller left to + * fail, but is reported through `onStandbyError` so a takeover that can never + * succeed (an endpoint directory it cannot write, a claim that never clears) + * does not look like a healthy wait. Bound to the scope: `close()` interrupts + * it before reading the listener, and the layer scope interrupts it on + * dispose. + */ +const standByForEndpoint = Effect.fnUntraced(function*( + endpoint: string, + acquire: Effect.Effect, + roleListeners: ReadonlySet<(role: EventRuntimeServerRole) => void>, + report: StandbyErrorReporter, +): Effect.fn.Return { + const reportAndStandBy = (error: EventRuntimeTransportError): Effect.Effect => Effect.sync(() => { + report(error); + return undefined; + }); + while (true) { + const jitter = yield* Random.nextIntBetween(0, STANDBY_PROBE_JITTER_MS); + yield* Effect.sleep(Duration.sum(STANDBY_PROBE_INTERVAL, Duration.millis(jitter))); + const endpointState = yield* probeEndpoint(endpoint).pipe(Effect.catch(reportAndStandBy)); + if (endpointState === undefined || endpointState === 'live') continue; + const listener = yield* acquire.pipe( + Effect.catchIf(isEndpointOwned, () => Effect.succeed(undefined)), + Effect.catch(reportAndStandBy), + ); + if (listener === undefined) continue; + yield* notifyRoleListeners(roleListeners, 'owner', report); + return; + } +}); const eventSocketLayer = ( options: CreateEventRuntimeServerOptions, testHooks?: EventRuntimeServerTestHooks, ): Layer.Layer => - Layer.effect(EventSocketService, Effect.acquireRelease(openServer(options, testHooks), shutdownServer)); + Layer.effect(EventSocketService, Effect.gen(function*() { + const endpoint = eventRuntimeEndpoint(options.endpointId); + const listener = yield* Ref.make(undefined); + const roleListeners = new Set<(role: EventRuntimeServerRole) => void>(); + // Binding and recording the listener are one uninterruptible acquire, so + // a `close()` that interrupts a takeover mid-flight still finds — and + // removes — the socket that takeover had just bound. + const acquire = Effect.acquireRelease( + openServer(options, testHooks).pipe(Effect.tap((bound) => Ref.set(listener, bound))), + shutdownServer, + ); + if (options.whenOwned !== 'standby') { + yield* acquire; + return { endpoint, listener, roleListeners, standby: undefined }; + } + const opened = yield* acquire.pipe(Effect.catchIf(isEndpointOwned, () => Effect.succeed(undefined))); + if (opened !== undefined) return { endpoint, listener, roleListeners, standby: undefined }; + const standby = yield* Effect.forkScoped( + standByForEndpoint(endpoint, acquire, roleListeners, guardedStandbyErrorReporter(options.onStandbyError)), + ); + return { endpoint, listener, roleListeners, standby }; + })); const createEventRuntimeServerWithHooks = async ( options: CreateEventRuntimeServerOptions, testHooks?: EventRuntimeServerTestHooks, ): Promise => { const runtime: ScopedEffectRuntime = makeScopedEffectRuntime(eventSocketLayer(options, testHooks)); - const service = await runtime.run(EventSocketService); + const state = await runtime.run(EventSocketService); let closing: Promise | undefined; return Object.freeze({ close: () => { - closing ??= runtime.run(closeServer(service, testHooks)).finally(() => runtime.close()); + closing ??= runtime.run(closeServer(state, testHooks)).finally(() => runtime.close()); return closing; }, - endpoint: service.endpoint, + endpoint: state.endpoint, + onRoleChange: (listener: (role: EventRuntimeServerRole) => void) => { + state.roleListeners.add(listener); + return () => { + state.roleListeners.delete(listener); + }; + }, + role: () => (Ref.getUnsafe(state.listener) === undefined ? 'standby' : 'owner'), }); }; diff --git a/packages/agent-bundle/src/mcp-server-runtime.ts b/packages/agent-bundle/src/mcp-server-runtime.ts index 28dcaab93..46fc32f4c 100644 --- a/packages/agent-bundle/src/mcp-server-runtime.ts +++ b/packages/agent-bundle/src/mcp-server-runtime.ts @@ -29,7 +29,7 @@ import { runAgentRequest, unavailable, } from '@agent-bundle/runtime'; -import type { createEventRuntimeServer } from './events/ipc.ts'; +import type { createEventRuntimeServer, EventRuntimeTransportError } from './events/ipc.ts'; import type { createCanonicalEventProps, projectEventDocument } from './events/project.ts'; import { createTaskAugmentedMcpServer, type TaskAugmentedMcpServer } from './mcp-tasks.ts'; import { canonicalAgentEvents, type CanonicalAgentEvent } from './routes/public.ts'; @@ -731,6 +731,35 @@ const noticeDiagnostic = (line: string): void => { const describeError = (error: unknown): string => (error instanceof Error ? error.message : String(error)); +/** stderr only: stdout is the protocol stream. */ +const eventRuntimeDiagnostic = (line: string): void => { + process.stderr.write(`agent-bundle event runtime: ${line}\n`); +}; + +/** How long one takeover-failure message stays reported before the same message earns another line. */ +const STANDBY_ERROR_REPEAT_INTERVAL_MS = 30_000; + +/** + * Turns the standby loop's recoverable failures into stderr lines without + * repeating one per probe tick: a message is reported when first seen and + * again once `STANDBY_ERROR_REPEAT_INTERVAL_MS` has passed since its last + * line, so a takeover that keeps failing the same way stays visible at one + * line per half minute while a new failure mode is reported at once. + */ +export const createStandbyErrorReporter = ( + report: (line: string) => void, + now: () => number = Date.now, +): ((error: EventRuntimeTransportError) => void) => { + const lastReportedAt = new Map(); + return (error) => { + const at = now(); + const previous = lastReportedAt.get(error.message); + if (previous !== undefined && at - previous < STANDBY_ERROR_REPEAT_INTERVAL_MS) return; + lastReportedAt.set(error.message, at); + report(`takeover failed, still standing by: ${error.message}`); + }; +}; + /** * Installs `resources/subscribe` / `resources/unsubscribe` for the notice * inbox URI and returns the post-render observation that emits @@ -878,6 +907,15 @@ const canonicalEvent = (event: string): CanonicalAgentEvent => { * The shared event runtime for one artifact: native envelopes arrive over the * IPC socket, render through the same dispatcher every route uses, and project * back into the host's own hook response shape. + * + * One process per install root owns the socket. Hosts routinely launch several + * sessions of the same plugin from one install (every Claude Code session + * Claude Desktop starts, Cursor importing a Claude-installed plugin), so a + * second server finding the socket owned is normal, not fatal: it starts in + * the `standby` role, serves its own MCP session, and takes the socket over + * when the owner exits. Both role changes, and any takeover failure other than + * "still owned", are announced on stderr only — the process's stdout is the + * MCP protocol stream. */ const startEventRuntime = async ( events: GeneratedEventRuntimeBinding, @@ -888,7 +926,10 @@ const startEventRuntime = async ( pluginRoot: Observed | undefined, ): Promise<{ readonly close: () => Promise }> => { const startedAt = new Date().toISOString(); - return events.createEventRuntimeServer({ + // The socket path is known once the server exists; the standby loop only + // reports after its first probe interval, by which time it is. + let endpointLabel = events.endpointId; + const server = await events.createEventRuntimeServer({ artifactEpoch: events.artifactEpoch, endpointId: events.endpointId, handle: async (request, signal) => settled(async () => { @@ -966,7 +1007,19 @@ const startEventRuntime = async ( pid: process.pid, startedAt, }), + onStandbyError: createStandbyErrorReporter((line) => { + eventRuntimeDiagnostic(`${endpointLabel} ${line}`); + }), + whenOwned: 'standby', }); + endpointLabel = server.endpoint; + if (server.role() === 'standby') { + eventRuntimeDiagnostic(`${server.endpoint} is owned by another process; standing by`); + server.onRoleChange((role) => { + if (role === 'owner') eventRuntimeDiagnostic(`${server.endpoint} was released by its owner; took it over`); + }); + } + return server; }; /** diff --git a/packages/agent-bundle/src/test/mcp.ts b/packages/agent-bundle/src/test/mcp.ts index e2086b60a..36c0a3219 100644 --- a/packages/agent-bundle/src/test/mcp.ts +++ b/packages/agent-bundle/src/test/mcp.ts @@ -551,7 +551,11 @@ export const openInMemoryMcpServer = async < allowedTargets: [options.lineageHost], artifactEpoch, createCanonicalEventProps: (() => { throw new Error('in-memory lineage sessions dispatch no events'); }) as never, - createEventRuntimeServer: (async () => ({ close: async () => undefined })) as never, + createEventRuntimeServer: (async () => ({ + close: async () => undefined, + onRoleChange: () => () => undefined, + role: () => 'owner', + })) as never, endpointId: `${artifactEpoch}:in-memory`, projectEventDocument: (() => undefined) as never, target: options.lineageHost, diff --git a/packages/agent-bundle/tests/event-ipc.test.ts b/packages/agent-bundle/tests/event-ipc.test.ts index 76caff7e7..7134072f7 100644 --- a/packages/agent-bundle/tests/event-ipc.test.ts +++ b/packages/agent-bundle/tests/event-ipc.test.ts @@ -5,7 +5,7 @@ import { createConnection, type Socket } from 'node:net'; import { dirname } from 'node:path'; import { setTimeout as delay } from 'node:timers/promises'; -import { Effect } from 'effect'; +import { Duration, Effect } from 'effect'; import { expect, it } from 'effect-rstest'; import { @@ -13,10 +13,12 @@ import { createEventRuntimeServerForTest, eventRuntimeEndpoint, type EventRuntimeServer, + type EventRuntimeServerRole, EventRuntimeTransportError, requestEventRuntime, requestEventRuntimeStatus, } from '../src/events/ipc.ts'; +import { deferred, eventually, within } from './support/eventually.ts'; type ContenderOutcome = | Readonly<{ readonly server: EventRuntimeServer; readonly status: 'opened' }> @@ -134,6 +136,27 @@ const writeEndpointClaim = async (endpointId: string, contents: string): Promise return claimPath; }; +/** One `session/start` hook request against whichever server owns `endpointId`. */ +const askRuntime = (endpointId: string): Effect.Effect => Effect.promise(() => requestEventRuntime({ + artifactEpoch: 'epoch-1', + endpointId, + event: 'session/start', + hostContractRevision: '2.1.250', + native: { hook_event_name: 'SessionStart' }, + signal: new AbortController().signal, + target: 'claude', + timeoutMs: 1_000, +})); + +/** Resolves with the next role the server reports; the standby loop settles within a probe interval plus jitter. */ +const nextRole = (server: EventRuntimeServer): Promise => within( + new Promise((resolve) => { server.onRoleChange(resolve); }), + 5_000, +); + +const expectMissing = (path: string): Effect.Effect => Effect.promise(() => + expect(stat(path)).rejects.toMatchObject({ code: 'ENOENT' })); + const expectBoundedClaimFailure = async (endpointId: string): Promise => { await expect(createEventRuntimeServer({ artifactEpoch: 'epoch-1', @@ -278,6 +301,371 @@ it.live('rejects a second live server without disturbing the endpoint owner', () })); })); +it.live('keeps rejecting a second live server when the caller asks for the fail policy explicitly', () => Effect.gen(function*() { + if (process.platform === 'win32') return; + const endpointId = `event-ipc-owner-explicit-fail-${crypto.randomUUID()}`; + yield* Effect.acquireRelease( + Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'first' }), + })), + (server) => Effect.promise(() => server.close()), + ); + const alreadyRunning = yield* Effect.tryPromise({ + try: () => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'second' }), + whenOwned: 'fail', + }), + catch: (error) => error, + }).pipe(Effect.flip); + expect(alreadyRunning).toMatchObject({ + code: 'runtime-failed', + message: 'Event runtime endpoint already has a live server.', + name: EventRuntimeTransportError.name, + }); + expect(yield* askRuntime(endpointId)).toEqual({ owner: 'first' }); +})); + +it.live('stands by behind a live owner and takes the endpoint over once the owner closes', () => Effect.gen(function*() { + if (process.platform === 'win32') return; + const endpointId = `event-ipc-standby-${crypto.randomUUID()}`; + const endpoint = eventRuntimeEndpoint(endpointId); + // A free endpoint under the standby policy opens as owner straight away. + const owner = yield* Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'first' }), + whenOwned: 'standby', + })); + expect(owner.role()).toBe('owner'); + + yield* Effect.scoped(Effect.gen(function*() { + const standby = yield* Effect.acquireRelease( + Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'second' }), + whenOwned: 'standby', + })), + (server) => Effect.promise(() => server.close()), + ); + const roles: EventRuntimeServerRole[] = []; + standby.onRoleChange((role) => { roles.push(role); }); + const takeover = nextRole(standby); + expect(standby.role()).toBe('standby'); + expect(standby.endpoint).toBe(endpoint); + // The owner keeps answering; the standby bound nothing. + expect(yield* askRuntime(endpointId)).toEqual({ owner: 'first' }); + + yield* Effect.promise(() => owner.close()); + expect(yield* Effect.promise(() => takeover)).toBe('owner'); + expect(standby.role()).toBe('owner'); + expect(roles).toEqual(['owner']); + expect(yield* askRuntime(endpointId)).toEqual({ owner: 'second' }); + const bound = yield* Effect.promise(() => stat(endpoint)); + expect(bound.mode & 0o777).toBe(0o600); + })); + // The last owner out removes the socket it bound, and no claim lingers. + yield* expectMissing(endpoint); + yield* expectMissing(`${endpoint}.lock`); +}), 20_000); + +it.live('closes a standby before any takeover without disturbing the owner or leaving files behind', () => Effect.gen(function*() { + if (process.platform === 'win32') return; + const endpointId = `event-ipc-standby-close-${crypto.randomUUID()}`; + const endpoint = eventRuntimeEndpoint(endpointId); + const owner = yield* Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'first' }), + })); + const standby = yield* Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'second' }), + whenOwned: 'standby', + })); + expect(standby.role()).toBe('standby'); + yield* Effect.promise(() => standby.close()); + expect(standby.role()).toBe('standby'); + expect(yield* askRuntime(endpointId)).toEqual({ owner: 'first' }); + yield* expectMissing(`${endpoint}.lock`); + + yield* Effect.promise(() => owner.close()); + yield* expectMissing(endpoint); + // A closed standby's loop is gone: nothing takes the freed endpoint over. + yield* Effect.sleep(Duration.seconds(2)); + yield* expectMissing(endpoint); + yield* expectMissing(`${endpoint}.lock`); +}), 20_000); + +it.live('lets exactly one of several standbys take over and keeps the rest standing by', () => Effect.gen(function*() { + if (process.platform === 'win32') return; + const endpointId = `event-ipc-standby-race-${crypto.randomUUID()}`; + const endpoint = eventRuntimeEndpoint(endpointId); + const owner = yield* Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'first' }), + })); + const standbys = yield* Effect.forEach(['second', 'third'] as const, (label) => Effect.acquireRelease( + Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: label }), + whenOwned: 'standby', + })), + (server) => Effect.promise(() => server.close()), + )); + expect(standbys.map((server) => server.role())).toEqual(['standby', 'standby']); + const takeovers = standbys.map((server) => nextRole(server).then(() => server)); + + yield* Effect.promise(() => owner.close()); + const winner = yield* Effect.promise(() => Promise.race(takeovers)); + // Give the loser at least one more probe after the winner bound the socket: + // it must see a live endpoint and stay standby rather than unlink the winner. + yield* Effect.sleep(Duration.millis(1_600)); + const roles = standbys.map((server) => server.role()); + expect(roles.filter((role) => role === 'owner')).toHaveLength(1); + expect(winner.role()).toBe('owner'); + const loser = standbys.find((server) => server !== winner)!; + expect(loser.role()).toBe('standby'); + const served = yield* askRuntime(endpointId); + expect([{ owner: 'second' }, { owner: 'third' }]).toContainEqual(served); + + // The loser is next in line: closing the new owner hands the endpoint on. + const succession = nextRole(loser); + yield* Effect.promise(() => winner.close()); + expect(yield* Effect.promise(() => succession)).toBe('owner'); + expect(yield* askRuntime(endpointId)).not.toEqual(served); + yield* Effect.promise(() => loser.close()); + yield* expectMissing(endpoint); + yield* expectMissing(`${endpoint}.lock`); +}), 30_000); + +it.live('holds the endpoint claim across close() so no standby can bind while the old owner is still tearing down', () => Effect.gen(function*() { + if (process.platform === 'win32') return; + const endpointId = `event-ipc-close-takeover-race-${crypto.randomUUID()}`; + const endpoint = eventRuntimeEndpoint(endpointId); + const removalStarted = deferred(); + const removalMayContinue = deferred(); + // The owner's close pauses after it has stopped listening — the path is + // already unlinked — and before its identity-checked removal: the window in + // which an unserialized takeover would bind a new socket at the path, quite + // possibly on the inode the kernel just freed, and hand it to the resumed + // identity check as this server's own. + const owner = yield* Effect.promise(() => createEventRuntimeServerForTest({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'first' }), + }, { + beforeEndpointRemoval: async () => { + removalStarted.resolve(); + await removalMayContinue.promise; + }, + })); + const standbyErrors: EventRuntimeTransportError[] = []; + const standby = yield* Effect.acquireRelease( + Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'second' }), + onStandbyError: (error) => { standbyErrors.push(error); }, + whenOwned: 'standby', + })), + (server) => Effect.promise(() => server.close()), + ); + expect(standby.role()).toBe('standby'); + + const closing = owner.close(); + yield* Effect.promise(() => removalStarted.promise); + // Let the standby probe and spend its whole bounded claim-retry budget + // inside that window: the closing owner's claim must hold it off, which it + // reports as a claim that did not clear. + yield* Effect.promise(() => eventually(() => standbyErrors.length > 0 || standby.role() === 'owner', 10_000)); + expect(standby.role()).toBe('standby'); + expect(yield* Effect.promise(() => requestEventRuntimeStatus({ endpoint, timeoutMs: 500 }))) + .toEqual({ status: 'unavailable' }); + // The attempt it held off was reported, not swallowed. + expect(standbyErrors.map((error) => error.message)) + .toContain('Event runtime endpoint claim did not clear after bounded retries.'); + removalMayContinue.resolve(); + yield* Effect.promise(() => closing); + + // Once the old owner has released the claim the standby takes over, and its + // socket is the one on the path. + yield* Effect.promise(() => eventually(() => standby.role() === 'owner', 5_000)); + const bound = yield* Effect.promise(() => stat(endpoint)); + expect(bound.isSocket()).toBe(true); + expect(yield* askRuntime(endpointId)).toEqual({ owner: 'second' }); + yield* Effect.promise(() => standby.close()); + yield* expectMissing(endpoint); + yield* expectMissing(`${endpoint}.lock`); +}), 30_000); + +it.live('reports a takeover that fails for a reason other than "still owned" and keeps standing by until one succeeds', () => Effect.gen(function*() { + if (process.platform === 'win32') return; + const endpointId = `event-ipc-standby-error-${crypto.randomUUID()}`; + const endpoint = eventRuntimeEndpoint(endpointId); + const owner = yield* Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'first' }), + })); + const hookFailure = new Error('probe hook failed once'); + let failedAttempts = 0; + const standbyErrors: EventRuntimeTransportError[] = []; + const standby = yield* Effect.acquireRelease( + Effect.promise(() => createEventRuntimeServerForTest({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'second' }), + onStandbyError: (error) => { standbyErrors.push(error); }, + whenOwned: 'standby', + }, { + // The first takeover attempt — the first open that finds the endpoint + // free — fails before it claims anything; the next one is left alone. + afterEndpointProbe: async (state) => { + if (state === 'live' || failedAttempts > 0) return; + failedAttempts += 1; + throw hookFailure; + }, + })), + (server) => Effect.promise(() => server.close()), + ); + expect(standby.role()).toBe('standby'); + const takeover = nextRole(standby); + + yield* Effect.promise(() => owner.close()); + expect(yield* Effect.promise(() => takeover)).toBe('owner'); + expect(failedAttempts).toBe(1); + expect(standbyErrors).toHaveLength(1); + expect(standbyErrors[0]).toBeInstanceOf(EventRuntimeTransportError); + expect(standbyErrors[0]).toMatchObject({ + cause: hookFailure, + code: 'runtime-failed', + message: 'Event runtime endpoint probe hook failed.', + }); + expect(yield* askRuntime(endpointId)).toEqual({ owner: 'second' }); + yield* expectMissing(`${endpoint}.lock`); +}), 20_000); + +it.live('keeps standing by and takes over later even when the error reporter itself throws', () => Effect.gen(function*() { + if (process.platform === 'win32') return; + const endpointId = `event-ipc-throwing-reporter-${crypto.randomUUID()}`; + const owner = yield* Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'first' }), + })); + let failedAttempts = 0; + const reported: EventRuntimeTransportError[] = []; + const standby = yield* Effect.acquireRelease( + Effect.promise(() => createEventRuntimeServerForTest({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'second' }), + onStandbyError: (error) => { + reported.push(error); + throw new Error('reporter exploded'); + }, + whenOwned: 'standby', + }, { + afterEndpointProbe: async (state) => { + if (state === 'live' || failedAttempts > 0) return; + failedAttempts += 1; + throw new Error('probe hook failed once'); + }, + })), + (server) => Effect.promise(() => server.close()), + ); + expect(standby.role()).toBe('standby'); + const takeover = nextRole(standby); + + yield* Effect.promise(() => owner.close()); + // The first attempt's failure reached the reporter, whose throw did not end the loop. + expect(yield* Effect.promise(() => takeover)).toBe('owner'); + expect(failedAttempts).toBe(1); + expect(reported.map((error) => error.message)).toEqual(['Event runtime endpoint probe hook failed.']); + expect(yield* askRuntime(endpointId)).toEqual({ owner: 'second' }); +}), 20_000); + +it.live('still reaches a later role listener when an earlier listener and the error reporter both throw', () => Effect.gen(function*() { + if (process.platform === 'win32') return; + const endpointId = `event-ipc-throwing-listener-and-reporter-${crypto.randomUUID()}`; + const owner = yield* Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'first' }), + })); + let reports = 0; + const standby = yield* Effect.acquireRelease( + Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'second' }), + onStandbyError: () => { + reports += 1; + throw new Error('reporter exploded'); + }, + whenOwned: 'standby', + })), + (server) => Effect.promise(() => server.close()), + ); + standby.onRoleChange(() => { throw new Error('listener exploded'); }); + const roles: EventRuntimeServerRole[] = []; + standby.onRoleChange((role) => { roles.push(role); }); + + yield* Effect.promise(() => owner.close()); + yield* Effect.promise(() => eventually(() => roles.length > 0, 5_000)); + expect(roles).toEqual(['owner']); + expect(reports).toBe(1); + expect(standby.role()).toBe('owner'); + expect(yield* askRuntime(endpointId)).toEqual({ owner: 'second' }); +}), 20_000); + +it.live('keeps notifying the remaining role listeners when an earlier one throws, and reports the throw', () => Effect.gen(function*() { + if (process.platform === 'win32') return; + const endpointId = `event-ipc-role-listener-${crypto.randomUUID()}`; + const owner = yield* Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'first' }), + })); + const standbyErrors: EventRuntimeTransportError[] = []; + const standby = yield* Effect.acquireRelease( + Effect.promise(() => createEventRuntimeServer({ + artifactEpoch: 'epoch-1', + endpointId, + handle: async () => ({ owner: 'second' }), + onStandbyError: (error) => { standbyErrors.push(error); }, + whenOwned: 'standby', + })), + (server) => Effect.promise(() => server.close()), + ); + const listenerFailure = new Error('listener exploded'); + standby.onRoleChange(() => { throw listenerFailure; }); + const roles: EventRuntimeServerRole[] = []; + standby.onRoleChange((role) => { roles.push(role); }); + const takeover = nextRole(standby); + + yield* Effect.promise(() => owner.close()); + expect(yield* Effect.promise(() => takeover)).toBe('owner'); + expect(roles).toEqual(['owner']); + expect(standby.role()).toBe('owner'); + expect(standbyErrors).toHaveLength(1); + expect(standbyErrors[0]).toBeInstanceOf(EventRuntimeTransportError); + expect(standbyErrors[0]).toMatchObject({ + cause: listenerFailure, + code: 'runtime-failed', + message: 'Event runtime role listener failed.', + }); + expect(yield* askRuntime(endpointId)).toEqual({ owner: 'second' }); +}), 20_000); + it.live('reports a failed endpoint removal from close() after the server has stopped listening', () => Effect.gen(function*() { if (process.platform === 'win32') return; const endpointId = `event-ipc-close-failure-${crypto.randomUUID()}`; diff --git a/packages/agent-bundle/tests/generated-route-server.test.ts b/packages/agent-bundle/tests/generated-route-server.test.ts index c118f627e..2baf377e5 100644 --- a/packages/agent-bundle/tests/generated-route-server.test.ts +++ b/packages/agent-bundle/tests/generated-route-server.test.ts @@ -13,6 +13,7 @@ import { requestEventRuntime, requestEventRuntimeStatus, } from '../src/events/ipc.ts'; +import { eventuallyPasses } from './support/eventually.ts'; const roots: string[] = []; @@ -236,6 +237,40 @@ const writeGeneratedProject = async ( ]); }; +interface GeneratedEntryConnection { + readonly client: Client; + readonly close: () => Promise; + readonly pid: number | undefined; + /** Everything the server process wrote to stderr so far — stdout is the protocol wire. */ + readonly stderr: () => string; +} + +/** Spawns one built stdio entry and completes `initialize` against it. */ +const connectGeneratedEntry = async ( + entry: string, + name = 'generated-route-test', +): Promise => { + const client = new Client({ name, version: '0.0.0' }); + const transport = new StdioClientTransport({ args: [entry], command: process.execPath, stderr: 'pipe' }); + let diagnostics = ''; + transport.stderr?.on('data', (chunk) => { diagnostics += String(chunk); }); + try { + await client.connect(transport); + } catch (error) { + throw new Error(`Generated route server failed to connect: ${diagnostics}`, { cause: error }); + } + let closing: Promise | undefined; + return { + client, + close: () => { + closing ??= client.close(); + return closing; + }, + pid: transport.pid ?? undefined, + stderr: () => diagnostics, + }; +}; + const connectGeneratedServer = async ( root: string, target: 'cursor' | 'portable' = 'portable', @@ -248,25 +283,12 @@ const connectGeneratedServer = async ( const compiled = await build({ output, root, targets: [target] }); const server = compiled.model.mcpServers[0]; if (server?.args?.[0] === undefined) throw new Error('expected a generated MCP entry'); - const client = new Client({ name: 'generated-route-test', version: '0.0.0' }); - const transport = new StdioClientTransport({ - args: [join(output, target, server.args[0])], - command: process.execPath, - stderr: 'pipe', - }); - let diagnostics = ''; - transport.stderr?.on('data', (chunk) => { diagnostics += String(chunk); }); - try { - await client.connect(transport); - } catch (error) { - throw new Error(`Generated route server failed to connect: ${diagnostics}`, { cause: error }); - } + const entry = join(output, target, server.args[0]); + const connection = await connectGeneratedEntry(entry); return { - client, - close: async () => { - await client.close(); - }, - endpointId: `${compiled.build.manifest.project.revision}:${target}:${dirname(dirname(resolve(join(output, target, server.args[0]))))}`, + client: connection.client, + close: connection.close, + endpointId: `${compiled.build.manifest.project.revision}:${target}:${dirname(dirname(resolve(entry)))}`, }; }; @@ -877,6 +899,74 @@ it('fails closed when the generated runtime worker restarts', { retry: 2, timeou } }); +it('keeps a second generated server from the same install alive while the first owns the event runtime socket, then hands the socket over', { retry: 2, timeout: 90_000 }, async () => { + const root = await mkdtemp(join(tmpdir(), 'agent-bundle-generated-standby-')); + roots.push(root); + await writeGeneratedProject(root, { + 'src/mcp/curator/tools/ping.tsx': [ + "import { Agent } from '@agent-bundle/runtime';", + "import { createElement } from 'react';", + "import { z } from 'zod';", + "export const config = { annotations: { readOnlyHint: true }, description: 'Answer from this process.' };", + 'export const inputSchema = z.object({}).strict();', + 'export const resultSchema = z.object({ pid: z.number() }).strict();', + 'export default async function Ping() {', + " return createElement(Agent.Result, { value: { pid: process.pid } }, createElement(Agent.Text, null, 'pong'));", + '}', + '', + ].join('\n'), + 'src/events/session/start.tsx': [ + "import { Agent } from '@agent-bundle/runtime';", + "import { createElement } from 'react';", + "export const config = { targets: ['cursor'] };", + "export default async function SessionStart() { return createElement(Agent.Text, null, 'ready'); }", + '', + ].join('\n'), + }, 'cursor'); + const output = join(root, 'artifact'); + const compiled = await build({ output, root, targets: ['cursor'] }); + const server = compiled.model.mcpServers[0]; + if (server?.args?.[0] === undefined) throw new Error('expected a generated MCP entry'); + const entry = join(output, 'cursor', server.args[0]); + const endpointId = `${compiled.build.manifest.project.revision}:cursor:${dirname(dirname(resolve(entry)))}`; + const endpoint = eventRuntimeEndpoint(endpointId); + const status = (): Promise => requestEventRuntimeStatus({ endpointId, timeoutMs: 1_000 }); + + // Two host sessions launch the same plugin from the same install root: the + // second finds the event runtime socket owned and must still come up. + const first = await connectGeneratedEntry(entry, 'generated-standby-first'); + let second: GeneratedEntryConnection | undefined; + try { + second = await connectGeneratedEntry(entry, 'generated-standby-second'); + const [firstTools, secondTools] = await Promise.all([first.client.listTools(), second.client.listTools()]); + expect(firstTools.tools.map((tool) => tool.name)).toEqual(['ping']); + expect(secondTools.tools.map((tool) => tool.name)).toEqual(['ping']); + await expect(second.client.callTool({ arguments: {}, name: 'ping' }, { signal: AbortSignal.timeout(10_000) })) + .resolves.toMatchObject({ structuredContent: { pid: second.pid } }); + await eventuallyPasses(() => { + expect(second!.stderr()).toContain(`agent-bundle event runtime: ${endpoint} is owned by another process; standing by`); + }, { attempts: 100, delayMs: 50 }); + expect(first.stderr()).not.toContain('standing by'); + await expect(status()).resolves.toMatchObject({ pid: first.pid, status: 'available' }); + + // The owner exits; the standby takes the socket over without restarting. + await first.close(); + await eventuallyPasses(async () => { + await expect(status()).resolves.toMatchObject({ pid: second!.pid, status: 'available' }); + }, { attempts: 100, delayMs: 100 }); + expect(second.stderr()).toContain(`agent-bundle event runtime: ${endpoint} was released by its owner; took it over`); + await expect(second.client.callTool({ arguments: {}, name: 'ping' }, { signal: AbortSignal.timeout(10_000) })) + .resolves.toMatchObject({ structuredContent: { pid: second.pid } }); + } finally { + await first.close(); + await second?.close(); + } + // The last owner out removes the socket it bound. + await eventuallyPasses(async () => { + await expect(stat(endpoint)).rejects.toMatchObject({ code: 'ENOENT' }); + }, { attempts: 100, delayMs: 50 }); +}); + it('renders one tool/after event route through two native thin clients', { retry: 2, timeout: 90_000 }, async () => { const root = await mkdtemp(join(tmpdir(), 'agent-bundle-generated-events-')); roots.push(root); diff --git a/packages/agent-bundle/tests/mcp-server-runtime.test.ts b/packages/agent-bundle/tests/mcp-server-runtime.test.ts index a4db61c7b..9d358bf6b 100644 --- a/packages/agent-bundle/tests/mcp-server-runtime.test.ts +++ b/packages/agent-bundle/tests/mcp-server-runtime.test.ts @@ -10,9 +10,15 @@ import type { AgentNoticePrincipal, } from '@agent-bundle/runtime/notices'; +import { + type createEventRuntimeServer, + type EventRuntimeServerRole, + EventRuntimeTransportError, +} from '../src/events/ipc.ts'; import { advertisedOutputSchema, createGeneratedRouteMcpServer, + createStandbyErrorReporter, type GeneratedNoticeDeliveryBinding, type GeneratedNoticeInboxSignalOutcome, type GeneratedNoticePrincipal, @@ -291,6 +297,8 @@ describe('generated server teardown', () => { order.push('events'); throw new Error('socket teardown failed'); }, + onRoleChange: () => () => undefined, + role: () => 'owner', })) as never, endpointId: 'teardown-test', projectEventDocument: (() => { @@ -338,3 +346,92 @@ describe('generated server teardown', () => { await client.close(); }); }); + +describe('createStandbyErrorReporter', () => { + const failure = (message: string): EventRuntimeTransportError => new EventRuntimeTransportError('runtime-failed', message); + + it('reports each distinct takeover failure once, and again only after the repeat interval', () => { + let now = 1_000; + const lines: string[] = []; + const report = createStandbyErrorReporter((line) => { lines.push(line); }, () => now); + + report(failure('Unable to prepare the event runtime endpoint.')); + report(failure('Unable to prepare the event runtime endpoint.')); + report(failure('Event runtime endpoint claim did not clear after bounded retries.')); + report(failure('Unable to prepare the event runtime endpoint.')); + expect(lines).toEqual([ + 'takeover failed, still standing by: Unable to prepare the event runtime endpoint.', + 'takeover failed, still standing by: Event runtime endpoint claim did not clear after bounded retries.', + ]); + + now += 29_999; + report(failure('Unable to prepare the event runtime endpoint.')); + expect(lines).toHaveLength(2); + now += 1; + report(failure('Unable to prepare the event runtime endpoint.')); + expect(lines).toHaveLength(3); + // The other message keeps its own clock. + report(failure('Event runtime endpoint claim did not clear after bounded retries.')); + expect(lines).toHaveLength(4); + }); +}); + +describe('generated server standby diagnostics', () => { + it('announces standby, takeover failures, and the takeover on stderr only', async () => { + const { host, notices } = stubs(); + let captured: Parameters[0] | undefined; + let roleListener: ((role: EventRuntimeServerRole) => void) | undefined; + const stderr: string[] = []; + const write = process.stderr.write; + process.stderr.write = ((chunk: string | Uint8Array) => { + stderr.push(String(chunk)); + return true; + }) as typeof process.stderr.write; + try { + const server = await createGeneratedRouteMcpServer({ + artifactEpoch: 'epoch', + events: { + allowedTargets: ['claude'], + artifactEpoch: 'epoch', + createCanonicalEventProps: (() => { + throw new Error('not invoked'); + }) as never, + createEventRuntimeServer: (async (options: Parameters[0]) => { + captured = options; + return { + close: async () => undefined, + endpoint: '/tmp/agent-bundle-test/event-standby.sock', + onRoleChange: (listener: (role: EventRuntimeServerRole) => void) => { + roleListener = listener; + return () => undefined; + }, + role: () => 'standby', + }; + }) as never, + endpointId: 'standby-diagnostics-test', + projectEventDocument: (() => { + throw new Error('not invoked'); + }) as never, + target: 'claude', + }, + host, + notices, + plugin: { name: 'standby', version: '0.0.0' }, + routes: {}, + }); + expect(captured?.whenOwned).toBe('standby'); + const prepareFailure = new EventRuntimeTransportError('runtime-failed', 'Unable to prepare the event runtime endpoint.'); + captured?.onStandbyError?.(prepareFailure); + captured?.onStandbyError?.(prepareFailure); + roleListener?.('owner'); + await server.close(); + } finally { + process.stderr.write = write; + } + expect(stderr.filter((line) => line.startsWith('agent-bundle event runtime:'))).toEqual([ + 'agent-bundle event runtime: /tmp/agent-bundle-test/event-standby.sock is owned by another process; standing by\n', + 'agent-bundle event runtime: /tmp/agent-bundle-test/event-standby.sock takeover failed, still standing by: Unable to prepare the event runtime endpoint.\n', + 'agent-bundle event runtime: /tmp/agent-bundle-test/event-standby.sock was released by its owner; took it over\n', + ]); + }); +}); diff --git a/website/docs/en/guide/authoring/hooks.mdx b/website/docs/en/guide/authoring/hooks.mdx index 123cb974a..6132ab1b0 100644 --- a/website/docs/en/guide/authoring/hooks.mdx +++ b/website/docs/en/guide/authoring/hooks.mdx @@ -337,6 +337,25 @@ down first, so it never leaves a live server behind the error. A stale socket or crashed process is recovered by the next server, which probes the endpoint and reclaims a claim whose owner is provably dead. +**Concurrent sessions.** Hosts routinely run several sessions of one plugin from the same +install root — every Claude Code session Claude Desktop launches shares one +`--plugin-dir`, and Cursor imports Claude-installed plugins from that same cache — so several +generated MCP server processes with the same endpoint are normal. Exactly one of them owns +the socket at a time. A server that starts while another process owns the endpoint does +**not** exit: it comes up in the `standby` role, serves its own MCP session as usual, writes +`agent-bundle event runtime: is owned by another process; standing by` to stderr, +and probes the endpoint about once a second. The moment the owner exits, a standby takes the +socket over through the same claim-and-listen path an unowned start uses (and logs +`… was released by its owner; took it over`); if several stand by, the claim lock lets exactly +one win and the rest keep waiting. Hook wrappers need no change: while an owner is alive they +are answered as before, and during the takeover gap (at most about a second) they see +`runtime-unavailable`, which is exactly the case the `fallback: 'standalone'` route above +covers — a shared-only route exits nonzero for that one event. A takeover that fails for any +reason other than the socket still being owned (a socket directory it cannot write, a claim +that never clears) is reported on the server's stderr — one line per distinct failure, repeated +at most every 30 seconds — while the server keeps standing by. A startup that fails for any +other reason still fails closed, as does a stale socket the owner cannot remove. + A route with `runtime: 'standalone'` bundles its module into the wrapper itself: the same canonical identity, the same projection, no shared process state. `AB4817` refuses a route that requires the shared runtime on a target where no generated MCP entry hosts it and no diff --git a/website/docs/zh/guide/authoring/hooks.mdx b/website/docs/zh/guide/authoring/hooks.mdx index 73c08ed75..d7f276dc6 100644 --- a/website/docs/zh/guide/authoring/hooks.mdx +++ b/website/docs/zh/guide/authoring/hooks.mdx @@ -293,6 +293,19 @@ the event runtime endpoint.` / `Unable to release the event runtime endpoint cla 启动失败时会先关掉已经起来的监听器,因此绝不会在错误背后留下一个活的服务端。崩溃进程留下的过期 socket 或锁会由下一个服务端恢复:它探测端点,并回收所有者已被证明死亡的认领。 +**并发会话。** 宿主经常从同一个安装根目录运行同一插件的多个会话——Claude Desktop 启动的每个 +Claude Code 会话共享同一个 `--plugin-dir`,Cursor 也会从同一个缓存导入 Claude 安装的插件——因此多个 +端点相同的生成 MCP 服务端进程是常态。任一时刻恰好只有一个进程拥有 socket。启动时发现端点已被另一个 +进程拥有的服务端**不会**退出:它以 `standby` 角色启动,照常服务自己的 MCP 会话,向 stderr 写入 +`agent-bundle event runtime: is owned by another process; standing by`,并大约每秒探测一次 +端点。所有者一退出,某个待命者就会通过与无主启动完全相同的认领并监听路径接管 socket(并记录 +`… was released by its owner; took it over`);若有多个待命者,认领锁保证恰好一个胜出,其余继续等待。 +钩子包装层无需任何改动:只要所有者活着,它们的回答与以前一样;在接管间隙(最多约一秒)里它们会看到 +`runtime-unavailable`,这正是上文 `fallback: 'standalone'` 路由覆盖的情形——仅共享的路由则对这一个事件 +以非零退出。接管因"socket 仍被占有"之外的任何原因失败时(无法写入的 socket 目录、永远不清除的认领), +服务端会把失败报告到自己的 stderr——每种不同的失败一行,最多每 30 秒重复一次——并继续待命。因其他 +任何原因失败的启动仍然失败即关闭,所有者无法删除的过期 socket 也一样。 + `runtime: 'standalone'` 的路由把它的模块打进包装层自身:同样的规范身份、同样的投影、没有共享的进程状态。 `AB4817` 会拒绝这样的路由:它需要共享运行时,但在某个 target 上没有生成的 MCP 入口承载它,也没有独立 回退。