From 5f43a1ab8cc7ad7a9a539453b7a0f57ebdd0460e Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Mon, 28 Sep 2026 15:01:43 +0000 Subject: [PATCH 1/8] Retain Session executors behind a shared Harness contract --- CONTRIBUTING.md | 375 +++++++++--------- README.md | 1 + .../internal/agent/claudesdk/cancellation.go | 35 +- .../agent/claudesdk/cancellation_test.go | 17 +- .../agent/claudesdk/commands_session_test.go | 14 +- .../internal/agent/claudesdk/executor.go | 314 +++++++++++++++ .../agent/claudesdk/executor_fixture_test.go | 53 +++ .../claudesdk/executor_live_linux_test.go | 199 ++++++++++ .../internal/agent/claudesdk/executor_test.go | 285 +++++++++++++ .../internal/agent/claudesdk/executor_turn.go | 215 ++++++++++ .../internal/agent/claudesdk/functions.go | 9 +- .../agent/claudesdk/functions_test.go | 4 +- .../internal/agent/claudesdk/preparation.go | 193 +++------ .../claudesdk/preparation_fixture_test.go | 16 +- .../agent/claudesdk/preparation_test.go | 10 +- .../internal/agent/claudesdk/readiness.go | 2 +- .../agent/claudesdk/readiness_test.go | 6 +- .../internal/agent/claudesdk/runtime_cache.go | 52 +++ .../internal/agent/claudesdk/session.go | 261 ++---------- .../internal/agent/claudesdk/session_test.go | 15 +- .../internal/agent/claudesdk/steering.go | 12 +- .../internal/agent/claudesdk/steering_test.go | 13 +- .../agent/claudesdk/workspace_directory.go | 3 + .../claudesdk/workspace_directory_test.go | 12 +- .../agent/claudesdk/workspace_launch_test.go | 7 +- .../agent/claudesdk/workspace_read.go | 3 + .../agent/claudesdk/workspace_read_test.go | 14 +- .../claudesdk/workspace_structured_test.go | 2 +- .../internal/agent/codex/executor.go | 156 ++++++++ .../agent/codex/executor_native_test.go | 164 ++++++++ .../internal/agent/codex/executor_test.go | 187 +++++++++ .../internal/agent/codex/executor_turn.go | 163 ++++++++ .../internal/agent/codex/function_receipts.go | 8 + .../internal/agent/codex/preparation.go | 19 +- .../agent/codex/preparation_helpers_test.go | 34 ++ .../agent/codex/preparation_router_test.go | 13 +- .../internal/agent/codex/prepared.go | 4 +- .../parsar-daemon/internal/agent/codex/rpc.go | 39 +- .../internal/agent/codex/server_requests.go | 16 + .../internal/agent/codex/session.go | 30 +- .../internal/agent/codex/session_cancel.go | 3 + .../internal/agent/codex/session_run.go | 41 +- .../internal/agent/codex/session_steering.go | 6 +- .../internal/agent/codex/session_usage.go | 2 +- apps/parsar-daemon/internal/agent/executor.go | 61 +++ .../internal/agent/mcode/connection.go | 146 +++++++ .../agent/mcode/environment_mcp_test.go | 10 +- .../internal/agent/mcode/execution.go | 2 +- .../internal/agent/mcode/executor.go | 173 ++++++++ .../agent/mcode/executor_native_test.go | 176 ++++++++ .../internal/agent/mcode/executor_test.go | 243 ++++++++++++ .../internal/agent/mcode/executor_turn.go | 153 +++++++ .../internal/agent/mcode/preparation.go | 146 ++----- .../internal/agent/mcode/preparation_test.go | 2 +- .../internal/agent/mcode/session.go | 147 +++---- .../internal/agent/mcode/session_test.go | 34 +- .../internal/agent/mcode/steering.go | 37 +- .../internal/agent/mcode/subagent_cancel.go | 27 +- .../internal/agent/mcode/subagents.go | 8 +- .../internal/agent/mcode/subagents_test.go | 35 ++ apps/parsar-daemon/internal/agent/registry.go | 3 + .../internal/agent/registry_test.go | 21 + .../internal/cli/agent_registration.go | 6 + apps/parsar-daemon/internal/cli/authoring.go | 3 + .../internal/cli/capability_downloads.go | 62 ++- .../internal/cli/capability_downloads_test.go | 21 + apps/parsar-daemon/internal/cli/claude_sdk.go | 3 + .../internal/cli/claude_sdk_test.go | 3 + apps/parsar-daemon/internal/cli/connect.go | 9 +- .../internal/cli/connect_cleanup.go | 24 ++ .../internal/cli/connect_cleanup_test.go | 196 +++++++++ .../internal/cli/connect_suspend.go | 11 +- .../internal/cli/skill_upload.go | 25 +- .../internal/dispatch/executor.go | 373 +++++++++++++++++ .../internal/dispatch/executor_test.go | 312 +++++++++++++++ .../internal/dispatch/local_directory_test.go | 2 +- .../internal/dispatch/mcp_http_test.go | 2 +- .../internal/dispatch/preparation.go | 9 + .../dispatch/preparation_cancel_test.go | 34 +- .../internal/dispatch/preparation_cleanup.go | 3 + .../dispatch/preparation_cleanup_test.go | 97 +---- .../preparation_executor_fixture_test.go | 133 +++++++ .../internal/dispatch/preparation_start.go | 159 +++----- .../internal/dispatch/preparation_test.go | 113 ++---- .../internal/dispatch/prepared_handoff.go | 144 +++++-- .../dispatch/prepared_handoff_test.go | 10 +- .../parsar-daemon/internal/dispatch/router.go | 2 + .../internal/dispatch/shutdown.go | 7 + .../internal/dispatch/steering.go | 4 +- .../internal/dispatch/suspend.go | 15 + .../dispatch/workspace_directory_test.go | 2 +- .../internal/dispatch/workspace_read.go | 3 + .../internal/dispatch/workspace_write.go | 8 +- .../internal/localworkspace/binding.go | 2 +- .../internal/localworkspace/binding_test.go | 8 + .../parsar-daemon/testdata/onboarding/main.go | 87 +++- contracts/agents-api/harness-onboarding.md | 228 +++++++---- contracts/agents-api/harnesses.md | 25 +- contracts/agents-api/message-input.md | 2 +- internal/agentdaemon/proto/envelope_test.go | 4 +- internal/agentdaemon/proto/preparation.go | 16 +- internal/agentdaemon/proto/version.go | 2 +- packages/claude-sdk-adapter/src/adapter.ts | 107 +++-- .../src/executor_protocol.ts | 129 ++++++ packages/claude-sdk-adapter/src/inputs.ts | 5 + packages/claude-sdk-adapter/src/main.ts | 12 +- packages/claude-sdk-adapter/src/request.ts | 9 +- .../claude-sdk-adapter/src/runtime_check.ts | 2 +- packages/claude-sdk-adapter/src/subagents.ts | 5 + packages/claude-sdk-adapter/src/usage.ts | 11 + .../tests/executor.test.mjs | 143 +++++++ .../claude-sdk-adapter/tests/usage.test.mjs | 16 +- scripts/check-claude-sdk-runtime.mjs | 4 +- .../agents-api/internal/execution/delivery.go | 54 +++ .../execution/directory_preparation.go | 2 +- .../internal/execution/dispatcher.go | 10 +- .../execution/environment_admission.go | 6 +- .../execution/executor_observation.go | 41 ++ .../execution/executor_preparation.go | 63 +++ .../internal/execution/notifications.go | 50 +++ .../internal/execution/notifications_test.go | 50 +++ .../internal/execution/preparation.go | 42 +- .../internal/execution/prepared_dispatch.go | 7 +- .../agents-api/internal/execution/request.go | 2 +- .../agents-api/internal/execution/support.go | 3 + .../agents-api/internal/execution/worker.go | 119 ++++-- .../internal/execution/worker_schedule.go | 6 +- .../internal/execution/worker_wakeup.go | 25 ++ .../internal/execution/worker_wakeup_test.go | 35 ++ .../internal/store/claude_execution_test.go | 4 +- .../internal/store/claude_mcp_test.go | 6 +- .../internal/store/command_output_test.go | 2 +- .../internal/store/dispatch_test.go | 29 +- .../store/environment_expiry_dispatch_test.go | 6 +- .../store/environment_worker_helpers_test.go | 5 + .../store/environment_worker_scan_test.go | 4 +- .../internal/store/environment_worker_test.go | 6 +- .../internal/store/execution_events_test.go | 4 +- .../internal/store/execution_messages_test.go | 6 +- .../internal/store/execution_tools_test.go | 2 +- .../internal/store/executor_fixture_test.go | 63 +++ .../internal/store/executor_recovery_test.go | 58 +++ .../internal/store/function_execution_test.go | 10 +- .../store/function_input_execution_test.go | 2 +- .../internal/store/function_worker_test.go | 10 +- .../internal/store/harness_onboarding_test.go | 29 +- .../internal/store/public_execution_test.go | 6 +- .../store/session_deletion_execution_test.go | 8 +- .../internal/store/steering_receipts_test.go | 2 +- .../store/structured_output_dispatch_test.go | 2 +- .../internal/store/subagent_dispatch_test.go | 2 +- .../internal/store/worker_input_race_test.go | 2 +- .../internal/store/worker_lease_loss_test.go | 4 +- .../internal/store/worker_wakeup_test.go | 141 +++++++ .../agents-api/tests/official_user_runtime.py | 36 +- 155 files changed, 6522 insertions(+), 1505 deletions(-) create mode 100644 apps/parsar-daemon/internal/agent/claudesdk/executor.go create mode 100644 apps/parsar-daemon/internal/agent/claudesdk/executor_fixture_test.go create mode 100644 apps/parsar-daemon/internal/agent/claudesdk/executor_live_linux_test.go create mode 100644 apps/parsar-daemon/internal/agent/claudesdk/executor_test.go create mode 100644 apps/parsar-daemon/internal/agent/claudesdk/executor_turn.go create mode 100644 apps/parsar-daemon/internal/agent/claudesdk/runtime_cache.go create mode 100644 apps/parsar-daemon/internal/agent/codex/executor.go create mode 100644 apps/parsar-daemon/internal/agent/codex/executor_native_test.go create mode 100644 apps/parsar-daemon/internal/agent/codex/executor_test.go create mode 100644 apps/parsar-daemon/internal/agent/codex/executor_turn.go create mode 100644 apps/parsar-daemon/internal/agent/executor.go create mode 100644 apps/parsar-daemon/internal/agent/mcode/connection.go create mode 100644 apps/parsar-daemon/internal/agent/mcode/executor.go create mode 100644 apps/parsar-daemon/internal/agent/mcode/executor_native_test.go create mode 100644 apps/parsar-daemon/internal/agent/mcode/executor_test.go create mode 100644 apps/parsar-daemon/internal/agent/mcode/executor_turn.go create mode 100644 apps/parsar-daemon/internal/cli/connect_cleanup.go create mode 100644 apps/parsar-daemon/internal/cli/connect_cleanup_test.go create mode 100644 apps/parsar-daemon/internal/dispatch/executor.go create mode 100644 apps/parsar-daemon/internal/dispatch/executor_test.go create mode 100644 apps/parsar-daemon/internal/dispatch/preparation_executor_fixture_test.go create mode 100644 packages/claude-sdk-adapter/src/executor_protocol.ts create mode 100644 packages/claude-sdk-adapter/tests/executor.test.mjs create mode 100644 services/agents-api/internal/execution/executor_observation.go create mode 100644 services/agents-api/internal/execution/executor_preparation.go create mode 100644 services/agents-api/internal/execution/notifications.go create mode 100644 services/agents-api/internal/execution/notifications_test.go create mode 100644 services/agents-api/internal/execution/worker_wakeup.go create mode 100644 services/agents-api/internal/execution/worker_wakeup_test.go create mode 100644 services/agents-api/internal/store/executor_fixture_test.go create mode 100644 services/agents-api/internal/store/executor_recovery_test.go create mode 100644 services/agents-api/internal/store/worker_wakeup_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 80420dc72..769981bcf 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -473,7 +473,7 @@ forwarding. The explicit daemon-executor decision supersedes the previous native executor interoperability requirement. The superseded execution route is removed; retain reusable filesystem helpers, necessary regressions and historical evidence without a compatibility layer. -The private daemon wire protocol is 0.5.0. Initial, prepared and active input use +The private daemon wire protocol is 0.6.0. Initial, prepared and active input use the same ordered MessageInput contract, replacing scalar prompts and attachments. User-message boundaries and text/image order remain intact through Core and the Runtime wire; adapters own native conversion and receipt aggregation. Text-only @@ -485,8 +485,8 @@ input list and uses blank-line separators between messages; this does not preser independent native user-message boundaries. No old wire fallback is maintained. Deploy Core and daemon together; the existing major/minor WebSocket check rejects older major/minor peers before dispatch rather than ignoring removed fields. -The independently packaged Claude bridge uses protocol 2 for ordered input; -readiness rejects packages reporting the old string-input protocol. +The independently packaged Claude bridge uses protocol 3 for a prepared Executor +and separately identified Turns; readiness rejects other protocol versions. Image-bearing messages require a qualified profile/placement before persistence and image support from the selected Runtime before native delivery. These checks apply to that operation only; ordinary text retains offline queueing. Initial, @@ -1032,7 +1032,7 @@ a chain of old writable disks across suspension cycles. No Kubernetes, distribut scheduler or snapshot replication belongs in this V1 profile. Queued work and live Environment file access request wake. History and published -artifact reads do not. Planned suspension uses private daemon wire 0.5.0 with an +artifact reads do not. Planned suspension uses private daemon wire 0.6.0 with an Environment and suspension token; a PID/start-time fenced local control signal wakes the parked daemon, which reauthenticates before admitting new work. A transient disconnect before confirmation retries the same armed suspension with @@ -1561,103 +1561,106 @@ reconciles old connection observations before admitting new ones. A connection o heartbeat does not prove native readiness or process quiescence. Failed or stale observations cannot establish a current connection. -Preparation resolves the immutable local binding and holds the existing native -resource without creating model work. Start transfers that resource once; failed -start and abandoned preparation retain the established cleanup rules. Strict resume -uses the bound native history and never falls back to a new Session. - -Every executable preparation must implement `PreparedCancellation`; read-only -preparations may implement only `Prepared`. The Router rejects and closes an -executable preparation before `ready` when that contract is missing. Codex fences -future Start under the transfer lock; unused resources use preparation teardown, -while transferred resources use the existing Session cancellation path inside the -adapter. `Close` remains inert after transfer. -`CancellationOutcome` exposes observed content, Usage and verified native identity; -an unstarted resource has no measured Usage or observed resume identity. This is -best-effort cancellation and an observed snapshot, not immutable final output, -notification drain, caller-deadline compliance or remote process quiescence. -From Start admission onward, the exact `PreparedCancellation` object is the sole -release target: a late Session never receives fallback `Cancel`, and the Router -never falls back to preparation `Close` or owner-context cancellation. Successful -`Cancel` means local cleanup is complete and no more output writes can occur. A -failed or timed-out call returns without waiting for output, retains execution -ownership, and permits only a later explicit serialized retry on that same object. -Before publication, failed cleanup also retains the preparation slot. Successful -publication permanently transfers resource tracking to the Run; subsequent release -does not restore preparation ownership or make its old handle cancel that Run. -Forwarded permission and user-choice observations from that cancelled handoff do -not register actionable interactions. Codex prepared cancellation also waits for -the transferred Session's local cleanup, which can finish after output closes. -Codex cancellation has one continuing native owner: caller deadlines bound only -their wait, leaving child observation and cleanup alive for an explicit retry. -Only observed child terminal facts can settle the child; actual observation -failures remain failures. Native process-exit confirmation remains retryable. -One prepared-output consumer starts before `Prepared.Start`, so native output beyond -the 64-frame channel capacity cannot deadlock Start. It retains the first terminal -frame, drains later output, and forwards accepted frames before the observed cancellation -outcome receipt. Missing capability, failed cancellation or failed forwarding cannot -produce an applied receipt. An unused resource may supply an empty observed outcome; -the Router never fabricates one. - -The returned Session remains private until the `started` status send succeeds. The -handoff has one permanent release claim, one current native attempt and one -success-only settlement. Function results, permission decisions, user-choice -decisions and steering admitted before the claim hold the same operation barrier -through native submission, replay bookkeeping and receipt delivery. Natural -completion waits for Start publication before cancellation; an abort wakes that -same attempt and may fence a concurrent Start. The initial started-status send is -bounded by the preparation deadline, which is rechecked at publication commit. -Delayed expiry callbacks cannot cancel a successfully published handoff. The successful attempt waits for Start and the sole output -consumer, then forwards the retained terminal frame and removes the Run. Internal -paths join the current attempt; only an explicit cancel, Release, expiry, device -shutdown or later Router Shutdown may retry a failed attempt. Workspace reads require -a published, open Session but keep their independent bounded lifecycle. - -Receipt settlement waits at most ten seconds, with a separate five-second send -budget and the existing gateway settlement deadline. A timeout does not free the -resource or stop tracking late cleanup. Shutdown cancels receipt waits while owned -Start/cleanup work remains tracked. This ordering covers cancellation received while -Start is pending; ordinary post-transfer cancellation, complete native output and -remote process exit retain their separate limitations. - -The private daemon preparation controls reuse execution configuration but reject -input, RunID, Conversation, attachments and product authoring. The local profile -requires an exact Environment binding, stable state key, strict resume and completion -release. Its separate capability is registered through an execution-only factory -and preserved through the product registry wrapper and heartbeat mapping. Native -details remain inside the adapter; this private profile does not narrow upstream. - -Preparation request IDs correlate only control responses. The daemon returns a -fresh opaque handle before slow work; Start supplies that handle and the actual -RunID/prompt. Handles belong to one daemon connection. Gateway preparation -subscriptions do not register Runs. Per-handle revisions order asynchronous status -snapshots; reject responses describe control errors without inventing run events. -At most four native preparations may be preparing, ready, starting or closing. -Start admission expires five minutes after acceptance; retries do not extend it. -Failed cleanup retains the native resource and its capacity until Close succeeds; -terminal handles with retained resources cannot be pruned or started again. -At most 64 request records are retained; retired request IDs may allocate a fresh -handle, while old handles cannot consume replacements. This is not durable -exactly-once preparation or cross-connection recovery. - -Preparation and Start execute outside the receive loop and router lock, with -tracked lifetime work. Start reserves the real RunID and fixed cancellation target -before native work; cancellation in that phase uses the adapter contract across -the transfer. A late result cannot resurrect released ownership. Shutdown claims -or retries every prepared release under the lock before closing its cancellation -signal. A failed native attempt returns Shutdown without waiting on an output -consumer that may still be blocked; a later Shutdown retries the same target. -Successful publication stops the preparation deadline and uses the same prepared -output consumer and completion release; -later preparation Release cannot cancel that Run. Released/expired status makes -the handle unusable; asynchronous native cleanup still counts toward capacity and -does not promise immediate OS quiescence. Release retries retained cleanup. -Concurrent Shutdown calls join one tracked attempt within their caller deadlines; -a later call retries failed preparation cleanup and reports any remaining error. -A caller timeout does not discard ownership or repeat in-flight cleanup. These -records remain connection-local, not a persistent remote retirement fence. -The public idle-text path uses this -admission/start wiring; complete Environment lifecycle remains required work. +A Session owns one reusable Executor in its connected Runtime. A Turn owns one +input execution, its output stream and its cancellation. `agent.ExecutorFactory` +prepares the fixed configuration; `Executor.StartTurn` creates a new `agent.Turn` +without replacing healthy native resources. Normal completion settles only the +Turn. `Executor.Close` releases native resources on idle expiry, environment +shutdown or confirmed invalidation. Core does not keep a second Executor cache. +The same lifecycle applies after managed or user-managed environments connect, +and to the qualified no-environment profiles. Resource management owns machine +selection, allocation and Environment creation/reclamation. Closing an Executor +does not release the Environment allocation or delete its workspace. Environment +reclamation explicitly coordinates with Runtime execution. Connection, installed +capability snapshot, Session Executor and Turn retain separate lifetimes. + +The Runtime binds its Executor record to Session, Environment, connection and +immutable execution configuration. Resume identity and prior-Turn recovery flags +are continuity assertions, not configuration changes. A supplied native identity +must match the retained owner; exact-history recovery never starts a new root +when existing history is required. A configuration conflict is an error, not a +hot switch. Lost connections retire their owners and handles. Old timers, output +and cancellation cannot affect replacements. + +Each Turn receives a fresh wrapper, output channel and receipt state. Optional +steering, functions, permissions and user-choice interfaces belong to that fixed +Turn. Native callbacks capture the originating Turn before asynchronous work; +late events cannot be assigned to whichever Turn happens to be active. Native +processes, query/transport connections, fixed capability configuration and native +Session identity belong to the Executor. Do not reset completed `sync.Once` +values or repurpose an old Turn object. + +`StartTurn` returning nil guarantees that no native input was submitted and the +output channel was not retained. The Runtime then closes that channel. Once input +may have been submitted, return a non-nil Turn even with an error: the Turn owns +exactly-once output closure and remains tracked until settlement. Unknown input +is never replayed. A definite `executor_unavailable` Start rejection permits one +common recovery attempt only after the previous Executor has been closed and no +input was submitted. Recheck the same physical peer and current authorization. + +`Turn.Cancel` targets only that Turn and does not close a healthy Executor. +`AwaitSettlement` applies after both natural completion and cancellation. Success +means output can no longer be written and the Turn's native events, input, +functions, interactions and child work have settled. `Reusable=true` additionally +confirms that the native owner can accept the next Turn. `Reusable=false` requires +a reason and subsequent confirmed Executor close. An error means settlement is +unconfirmed; it cannot free ownership or capacity. Caller deadlines stop waiting, +not tracked cleanup. Retry the same cleanup target serially. Failed cleanup +blocks replacement and retains its resource slot. + +One output consumer starts before native Start, drains the bounded 64-frame +channel, and retains the terminal observation until Start publication, Turn +settlement and admitted operation receipts finish. Natural Done never calls +Cancel. Input, function and interaction admission close before settlement, and +operations already admitted hold their barrier through native receipts and +outbound acknowledgement. Only then forward Done or an applied cancellation +receipt. Preserve the ten-second settlement wait and separate five-second receipt +send budget; timeout is not proof of quiescence. The observed cancellation outcome +retains native identity, Usage and output without fabricating missing evidence. + +Private preparation controls reserve a per-Turn admission, not a new Executor. +They carry an explicit Session identity and immutable configuration without model +input or Run identity. A fresh request returns a connection-local admission handle +and the owning Executor ID. Start supplies both identities and its actual Run ID +and ordered MessageInput. Per-admission revisions order status observations; +rejections describe control errors without inventing Run events. A reused healthy +Executor returns ready without native preparation. An admission release abandons +that admission; it does not close the Session's healthy idle Executor or cancel a +later Turn. Cancellation uses the exact Run identity. + +Preparations and Start execute outside the receive loop and Router lock. Admission +expires after five minutes; retries do not extend that deadline. Bound active +preparation and execution separately from idle retained resources, and count +closing or uncertain resources until cleanup succeeds. At most 64 admission +records are retained; old handles never consume replacement admissions. Idle expiry +is a Runtime resource policy, not Core active-Turn concurrency. Shutdown tracks and +closes active and idle Executors, retains failed close targets, and allows a later +serialized retry. Ordinary disconnection closes the failed transport and keeps +the exact Router until shutdown succeeds. A wait timeout or failed cleanup cannot +authorize reconnect; process shutdown also keeps waiting rather than silently +discarding owned native resources. These records are connection-local, not durable +input replay. + +Read-only workspace preparations remain separate bounded filesystem operations; +they cannot start model work. Workspace operations retain exact binding and +settlement rules across Turn boundaries and Executor closure. + +Successful input commits send a coalesced hint to the existing Worker scheduler. +The scheduler keeps lease, capacity, cursor fairness and per-Session ownership +checks; a hint does not admit work itself. If capacity is occupied, preserve one +rescan for completion without turning failed preparation into a busy retry loop. +HTTP readiness waits and active input delivery subscribe before reading relevant +state and wake after committed promotion or input. They recheck storage after each +hint. Polling remains the fallback for external writers, expiry and lost hints; +notifications contain no execution authority and no durable input data. + +Core readiness, Start acknowledgement and input-to-first-text logs use a single +process monotonic clock. Start acknowledgement confirms adapter ownership, not +model input consumption. +Runtime logs identify Executor creation, reuse, idle and close independently of +Turn completion. Do not call these durations model-only latency or subtract clocks +from different machines. Native process creation and same-owner successive Turns, +real provider results and same-condition timings must substantiate reuse claims. The Dispatcher prepares pending Environment input only on its exact enrolled or managed device. Preserve the same physical peer and preparation handle through @@ -2579,31 +2582,28 @@ replaced; do not carry obsolete compatibility code forward to satisfy this secti require native steering receipts. Commit terminal outcome and native Session ID together under the admission lock; unapplied messages prevent successful completion. Resolve credentials separately from the immutable non-secret snapshot. -- Internal execution requires the advertised `durable_turns` engine capability, - strict resume, and optional cancellation receipts - and `release_on_completion` support. Reject unadvertised peers before claiming; - failed strict resumes must not fall back to a new native thread. Release the native writer before forwarding - completion, so the next Turn can resume its durable native ID. For - release_on_completion Runs, close new steering admission and finish all - existing steering receipt sends before releasing the executor and forwarding - Done. Cached input identities and conflicts remain readable while completing; +- Internal execution requires advertised durable Turns, strict resume, preparation, + applied input receipts and the qualified operation capabilities. Reject + unadvertised peers before claiming; failed strict resume cannot start unrelated + history. A completed Turn closes steering admission, settles existing native + operations and receipt sends, and closes its output before the retained Executor + can accept another Turn. Normal completion does not cancel the Executor. + Cached input identities and conflicts remain readable while completing; queue/write success is not consumption. The receipt worker stays busy through its send, and router shutdown cancels its native and transport waits. - `durable_input_receipts` is required before execution binding/claiming. The - per-input `durable_receipt` opt-in requires release-on-completion and a phased - adapter. Its ten-second transport timer stops only after a complete native write; - a separate `written` acknowledgement stops the API's thirty-second delivery timer. - Neither that phase nor legacy `in_flight` advances the input cursor. Await final - native acceptance/consumption under the Run lifetime without automatic redelivery. - Receipt sends retain a separate five-second shutdown-aware context, and Done - retains a fifteen-second final settlement bound. Once cancellation is sent, its - receipt owns the terminal outcome even if an input becomes unknown first. - Calls without the opt-in retain their existing response deadlines. Existing product - requests retain their default idle-process policy. Native history still requires - the device's persisted engine files; IDs alone cannot restore deleted history. - Cancellation receipts carry the stopped engine's continuity snapshot when no - Done is emitted. Preserve separately reported usage on failure; do not add the - same counters again when Done also includes them. + The per-input `durable_receipt` opt-in requires a phased adapter. Its ten-second + transport timer stops only after a complete native write; a separate `written` + acknowledgement stops the API's thirty-second delivery timer. Neither phase + advances the input cursor. Await final native acceptance/consumption under the + Turn lifetime without automatic redelivery. Receipt sends retain a separate + five-second shutdown-aware context, and Done retains a fifteen-second final + settlement bound. Once cancellation is sent, its receipt owns the terminal + outcome even if an input becomes unknown first. Calls without the opt-in retain + their existing response deadlines. Native history still requires the device's + persisted engine files; IDs alone cannot restore deleted history. Cancellation + receipts carry the stopped Turn's confirmed continuity snapshot when no Done is + emitted. Preserve separately reported usage on failure; do not add the same + counters again when Done also includes them. Usage frames carry cumulative snapshots for the current execution, not deltas. Adapters publish observed snapshots promptly through the same ordered stream; waiting for Done unnecessarily loses known measurements if the Runtime stops. @@ -2983,14 +2983,14 @@ Owned output pipes remain readable after the leader exits. Consumers must drain stdout and stderr before calling `Wait`, which joins the cached process result and closes the readers. `Done` reports leader reaping and group cleanup signals; it is not a native execution receipt or proof of persisted history. SDK adapters -must close their query, await their native child and drain observations before -publishing completion. Process groups are lifecycle supervision, not OS isolation +must settle each Turn and drain its observations before publishing completion. +Executor close additionally closes the query and awaits the native child. Process groups are lifecycle supervision, not OS isolation or containment of descendants that deliberately leave the group. ### Harness qualification and onboarding Codex, Claude and future harnesses have equal architectural status. The common -Runtime wire protocol and Factory/Session/Prepared interfaces own lifecycle, +Runtime wire protocol and Executor/Turn interfaces own lifecycle, input receipts, cancellation, recovery and resource access; each native adapter retains its implementation and model/tool loop. A new engine supplies an adapter, a qualified profile in `services/agents-api/internal/engine`, registration and @@ -3038,9 +3038,12 @@ workspace, functions or MCP. Enabled Subagents use the separately qualified common observation path below. Native configuration disables file/shell authority and external capability discovery; the child receives a private Session home and a restricted -environment. Active-input application requires a native ACP receipt, cancellation -settles the process and output, and continuation requires the exact owned native -history. Do not infer history IDs or qualify hosted execution from this text +environment. Active-input application requires a native ACP receipt. Normal +Turns retain one ACP connection and native Session. Cancellation requires native +root and child completion evidence before reuse. Without a qualified root-state +reader, the ACP cancellation response is insufficient: stop the invalid native +process and wait for its exit before settling the Turn as non-reusable. Common +Runtime recovery then loads the exact owned native history in a new Executor. Do not infer history IDs or qualify hosted execution from this text profile. See [deployment and acceptance](services/agents-api/deploy/mcode/README.md). The MiniMax workspace profile builds one CLI from the fixed upstream source and @@ -3190,11 +3193,11 @@ qualification limits are recorded in [the coverage note](contracts/agents-api/st ### Claude SDK adapter foundation `packages/claude-sdk-adapter` privately owns the pinned official TypeScript SDK -and native message translation. The Go `claudesdk.NewFactory` uses the shared +and native message translation. The Go `claudesdk.NewExecutorFactory` uses the shared owned process runner and emits the existing daemon delta/error/Done frames. -The SDK owns the model loop. Its narrow stdio protocol carries a start request, -text deltas, function calls/results/receipts, active text input/receipts, usage -snapshots and one terminal result/error; native translation stays inside the adapter. +The SDK owns the model loop. Its narrow stdio protocol carries Executor preparation and identified Turn starts, +text deltas, function calls/results/receipts, active input/receipts, usage snapshots +and terminal result/error plus settlement; native translation stays inside the adapter. With `observe_messages`, it also emits the existing neutral `output_message` start/completion snapshots and tags deltas with the native Messages API message ID, not the SDK event UUID. Text blocks in one native message share that identity. @@ -3202,7 +3205,8 @@ The SDK's per-block assistant snapshots replace draft block text; only native `message_stop` completes the message, without replaying its text as another delta. Thinking/tool-only messages produce no text Items; interrupted messages retain their streamed partial text. No phase is inferred from the final result. -SDK/native child release and output draining precede daemon completion. +Turn-owned native work and output draining precede reuse. Executor close releases +the SDK Query and native process. `claudesdk.Config.Workspace` is a private, trusted operator binding for one qualified placement. It enables native Bash/Read/Edit and declared host functions @@ -3237,50 +3241,39 @@ public preparation, shared placement quotas, command Items and Files ownership. `TestLiveClaudeWorkspaceFactory` is explicit real-provider acceptance inside a qualified placement, including effects, cancellation and same-history continuation. -The private workspace bridge also accepts `prepare` without a prompt. It freezes +The private bridge accepts `executor_prepare` without model input. It freezes validated configuration and resume identity, checks required history, and retains -one native process through the pinned SDK's `startup`/`query` API. Preparation -requires initialization and acknowledgement of the required hooks while the input -iterator remains empty. Its `prepared` receipt permits one later `start` containing -only the initial prompt; configuration replacement, premature or duplicate start -is rejected. Native Session identity and actual tool inventory are still checked -at execution initialization before `input_ready`. Existing direct execution uses -the same observation and completion path. - -Unused EOF, owner signals, invalid control input and native exit release owned -resources before a terminal event. The private `workspace_prepare` runtime feature -identifies this bridge contract only. It does not register daemon preparation, -enable public admission, or project public Environment readiness. Preparation may -write native runtime metadata outside the workspace and perform startup traffic; -it does not prove provider authentication, complete sandbox health or tenant -placement authorization. - -`claudesdk.NewPreparationFactory` privately binds that workspace bridge to -`agent.Prepared`; the existing workspace factory uses the same Prepare/Start path. -Preparation receives configuration and required resume identity without a RunID or -prompt, checks the installed `workspace_prepare` feature, and owns the process until -one successful Start transfers it. The selected configuration and environment are -fixed before returning; a later Start supplies only its actual RunID, prompt and -output channel. Early preparation failure returns without an executing Session or -fabricated completion. The non-workspace direct factory keeps its existing path. - -The preparation owner context spans the eventual Session. Start's context bounds -that operation only, and Close is inert after successful transfer. Abandoned or -failed preparation, owner cancellation and native exit release owned work. The -same output consumer and cancellation settlement follow the resource across Start; -an unstarted preparation has no measured Usage or observed native Session identity. -A cancellation deadline cannot establish cleanup completion while cleanup remains -pending. This adapter ownership seam does not register a daemon capability or -supply per-Session placement authorization, public admission or an idle Files owner. - -The optional private `agent.WorkspaceReader` on this preparation and transferred -Session requires the packaged `workspace_read` feature. It sends bounded relative +one native process and SDK Query across Turns. Preparation requires initialization +and acknowledgement of required hooks while the input iterator remains empty. +An `executor_ready` receipt permits later `turn_start` messages containing only +Turn identity and ordered input; configuration replacement and concurrent starts +are rejected. Every Turn event carries its originating `turn_id`. Native Session +identity and actual tool inventory are checked before `input_ready`. + +Each Turn ends with a result/error and `turn_settled`, independently of process +exit. The outer input iterator remains open for later Turns. `turn_cancel` invokes +the native interrupt control for that exact Turn. Unconfirmed input, native child +work or queue state invalidates the Executor and requires close before replacement. +EOF, owner signals and invalid control input close owned resources. Preparation +may write native metadata and perform startup traffic; readiness does not prove +provider authentication, complete sandbox health or placement authorization. + +`claudesdk.NewExecutorFactory` binds this bridge to `agent.Executor`. Direct-call +and read-only preparation wrappers delegate to the same implementation. Runtime +execution uses the Executor registry for both none and workspace configurations. +Its owner context spans all Turns; a Turn's caller cannot replace fixed resources. +A failed preparation returns its Executor when cleanup remains unconfirmed. +Installed runtime checks are cached by package/file identity, while capability +and request validation still run for each Executor configuration. + +The optional private `agent.WorkspaceReader` on this Executor and its delegated wrappers +requires the packaged `workspace_read` feature. It sends bounded relative paths to that same SDK Query's native `readFile` control. Only the adapter combines the path with the frozen workspace root; callers cannot replace the placement. The pinned native read handler awaits file-handle close before its successful base64 response. The adapter validates bytes and truncation, bounds each result to 1 MiB and each request to 8 KiB, and admits one read at a time. Its continuous -bridge output consumer retains read receipts during preparation and across Start. +bridge output consumer retains read receipts during preparation and across Turns. Caller cancellation detaches observation without cancelling the Run or discarding an admitted waiter; its original deadline still applies. Owner closure stops admission. Native null, malformed receipts, timeout and interrupted delivery remain @@ -3293,7 +3286,7 @@ before input and during real execution, plus effects before cancellation and rea on fresh-process history continuation. The optional private `agent.WorkspaceDirectoryLister` requires the Linux-only -`workspace_directory` bridge feature on the same preparation or transferred Session. +`workspace_directory` bridge feature on the same Executor. The pinned SDK has no directory-enumeration control; its fuzzy file suggestions are not an inventory. This narrow adapter operation therefore reads metadata inside the already qualified co-located mount/process boundary. It pins the configured workspace @@ -3310,7 +3303,7 @@ ordering, snapshots, recursion, or public pagination. Missing and permission err are returned only from distinguishable filesystem outcomes; unknown results stop the owner. Each operation closes its directory and intermediate descriptors before a successful receipt; the root descriptor remains owned until bridge release. -Caller cancellation, Start transfer and owner shutdown retain the existing workspace +Caller cancellation, Turn transitions and owner shutdown retain the existing workspace read settlement rules. This adapter gap fill alone does not enable public Claude Files; the dedicated Runtime integration supplies public placement and ownership. @@ -3516,9 +3509,10 @@ also retain all snapshots in order under `claude_sdk_results`. Main-loop `usage` is per native turn, while query-pipeline `modelUsage` and estimated `total_cost_usd` are cumulative within the query. Retain subtype/error provenance and earlier snapshots even when a later failure reports zero counters. Reuse the latest full -snapshot set in Usage and Done; never sum cumulative measurements. Each factory -invocation owns one SDK query, including cold resume, so no prior query counters -are carried forward. Missing native results do not imply zero consumption. SDK estimates stay +snapshot set in Usage and Done; never sum cumulative measurements. Each Executor owns one SDK query, including cold resume. Raw snapshots explicitly +identify per-native-turn usage versus query-cumulative model usage and cost. A new +Turn has a fresh snapshot list, but query totals may include earlier Turns; never +represent those totals as consumption by the current Turn. Missing native results do not imply zero consumption. SDK estimates stay in raw evidence, outside the billed cost field; do not select an arbitrary model or invent missing public token breakdowns. The API does not parse native counters. Precise public usage projection, unreported costs and crash/partial accounting @@ -3534,18 +3528,19 @@ confirm applied input. Typed mid-turn folds may appear only on the native result Preserve that receipt even when the result reports failure. Check pending functions after the query drains: the SDK may dispatch later-turn callbacks before the earlier result handler finishes. -Keep the iterator open until every submitted input has a consuming result, even -when an earlier result reports an empty native queue. Close admission before -releasing final receipt waiters; drain and release the SDK/native processes before -one daemon Done. Cancellation resolves unconfirmed pending receipts as unknown and ends the -owned execution. Successful private SDK Cancel waits for owned-process exit and -stdout/stderr drain, then exposes the same settled CancellationOutcome as Done. -It retains native identity verified at input readiness, partial text and observed -Usage even on cancellation/failure; requested resume identity alone is not evidence. -A caller deadline before settlement reports failure/unknown, while cleanup continues. -Settlement precedes terminal publication so router completion cleanup cannot wait -on its own Done consumer. Cancellation releases intermediate event backpressure; -the settled outcome remains readable even when connection loss prevents publication. +Keep Turn input admission open until every submitted input has a consuming result, +even when an earlier result reports an empty native queue. Close admission before +releasing final receipt waiters. The outer SDK iterator remains open across Turns. +Cancellation resolves unconfirmed pending receipts as unknown and interrupts the +exact native Turn. Successful Cancel requires confirmed Turn settlement; process +exit alone cannot establish a successful cancellation. Reuse also requires an +empty confirmed native queue and settled child work. Otherwise close the Executor. +The settled CancellationOutcome retains verified native identity, partial text +and observed Usage; a requested resume identity alone is not evidence. Caller +wait expiry reports failure/unknown while cleanup retains ownership. Output +backpressure cannot turn missing native facts into confirmed settlement. Closed +Turn output precedes successful AwaitSettlement; the settled outcome remains +readable when connection loss prevents publication. A receipt timeout after a full write preserves the process and pending identity without redelivery; a blocked write is cancelled and released. The private adapter permits one input awaiting consumption and at most 63 extra diff --git a/README.md b/README.md index fb6bd0525..116f30d7a 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ native differences. | [API reference](docs/api/README.md) | The `/v1`, `/core/v1` and `/api/v1` namespaces | | [Nodes and sandbox backends: operator reference](services/agents-api/HOSTED-SANDBOX-MANAGER.md) | Node protocol, manual registration, placement, maintenance | | [Protocol coverage](contracts/agents-api/README.md) and [harness selection](contracts/agents-api/harness-selection.md) | Supported operations and native differences | +| [Add a Harness](contracts/agents-api/harness-onboarding.md) | Required adapter interfaces, lifecycle, capabilities, registration and acceptance | | [Landing page source](site/index.html) | The public site | ### Maintainers and advanced deployments diff --git a/apps/parsar-daemon/internal/agent/claudesdk/cancellation.go b/apps/parsar-daemon/internal/agent/claudesdk/cancellation.go index b2c432aa8..972e8af08 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/cancellation.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/cancellation.go @@ -2,28 +2,51 @@ package claudesdk import ( "context" + "errors" + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" ) -// Cancel confirms process exit and output drain, independently of terminal delivery. +// Cancel addresses this fixed Turn. Settlement remains observable after the caller detaches. func (s *session) Cancel(ctx context.Context) error { if ctx == nil { ctx = context.Background() } select { case <-s.settled: - return nil + return s.settlementErr default: } - s.process.Cancel() + if err := ctx.Err(); err != nil { + return err + } + if s.owner == nil { + return errors.New("claudesdk: Turn owner is unavailable") + } + s.cancelOnce.Do(func() { + close(s.cancelOutput) + s.owner.mu.Lock() + current := s.owner.active == s && !s.owner.invalid + s.owner.mu.Unlock() + if current { + stopWrite := context.AfterFunc(ctx, s.invalidate) + defer stopWrite() + if err := s.owner.write(struct { + Type string `json:"type"` + TurnID string `json:"turn_id"` + }{"turn_cancel", s.runID}); err != nil { + s.invalidate() + } + } + }) select { case <-s.settled: - return nil + return s.settlementErr case <-ctx.Done(): select { case <-s.settled: - return nil + return s.settlementErr default: return ctx.Err() } @@ -40,3 +63,5 @@ func (s *session) CancellationOutcome() proto.DonePayload { return proto.DonePayload{} } } + +var _ agent.Turn = (*session)(nil) diff --git a/apps/parsar-daemon/internal/agent/claudesdk/cancellation_test.go b/apps/parsar-daemon/internal/agent/claudesdk/cancellation_test.go index 0f94f90ad..b8e64ed3a 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/cancellation_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/cancellation_test.go @@ -3,16 +3,15 @@ package claudesdk import ( + "bufio" "bytes" "context" "encoding/json" "errors" "os" - "os/signal" "path/filepath" "reflect" "strings" - "syscall" "testing" "time" @@ -171,12 +170,16 @@ func cancellationRequest() proto.PromptRequestPayload { return proto.PromptRequestPayload{RunID: "run", Input: proto.TextInput("hello"), AgentSessionID: "native-session", AgentOptions: map[string]any{"model": "fake-model", "system_prompt": "instructions"}} } -func runCancellationHelper(request startRequest, mode string, emit func(bridgeEvent)) { +func runCancellationHelper(request startRequest, mode string, scanner *bufio.Scanner, emit func(bridgeEvent)) { if mode == "cancellation-wait" { - stopped := make(chan os.Signal, 1) - signal.Notify(stopped, syscall.SIGTERM) emit(bridgeEvent{Type: "delta", Delta: "partial"}) - <-stopped + if !scanner.Scan() { + return + } + var cancel map[string]any + if json.Unmarshal(scanner.Bytes(), &cancel) != nil || cancel["type"] != "turn_cancel" { + os.Exit(9) + } // These valid observations were in flight when cancellation started. emit(bridgeEvent{Type: "input_ready", SessionID: request.Resume}) emit(bridgeEvent{Type: "usage", ResultID: "native-result", SessionID: request.Resume, Usage: json.RawMessage(usageFixture)}) @@ -189,6 +192,8 @@ func runCancellationHelper(request startRequest, mode string, emit func(bridgeEv } _, _ = os.Stderr.WriteString(strings.Repeat("x", 2*1024*1024)) emit(bridgeEvent{Type: "delta", Delta: "drained"}) + emit(bridgeEvent{Type: "input_closed", SessionID: request.Resume}) + emit(bridgeEvent{Type: "error", Code: "cancelled"}) return } if mode != "cancellation-before-identity" { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/commands_session_test.go b/apps/parsar-daemon/internal/agent/claudesdk/commands_session_test.go index f87ed4969..5a2d6e41b 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/commands_session_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/commands_session_test.go @@ -3,10 +3,10 @@ package claudesdk import ( + "bufio" "context" "encoding/json" "os" - "os/signal" "path/filepath" "strings" "syscall" @@ -155,13 +155,7 @@ func TestWorkspaceCommandCancellationAndBridgeFailuresCloseOnlyPendingCalls(t *t } } -func runCommandsHelper(request startRequest, mode string, emit func(bridgeEvent)) { - var stopping chan os.Signal - if mode == "commands-cancel" || mode == "commands-drained" { - stopping = make(chan os.Signal, 1) - signal.Notify(stopping, syscall.SIGTERM) - defer signal.Stop(stopping) - } +func runCommandsHelper(request startRequest, mode string, scanner *bufio.Scanner, emit func(bridgeEvent)) { if mode != "commands-before-ready" { emit(bridgeEvent{Type: "input_ready", SessionID: request.Resume}) } @@ -181,7 +175,9 @@ func runCommandsHelper(request startRequest, mode string, emit func(bridgeEvent) } switch mode { case "commands-cancel", "commands-drained": - <-stopping + if !scanner.Scan() { + return + } if mode == "commands-drained" { interrupted := commandEvent("pending", "after", "incomplete", "sleep 30") interrupted.Observation.Output = json.RawMessage(`"observed while draining"`) diff --git a/apps/parsar-daemon/internal/agent/claudesdk/executor.go b/apps/parsar-daemon/internal/agent/claudesdk/executor.go new file mode 100644 index 000000000..0abc0ca47 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/claudesdk/executor.go @@ -0,0 +1,314 @@ +package claudesdk + +import ( + "context" + "encoding/json" + "errors" + "io" + "slices" + "strings" + "sync" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +type executor struct { + mu sync.Mutex + base *session + start startRequest + active *session + ready chan error + done chan struct{} + invalid bool + nativeID string +} + +func NewExecutorFactory(config Config) agent.ExecutorFactory { + config.Env = slices.Clone(config.Env) + if config.Workspace != nil { + workspace := *config.Workspace + workspace.ProtectedDirs = slices.Clone(workspace.ProtectedDirs) + config.Workspace = &workspace + } + checked := &runtimeCheckCache{} + return func(ctx context.Context, req proto.PromptRequestPayload) (agent.Executor, error) { + if ctx == nil { + ctx = context.Background() + } + if req.RunID != "" || len(req.Input) != 0 || req.ConversationID != "" { + return nil, errors.New("claudesdk: Executor preparation cannot submit input") + } + start, env, err := prepareConfiguration(config, req) + if err != nil { + return nil, err + } + info, err := checked.check(ctx, config) + if err != nil { + return nil, err + } + if err = validateExecutorFeatures(info, start); err != nil { + return nil, err + } + start.Type = "executor_prepare" + base, err := launch(ctx, config, start, env) + if err != nil { + return nil, err + } + base.reads.supported = slices.Contains(info.Features, "workspace_read") + base.directories.supported = slices.Contains(info.Features, "workspace_directory") + e := &executor{base: base, start: start, ready: make(chan error, 1), done: make(chan struct{}), nativeID: start.Resume} + go e.read() + if err = e.write(start); err == nil { + select { + case err = <-e.ready: + case <-ctx.Done(): + err = ctx.Err() + } + } + if err != nil { + closeCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if closeErr := e.Close(closeCtx); closeErr != nil { + return e, errors.Join(err, closeErr) + } + return nil, err + } + return e, nil + } +} + +func validateExecutorFeatures(info RuntimeInfo, start startRequest) error { + if info.Protocol != 3 { + return errors.New("claudesdk: Executor bridge protocol is unavailable") + } + if start.Workspace != nil && !info.supportsWorkspacePreparation() { + return errors.New("claudesdk: workspace preparation is unavailable") + } + if start.OutputFormat != nil && (!info.SupportsStructuredOutput() || start.Workspace != nil && !info.SupportsWorkspaceStructuredOutput()) { + return errors.New("claudesdk: packaged runtime does not support workspace structured output") + } + if start.Subagents != nil && !info.SupportsSubagents() { + return errors.New("claudesdk: subagent resources are unavailable") + } + if start.Workspace != nil && start.observeFunctions && !info.supportsWorkspaceCommands() { + return errors.New("claudesdk: packaged runtime does not support workspace command observations") + } + if start.Workspace != nil && len(start.Functions) > 0 && !info.SupportsWorkspaceFunctions() { + return errors.New("claudesdk: workspace functions are unavailable") + } + if start.MCPHTTPServers != nil { + if !info.SupportsHTTPMCP() { + return errors.New("claudesdk: packaged runtime does not support HTTP MCP") + } + for _, server := range *start.MCPHTTPServers { + if server.Required && !info.SupportsHTTPMCPRequired() { + return errors.New("claudesdk: packaged runtime does not support required HTTP MCP") + } + if server.BearerTokenEnvVar != "" && !info.SupportsHTTPMCPBearer() { + return errors.New("claudesdk: packaged runtime does not support authenticated HTTP MCP") + } + } + } + return nil +} + +func (e *executor) write(value any) error { + e.base.writeMu.Lock() + defer e.base.writeMu.Unlock() + return json.NewEncoder(e.base.process.Stdin).Encode(value) +} + +func (e *executor) read() { + stderrDone := make(chan struct{}) + go func() { _, _ = io.Copy(io.Discard, e.base.process.Stderr); close(stderrDone) }() + scanner := e.base.bridgeOutput() + ready := false + readyFailure := errors.New("claudesdk: Executor readiness failed") + for scanner.Scan() { + raw := append([]byte(nil), scanner.Bytes()...) + var event bridgeEvent + if json.Unmarshal(raw, &event) != nil { + e.base.process.Cancel() + break + } + if !ready { + if event.Type != "executor_ready" || event.Protocol != 3 || event.TurnID != "" { + if event.Type == "error" { + readyFailure = bridgeFailure(event.Code) + } + e.base.process.Cancel() + break + } + ready = true + e.ready <- nil + continue + } + e.mu.Lock() + turn := e.active + valid := turn != nil && !turn.nativeEnded && event.TurnID == turn.runID + if valid && event.Type == "turn_settled" { + turn.nativeEnded = true + } + e.mu.Unlock() + if !valid { + e.base.process.Cancel() + break + } + select { + case turn.frames <- raw: + case <-e.base.process.Context().Done(): + } + } + e.base.process.Cancel() + for scanner.Scan() { + } + <-stderrDone + _ = e.base.process.Wait() + e.base.stopWorkspaceReads() + e.base.stopWorkspaceDirectories() + e.mu.Lock() + e.invalid = true + if e.active != nil { + close(e.active.frames) + } + e.mu.Unlock() + if !ready { + e.ready <- readyFailure + } + close(e.done) +} + +func (e *executor) StartTurn(ctx context.Context, run string, input proto.MessageInput, out chan<- proto.Envelope) (agent.Turn, error) { + if ctx == nil { + ctx = context.Background() + } + if strings.TrimSpace(run) == "" || input.Validate() != nil || out == nil || ctx.Err() != nil { + return nil, errors.New("claudesdk: Turn requires live context, identity, input and output") + } + e.mu.Lock() + if e.invalid || e.active != nil || e.base.process.Context().Err() != nil { + e.mu.Unlock() + return nil, errors.New("claudesdk: Executor is unavailable") + } + s := &session{owner: e, runID: run, process: e.base.process, writeMu: e.base.writeMu, frames: make(chan []byte, 64), functions: functionState{calls: map[string]*pendingFunction{}}, settled: make(chan struct{}), outputDone: make(chan struct{}), cancelOutput: make(chan struct{})} + start := e.start + start.Resume = e.nativeID + e.active = s + e.mu.Unlock() + go s.runTurn(start, out) + // Once the write is attempted, a lost receipt has an unknown input outcome. + stopWrite := context.AfterFunc(ctx, s.invalidate) + err := e.write(struct { + Type string `json:"type"` + TurnID string `json:"turn_id"` + Input proto.MessageInput `json:"input"` + }{"turn_start", run, input}) + stopWrite() + if err != nil { + s.invalidate() + return s, errors.New("claudesdk: Turn input delivery is unknown") + } + return s, nil +} + +func (e *executor) finishTurn(s *session) { + e.mu.Lock() + if e.active != s { + e.mu.Unlock() + return + } + if native, _ := s.outcome.Metadata[proto.DoneMetaAgentSessionID].(string); native != "" { + if e.nativeID != "" && e.nativeID != native { + e.invalid = true + } else { + e.nativeID = native + } + } + if !s.turnSettlement.Reusable { + e.invalid = true + } + invalid := e.invalid + if invalid { + s.turnSettlement.Reusable = false + if s.turnSettlement.Reason == "" { + s.turnSettlement.Reason = "executor_invalidated" + } + } + e.active = nil + e.mu.Unlock() + if invalid { + e.base.process.Cancel() + <-e.done + } +} + +func (e *executor) retire() { + e.mu.Lock() + e.invalid = true + e.mu.Unlock() + e.base.process.Cancel() + <-e.done +} + +func (e *executor) Close(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + e.mu.Lock() + e.invalid = true + active := e.active + e.mu.Unlock() + e.base.process.Cancel() + select { + case <-e.done: + if active != nil { + select { + case <-active.outputDone: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *session) AwaitSettlement(ctx context.Context) (agent.TurnSettlement, error) { + if ctx == nil { + ctx = context.Background() + } + select { + case <-s.outputDone: + return s.turnSettlement, errors.Join(s.settlementErr, s.outputErr) + case <-ctx.Done(): + return agent.TurnSettlement{}, ctx.Err() + } +} + +var _ agent.Executor = (*executor)(nil) +var _ agent.Turn = (*session)(nil) + +// A timed-out operation retains its original Turn identity. Its callback cannot +// retire a healthy successor after that operation has otherwise settled. +func (s *session) invalidate() { + if s.owner == nil { + s.process.Cancel() + return + } + s.owner.mu.Lock() + defer s.owner.mu.Unlock() + if s.owner.active == s { + s.owner.invalid = true + s.process.Cancel() + } +} + +func (e *executor) ReadWorkspaceFile(ctx context.Context, path string, maxBytes int) (agent.WorkspaceReadResult, error) { + return e.base.ReadWorkspaceFile(ctx, path, maxBytes) +} +func (e *executor) ListWorkspaceDirectory(ctx context.Context, path string, maxEntries int) (agent.WorkspaceDirectoryResult, error) { + return e.base.ListWorkspaceDirectory(ctx, path, maxEntries) +} diff --git a/apps/parsar-daemon/internal/agent/claudesdk/executor_fixture_test.go b/apps/parsar-daemon/internal/agent/claudesdk/executor_fixture_test.go new file mode 100644 index 000000000..5b2c9244d --- /dev/null +++ b/apps/parsar-daemon/internal/agent/claudesdk/executor_fixture_test.go @@ -0,0 +1,53 @@ +//go:build unix + +package claudesdk + +import ( + "bufio" + "encoding/json" + "os" +) + +func helperTurn(scanner *bufio.Scanner, request *startRequest) (func(bridgeEvent), func()) { + _ = json.NewEncoder(os.Stdout).Encode(bridgeEvent{Type: "executor_ready", Protocol: 3}) + if !scanner.Scan() { + os.Exit(4) + } + var start struct { + Type string `json:"type"` + TurnID string `json:"turn_id"` + Input json.RawMessage `json:"input"` + } + if json.Unmarshal(scanner.Bytes(), &start) != nil || start.Type != "turn_start" || start.TurnID == "" || json.Unmarshal(start.Input, &request.Input) != nil { + os.Exit(5) + } + return helperTurnOutput(scanner, start.TurnID) +} + +func helperTurnOutput(scanner *bufio.Scanner, id string) (func(bridgeEvent), func()) { + terminal := false + reusable := true + encode := func(event bridgeEvent) { + event.TurnID = id + _ = json.NewEncoder(os.Stdout).Encode(event) + if event.Type == "result" || event.Type == "error" { + terminal = true + if event.Type == "error" && event.Code != "cancelled" { + reusable = false + } + } + } + encode(bridgeEvent{Type: "turn_started"}) + return encode, func() { + if !terminal { + return + } + reason := "" + if !reusable { + reason = "native_error" + } + encode(bridgeEvent{Type: "turn_settled", Reusable: &reusable, Reason: reason}) + for scanner.Scan() { + } + } +} diff --git a/apps/parsar-daemon/internal/agent/claudesdk/executor_live_linux_test.go b/apps/parsar-daemon/internal/agent/claudesdk/executor_live_linux_test.go new file mode 100644 index 000000000..a2ac5f536 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/claudesdk/executor_live_linux_test.go @@ -0,0 +1,199 @@ +//go:build linux + +package claudesdk + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/google/uuid" +) + +// Explicit opt-in acceptance; normal test runs never contact a provider. +func TestLiveClaudeExecutorReuseAndCancel(t *testing.T) { + entry, keyPath, proof := os.Getenv("OAC_TEST_CLAUDE_EXECUTOR_ENTRYPOINT"), os.Getenv("OAC_TEST_CLAUDE_EXECUTOR_KEY_FILE"), os.Getenv("OAC_TEST_CLAUDE_EXECUTOR_PROOF_DIR") + if entry == "" || keyPath == "" || proof == "" { + t.Skip("explicit installed runtime, private key file and proof directory required") + } + if !filepath.IsAbs(proof) { + t.Fatal("absolute proof directory required") + } + if err := os.MkdirAll(proof, 0700); err != nil { + t.Fatal(err) + } + key, err := os.ReadFile(keyPath) + if err != nil { + t.Fatal("cannot read selected credential file") + } + t.Setenv("OAC_RUNTIME_HOME", proof) + config := Config{Node: os.Getenv("OAC_TEST_CLAUDE_EXECUTOR_NODE"), Entrypoint: entry, StateDir: filepath.Join(proof, "state"), Env: []string{ + "ANTHROPIC_BASE_URL=https://api.moonshot.cn/anthropic", "ANTHROPIC_AUTH_TOKEN=" + strings.TrimSpace(string(key)), "ANTHROPIC_API_KEY=", "CLAUDE_CODE_OAUTH_TOKEN=", "CLAUDE_CODE_DISABLE_EXPERIMENTAL_BETAS=1", + "ANTHROPIC_DEFAULT_SONNET_MODEL=kimi-k3", "ANTHROPIC_DEFAULT_OPUS_MODEL=kimi-k3", "ANTHROPIC_DEFAULT_HAIKU_MODEL=kimi-k3", + }} + if proxy := os.Getenv("OAC_TEST_CLAUDE_EXECUTOR_HTTP_PROXY"); proxy != "" { + config.Env = append(config.Env, "HTTP_PROXY="+proxy, "HTTPS_PROXY="+proxy, "http_proxy="+proxy, "https_proxy="+proxy) + } + ctx, cancel := context.WithTimeout(t.Context(), 4*time.Minute) + defer cancel() + readiness, err := CheckRuntime(ctx, config) + if err != nil { + t.Fatal("installed runtime readiness failed", err) + } + type turnEvidence struct { + Run string `json:"run"` + NodePID int `json:"node_pid"` + NativePIDs []int `json:"native_pids"` + FirstTextMS int64 `json:"first_text_ms"` + ElapsedMS int64 `json:"elapsed_ms"` + CancelMS int64 `json:"cancel_ms,omitempty"` + Cancelled bool `json:"cancelled"` + CancellationError string `json:"cancellation_error,omitempty"` + Settlement agent.TurnSettlement `json:"settlement"` + SettlementError string `json:"settlement_error,omitempty"` + Done proto.DonePayload `json:"done"` + Errors []string `json:"errors,omitempty"` + } + evidence := struct { + Runtime RuntimeInfo `json:"runtime"` + PrepareMS int64 `json:"prepare_ms"` + Turns []turnEvidence `json:"turns"` + Recovered bool `json:"recovered_after_cancel"` + Closed bool `json:"closed"` + }{Runtime: readiness} + persist := func() { + raw, _ := json.MarshalIndent(evidence, "", " ") + _ = os.WriteFile(filepath.Join(proof, "executor-evidence.json"), raw, 0600) + } + defer persist() + request := proto.PromptRequestPayload{StrictResume: true, DisableExecutionEnvironment: true, DisableSubagents: true, ObserveMessages: true, ExecutionControls: &proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium"}, AgentOptions: map[string]any{"model": "kimi-k3", "system_prompt": "Follow requested formats briefly. Remember the exact verification marker across the conversation. Use no tools."}} + factory := NewExecutorFactory(config) + prepared := time.Now() + owner, err := factory(ctx, request) + if err != nil { + t.Fatal("executor preparation failed", err) + } + evidence.PrepareMS = time.Since(prepared).Milliseconds() + defer func() { + closeCtx, stop := context.WithTimeout(context.Background(), 10*time.Second) + defer stop() + evidence.Closed = owner.Close(closeCtx) == nil + persist() + }() + nativeChildren := func(pid int) []int { + raw, _ := os.ReadFile(fmt.Sprintf("/proc/%d/task/%d/children", pid, pid)) + var ids []int + for _, value := range strings.Fields(string(raw)) { + child, _ := strconv.Atoi(value) + args, _ := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", child)) + if bytes.Contains(args, []byte("\x00--input-format\x00stream-json\x00")) && bytes.Contains(args, []byte("\x00--output-format\x00stream-json\x00")) { + ids = append(ids, child) + } + } + slices.Sort(ids) + return ids + } + run := func(id, prompt string, cancelOnText bool) turnEvidence { + t.Helper() + started := time.Now() + record := turnEvidence{Run: id, NodePID: owner.(*executor).base.process.Cmd.Process.Pid} + output := make(chan proto.Envelope, 128) + turn, err := owner.StartTurn(ctx, id, proto.TextInput(prompt), output) + if err != nil || turn == nil { + t.Fatal("Turn start failed", err) + } + cancellation := make(chan error, 1) + var cancelAt time.Time + for event := range output { + if event.ID != id { + t.Fatal("output crossed Turn identity") + } + if event.Type == proto.TypeDelta && record.FirstTextMS == 0 { + record.FirstTextMS = time.Since(started).Milliseconds() + record.NativePIDs = nativeChildren(record.NodePID) + if cancelOnText { + record.Cancelled = true + cancelAt = time.Now() + go func() { + cancelCtx, stop := context.WithTimeout(ctx, 30*time.Second) + defer stop() + cancellation <- turn.Cancel(cancelCtx) + }() + } + } + if event.Type == proto.TypeError { + var failure proto.ErrorPayload + _ = event.DecodePayload(&failure) + record.Errors = append(record.Errors, failure.Error) + } + if event.Type == proto.TypeDone { + _ = event.DecodePayload(&record.Done) + } + } + if record.Cancelled { + err := <-cancellation + record.CancelMS = time.Since(cancelAt).Milliseconds() + if err != nil { + record.CancellationError = err.Error() + } + } + record.Settlement, err = turn.AwaitSettlement(ctx) + if err != nil { + record.SettlementError = err.Error() + } + record.ElapsedMS = time.Since(started).Milliseconds() + evidence.Turns = append(evidence.Turns, record) + persist() + t.Logf("turn=%s bridge_pid=%d native_pids=%v reusable=%t first_text_ms=%d total_ms=%d", id, record.NodePID, record.NativePIDs, record.Settlement.Reusable, record.FirstTextMS, record.ElapsedMS) + return record + } + marker := "REUSE-" + uuid.NewString() + first := run("first", "Remember this marker: "+marker+". Reply with exactly the marker.", false) + if first.SettlementError != "" || !first.Settlement.Reusable || !strings.Contains(first.Done.Content, marker) { + t.Fatal("first Turn failed or was not reusable") + } + second := run("second", "What exact marker did I give you? Reply with only that marker.", false) + if second.SettlementError != "" || !second.Settlement.Reusable || !strings.Contains(second.Done.Content, marker) || first.NodePID != second.NodePID || len(first.NativePIDs) == 0 || !slices.Equal(first.NativePIDs, second.NativePIDs) { + t.Fatal("ordinary Turns did not retain native execution and history") + } + interrupted := run("cancel", "List the numbers 1 through 10000, one number per line, without stopping early.", true) + if !interrupted.Cancelled { + t.Fatal("native output ended before cancellation boundary") + } + if interrupted.SettlementError != "" || !interrupted.Settlement.Reusable { + if interrupted.Settlement.Reason == "" && interrupted.SettlementError == "" { + t.Fatal("invalid executor omitted its reason") + } + if err := owner.Close(ctx); err != nil { + t.Fatal("invalid owner cleanup was not confirmed", err) + } + native, _ := interrupted.Done.Metadata[proto.DoneMetaAgentSessionID].(string) + if native == "" { + native, _ = second.Done.Metadata[proto.DoneMetaAgentSessionID].(string) + } + request.AgentSessionID = native + request.RequireExistingNativeSession = true + owner, err = factory(ctx, request) + if err != nil { + t.Fatal("history recovery failed", err) + } + evidence.Recovered = true + } + continued := run("continued", "What exact marker did I originally give you? Reply with only the marker.", false) + if continued.SettlementError != "" || !continued.Settlement.Reusable || !strings.Contains(continued.Done.Content, marker) { + t.Fatal("history did not continue after cancellation") + } + if !evidence.Recovered && (continued.NodePID != second.NodePID || !slices.Equal(continued.NativePIDs, second.NativePIDs)) { + t.Fatal("reusable cancellation replaced native execution") + } +} diff --git a/apps/parsar-daemon/internal/agent/claudesdk/executor_test.go b/apps/parsar-daemon/internal/agent/claudesdk/executor_test.go new file mode 100644 index 000000000..755a0bc53 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/claudesdk/executor_test.go @@ -0,0 +1,285 @@ +//go:build unix + +package claudesdk + +import ( + "bufio" + "context" + "encoding/json" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +func persistentConfig(t *testing.T, mode string) (Config, proto.PromptRequestPayload) { + t.Helper() + root := t.TempDir() + t.Setenv("OAC_RUNTIME_HOME", root) + entry := filepath.Join(root, "worker") + if err := os.WriteFile(entry, []byte("version-one"), 0600); err != nil { + t.Fatal(err) + } + return Config{Node: os.Args[0], Entrypoint: entry, StateDir: filepath.Join(root, "state"), Env: []string{"GO_CLAUDE_EXECUTOR_HELPER=1", "SDK_EXECUTOR_DIR=" + root, "SDK_EXECUTOR_MODE=" + mode, "GORACE=atexit_sleep_ms=0"}}, proto.PromptRequestPayload{DisableExecutionEnvironment: true, AgentOptions: map[string]any{"model": "fixture"}} +} + +func runPersistentExecutorHelper() { + root := os.Getenv("SDK_EXECUTOR_DIR") + if len(os.Args) > 1 && strings.HasSuffix(os.Args[1], "runtime_check.js") { + file, _ := os.OpenFile(filepath.Join(root, "probes"), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if file != nil { + _, _ = file.WriteString("probe\n") + _ = file.Close() + } + printlnReport := json.NewEncoder(os.Stdout) + _ = printlnReport.Encode(RuntimeInfo{Type: "runtime_ready", Protocol: 3, Node: "fixture", SDK: "fixture", MCP: "fixture", Native: "fixture"}) + return + } + scanner := bufio.NewScanner(os.Stdin) + if !scanner.Scan() { + return + } + var prepare map[string]json.RawMessage + if json.Unmarshal(scanner.Bytes(), &prepare) != nil || string(prepare["type"]) != `"executor_prepare"` || prepare["input"] != nil || prepare["run_id"] != nil { + os.Exit(4) + } + _ = os.WriteFile(filepath.Join(root, "prepared"), []byte(strconv.Itoa(os.Getpid())), 0600) + encode := func(event bridgeEvent) { _ = json.NewEncoder(os.Stdout).Encode(event) } + encode(bridgeEvent{Type: "executor_ready", Protocol: 3}) + if os.Getenv("SDK_EXECUTOR_MODE") == "block" { + time.Sleep(time.Hour) + return + } + var active, old string + settle := func(cancel bool) { + encode(bridgeEvent{Type: "input_closed", TurnID: active, SessionID: "native-persistent"}) + if cancel { + encode(bridgeEvent{Type: "error", TurnID: active, Code: "cancelled"}) + } else { + encode(bridgeEvent{Type: "result", TurnID: active, SessionID: "native-persistent", Text: active}) + } + yes := true + encode(bridgeEvent{Type: "turn_settled", TurnID: active, Reusable: &yes}) + old, active = active, "" + } + for scanner.Scan() { + var command struct { + Type string `json:"type"` + TurnID string `json:"turn_id"` + Input proto.MessageInput `json:"input"` + } + if json.Unmarshal(scanner.Bytes(), &command) != nil { + return + } + switch command.Type { + case "turn_start": + if active != "" { + os.Exit(5) + } + active = command.TurnID + if os.Getenv("SDK_EXECUTOR_MODE") == "late" && old != "" { + encode(bridgeEvent{Type: "delta", TurnID: old, Delta: "late"}) + continue + } + encode(bridgeEvent{Type: "turn_started", TurnID: active}) + encode(bridgeEvent{Type: "input_ready", TurnID: active, SessionID: "native-persistent"}) + encode(bridgeEvent{Type: "delta", TurnID: active, Delta: "partial"}) + if len(command.Input) > 0 && len(command.Input[0].Content) > 0 && command.Input[0].Content[0].Text != nil && *command.Input[0].Content[0].Text == "wait" { + continue + } + settle(false) + case "turn_cancel": + if active == command.TurnID { + settle(true) + } + } + } +} + +func consumeExecutorTurn(t *testing.T, owner agent.Executor, id, input string) (agent.Turn, <-chan proto.Envelope) { + t.Helper() + out := make(chan proto.Envelope, 16) + turn, err := owner.StartTurn(t.Context(), id, proto.TextInput(input), out) + if err != nil || turn == nil { + t.Fatalf("StartTurn: %v", err) + } + return turn, out +} +func awaitExecutorTurn(t *testing.T, turn agent.Turn, out <-chan proto.Envelope, reusable bool) { + t.Helper() + for event := range out { + if event.Type == proto.TypeDone { + var done proto.DonePayload + _ = event.DecodePayload(&done) + if done.Metadata[proto.DoneMetaAgentSessionID] != "native-persistent" { + t.Fatal("native continuity missing", done) + } + } + } + settlement, err := turn.AwaitSettlement(t.Context()) + if err != nil || settlement.Reusable != reusable { + t.Fatalf("settlement: %+v %v", settlement, err) + } +} +func TestExecutorRetainsProcessAcrossTurnsAndCancellation(t *testing.T) { + config, req := persistentConfig(t, "") + owner, err := NewExecutorFactory(config)(t.Context(), req) + if err != nil { + t.Fatal(err) + } + defer owner.Close(context.Background()) + pid := owner.(*executor).base.process.Cmd.Process.Pid + first, out := consumeExecutorTurn(t, owner, "first", "hello") + awaitExecutorTurn(t, first, out, true) + next, out := consumeExecutorTurn(t, owner, "next", "wait") + if event := <-out; event.Type != proto.TypeDelta { + t.Fatal(event.Type) + } + if err := first.Cancel(t.Context()); err != nil { + t.Fatal(err) + } + if err := next.Cancel(t.Context()); err != nil { + t.Fatal(err) + } + awaitExecutorTurn(t, next, out, true) + third, out := consumeExecutorTurn(t, owner, "third", "hello") + awaitExecutorTurn(t, third, out, true) + if owner.(*executor).base.process.Cmd.Process.Pid != pid { + t.Fatal("native process was replaced") + } + select { + case <-owner.(*executor).base.process.Done(): + t.Fatal("native process exited between Turns") + default: + } +} +func TestExecutorLateTurnEventInvalidatesWithoutRetargeting(t *testing.T) { + config, req := persistentConfig(t, "late") + owner, err := NewExecutorFactory(config)(t.Context(), req) + if err != nil { + t.Fatal(err) + } + defer owner.Close(context.Background()) + first, out := consumeExecutorTurn(t, owner, "first", "hello") + awaitExecutorTurn(t, first, out, true) + second, out := consumeExecutorTurn(t, owner, "second", "hello") + for frame := range out { + if frame.Type == proto.TypeDelta { + t.Fatal("old Turn event reached successor") + } + } + if _, err := second.AwaitSettlement(t.Context()); err == nil { + t.Fatal("unknown native outcome marked settled") + } + next := make(chan proto.Envelope, 1) + if turn, err := owner.StartTurn(t.Context(), "third", proto.TextInput("hello"), next); turn != nil || err == nil { + t.Fatal("invalid executor accepted input") + } +} +func TestExecutorCachesReadinessUntilInstalledArtifactChanges(t *testing.T) { + config, req := persistentConfig(t, "") + factory := NewExecutorFactory(config) + for range 2 { + owner, err := factory(t.Context(), req) + if err != nil { + t.Fatal(err) + } + if err := owner.Close(t.Context()); err != nil { + t.Fatal(err) + } + } + count := func() int { + raw, _ := os.ReadFile(filepath.Join(filepath.Dir(config.Entrypoint), "probes")) + return strings.Count(string(raw), "probe\n") + } + if count() != 1 { + t.Fatal("unchanged registered runtime was reprobed") + } + time.Sleep(time.Millisecond) + if err := os.WriteFile(config.Entrypoint, []byte("version-two-changed"), 0600); err != nil { + t.Fatal(err) + } + owner, err := factory(t.Context(), req) + if err != nil { + t.Fatal(err) + } + defer owner.Close(context.Background()) + if count() != 2 { + t.Fatal("changed installation reused stale readiness") + } +} + +func TestExecutorSeparatesPreInputRejectionFromUnknownWrite(t *testing.T) { + config, req := persistentConfig(t, "block") + owner, err := NewExecutorFactory(config)(t.Context(), req) + if err != nil { + t.Fatal(err) + } + defer owner.Close(context.Background()) + rejected := make(chan proto.Envelope, 1) + cancelled, stop := context.WithCancel(t.Context()) + stop() + if turn, err := owner.StartTurn(cancelled, "not-written", proto.TextInput("hello"), rejected); turn != nil || err == nil { + t.Fatal("cancelled admission reached native input") + } + select { + case <-rejected: + t.Fatal("nil Turn consumed caller output") + default: + } + e := owner.(*executor) + if err := e.base.process.Stdin.Close(); err != nil { + t.Fatal(err) + } + out := make(chan proto.Envelope, 8) + turn, err := owner.StartTurn(t.Context(), "unknown-write", proto.TextInput("hello"), out) + if turn == nil || err == nil { + t.Fatalf("attempted write must preserve unknown Turn ownership: %T %v", turn, err) + } + for range out { + } + if _, err := turn.AwaitSettlement(t.Context()); err == nil { + t.Fatal("lost input outcome became confirmed") + } +} + +func TestExecutorCancellationDeadlineInterruptsBlockedTransport(t *testing.T) { + config, req := persistentConfig(t, "block") + owner, err := NewExecutorFactory(config)(t.Context(), req) + if err != nil { + t.Fatal(err) + } + defer owner.Close(context.Background()) + turn, out := consumeExecutorTurn(t, owner, "blocked", "hello") + e := owner.(*executor) + written := make(chan error, 1) + go func() { written <- e.write(map[string]string{"padding": strings.Repeat("x", 2*1024*1024)}) }() + deadline := time.Now().Add(time.Second) + for e.base.writeMu.TryLock() { + e.base.writeMu.Unlock() + if time.Now().After(deadline) { + t.Fatal("transport write never started") + } + time.Sleep(time.Millisecond) + } + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Millisecond) + defer cancel() + returned := make(chan error, 1) + go func() { returned <- turn.Cancel(ctx) }() + select { + case err := <-returned: + if err == nil { + t.Fatal("blocked cancellation invented settlement") + } + case <-time.After(time.Second): + t.Fatal("Cancel ignored its transport deadline") + } + <-written + for range out { + } +} diff --git a/apps/parsar-daemon/internal/agent/claudesdk/executor_turn.go b/apps/parsar-daemon/internal/agent/claudesdk/executor_turn.go new file mode 100644 index 000000000..14f5067ac --- /dev/null +++ b/apps/parsar-daemon/internal/agent/claudesdk/executor_turn.go @@ -0,0 +1,215 @@ +package claudesdk + +import ( + "encoding/json" + "fmt" + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "strings" + "time" +) + +func (s *session) runTurn(start startRequest, out chan<- proto.Envelope) { + defer close(s.outputDone) + defer s.owner.finishTurn(s) + defer close(out) + defer s.stopFunctions() + defer s.stopSteering() + var failure error + outputLost := false + terminalLost := false + emit := func(kind string, payload any) { + event, err := proto.NewEnvelope(kind, s.runID, payload) + if err != nil { + return + } + select { + case out <- event: + return + default: + } + var cancelled <-chan struct{} + if kind != proto.TypeDone && kind != proto.TypeError { + cancelled = s.cancelOutput + } + timer := time.NewTimer(5 * time.Second) + defer timer.Stop() + select { + case out <- event: + case <-cancelled: + outputLost = true + case <-timer.C: + outputLost = true + if kind == proto.TypeDone || kind == proto.TypeError { + terminalLost = true + } + s.invalidate() + } + } + var content strings.Builder + var result *bridgeEvent + var usage proto.Usage + var usageSession string + usageIDs := map[string]bool{} + var sequence uint64 + terminal := false + mcp := mcpState{calls: map[string]proto.ToolObservation{}} + commands := commandState{calls: map[string]proto.ToolObservation{}} + settlementReceived := false + reusable := false + reason := "bridge_interrupted" + for raw := range s.frames { + var event bridgeEvent + if err := json.Unmarshal(raw, &event); err != nil || event.TurnID != s.runID { + failure = fmt.Errorf("claudesdk: invalid Turn event identity") + s.invalidate() + break + } + if event.Type == "turn_started" && !terminal { + continue + } + if event.Type == "turn_settled" { + if !terminal || event.Reusable == nil || (!*event.Reusable && event.Reason == "") { + failure = fmt.Errorf("claudesdk: invalid Turn settlement") + s.invalidate() + break + } + settlementReceived, reusable, reason = true, *event.Reusable, event.Reason + break + } + if terminal { + failure = fmt.Errorf("claudesdk: invalid SDK bridge output") + s.invalidate() + break + } + switch event.Type { + case "command_observation": + if err := commands.receive(event, start, s.inputSessionID(), emit); err != nil { + failure = err + s.invalidate() + } + case "mcp_observation": + if err := mcp.receive(event, start, emit); err != nil { + failure = err + s.invalidate() + } + case "delta": + if start.ObserveMessages && event.ItemID == "" || !start.ObserveMessages && event.ItemID != "" { + failure = fmt.Errorf("claudesdk: invalid message delta identity") + s.invalidate() + break + } + content.WriteString(event.Delta) + sequence++ + emit(proto.TypeDelta, proto.DeltaPayload{ItemID: event.ItemID, Delta: event.Delta, Sequence: sequence}) + case "output_message": + message := event.Message + if !start.ObserveMessages || message == nil || message.ID == "" || + (message.Status != "in_progress" && message.Status != "completed") || + (message.Status == "completed") != (message.Text != nil) { + failure = fmt.Errorf("claudesdk: invalid message observation") + s.invalidate() + break + } + emit(proto.TypeOutputMessage, message) + case "function_call", "function_applied": + if err := s.receiveFunction(event, start, emit); err != nil { + failure = err + s.invalidate() + } + case "input_ready", "input_closed", "input_applied", "input_rejected": + if err := s.receiveInput(event, start); err != nil { + failure = err + s.invalidate() + } + case proto.TypeSubagentIdentity, proto.TypeSubagentTurn, proto.TypeSubagentItem, proto.TypeSubagentCoordination: + if start.Subagents == nil || !json.Valid(event.Fact) { + failure = fmt.Errorf("claudesdk: unrequested native child observation") + s.invalidate() + break + } + emit(event.Type, event.Fact) + case "usage": + if event.ResultID == "" || !s.matchesInputSession(event.SessionID) || event.SessionID == "" || (start.Resume != "" && event.SessionID != start.Resume) || + (usageSession != "" && usageSession != event.SessionID) || usageIDs[event.ResultID] { + failure = fmt.Errorf("claudesdk: invalid usage identity or duplicate result") + s.invalidate() + break + } + nextUsage, err := appendNativeUsage(usage, event.Usage) + failure = err + if failure != nil { + s.invalidate() + break + } + usage = nextUsage + usageIDs[event.ResultID] = true + usageSession = event.SessionID + emit(proto.TypeUsage, proto.UsagePayload{Usage: usage}) + case "result": + if !s.matchesInputSession(event.SessionID) || event.SessionID == "" || start.Resume != "" && event.SessionID != start.Resume || usageSession != "" && event.SessionID != usageSession || !s.functionsComplete() || !s.steeringComplete() || !mcp.complete() || !commands.complete() { + failure = fmt.Errorf("claudesdk: invalid native completion or unconfirmed input/result") + s.invalidate() + } else { + result = &event + } + terminal = true + case "error": + failure = bridgeFailure(event.Code) + terminal = true + default: + failure = fmt.Errorf("claudesdk: unknown SDK bridge event") + s.invalidate() + } + if failure != nil && !terminal { + break + } + } + if !settlementReceived && failure == nil { + failure = fmt.Errorf("claudesdk: Turn settlement is missing") + } + if result == nil && failure == nil { + failure = fmt.Errorf("claudesdk: SDK result is missing") + } + if !s.functionsComplete() || !s.steeringComplete() || !mcp.complete() || !commands.complete() { + reusable, reason = false, "unsettled_native_operations" + } + mcp.close(start, emit) + commands.close(start, emit) + s.stopFunctions() + s.stopSteering() + metadata := map[string]any{proto.DoneMetaAgentSessionType: "claude_session"} + if id := s.inputSessionID(); id != "" { + metadata[proto.DoneMetaAgentSessionID] = id + } + if failure == nil { + content.Reset() + content.WriteString(result.Text) + metadata[proto.DoneMetaAgentSessionID] = result.SessionID + } + s.outcome = proto.DonePayload{Content: content.String(), Usage: usage, Metadata: metadata} + // Publish the observed cancellation outcome before terminal delivery. + if !settlementReceived || !reusable || outputLost || s.process.Context().Err() != nil { + s.owner.retire() + reusable = false + if reason == "" { + reason = "bridge_interrupted" + } + } + if !settlementReceived { + s.settlementErr = fmt.Errorf("claudesdk: native Turn settlement is unconfirmed") + } + close(s.settled) + if failure != nil { + emit(proto.TypeError, proto.ErrorPayload{Error: failure.Error()}) + } + emit(proto.TypeDone, s.outcome) + if outputLost { + s.owner.retire() + reusable, reason = false, "output_delivery_interrupted" + } + if terminalLost { + s.outputErr = fmt.Errorf("claudesdk: terminal output delivery failed") + } + s.turnSettlement = agent.TurnSettlement{Reusable: reusable, Reason: reason} +} diff --git a/apps/parsar-daemon/internal/agent/claudesdk/functions.go b/apps/parsar-daemon/internal/agent/claudesdk/functions.go index b135f8ec5..74df35648 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/functions.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/functions.go @@ -115,9 +115,10 @@ func (s *session) SubmitFunctionResult(ctx context.Context, result proto.Functio return err } data, err := json.Marshal(struct { - Type string `json:"type"` + Type string `json:"type"` + TurnID string `json:"turn_id"` proto.FunctionResultPayload - }{Type: "function_result", FunctionResultPayload: result}) + }{Type: "function_result", TurnID: s.runID, FunctionResultPayload: result}) if err != nil { return err } @@ -139,7 +140,7 @@ func (s *session) SubmitFunctionResult(ctx context.Context, result proto.Functio ctx, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() // A lost receipt has an unknown outcome; terminate this execution instead of resending. - stop := context.AfterFunc(ctx, s.process.Cancel) + stop := context.AfterFunc(ctx, s.invalidate) defer stop() s.writeMu.Lock() if err = ctx.Err(); err == nil { @@ -147,7 +148,7 @@ func (s *session) SubmitFunctionResult(ctx context.Context, result proto.Functio } s.writeMu.Unlock() if err != nil { - s.process.Cancel() + s.invalidate() return fmt.Errorf("claudesdk: function result transport failed") } select { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/functions_test.go b/apps/parsar-daemon/internal/agent/claudesdk/functions_test.go index cc5c6a71a..f64954b70 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/functions_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/functions_test.go @@ -127,7 +127,9 @@ func runFunctionHelper(request startRequest, mode string, scanner *bufio.Scanner } if mode == "functions-cancel" { emit(bridgeEvent{Type: "delta", Delta: "waiting for confirmation"}) - time.Sleep(time.Hour) + if scanner.Scan() { + emit(bridgeEvent{Type: "error", Code: "cancelled"}) + } return } if mode == "functions-no-receipt" { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/preparation.go b/apps/parsar-daemon/internal/agent/claudesdk/preparation.go index cdd683865..36b228698 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/preparation.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/preparation.go @@ -2,129 +2,66 @@ package claudesdk import ( "context" - "encoding/json" - "fmt" - "slices" - "strings" + "errors" "sync" "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" ) +// prepared is a single admission for direct adapter callers. It uses the same +// Executor as pooled Runtime execution; workspace reads retain the owner. type prepared struct { - mu sync.Mutex - session *session - ready chan struct{} - started chan struct{} - binding *preparedStart - closed bool - failure error + mu sync.Mutex + executor *executor + session *session + binding *preparedStart + closed bool } type preparedStart struct { - runID string - prompt proto.MessageInput - out chan<- proto.Envelope + turn agent.Turn + done chan struct{} } -var _ agent.PreparedCancellation = (*prepared)(nil) - -// NewPreparationFactory retains a trusted workspace process without submitting model input. func NewPreparationFactory(config Config) agent.PreparationFactory { - return func(owner context.Context, req proto.PromptRequestPayload) (agent.Prepared, error) { - if owner == nil { - owner = context.Background() - } - if config.Workspace == nil || req.RunID != "" || len(req.Input) != 0 || req.ConversationID != "" { - return nil, fmt.Errorf("claudesdk: preparation requires workspace configuration without input or conversation") - } - snapshot := config - snapshot.Env = slices.Clone(config.Env) - workspace := *config.Workspace - workspace.ProtectedDirs = slices.Clone(workspace.ProtectedDirs) - snapshot.Workspace = &workspace - start, env, err := prepareConfiguration(snapshot, req) - if err != nil { - return nil, err - } - start.Type = "prepare" - info, err := CheckRuntime(owner, snapshot) - if err != nil || !info.supportsWorkspacePreparation() { - return nil, fmt.Errorf("claudesdk: packaged runtime does not support workspace preparation") + factory := NewExecutorFactory(config) + return func(ctx context.Context, req proto.PromptRequestPayload) (agent.Prepared, error) { + if config.Workspace == nil { + return nil, errors.New("claudesdk: workspace preparation requires a bound workspace") } - if start.OutputFormat != nil && !info.SupportsWorkspaceStructuredOutput() { - return nil, fmt.Errorf("claudesdk: packaged runtime does not support workspace structured output") - } - if start.Subagents != nil && !info.SupportsSubagents() { - return nil, fmt.Errorf("claudesdk: packaged runtime does not support subagent resources") - } - if start.observeFunctions && !info.supportsWorkspaceCommands() { - return nil, fmt.Errorf("claudesdk: packaged runtime does not support workspace command observations") - } - if len(start.Functions) > 0 && !info.SupportsWorkspaceFunctions() { - return nil, fmt.Errorf("claudesdk: packaged runtime does not support workspace functions") - } - s, err := launch(owner, snapshot, start, env) - if err != nil { + resource, err := factory(ctx, req) + if resource == nil { return nil, err } - s.reads.supported = slices.Contains(info.Features, "workspace_read") - s.directories.supported = slices.Contains(info.Features, "workspace_directory") - p := &prepared{session: s, ready: make(chan struct{}), started: make(chan struct{})} - go s.run(owner, "", start, nil, p) - select { - case <-p.ready: - if owner.Err() == nil { - return p, nil - } - _ = p.Close() - return nil, owner.Err() - case <-s.settled: - return nil, p.failure - } + e := resource.(*executor) + return &prepared{executor: e, session: e.base}, err } } -// Start transfers ownership once; ctx bounds this operation, not the Session lifetime. -func (p *prepared) Start(ctx context.Context, runID string, prompt proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { - if ctx == nil { - ctx = context.Background() - } +func (p *prepared) Start(ctx context.Context, run string, input proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { p.mu.Lock() if p.closed || p.binding != nil { p.mu.Unlock() - return nil, fmt.Errorf("claudesdk: preparation is no longer available") - } - var err error - if strings.TrimSpace(runID) == "" || prompt.Validate() != nil || out == nil { - err = fmt.Errorf("claudesdk: start requires a run identity, prompt and output channel") - } else if ctx.Err() != nil { - err = ctx.Err() - } else if p.session.process.Context().Err() != nil { - err = p.session.process.Context().Err() - } else { - select { - case <-p.session.process.Done(): - err = fmt.Errorf("claudesdk: prepared process has exited") - case <-p.session.settled: - err = fmt.Errorf("claudesdk: preparation has ended") - default: - } + return nil, errors.New("claudesdk: admission is unavailable") } - if err != nil { - p.closed = true - p.mu.Unlock() - _ = p.session.Cancel(context.Background()) - return nil, err + binding := &preparedStart{done: make(chan struct{})} + p.binding = binding + p.mu.Unlock() + turn, err := p.executor.StartTurn(ctx, run, input, out) + p.mu.Lock() + binding.turn = turn + if turn == nil { + p.binding = nil } - p.binding = &preparedStart{runID: runID, prompt: prompt, out: out} - close(p.started) + close(binding.done) p.mu.Unlock() - return p.session, nil + if turn != nil { + go func() { _, _ = turn.AwaitSettlement(context.Background()); _ = p.executor.Close(context.Background()) }() + } + return turn, err } -// Close settles unused ownership and is inert after transfer. func (p *prepared) Close() error { p.mu.Lock() if p.binding != nil { @@ -133,51 +70,47 @@ func (p *prepared) Close() error { } p.closed = true p.mu.Unlock() - return p.session.Cancel(context.Background()) + return p.executor.Close(context.Background()) } -// Cancel follows the resource across transfer and waits for the existing output settlement. func (p *prepared) Cancel(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } p.mu.Lock() p.closed = true + binding := p.binding p.mu.Unlock() - return p.session.Cancel(ctx) -} - -func (p *prepared) CancellationOutcome() proto.DonePayload { - return p.session.CancellationOutcome() -} - -func (p *prepared) awaitStart(scanner *bridgeOutput, failure error) (*preparedStart, error) { - if failure == nil { - if scanner.Scan() { - var event bridgeEvent - if json.Unmarshal(scanner.Bytes(), &event) != nil { - failure = fmt.Errorf("claudesdk: invalid SDK bridge output") - } else if event.Type == "error" { - failure = bridgeFailure(event.Code) - } else if event.Type != "prepared" { - failure = fmt.Errorf("claudesdk: SDK preparation receipt is missing") - } - } else { - failure = fmt.Errorf("claudesdk: SDK preparation receipt is missing") + if binding == nil { + return p.executor.Close(ctx) + } + select { + case <-binding.done: + default: + if err := p.executor.Close(ctx); err != nil { + return err + } + select { + case <-binding.done: + case <-ctx.Done(): + return ctx.Err() } } - if failure != nil { - p.session.process.Cancel() - return nil, failure + if binding.turn == nil { + return p.executor.Close(ctx) } - close(p.ready) - select { - case <-p.started: - case <-p.session.process.Context().Done(): - case <-p.session.process.Done(): + if err := binding.turn.Cancel(ctx); err != nil { + return err } + p.executor.retire() + return nil +} + +func (p *prepared) CancellationOutcome() proto.DonePayload { p.mu.Lock() defer p.mu.Unlock() - if p.binding != nil { - return p.binding, nil + if p.binding == nil || p.binding.turn == nil { + return proto.DonePayload{} } - p.closed = true - return nil, fmt.Errorf("claudesdk: preparation ended before start") + return p.binding.turn.CancellationOutcome() } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/preparation_fixture_test.go b/apps/parsar-daemon/internal/agent/claudesdk/preparation_fixture_test.go index 2c02bd316..a9e8e0af6 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/preparation_fixture_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/preparation_fixture_test.go @@ -50,7 +50,7 @@ func runPreparationHelper() { features = append(features, "workspace_structured_output") } } - _ = json.NewEncoder(os.Stdout).Encode(RuntimeInfo{Type: "runtime_ready", Protocol: 2, Node: "fixture", SDK: "fixture", MCP: "fixture", Native: "fixture", Features: features}) + _ = json.NewEncoder(os.Stdout).Encode(RuntimeInfo{Type: "runtime_ready", Protocol: 3, Node: "fixture", SDK: "fixture", MCP: "fixture", Native: "fixture", Features: features}) return } state := os.Getenv("CLAUDE_CONFIG_DIR") @@ -63,7 +63,7 @@ func runPreparationHelper() { _ = os.WriteFile(filepath.Join(state, "prepare.json"), raw, 0o600) var fields map[string]json.RawMessage var request startRequest - if json.Unmarshal(raw, &fields) != nil || json.Unmarshal(raw, &request) != nil || request.Type != "prepare" || fields["input"] != nil || fields["run_id"] != nil || request.Workspace == nil { + if json.Unmarshal(raw, &fields) != nil || json.Unmarshal(raw, &request) != nil || request.Type != "executor_prepare" || fields["input"] != nil || fields["run_id"] != nil || request.Workspace == nil { os.Exit(3) } emit := func(event bridgeEvent) { _ = json.NewEncoder(os.Stdout).Encode(event) } @@ -84,7 +84,7 @@ func runPreparationHelper() { time.Sleep(time.Millisecond) } } - emit(bridgeEvent{Type: "prepared"}) + emit(bridgeEvent{Type: "executor_ready", Protocol: 3}) if strings.HasPrefix(mode, "directory-") { runWorkspaceDirectoryHelper(scanner, state, mode) return @@ -99,15 +99,19 @@ func runPreparationHelper() { raw = append([]byte{}, scanner.Bytes()...) _ = os.WriteFile(filepath.Join(state, "start.json"), raw, 0o600) fields = nil - if json.Unmarshal(raw, &fields) != nil || len(fields) != 2 || string(fields["type"]) != `"start"` || string(fields["input"]) != `[{"content":[{"type":"input_text","text":"hello"}]}]` { + if json.Unmarshal(raw, &fields) != nil || len(fields) != 3 || string(fields["type"]) != `"turn_start"` || string(fields["input"]) != `[{"content":[{"type":"input_text","text":"hello"}]}]` { os.Exit(4) } + var turnID string + _ = json.Unmarshal(fields["turn_id"], &turnID) + emit, finish := helperTurnOutput(scanner, turnID) + defer finish() if mode == "cancellation" { - runCancellationHelper(request, "cancellation-wait", emit) + runCancellationHelper(request, "cancellation-wait", scanner, emit) return } if strings.HasPrefix(mode, "commands") { - runCommandsHelper(request, mode, emit) + runCommandsHelper(request, mode, scanner, emit) return } emit(bridgeEvent{Type: "input_ready", SessionID: request.Resume}) diff --git a/apps/parsar-daemon/internal/agent/claudesdk/preparation_test.go b/apps/parsar-daemon/internal/agent/claudesdk/preparation_test.go index 610c9154f..a5e1406b6 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/preparation_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/preparation_test.go @@ -78,7 +78,7 @@ func TestPreparationWaitsForReceiptAndRetainsConfiguration(t *testing.T) { if err != nil { t.Fatal(err) } - if s != p.session || s.(*session).process.Cmd.Process.Pid != pid { + if s.(*session).owner != p.executor || s.(*session).process.Cmd.Process.Pid != pid { t.Fatal("Start replaced the prepared native process") } if err := p.Close(); err != nil { @@ -184,12 +184,13 @@ func TestPreparationFailureAndUnusedRelease(t *testing.T) { if startErr == nil { t.Fatal("invalid or cancelled Start succeeded") } + err = p.Close() } if err != nil { t.Fatal(err) } select { - case <-p.session.settled: + case <-p.executor.done: case <-time.After(5 * time.Second): t.Fatal("unused process was not settled") } @@ -280,7 +281,10 @@ func TestPreparedStartRacesCloseAndCancellation(t *testing.T) { } } if err := p.Cancel(t.Context()); err != nil { - t.Fatal(err) + // A racing Close can confirm resource cleanup without proving the Turn result. + if closeErr := p.executor.Close(t.Context()); closeErr != nil { + t.Fatal(closeErr) + } } select { case <-p.session.process.Done(): diff --git a/apps/parsar-daemon/internal/agent/claudesdk/readiness.go b/apps/parsar-daemon/internal/agent/claudesdk/readiness.go index 878f3ed16..ac9a91908 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/readiness.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/readiness.go @@ -132,7 +132,7 @@ func CheckRuntime(ctx context.Context, config Config) (RuntimeInfo, error) { return RuntimeInfo{}, fmt.Errorf("claudesdk: runtime check failed") } var info RuntimeInfo - if json.Unmarshal(raw, &info) != nil || info.Type != "runtime_ready" || info.Protocol != 2 || + if json.Unmarshal(raw, &info) != nil || info.Type != "runtime_ready" || info.Protocol != 3 || info.Node == "" || info.SDK == "" || info.MCP == "" || info.Native == "" { return RuntimeInfo{}, fmt.Errorf("claudesdk: invalid runtime readiness report") } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/readiness_test.go b/apps/parsar-daemon/internal/agent/claudesdk/readiness_test.go index a4e14866c..2e4825cc0 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/readiness_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/readiness_test.go @@ -15,7 +15,7 @@ import ( "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" ) -const readyReport = `{"type":"runtime_ready","protocol":2,"node":"22.22.2","sdk":"0.3.269","mcp":"1.30.0","native":"2.1.269 (Claude Code)"}` +const readyReport = `{"type":"runtime_ready","protocol":3,"node":"22.22.2","sdk":"0.3.269","mcp":"1.30.0","native":"2.1.269 (Claude Code)"}` func TestRequiredMCPNeedsQualifiedRuntime(t *testing.T) { root := t.TempDir() @@ -128,11 +128,11 @@ func runReadinessHelper() { os.Exit(5) } } - _, _ = fmt.Fprintln(os.Stdout, strings.Replace(readyReport, `"protocol":2`, `"protocol":2,"features":["mcp_http_tools"]`, 1)) + _, _ = fmt.Fprintln(os.Stdout, strings.Replace(readyReport, `"protocol":3`, `"protocol":3,"features":["mcp_http_tools"]`, 1)) case "malformed": _, _ = fmt.Fprintln(os.Stdout, "not-json") case "wrong-protocol": - _, _ = fmt.Fprintln(os.Stdout, strings.Replace(readyReport, `"protocol":2`, `"protocol":1`, 1)) + _, _ = fmt.Fprintln(os.Stdout, strings.Replace(readyReport, `"protocol":3`, `"protocol":1`, 1)) case "missing-version": _, _ = fmt.Fprintln(os.Stdout, strings.Replace(readyReport, `"mcp":"1.30.0"`, `"mcp":""`, 1)) case "multiple": diff --git a/apps/parsar-daemon/internal/agent/claudesdk/runtime_cache.go b/apps/parsar-daemon/internal/agent/claudesdk/runtime_cache.go new file mode 100644 index 000000000..82b07f95e --- /dev/null +++ b/apps/parsar-daemon/internal/agent/claudesdk/runtime_cache.go @@ -0,0 +1,52 @@ +package claudesdk + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "sync" +) + +type runtimeCheckCache struct { + mu sync.Mutex + stamp string + info RuntimeInfo +} + +// Registration retains the package check. Replacing an installed entrypoint, +// readiness companion, package manifest or Node binary invalidates the result. +func (c *runtimeCheckCache) check(ctx context.Context, config Config) (RuntimeInfo, error) { + node := config.Node + if node == "" { + node = "node" + } + node, err := exec.LookPath(node) + if err != nil { + return RuntimeInfo{}, err + } + root := filepath.Dir(config.Entrypoint) + stamp := "" + for _, path := range []string{node, config.Entrypoint, filepath.Join(root, "runtime_check.js"), filepath.Join(root, "../package.json"), filepath.Join(root, "../runtime-manifest.json")} { + info, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + stamp += path + ":missing;" + continue + } + return RuntimeInfo{}, err + } + stamp += fmt.Sprintf("%s:%d:%d;", path, info.Size(), info.ModTime().UnixNano()) + } + c.mu.Lock() + defer c.mu.Unlock() + if c.stamp == stamp && c.info.Protocol == 3 { + return c.info, nil + } + info, err := CheckRuntime(ctx, config) + if err == nil { + c.info, c.stamp = info, stamp + } + return info, err +} diff --git a/apps/parsar-daemon/internal/agent/claudesdk/session.go b/apps/parsar-daemon/internal/agent/claudesdk/session.go index 267811f80..dc612355f 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/session.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/session.go @@ -4,8 +4,6 @@ import ( "context" "encoding/json" "fmt" - "io" - "strings" "sync" "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" @@ -14,10 +12,21 @@ import ( ) type session struct { + owner *executor + runID string + frames chan []byte + outputDone chan struct{} + turnSettlement agent.TurnSettlement + settlementErr error + outputErr error + cancelOnce sync.Once + cancelOutput chan struct{} + nativeEnded bool + reads workspaceReadState directories workspaceDirectoryState process *clirunner.Process - writeMu sync.Mutex + writeMu *sync.Mutex functions functionState steering steeringState settled chan struct{} @@ -25,59 +34,32 @@ type session struct { } func NewFactory(config Config) agent.Factory { + prepareExecutor := NewExecutorFactory(config) return func(ctx context.Context, req proto.PromptRequestPayload, out chan<- proto.Envelope) (agent.Session, error) { - if ctx == nil { - ctx = context.Background() - } - if out == nil { - return nil, fmt.Errorf("claudesdk: output channel is required") - } - if config.Workspace != nil { - runID, prompt := req.RunID, req.Input - if strings.TrimSpace(runID) == "" || prompt.Validate() != nil { - return nil, fmt.Errorf("claudesdk: run id and prompt are required") - } - req.RunID, req.Input = "", nil - prepared, err := NewPreparationFactory(config)(ctx, req) - if err != nil { - return nil, err - } - defer prepared.Close() - return prepared.Start(ctx, runID, prompt, out) - } - start, env, err := prepare(config, req) + run, input := req.RunID, req.Input + req.RunID, req.ConversationID, req.Input = "", "", nil + resource, err := prepareExecutor(ctx, req) if err != nil { return nil, err } - if start.MCPHTTPServers != nil || start.Subagents != nil { - info, err := CheckRuntime(ctx, config) - if start.Subagents != nil && (err != nil || !info.SupportsSubagents()) { - return nil, fmt.Errorf("claudesdk: packaged runtime does not support subagent resources") - } - if start.MCPHTTPServers != nil && (err != nil || !info.SupportsHTTPMCP()) { - return nil, fmt.Errorf("claudesdk: packaged runtime does not support HTTP MCP") - } - if start.MCPHTTPServers != nil { - for _, server := range *start.MCPHTTPServers { - if server.Required && !info.SupportsHTTPMCPRequired() { - return nil, fmt.Errorf("claudesdk: packaged runtime does not support required HTTP MCP") - } - if server.BearerTokenEnvVar != "" && !info.SupportsHTTPMCPBearer() { - return nil, fmt.Errorf("claudesdk: packaged runtime does not support authenticated HTTP MCP") - } - } - } - } - s, err := launch(ctx, config, start, env) - if err != nil { + turn, err := resource.StartTurn(ctx, run, input, out) + if turn == nil { + _ = resource.Close(context.Background()) return nil, err } - go s.run(ctx, req.RunID, start, out, nil) - return s, nil + // Factory callers own one execution. Both entrypoints use the same + // Executor implementation; Runtime pooling uses NewExecutorFactory. + go func() { _, _ = turn.AwaitSettlement(context.Background()); _ = resource.Close(context.Background()) }() + return turn, err } } type bridgeEvent struct { + TurnID string `json:"turn_id"` + Reusable *bool `json:"reusable"` + Reason string `json:"reason"` + Protocol int `json:"protocol"` + Fact json.RawMessage `json:"fact"` InputID string `json:"input_id"` ResultID string `json:"result_id"` @@ -97,191 +79,6 @@ type bridgeEvent struct { Observation *proto.ToolObservation `json:"observation"` } -func (s *session) run(ctx context.Context, runID string, start startRequest, out chan<- proto.Envelope, prepared *prepared) { - defer func() { - if out != nil { - close(out) - } - }() - defer s.stopFunctions() - defer s.stopSteering() - defer s.stopWorkspaceReads() - defer s.stopWorkspaceDirectories() - emit := func(kind string, payload any) { - event, err := proto.NewEnvelope(kind, runID, payload) - if err != nil { - return - } - // Cancellation must drain native output even if the event consumer stops. - // Terminal publication follows settlement and cannot hold up Cancel. - var stopping <-chan struct{} - if kind != proto.TypeError && kind != proto.TypeDone { - stopping = s.process.Context().Done() - } - // Preserve drained observations when the consumer can accept them immediately. - select { - case out <- event: - return - default: - } - select { - case out <- event: - case <-ctx.Done(): - case <-stopping: - } - } - stderrDone := make(chan struct{}) - go func() { _, _ = io.Copy(io.Discard, s.process.Stderr); close(stderrDone) }() - var failure error - if err := json.NewEncoder(s.process.Stdin).Encode(start); err != nil { - failure = fmt.Errorf("claudesdk: cannot submit SDK input") - s.process.Cancel() - } - scanner := s.bridgeOutput() - if prepared != nil { - binding, err := prepared.awaitStart(scanner, failure) - if binding == nil { - prepared.failure = s.drain(scanner, stderrDone, err) - close(s.settled) - return - } - runID, out = binding.runID, binding.out - s.writeMu.Lock() - err = json.NewEncoder(s.process.Stdin).Encode(struct { - Type string `json:"type"` - Input proto.MessageInput `json:"input"` - }{Type: "start", Input: binding.prompt}) - s.writeMu.Unlock() - if err != nil { - failure = fmt.Errorf("claudesdk: cannot submit SDK input") - s.process.Cancel() - } - } - var content strings.Builder - var result *bridgeEvent - var usage proto.Usage - var usageSession string - usageIDs := map[string]bool{} - var sequence uint64 - terminal := false - mcp := mcpState{calls: map[string]proto.ToolObservation{}} - commands := commandState{calls: map[string]proto.ToolObservation{}} - for scanner.Scan() { - var event bridgeEvent - if err := json.Unmarshal(scanner.Bytes(), &event); err != nil || terminal { - failure = fmt.Errorf("claudesdk: invalid SDK bridge output") - s.process.Cancel() - break - } - switch event.Type { - case "command_observation": - if err := commands.receive(event, start, s.inputSessionID(), emit); err != nil { - failure = err - s.process.Cancel() - } - case "mcp_observation": - if err := mcp.receive(event, start, emit); err != nil { - failure = err - s.process.Cancel() - } - case "delta": - if start.ObserveMessages && event.ItemID == "" || !start.ObserveMessages && event.ItemID != "" { - failure = fmt.Errorf("claudesdk: invalid message delta identity") - s.process.Cancel() - break - } - content.WriteString(event.Delta) - sequence++ - emit(proto.TypeDelta, proto.DeltaPayload{ItemID: event.ItemID, Delta: event.Delta, Sequence: sequence}) - case "output_message": - message := event.Message - if !start.ObserveMessages || message == nil || message.ID == "" || - (message.Status != "in_progress" && message.Status != "completed") || - (message.Status == "completed") != (message.Text != nil) { - failure = fmt.Errorf("claudesdk: invalid message observation") - s.process.Cancel() - break - } - emit(proto.TypeOutputMessage, message) - case "function_call", "function_applied": - if err := s.receiveFunction(event, start, emit); err != nil { - failure = err - s.process.Cancel() - } - case "input_ready", "input_closed", "input_applied", "input_rejected": - if err := s.receiveInput(event, start); err != nil { - failure = err - s.process.Cancel() - } - case proto.TypeSubagentIdentity, proto.TypeSubagentTurn, proto.TypeSubagentItem, proto.TypeSubagentCoordination: - if start.Subagents == nil || !json.Valid(event.Fact) { - failure = fmt.Errorf("claudesdk: unrequested native child observation") - s.process.Cancel() - break - } - emit(event.Type, event.Fact) - case "usage": - if event.ResultID == "" || !s.matchesInputSession(event.SessionID) || event.SessionID == "" || (start.Resume != "" && event.SessionID != start.Resume) || - (usageSession != "" && usageSession != event.SessionID) || usageIDs[event.ResultID] { - failure = fmt.Errorf("claudesdk: invalid usage identity or duplicate result") - s.process.Cancel() - break - } - nextUsage, err := appendNativeUsage(usage, event.Usage) - failure = err - if failure != nil { - s.process.Cancel() - break - } - usage = nextUsage - usageIDs[event.ResultID] = true - usageSession = event.SessionID - emit(proto.TypeUsage, proto.UsagePayload{Usage: usage}) - case "result": - if !s.matchesInputSession(event.SessionID) || event.SessionID == "" || start.Resume != "" && event.SessionID != start.Resume || usageSession != "" && event.SessionID != usageSession || !s.functionsComplete() || !s.steeringComplete() || !mcp.complete() || !commands.complete() { - failure = fmt.Errorf("claudesdk: invalid native completion or unconfirmed input/result") - s.process.Cancel() - } else { - result = &event - } - terminal = true - case "error": - failure = bridgeFailure(event.Code) - terminal = true - default: - failure = fmt.Errorf("claudesdk: unknown SDK bridge event") - s.process.Cancel() - } - if failure != nil && !terminal { - break - } - } - failure = s.drain(scanner, stderrDone, failure) - if result == nil && failure == nil { - failure = fmt.Errorf("claudesdk: SDK result is missing") - } - mcp.close(start, emit) - commands.close(start, emit) - s.stopFunctions() - s.stopSteering() - metadata := map[string]any{proto.DoneMetaAgentSessionType: "claude_session"} - if id := s.inputSessionID(); id != "" { - metadata[proto.DoneMetaAgentSessionID] = id - } - if failure == nil { - content.Reset() - content.WriteString(result.Text) - metadata[proto.DoneMetaAgentSessionID] = result.SessionID - } - s.outcome = proto.DonePayload{Content: content.String(), Usage: usage, Metadata: metadata} - // Router completion cleanup calls Cancel while consuming Done. Settle first. - close(s.settled) - if failure != nil { - emit(proto.TypeError, proto.ErrorPayload{Error: failure.Error()}) - } - emit(proto.TypeDone, s.outcome) -} - func launch(ctx context.Context, config Config, start startRequest, env []string) (*session, error) { binary := config.Node if binary == "" { @@ -291,7 +88,7 @@ func launch(ctx context.Context, config Config, start startRequest, env []string if err != nil { return nil, err } - return &session{process: process, functions: functionState{calls: map[string]*pendingFunction{}}, settled: make(chan struct{})}, nil + return &session{process: process, writeMu: &sync.Mutex{}, functions: functionState{calls: map[string]*pendingFunction{}}, settled: make(chan struct{})}, nil } func (s *session) drain(scanner *bridgeOutput, stderrDone <-chan struct{}, failure error) error { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/session_test.go b/apps/parsar-daemon/internal/agent/claudesdk/session_test.go index 2a736f7f2..eacdb8e16 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/session_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/session_test.go @@ -101,6 +101,10 @@ func TestTextFactoryRejectsUnsupportedInput(t *testing.T) { } func TestMain(m *testing.M) { + if os.Getenv("GO_CLAUDE_EXECUTOR_HELPER") == "1" { + runPersistentExecutorHelper() + os.Exit(0) + } if os.Getenv("GO_CLAUDE_PREPARATION_HELPER") == "1" { runPreparationHelper() os.Exit(0) @@ -117,18 +121,23 @@ func TestMain(m *testing.M) { } func runSDKHelper() { + if len(os.Args) > 1 && strings.HasSuffix(os.Args[1], "runtime_check.js") { + fmt.Fprintln(os.Stdout, strings.TrimSuffix(readyReport, "}")+`,"features":["structured_output","subagents"]}`) + return + } scanner := bufio.NewScanner(os.Stdin) if !scanner.Scan() { os.Exit(2) } var request startRequest - if json.Unmarshal(scanner.Bytes(), &request) != nil || request.Type != "start" || *request.Input[0].Content[0].Text != "hello" || request.Model != "fake-model" || request.SystemPrompt != "instructions" { + if json.Unmarshal(scanner.Bytes(), &request) != nil || request.Type != "executor_prepare" || request.Model != "fake-model" || request.SystemPrompt != "instructions" { os.Exit(3) } - encode := func(event bridgeEvent) { _ = json.NewEncoder(os.Stdout).Encode(event) } + encode, finish := helperTurn(scanner, &request) + defer finish() mode := os.Getenv("SDK_HELPER_MODE") if strings.HasPrefix(mode, "cancellation-") { - runCancellationHelper(request, mode, encode) + runCancellationHelper(request, mode, scanner, encode) return } if strings.HasPrefix(mode, "steering-") { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/steering.go b/apps/parsar-daemon/internal/agent/claudesdk/steering.go index b87b16f1d..c9d2c0eae 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/steering.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/steering.go @@ -34,6 +34,11 @@ func (s *session) Steer(ctx context.Context, input proto.PromptSteerPayload) err // SteerWithReceipt separates a complete bridge write from native consumption. func (s *session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerPayload, written func()) error { + select { + case <-s.cancelOutput: + return agent.ErrSteeringInactive + default: + } if ctx == nil { ctx = context.Background() } @@ -42,9 +47,10 @@ func (s *session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerP } data, err := json.Marshal(struct { Type string `json:"type"` + TurnID string `json:"turn_id"` InputID string `json:"input_id"` Input proto.MessageInput `json:"input"` - }{Type: "steer", InputID: input.InputID, Input: input.Input}) + }{Type: "steer", TurnID: s.runID, InputID: input.InputID, Input: input.Input}) if err != nil || len(data) > 1024*1024 { return fmt.Errorf("%w: input exceeds bridge limit", agent.ErrSteeringRejected) } @@ -74,7 +80,7 @@ func (s *session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerP s.steering.mu.Unlock() // Cancellation must release a blocked write, but a lost receipt after a full // write preserves the process and unknown outcome without automatic redelivery. - stop := context.AfterFunc(ctx, s.process.Cancel) + stop := context.AfterFunc(ctx, s.invalidate) s.writeMu.Lock() if err = ctx.Err(); err == nil { _, err = s.process.Stdin.Write(append(data, '\n')) @@ -82,7 +88,7 @@ func (s *session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerP s.writeMu.Unlock() stop() if err != nil { - s.process.Cancel() + s.invalidate() return fmt.Errorf("claudesdk: input transport failed") } if written != nil { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/steering_test.go b/apps/parsar-daemon/internal/agent/claudesdk/steering_test.go index 256856efc..7bd7f68d3 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/steering_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/steering_test.go @@ -147,10 +147,11 @@ func TestSteeringReceiptsAndLifecycle(t *testing.T) { if err := s.Steer(ctx, input); !errors.Is(err, agent.ErrSteeringInactive) { t.Fatal("completed execution accepted input", err) } - select { - case <-s.process.Done(): - default: - t.Fatal("completion preceded process release") + if _, err := s.AwaitSettlement(ctx); err != nil && (mode == "success" || mode == "phased" || mode == "timeout") { + t.Fatal(err) + } + if err := s.owner.Close(ctx); err != nil { + t.Fatal(err) } }) } @@ -186,7 +187,9 @@ func runSteeringHelper(request startRequest, mode string, scanner *bufio.Scanner os.Exit(3) } if mode == "steering-cancel" { - time.Sleep(time.Hour) + if scanner.Scan() { + emit(bridgeEvent{Type: "error", Code: "cancelled"}) + } return } if mode == "steering-timeout" { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory.go index 14507ab80..4cbf89803 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory.go @@ -57,6 +57,9 @@ func (p *prepared) ListWorkspaceDirectory(ctx context.Context, path string, maxE } func (s *session) ListWorkspaceDirectory(ctx context.Context, path string, maxEntries int) (agent.WorkspaceDirectoryResult, error) { + if s.owner != nil { + return s.owner.base.ListWorkspaceDirectory(ctx, path, maxEntries) + } read, err := s.admitWorkspaceDirectory(ctx, path, maxEntries) if err != nil { return agent.WorkspaceDirectoryResult{}, err diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory_test.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory_test.go index f31003a9c..2cb969cf2 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory_test.go @@ -20,14 +20,24 @@ import ( ) func runWorkspaceDirectoryHelper(scanner *bufio.Scanner, state, mode string) { + turnID := "" for scanner.Scan() { var req struct { Type, ID string + TurnID string `json:"turn_id"` Path string `json:"directory"` Max int `json:"max_entries"` } _ = json.Unmarshal(scanner.Bytes(), &req) - if req.Type == "start" { + if req.Type == "turn_cancel" { + reusable := false + _ = json.NewEncoder(os.Stdout).Encode(bridgeEvent{Type: "error", TurnID: turnID, Code: "cancelled"}) + _ = json.NewEncoder(os.Stdout).Encode(bridgeEvent{Type: "turn_settled", TurnID: turnID, Reusable: &reusable, Reason: "fixture_closed"}) + return + } + if req.Type == "turn_start" { + turnID = req.TurnID + _ = json.NewEncoder(os.Stdout).Encode(bridgeEvent{Type: "turn_started", TurnID: turnID}) _ = os.WriteFile(filepath.Join(state, "directory-started"), []byte("{}"), 0600) continue } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_launch_test.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_launch_test.go index 832b1dcce..556db3e7f 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_launch_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_launch_test.go @@ -24,12 +24,13 @@ test "$ANTHROPIC_AUTH_TOKEN" = selected-provider-fixture || exit 23 test "$TMPDIR" != "$CLAUDE_CONFIG_DIR/tmp" || exit 24 case "$1" in */runtime_check.js) - printf '%s\n' '{"type":"runtime_ready","protocol":2,"node":"fixture","sdk":"fixture","mcp":"fixture","native":"fixture","features":["workspace_tools","workspace_prepare"]}' ;; + printf '%s\n' '{"type":"runtime_ready","protocol":3,"node":"fixture","sdk":"fixture","mcp":"fixture","native":"fixture","features":["workspace_tools","workspace_prepare"]}' ;; *) IFS= read -r request - printf '%s\n' '{"type":"prepared"}' + printf '%s\n' '{"type":"executor_ready","protocol":3}' IFS= read -r request - printf '%s\n' '{"type":"result","session_id":"native","text":"completed"}' ;; + printf '%s\n' '{"type":"result","turn_id":"run","session_id":"native","text":"completed"}' + printf '%s\n' '{"type":"turn_settled","turn_id":"run","reusable":true,"reason":""}' ;; esac ` if err := os.WriteFile(config.Node, []byte(script), 0o700); err != nil { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_read.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_read.go index 9ed4b64c7..5f0a4052d 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_read.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_read.go @@ -57,6 +57,9 @@ func (p *prepared) ReadWorkspaceFile(ctx context.Context, path string, maxBytes } func (s *session) ReadWorkspaceFile(ctx context.Context, path string, maxBytes int) (agent.WorkspaceReadResult, error) { + if s.owner != nil { + return s.owner.base.ReadWorkspaceFile(ctx, path, maxBytes) + } read, err := s.admitWorkspaceRead(ctx, path, maxBytes) if err != nil { return agent.WorkspaceReadResult{}, err diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_read_test.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_read_test.go index 61209b63b..ca46342ef 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_read_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_read_test.go @@ -20,13 +20,23 @@ import ( ) func runWorkspaceReadHelper(scanner *bufio.Scanner, state, mode string) { + turnID := "" for scanner.Scan() { var req struct { Type, ID, Path string - Max int `json:"max_bytes"` + TurnID string `json:"turn_id"` + Max int `json:"max_bytes"` } _ = json.Unmarshal(scanner.Bytes(), &req) - if req.Type == "start" { + if req.Type == "turn_cancel" { + reusable := false + _ = json.NewEncoder(os.Stdout).Encode(bridgeEvent{Type: "error", TurnID: turnID, Code: "cancelled"}) + _ = json.NewEncoder(os.Stdout).Encode(bridgeEvent{Type: "turn_settled", TurnID: turnID, Reusable: &reusable, Reason: "fixture_closed"}) + return + } + if req.Type == "turn_start" { + turnID = req.TurnID + _ = json.NewEncoder(os.Stdout).Encode(bridgeEvent{Type: "turn_started", TurnID: turnID}) _ = os.WriteFile(filepath.Join(state, "read-started"), []byte("{}"), 0600) continue } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_structured_test.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_structured_test.go index a9f7a8ebc..b8ad20f00 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_structured_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_structured_test.go @@ -52,7 +52,7 @@ func TestWorkspaceStructuredPreparationQualificationAndFrozenSchema(t *testing.T } } var started map[string]json.RawMessage - if err := json.Unmarshal(waitPreparationFile(t, filepath.Join(config.StateDir, "start.json")), &started); err != nil || len(started) != 2 || started["output_format"] != nil { + if err := json.Unmarshal(waitPreparationFile(t, filepath.Join(config.StateDir, "start.json")), &started); err != nil || len(started) != 3 || started["output_format"] != nil { t.Fatal("Start replaced the prepared configuration", err) } }) diff --git a/apps/parsar-daemon/internal/agent/codex/executor.go b/apps/parsar-daemon/internal/agent/codex/executor.go new file mode 100644 index 000000000..954a29d0f --- /dev/null +++ b/apps/parsar-daemon/internal/agent/codex/executor.go @@ -0,0 +1,156 @@ +package codex + +import ( + "context" + "errors" + "strings" + "sync" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +// Executor owns the prepared process, fixed plan and native thread. Each StartTurn +// creates independent receipt, observation, cancellation and output ownership. +type Executor struct { + mu sync.Mutex + prepared *Prepared + active *Session + closed bool + closeMu sync.Mutex +} + +func PrepareExecutor(ctx context.Context, req proto.PromptRequestPayload) (agent.Executor, error) { + return newExecutor(ctx, req, defaultSessionConfig()) +} + +func newExecutor(ctx context.Context, req proto.PromptRequestPayload, cfg sessionConfig) (*Executor, error) { + p, err := newPreparation(ctx, req, cfg) + if err != nil { + if p != nil { + return &Executor{prepared: p}, err + } + return nil, err + } + p.mu.Lock() + p.started = true + close(p.transferred) + p.mu.Unlock() + return &Executor{prepared: p}, nil +} + +func (e *Executor) StartTurn(ctx context.Context, runID string, input proto.MessageInput, out chan<- proto.Envelope) (agent.Turn, error) { + if out == nil || strings.TrimSpace(runID) == "" || input.Validate() != nil { + return nil, errors.New("codex: start requires a run identity, input and output") + } + if err := ctx.Err(); err != nil { + return nil, err + } + e.mu.Lock() + base := e.prepared.session + if e.closed || base.cancelCtx.Err() != nil || !base.rpc.Alive() { + e.mu.Unlock() + return nil, errors.New("codex: executor unavailable") + } + previous := e.active + if previous != nil { + select { + case <-previous.waitDone: + default: + e.mu.Unlock() + return nil, errors.New("codex: previous turn has not settled") + } + if previous.settlementErr != nil || !previous.settlement.Reusable { + e.mu.Unlock() + return nil, errors.New("codex: executor requires retirement") + } + } + functions := &functionCalls{definitions: base.functions.definitions, names: base.functions.names, pending: map[string]*pendingFunction{}} + turnCtx, cancel := context.WithCancel(base.cancelCtx) + s := &Session{executor: e, nativeHome: base.nativeHome, toolEnvironment: base.toolEnvironment, + functions: functions, observeMessages: base.observeMessages, observeTools: base.observeTools, + observeToolObservations: base.observeToolObservations, observeSubagentIdentities: base.observeSubagentIdentities, + cfg: base.cfg, rpc: base.rpc, cancelCtx: turnCtx, cancelFn: cancel, + waitDone: make(chan struct{}), outputDone: make(chan struct{}), cleanup: func() {}, + bufs: NewItemBuffers(), resolvedModel: base.resolvedModel, interactions: newPendingCodexInteractions(), runID: runID, out: out} + if previous != nil { + s.threadID = previous.currentThreadID() + s.retiredTurns = make(map[string]bool, len(previous.retiredTurns)+1) + for id := range previous.retiredTurns { + s.retiredTurns[id] = true + } + previous.steering.mu.Lock() + s.retiredTurns[previous.steering.id] = true + previous.steering.mu.Unlock() + s.resolvedModel = previous.resolvedModel + previous.usageMu.Lock() + s.usageTotal = previous.usageTotal + previous.usageMu.Unlock() + } + e.active = s + if s.observeSubagentIdentities { + s.startSubagentObservations() + } + s.registerHandlers() + e.mu.Unlock() + req := proto.PromptRequestPayload{RunID: runID, Input: input, AgentSessionID: e.prepared.resumeID, StrictResume: e.prepared.strictResume, RequireExistingNativeSession: e.prepared.requireExistingNativeSession} + // Ownership precedes any native submission. Even an uncertain start returns the + // exact Turn so its caller can await settlement without replaying the input. + err := s.startNative(ctx, e.prepared.plan, req) + if err != nil { + s.emitTerminal("codex: native start failed", true) + s.finishAfterTerminal() + } + go s.settleExecutorTurn(err) + return s, err +} + +func (e *Executor) Close(ctx context.Context) error { + e.mu.Lock() + e.closed = true + e.mu.Unlock() + done := make(chan error, 1) + go func() { + e.closeMu.Lock() + defer e.closeMu.Unlock() + base := e.prepared.session + e.mu.Lock() + running := e.active + e.mu.Unlock() + if running != nil { + select { + case <-running.waitDone: + default: + if err := running.Cancel(ctx); err != nil { + done <- err + return + } + } + } + base.cancelFn() + err := base.rpc.Close() + e.mu.Lock() + active := e.active + e.mu.Unlock() + if err == nil && active != nil { + _, err = active.AwaitSettlement(ctx) + } + if err == nil { + err = base.rpc.awaitReaders(ctx) + } + if err == nil { + e.prepared.plan.Cleanup() + } + done <- err + }() + select { + case err := <-done: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +var _ agent.Executor = (*Executor)(nil) + +func NewExecutorFactory() agent.ExecutorFactory { return PrepareExecutor } diff --git a/apps/parsar-daemon/internal/agent/codex/executor_native_test.go b/apps/parsar-daemon/internal/agent/codex/executor_native_test.go new file mode 100644 index 000000000..5d4ecbc97 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/codex/executor_native_test.go @@ -0,0 +1,164 @@ +package codex + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + obslog "github.com/MiniMax-AI-Dev/parsar/internal/obs/log" +) + +// This opt-in test uses the real pinned harness and an explicitly configured +// Responses provider. Credentials are read from a private file, never arguments. +func TestExecutorNativeReuse(t *testing.T) { + if os.Getenv("OAC_TEST_CODEX_EXECUTOR_NATIVE") != "1" { + t.Skip("requires an authorized native model acceptance environment") + } + binary, root := os.Getenv("OAC_TEST_CODEX_BINARY"), os.Getenv("OAC_TEST_CODEX_LIVE_ROOT") + model, endpoint := os.Getenv("OAC_TEST_CODEX_MODEL"), os.Getenv("OAC_TEST_CODEX_BASE_URL") + if binary == "" || root == "" || model == "" || endpoint == "" { + t.Fatal("missing explicit native acceptance configuration") + } + version, err := exec.Command(binary, "--version").Output() + if err != nil || strings.TrimSpace(string(version)) != "codex-cli 0.153.4" { + t.Fatal("native artifact is not Codex 0.153.4") + } + key, err := os.ReadFile(os.Getenv("OAC_TEST_CODEX_KEY_FILE")) + if err != nil || strings.TrimSpace(string(key)) == "" { + t.Fatal("private provider credential unavailable") + } + if err = os.MkdirAll(root, 0700); err != nil { + t.Fatal("cannot create isolated acceptance root") + } + isolated, err := os.MkdirTemp(root, "run-") + if err != nil { + t.Fatal("cannot create isolated acceptance workspace") + } + t.Cleanup(func() { _ = os.RemoveAll(isolated) }) + t.Setenv("OAC_RUNTIME_HOME", isolated) + for _, name := range []string{"CODEX_EXEC_SERVER_URL", "CODEX_EXEC_SERVER_NOISE_REGISTRY_URL", "CODEX_EXEC_SERVER_NOISE_ENVIRONMENT_ID", "CODEX_EXEC_SERVER_NOISE_AUTH_TOKEN"} { + t.Setenv(name, "") + } + cfg := defaultSessionConfig() + cfg.codexBinary = binary + cfg.logger = obslog.Discard() + req := proto.PromptRequestPayload{ + AgentKind: "codex", AgentStateKey: "executor-native", WorkDir: filepath.Join(isolated, "workspace"), + StrictResume: true, DisableExecutionEnvironment: true, DisableSubagents: true, ObserveMessages: true, + AgentOptions: map[string]any{"model": model, "codex_provider": map[string]any{"base_url": endpoint, "wire_api": "responses", "bearer_token": strings.TrimSpace(string(key))}}, + FunctionTools: []proto.FunctionTool{{Name: "hold", Description: "Wait until the host supplies a result.", Parameters: json.RawMessage("{\"type\":\"object\",\"properties\":{},\"additionalProperties\":false}")}}, + } + ctx, cancel := context.WithTimeout(t.Context(), 4*time.Minute) + defer cancel() + began := time.Now() + e, err := newExecutor(ctx, req, cfg) + if err != nil { + t.Fatalf("native prepare failed (%T)", err) + } + t.Cleanup(func() { + c, stop := context.WithTimeout(context.Background(), 20*time.Second) + defer stop() + if err := e.Close(c); err != nil { + t.Errorf("native owner cleanup failed (%T)", err) + } + }) + t.Logf("native_version=0.153.4 prepare_ms=%d", time.Since(began).Milliseconds()) + pid := e.prepared.session.rpc.cmd.Process.Pid + var thread string + run := func(id, prompt string, old agent.Turn, interrupt bool) (agent.Turn, string) { + t.Helper() + output := make(chan proto.Envelope, 512) + started := time.Now() + turn, err := e.StartTurn(ctx, id, proto.TextInput(prompt), output) + if err != nil { + t.Fatalf("%s start failed (%T)", id, err) + } + if old != nil { + if err := old.Cancel(ctx); err != nil { + t.Fatalf("%s stale cancellation failed", id) + } + } + done := make(chan struct{}) + called := make(chan struct{}, 1) + var text string + var terminal int + var first time.Duration + go func() { + defer close(done) + for event := range output { + if first == 0 { + first = time.Since(started) + } + if event.Type == proto.TypeFunctionCall { + select { + case called <- struct{}{}: + default: + } + } + if event.Type == proto.TypeDone { + var result proto.DonePayload + _ = event.DecodePayload(&result) + text = result.Content + terminal++ + } + } + }() + if interrupt { + select { + case <-called: + case <-done: + t.Fatal("native cancellation did not reach its function") + case <-ctx.Done(): + t.Fatal("native function admission timed out") + } + cancelStarted := time.Now() + if err := turn.Cancel(ctx); err != nil { + t.Fatalf("native interrupt failed (%T)", err) + } + t.Logf("turn=%s cancel_settled_ms=%d", id, time.Since(cancelStarted).Milliseconds()) + } + settlement, err := turn.AwaitSettlement(ctx) + if err != nil || !settlement.Reusable { + t.Fatalf("%s native settlement not reusable (error=%t reason=%s)", id, err != nil, settlement.Reason) + } + <-done + if terminal != 1 { + t.Fatalf("%s emitted %d terminals", id, terminal) + } + s := turn.(*Session) + if thread == "" { + thread = s.currentThreadID() + } + if thread == "" || s.currentThreadID() != thread || !s.rpc.Alive() || s.rpc.cmd.Process.Pid != pid { + t.Fatal("native owner/thread changed") + } + t.Logf("turn=%s first_event_ms=%d settled_ms=%d same_process=true same_thread=true", id, first.Milliseconds(), time.Since(started).Milliseconds()) + return turn, text + } + first, _ := run("cold", "Remember the exact code CEDAR-4729 for this conversation. Reply only SAVED. Do not call tools.", nil, false) + _, answer := run("warm", "What is the exact code I asked you to remember? Reply only the code. Do not call tools.", nil, false) + if strings.TrimSpace(answer) != "CEDAR-4729" { + t.Fatal("warm native Turn did not retain conversation memory") + } + cancelled, _ := run("cancel", "Call the hold function now with an empty object. Wait for its result before responding.", nil, true) + _, answer = run("followup", "What is the exact code I asked you to remember? Reply only the code. Do not call tools.", cancelled, false) + if strings.TrimSpace(answer) != "CEDAR-4729" { + t.Fatal("post-cancellation native Turn lost conversation memory") + } + if err := first.Cancel(ctx); err != nil { + t.Fatal("retired first Turn cancellation failed") + } + if err := e.Close(ctx); err != nil { + t.Fatal("native final cleanup failed") + } + if e.prepared.session.rpc.Alive() { + t.Fatal("native child remained alive after owner close") + } +} diff --git a/apps/parsar-daemon/internal/agent/codex/executor_test.go b/apps/parsar-daemon/internal/agent/codex/executor_test.go new file mode 100644 index 000000000..39413ecfa --- /dev/null +++ b/apps/parsar-daemon/internal/agent/codex/executor_test.go @@ -0,0 +1,187 @@ +package codex + +import ( + "context" + "errors" + "os" + "os/exec" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +func executorFixture(t *testing.T, mode string) (*Executor, string) { + t.Helper() + req, cfg, root := preparationFixture(t) + t.Setenv("OAC_TEST_EXECUTOR_MODE", mode) + e, err := newExecutor(t.Context(), req, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := e.Close(ctx); err != nil { + t.Error(err) + } + }) + return e, root +} +func awaitExecutorTurn(t *testing.T, turn agent.Turn, out <-chan proto.Envelope) agent.TurnSettlement { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + settlement, err := turn.AwaitSettlement(ctx) + if err != nil { + t.Fatal(err) + } + count := 0 + for frame := range out { + if frame.Type == proto.TypeDone { + count++ + } + } + if count != 1 { + t.Fatalf("terminal count %d", count) + } + return settlement +} +func TestExecutorNormalTurnsKeepProcessAndThread(t *testing.T) { + e, root := executorFixture(t, "complete") + var previous agent.Turn + for _, id := range []string{"one", "two"} { + out := make(chan proto.Envelope, 20) + turn, err := e.StartTurn(t.Context(), id, proto.TextInput("answer"), out) + if err != nil { + t.Fatal(err) + } + if !awaitExecutorTurn(t, turn, out).Reusable { + t.Fatal("healthy turn was not reusable") + } + if previous != nil { + if err := previous.Cancel(t.Context()); err != nil { + t.Fatal(err) + } + } + previous = turn + } + counts := map[string]int{} + pids := map[int]bool{} + for _, f := range preparationFrames(t, root) { + counts[f.Method]++ + pids[f.PID] = true + } + if counts["initialize"] != 1 || counts["thread/start"] != 1 || counts["turn/start"] != 2 || counts["turn/interrupt"] != 0 || len(pids) != 1 { + t.Fatal(counts, pids) + } + if !e.prepared.session.rpc.Alive() { + t.Fatal("normal completion closed executor") + } +} +func TestExecutorCancellationSettlesThenReuses(t *testing.T) { + e, root := executorFixture(t, "complete") + out := make(chan proto.Envelope, 20) + first, err := e.StartTurn(t.Context(), "cancel", proto.TextInput("hold"), out) + if err != nil { + t.Fatal(err) + } + if _, err := e.StartTurn(t.Context(), "overlap", proto.TextInput("answer"), make(chan proto.Envelope, 20)); err == nil { + t.Fatal("overlapping turn admitted") + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + if err := first.Cancel(ctx); err != nil { + t.Fatal(err) + } + if !awaitExecutorTurn(t, first, out).Reusable { + t.Fatal("cancelled healthy executor unavailable") + } + secondOut := make(chan proto.Envelope, 20) + second, err := e.StartTurn(t.Context(), "next", proto.TextInput("hold"), secondOut) + if err != nil { + t.Fatal(err) + } + if err := first.Cancel(ctx); err != nil { + t.Fatal(err) + } + // A stale handle cannot interrupt its successor. + interrupts := 0 + for _, f := range preparationFrames(t, root) { + if f.Method == "turn/interrupt" { + interrupts++ + } + } + if interrupts != 1 { + t.Fatalf("stale cancellation sent %d interrupts", interrupts) + } + if err := second.Cancel(ctx); err != nil { + t.Fatal(err) + } + if !awaitExecutorTurn(t, second, secondOut).Reusable { + t.Fatal("second cancellation did not settle") + } +} +func TestExecutorStartErrorsRetainExactOwnership(t *testing.T) { + for _, mode := range []string{"start-error", "disconnect"} { + t.Run(mode, func(t *testing.T) { + e, _ := executorFixture(t, mode) + out := make(chan proto.Envelope, 20) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + turn, err := e.StartTurn(ctx, "before", proto.TextInput("answer"), out) + if turn != nil || !errors.Is(err, context.Canceled) { + t.Fatal(turn, err) + } + select { + case <-out: + t.Fatal("pre-admission output was retained or closed") + default: + } + turn, err = e.StartTurn(t.Context(), "after", proto.TextInput("answer"), out) + if turn == nil || err == nil { + t.Fatal("uncertain submission lost owner", err) + } + if awaitExecutorTurn(t, turn, out).Reusable { + t.Fatal("failed submission was reusable") + } + }) + } +} + +func TestExecutorCloseRetainsPlanUntilReaped(t *testing.T) { + cmd := exec.Command(os.Args[0], "-test.run=^TestJSONRPCClientFakeCodexProcess$", "--") + cmd.Env = append(os.Environ(), "CODEX_RPC_FAKE_PROCESS=1", "GORACE=atexit_sleep_ms=0") + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + if err = cmd.Start(); err != nil { + t.Fatal(err) + } + rpc := NewJSONRPCClient(JSONRPCConfig{}) + rpc.cmd, rpc.stdin, rpc.alive = cmd, stdin, true + var reap sync.Once + t.Cleanup(func() { _ = cmd.Process.Kill(); reap.Do(rpc.waitChild) }) + _, cancel := context.WithCancel(t.Context()) + var cleaned atomic.Bool + e := &Executor{prepared: &Prepared{session: &Session{rpc: rpc, cancelFn: cancel}, plan: SessionPlan{Cleanup: func() { cleaned.Store(true) }}}} + ctx, stop := context.WithTimeout(t.Context(), 20*time.Millisecond) + defer stop() + if err = e.Close(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatal("unreaped close", err) + } + if cleaned.Load() { + t.Fatal("plan released before cleanup settled") + } + reap.Do(rpc.waitChild) + if err = e.Close(t.Context()); err != nil { + t.Fatal(err) + } + if !cleaned.Load() { + t.Fatal("cleanup retry did not release plan") + } +} diff --git a/apps/parsar-daemon/internal/agent/codex/executor_turn.go b/apps/parsar-daemon/internal/agent/codex/executor_turn.go new file mode 100644 index 000000000..216d26bbd --- /dev/null +++ b/apps/parsar-daemon/internal/agent/codex/executor_turn.go @@ -0,0 +1,163 @@ +package codex + +import ( + "context" + "encoding/json" + "errors" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" +) + +func (s *Session) AwaitSettlement(ctx context.Context) (agent.TurnSettlement, error) { + select { + case <-s.waitDone: + return s.settlement, s.settlementErr + case <-ctx.Done(): + return agent.TurnSettlement{}, ctx.Err() + } +} + +func (s *Session) settleExecutorTurn(startErr error) { + defer close(s.waitDone) + if startErr == nil { + select { + case <-s.outputDone: + case <-s.rpc.Done(): + s.emitTerminal("codex: connection closed before settlement", true) + case <-s.cancelCtx.Done(): + s.emitTerminal("codex: execution owner closed", true) + } + } + // Detach the old callbacks before waiting for their captured Turn and native + // replies. Later frames cannot acquire this Turn or redirect to a successor. + if !s.nativeSettled.Load() || startErr != nil || !s.rpc.Alive() { + s.cancelFn() + s.settlementErr = s.rpc.Close() + } + s.rpc.detachHandlers() + s.operationMu.Lock() + s.operationsClosed = true + s.operationMu.Unlock() + s.stopSteering() + s.stopFunctionCalls() + s.stopCodexInteractionTimers() + if !s.nativeSettled.Load() || startErr != nil || !s.rpc.Alive() { + s.cancelFn() + s.settlementErr = s.rpc.Close() + s.settlement = agent.TurnSettlement{Reason: "native_execution_unavailable"} + } else { + s.settlement = agent.TurnSettlement{Reusable: true} + } + s.closeRunOutput() + s.operations.Wait() + if s.subagents != nil { + s.subagents.mu.Lock() + err := s.subagents.cancelResult + s.subagents.mu.Unlock() + if err != nil { + s.settlement = agent.TurnSettlement{Reason: "child_settlement_unconfirmed"} + s.settlementErr = errors.Join(err, s.rpc.Close()) + } + } + // The terminal callback has returned before detachHandlers completes. A + // concurrent cancellation owns only this Turn and finishes before reuse. + s.operationMu.Lock() + ready := s.cancelReady + s.operationMu.Unlock() + if ready != nil { + <-ready + if s.cancelErr != nil { + s.settlement.Reusable = false + s.settlement.Reason = "cancellation_unconfirmed" + s.settlementErr = errors.Join(s.cancelErr, s.rpc.Close()) + } + } + s.cancelFn() +} + +func (s *Session) beginOperation() bool { + if s.executor == nil { + return true + } + s.operationMu.Lock() + defer s.operationMu.Unlock() + if s.operationsClosed { + return false + } + s.operations.Add(1) + return true +} +func (s *Session) endOperation() { + if s.executor != nil { + s.operations.Done() + } +} + +func (s *Session) cancelExecutorTurn(ctx context.Context) error { + s.operationMu.Lock() + if s.operationsClosed { + s.operationMu.Unlock() + _, err := s.AwaitSettlement(ctx) + return err + } + s.cancelOnce.Do(func() { + s.cancelled.Store(true) + s.cancelReady = make(chan struct{}) + go func() { + defer close(s.cancelReady) + s.stopFunctionCalls() + turnID, active := s.stopSteering() + s.stopCodexInteractionTimers() + if !s.terminal.Load() && active { + if turnID == "" { + s.cancelErr = errors.New("codex: cancellation has no native turn identity") + s.cancelFn() + _ = s.rpc.Close() + return + } + interruptCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + _, err := s.rpc.request(interruptCtx, "turn/interrupt", TurnInterruptParams{ThreadID: s.currentThreadID(), TurnID: turnID}, func(frame any) error { return s.rpc.writeFrameContext(interruptCtx, frame) }) + cancel() + if err != nil { + s.cancelErr = err + s.cancelFn() + _ = s.rpc.Close() + return + } + } + if s.subagents != nil { + s.cancelErr = s.cancelSubagentWork(s.cancelCtx) + } + }() + }) + s.operationMu.Unlock() + _, err := s.AwaitSettlement(ctx) + return err +} + +var _ agent.Turn = (*Session)(nil) + +// Server callbacks retain their originating Turn. MCP elicitation may be a +// standalone thread-scoped request in the native protocol; a supplied Turn ID +// must still match exactly. +func (s *Session) onServerRequest(method string, handler ServerRequestHandler) { + s.rpc.OnServerRequest(method, func(raw json.RawMessage, id any) (any, error) { + if s.executor != nil { + var scope struct { + ThreadID string `json:"threadId"` + TurnID string `json:"turnId"` + } + if json.Unmarshal(raw, &scope) != nil || !s.isRootThread(scope.ThreadID) || s.terminal.Load() || + (scope.TurnID != "" && !s.isRootTurn(scope.ThreadID, scope.TurnID)) || + (scope.TurnID == "" && method != "mcpServer/elicitation/request") { + return nil, errors.New("codex: request does not belong to the active turn") + } + } + if !s.beginOperation() { + return nil, errors.New("codex: turn has settled") + } + defer s.endOperation() + return handler(raw, id) + }) +} diff --git a/apps/parsar-daemon/internal/agent/codex/function_receipts.go b/apps/parsar-daemon/internal/agent/codex/function_receipts.go index 7a70518d4..8c594209d 100644 --- a/apps/parsar-daemon/internal/agent/codex/function_receipts.go +++ b/apps/parsar-daemon/internal/agent/codex/function_receipts.go @@ -26,6 +26,10 @@ type pendingFunction struct { } func (s *Session) SubmitFunctionResult(ctx context.Context, result proto.FunctionResultPayload) error { + if !s.beginOperation() { + return agent.ErrSteeringInactive + } + defer s.endOperation() if s.functions == nil { return agent.ErrUnknownFunctionCall } @@ -59,7 +63,11 @@ func (s *Session) SubmitFunctionResult(ctx context.Context, result proto.Functio // This operation owns both transport failure and native application. A generic // write deadline must not close the process after a native receipt has won. written := make(chan error, 1) + if !s.beginOperation() { + return agent.ErrUnknownFunctionCall + } go func() { + defer s.endOperation() if err := ctx.Err(); err != nil { written <- err return diff --git a/apps/parsar-daemon/internal/agent/codex/preparation.go b/apps/parsar-daemon/internal/agent/codex/preparation.go index 3c22dcb04..2d143f97b 100644 --- a/apps/parsar-daemon/internal/agent/codex/preparation.go +++ b/apps/parsar-daemon/internal/agent/codex/preparation.go @@ -130,10 +130,7 @@ func newPreparation(parent context.Context, req proto.PromptRequestPayload, cfg } if req.DisableExecutionEnvironment { if err := verifyNoExecutionEnvironment(cancelCtx, rpc); err != nil { - cancelFn() - _ = rpc.Close() - plan.Cleanup() - return nil, err + return p.preparationFailed(err) } } if s.observeSubagentIdentities { @@ -148,18 +145,12 @@ func newPreparation(parent context.Context, req proto.PromptRequestPayload, cfg } if plan.mcpServers != nil { if err := verifyMCPConfig(cancelCtx, rpc, plan); err != nil { - cancelFn() - _ = rpc.Close() - plan.Cleanup() - return nil, err + return p.preparationFailed(err) } } if len(skillRoots) > 0 { if err := setSkillExtraRoots(cancelCtx, rpc, skillRoots); err != nil { - cancelFn() - _ = rpc.Close() - plan.Cleanup() - return nil, fmt.Errorf("codex: register skill root: %w", err) + return p.preparationFailed(fmt.Errorf("codex: register skill root: %w", err)) } } @@ -168,6 +159,8 @@ func newPreparation(parent context.Context, req proto.PromptRequestPayload, cfg } func (p *Prepared) preparationFailed(cause error) (*Prepared, error) { - _ = p.Close() + if err := p.Close(); err != nil { + return p, errors.Join(cause, err) + } return nil, cause } diff --git a/apps/parsar-daemon/internal/agent/codex/preparation_helpers_test.go b/apps/parsar-daemon/internal/agent/codex/preparation_helpers_test.go index d1b5a7267..237d4f329 100644 --- a/apps/parsar-daemon/internal/agent/codex/preparation_helpers_test.go +++ b/apps/parsar-daemon/internal/agent/codex/preparation_helpers_test.go @@ -3,6 +3,7 @@ package codex import ( "bufio" "encoding/json" + "fmt" "os" "path/filepath" "strings" @@ -139,6 +140,9 @@ func TestPreparationFakeCodexProcess(t *testing.T) { frames := json.NewEncoder(log) output := json.NewEncoder(os.Stdout) scanner := bufio.NewScanner(os.Stdin) + turnNumber := 0 + currentTurn := "" + executorMode := os.Getenv("OAC_TEST_EXECUTOR_MODE") for scanner.Scan() { var frame preparationFrame if json.Unmarshal(scanner.Bytes(), &frame) != nil { @@ -182,11 +186,41 @@ func TestPreparationFakeCodexProcess(t *testing.T) { } result = map[string]any{"thread": map[string]string{"id": "fixture-native-thread"}, "model": "fixture-model"} case "turn/start": + if executorMode != "" { + turnNumber++ + currentTurn = fmt.Sprintf("fixture-turn-%d", turnNumber) + if turnNumber > 1 { + _ = output.Encode(map[string]any{"method": "turn/started", "params": map[string]any{"threadId": "fixture-native-thread", "turn": map[string]string{"id": fmt.Sprintf("fixture-turn-%d", turnNumber-1)}}}) + } + if executorMode == "disconnect" { + os.Exit(0) + } + if executorMode == "start-error" { + _ = output.Encode(map[string]any{"id": frame.ID, "error": map[string]any{"code": -32603, "message": "start rejected"}}) + continue + } + result = map[string]any{"turn": map[string]string{"id": currentTurn}} + break + } result = map[string]any{"turn": map[string]string{"id": "fixture-native-turn"}} } if output.Encode(map[string]any{"jsonrpc": "2.0", "id": frame.ID, "result": result}) != nil { os.Exit(5) } + + if executorMode != "" && (frame.Method == "turn/start" || frame.Method == "turn/interrupt") { + if frame.Method == "turn/start" { + _ = output.Encode(map[string]any{"method": "turn/started", "params": map[string]any{"threadId": "fixture-native-thread", "turn": map[string]string{"id": currentTurn}}}) + } + if frame.Method == "turn/interrupt" || !strings.Contains(string(frame.Params), "hold") { + status := "completed" + if frame.Method == "turn/interrupt" { + status = "interrupted" + } + _ = output.Encode(map[string]any{"method": "turn/completed", "params": map[string]any{"threadId": "fixture-native-thread", "turn": map[string]string{"id": currentTurn, "status": status}}}) + } + continue + } if frame.Method == "turn/start" { _ = output.Encode(map[string]any{"jsonrpc": "2.0", "method": "turn/started", "params": map[string]any{"threadId": "fixture-native-thread", "turn": map[string]string{"id": "fixture-native-turn"}}}) if os.Getenv("OAC_TEST_PREPARATION_OBSERVE") == "1" { diff --git a/apps/parsar-daemon/internal/agent/codex/preparation_router_test.go b/apps/parsar-daemon/internal/agent/codex/preparation_router_test.go index 15bf5d432..45eb0e8a3 100644 --- a/apps/parsar-daemon/internal/agent/codex/preparation_router_test.go +++ b/apps/parsar-daemon/internal/agent/codex/preparation_router_test.go @@ -30,6 +30,7 @@ func TestPreparationRouterRetainsActualNativeChild(t *testing.T) { for _, start := range []bool{false, true} { t.Run(map[bool]string{false: "disconnect-before-start", true: "transfer-and-cancel"}[start], func(t *testing.T) { req, cfg, root := preparationFixture(t) + t.Setenv("OAC_TEST_EXECUTOR_MODE", "complete") environment, session := uuid.NewString(), uuid.NewString() if err := os.MkdirAll(req.WorkDir, 0700); err != nil { t.Fatal(err) @@ -58,13 +59,13 @@ func TestPreparationRouterRetainsActualNativeChild(t *testing.T) { return nil, errors.New("ordinary Factory must not run") }) prepared := make(chan *Prepared, 1) - registry.RegisterPreparation("codex", false, func(ctx context.Context, req proto.PromptRequestPayload) (agent.Prepared, error) { - p, err := newPreparation(ctx, req, cfg) + registry.RegisterExecutor("codex", func(ctx context.Context, req proto.PromptRequestPayload) (agent.Executor, error) { + e, err := newExecutor(ctx, req, cfg) if err != nil { return nil, err } - prepared <- p - return p, nil + prepared <- e.prepared + return e, nil }) sender := make(preparationWireSender, 64) r, err := dispatch.New(dispatch.Config{Registry: registry, Sender: sender, LocalWorkspace: binding}) @@ -110,7 +111,7 @@ func TestPreparationRouterRetainsActualNativeChild(t *testing.T) { } } } - send(proto.TypeExecutionPrepare, "prepare-request", proto.ExecutionPreparePayload{Configuration: req}) + send(proto.TypeExecutionPrepare, "prepare-request", proto.ExecutionPreparePayload{SessionID: session, Configuration: req}) ready := await("ready") p := <-prepared assertPreparationOnly(t, root) @@ -119,7 +120,7 @@ func TestPreparationRouterRetainsActualNativeChild(t *testing.T) { t.Fatal("preparation became a Run") } if start { - input := proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "actual-run", Input: proto.TextInput("actual input")} + input := proto.ExecutionStartPayload{ExecutorID: ready.ExecutorID, Handle: ready.Handle, RunID: "actual-run", Input: proto.TextInput("hold")} send(proto.TypeExecutionStart, "prepare-request", input) await("started") frames := waitPreparationMethod(t, root, "turn/start") diff --git a/apps/parsar-daemon/internal/agent/codex/prepared.go b/apps/parsar-daemon/internal/agent/codex/prepared.go index a83cf6ffe..b4e3a0b78 100644 --- a/apps/parsar-daemon/internal/agent/codex/prepared.go +++ b/apps/parsar-daemon/internal/agent/codex/prepared.go @@ -88,7 +88,9 @@ func (p *Prepared) Close() error { p.mu.Unlock() p.session.cancelFn() err := p.session.rpc.Close() - p.plan.Cleanup() + if err == nil { + p.plan.Cleanup() + } return err } diff --git a/apps/parsar-daemon/internal/agent/codex/rpc.go b/apps/parsar-daemon/internal/agent/codex/rpc.go index ec52afebb..d078c4588 100644 --- a/apps/parsar-daemon/internal/agent/codex/rpc.go +++ b/apps/parsar-daemon/internal/agent/codex/rpc.go @@ -91,11 +91,13 @@ type JSONRPCClient struct { pendingMu sync.Mutex pending map[string]*pendingRequest + handlerWork sync.WaitGroup handlersMu sync.RWMutex notifHandlers map[string][]NotificationHandler serverReqHandlers map[string]ServerRequestHandler anyNotifHandler NotificationHandler + readers sync.WaitGroup closeOnce sync.Once doneCh chan struct{} } @@ -206,8 +208,9 @@ func (c *JSONRPCClient) Start(ctx context.Context, init InitializeParams) (Initi c.alive = true c.mu.Unlock() - go c.readStdoutLoop() - go c.pumpStderr() + c.readers.Add(2) + go func() { defer c.readers.Done(); c.readStdoutLoop() }() + go func() { defer c.readers.Done(); c.pumpStderr() }() go c.waitChild() initCtx, cancel := context.WithTimeout(ctx, rpcInitTimeout) @@ -415,7 +418,13 @@ func (c *JSONRPCClient) handleNotification(method string, rawFrame []byte) { c.handlersMu.RLock() handlers := append([]NotificationHandler{}, c.notifHandlers[method]...) any := c.anyNotifHandler + if len(handlers) > 0 || any != nil { + c.handlerWork.Add(1) + } c.handlersMu.RUnlock() + if len(handlers) > 0 || any != nil { + defer c.handlerWork.Done() + } if len(handlers) == 0 && any == nil { c.cfg.Logger.Debug("codex rpc unhandled notification", "tag", c.cfg.LogTag, "method", method) return @@ -438,6 +447,9 @@ func (c *JSONRPCClient) handleServerRequest(rawFrame []byte, rawID json.RawMessa c.handlersMu.RLock() h, ok := c.serverReqHandlers[method] + if ok { + c.handlerWork.Add(1) + } c.handlersMu.RUnlock() if !ok { // Method-not-found per JSON-RPC 2.0 — codex hangs if we drop it. @@ -445,6 +457,7 @@ func (c *JSONRPCClient) handleServerRequest(rawFrame []byte, rawID json.RawMessa return } go func() { + defer c.handlerWork.Done() defer func() { if r := recover(); r != nil { _ = c.SendServerError(id, -32603, fmt.Sprintf("panic in %s handler: %v", method, r), nil) @@ -525,3 +538,25 @@ func truncate(b []byte, n int) []byte { } return b[:n] } + +// detachHandlers fences registration before joining captured callbacks, including +// asynchronous server replies. New notifications have no Turn destination. +func (c *JSONRPCClient) detachHandlers() { + c.handlersMu.Lock() + clear(c.notifHandlers) + clear(c.serverReqHandlers) + c.anyNotifHandler = nil + c.handlersMu.Unlock() + c.handlerWork.Wait() +} + +func (c *JSONRPCClient) awaitReaders(ctx context.Context) error { + done := make(chan struct{}) + go func() { c.readers.Wait(); close(done) }() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} diff --git a/apps/parsar-daemon/internal/agent/codex/server_requests.go b/apps/parsar-daemon/internal/agent/codex/server_requests.go index 69aa69d3c..656ac6ca7 100644 --- a/apps/parsar-daemon/internal/agent/codex/server_requests.go +++ b/apps/parsar-daemon/internal/agent/codex/server_requests.go @@ -195,6 +195,10 @@ func (s *Session) handleCodexUserInput(raw json.RawMessage, rpcID any) (any, err } func (s *Session) submitCodexPermission(requestID string, decision proto.PermissionDecisionPayload) error { + if !s.beginOperation() { + return agent.ErrSteeringInactive + } + defer s.endOperation() s.interactions.mu.Lock() pending, ok := s.interactions.permissions[requestID] if ok { @@ -243,6 +247,10 @@ func (s *Session) sendCodexPermissionReply(pending pendingCodexPermission, appro } func (s *Session) submitCodexUserInput(askID string, decision proto.PromptForUserChoiceDecisionPayload) error { + if !s.beginOperation() { + return agent.ErrSteeringInactive + } + defer s.endOperation() s.interactions.mu.Lock() pending, ok := s.interactions.asks[askID] if ok { @@ -311,6 +319,10 @@ func (s *Session) submitCodexUserInput(askID string, decision proto.PromptForUse } func (s *Session) expireCodexPermission(requestID string) { + if !s.beginOperation() { + return + } + defer s.endOperation() s.interactions.mu.Lock() pending, ok := s.interactions.permissions[requestID] if ok { @@ -326,6 +338,10 @@ func (s *Session) expireCodexPermission(requestID string) { } func (s *Session) expireCodexAsk(askID string) { + if !s.beginOperation() { + return + } + defer s.endOperation() s.interactions.mu.Lock() pending, ok := s.interactions.asks[askID] if ok { diff --git a/apps/parsar-daemon/internal/agent/codex/session.go b/apps/parsar-daemon/internal/agent/codex/session.go index f9f624f1d..bbd808693 100644 --- a/apps/parsar-daemon/internal/agent/codex/session.go +++ b/apps/parsar-daemon/internal/agent/codex/session.go @@ -63,6 +63,15 @@ func Factory(ctx context.Context, req proto.PromptRequestPayload, out chan<- pro // 5. turn/completed emits TypeDone + closes out. Cancel can short-cut // this by killing the child early. type Session struct { + retiredTurns map[string]bool + executor *Executor + outputDone chan struct{} + nativeSettled atomic.Bool + settlement agent.TurnSettlement + settlementErr error + operationMu sync.Mutex + operations sync.WaitGroup + operationsClosed bool toolEnvironment bool nativeHome string subagents *subagentObservations @@ -152,14 +161,14 @@ func (s *Session) registerHandlers() { rpc.OnNotification("error", s.onErrorNotif) rpc.OnNotification("hook/completed", s.onToolEnvironmentHook) - rpc.OnServerRequest("item/commandExecution/requestApproval", s.handleCodexCommandApproval) - rpc.OnServerRequest("item/fileChange/requestApproval", s.handleCodexFileApproval) - rpc.OnServerRequest("item/permissions/requestApproval", s.handleCodexPermissionsApproval) + s.onServerRequest("item/commandExecution/requestApproval", s.handleCodexCommandApproval) + s.onServerRequest("item/fileChange/requestApproval", s.handleCodexFileApproval) + s.onServerRequest("item/permissions/requestApproval", s.handleCodexPermissionsApproval) // Older app-server releases used this unseparated method name. - rpc.OnServerRequest("item/permissionsRequestApproval", s.handleCodexPermissionsApproval) - rpc.OnServerRequest("item/tool/requestUserInput", s.handleCodexUserInput) - rpc.OnServerRequest("item/tool/call", s.handleFunctionCall) - rpc.OnServerRequest("mcpServer/elicitation/request", s.handleCodexMCPElicitation) + s.onServerRequest("item/permissionsRequestApproval", s.handleCodexPermissionsApproval) + s.onServerRequest("item/tool/requestUserInput", s.handleCodexUserInput) + s.onServerRequest("item/tool/call", s.handleFunctionCall) + s.onServerRequest("mcpServer/elicitation/request", s.handleCodexMCPElicitation) } func (s *Session) onTurnStarted(raw json.RawMessage) { @@ -193,6 +202,7 @@ func (s *Session) onTurnCompleted(raw json.RawMessage) { if json.Unmarshal(raw, &p) != nil || !s.isRootTurn(p.ThreadID, p.Turn.ID) { return } + s.nativeSettled.Store(true) s.stopSteering() usage := p.Turn.Usage @@ -392,6 +402,9 @@ func (s *Session) emitUsage(u TurnUsage) { } func (s *Session) emitTerminal(message string, asError bool) { + if s.executor != nil { + defer s.finishAfterTerminal() + } if !s.terminal.CompareAndSwap(false, true) { return } @@ -442,6 +455,9 @@ func (s *Session) closeOut() { s.outMu.Lock() s.outClosed = true close(s.out) + if s.outputDone != nil { + close(s.outputDone) + } s.outMu.Unlock() }) } diff --git a/apps/parsar-daemon/internal/agent/codex/session_cancel.go b/apps/parsar-daemon/internal/agent/codex/session_cancel.go index 2b1fe0080..25024d5b5 100644 --- a/apps/parsar-daemon/internal/agent/codex/session_cancel.go +++ b/apps/parsar-daemon/internal/agent/codex/session_cancel.go @@ -7,6 +7,9 @@ import ( ) func (s *Session) Cancel(ctx context.Context) error { + if s.executor != nil { + return s.cancelExecutorTurn(ctx) + } s.cancelled.Store(true) s.cancelOnce.Do(func() { s.cancelReady = make(chan struct{}) diff --git a/apps/parsar-daemon/internal/agent/codex/session_run.go b/apps/parsar-daemon/internal/agent/codex/session_run.go index 170708625..21736ba88 100644 --- a/apps/parsar-daemon/internal/agent/codex/session_run.go +++ b/apps/parsar-daemon/internal/agent/codex/session_run.go @@ -16,15 +16,32 @@ func (s *Session) run(plan SessionPlan, req proto.PromptRequestPayload) { defer s.cleanup() defer s.closeRunOutput() - if err := s.resolveThread(req, plan); err != nil { + if err := s.startNative(s.cancelCtx, plan, req); err != nil { s.emitTerminal(err.Error(), true) return } + // Block until terminal handlers close the RPC child or cancellation arrives. + select { + case <-s.rpc.Done(): + if !s.cancelled.Load() && s.cancelCtx.Err() == nil { + s.emitTerminal("codex: connection closed before the run completed", true) + } + case <-s.cancelCtx.Done(): + _ = s.rpc.Close() + } +} + +func (s *Session) startNative(ctx context.Context, plan SessionPlan, req proto.PromptRequestPayload) error { + if s.currentThreadID() == "" { + if err := s.resolveThread(req, plan); err != nil { + return err + } + } + input, err := nativeInput(req.Input) if err != nil { - s.emitTerminal("codex: empty prompt", true) - return + return err } turnParams := TurnStartParams{ ThreadID: s.currentThreadID(), @@ -33,8 +50,7 @@ func (s *Session) run(plan SessionPlan, req proto.PromptRequestPayload) { if plan.CollaborationMode != "" { model := strings.TrimSpace(s.resolvedModel) if model == "" { - s.emitTerminal("codex: collaboration mode requires a resolved model", true) - return + return fmt.Errorf("codex: collaboration mode requires a resolved model") } var developerInstructions *string if plan.SystemPrompt != "" { @@ -48,22 +64,13 @@ func (s *Session) run(plan SessionPlan, req proto.PromptRequestPayload) { }, } } - turnCtx, turnCancel := context.WithTimeout(s.cancelCtx, 10*time.Second) + turnCtx, turnCancel := context.WithTimeout(ctx, 10*time.Second) _, ackErr := s.rpc.requestWithResult(turnCtx, "turn/start", turnParams, s.bindTurnResult) turnCancel() if ackErr != nil { s.cfg.logger.Warn("codex: turn/start ack failed", "run_id", s.runID, "err", ackErr) - s.emitTerminal(fmt.Sprintf("codex: turn/start: %v", ackErr), true) - return + return fmt.Errorf("codex: turn/start: %w", ackErr) } - // Block until terminal handlers close the RPC child or cancellation arrives. - select { - case <-s.rpc.Done(): - if !s.cancelled.Load() && s.cancelCtx.Err() == nil { - s.emitTerminal("codex: connection closed before the run completed", true) - } - case <-s.cancelCtx.Done(): - _ = s.rpc.Close() - } + return nil } diff --git a/apps/parsar-daemon/internal/agent/codex/session_steering.go b/apps/parsar-daemon/internal/agent/codex/session_steering.go index 4ad2716fe..f0d6be931 100644 --- a/apps/parsar-daemon/internal/agent/codex/session_steering.go +++ b/apps/parsar-daemon/internal/agent/codex/session_steering.go @@ -38,6 +38,10 @@ func (s *Session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerP } func (s *Session) steer(ctx context.Context, input proto.PromptSteerPayload, written func()) error { + if !s.beginOperation() { + return agent.ErrSteeringInactive + } + defer s.endOperation() ctx, cancel := context.WithCancel(ctx) defer cancel() s.steering.mu.Lock() @@ -96,7 +100,7 @@ func (s *Session) steer(ctx context.Context, input proto.PromptSteerPayload, wri } func (s *Session) startSteering(threadID, turnID string) bool { - if !s.isRootThread(threadID) || turnID == "" { + if !s.isRootThread(threadID) || turnID == "" || s.retiredTurns[turnID] { return false } s.steering.mu.Lock() diff --git a/apps/parsar-daemon/internal/agent/codex/session_usage.go b/apps/parsar-daemon/internal/agent/codex/session_usage.go index 309d05b99..17e672284 100644 --- a/apps/parsar-daemon/internal/agent/codex/session_usage.go +++ b/apps/parsar-daemon/internal/agent/codex/session_usage.go @@ -30,7 +30,7 @@ func (s *Session) onUsageUpdated(raw json.RawMessage) { s.usageMu.Unlock() return } - if s.terminal.Load() { + if s.terminal.Load() || s.retiredTurns[p.TurnID] { return } var observed *TurnUsage diff --git a/apps/parsar-daemon/internal/agent/executor.go b/apps/parsar-daemon/internal/agent/executor.go new file mode 100644 index 000000000..1e1346cf2 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/executor.go @@ -0,0 +1,61 @@ +package agent + +import ( + "context" + "fmt" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +// TurnSettlement describes native readiness after this Turn has fully drained. +// A false result must carry a reason; it requires confirmed Executor.Close. +type TurnSettlement struct { + Reusable bool + Reason string +} + +// Turn owns one output stream and never retargets cancellation to a successor. +type Turn interface { + Session + CancellationOutcome() proto.DonePayload + // Success confirms closed output and settled native input, function, + // interaction and child-work obligations. Errors cannot prove cancellation. + AwaitSettlement(context.Context) (TurnSettlement, error) +} + +// Executor retains a fixed native configuration across independently owned Turns. +type Executor interface { + // A nil Turn guarantees no input was submitted or output retained and leaves + // out with the caller. A non-nil Turn owns out, including on error; input + // may have been submitted and must never be replayed automatically. + StartTurn(context.Context, string, proto.MessageInput, chan<- proto.Envelope) (Turn, error) + // Success confirms all native work and owned output streams have stopped. + // It does not establish an otherwise unknown Turn result. An error retains + // resource ownership; another caller may await or retry Close. + Close(context.Context) error +} + +// A failed factory retains any unconfirmed cleanup in its non-nil Executor. +type ExecutorFactory func(context.Context, proto.PromptRequestPayload) (Executor, error) + +func (r *Registry) RegisterExecutor(kind string, factory ExecutorFactory) { + r.mu.Lock() + defer r.mu.Unlock() + info, exists := r.kinds[kind] + if !exists || factory == nil { + panic("agent.Registry.RegisterExecutor: registered kind and factory required") + } + r.executors[kind] = factory + info.Capabilities.Preparation = true + r.kinds[kind] = info +} + +func (r *Registry) ResolveExecutor(kind string) (ExecutorFactory, error) { + r.mu.RLock() + defer r.mu.RUnlock() + factory := r.executors[kind] + if factory == nil { + return nil, fmt.Errorf("agent: executor unavailable for %q", kind) + } + return factory, nil +} diff --git a/apps/parsar-daemon/internal/agent/mcode/connection.go b/apps/parsar-daemon/internal/agent/mcode/connection.go new file mode 100644 index 000000000..420155530 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/mcode/connection.go @@ -0,0 +1,146 @@ +package mcode + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "strconv" + "sync" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent/clirunner" +) + +// connection is the process and ACP owner. It keeps draining while no Turn owns +// output; RPC identities remain unique across every Turn on this connection. +type connection struct { + process *clirunner.Process + exited chan struct{} + exitErr error + writeMu sync.Mutex + transportMu sync.Mutex + nextID int + responses map[string]chan rpcFrame + current *Session +} + +func (c *connection) read() { + defer close(c.exited) + stderrDone := make(chan struct{}) + go func() { defer close(stderrDone); _, _ = io.Copy(io.Discard, c.process.Stderr) }() + scanner := bufio.NewScanner(c.process.Stdout) + scanner.Buffer(make([]byte, 64*1024), 16*1024*1024) + var readErr error + for scanner.Scan() { + var frame rpcFrame + if json.Unmarshal(scanner.Bytes(), &frame) != nil { + readErr = fmt.Errorf("mcode: malformed ACP response") + c.process.Cancel() + break + } + c.transportMu.Lock() + response, owner := c.responses[string(frame.ID)], c.current + c.transportMu.Unlock() + if frame.Method == "" && response != nil { + select { + case response <- frame: + default: + } + continue + } + if owner == nil { + // A late request has no Turn authority. Never hold it for a successor. + if frame.Method != "" && len(frame.ID) > 0 { + if err := c.write(rpcFrame{JSONRPC: "2.0", ID: frame.ID, Error: &rpcError{Code: -32601, Message: "No active Turn"}}); err != nil { + readErr = err + c.process.Cancel() + break + } + } + continue + } + select { + case owner.frames <- frame: + case <-owner.finished: + case <-c.process.Context().Done(): + } + } + if scanner.Err() != nil { + readErr = fmt.Errorf("mcode: ACP stream read failed") + c.process.Cancel() + } + waitErr := c.process.Wait() + <-stderrDone + if readErr != nil { + c.exitErr = readErr + } else { + c.exitErr = waitErr + } +} + +func (c *connection) write(frame rpcFrame) error { + c.writeMu.Lock() + defer c.writeMu.Unlock() + return json.NewEncoder(c.process.Stdin).Encode(frame) +} + +func (c *connection) reserveResponse() (string, chan rpcFrame) { + c.transportMu.Lock() + defer c.transportMu.Unlock() + c.nextID++ + id := strconv.Itoa(c.nextID) + reply := make(chan rpcFrame, 1) + c.responses[id] = reply + return id, reply +} + +func (c *connection) removeResponse(id string) { + c.transportMu.Lock() + delete(c.responses, id) + c.transportMu.Unlock() +} + +func (c *connection) setCurrent(s *Session) { + c.transportMu.Lock() + c.current = s + c.transportMu.Unlock() +} + +// An ordered ACP control response fences notifications left in the pipe by the +// preceding prompt. No Turn is attached while this barrier drains them. +func (c *connection) barrier(ctx context.Context, session, model string) error { + id, reply := c.reserveResponse() + defer c.removeResponse(id) + raw, _ := json.Marshal(map[string]string{"sessionId": session, "configId": "model", "value": model}) + err := c.writeContext(ctx, rpcFrame{JSONRPC: "2.0", ID: json.RawMessage(id), Method: "session/set_config_option", Params: raw}) + if err != nil { + return err + } + select { + case frame := <-reply: + if frame.Error != nil { + return fmt.Errorf("mcode: native readiness barrier failed") + } + return nil + case <-c.exited: + return fmt.Errorf("mcode: native process exited") + case <-ctx.Done(): + return ctx.Err() + } +} + +// Drain the deadline callback before releasing the caller's Turn/start guard; +// an old writer deadline must never kill a subsequent owner. +func (c *connection) writeContext(ctx context.Context, frame rpcFrame) error { + callbackDone := make(chan struct{}) + stop := context.AfterFunc(ctx, func() { + defer close(callbackDone) + c.process.Cancel() + }) + err := c.write(frame) + if !stop() { + <-callbackDone + } + return err +} diff --git a/apps/parsar-daemon/internal/agent/mcode/environment_mcp_test.go b/apps/parsar-daemon/internal/agent/mcode/environment_mcp_test.go index a47e4eb1c..e9cb6f18c 100644 --- a/apps/parsar-daemon/internal/agent/mcode/environment_mcp_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/environment_mcp_test.go @@ -114,7 +114,7 @@ func TestEnvironmentMCPCancelSettlesPendingObservationBeforeDone(t *testing.T) { if err != nil { t.Fatal(err) } - t.Cleanup(func() { _ = session.Cancel(context.Background()) }) + t.Cleanup(func() { _ = resource.(*prepared).Cancel(context.Background()) }) select { case event := <-out: var call proto.ToolCallPayload @@ -124,8 +124,12 @@ func TestEnvironmentMCPCancelSettlesPendingObservationBeforeDone(t *testing.T) { case <-ctx.Done(): t.Fatal("MCP start was not observed") } - if err := session.Cancel(ctx); err != nil { - t.Fatal(err) + if err := session.Cancel(ctx); err == nil { + t.Fatal("unconfirmed MCP completion authorized reusable cancellation") + } + settlement, err := session.(*Session).AwaitSettlement(ctx) + if err != nil || settlement.Reusable { + t.Fatal("pending MCP work retained a reusable owner", settlement, err) } closed, done := false, false for event := range out { diff --git a/apps/parsar-daemon/internal/agent/mcode/execution.go b/apps/parsar-daemon/internal/agent/mcode/execution.go index b8c94bbed..6a3eb80c4 100644 --- a/apps/parsar-daemon/internal/agent/mcode/execution.go +++ b/apps/parsar-daemon/internal/agent/mcode/execution.go @@ -13,7 +13,7 @@ func SupportsExecution(version string) bool { } func validateExecutionRequest(req proto.PromptRequestPayload) error { - if !req.ReleaseOnCompletion || !req.DisableExecutionEnvironment || req.WorkDir != "" || req.AgentStateKey == "" || req.LocalEnvironment != nil || req.RequireExistingNativeSession || len(req.FunctionTools) != 0 || (req.MCPHTTPServers != nil && len(*req.MCPHTTPServers) != 0) { + if !req.DisableExecutionEnvironment || req.WorkDir != "" || req.AgentStateKey == "" || req.LocalEnvironment != nil || req.RequireExistingNativeSession || len(req.FunctionTools) != 0 || (req.MCPHTTPServers != nil && len(*req.MCPHTTPServers) != 0) { return fmt.Errorf("mcode: unsupported execution configuration") } if req.ExecutionControls == nil || req.ExecutionControls.OutputFormat != nil || req.ExecutionControls.WebSearch != "disabled" || (req.ExecutionControls.TextVerbosity != "" && req.ExecutionControls.TextVerbosity != "medium") { diff --git a/apps/parsar-daemon/internal/agent/mcode/executor.go b/apps/parsar-daemon/internal/agent/mcode/executor.go new file mode 100644 index 000000000..632bb6a40 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/mcode/executor.go @@ -0,0 +1,173 @@ +package mcode + +import ( + "context" + "errors" + "fmt" + "strings" + "sync" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +var errTurnCancelled = errors.New("mcode: Turn cancelled before submission") + +type executor struct { + mu sync.Mutex + connection *connection + req proto.PromptRequestPayload + opts launchOptions + nativeSession, model string + active *Session + starting, closed, invalid bool +} + +var _ agent.Executor = (*executor)(nil) +var _ agent.Turn = (*Session)(nil) + +// NewExecutorFactory fixes the deployment workspace once; nil selects none. +func NewExecutorFactory(config *WorkspaceConfig) agent.ExecutorFactory { + var frozen *WorkspaceConfig + if config != nil { + value := *config + value.ProtectedDirs = append([]string(nil), config.ProtectedDirs...) + value.AllowedDomains = append([]string(nil), config.AllowedDomains...) + frozen = &value + } + return func(ctx context.Context, req proto.PromptRequestPayload) (agent.Executor, error) { + if ctx == nil { + ctx = context.Background() + } + if req.RunID != "" || len(req.Input) != 0 || req.ConversationID != "" { + return nil, fmt.Errorf("mcode: Executor configuration cannot contain Turn input") + } + var err error + var opts launchOptions + binary := defaultBinary() + if frozen == nil { + opts, err = prepareOptions(ctx, req) + } else { + binary = frozen.Binary + opts, err = prepareWorkspaceOptions(ctx, *frozen, req) + } + if err != nil { + return nil, err + } + return newExecutor(ctx, req, opts, binary) + } +} + +func newExecutor(ctx context.Context, req proto.PromptRequestPayload, opts launchOptions, binary string) (*executor, error) { + bootstrap, err := launch(ctx, req, opts, binary, nil) + if err != nil { + return nil, err + } + e := &executor{connection: bootstrap.connection, req: req, opts: opts} + err = bootstrap.prepareNative() + e.connection.setCurrent(nil) + close(bootstrap.finished) + if err == nil { + err = ctx.Err() + } + if err != nil { + cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if closeErr := e.Close(cleanup); closeErr != nil { + return e, err + } + return nil, err + } + e.nativeSession, e.model = bootstrap.sessionID, bootstrap.nativeModel + return e, nil +} + +func (e *executor) StartTurn(ctx context.Context, runID string, input proto.MessageInput, out chan<- proto.Envelope) (agent.Turn, error) { + if ctx == nil { + ctx = context.Background() + } + if strings.TrimSpace(runID) == "" || input.Validate() != nil || out == nil || ctx.Err() != nil { + return nil, fmt.Errorf("mcode: Turn requires live context, identity, input and output") + } + text, err := input.TextOnly() + if err != nil { + return nil, err + } + e.mu.Lock() + if e.closed || e.invalid || e.starting || e.active != nil { + e.mu.Unlock() + return nil, fmt.Errorf("mcode: Executor is unavailable") + } + e.starting = true + e.mu.Unlock() + + // No model input is sent and no output channel retained until this fence. + barrier, cancel := context.WithTimeout(ctx, 60*time.Second) + err = e.connection.barrier(barrier, e.nativeSession, e.model) + cancel() + e.mu.Lock() + defer e.mu.Unlock() + e.starting = false + if err != nil || e.closed || ctx.Err() != nil { + if err != nil { + e.invalid = true + return nil, err + } + return nil, fmt.Errorf("mcode: Executor closed before input") + } + select { + case <-e.connection.exited: + e.invalid = true + return nil, fmt.Errorf("mcode: native process exited") + default: + } + req := e.req + req.RunID, req.Input = runID, proto.TextInput(text) + s := newTurnSession(e.connection.process.Context(), req, e.opts, e.connection, out) + s.executor, s.sessionID, s.nativeModel = e, e.nativeSession, e.model + s.settled, s.inputDone = make(chan struct{}), make(chan struct{}) + s.outputContext, s.outputCancel = context.WithCancel(context.Background()) + e.active = s + e.connection.setCurrent(s) + go s.runExecutorTurn() + go func() { + select { + case <-ctx.Done(): + cancellation, stop := context.WithTimeout(context.Background(), 10*time.Second) + defer stop() + _ = s.Cancel(cancellation) + case <-s.settled: + } + }() + return s, nil +} + +// Close keeps the native owner reachable until process, pipes and active Turn +// settlement all finish. Repeating it waits on those same owners. +func (e *executor) Close(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + e.mu.Lock() + e.closed = true + current := e.active + if current != nil { + current.outputCancel() + } + e.connection.process.Cancel() + e.mu.Unlock() + select { + case <-e.connection.exited: + case <-ctx.Done(): + return ctx.Err() + } + if current != nil { + select { + case <-current.settled: + case <-ctx.Done(): + return ctx.Err() + } + } + return nil +} diff --git a/apps/parsar-daemon/internal/agent/mcode/executor_native_test.go b/apps/parsar-daemon/internal/agent/mcode/executor_native_test.go new file mode 100644 index 000000000..4a1e23949 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/mcode/executor_native_test.go @@ -0,0 +1,176 @@ +package mcode + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +// This opt-in test makes real model calls and uses an isolated native home. +// Provider options are read from a private file and never included in failures. +func TestNativeMCodeExecutorReuse(t *testing.T) { + binary, options := os.Getenv("OAC_RUNTIME_MCODE_BIN"), os.Getenv("OAC_TEST_MCODE_REAL_OPTIONS") + if binary == "" || options == "" { + t.Skip("native executable and private provider options required") + } + ctx, cancel := context.WithTimeout(t.Context(), 8*time.Minute) + defer cancel() + if version, err := CheckCLIAvailable(ctx, binary); err != nil || version != SupportedVersion { + t.Fatal("pinned native version verification failed") + } + raw, err := os.ReadFile(options) + if err != nil { + t.Fatal("private provider options unavailable") + } + req := executionRequest(t) + req.ReleaseOnCompletion = false + req.RunID, req.Input, req.ConversationID = "", nil, "" + if json.Unmarshal(raw, &req.AgentOptions) != nil { + t.Fatal("invalid private provider options") + } + value, err := NewExecutorFactory(nil)(ctx, req) + if err != nil { + t.Fatal("native Executor preparation failed") + } + e := value.(*executor) + defer func() { + cleanup, stop := context.WithTimeout(context.Background(), 15*time.Second) + defer stop() + if e.Close(cleanup) != nil { + t.Error("native owner cleanup failed") + } + }() + pid, nativeID := e.connection.process.Cmd.Process.Pid, e.nativeSession + assertOwner := func() { + t.Helper() + if e.connection.process.Cmd.Process.Pid != pid || e.nativeSession != nativeID { + t.Fatal("native owner changed") + } + select { + case <-e.connection.exited: + t.Fatal("native process exited") + default: + } + } + run := func(id, prompt string) agent.Turn { + t.Helper() + out := make(chan proto.Envelope, 256) + turn, startErr := e.StartTurn(ctx, id, proto.TextInput(prompt), out) + if startErr != nil || turn == nil { + t.Fatal("native Turn admission failed") + } + content := "" + doneCount := 0 + for event := range out { + if event.ID != id { + t.Fatal("event crossed Turn boundary") + } + switch event.Type { + case proto.TypeError: + t.Fatal("native Turn reported an error") + case proto.TypeDone: + var done proto.DonePayload + if json.Unmarshal(event.Payload, &done) != nil { + t.Fatal("invalid completion") + } + content = done.Content + doneCount++ + } + } + settlement, settleErr := turn.AwaitSettlement(ctx) + if settleErr != nil || !settlement.Reusable { + t.Fatal("normal Turn did not establish reusable settlement") + } + if doneCount != 1 || !strings.Contains(content, "ORCHID-72-BLUE") { + t.Fatal("native conversation did not retain the private test marker") + } + assertOwner() + t.Logf("Verified marker recall and retained native owner for %s", id) + return turn + } + first := run("first", "Remember the exact marker ORCHID-72-BLUE for this conversation. Reply with only that marker.") + run("second", "What exact marker did I ask you to remember? Reply with only that marker.") + out := make(chan proto.Envelope, 256) + interrupted, err := e.StartTurn(ctx, "cancelled", proto.TextInput("Produce 2000 numbered lines now. Each line must contain a different sentence about rain. Start immediately at line 1 and continue without stopping or summarizing."), out) + if err != nil || interrupted == nil { + t.Fatal("cancellation Turn admission failed") + } + streaming := false + for !streaming { + select { + case event, ok := <-out: + if !ok || event.Type == proto.TypeDone || event.Type == proto.TypeError { + t.Fatal("Turn ended before live cancellation could be tested") + } + if event.Type == proto.TypeDelta { + streaming = true + } + case <-ctx.Done(): + t.Fatal("native stream did not begin before deadline") + } + } + select { + case <-interrupted.(*Session).finished: + t.Fatal("cancellation was not in flight") + default: + } + staleCtx, staleStop := context.WithTimeout(ctx, 5*time.Second) + if first.Cancel(staleCtx) != nil { + staleStop() + t.Fatal("stale cancellation failed") + } + staleStop() + interrupted.(*Session).mu.Lock() + wasCancelled := interrupted.(*Session).cancelled + interrupted.(*Session).mu.Unlock() + if wasCancelled { + t.Fatal("stale cancellation targeted current Turn") + } + stopCtx, stop := context.WithTimeout(ctx, 30*time.Second) + cancelErr := interrupted.Cancel(stopCtx) + settlement, settleErr := interrupted.AwaitSettlement(stopCtx) + stop() + if settleErr != nil || !settlement.Reusable { + if cancelErr == nil { + t.Fatal("uncertain cancellation reported success") + } + t.Log("Native cancellation cannot establish reusable settlement; successor must use recovery") + successor := make(chan proto.Envelope, 1) + next, nextErr := e.StartTurn(ctx, "unsafe-successor", proto.TextInput("must not execute"), successor) + if next != nil || nextErr == nil { + t.Fatal("unsettled native owner accepted a successor") + } + close(successor) + cleanup, cleanupStop := context.WithTimeout(ctx, 15*time.Second) + if e.Close(cleanup) != nil { + cleanupStop() + t.Fatal("invalid native owner did not close") + } + cleanupStop() + req.AgentSessionID = nativeID + recovered, recoverErr := NewExecutorFactory(nil)(ctx, req) + if recoverErr != nil || recovered == nil { + t.Fatal("exact native history recovery failed") + } + e = recovered.(*executor) + if e.nativeSession != nativeID || e.connection.process.Cmd.Process.Pid == pid { + t.Fatal("recovery did not replace the process while retaining native history") + } + pid = e.connection.process.Cmd.Process.Pid + run("recovery", "What exact marker did I ask you to remember at the beginning? Reply with only that marker.") + t.Log("Verified exact history continuation in a replacement native owner after uncertain cancellation") + return + } + if cancelErr != nil { + t.Fatal("settled cancellation reported an error") + } + assertOwner() + run("continuation", "What exact marker did I ask you to remember at the beginning? Reply with only that marker.") + t.Log("Verified native process and session reuse across two normal Turns, active cancellation, and continuation") +} diff --git a/apps/parsar-daemon/internal/agent/mcode/executor_test.go b/apps/parsar-daemon/internal/agent/mcode/executor_test.go new file mode 100644 index 000000000..17f532041 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/mcode/executor_test.go @@ -0,0 +1,243 @@ +package mcode + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +func executorFixture(t *testing.T, scenario string, workspace bool) (*executor, string) { + t.Helper() + config, req, record := workspaceFixture(t) + req.ReleaseOnCompletion = false + script, err := os.ReadFile(config.Binary) + if err != nil { + t.Fatal(err) + } + if err = os.WriteFile(config.Binary, []byte(strings.Replace(string(script), "HELPER=prepared", "HELPER="+scenario, 1)), 0700); err != nil { + t.Fatal(err) + } + var factory agent.ExecutorFactory + if workspace { + factory = NewExecutorFactory(&config) + } else { + req = executionRequest(t) + req.ReleaseOnCompletion = false + req.RunID, req.Input, req.ConversationID = "", nil, "" + t.Setenv("OAC_RUNTIME_MCODE_BIN", config.Binary) + factory = NewExecutorFactory(nil) + } + value, err := factory(t.Context(), req) + if err != nil { + t.Fatal(err) + } + e := value.(*executor) + t.Cleanup(func() { + cleanup, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := e.Close(cleanup); err != nil { + t.Error(err) + } + }) + return e, record +} + +func runExecutorFixtureTurn(t *testing.T, e *executor, id, text string) (*Session, []proto.Envelope) { + t.Helper() + out := make(chan proto.Envelope, 256) + turn, err := e.StartTurn(t.Context(), id, proto.TextInput(text), out) + if err != nil { + t.Fatal(err) + } + var events []proto.Envelope + for event := range out { + events = append(events, event) + } + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + settlement, err := turn.AwaitSettlement(ctx) + if err != nil || !settlement.Reusable { + t.Fatalf("settlement = %+v, %v", settlement, err) + } + return turn.(*Session), events +} + +func TestExecutorReusesNativeOwnerWithFreshTurnsAndIdleDrain(t *testing.T) { + for _, workspace := range []bool{false, true} { + t.Run(map[bool]string{false: "none", true: "workspace"}[workspace], func(t *testing.T) { + e, record := executorFixture(t, "executor-reuse", workspace) + pid := e.connection.process.Cmd.Process.Pid + first, _ := runExecutorFixtureTurn(t, e, "first", "one") + second, events := runExecutorFixtureTurn(t, e, "second", "two") + if first == second || e.connection.process.Cmd.Process.Pid != pid { + t.Fatal("Turn reused mutable state or rebuilt the process") + } + text, tools, done := "", 0, 0 + for _, event := range events { + if event.ID != "second" { + t.Fatal("event escaped its Turn", event.ID) + } + switch event.Type { + case proto.TypeError: + t.Fatalf("execution failed: %s", event.Payload) + case proto.TypeDelta: + var delta proto.DeltaPayload + _ = json.Unmarshal(event.Payload, &delta) + text += delta.Delta + if delta.Sequence != 1 { + t.Fatal("sequence carried over from old Turn") + } + case proto.TypeToolCall: + tools++ + case proto.TypeDone: + done++ + } + } + if text != "two" || tools != 2 || done != 1 { + t.Fatalf("cross-Turn state: text=%q tools=%d done=%d", text, tools, done) + } + raw, _ := os.ReadFile(record) + if strings.Count(string(raw), "initialize\n") != 1 || strings.Count(string(raw), "session/new\n") != 1 || strings.Count(string(raw), "session/prompt\n") != 2 { + t.Fatalf("native owner was recreated: %s", raw) + } + select { + case <-e.connection.exited: + t.Fatal("completion killed native owner") + default: + } + }) + } +} + +func TestExecutorCancellationWithoutRootProofInvalidatesAndLateCancelCannotRetarget(t *testing.T) { + e, record := executorFixture(t, "executor-cancel", true) + first, _ := runExecutorFixtureTurn(t, e, "first", "one") + out := make(chan proto.Envelope, 32) + second, err := e.StartTurn(t.Context(), "second", proto.TextInput("wait"), out) + if err != nil { + t.Fatal(err) + } + select { + case event := <-out: + if event.Type != proto.TypeDelta { + t.Fatal(event.Type) + } + case <-time.After(3 * time.Second): + t.Fatal("second Turn did not start") + } + if err = first.Cancel(t.Context()); err != nil { + t.Fatal(err) + } + raw, _ := os.ReadFile(record) + if strings.Contains(string(raw), "session/cancel") { + t.Fatal("late cancellation targeted the new Turn") + } + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + if err = second.Cancel(ctx); err == nil { + t.Fatal("unproven root settlement reported reusable cancellation") + } + settlement, err := second.AwaitSettlement(ctx) + if err != nil || settlement.Reusable { + t.Fatal(settlement, err) + } + select { + case <-e.connection.exited: + default: + t.Fatal("nonreusable settlement preceded native process exit") + } + if got := second.CancellationOutcome(); got.Content != "ready" || got.Metadata[proto.DoneMetaAgentSessionID] != "native-1" { + t.Fatal("cancel lost settled outcome", got) + } + for range out { + } + thirdOut := make(chan proto.Envelope, 1) + third, err := e.StartTurn(t.Context(), "third", proto.TextInput("three"), thirdOut) + if third != nil || err == nil { + t.Fatal("unproven native owner admitted a successor") + } + close(thirdOut) + raw, _ = os.ReadFile(record) + if strings.Count(string(raw), "session/cancel\n") != 1 || strings.Count(string(raw), "initialize\n") != 1 { + t.Fatalf("cancellation replaced owner: %s", raw) + } +} + +func TestExecutorStartFailureOwnership(t *testing.T) { + t.Run("validation", func(t *testing.T) { + e, record := executorFixture(t, "executor-reuse", true) + out := make(chan proto.Envelope, 1) + turn, err := e.StartTurn(t.Context(), "", proto.TextInput("invalid"), out) + if err == nil || turn != nil { + t.Fatal("invalid Start acquired output") + } + close(out) + raw, _ := os.ReadFile(record) + if strings.Contains(string(raw), "session/prompt") { + t.Fatal("invalid input was sent") + } + runExecutorFixtureTurn(t, e, "healthy", "valid") + }) + t.Run("before input", func(t *testing.T) { + e, record := executorFixture(t, "executor-preexit", true) + out := make(chan proto.Envelope, 1) + turn, err := e.StartTurn(t.Context(), "run", proto.TextInput("input"), out) + if err == nil || turn != nil { + t.Fatal("failed readiness retained caller output") + } + close(out) + raw, _ := os.ReadFile(record) + if strings.Contains(string(raw), "session/prompt") { + t.Fatal("pre-input failure sent input") + } + }) + t.Run("after submission", func(t *testing.T) { + e, record := executorFixture(t, "executor-exit", true) + out := make(chan proto.Envelope, 32) + turn, err := e.StartTurn(t.Context(), "run", proto.TextInput("input"), out) + if err != nil || turn == nil { + t.Fatal("submitted Turn lost ownership", err) + } + for range out { + } + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + settlement, err := turn.AwaitSettlement(ctx) + if err != nil || settlement.Reusable { + t.Fatal("exited process remained reusable", settlement, err) + } + raw, _ := os.ReadFile(record) + if strings.Count(string(raw), "session/prompt\n") != 1 { + t.Fatal("uncertain input was replayed") + } + rejected := make(chan proto.Envelope) + again, err := e.StartTurn(t.Context(), "again", proto.TextInput("again"), rejected) + if again != nil || err == nil { + t.Fatal("invalid owner accepted another Turn") + } + close(rejected) + }) +} + +func TestExecutorCloseRetainsOwnerAfterDeadline(t *testing.T) { + e, _ := executorFixture(t, "executor-reuse", true) + cancelled, cancel := context.WithCancel(t.Context()) + cancel() + _ = e.Close(cancelled) + ctx, stop := context.WithTimeout(t.Context(), 5*time.Second) + defer stop() + if err := e.Close(ctx); err != nil { + t.Fatal(err) + } + select { + case <-e.connection.exited: + default: + t.Fatal("Close lost native cleanup ownership") + } +} diff --git a/apps/parsar-daemon/internal/agent/mcode/executor_turn.go b/apps/parsar-daemon/internal/agent/mcode/executor_turn.go new file mode 100644 index 000000000..540a6202b --- /dev/null +++ b/apps/parsar-daemon/internal/agent/mcode/executor_turn.go @@ -0,0 +1,153 @@ +package mcode + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +func (s *Session) runExecutorTurn() { + err := s.captureSubagentBaseline() + if err == nil { + err = s.executePrompt() + } + if errors.Is(err, errTurnCancelled) { + err = nil + } + s.mu.Lock() + s.closing, s.steeringReady = true, false + close(s.inputDone) + s.mu.Unlock() + // Written steering requests retain their original receipt owner even after + // prompt completion. No successor starts until all callers have settled. + s.operations.Wait() + if s.req.StrictResume && !s.req.DisableSubagents && s.subagentHistoryReady { + childErr := s.settleSubagents() + s.mu.Lock() + s.subagentSettlementError = childErr + s.mu.Unlock() + if childErr != nil { + err = childErr + } + } + reusable := err == nil && s.process.Context().Err() == nil + if len(s.tools) > 0 { + reusable = false + } + s.finishEnvironmentMCP() + s.mu.Lock() + // ACP cancelled can precede native root Turn settlement in 0.4.12. + // Without the qualified history reader, retain no claim of safe reuse. + if (s.cancelled && s.req.DisableSubagents) || s.inputUncertain || len(s.permissions) != 0 || len(s.questions) != 0 { + reusable = false + } + metadata := map[string]any{proto.DoneMetaAgentSessionType: "mcode", proto.DoneMetaAgentSessionID: s.sessionID} + s.outcome = proto.DonePayload{Content: s.content.String(), Metadata: metadata, SourceCompletedAtMS: s.rootCompletedAtMS} + outcome := s.outcome + s.permissions, s.questions = map[string]pendingPermission{}, map[string]pendingQuestion{} + s.mu.Unlock() + if !reusable { + // Unknown quiescence invalidates this owner. A successful settlement + // still proves its native process group and pipes have stopped. + s.process.Cancel() + <-s.exited + } + if err != nil { + s.emit(proto.TypeError, proto.ErrorPayload{Error: err.Error()}) + } + s.emit(proto.TypeDone, outcome) + close(s.out) + close(s.finished) + s.executor.mu.Lock() + s.connection.setCurrent(nil) + if !reusable { + s.executor.invalid = true + } + s.executor.active = nil + s.settlement = agent.TurnSettlement{Reusable: reusable} + if !reusable { + s.settlement.Reason = "native Turn did not establish reusable settlement" + } + close(s.settled) + s.executor.mu.Unlock() + s.outputCancel() +} + +func (s *Session) captureSubagentBaseline() error { + if !s.req.StrictResume || s.req.DisableSubagents { + return nil + } + snapshot, err := s.readSubagents(s.ctx) + s.subagentHistoryReady = err == nil + s.previousNativeTurns = map[string]bool{} + for _, session := range snapshot.Sessions { + if session.ID == s.sessionID { + for _, turn := range session.Turns { + s.previousNativeTurns[turn.ID] = true + } + } + } + return err +} + +func (s *Session) AwaitSettlement(ctx context.Context) (agent.TurnSettlement, error) { + if s.settled == nil { + return agent.TurnSettlement{}, fmt.Errorf("mcode: Turn has no Executor owner") + } + select { + case <-s.settled: + return s.settlement, nil + case <-ctx.Done(): + return agent.TurnSettlement{}, ctx.Err() + } +} + +func (s *Session) cancelTurn(ctx context.Context) error { + e := s.executor + e.mu.Lock() + if e.active != s { + e.mu.Unlock() + _, err := s.AwaitSettlement(ctx) + return err + } + s.mu.Lock() + first := !s.cancelled && !s.closing + s.cancelled = true + if first { + s.operations.Add(1) + } + s.mu.Unlock() + // Cancellation releases event backpressure but does not cancel native owner + // context. ACP prompt completion and child/tool settlement decide reuse. + s.outputCancel() + var err error + if first { + raw, _ := json.Marshal(map[string]string{"sessionId": s.sessionID}) + err = s.writeContext(ctx, rpcFrame{JSONRPC: "2.0", Method: "session/cancel", Params: raw}) + } + e.mu.Unlock() + if first { + if err == nil && s.req.StrictResume && !s.req.DisableSubagents { + err = s.stopSubagents(ctx) + } + if err != nil { + s.markInputUncertain() + } + s.operations.Done() + } + if err != nil { + return err + } + settlement, err := s.AwaitSettlement(ctx) + if err != nil { + return err + } + if !settlement.Reusable { + return fmt.Errorf("mcode: cancellation did not establish reusable settlement") + } + return nil +} diff --git a/apps/parsar-daemon/internal/agent/mcode/preparation.go b/apps/parsar-daemon/internal/agent/mcode/preparation.go index 97e614d82..8d9672548 100644 --- a/apps/parsar-daemon/internal/agent/mcode/preparation.go +++ b/apps/parsar-daemon/internal/agent/mcode/preparation.go @@ -3,145 +3,85 @@ package mcode import ( "context" "fmt" - "strings" "sync" "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" ) +// Prepared retains its single admission API over the same native Executor. +// Runtime's Session lifecycle uses Executor directly. type prepared struct { - mu sync.Mutex - session *Session - ready chan struct{} - started chan struct{} - failure error - closed bool - binding *preparedStart -} - -type preparedStart struct { - runID string - prompt proto.MessageInput - out chan<- proto.Envelope + mu sync.Mutex + executor *executor + session *Session + started, closed bool } func NewPreparationFactory(config WorkspaceConfig) agent.PreparationFactory { + factory := NewExecutorFactory(&config) return func(ctx context.Context, req proto.PromptRequestPayload) (agent.Prepared, error) { - if ctx == nil { - ctx = context.Background() - } - if req.RunID != "" || len(req.Input) != 0 || req.ConversationID != "" { - return nil, fmt.Errorf("mcode: preparation cannot contain input or product context") - } - opts, err := prepareWorkspaceOptions(ctx, config, req) - if err != nil { - return nil, err - } - s, err := launch(ctx, req, opts, config.Binary, nil) + value, err := factory(ctx, req) if err != nil { + if value != nil { + return &prepared{executor: value.(*executor)}, err + } return nil, err } - p := &prepared{session: s, ready: make(chan struct{}), started: make(chan struct{})} - go s.run(p) - <-p.ready - if p.failure != nil { - <-s.finished - return nil, p.failure - } - if ctx.Err() != nil { - _ = p.Close() - return nil, ctx.Err() - } - return p, nil + e := value.(*executor) + return &prepared{executor: e, session: newTurnSession(ctx, req, e.opts, e.connection, nil)}, nil } } -func (p *prepared) Start(ctx context.Context, runID string, prompt proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { - if ctx == nil { - ctx = context.Background() - } +func (p *prepared) Start(ctx context.Context, runID string, input proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { p.mu.Lock() defer p.mu.Unlock() - if p.closed || p.binding != nil { + if p.closed || p.started { return nil, fmt.Errorf("mcode: preparation is no longer available") } - if strings.TrimSpace(runID) == "" || prompt.Validate() != nil || out == nil || ctx.Err() != nil { - return nil, fmt.Errorf("mcode: start requires a live context, identity, prompt and output") - } - select { - case <-p.session.process.Context().Done(): - return nil, fmt.Errorf("mcode: prepared process ended") - case <-p.session.exited: - return nil, fmt.Errorf("mcode: prepared process exited") - default: + turn, err := p.executor.StartTurn(ctx, runID, input, out) + if turn != nil { + p.started, p.session = true, turn.(*Session) } - p.binding = &preparedStart{runID: runID, prompt: prompt, out: out} - close(p.started) - return p.session, nil + return turn, err } func (p *prepared) Close() error { p.mu.Lock() - if p.binding != nil { - p.mu.Unlock() - return nil + started := p.started + if !started { + p.closed = true } - p.closed = true p.mu.Unlock() - return p.closeUnused(context.Background()) + if started { + return nil + } + return p.executor.Close(context.Background()) } func (p *prepared) Cancel(ctx context.Context) error { p.mu.Lock() p.closed = true - started := p.binding != nil + current, started := p.session, p.started p.mu.Unlock() - if !started { - return p.closeUnused(ctx) - } - return p.session.Cancel(ctx) -} - -// An unconsumed preparation has no model input or child work to settle. -func (p *prepared) closeUnused(ctx context.Context) error { - p.session.process.Cancel() - select { - case <-p.session.finished: - return nil - case <-ctx.Done(): - return ctx.Err() + if started { + if err := current.Cancel(ctx); err != nil { + // The single-use Prepared owns disposal as well as cancellation. + if closeErr := p.executor.Close(ctx); closeErr != nil { + return closeErr + } + return nil + } } + return p.executor.Close(ctx) } -func (p *prepared) CancellationOutcome() proto.DonePayload { return p.session.CancellationOutcome() } - -// The Session goroutine owns preparation, input submission and terminal output. -func (p *prepared) awaitStart(err error) error { - p.failure = err - close(p.ready) - if err != nil { - return err - } - for { - var ended error - select { - case <-p.started: - case <-p.session.process.Context().Done(): - ended = p.session.process.Context().Err() - case _, ok := <-p.session.frames: - if ok { - continue - } - ended = fmt.Errorf("mcode: prepared process exited") - } - p.mu.Lock() - if b := p.binding; b != nil { - p.session.req.RunID, p.session.req.Input, p.session.out = b.runID, b.prompt, b.out - } else { - p.closed = true - } - p.mu.Unlock() - return ended +func (p *prepared) CancellationOutcome() proto.DonePayload { + p.mu.Lock() + s := p.session + p.mu.Unlock() + if s == nil { + return proto.DonePayload{} } + return s.CancellationOutcome() } diff --git a/apps/parsar-daemon/internal/agent/mcode/preparation_test.go b/apps/parsar-daemon/internal/agent/mcode/preparation_test.go index 9ae612c19..65ea1a87e 100644 --- a/apps/parsar-daemon/internal/agent/mcode/preparation_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/preparation_test.go @@ -88,8 +88,8 @@ func TestPreparedWorkspaceHasOneInputAndOutputOwner(t *testing.T) { done++ select { case <-p.session.exited: + t.Fatal("successful Turn disposed the reusable native owner") default: - t.Fatal("Done before native settlement") } var d proto.DonePayload _ = json.Unmarshal(e.Payload, &d) diff --git a/apps/parsar-daemon/internal/agent/mcode/session.go b/apps/parsar-daemon/internal/agent/mcode/session.go index 476ace40b..89f51251f 100644 --- a/apps/parsar-daemon/internal/agent/mcode/session.go +++ b/apps/parsar-daemon/internal/agent/mcode/session.go @@ -1,15 +1,12 @@ package mcode import ( - "bufio" "context" "encoding/json" "errors" "fmt" - "io" "net/url" "slices" - "strconv" "strings" "sync" "time" @@ -20,22 +17,29 @@ import ( ) type Session struct { - ctx context.Context - req proto.PromptRequestPayload - opts launchOptions - process *clirunner.Process + ctx context.Context + req proto.PromptRequestPayload + opts launchOptions + *connection + executor *executor + settlement agent.TurnSettlement + settled chan struct{} + inputDone chan struct{} + outputCancel context.CancelFunc + operations sync.WaitGroup + closing bool + cancelled bool + inputUncertain bool out chan<- proto.Envelope frames chan rpcFrame - exited chan struct{} finished chan struct{} - writeMu sync.Mutex mu sync.Mutex sessionID string + nativeModel string + outputContext context.Context + outcome proto.DonePayload permissions map[string]pendingPermission questions map[string]pendingQuestion - exitErr error - nextID int - responses map[string]chan rpcFrame steeringReady bool steeringTurn string sequence uint64 @@ -70,7 +74,7 @@ func newSession(ctx context.Context, req proto.PromptRequestPayload, out chan<- if err != nil { return nil, err } - go s.run(nil) + go s.run() return s, nil } @@ -79,56 +83,18 @@ func launch(ctx context.Context, req proto.PromptRequestPayload, opts launchOpti if err != nil { return nil, err } - s := &Session{ctx: ctx, req: req, opts: opts, process: process, out: out, frames: make(chan rpcFrame, 32), exited: make(chan struct{}), finished: make(chan struct{}), responses: map[string]chan rpcFrame{}, permissions: map[string]pendingPermission{}, questions: map[string]pendingQuestion{}, tools: map[string]toolUpdate{}, completedTools: map[string]bool{}} - go func() { _, _ = io.Copy(io.Discard, process.Stderr) }() - go s.read() + c := &connection{process: process, exited: make(chan struct{}), responses: map[string]chan rpcFrame{}} + s := newTurnSession(ctx, req, opts, c, out) + c.current = s + go c.read() return s, nil } -func (s *Session) read() { - defer close(s.exited) - defer close(s.frames) - scanner := bufio.NewScanner(s.process.Stdout) - scanner.Buffer(make([]byte, 64*1024), 16*1024*1024) - var readErr error - for scanner.Scan() { - var frame rpcFrame - if err := json.Unmarshal(scanner.Bytes(), &frame); err != nil { - readErr = fmt.Errorf("mcode: malformed ACP response") - s.process.Cancel() - break - } - if frame.Method == "" { - s.mu.Lock() - response := s.responses[string(frame.ID)] - s.mu.Unlock() - if response != nil { - select { - case response <- frame: - default: - } - continue - } - } - select { - case s.frames <- frame: - case <-s.finished: - case <-s.process.Context().Done(): - } - } - if scanner.Err() != nil { - readErr = fmt.Errorf("mcode: ACP stream read failed") - s.process.Cancel() - } - waitErr := s.process.Wait() - if readErr != nil { - s.exitErr = readErr - } else { - s.exitErr = waitErr - } +func newTurnSession(ctx context.Context, req proto.PromptRequestPayload, opts launchOptions, c *connection, out chan<- proto.Envelope) *Session { + return &Session{ctx: ctx, req: req, opts: opts, connection: c, out: out, frames: make(chan rpcFrame, 32), finished: make(chan struct{}), permissions: map[string]pendingPermission{}, questions: map[string]pendingQuestion{}, tools: map[string]toolUpdate{}, completedTools: map[string]bool{}} } -func (s *Session) run(p *prepared) { +func (s *Session) run() { defer func() { if s.out != nil { close(s.out) @@ -136,9 +102,6 @@ func (s *Session) run(p *prepared) { }() defer close(s.finished) err := s.prepareNative() - if p != nil { - err = p.awaitStart(err) - } if err == nil { if s.req.StrictResume && !s.req.DisableSubagents { var snapshot nativeSubagentSnapshot @@ -166,7 +129,7 @@ func (s *Session) run(p *prepared) { err = observationErr } } - if p != nil || err != nil { + if err != nil { s.process.Cancel() <-s.exited } @@ -241,6 +204,7 @@ func (s *Session) prepareNative() error { if err != nil { return err } + s.nativeModel = model if err := s.call("session/set_config_option", map[string]any{"sessionId": session.SessionID, "configId": "model", "value": model}, nil, false); err != nil { return err } @@ -264,6 +228,12 @@ func (s *Session) executePrompt() error { if err != nil { return err } + s.mu.Lock() + cancelled := s.cancelled + s.mu.Unlock() + if result.StopReason == "cancelled" && cancelled { + return nil + } if result.StopReason != "end_turn" { return fmt.Errorf("mcode: prompt stopped (%s)", result.StopReason) } @@ -304,7 +274,21 @@ func (s *Session) call(method string, params any, result any, prompt bool) error if err != nil { return err } - if err := s.write(rpcFrame{JSONRPC: "2.0", ID: json.RawMessage(id), Method: method, Params: raw}); err != nil { + if prompt && s.executor != nil { + s.executor.mu.Lock() + s.mu.Lock() + cancelled := s.cancelled + s.mu.Unlock() + if cancelled || s.executor.closed { + s.executor.mu.Unlock() + return errTurnCancelled + } + err = s.write(rpcFrame{JSONRPC: "2.0", ID: json.RawMessage(id), Method: method, Params: raw}) + s.executor.mu.Unlock() + } else { + err = s.write(rpcFrame{JSONRPC: "2.0", ID: json.RawMessage(id), Method: method, Params: raw}) + } + if err != nil { return err } ctx := s.process.Context() @@ -317,6 +301,8 @@ func (s *Session) call(method string, params any, result any, prompt bool) error select { case <-ctx.Done(): return fmt.Errorf("mcode: %s: %w", method, ctx.Err()) + case <-s.exited: + return fmt.Errorf("mcode: ACP process exited: %v", s.exitErr) case frame, ok := <-s.frames: if !ok { <-s.exited @@ -344,24 +330,30 @@ func (s *Session) call(method string, params any, result any, prompt bool) error } } -func (s *Session) write(frame rpcFrame) error { - s.writeMu.Lock() - defer s.writeMu.Unlock() - return json.NewEncoder(s.process.Stdin).Encode(frame) -} - func (s *Session) emit(kind string, payload any) { + ctx := s.ctx + if s.outputContext != nil { + ctx = s.outputContext + } env, err := proto.NewEnvelope(kind, s.req.RunID, payload) if err != nil { return } select { case s.out <- env: - case <-s.ctx.Done(): + return + default: + } + select { + case s.out <- env: + case <-ctx.Done(): } } func (s *Session) Cancel(ctx context.Context) error { + if s.executor != nil { + return s.cancelTurn(ctx) + } if s.req.StrictResume && !s.req.DisableSubagents { if err := s.cancelSubagents(ctx); err != nil { s.process.Cancel() @@ -409,18 +401,3 @@ func (s *Session) SubmitPermission(_ context.Context, id string, decision proto. delete(s.permissions, id) return nil } - -func (s *Session) reserveResponse() (string, chan rpcFrame) { - s.mu.Lock() - defer s.mu.Unlock() - s.nextID++ - id := strconv.Itoa(s.nextID) - reply := make(chan rpcFrame, 1) - s.responses[id] = reply - return id, reply -} -func (s *Session) removeResponse(id string) { - s.mu.Lock() - delete(s.responses, id) - s.mu.Unlock() -} diff --git a/apps/parsar-daemon/internal/agent/mcode/session_test.go b/apps/parsar-daemon/internal/agent/mcode/session_test.go index 2e347057a..641fe93a0 100644 --- a/apps/parsar-daemon/internal/agent/mcode/session_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/session_test.go @@ -198,6 +198,7 @@ func TestMCodeProcess(t *testing.T) { } scanner := bufio.NewScanner(os.Stdin) var promptID json.RawMessage + prompts, configurations := 0, 0 for scanner.Scan() { var frame rpcFrame if json.Unmarshal(scanner.Bytes(), &frame) != nil { @@ -248,6 +249,10 @@ func TestMCodeProcess(t *testing.T) { result = map[string]any{"configOptions": []map[string]any{{"id": "permissionMode"}}} } case "session/set_config_option": + configurations++ + if scenario == "executor-preexit" && configurations > 1 { + os.Exit(0) + } var params map[string]string _ = json.Unmarshal(frame.Params, ¶ms) if params["configId"] == "model" && params["value"] != "m:custom_provider%3Aoac:fixture:v:" { @@ -258,10 +263,33 @@ func TestMCodeProcess(t *testing.T) { Prompt []map[string]string `json:"prompt"` } _ = json.Unmarshal(frame.Params, &input) - if strict := scenario == "strict-cancel" || strings.HasPrefix(scenario, "prepared"); (strict && len(input.Prompt) != 2) || (!strict && len(input.Prompt) != 1) { + if strict := scenario == "strict-cancel" || (strings.HasPrefix(scenario, "prepared") || strings.HasPrefix(scenario, "executor")); (strict && len(input.Prompt) != 2) || (!strict && len(input.Prompt) != 1) { os.Exit(9) } + if strings.HasPrefix(scenario, "executor") { + prompts++ + promptID = frame.ID + if scenario == "executor-exit" { + os.Exit(0) + } + if input.Prompt[0]["text"] == "wait" { + update("agent_message_chunk", map[string]any{"content": map[string]string{"type": "text", "text": "ready"}}) + continue + } + update("agent_message_chunk", map[string]any{"content": map[string]string{"type": "text", "text": input.Prompt[0]["text"]}}) + update("tool_call", map[string]any{"toolCallId": "repeated-call", "name": "Read", "status": "in_progress", "rawInput": map[string]any{}}) + update("tool_call_update", map[string]any{"toolCallId": "repeated-call", "status": "completed"}) + raw, _ := json.Marshal(map[string]string{"stopReason": "end_turn"}) + send(rpcFrame{JSONRPC: "2.0", ID: frame.ID, Result: raw}) + // These frames precede the next control barrier on the wire and + // must never become the following Turn's output. + for range 100 { + update("agent_message_chunk", map[string]any{"content": map[string]string{"type": "text", "text": "OLD"}}) + } + continue + } if scenario == "prepared-mcp-cancel" { + promptID = frame.ID update("tool_call", map[string]any{"toolCallId": "native-call", "name": "mcp__proof_server__read_status", "status": "in_progress", "rawInput": map[string]any{}}) continue } @@ -294,6 +322,10 @@ func TestMCodeProcess(t *testing.T) { } update("usage_update", map[string]any{"used": 2000, "size": 64000, "cost": map[string]any{"amount": 2, "currency": "USD"}}) result = map[string]string{"stopReason": "end_turn"} + case "session/cancel": + raw, _ := json.Marshal(map[string]string{"stopReason": "cancelled"}) + send(rpcFrame{JSONRPC: "2.0", ID: promptID, Result: raw}) + continue case "mcode/session/steer": if scenario == "steer-lost" { os.Exit(0) diff --git a/apps/parsar-daemon/internal/agent/mcode/steering.go b/apps/parsar-daemon/internal/agent/mcode/steering.go index 7479b2964..cb16925bc 100644 --- a/apps/parsar-daemon/internal/agent/mcode/steering.go +++ b/apps/parsar-daemon/internal/agent/mcode/steering.go @@ -18,6 +18,16 @@ func (s *Session) Steer(ctx context.Context, input proto.PromptSteerPayload) err // Native acceptance belongs to the active ACP Turn; it does not promise model consumption. func (s *Session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerPayload, written func()) error { + if s.executor != nil { + s.mu.Lock() + if s.closing || s.cancelled { + s.mu.Unlock() + return agent.ErrSteeringInactive + } + s.operations.Add(1) + s.mu.Unlock() + defer s.operations.Done() + } text, err := input.Input.TextOnly() if strings.TrimSpace(input.InputID) == "" || err != nil { return agent.ErrSteeringRejected @@ -40,10 +50,9 @@ func (s *Session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerP if err := ctx.Err(); err != nil { return err } - stop := context.AfterFunc(ctx, s.process.Cancel) - err = s.write(rpcFrame{JSONRPC: "2.0", ID: json.RawMessage(id), Method: "mcode/session/steer", Params: raw}) - stop() + err = s.writeContext(ctx, rpcFrame{JSONRPC: "2.0", ID: json.RawMessage(id), Method: "mcode/session/steer", Params: raw}) if err != nil { + s.markInputUncertain() return fmt.Errorf("mcode: input transport failed") } if written != nil { @@ -53,7 +62,7 @@ func (s *Session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerP var stopped error select { case frame = <-response: - case <-s.finished: + case <-s.inputSettlementDone(): stopped = fmt.Errorf("mcode: run ended with unknown input outcome") case <-s.exited: stopped = fmt.Errorf("mcode: input transport closed") @@ -65,6 +74,7 @@ func (s *Session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerP select { case frame = <-response: default: + s.markInputUncertain() return stopped } } @@ -72,6 +82,7 @@ func (s *Session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerP if frame.Error.Code == -32602 || frame.Error.Code == -32601 { return agent.ErrSteeringRejected } + s.markInputUncertain() return fmt.Errorf("mcode: native steering failed with unknown input outcome") } var result struct { @@ -79,23 +90,41 @@ func (s *Session) SteerWithReceipt(ctx context.Context, input proto.PromptSteerP Mode string `json:"mode"` } if json.Unmarshal(frame.Result, &result) != nil || result.TurnID == "" || (result.Mode != "steered" && result.Mode != "duplicate") { + s.markInputUncertain() return fmt.Errorf("mcode: invalid steering receipt") } s.mu.Lock() defer s.mu.Unlock() if s.steeringTurn != "" && s.steeringTurn != result.TurnID { + s.inputUncertain = true return fmt.Errorf("mcode: steering turn changed") } s.steeringTurn = result.TurnID return nil } +func (s *Session) inputSettlementDone() <-chan struct{} { + if s.inputDone != nil { + return s.inputDone + } + return s.finished +} + func (s *Session) CancellationOutcome() proto.DonePayload { s.mu.Lock() defer s.mu.Unlock() + if s.outcome.Metadata != nil { + return s.outcome + } metadata := map[string]any{proto.DoneMetaAgentSessionType: "mcode"} if s.sessionID != "" { metadata[proto.DoneMetaAgentSessionID] = s.sessionID } return proto.DonePayload{Metadata: metadata} } + +func (s *Session) markInputUncertain() { + s.mu.Lock() + s.inputUncertain = true + s.mu.Unlock() +} diff --git a/apps/parsar-daemon/internal/agent/mcode/subagent_cancel.go b/apps/parsar-daemon/internal/agent/mcode/subagent_cancel.go index 1469c1ff0..bd1b88e68 100644 --- a/apps/parsar-daemon/internal/agent/mcode/subagent_cancel.go +++ b/apps/parsar-daemon/internal/agent/mcode/subagent_cancel.go @@ -7,6 +7,21 @@ import ( ) func (s *Session) cancelSubagents(ctx context.Context) error { + if err := s.stopSubagents(ctx); err != nil { + return err + } + select { + case <-s.finished: + s.mu.Lock() + err := s.subagentSettlementError + s.mu.Unlock() + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (s *Session) stopSubagents(ctx context.Context) error { select { case <-s.finished: return nil @@ -21,7 +36,7 @@ func (s *Session) cancelSubagents(ctx context.Context) error { id, response := s.reserveResponse() defer s.removeResponse(id) raw, _ := json.Marshal(map[string]string{"sessionId": native}) - if err := s.write(rpcFrame{JSONRPC: "2.0", ID: json.RawMessage(id), Method: "mcode/session/delegation/stop", Params: raw}); err != nil { + if err := s.writeContext(ctx, rpcFrame{JSONRPC: "2.0", ID: json.RawMessage(id), Method: "mcode/session/delegation/stop", Params: raw}); err != nil { return err } select { @@ -40,13 +55,5 @@ func (s *Session) cancelSubagents(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() } - select { - case <-s.finished: - s.mu.Lock() - err := s.subagentSettlementError - s.mu.Unlock() - return err - case <-ctx.Done(): - return ctx.Err() - } + return nil } diff --git a/apps/parsar-daemon/internal/agent/mcode/subagents.go b/apps/parsar-daemon/internal/agent/mcode/subagents.go index 33fe2ed9e..645525f54 100644 --- a/apps/parsar-daemon/internal/agent/mcode/subagents.go +++ b/apps/parsar-daemon/internal/agent/mcode/subagents.go @@ -121,11 +121,9 @@ func (s *Session) settleSubagents() error { settled = false } } - if session.ID != snapshot.Root { - for _, turn := range session.Turns { - if turn.Status == "accepted" { - settled = false - } + for _, turn := range session.Turns { + if turn.Status == "accepted" { + settled = false } } } diff --git a/apps/parsar-daemon/internal/agent/mcode/subagents_test.go b/apps/parsar-daemon/internal/agent/mcode/subagents_test.go index de4b4b414..9268c5837 100644 --- a/apps/parsar-daemon/internal/agent/mcode/subagents_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/subagents_test.go @@ -3,8 +3,11 @@ package mcode import ( "context" "encoding/json" + "os" + "path/filepath" "reflect" "testing" + "time" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" ) @@ -65,3 +68,35 @@ func TestSubagentSnapshotRejectsMissingParentProvenance(t *testing.T) { t.Fatal("accepted child without original spawning provenance") } } + +// The ACP cancelled response may precede the root native Turn becoming terminal. +func TestSubagentSettlementIncludesActiveRootTurn(t *testing.T) { + dir := t.TempDir() + node, bridge := filepath.Join(dir, "node"), filepath.Join(dir, "bridge.mjs") + for name, data := range map[string]string{ + node: "#!/bin/sh\ncat \"$3/snapshot.json\"\n", + bridge: "", + filepath.Join(dir, "subagent-snapshot.mjs"): "", + } { + if err := os.WriteFile(name, []byte(data), 0700); err != nil { + t.Fatal(err) + } + } + t.Setenv("OAC_RUNTIME_MCODE_NODE", node) + t.Setenv("OAC_RUNTIME_MCODE_WORKSPACE_BRIDGE", bridge) + for _, status := range []string{"accepted", "aborted"} { + t.Run(status, func(t *testing.T) { + raw := []byte(`{"version":1,"complete":true,"rootSessionId":"root","sessions":[{"id":"root","turns":[{"id":"current","status":"` + status + `"}]}]}`) + if err := os.WriteFile(filepath.Join(dir, "snapshot.json"), raw, 0600); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + s := &Session{ctx: ctx, sessionID: "root", opts: launchOptions{DataDir: dir}, frames: make(chan rpcFrame)} + err := s.settleSubagents() + if (err != nil) != (status == "accepted") { + t.Fatalf("root %s settlement error = %v", status, err) + } + }) + } +} diff --git a/apps/parsar-daemon/internal/agent/registry.go b/apps/parsar-daemon/internal/agent/registry.go index d9f6459f5..97d79a4c9 100644 --- a/apps/parsar-daemon/internal/agent/registry.go +++ b/apps/parsar-daemon/internal/agent/registry.go @@ -58,6 +58,7 @@ type Registry struct { mu sync.RWMutex factories map[string]Factory preparers map[string]PreparationFactory + executors map[string]ExecutorFactory kinds map[string]proto.SupportedAgentKind } @@ -65,6 +66,7 @@ func NewRegistry() *Registry { return &Registry{ factories: make(map[string]Factory), preparers: make(map[string]PreparationFactory), + executors: make(map[string]ExecutorFactory), kinds: make(map[string]proto.SupportedAgentKind), } } @@ -90,6 +92,7 @@ func (r *Registry) RegisterKind(info proto.SupportedAgentKind, f Factory) { defer r.mu.Unlock() r.factories[kind] = f delete(r.preparers, kind) + delete(r.executors, kind) info.Capabilities.Preparation = false info.Capabilities.WorkspaceReadPreparation = false r.kinds[kind] = info diff --git a/apps/parsar-daemon/internal/agent/registry_test.go b/apps/parsar-daemon/internal/agent/registry_test.go index 58c4488a4..e1ab3e8f7 100644 --- a/apps/parsar-daemon/internal/agent/registry_test.go +++ b/apps/parsar-daemon/internal/agent/registry_test.go @@ -136,3 +136,24 @@ func TestRegistrySupportedAgentKindsReportsDescriptors(t *testing.T) { t.Fatalf("opencode descriptor not preserved: %#v", got[1]) } } + +func TestRegistryExecutorRequiresExplicitRegistration(t *testing.T) { + registry := agent.NewRegistry() + registry.Register("native", stubFactory("native")) + if _, err := registry.ResolveExecutor("native"); err == nil { + t.Fatal("legacy factory implied reusable execution") + } + expected := errors.New("executor factory") + registry.RegisterExecutor("native", func(context.Context, proto.PromptRequestPayload) (agent.Executor, error) { return nil, expected }) + factory, err := registry.ResolveExecutor("native") + if err != nil { + t.Fatal(err) + } + if _, err := factory(t.Context(), proto.PromptRequestPayload{}); !errors.Is(err, expected) { + t.Fatal(err) + } + registry.Register("native", stubFactory("replacement")) + if _, err := registry.ResolveExecutor("native"); err == nil { + t.Fatal("replacing a kind retained its old executor capability") + } +} diff --git a/apps/parsar-daemon/internal/cli/agent_registration.go b/apps/parsar-daemon/internal/cli/agent_registration.go index 17126ff5a..ea93bbf30 100644 --- a/apps/parsar-daemon/internal/cli/agent_registration.go +++ b/apps/parsar-daemon/internal/cli/agent_registration.go @@ -16,6 +16,9 @@ func registerAgentKinds(registry *agent.Registry, agentCLIs agentCLIDiscovery, s registerProductAgentKind(registry, agentCLIs.ClaudeCode, withSkillUploadServer(withCapabilityDownloads(claudecode.Factory, serverURL), serverURL)) registerProductAgentKind(registry, agentCLIs.OpenCode, withSkillUploadServer(withCapabilityDownloads(opencodeagent.Factory, serverURL), serverURL)) registerProductAgentKind(registry, agentCLIs.Codex, withSkillUploadServer(withCapabilityDownloads(codex.Factory, serverURL), serverURL)) + if agentCLIs.Codex.Available { + registry.RegisterExecutor("codex", withExecutorCapabilities(codex.NewExecutorFactory(), serverURL)) + } if agentCLIs.Codex.Available && agentCLIs.Codex.Capabilities.LocalEnvironment { registry.RegisterPreparation("codex", true, func(ctx context.Context, req proto.PromptRequestPayload) (agent.Prepared, error) { prepared, err := codex.Prepare(ctx, req) @@ -32,6 +35,9 @@ func registerAgentKinds(registry *agent.Registry, agentCLIs agentCLIDiscovery, s } else { registerProductAgentKind(registry, agentCLIs.MCode, withSkillUploadServer(withCapabilityDownloads(mcode.Factory, serverURL), serverURL)) } + if agentCLIs.MCode.Available { + registry.RegisterExecutor("mcode", withExecutorCapabilities(mcode.NewExecutorFactory(agentCLIs.MCodeWorkspace), serverURL)) + } registerClaudeSDK(registry, agentCLIs.ClaudeSDK) } diff --git a/apps/parsar-daemon/internal/cli/authoring.go b/apps/parsar-daemon/internal/cli/authoring.go index b6b456906..e8fb938ce 100644 --- a/apps/parsar-daemon/internal/cli/authoring.go +++ b/apps/parsar-daemon/internal/cli/authoring.go @@ -62,6 +62,9 @@ func authoringRegistry(registry *agent.Registry, bridge *authoring.Bridge) *agen factory = withAuthoringBridge(factory, bridge) } wrapped.RegisterKind(info, factory) + if executor, err := registry.ResolveExecutor(info.Kind); err == nil { + wrapped.RegisterExecutor(info.Kind, executor) + } if prepare, err := registry.ResolvePreparation(info.Kind); err == nil { wrapped.RegisterPreparation(info.Kind, info.Capabilities.WorkspaceReadPreparation, prepare) } diff --git a/apps/parsar-daemon/internal/cli/capability_downloads.go b/apps/parsar-daemon/internal/cli/capability_downloads.go index 2f5e95819..e92fdfc0c 100644 --- a/apps/parsar-daemon/internal/cli/capability_downloads.go +++ b/apps/parsar-daemon/internal/cli/capability_downloads.go @@ -20,28 +20,7 @@ func withCapabilityDownloads(factory agent.Factory, serverURL string) agent.Fact return factory } return func(ctx context.Context, req proto.PromptRequestPayload, out chan<- proto.Envelope) (agent.Session, error) { - opts := maps.Clone(req.AgentOptions) - for _, key := range []string{"skills", "plugins"} { - items, ok := opts[key].([]any) - if !ok { - continue - } - items = slices.Clone(items) - for i, item := range items { - descriptor, ok := item.(map[string]any) - if !ok { - continue - } - raw, _ := descriptor["download_url"].(string) - if resolved := capabilityDownloadURL(raw, base); resolved != raw { - updated := maps.Clone(descriptor) - updated["download_url"] = resolved - items[i] = updated - } - } - opts[key] = items - } - req.AgentOptions = opts + req = capabilityDownloadRequest(req, base) return factory(ctx, req, out) } } @@ -65,3 +44,42 @@ func capabilityDownloadURL(raw string, base *url.URL) string { u.RawPath = "" return u.String() } + +func capabilityDownloadRequest(req proto.PromptRequestPayload, base *url.URL) proto.PromptRequestPayload { + opts := maps.Clone(req.AgentOptions) + for _, key := range []string{"skills", "plugins"} { + items, ok := opts[key].([]any) + if !ok { + continue + } + items = slices.Clone(items) + for i, item := range items { + descriptor, ok := item.(map[string]any) + if !ok { + continue + } + raw, _ := descriptor["download_url"].(string) + if resolved := capabilityDownloadURL(raw, base); resolved != raw { + updated := maps.Clone(descriptor) + updated["download_url"] = resolved + items[i] = updated + } + } + opts[key] = items + } + req.AgentOptions = opts + return req +} + +// Executor setup applies the same immutable capability snapshot as ordinary execution. +func withExecutorCapabilities(factory agent.ExecutorFactory, serverURL string) agent.ExecutorFactory { + base, err := url.Parse(serverURL) + valid := err == nil && base.Host != "" && (base.Scheme == "http" || base.Scheme == "https") + return func(ctx context.Context, req proto.PromptRequestPayload) (agent.Executor, error) { + if valid { + req = capabilityDownloadRequest(req, base) + } + req = skillUploadRequest(req, serverURL) + return factory(ctx, req) + } +} diff --git a/apps/parsar-daemon/internal/cli/capability_downloads_test.go b/apps/parsar-daemon/internal/cli/capability_downloads_test.go index 5dcaf97e8..fb63b1775 100644 --- a/apps/parsar-daemon/internal/cli/capability_downloads_test.go +++ b/apps/parsar-daemon/internal/cli/capability_downloads_test.go @@ -92,3 +92,24 @@ func TestCapabilityDownloadsInstallZIPThroughPairedServer(t *testing.T) { t.Fatal("rewrote the original request options") } } + +func TestExecutorPreparationUsesCapabilitySnapshotTransform(t *testing.T) { + originalURL := "http://localhost:8080/internal/blobs/pg:skill" + descriptor := map[string]any{"download_url": originalURL} + source := proto.PromptRequestPayload{AgentOptions: map[string]any{"skills": []any{descriptor}}} + calls := 0 + factory := withExecutorCapabilities(func(_ context.Context, req proto.PromptRequestPayload) (agent.Executor, error) { + calls++ + actual := req.AgentOptions["skills"].([]any)[0].(map[string]any)["download_url"] + if actual != "http://paired-core:8080/internal/blobs/pg:skill" { + t.Fatalf("executor capability URL = %v", actual) + } + return nil, nil + }, "http://paired-core:8080") + if _, err := factory(t.Context(), source); err != nil { + t.Fatal(err) + } + if calls != 1 || descriptor["download_url"] != originalURL { + t.Fatal("preparation changed the source snapshot or repeated setup") + } +} diff --git a/apps/parsar-daemon/internal/cli/claude_sdk.go b/apps/parsar-daemon/internal/cli/claude_sdk.go index add58c1b1..746288935 100644 --- a/apps/parsar-daemon/internal/cli/claude_sdk.go +++ b/apps/parsar-daemon/internal/cli/claude_sdk.go @@ -128,6 +128,9 @@ func registerClaudeSDK(registry *agent.Registry, discovery *claudeSDKDiscovery) } } registry.RegisterKind(discovery.Info, factory) + if discovery.Info.Available { + registry.RegisterExecutor("claude_sdk", claudesdk.NewExecutorFactory(discovery.Config)) + } if discovery.Info.Available && discovery.Info.Capabilities.LocalEnvironment { registry.RegisterPreparation("claude_sdk", true, claudesdk.NewPreparationFactory(discovery.Config)) } diff --git a/apps/parsar-daemon/internal/cli/claude_sdk_test.go b/apps/parsar-daemon/internal/cli/claude_sdk_test.go index c691d9292..ce8eaa35f 100644 --- a/apps/parsar-daemon/internal/cli/claude_sdk_test.go +++ b/apps/parsar-daemon/internal/cli/claude_sdk_test.go @@ -78,6 +78,9 @@ func TestClaudeSDKDiscoveryAndRegistration(t *testing.T) { t.Fatal(err) } info := discovery.ClaudeSDK.Info + if info.Available { + info.Capabilities.Preparation = true + } if info.Available != tc.ready { t.Fatal(info) } diff --git a/apps/parsar-daemon/internal/cli/connect.go b/apps/parsar-daemon/internal/cli/connect.go index 345ec2b48..61f0ebe4a 100644 --- a/apps/parsar-daemon/internal/cli/connect.go +++ b/apps/parsar-daemon/internal/cli/connect.go @@ -387,8 +387,8 @@ func mainLoopRemote(rc *runContext, profile string, prof auth.Profile, agentCLIs // pumpConn runs the per-connection workload: a dispatch.Router fed by // conn.Recv(), heartbeats every boot.HeartbeatInterval(), and a -// graceful router.Shutdown on exit so any in-flight subprocesses get -// SIGTERM. +// confirmed router.Shutdown before returning ownership to the reconnect loop. +// Failed cleanup keeps this exact Router alive, including after a shutdown signal. func pumpConn(parentCtx context.Context, conn *transport.Conn, registry *agent.Registry, boot *transport.BootstrapResponse, agentCLIs agentCLIDiscovery) error { local, err := localworkspace.Load() if err != nil { @@ -406,9 +406,8 @@ func pumpConn(parentCtx context.Context, conn *transport.Conn, registry *agent.R return fmt.Errorf("router init: %w", err) } defer func() { - shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - _ = router.Shutdown(shutdownCtx) - cancel() + _ = conn.Close() + shutdownRouterUntilConfirmed(router.Shutdown, time.Second) }() conn.StartHeartbeats(parentCtx, boot.HeartbeatInterval(), func() proto.HeartbeatPayload { diff --git a/apps/parsar-daemon/internal/cli/connect_cleanup.go b/apps/parsar-daemon/internal/cli/connect_cleanup.go new file mode 100644 index 000000000..7909d7347 --- /dev/null +++ b/apps/parsar-daemon/internal/cli/connect_cleanup.go @@ -0,0 +1,24 @@ +package cli + +import ( + "context" + "time" + + obslog "github.com/MiniMax-AI-Dev/parsar/internal/obs/log" +) + +// A wait deadline does not revoke native ownership. Keep retrying the same +// Router until it confirms cleanup; reconnect and process exit both wait here. +// Router.Shutdown serializes attempts and retains resources after a failure. +func shutdownRouterUntilConfirmed(shutdown func(context.Context) error, retryDelay time.Duration) { + for { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + err := shutdown(ctx) + cancel() + if err == nil { + return + } + obslog.Bg().Warn("router cleanup unconfirmed; reconnect remains blocked", "err", err) + time.Sleep(retryDelay) + } +} diff --git a/apps/parsar-daemon/internal/cli/connect_cleanup_test.go b/apps/parsar-daemon/internal/cli/connect_cleanup_test.go new file mode 100644 index 000000000..e5da165fd --- /dev/null +++ b/apps/parsar-daemon/internal/cli/connect_cleanup_test.go @@ -0,0 +1,196 @@ +package cli + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "strings" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/transport" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/gorilla/websocket" + "sync/atomic" + "testing" + "time" +) + +func TestReconnectWaitsForConfirmedRouterCleanup(t *testing.T) { + for _, failure := range []error{errors.New("native close failed"), context.DeadlineExceeded} { + t.Run(failure.Error(), func(t *testing.T) { + var calls atomic.Int32 + retry := make(chan struct{}) + confirm := make(chan struct{}) + returned := make(chan struct{}) + go func() { + shutdownRouterUntilConfirmed(func(ctx context.Context) error { + if ctx.Err() != nil { + t.Error("cleanup inherited a cancelled connection context") + } + if _, ok := ctx.Deadline(); !ok { + t.Error("cleanup wait has no deadline") + } + if calls.Add(1) == 1 { + return failure + } + close(retry) + <-confirm + return nil + }, time.Millisecond) + close(returned) + }() + select { + case <-retry: + case <-time.After(time.Second): + t.Fatal("same cleanup was not retried") + } + select { + case <-returned: + t.Fatal("reconnect released unconfirmed ownership") + default: + } + close(confirm) + select { + case <-returned: + case <-time.After(time.Second): + t.Fatal("confirmed cleanup did not release reconnect") + } + if calls.Load() != 2 { + t.Fatalf("cleanup attempts=%d", calls.Load()) + } + }) + } +} + +type cleanupExecutor struct { + closes atomic.Int32 + retry chan struct{} + confirm chan struct{} +} + +func (e *cleanupExecutor) StartTurn(context.Context, string, proto.MessageInput, chan<- proto.Envelope) (agent.Turn, error) { + return nil, errors.New("unexpected Turn") +} +func (e *cleanupExecutor) Close(context.Context) error { + if e.closes.Add(1) == 1 { + return errors.New("native cleanup temporarily unavailable") + } + close(e.retry) + <-e.confirm + return nil +} + +func TestDisconnectedPumpRetainsExactExecutorUntilCleanup(t *testing.T) { + for _, suspend := range []bool{false, true} { + name := "ordinary" + if suspend { + name = "suspension-enabled" + } + t.Run(name, func(t *testing.T) { testDisconnectedPumpCleanup(t, suspend) }) + } +} + +func testDisconnectedPumpCleanup(t *testing.T, suspend bool) { + t.Setenv("OAC_RUNTIME_WORKSPACE", "") + peers := make(chan *websocket.Conn, 1) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + peer, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + if err == nil { + peers <- peer + } + })) + defer server.Close() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + dial := func(ctx context.Context) (*transport.Conn, error) { + return transport.Dial(ctx, transport.DialOptions{ + WSURL: "ws" + strings.TrimPrefix(server.URL, "http"), DeviceID: "device", + Credential: "credential", DaemonVersion: proto.Version, + }) + } + owner := &cleanupExecutor{retry: make(chan struct{}), confirm: make(chan struct{})} + registry := agent.NewRegistry() + registry.RegisterKind(proto.SupportedAgentKind{Kind: "cleanup", Available: true, Capabilities: proto.AgentKindCapabilities{EnvironmentNone: true}}, + func(context.Context, proto.PromptRequestPayload, chan<- proto.Envelope) (agent.Session, error) { + return nil, errors.New("unexpected legacy factory") + }) + var factories atomic.Int32 + registry.RegisterExecutor("cleanup", func(context.Context, proto.PromptRequestPayload) (agent.Executor, error) { + factories.Add(1) + return owner, nil + }) + finished := make(chan error, 1) + go func() { + boot := &transport.BootstrapResponse{HeartbeatSeconds: 60} + if suspend { + control := &suspendControl{signal: make(chan os.Signal, 1)} + finished <- runSuspendLoop(ctx, dial, registry, boot, agentCLIDiscovery{}, control) + return + } + conn, err := dial(ctx) + if err != nil { + finished <- err + return + } + defer conn.Close() + finished <- pumpConn(ctx, conn, registry, boot, agentCLIDiscovery{}) + }() + var peer *websocket.Conn + select { + case peer = <-peers: + case <-time.After(3 * time.Second): + t.Fatal("initial connection missing") + } + defer peer.Close() + env, err := proto.NewEnvelope(proto.TypeExecutionPrepare, "prepare", proto.ExecutionPreparePayload{SessionID: "cleanup", + Configuration: proto.PromptRequestPayload{AgentKind: "cleanup", AgentStateKey: "agents-api-cleanup", StrictResume: true, DisableExecutionEnvironment: true}}) + if err != nil { + t.Fatal(err) + } + if err := peer.WriteJSON(env); err != nil { + t.Fatal(err) + } + _ = peer.SetReadDeadline(time.Now().Add(3 * time.Second)) + for { + var result proto.Envelope + if err := peer.ReadJSON(&result); err != nil { + t.Fatal(err) + } + if result.Type != proto.TypePreparationStatus { + continue + } + var status proto.PreparationStatusPayload + if err := result.DecodePayload(&status); err != nil { + t.Fatal(err) + } + if status.State == "ready" { + break + } + if status.State == "failed" { + t.Fatalf("preparation failed: %+v", status) + } + } + _ = peer.Close() + select { + case <-owner.retry: + case <-time.After(3 * time.Second): + t.Fatal("original executor cleanup was not retried") + } + select { + case err := <-finished: + t.Fatalf("pump discarded native ownership: %v", err) + default: + } + cancel() + close(owner.confirm) + select { + case <-finished: + case <-time.After(3 * time.Second): + t.Fatal("pump did not return after confirmed cleanup") + } + if factories.Load() != 1 || owner.closes.Load() != 2 { + t.Fatalf("factory=%d close=%d", factories.Load(), owner.closes.Load()) + } +} diff --git a/apps/parsar-daemon/internal/cli/connect_suspend.go b/apps/parsar-daemon/internal/cli/connect_suspend.go index 38a391252..c471a12f0 100644 --- a/apps/parsar-daemon/internal/cli/connect_suspend.go +++ b/apps/parsar-daemon/internal/cli/connect_suspend.go @@ -55,10 +55,8 @@ func newSuspendedRouter(conn *transport.Conn, registry *agent.Registry) (*suspen return &suspendedRouter{router: router, bridge: bridge, sender: sender, registry: wrapped, local: local}, nil } -func (s *suspendedRouter) shutdown() error { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - return s.router.Shutdown(ctx) +func (s *suspendedRouter) shutdown() { + shutdownRouterUntilConfirmed(s.router.Shutdown, time.Second) } // runSuspendLoop uses the ordinary connection authentication and dispatch chain. @@ -78,10 +76,7 @@ func runSuspendLoop(ctx context.Context, dial transport.DialFn, registry *agent. return err } err = serveSuspendLifecycle(ctx, conn, dial, state, boot, discovery, control) - closeErr := state.shutdown() - if closeErr != nil { - return errors.Join(err, closeErr) - } + state.shutdown() if ctx.Err() != nil { return nil } diff --git a/apps/parsar-daemon/internal/cli/skill_upload.go b/apps/parsar-daemon/internal/cli/skill_upload.go index 935b35cd3..c77b0fefe 100644 --- a/apps/parsar-daemon/internal/cli/skill_upload.go +++ b/apps/parsar-daemon/internal/cli/skill_upload.go @@ -12,16 +12,7 @@ import ( func withSkillUploadServer(factory agent.Factory, serverURL string) agent.Factory { return func(ctx context.Context, req proto.PromptRequestPayload, out chan<- proto.Envelope) (agent.Session, error) { - env, _ := req.AgentOptions["env"].(map[string]any) - if token, _ := env["PARSAR_CAPABILITY_UPLOAD_TOKEN"].(string); token != "" { - env = maps.Clone(env) - env["PARSAR_SERVER_URL"] = serverURL - if executable, err := os.Executable(); err == nil { - addCompanionCLIPath(env, filepath.Dir(executable)) - } - req.AgentOptions = maps.Clone(req.AgentOptions) - req.AgentOptions["env"] = env - } + req = skillUploadRequest(req, serverURL) return factory(ctx, req, out) } } @@ -37,3 +28,17 @@ func addCompanionCLIPath(env map[string]any, dir string) { } env["PATH"] = dir + string(os.PathListSeparator) + path } + +func skillUploadRequest(req proto.PromptRequestPayload, serverURL string) proto.PromptRequestPayload { + env, _ := req.AgentOptions["env"].(map[string]any) + if token, _ := env["PARSAR_CAPABILITY_UPLOAD_TOKEN"].(string); token != "" { + env = maps.Clone(env) + env["PARSAR_SERVER_URL"] = serverURL + if executable, err := os.Executable(); err == nil { + addCompanionCLIPath(env, filepath.Dir(executable)) + } + req.AgentOptions = maps.Clone(req.AgentOptions) + req.AgentOptions["env"] = env + } + return req +} diff --git a/apps/parsar-daemon/internal/dispatch/executor.go b/apps/parsar-daemon/internal/dispatch/executor.go new file mode 100644 index 000000000..22fa3d708 --- /dev/null +++ b/apps/parsar-daemon/internal/dispatch/executor.go @@ -0,0 +1,373 @@ +package dispatch + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "strings" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/google/uuid" +) + +const executorIdleCapacity = 16 + +type executorState struct { + id, sessionID, environmentID, stateKey string + fingerprint [32]byte + native agent.Executor + nativeID string + ctx context.Context + cancel context.CancelFunc + admission *preparationState + run *sessionState + preparing bool + invalid bool + closeDone chan struct{} + closeReason string + closeErr error + timer *time.Timer + idleLease uint64 +} + +func executorFingerprint(req proto.PromptRequestPayload) ([32]byte, error) { + req.RunID, req.Input = "", nil + req.AgentSessionID = "" + req.RequireExistingNativeSession = false + req.ReleaseOnCompletion = false + data, err := json.Marshal(req) + return sha256.Sum256(data), err +} + +func (r *Router) handleExecutorPrepare(ctx context.Context, env proto.Envelope, input proto.ExecutionPreparePayload) error { + req := input.Configuration + if strings.TrimSpace(input.SessionID) == "" || req.RunID != "" || len(req.Input) != 0 || req.ConversationID != "" || req.WorkspaceAuthoring || req.AgentStateKey != "agents-api-"+input.SessionID || !req.StrictResume { + return r.rejectPreparation(env, "invalid_configuration") + } + caps := r.availableCapabilities(req.AgentKind) + factory, err := r.registry.ResolveExecutor(req.AgentKind) + if err != nil || !caps.Preparation { + return r.rejectPreparation(env, "unsupported_preparation") + } + req, err = r.localWorkspace.Configure(req) + if err != nil { + return r.rejectPreparation(env, "invalid_configuration") + } + if validateExecutionEnvironment(req, caps) != nil || len(req.FunctionTools) > 0 && !caps.FunctionTools { + return r.rejectPreparation(env, "unsupported_configuration") + } + fingerprint, err := executorFingerprint(req) + if err != nil { + return r.rejectPreparation(env, "invalid_configuration") + } + encoded, err := json.Marshal(input) + if err != nil { + return r.rejectPreparation(env, "invalid_configuration") + } + requestFingerprint := sha256.Sum256(encoded) + r.mu.Lock() + if r.closed || r.suspension != nil { + r.mu.Unlock() + return ErrRouterClosed + } + if r.workspaceWrite != nil || r.workspaceExport != nil { + r.mu.Unlock() + return r.rejectPreparation(env, "resource_unavailable") + } + r.prunePreparationsLocked() + if old := r.preparationRequests[env.ID]; old != nil { + matches, status := old.fingerprint == requestFingerprint, old.status + r.mu.Unlock() + if !matches { + return r.rejectPreparation(env, "request_conflict") + } + r.publishPreparation(old, status) + return nil + } + if len(r.preparations) >= preparationRecords { + r.mu.Unlock() + return r.rejectPreparation(env, "preparation_capacity") + } + active := 0 + for _, candidate := range r.executors { + if candidate.sessionID != input.SessionID && candidate.stateKey == req.AgentStateKey { + r.mu.Unlock() + return r.rejectPreparation(env, "session_binding_conflict") + } + } + for _, owner := range r.executors { + if owner.preparing || owner.admission != nil || owner.run != nil || owner.invalid { + active++ + } + } + if active >= preparationCapacity { + r.mu.Unlock() + return r.rejectPreparation(env, "preparation_capacity") + } + owner := r.executors[input.SessionID] + reused := owner != nil + if owner != nil { + code := "" + switch { + case owner.environmentID != req.EnvironmentID() || owner.fingerprint != fingerprint: + code = "configuration_conflict" + case owner.invalid: + code = "executor_cleanup_unconfirmed" + case owner.preparing || owner.admission != nil || owner.run != nil: + code = "resource_unavailable" + case req.AgentSessionID != owner.nativeID: + code = "native_session_conflict" + case req.RequireExistingNativeSession && owner.nativeID == "": + code = "history_unavailable" + } + if code != "" { + r.mu.Unlock() + return r.rejectPreparation(env, code) + } + if owner.timer != nil { + owner.timer.Stop() + } + owner.idleLease++ + } else { + if len(r.executors) >= executorIdleCapacity+preparationCapacity { + r.mu.Unlock() + return r.rejectPreparation(env, "executor_capacity") + } + ownerCtx, cancel := context.WithCancel(context.WithoutCancel(ctx)) + owner = &executorState{id: uuid.NewString(), sessionID: input.SessionID, environmentID: req.EnvironmentID(), stateKey: req.AgentStateKey, fingerprint: fingerprint, ctx: ownerCtx, cancel: cancel, preparing: true, nativeID: req.AgentSessionID} + r.executors[input.SessionID] = owner + r.log.Info("executor owner_created", "executor_id", owner.id, "session_id", owner.sessionID) + } + operation, cancel := context.WithCancel(context.WithoutCancel(ctx)) + p := &preparationState{requestID: env.ID, trace: env.Trace, fingerprint: requestFingerprint, ctx: operation, cancel: cancel, stateKey: req.AgentStateKey, environmentID: req.EnvironmentID(), executor: owner, owns: true, busy: !reused, deadline: time.Now().Add(r.preparationTimeout)} + state := "preparing" + if reused { + state = "ready" + } + p.status = proto.PreparationStatusPayload{Handle: uuid.NewString(), ExecutorID: owner.id, Revision: 1, State: state, Reused: reused, ExpiresAt: p.deadline.UnixMilli()} + owner.admission = p + r.preparations[p.status.Handle], r.preparationRequests[env.ID] = p, p + p.timer = time.AfterFunc(r.preparationTimeout, func() { r.releasePreparation(p, "expired", "", true, true) }) + if !reused { + r.shutdownWG.Add(1) + } + status := p.status + r.mu.Unlock() + if reused { + r.publishPreparation(p, status) + } else { + go r.prepareExecutor(p, req, factory) + } + return nil +} + +func (r *Router) prepareExecutor(p *preparationState, req proto.PromptRequestPayload, factory agent.ExecutorFactory) { + defer r.shutdownWG.Done() + owner := p.executor + started := time.Now() + r.mu.Lock() + initial := p.status + r.mu.Unlock() + if !r.sendPreparation(p.requestID, p.trace, initial) { + r.abandonExecutorAdmission(p, "failed", "status_delivery_failed", false) + } + var native agent.Executor + var err error + if owner.ctx.Err() == nil { + native, err = factory(owner.ctx, req) + } else { + err = owner.ctx.Err() + } + r.mu.Lock() + owner.native, owner.preparing = native, false + r.log.Info("executor native_prepare", "executor_id", owner.id, "session_id", owner.sessionID, "duration_ms", time.Since(started).Milliseconds(), "success", err == nil && native != nil) + ready := native != nil && err == nil && !owner.invalid && !r.closed && r.suspension == nil && p.status.State == "preparing" && p.ctx.Err() == nil + p.busy = false + if ready { + p.status.State, p.status.Revision = "ready", p.status.Revision+1 + } else { + owner.invalid = true + owner.closeReason = "preparation_failed" + if p.status.State == "preparing" { + p.status.State, p.status.ErrorCode, p.status.Revision = "failed", "preparation_failed", p.status.Revision+1 + } + p.timer.Stop() + } + status := p.status + r.mu.Unlock() + if !ready { + closeErr := r.closeExecutor(owner) + r.mu.Lock() + p.owns, p.closeErr = closeErr != nil, closeErr + if closeErr != nil { + p.status.State, p.status.ErrorCode, p.status.Revision = "failed", "executor_cleanup_unconfirmed", p.status.Revision+1 + } + status = p.status + r.mu.Unlock() + } + if !r.sendPreparation(p.requestID, p.trace, status) && ready { + r.abandonExecutorAdmission(p, "failed", "status_delivery_failed", false) + } +} + +func (r *Router) abandonExecutorAdmission(p *preparationState, state, code string, publish bool) { + r.mu.Lock() + owner := p.executor + if p.handoff != nil || p.status.State == "started" { + status := p.status + r.mu.Unlock() + if publish && state == "released" { + r.publishPreparation(p, status) + } + return + } + if p.status.State == "preparing" || p.status.State == "ready" { + p.status.State, p.status.ErrorCode, p.status.Revision = state, code, p.status.Revision+1 + p.timer.Stop() + p.cancel() + p.owns = false + if owner.admission == p { + owner.admission = nil + } + if owner.preparing { + owner.invalid = true + owner.cancel() + } else if !owner.invalid { + r.scheduleExecutorIdleLocked(owner) + } + } + status := p.status + r.mu.Unlock() + if publish { + r.publishPreparation(p, status) + } +} + +func (r *Router) scheduleExecutorIdleLocked(owner *executorState) { + if owner.invalid || owner.preparing || owner.run != nil || owner.admission != nil { + return + } + if owner.timer != nil { + owner.timer.Stop() + } + idle := 0 + for _, candidate := range r.executors { + if candidate != owner && candidate.run == nil && candidate.admission == nil && !candidate.preparing { + idle++ + } + } + if idle >= executorIdleCapacity { + owner.invalid = true + owner.closeReason = "idle_capacity" + r.shutdownWG.Add(1) + go func() { defer r.shutdownWG.Done(); _ = r.closeExecutor(owner) }() + return + } + owner.idleLease++ + lease := owner.idleLease + r.log.Info("executor owner_idle", "executor_id", owner.id, "session_id", owner.sessionID) + owner.timer = time.AfterFunc(r.idleTimeout, func() { + r.mu.Lock() + if r.executors[owner.sessionID] != owner || owner.idleLease != lease || owner.run != nil || owner.admission != nil || owner.invalid { + r.mu.Unlock() + return + } + owner.invalid = true + owner.closeReason = "idle_expired" + r.shutdownWG.Add(1) + r.mu.Unlock() + defer r.shutdownWG.Done() + _ = r.closeExecutor(owner) + }) +} + +// Close failures keep the exact owner; a later call retries only settled failures. +func (r *Router) closeExecutor(owner *executorState) error { + r.mu.Lock() + owner.invalid = true + if owner.closeReason == "" { + owner.closeReason = "invalidated" + } + if owner.timer != nil { + owner.timer.Stop() + } + if owner.preparing { + owner.cancel() + r.mu.Unlock() + return errors.New("executor preparation is still settling") + } + if done := owner.closeDone; done != nil { + select { + case <-done: + if owner.closeErr == nil { + r.mu.Unlock() + return nil + } + default: + r.mu.Unlock() + <-done + r.mu.Lock() + err := owner.closeErr + r.mu.Unlock() + return err + } + } + done := make(chan struct{}) + owner.closeDone = done + native := owner.native + r.mu.Unlock() + var err error + if native != nil { + ctx, cancel := context.WithTimeout(context.Background(), preparedCancelTimeout) + err = native.Close(ctx) + cancel() + } + r.mu.Lock() + owner.closeErr = err + if err == nil { + owner.cancel() + } + r.log.Info("executor owner_closed", "executor_id", owner.id, "session_id", owner.sessionID, "confirmed", err == nil, "reason", owner.closeReason) + if err == nil && owner.run == nil && r.executors[owner.sessionID] == owner { + delete(r.executors, owner.sessionID) + } + close(done) + r.mu.Unlock() + return err +} + +func (r *Router) closeIdleExecutorsLocked() []*executorState { + var owners []*executorState + for _, owner := range r.executors { + owner.invalid = true + owner.closeReason = "shutdown" + if owner.preparing { + owner.cancel() + } + if owner.timer != nil { + owner.timer.Stop() + } + if owner.admission != nil && owner.run == nil { + p := owner.admission + p.cancel() + p.timer.Stop() + p.status.State, p.status.ErrorCode, p.status.Revision = "failed", "connection_closed", p.status.Revision+1 + p.owns = false + owner.admission = nil + } + if !owner.preparing && owner.run == nil { + r.shutdownWG.Add(1) + owners = append(owners, owner) + } + } + return owners +} + +func (r *Router) closeIdleExecutors(owners []*executorState) { + for _, owner := range owners { + go func() { defer r.shutdownWG.Done(); _ = r.closeExecutor(owner) }() + } +} diff --git a/apps/parsar-daemon/internal/dispatch/executor_test.go b/apps/parsar-daemon/internal/dispatch/executor_test.go new file mode 100644 index 000000000..db02f9ff8 --- /dev/null +++ b/apps/parsar-daemon/internal/dispatch/executor_test.go @@ -0,0 +1,312 @@ +package dispatch_test + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/dispatch" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +type reusableExecutor struct { + starts chan *reusableTurn + closes atomic.Int32 + mu sync.Mutex + closeErr error + startErr error +} + +func (e *reusableExecutor) Close(context.Context) error { + e.closes.Add(1) + e.mu.Lock() + defer e.mu.Unlock() + return e.closeErr +} +func (e *reusableExecutor) StartTurn(_ context.Context, id string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Turn, error) { + if e.startErr != nil { + return nil, e.startErr + } + t := &reusableTurn{id: id, out: out, settled: make(chan struct{})} + e.starts <- t + return t, nil +} + +type reusableTurn struct { + id string + out chan<- proto.Envelope + settled chan struct{} + once sync.Once + cancels atomic.Int32 +} + +func (t *reusableTurn) finish() { + t.once.Do(func() { + frame, _ := proto.NewEnvelope(proto.TypeDone, t.id, t.CancellationOutcome()) + t.out <- frame + close(t.out) + close(t.settled) + }) +} +func (t *reusableTurn) Cancel(context.Context) error { t.cancels.Add(1); t.finish(); return nil } +func (t *reusableTurn) CancellationOutcome() proto.DonePayload { + return proto.DonePayload{Content: t.id, Metadata: map[string]any{proto.DoneMetaAgentSessionID: "native-session"}} +} +func (t *reusableTurn) AwaitSettlement(ctx context.Context) (agent.TurnSettlement, error) { + select { + case <-t.settled: + return agent.TurnSettlement{Reusable: true}, nil + case <-ctx.Done(): + return agent.TurnSettlement{}, ctx.Err() + } +} + +func executorRequest() proto.ExecutionPreparePayload { + return proto.ExecutionPreparePayload{SessionID: "session", Configuration: proto.PromptRequestPayload{AgentKind: "reusable", AgentStateKey: "agents-api-session", StrictResume: true, DisableExecutionEnvironment: true}} +} +func executorRouter(t *testing.T, owner *reusableExecutor, idle time.Duration) (*dispatch.Router, *recSender, *atomic.Int32) { + t.Helper() + calls := &atomic.Int32{} + reg := agent.NewRegistry() + reg.RegisterKind(proto.SupportedAgentKind{Kind: "reusable", Available: true, Capabilities: proto.AgentKindCapabilities{EnvironmentNone: true}}, func(context.Context, proto.PromptRequestPayload, chan<- proto.Envelope) (agent.Session, error) { + return nil, errors.New("ordinary factory is forbidden") + }) + reg.RegisterExecutor("reusable", func(context.Context, proto.PromptRequestPayload) (agent.Executor, error) { + calls.Add(1) + return owner, nil + }) + sender := &recSender{} + router, err := dispatch.New(dispatch.Config{Registry: reg, Sender: sender, IdleTimeout: idle}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + owner.mu.Lock() + owner.closeErr = nil + owner.mu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := router.Shutdown(ctx); err != nil { + t.Error(err) + } + }) + return router, sender, calls +} +func executorAdmission(t *testing.T, r *dispatch.Router, s *recSender, key string, req proto.ExecutionPreparePayload) proto.PreparationStatusPayload { + t.Helper() + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, key, req)); err != nil { + t.Fatal(err) + } + return waitPreparationStatus(t, s, key, "ready", "") +} +func startExecutorTurn(t *testing.T, r *dispatch.Router, s *recSender, key, id string, p proto.PreparationStatusPayload) { + t.Helper() + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, key, proto.ExecutionStartPayload{Handle: p.Handle, ExecutorID: p.ExecutorID, RunID: id, Input: proto.TextInput("input")})); err != nil { + t.Fatal(err) + } + waitPreparationStatus(t, s, key, "started", "") +} +func TestExecutorRetainsOwnerAcrossIndependentTurns(t *testing.T) { + owner := &reusableExecutor{starts: make(chan *reusableTurn, 2)} + r, s, calls := executorRouter(t, owner, time.Minute) + first := executorAdmission(t, r, s, "first", executorRequest()) + startExecutorTurn(t, r, s, "first", "one", first) + turn := <-owner.starts + turn.finish() + waitFor(t, func() bool { return r.ActiveRuns() == 0 }, "first Turn settled") + if owner.closes.Load() != 0 || turn.cancels.Load() != 0 { + t.Fatal("normal completion tore down or cancelled the executor") + } + _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionRelease, "first", proto.ExecutionReleasePayload{Handle: first.Handle})) + req := executorRequest() + req.Configuration.AgentSessionID = "native-session" + req.Configuration.RequireExistingNativeSession = true + second := executorAdmission(t, r, s, "second", req) + if second.Handle == first.Handle || second.ExecutorID != first.ExecutorID || !second.Reused || calls.Load() != 1 { + t.Fatalf("reuse identities: first=%+v second=%+v calls=%d", first, second, calls.Load()) + } + startExecutorTurn(t, r, s, "second", "two", second) + next := <-owner.starts + _ = turn.Cancel(t.Context()) + if next.cancels.Load() != 0 { + t.Fatal("old Turn cancellation reached its successor") + } + next.finish() + waitFor(t, func() bool { return r.ActiveRuns() == 0 }, "second Turn settled") + if owner.closes.Load() != 0 { + t.Fatal("executor closed between Turns") + } +} +func TestExecutorCancelSettlesTurnWithoutClosingOwner(t *testing.T) { + owner := &reusableExecutor{starts: make(chan *reusableTurn, 1)} + r, s, _ := executorRouter(t, owner, time.Minute) + p := executorAdmission(t, r, s, "first", executorRequest()) + startExecutorTurn(t, r, s, "first", "run", p) + turn := <-owner.starts + if err := r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "cancel"})); err != nil { + t.Fatal(err) + } + waitFor(t, func() bool { return len(cancellationAcks(s)) == 1 }, "cancellation receipt") + ack := cancellationAcks(s)[0] + if !ack.Applied || ack.Outcome == nil || turn.cancels.Load() != 1 || owner.closes.Load() != 0 { + t.Fatalf("cancel=%+v closes=%d", ack, owner.closes.Load()) + } +} +func TestExecutorRejectsConfigurationAndNativeIdentityChanges(t *testing.T) { + owner := &reusableExecutor{starts: make(chan *reusableTurn, 1)} + r, s, calls := executorRouter(t, owner, time.Minute) + p := executorAdmission(t, r, s, "first", executorRequest()) + startExecutorTurn(t, r, s, "first", "one", p) + (<-owner.starts).finish() + waitFor(t, func() bool { return r.ActiveRuns() == 0 }, "Turn settlement") + for _, kind := range []string{"configuration", "native"} { + req := executorRequest() + if kind == "configuration" { + req.Configuration.AgentOptions = map[string]any{"model": "changed"} + } else { + req.Configuration.AgentSessionID = "other-native" + } + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, kind, req)); err == nil { + t.Fatalf("accepted changed %s", kind) + } + } + if calls.Load() != 1 || owner.closes.Load() != 0 { + t.Fatal("conflict replaced executor") + } +} +func TestExecutorIdleExpiryClosesRetainedOwner(t *testing.T) { + owner := &reusableExecutor{starts: make(chan *reusableTurn, 1)} + r, s, _ := executorRouter(t, owner, 30*time.Millisecond) + p := executorAdmission(t, r, s, "first", executorRequest()) + startExecutorTurn(t, r, s, "first", "one", p) + (<-owner.starts).finish() + waitFor(t, func() bool { return owner.closes.Load() == 1 }, "idle executor close") +} +func TestExecutorPreInputFailureConfirmsCloseBeforeRetrySignal(t *testing.T) { + for _, unconfirmed := range []bool{false, true} { + t.Run(map[bool]string{false: "closed", true: "unconfirmed"}[unconfirmed], func(t *testing.T) { + owner := &reusableExecutor{starts: make(chan *reusableTurn, 1), startErr: errors.New("native exited")} + if unconfirmed { + owner.closeErr = errors.New("cleanup unknown") + } + r, s, _ := executorRouter(t, owner, time.Minute) + p := executorAdmission(t, r, s, "first", executorRequest()) + _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "first", proto.ExecutionStartPayload{Handle: p.Handle, ExecutorID: p.ExecutorID, RunID: "run", Input: proto.TextInput("input")})) + status := waitPreparationStatus(t, s, "first", "rejected", "") + want := "executor_unavailable" + if unconfirmed { + want = "executor_cleanup_unconfirmed" + } + if status.ErrorCode != want || owner.closes.Load() == 0 { + t.Fatalf("status=%+v closes=%d", status, owner.closes.Load()) + } + if unconfirmed { + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "retry", executorRequest())); err == nil { + t.Fatal("replaced unconfirmed owner") + } + } + }) + } +} + +func poolRouter(t *testing.T, factory agent.ExecutorFactory) (*dispatch.Router, *recSender) { + t.Helper() + registry := agent.NewRegistry() + registry.RegisterKind(proto.SupportedAgentKind{Kind: "reusable", Available: true, Capabilities: proto.AgentKindCapabilities{EnvironmentNone: true}}, func(context.Context, proto.PromptRequestPayload, chan<- proto.Envelope) (agent.Session, error) { + return nil, errors.New("unexpected legacy factory") + }) + registry.RegisterExecutor("reusable", factory) + sender := &recSender{} + router, err := dispatch.New(dispatch.Config{Registry: registry, Sender: sender, IdleTimeout: time.Minute}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := router.Shutdown(ctx); err != nil { + t.Error(err) + } + }) + return router, sender +} +func poolRequest(id string) proto.ExecutionPreparePayload { + req := executorRequest() + req.SessionID = id + req.Configuration.AgentStateKey = "agents-api-" + id + return req +} + +func TestExecutorIdleAndActiveCapacitiesAreIndependent(t *testing.T) { + var mu sync.Mutex + var owners []*reusableExecutor + r, s := poolRouter(t, func(context.Context, proto.PromptRequestPayload) (agent.Executor, error) { + e := &reusableExecutor{} + mu.Lock() + owners = append(owners, e) + mu.Unlock() + return e, nil + }) + for i := 0; i < 17; i++ { + id := fmt.Sprint("idle-", i) + ready := executorAdmission(t, r, s, id, poolRequest(id)) + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionRelease, id, proto.ExecutionReleasePayload{Handle: ready.Handle})); err != nil { + t.Fatal(err) + } + } + waitFor(t, func() bool { + mu.Lock() + defer mu.Unlock() + var closed int32 + for _, e := range owners { + closed += e.closes.Load() + } + return closed == 1 + }, "idle capacity eviction") + for i := 0; i < 4; i++ { + id := fmt.Sprint("active-", i) + executorAdmission(t, r, s, id, poolRequest(id)) + } + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "overflow", poolRequest("overflow"))); err == nil { + t.Fatal("active capacity exceeded") + } +} + +func TestExecutorRejectsSessionStateScopeMismatch(t *testing.T) { + e := &reusableExecutor{} + r, s, calls := executorRouter(t, e, time.Minute) + executorAdmission(t, r, s, "one", executorRequest()) + req := executorRequest() + req.SessionID = "another" + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "scope", req)); err == nil { + t.Fatal("two sessions shared native state") + } + if calls.Load() != 1 { + t.Fatal("mismatched session invoked native preparation") + } +} + +func TestExecutorRejectsOutputFromAnotherTurn(t *testing.T) { + e := &reusableExecutor{starts: make(chan *reusableTurn, 1)} + r, s, _ := executorRouter(t, e, time.Minute) + p := executorAdmission(t, r, s, "one", executorRequest()) + startExecutorTurn(t, r, s, "one", "current", p) + turn := <-e.starts + wrong, _ := proto.NewEnvelope(proto.TypeDelta, "old-turn", proto.DeltaPayload{Delta: "late"}) + turn.out <- wrong + turn.finish() + waitFor(t, func() bool { return r.ActiveRuns() == 0 }, "invalid native event cleanup") + if e.closes.Load() != 1 { + t.Fatal("invalid event left executor reusable") + } + for _, frame := range s.snapshot() { + if frame.ID == "old-turn" { + t.Fatal("old Turn output escaped") + } + } +} diff --git a/apps/parsar-daemon/internal/dispatch/local_directory_test.go b/apps/parsar-daemon/internal/dispatch/local_directory_test.go index 0515a59d6..56605ca58 100644 --- a/apps/parsar-daemon/internal/dispatch/local_directory_test.go +++ b/apps/parsar-daemon/internal/dispatch/local_directory_test.go @@ -57,7 +57,7 @@ func TestLocalDirectoryPreparationNeedsNoHarnessAndRejectsOtherOwners(t *testing if got := waitWorkspaceRead(t, sender, "foreign"); got.Outcome != "rejected" { t.Fatal("foreign directory accepted", got) } - if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "idle", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "forbidden", Input: proto.TextInput("work")})); err == nil { + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "idle", proto.ExecutionStartPayload{Handle: ready.Handle, ExecutorID: ready.ExecutorID, RunID: "forbidden", Input: proto.TextInput("work")})); err == nil { t.Fatal("read preparation admitted execution") } bad := request diff --git a/apps/parsar-daemon/internal/dispatch/mcp_http_test.go b/apps/parsar-daemon/internal/dispatch/mcp_http_test.go index 127a7f96d..13bb7e76e 100644 --- a/apps/parsar-daemon/internal/dispatch/mcp_http_test.go +++ b/apps/parsar-daemon/internal/dispatch/mcp_http_test.go @@ -102,7 +102,7 @@ func TestLocalMCPRejectsBeforePreparationFactory(t *testing.T) { t.Error("ordinary factory called") return nil, errors.New("unexpected") }) - h.reg.RegisterPreparation("prepared", false, func(context.Context, proto.PromptRequestPayload) (agent.Prepared, error) { + h.reg.RegisterExecutor("prepared", func(context.Context, proto.PromptRequestPayload) (agent.Executor, error) { entered <- struct{}{} return nil, errors.New("controlled stop") }) diff --git a/apps/parsar-daemon/internal/dispatch/preparation.go b/apps/parsar-daemon/internal/dispatch/preparation.go index acfe25be6..26e230cab 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation.go +++ b/apps/parsar-daemon/internal/dispatch/preparation.go @@ -19,6 +19,7 @@ const preparationRecords = 64 // All mutable fields are protected by Router.mu. owns includes resources whose // cancellation is underway; a slow close cannot bypass the capacity bound. type preparationState struct { + executor *executorState requestID string trace string fingerprint [32]byte @@ -44,6 +45,9 @@ func (r *Router) handleExecutionPrepare(ctx context.Context, env proto.Envelope) return r.rejectPreparation(env, "invalid_request") } req := input.Configuration + if !req.WorkspaceReadOnly { + return r.handleExecutorPrepare(ctx, env, input) + } caps := r.availableCapabilities(req.AgentKind) prepare, err := r.registry.ResolvePreparation(req.AgentKind) if err != nil || !caps.Preparation { @@ -179,6 +183,11 @@ func (r *Router) releasePreparation(p *preparationState, state, code string, pub r.mu.Unlock() return } + if p.executor != nil { + r.mu.Unlock() + r.abandonExecutorAdmission(p, state, code, publish) + return + } if p.handoff != nil { if active := r.sessions[p.status.RunID]; active != nil && active.preparedHandoff == p.handoff { if state == "expired" && p.handoff.published { diff --git a/apps/parsar-daemon/internal/dispatch/preparation_cancel_test.go b/apps/parsar-daemon/internal/dispatch/preparation_cancel_test.go index d5fd98ec6..ecba30e74 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation_cancel_test.go +++ b/apps/parsar-daemon/internal/dispatch/preparation_cancel_test.go @@ -3,7 +3,6 @@ package dispatch_test import ( "context" "errors" - "fmt" "reflect" "sync" "sync/atomic" @@ -49,7 +48,7 @@ func startCancellationPreparation(t *testing.T, r *dispatch.Router, sender *recS t.Fatal(err) } ready := waitPreparationStatus(t, sender, "request", "ready", "") - if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "run", Input: proto.TextInput("input")})); err != nil { + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, ExecutorID: ready.ExecutorID, RunID: "run", Input: proto.TextInput("input")})); err != nil { t.Fatal(err) } return ready @@ -167,7 +166,7 @@ func TestPreparedCancellationWaitsForOutputAndCleanup(t *testing.T) { } func TestPreparedCancellationBeforeTransferPreservesUnknownOutcome(t *testing.T) { - for _, boundary := range []string{"start_failure", "release", "expiry", "shutdown"} { + for _, boundary := range []string{"start_failure", "release", "shutdown"} { t.Run(boundary, func(t *testing.T) { sender := &recSender{} entered, cancelled, allowReturn := make(chan struct{}), make(chan struct{}), make(chan struct{}) @@ -195,7 +194,7 @@ func TestPreparedCancellationBeforeTransferPreservesUnknownOutcome(t *testing.T) switch boundary { case "release": _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionRelease, "request", proto.ExecutionReleasePayload{Handle: ready.Handle})) - waitPreparationStatus(t, sender, "request", "released", "") + // An admission release does not terminate a handed-off Turn. case "expiry": waitPreparationStatus(t, sender, "request", "expired", "") case "shutdown": @@ -248,6 +247,9 @@ func TestPreparedCancellationFailuresRemainConservative(t *testing.T) { startCancellationPreparation(t, r, sender.recSender) <-entered _ = r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "cancel"})) + if failure == "cancel_failed" { + waitFor(t, func() bool { return p.calls.Load() == 1 }, "failed pre-transfer cleanup") + } close(allowReturn) waitFor(t, func() bool { return len(cancellationAcks(sender.recSender)) == 1 }, "failed cancellation") ack := cancellationAcks(sender.recSender)[0] @@ -255,17 +257,21 @@ func TestPreparedCancellationFailuresRemainConservative(t *testing.T) { t.Fatalf("uncertain cancellation accepted: %+v", ack) } if failure == "cancel_failed" { - if r.ActiveRuns() != 1 || session.cancels() != 0 { - t.Fatal("failed attempt released ownership or used a fallback target") + waitFor(t, func() bool { p.mu.Lock(); defer p.mu.Unlock(); return p.session != nil }, "late Turn returned") + if r.ActiveRuns() != 1 { + t.Fatal("unconfirmed cleanup lost owner") } _ = r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "retry"})) - waitFor(t, func() bool { return len(cancellationAcks(sender.recSender)) == 2 && r.ActiveRuns() == 0 }, "same-target cancellation retry") - if retry := cancellationAcks(sender.recSender)[1]; !retry.Applied || retry.ErrorCode != "" || session.cancels() != 1 || p.calls.Load() != 2 { - t.Fatalf("same target was not retried exactly once: ack=%+v calls=%d session=%d", retry, p.calls.Load(), session.cancels()) + waitFor(t, func() bool { return len(cancellationAcks(sender.recSender)) == 2 }, "cleanup retry receipt") + if !cancellationAcks(sender.recSender)[1].Applied { + t.Fatal("confirmed retry was rejected") } - } else if r.ActiveRuns() != 0 || session.cancels() != 1 { - t.Fatal("successful cleanup did not settle output failure") } + waitFor(t, func() bool { return r.ActiveRuns() == 0 }, "confirmed resource cleanup") + if session.cancels() != 1 { + t.Fatal("confirmed cleanup did not stop the native Turn") + } + }) } } @@ -310,11 +316,7 @@ func TestPreparedCancellationTimeoutKeepsCapacityUntilStartReturns(t *testing.T) }) startCancellationPreparation(t, r, sender) <-entered - for i := 0; i < 3; i++ { - id := fmt.Sprint(i) - _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, id, preparationRequest())) - waitPreparationStatus(t, sender, id, "ready", "") - } + _ = r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "cancel"})) deadline := time.Now().Add(12 * time.Second) for len(cancellationAcks(sender)) == 0 && time.Now().Before(deadline) { diff --git a/apps/parsar-daemon/internal/dispatch/preparation_cleanup.go b/apps/parsar-daemon/internal/dispatch/preparation_cleanup.go index 2c3cd176b..2a51c0e8c 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation_cleanup.go +++ b/apps/parsar-daemon/internal/dispatch/preparation_cleanup.go @@ -32,6 +32,9 @@ func (r *Router) closePendingPreparationsLocked() []*preparationState { var closeNow []*preparationState for _, p := range r.preparations { p.timer.Stop() + if p.executor != nil { + continue + } if !p.owns { continue } diff --git a/apps/parsar-daemon/internal/dispatch/preparation_cleanup_test.go b/apps/parsar-daemon/internal/dispatch/preparation_cleanup_test.go index 7c08293d6..a36355bd5 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation_cleanup_test.go +++ b/apps/parsar-daemon/internal/dispatch/preparation_cleanup_test.go @@ -55,7 +55,7 @@ func TestPreparedCancellationDoesNotAcknowledgeFailedCleanup(t *testing.T) { if owned, _ := r.PreparationOwnershipForTest(ready.Handle); !owned { t.Fatal("cancel receipt discarded unsettled preparation") } - _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionRelease, "request", proto.ExecutionReleasePayload{Handle: ready.Handle})) + _ = r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "cancel-retry"})) waitFor(t, func() bool { owned, _ := r.PreparationOwnershipForTest(ready.Handle); return !owned }, "retained cancellation cleanup") } @@ -69,7 +69,7 @@ func TestShutdownRetriesFailedPreparedCancellationOnSameTarget(t *testing.T) { return session, nil } p.cancel = func(ctx context.Context) error { - if p.calls.Load() == 1 { + if p.calls.Load() <= 2 { return want } return session.Cancel(ctx) @@ -80,7 +80,7 @@ func TestShutdownRetriesFailedPreparedCancellationOnSameTarget(t *testing.T) { if err := r.Shutdown(t.Context()); !errors.Is(err, want) { t.Fatalf("first shutdown lost native failure: %v", err) } - if r.ActiveRuns() != 1 || p.calls.Load() != 1 || session.cancels() != 0 { + if r.ActiveRuns() != 1 || p.calls.Load() != 2 || session.cancels() != 0 { t.Fatal("failed shutdown released ownership or changed release target") } select { @@ -91,78 +91,11 @@ func TestShutdownRetriesFailedPreparedCancellationOnSameTarget(t *testing.T) { if err := r.Shutdown(t.Context()); err != nil { t.Fatal(err) } - if r.ActiveRuns() != 0 || p.calls.Load() != 2 || session.cancels() != 1 { + if r.ActiveRuns() != 0 || p.calls.Load() != 3 || session.cancels() != 1 { t.Fatalf("shutdown did not retry the same target once: active=%d target=%d session=%d", r.ActiveRuns(), p.calls.Load(), session.cancels()) } } -func TestPreparationCloseFailureRetainsCapacityAndRetries(t *testing.T) { - sender := &recSender{} - entered, release := make(chan struct{}), make(chan struct{}) - var once sync.Once - unblock := func() { once.Do(func() { close(release) }) } - p := &retryablePreparation{close: func(call int32) error { - if call == 1 { - return errors.New("cleanup incomplete") - } - if call == 2 { - close(entered) - <-release - } - return nil - }} - var factories atomic.Int32 - r := preparationRouter(t, sender, time.Minute, func(context.Context, proto.PromptRequestPayload) (agent.Prepared, error) { - if factories.Add(1) == 1 { - return p, nil - } - return &controlledPreparation{closed: make(chan struct{})}, nil - }) - t.Cleanup(unblock) - var handle string - for i := range 4 { - id := fmt.Sprint(i) - _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, id, preparationRequest())) - ready := waitPreparationStatus(t, sender, id, "ready", "") - if i == 0 { - handle = ready.Handle - } - } - request := mustEnv(t, proto.TypeExecutionRelease, "0", proto.ExecutionReleasePayload{Handle: handle}) - _ = r.Handle(t.Context(), request) - waitFor(t, func() bool { _, busy := r.PreparationOwnershipForTest(handle); return p.calls.Load() == 1 && !busy }, "failed cleanup return") - if owned, _ := r.PreparationOwnershipForTest(handle); !owned { - t.Fatal("failed cleanup discarded ownership") - } - replacement := mustEnv(t, proto.TypeExecutionPrepare, "replacement", preparationRequest()) - if err := r.Handle(t.Context(), replacement); err == nil { - t.Fatal("failed cleanup released capacity") - } - start := proto.ExecutionStartPayload{Handle: handle, RunID: "run", Input: proto.TextInput("do not execute")} - if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "0", start)); err == nil { - t.Fatal("failed cleanup allowed Start") - } - _ = r.Handle(t.Context(), request) - select { - case <-entered: - case <-time.After(time.Second): - t.Fatal("release did not retry retained resource") - } - _ = r.Handle(t.Context(), request) - if p.calls.Load() != 2 { - t.Fatal("release started overlapping cleanup") - } - unblock() - waitFor(t, func() bool { owned, _ := r.PreparationOwnershipForTest(handle); return !owned }, "successful cleanup") - if err := r.Handle(t.Context(), replacement); err != nil { - t.Fatalf("settled cleanup retained capacity: %v", err) - } - waitPreparationStatus(t, sender, "replacement", "ready", "") - if p.starts.Load() != 0 { - t.Fatal("cleanup restarted native execution") - } -} - func TestShutdownRetriesFailedPreparationCleanup(t *testing.T) { want := errors.New("native cleanup incomplete") p := &retryablePreparation{close: func(call int32) error { @@ -174,18 +107,16 @@ func TestShutdownRetriesFailedPreparationCleanup(t *testing.T) { sender := &recSender{} r := preparationRouter(t, sender, time.Minute, func(context.Context, proto.PromptRequestPayload) (agent.Prepared, error) { return p, nil }) _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "request", preparationRequest())) - handle := waitPreparationStatus(t, sender, "request", "ready", "").Handle + waitPreparationStatus(t, sender, "request", "ready", "") if err := r.Shutdown(t.Context()); !errors.Is(err, want) { t.Fatalf("shutdown lost cleanup error: %v", err) } - if owned, _ := r.PreparationOwnershipForTest(handle); !owned { - t.Fatal("shutdown discarded unsettled resource") - } + if err := r.Shutdown(t.Context()); err != nil { t.Fatal(err) } - if owned, _ := r.PreparationOwnershipForTest(handle); owned || p.calls.Load() != 2 { - t.Fatalf("retry did not settle original resource: owned=%t calls=%d", owned, p.calls.Load()) + if p.calls.Load() != 2 { + t.Fatalf("retry did not settle original resource: calls=%d", p.calls.Load()) } } @@ -239,7 +170,7 @@ func TestPublishedPreparedRunRetainsRetryAfterHandleRetirement(t *testing.T) { return session, nil } p.cancel = func(ctx context.Context) error { - if p.calls.Load() == 1 { + if p.calls.Load() <= 2 { return errors.New("cleanup incomplete") } return session.Cancel(ctx) @@ -260,7 +191,7 @@ func TestPublishedPreparedRunRetainsRetryAfterHandleRetirement(t *testing.T) { } // A later admission retires the expired preparation record. The Run still // owns the exact cancellation target independently of that old handle. - if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "replacement", preparationRequest())); err != nil { + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "replacement", readOnlyPreparationRequest())); err != nil { t.Fatal(err) } waitPreparationStatus(t, sender, "replacement", "ready", "") @@ -278,7 +209,7 @@ func TestPublishedPreparedRunRetainsRetryAfterHandleRetirement(t *testing.T) { } } } - if p.calls.Load() != 2 || session.cancels() != 1 || r.ActiveRuns() != 0 { + if p.calls.Load() != 3 || session.cancels() != 1 || r.ActiveRuns() != 0 { t.Fatal("Run cancellation did not retry and settle the same target") } select { @@ -287,3 +218,9 @@ func TestPublishedPreparedRunRetainsRetryAfterHandleRetirement(t *testing.T) { default: } } + +func readOnlyPreparationRequest() proto.ExecutionPreparePayload { + req := preparationRequest() + req.Configuration.WorkspaceReadOnly = true + return req +} diff --git a/apps/parsar-daemon/internal/dispatch/preparation_executor_fixture_test.go b/apps/parsar-daemon/internal/dispatch/preparation_executor_fixture_test.go new file mode 100644 index 000000000..c82ac841e --- /dev/null +++ b/apps/parsar-daemon/internal/dispatch/preparation_executor_fixture_test.go @@ -0,0 +1,133 @@ +package dispatch_test + +import ( + "context" + "errors" + "sync" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +// Old fault-injection fixtures model disposable executors, not reusable native implementations. +func preparationExecutorFixture(factory agent.PreparationFactory) agent.ExecutorFactory { + return func(ctx context.Context, req proto.PromptRequestPayload) (agent.Executor, error) { + prepared, err := factory(ctx, req) + if prepared == nil { + return nil, err + } + owner := &preparationExecutor{prepared: prepared} + if _, ok := prepared.(agent.PreparedCancellation); !ok { + return owner, errors.New("fixture has no cancellation target") + } + return owner, err + } +} + +type preparationExecutor struct { + prepared agent.Prepared + mu sync.Mutex + cancelled bool +} + +func (e *preparationExecutor) Close(ctx context.Context) error { + e.mu.Lock() + defer e.mu.Unlock() + if e.cancelled { + return nil + } + var err error + if resource, ok := e.prepared.(*retryablePreparation); ok { + err = resource.Close() + } else if cancel, ok := e.prepared.(agent.PreparedCancellation); ok { + err = cancel.Cancel(ctx) + } else { + err = e.prepared.Close() + } + e.cancelled = err == nil + return err +} + +func (e *preparationExecutor) StartTurn(ctx context.Context, id string, input proto.MessageInput, out chan<- proto.Envelope) (agent.Turn, error) { + upstream := make(chan proto.Envelope, 64) + terminal := make(chan struct{}) + var once sync.Once + go func() { + defer close(out) + defer once.Do(func() { close(terminal) }) + for event := range upstream { + if event.Type == proto.TypeDone { + once.Do(func() { close(terminal) }) + } + out <- event + } + }() + session, err := e.prepared.Start(ctx, id, input, upstream) + if session == nil { + close(upstream) + // This fixture's old Start may have produced output before rejecting; retain its Turn. + return &preparationTurn{owner: e, settled: terminal}, err + } + return &preparationTurn{Session: session, owner: e, settled: terminal}, err +} + +type preparationTurn struct { + agent.Session + owner *preparationExecutor + settled <-chan struct{} +} + +func (t *preparationTurn) Cancel(ctx context.Context) error { return t.owner.Close(ctx) } +func (t *preparationTurn) CancellationOutcome() proto.DonePayload { + if target, ok := t.owner.prepared.(agent.PreparedCancellation); ok { + return target.CancellationOutcome() + } + return proto.DonePayload{} +} +func (t *preparationTurn) AwaitSettlement(ctx context.Context) (agent.TurnSettlement, error) { + select { + case <-t.settled: + return agent.TurnSettlement{Reason: "disposable fault fixture"}, t.owner.Close(ctx) + case <-ctx.Done(): + return agent.TurnSettlement{}, ctx.Err() + } +} +func (t *preparationTurn) SubmitFunctionResult(ctx context.Context, p proto.FunctionResultPayload) error { + if target, ok := t.Session.(agent.FunctionResultSubmitter); ok { + return target.SubmitFunctionResult(ctx, p) + } + return agent.ErrUnknownFunctionCall +} +func (t *preparationTurn) Steer(ctx context.Context, p proto.PromptSteerPayload) error { + if target, ok := t.Session.(agent.Steerer); ok { + return target.Steer(ctx, p) + } + return agent.ErrSteeringInactive +} +func (t *preparationTurn) SteerWithReceipt(ctx context.Context, p proto.PromptSteerPayload, write func()) error { + if target, ok := t.Session.(agent.DurableSteerer); ok { + return target.SteerWithReceipt(ctx, p, write) + } + if target, ok := t.Session.(agent.Steerer); ok { + return target.Steer(ctx, p) + } + return agent.ErrSteeringInactive +} +func (t *preparationTurn) SubmitPermission(ctx context.Context, id string, p proto.PermissionDecisionPayload) error { + if target, ok := t.Session.(agent.PermissionResponder); ok { + return target.SubmitPermission(ctx, id, p) + } + return agent.ErrUnknownPermission +} +func (t *preparationTurn) SubmitPromptForUserChoice(ctx context.Context, id string, p proto.PromptForUserChoiceDecisionPayload) error { + if target, ok := t.Session.(agent.UserChoiceResponder); ok { + return target.SubmitPromptForUserChoice(ctx, id, p) + } + return agent.ErrUnknownAsk +} +func (t *preparationTurn) ReadWorkspaceFile(ctx context.Context, path string, limit int) (agent.WorkspaceReadResult, error) { + if target, ok := t.Session.(agent.WorkspaceReader); ok { + return target.ReadWorkspaceFile(ctx, path, limit) + } + return agent.WorkspaceReadResult{}, agent.ErrWorkspaceReadUnsupported +} diff --git a/apps/parsar-daemon/internal/dispatch/preparation_start.go b/apps/parsar-daemon/internal/dispatch/preparation_start.go index 0607d407f..bd1e64894 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation_start.go +++ b/apps/parsar-daemon/internal/dispatch/preparation_start.go @@ -14,17 +14,17 @@ import ( func (r *Router) handleExecutionStart(_ context.Context, env proto.Envelope) error { var input proto.ExecutionStartPayload - if env.DecodePayload(&input) != nil || input.Handle == "" || strings.TrimSpace(input.RunID) == "" || input.Input.Validate() != nil { + if env.DecodePayload(&input) != nil || input.Handle == "" || input.ExecutorID == "" || strings.TrimSpace(input.RunID) == "" || input.Input.Validate() != nil { return r.rejectPreparation(env, "invalid_start") } encoded, _ := json.Marshal(input) fingerprint := sha256.Sum256(encoded) r.mu.Lock() - if r.closed { + if r.closed || r.suspension != nil { r.mu.Unlock() return ErrRouterClosed } - if r.workspaceExport != nil { + if r.workspaceWrite != nil || r.workspaceExport != nil { r.mu.Unlock() return r.rejectPreparation(env, "resource_unavailable") } @@ -37,23 +37,25 @@ func (r *Router) handleExecutionStart(_ context.Context, env proto.Envelope) err r.mu.Unlock() return r.rejectPreparation(env, "read_only_preparation") } + owner := p.executor + if owner == nil || owner.id != input.ExecutorID { + r.mu.Unlock() + return r.rejectPreparation(env, "unknown_executor") + } if p.status.State == "starting" || p.status.State == "started" { matches, status := p.startFingerprint == fingerprint, p.status - if matches { - if state := r.sessions[input.RunID]; state != nil && state.preparedHandoff != nil && - (state.session == nil || state.preparedHandoff.release != nil) { - r.mu.Unlock() - return nil - } - } + pending := p.handoff != nil && !p.handoff.published r.mu.Unlock() if !matches { return r.rejectPreparation(env, "start_conflict") } + if pending { + return nil + } r.publishPreparation(p, status) return nil } - if p.status.State != "ready" || p.ctx.Err() != nil { + if p.status.State != "ready" || p.ctx.Err() != nil || owner.admission != p || owner.run != nil || owner.invalid || owner.native == nil { r.mu.Unlock() return r.rejectPreparation(env, "preparation_not_ready") } @@ -66,24 +68,13 @@ func (r *Router) handleExecutionStart(_ context.Context, env proto.Envelope) err r.mu.Unlock() return r.rejectPreparation(env, "run_conflict") } - target, ok := p.prepared.(agent.PreparedCancellation) - if !ok { - r.mu.Unlock() - return r.rejectPreparation(env, "preparation_not_ready") - } p.status.State, p.status.RunID, p.status.Revision = "starting", input.RunID, p.status.Revision+1 p.startFingerprint, p.busy = fingerprint, true - state := &sessionState{ - runID: input.RunID, stateKey: p.stateKey, environmentID: p.environmentID, - out: make(chan proto.Envelope, 64), ctx: p.ctx, ctxCancel: p.cancel, - pendingIDs: make(map[string]struct{}), pendingAsks: make(map[string]struct{}), - traceparent: env.Trace, releaseOnCompletion: true, - } - state.preparedHandoff = newPreparedHandoff(p, target) - p.handoff = state.preparedHandoff + state := &sessionState{runID: input.RunID, stateKey: p.stateKey, environmentID: p.environmentID, out: make(chan proto.Envelope, 64), ctx: p.ctx, ctxCancel: p.cancel, pendingIDs: make(map[string]struct{}), pendingAsks: make(map[string]struct{}), traceparent: env.Trace} + state.preparedHandoff = newPreparedHandoff(p, owner.native) + p.handoff, owner.run = state.preparedHandoff, state r.sessions[input.RunID] = state status := p.status - // Track the Start owner and output consumer before either can publish. r.shutdownWG.Add(2) r.mu.Unlock() go r.startPreparedExecution(p, state, input, status) @@ -95,100 +86,74 @@ func (r *Router) startPreparedExecution(p *preparationState, state *sessionState handoff := state.preparedHandoff go r.forwardPreparedOutput(state) <-handoff.outputReady - - if !r.sendPreparation(p.requestID, p.trace, starting) { - r.mu.Lock() - handoff.outputErr = errors.Join(handoff.outputErr, errPreparedStatusDelivery) - if p.status.State == "starting" { - p.status.State, p.status.ErrorCode, p.status.Revision = "failed", "status_delivery_failed", p.status.Revision+1 - } - p.timer.Stop() - r.claimPreparedReleaseLocked(state, true, "", false) - r.mu.Unlock() - close(state.out) - close(handoff.startDone) - return - } - - // A release admitted before this point must not cause native work to start. + delivered := r.sendPreparation(p.requestID, p.trace, starting) r.mu.Lock() - blocked := handoff.release != nil && handoff.release.aborted() + blocked := !delivered || handoff.release != nil && handoff.release.aborted() || p.ctx.Err() != nil r.mu.Unlock() - var session agent.Session + var turn agent.Turn var startErr error if blocked { startErr = context.Canceled } else { - session, startErr = handoff.target.Start(p.ctx, input.RunID, input.Input, state.out) + turn, startErr = handoff.target.StartTurn(p.ctx, input.RunID, input.Input, state.out) } - r.mu.Lock() - release := handoff.release - notAborted := release == nil || !release.aborted() - started := startErr == nil && session != nil && p.status.State == "starting" && p.ctx.Err() == nil && notAborted - if started { + handoff.turn = turn + handoff.startErr = startErr + p.timer.Stop() + p.busy = false + if turn != nil { p.status.State, p.status.ErrorCode, p.status.Revision = "started", "", p.status.Revision+1 } else { - p.timer.Stop() - if p.status.State == "starting" { - p.status.State, p.status.ErrorCode, p.status.Revision = "failed", "start_failed", p.status.Revision+1 - } - r.claimPreparedReleaseLocked(state, true, "prepared execution could not start", false) + p.status.State, p.status.ErrorCode, p.status.Revision = "failed", "executor_unavailable", p.status.Revision+1 } status := p.status - // Keep release behind successful or failed status publication. This also - // orders an early native Done after the started status. - if started { - handoff.operations.RLock() - } r.mu.Unlock() - - // A nil Session leaves output ownership with the Router. A non-nil Session - // owns the close even when Start also returned an error. - if session == nil { + if turn == nil { close(state.out) - } - deadline := time.Now().Add(5 * time.Second) - if started && p.deadline.Before(deadline) { - deadline = p.deadline - } - delivered := r.sendPreparationUntil(p.requestID, p.trace, status, deadline) - - r.mu.Lock() - if started && !time.Now().Before(p.deadline) { - if p.status.State == "started" { - p.status.State, p.status.ErrorCode, p.status.Revision = "expired", "", p.status.Revision+1 + code := "executor_unavailable" + closeErr := r.closeExecutor(p.executor) + if closeErr != nil { + code = "executor_cleanup_unconfirmed" + } + if blocked { + code = "start_cancelled" + } + r.mu.Lock() + p.status.ErrorCode = code + release, _ := r.claimPreparedReleaseLocked(state, true, "", false) + close(handoff.startDone) + r.mu.Unlock() + if closeErr == nil { + <-release.settled + } + status = proto.PreparationStatusPayload{Handle: p.status.Handle, ExecutorID: p.executor.id, State: "rejected", ErrorCode: code, Operation: proto.TypeExecutionStart} + if delivered { + r.sendPreparation(p.requestID, p.trace, status) } - r.claimPreparedReleaseLocked(state, true, "", false) + return } - if started && handoff.release != nil && handoff.release.aborted() && p.status.State == "started" { - p.status.State, p.status.ErrorCode, p.status.Revision = "failed", "start_failed", p.status.Revision+1 + r.mu.Lock() + aborted := handoff.release != nil && handoff.release.aborted() + r.mu.Unlock() + if aborted { + status.State, status.ErrorCode = "failed", "start_cancelled" } - if !delivered { + delivered = delivered && !aborted && r.sendPreparationUntil(p.requestID, p.trace, status, p.deadline) + r.mu.Lock() + if !delivered && !aborted { handoff.outputErr = errors.Join(handoff.outputErr, errPreparedStatusDelivery) - if started && p.status.State == "started" { - p.status.State, p.status.ErrorCode, p.status.Revision = "failed", "status_delivery_failed", p.status.Revision+1 - } - r.claimPreparedReleaseLocked(state, true, "", false) - } else if started && p.status.State == "started" && - (handoff.release == nil || !handoff.release.aborted()) { - handoff.published = true - p.timer.Stop() } + handoff.published = turn != nil && delivered if handoff.published { - // Publication transfers resource tracking even when natural completion - // has already closed input admission. - if handoff.release == nil { - state.session = session - } - p.prepared = nil p.owns = false - p.busy = false - p.handoff = nil + } + if handoff.published && handoff.release == nil { + state.session = turn + } + if turn == nil || startErr != nil || !delivered { + r.claimPreparedReleaseLocked(state, true, "prepared execution could not start", false) } close(handoff.startDone) r.mu.Unlock() - if started { - handoff.operations.RUnlock() - } } diff --git a/apps/parsar-daemon/internal/dispatch/preparation_test.go b/apps/parsar-daemon/internal/dispatch/preparation_test.go index 4dee25e40..16134d129 100644 --- a/apps/parsar-daemon/internal/dispatch/preparation_test.go +++ b/apps/parsar-daemon/internal/dispatch/preparation_test.go @@ -3,7 +3,6 @@ package dispatch_test import ( "context" "errors" - "fmt" "os" "path/filepath" "sync" @@ -32,7 +31,9 @@ func (p *controlledPreparation) Close() error { if p.closeHook != nil { p.closeHook() } - close(p.closed) + if p.closed != nil { + close(p.closed) + } }) return nil } @@ -111,7 +112,7 @@ func localPreparationHarness(t *testing.T) *harness { } func preparationRequest() proto.ExecutionPreparePayload { - return proto.ExecutionPreparePayload{Configuration: proto.PromptRequestPayload{AgentKind: "prepared", AgentStateKey: "agents-api-" + preparationSessionID, StrictResume: true, ReleaseOnCompletion: true, LocalEnvironment: &proto.LocalEnvironment{ID: preparationEnvironmentID, NetworkAccess: "enabled"}}} + return proto.ExecutionPreparePayload{SessionID: preparationSessionID, Configuration: proto.PromptRequestPayload{AgentKind: "prepared", AgentStateKey: "agents-api-" + preparationSessionID, StrictResume: true, ReleaseOnCompletion: true, LocalEnvironment: &proto.LocalEnvironment{ID: preparationEnvironmentID, NetworkAccess: "enabled"}}} } func preparationRouter(t *testing.T, sender dispatch.Sender, timeout time.Duration, factory agent.PreparationFactory) *dispatch.Router { @@ -121,6 +122,7 @@ func preparationRouter(t *testing.T, sender dispatch.Sender, timeout time.Durati return nil, errors.New("ordinary Factory must not be used for preparation") }) reg.RegisterPreparation("prepared", true, factory) + reg.RegisterExecutor("prepared", preparationExecutorFixture(factory)) r, err := dispatch.New(dispatch.Config{Registry: reg, Sender: sender, PreparationTimeout: timeout, LocalWorkspace: preparationWorkspace(t)}) if err != nil { t.Fatal(err) @@ -225,7 +227,7 @@ func TestPreparationSingleTransferAndReleaseDoesNotCancelRun(t *testing.T) { if err := r.Handle(t.Context(), prepare); err != nil { t.Fatal(err) } - start := mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "real-run", Input: proto.TextInput("actual input")}) + start := mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, ExecutorID: ready.ExecutorID, RunID: "real-run", Input: proto.TextInput("actual input")}) if err := r.Handle(t.Context(), start); err != nil { t.Fatal(err) } @@ -276,7 +278,7 @@ func TestPreparationCancelDuringStartClosesLateSession(t *testing.T) { r := preparationRouter(t, sender, time.Minute, func(context.Context, proto.PromptRequestPayload) (agent.Prepared, error) { return p, nil }) _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "request", preparationRequest())) ready := waitPreparationStatus(t, sender, "request", "ready", "") - _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "real-run", Input: proto.TextInput("input")})) + _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, ExecutorID: ready.ExecutorID, RunID: "real-run", Input: proto.TextInput("input")})) <-entered if err := r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "real-run", proto.PromptCancelPayload{DeliveryID: "cancel"})); err != nil { t.Fatal(err) @@ -307,37 +309,6 @@ func TestPreparationCancelDuringStartClosesLateSession(t *testing.T) { } } -func TestPreparationCapacityAndConnectionOwnership(t *testing.T) { - sender := &recSender{} - var count atomic.Int32 - r := preparationRouter(t, sender, time.Minute, func(context.Context, proto.PromptRequestPayload) (agent.Prepared, error) { - count.Add(1) - return &controlledPreparation{closed: make(chan struct{})}, nil - }) - var handles []string - for i := 0; i < 4; i++ { - id := fmt.Sprint(i) - _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, id, preparationRequest())) - handles = append(handles, waitPreparationStatus(t, sender, id, "ready", "").Handle) - } - if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "overflow", preparationRequest())); err == nil { - t.Fatal("capacity unbounded") - } - if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "0", preparationRequest())); err != nil { - t.Fatal(err) - } - if count.Load() != 4 { - t.Fatal("retry recreated native resource") - } - other := preparationRouter(t, &recSender{}, time.Minute, func(context.Context, proto.PromptRequestPayload) (agent.Prepared, error) { - t.Error("unexpected new native resource") - return nil, errors.New("unexpected") - }) - if err := other.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "0", proto.ExecutionStartPayload{Handle: handles[0], RunID: "run", Input: proto.TextInput("input")})); err == nil { - t.Fatal("another connection consumed handle") - } -} - func TestPreparationExpiryAndOldHandleCannotStartReplacement(t *testing.T) { sender := &recSender{} created := make(chan *controlledPreparation, 2) @@ -350,13 +321,18 @@ func TestPreparationExpiryAndOldHandleCannotStartReplacement(t *testing.T) { _ = r.Handle(t.Context(), env) old := waitPreparationStatus(t, sender, "request", "ready", "") waitPreparationStatus(t, sender, "request", "expired", "") - waitPreparationClosed(t, <-created) + owner := <-created + select { + case <-owner.closed: + t.Fatal("admission expiry closed a healthy executor") + default: + } _ = r.Handle(t.Context(), env) - next := waitPreparationStatus(t, sender, "request", "preparing", old.Handle) + next := waitPreparationStatus(t, sender, "request", "ready", old.Handle) if next.Handle == old.Handle || next.ExpiresAt <= old.ExpiresAt { t.Fatal("replacement reused expired identity") } - if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: old.Handle, RunID: "late", Input: proto.TextInput("late")})); err == nil { + if err := r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: old.Handle, ExecutorID: old.ExecutorID, RunID: "late", Input: proto.TextInput("late")})); err == nil { t.Fatal("old handle started replacement") } } @@ -371,61 +347,25 @@ func (s failReadySender) Send(ctx context.Context, env proto.Envelope) error { return s.recSender.Send(ctx, env) } -func TestPreparationFailedReadyDeliveryClosesResource(t *testing.T) { +func TestPreparationFailedReadyDeliveryAbandonsAdmission(t *testing.T) { + created := make(chan struct{}) p := &controlledPreparation{closed: make(chan struct{})} - r := preparationRouter(t, failReadySender{&recSender{}}, time.Minute, func(context.Context, proto.PromptRequestPayload) (agent.Prepared, error) { return p, nil }) + r := preparationRouter(t, failReadySender{&recSender{}}, time.Minute, func(context.Context, proto.PromptRequestPayload) (agent.Prepared, error) { + close(created) + return p, nil + }) _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, "request", preparationRequest())) + <-created + waitFor(t, func() bool { return r.ActiveRuns() == 0 }, "abandoned admission") + if err := r.Shutdown(t.Context()); err != nil { + t.Fatal(err) + } waitPreparationClosed(t, p) if r.ActiveRuns() != 0 { t.Fatal("failed preparation became a run") } } -func TestPreparationCapacityIncludesClosingResources(t *testing.T) { - sender := &recSender{} - entered, unblock := make(chan struct{}), make(chan struct{}) - var releaseOnce sync.Once - release := func() { releaseOnce.Do(func() { close(unblock) }) } - defer release() - var count atomic.Int32 - r := preparationRouter(t, sender, time.Minute, func(context.Context, proto.PromptRequestPayload) (agent.Prepared, error) { - p := &controlledPreparation{closed: make(chan struct{})} - if count.Add(1) == 1 { - p.closeHook = func() { close(entered); <-unblock } - } - return p, nil - }) - var first proto.PreparationStatusPayload - for i := 0; i < 4; i++ { - id := fmt.Sprint(i) - _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionPrepare, id, preparationRequest())) - ready := waitPreparationStatus(t, sender, id, "ready", "") - if i == 0 { - first = ready - } - } - _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionRelease, "0", proto.ExecutionReleasePayload{Handle: first.Handle})) - select { - case <-entered: - case <-time.After(time.Second): - t.Fatal("cleanup did not start") - } - request := mustEnv(t, proto.TypeExecutionPrepare, "replacement", preparationRequest()) - if err := r.Handle(t.Context(), request); err == nil { - t.Fatal("closing resource returned capacity early") - } - release() - deadline := time.Now().Add(time.Second) - for time.Now().Before(deadline) { - if err := r.Handle(t.Context(), request); err == nil { - waitPreparationStatus(t, sender, "replacement", "ready", "") - return - } - time.Sleep(time.Millisecond) - } - t.Fatal("completed cleanup did not release capacity") -} - func TestPreparationRejectsInputAndProductConfiguration(t *testing.T) { for name, change := range map[string]func(*proto.PromptRequestPayload){ "run": func(p *proto.PromptRequestPayload) { p.RunID = "run" }, @@ -437,7 +377,6 @@ func TestPreparationRejectsInputAndProductConfiguration(t *testing.T) { }, "missing environment": func(p *proto.PromptRequestPayload) { p.LocalEnvironment = nil }, "resume": func(p *proto.PromptRequestPayload) { p.StrictResume = false }, - "release": func(p *proto.PromptRequestPayload) { p.ReleaseOnCompletion = false }, } { t.Run(name, func(t *testing.T) { r := preparationRouter(t, &recSender{}, time.Minute, func(context.Context, proto.PromptRequestPayload) (agent.Prepared, error) { diff --git a/apps/parsar-daemon/internal/dispatch/prepared_handoff.go b/apps/parsar-daemon/internal/dispatch/prepared_handoff.go index b5416df44..3df8b2934 100644 --- a/apps/parsar-daemon/internal/dispatch/prepared_handoff.go +++ b/apps/parsar-daemon/internal/dispatch/prepared_handoff.go @@ -40,12 +40,13 @@ func (release *preparedRelease) aborted() bool { } } -// preparedHandoff owns the exact PreparedCancellation from Start admission -// through native cleanup and output settlement. It never falls back to the -// returned Session or Prepared.Close. +// preparedHandoff owns one Turn admission and settlement on a retained Executor. type preparedHandoff struct { preparation *preparationState - target agent.PreparedCancellation + target agent.Executor + turn agent.Turn + startErr error + protocolErr error startDone chan struct{} outputReady chan struct{} @@ -62,7 +63,7 @@ type preparedHandoff struct { shutdownInterrupted bool } -func newPreparedHandoff(p *preparationState, target agent.PreparedCancellation) *preparedHandoff { +func newPreparedHandoff(p *preparationState, target agent.Executor) *preparedHandoff { return &preparedHandoff{ preparation: p, target: target, @@ -116,6 +117,9 @@ func (r *Router) claimPreparedReleaseLocked(state *sessionState, abort bool, fai } if abort && !release.aborted() { close(release.abort) + if handoff.turn == nil { + handoff.preparation.cancel() + } } if release.succeeded { return release, release.attempt @@ -142,66 +146,134 @@ func (r *Router) claimPreparedReleaseLocked(state *sessionState, abort bool, fai func (r *Router) runPreparedRelease(state *sessionState, handoff *preparedHandoff, release *preparedRelease, attempt *preparedReleaseAttempt) { defer r.shutdownWG.Done() - - // Natural completion must publish started before it can release the native - // resource. Abort is allowed to fence a Start that is still in progress. select { case <-handoff.startDone: case <-release.abort: + select { + case <-handoff.startDone: + default: + if err := r.closeExecutor(handoff.preparation.executor); err != nil { + r.mu.Lock() + attempt.err = err + handoff.preparation.busy = false + handoff.preparation.closeErr = err + close(attempt.done) + r.mu.Unlock() + return + } + } + <-handoff.startDone } + // Admission is closed; all earlier mutations retain their native receipt wait. handoff.operations.Lock() - ctx, cancel := context.WithTimeout(context.Background(), preparedCancelTimeout) - nativeErr := handoff.target.Cancel(ctx) - cancel() + handoff.operations.Unlock() + turn := handoff.turn + var settlement agent.TurnSettlement + var nativeErr error var outcome *proto.DonePayload + if turn == nil { + outcome = &proto.DonePayload{} + } + if turn != nil { + ctx, cancel := context.WithTimeout(context.Background(), preparedCancelTimeout) + cancelDone := make(chan error, 1) + settled := make(chan struct{}) + go func() { + select { + case <-release.abort: + err := turn.Cancel(ctx) + if err != nil { + cancel() + } + cancelDone <- err + case <-settled: + cancelDone <- nil + } + }() + settlement, nativeErr = turn.AwaitSettlement(ctx) + close(settled) + nativeErr = errors.Join(nativeErr, <-cancelDone) + cancel() + if nativeErr == nil { + value := turn.CancellationOutcome() + outcome = &value + } + } if nativeErr == nil { - observed := handoff.target.CancellationOutcome() - outcome = &observed + <-handoff.outputDone + } + r.mu.Lock() + owner := handoff.preparation.executor + if turn != nil && handoff.terminal == nil && !release.aborted() { + handoff.protocolErr = errors.New("executor output ended without a terminal result") + } + if handoff.protocolErr != nil { + release.failure = handoff.protocolErr.Error() + } + invalidate := nativeErr != nil || !settlement.Reusable || handoff.startErr != nil || handoff.outputErr != nil || handoff.protocolErr != nil || r.closed || owner.invalid + if invalidate { + owner.invalid = true + } + r.mu.Unlock() + var closeErr error + if invalidate { + closeErr = r.closeExecutor(owner) + // A failed Turn settlement is not repaired into a cancellation receipt by Close. + nativeErr = errors.Join(nativeErr, closeErr) } - handoff.operations.Unlock() - r.mu.Lock() attempt.err = nativeErr - if nativeErr != nil { + if closeErr != nil { handoff.preparation.busy = false handoff.preparation.closeErr = nativeErr close(attempt.done) r.mu.Unlock() return } - release.succeeded = true - release.outcome = outcome + if nativeErr != nil { + release.failure = "executor Turn settlement failed" + } + release.succeeded, release.outcome = true, outcome close(attempt.done) r.mu.Unlock() - - // Cancel success promises local cleanup and no further output writes. Wait - // for Start and the one output consumer before publishing terminal state. - <-handoff.startDone <-handoff.outputDone - r.mu.Lock() - outputErr := handoff.outputErr - terminal := handoff.terminal - closed := r.closed + outputErr, terminal, closed := handoff.outputErr, handoff.terminal, r.closed r.mu.Unlock() - var terminalErr error - if outputErr == nil && !closed { - terminalErr = r.forwardPreparedTerminal(state, release.failure, terminal, release.failure != "" && terminal == nil) + if outputErr == nil && !closed && turn != nil { + terminalErr = r.forwardPreparedTerminal(state, release.failure, terminal, terminal == nil) } r.cleanupSession(state) - r.mu.Lock() handoff.outputErr = errors.Join(handoff.outputErr, terminalErr) p := handoff.preparation - p.busy = false + p.busy, p.owns, p.handoff = false, false, nil p.closeErr = terminalErr - p.prepared = nil - p.owns = false - p.handoff = nil p.cancel() + owner.run = nil + if owner.invalid && owner.closeDone != nil && owner.closeErr == nil && r.executors[owner.sessionID] == owner { + delete(r.executors, owner.sessionID) + } + if owner.admission == p { + owner.admission = nil + } + if outcome != nil { + if id, ok := outcome.Metadata[proto.DoneMetaAgentSessionID].(string); ok && id != "" { + owner.nativeID = id + } + } + if terminalErr != nil { + owner.invalid = true + } + if !owner.invalid { + r.scheduleExecutorIdleLocked(owner) + } close(release.settled) r.mu.Unlock() + if terminalErr != nil { + _ = r.closeExecutor(owner) + } } func (r *Router) forwardPreparedOutput(state *sessionState) { @@ -230,7 +302,9 @@ func (r *Router) forwardPreparedOutput(state *sessionState) { return } r.mu.Lock() - if handoff.terminal != nil { + if env.ID != state.runID || handoff.terminal != nil { + handoff.protocolErr = errors.New("executor output crossed the Turn boundary") + r.claimPreparedReleaseLocked(state, true, handoff.protocolErr.Error(), false) r.mu.Unlock() continue } diff --git a/apps/parsar-daemon/internal/dispatch/prepared_handoff_test.go b/apps/parsar-daemon/internal/dispatch/prepared_handoff_test.go index d12c8246e..db420f8dc 100644 --- a/apps/parsar-daemon/internal/dispatch/prepared_handoff_test.go +++ b/apps/parsar-daemon/internal/dispatch/prepared_handoff_test.go @@ -208,7 +208,7 @@ func TestPreparedHandoffDuplicateStartDoesNotReexecuteDuringPublication(t *testi if session.functions.Load() != 0 || session.steers.Load() != 0 || session.reads.Load() != 0 || len(session.submissions()) != 0 || askCalls != 0 { t.Fatal("private Session accepted work before started publication") } - duplicate := mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "run", Input: proto.TextInput("input")}) + duplicate := mustEnv(t, proto.TypeExecutionStart, "request", proto.ExecutionStartPayload{Handle: ready.Handle, ExecutorID: ready.ExecutorID, RunID: "run", Input: proto.TextInput("input")}) if err := r.Handle(t.Context(), duplicate); err != nil { t.Fatal(err) } @@ -315,7 +315,7 @@ func TestPreparedHandoffEarlyDoneDetachesPublishedPreparation(t *testing.T) { return session, nil } p.cancel = func(ctx context.Context) error { - if p.calls.Load() == 1 { + if p.calls.Load() <= 2 { return errors.New("cleanup incomplete") } return session.Cancel(ctx) @@ -327,7 +327,7 @@ func TestPreparedHandoffEarlyDoneDetachesPublishedPreparation(t *testing.T) { waitPreparationStatus(t, sender, "request", "started", "") waitFor(t, func() bool { _, busy := r.PreparationOwnershipForTest(ready.Handle) - return p.calls.Load() == 1 && !busy + return p.calls.Load() == 2 && !busy }, "failed natural cleanup") if owned, _ := r.PreparationOwnershipForTest(ready.Handle); owned { t.Fatal("published Run retained preparation capacity after early Done") @@ -345,14 +345,14 @@ func TestPreparedHandoffEarlyDoneDetachesPublishedPreparation(t *testing.T) { } return count == 2 }, "published preparation release response") - if p.calls.Load() != 1 { + if p.calls.Load() != 2 { t.Fatal("old preparation handle retried native cancellation") } if err := r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: "retry"})); err != nil { t.Fatal(err) } waitFor(t, func() bool { return len(cancellationAcks(sender)) == 1 && r.ActiveRuns() == 0 }, "Run-owned retry settlement") - if p.calls.Load() != 2 || session.cancels() != 1 || !cancellationAcks(sender)[0].Applied { + if p.calls.Load() != 3 || session.cancels() != 1 || !cancellationAcks(sender)[0].Applied { t.Fatal("explicit Run cancellation did not retry the retained native target") } } diff --git a/apps/parsar-daemon/internal/dispatch/router.go b/apps/parsar-daemon/internal/dispatch/router.go index 61c97f360..8bcf14c12 100644 --- a/apps/parsar-daemon/internal/dispatch/router.go +++ b/apps/parsar-daemon/internal/dispatch/router.go @@ -47,6 +47,7 @@ type Router struct { shutdownWG dispatchWork // waits for all pump goroutines idleTimeout time.Duration closed bool + executors map[string]*executorState preparations map[string]*preparationState preparationRequests map[string]*preparationState preparationTimeout time.Duration @@ -135,6 +136,7 @@ func New(cfg Config) (*Router, error) { applied: make(map[string]appliedInteractionDecision), shutdownCh: make(chan struct{}), idleTimeout: idleTimeout, + executors: make(map[string]*executorState), preparations: make(map[string]*preparationState), preparationRequests: make(map[string]*preparationState), preparationTimeout: preparationTimeout, diff --git a/apps/parsar-daemon/internal/dispatch/shutdown.go b/apps/parsar-daemon/internal/dispatch/shutdown.go index a59808ead..f5ea7693c 100644 --- a/apps/parsar-daemon/internal/dispatch/shutdown.go +++ b/apps/parsar-daemon/internal/dispatch/shutdown.go @@ -57,12 +57,14 @@ func (r *Router) Shutdown(ctx context.Context) error { close(r.shutdownCh) } preparations := r.closePendingPreparationsLocked() + executors := r.closeIdleExecutorsLocked() attempt := &shutdownAttempt{done: make(chan struct{})} r.shutdownAttempt = attempt // Keep the WaitGroup non-zero until all cancellation dispatch is complete. r.shutdownWG.Add(1) r.mu.Unlock() + r.closeIdleExecutors(executors) for _, p := range preparations { go func() { defer r.shutdownWG.Done(); r.closePreparationResource(p) }() } @@ -110,6 +112,9 @@ func (r *Router) runShutdownAttempt(attempt *shutdownAttempt, victims []sessionC attempt.err = errors.Join(attempt.err, fmt.Errorf("dispatch: preparation %s: %w", p.status.Handle, cause)) } } + for _, owner := range r.executors { + attempt.err = errors.Join(attempt.err, fmt.Errorf("dispatch: executor %s cleanup unconfirmed: %w", owner.id, owner.closeErr)) + } close(attempt.done) r.mu.Unlock() } @@ -155,7 +160,9 @@ func (r *Router) handleDeviceShutdown(ctx context.Context, env proto.Envelope) e } r.idle = make(map[string]map[*sessionState]struct{}) preparations := r.closePendingPreparationsLocked() + executors := r.closeIdleExecutorsLocked() r.mu.Unlock() + r.closeIdleExecutors(executors) for _, p := range preparations { go func() { defer r.shutdownWG.Done(); r.closePreparationResource(p) }() } diff --git a/apps/parsar-daemon/internal/dispatch/steering.go b/apps/parsar-daemon/internal/dispatch/steering.go index 00265cb91..c6636fa1b 100644 --- a/apps/parsar-daemon/internal/dispatch/steering.go +++ b/apps/parsar-daemon/internal/dispatch/steering.go @@ -98,8 +98,8 @@ func (r *Router) queueSteering(ctx context.Context, env proto.Envelope, input pr return &ack } if input.DurableReceipt { - if _, ok := session.(agent.DurableSteerer); !ok || !state.releaseOnCompletion { - ack.ErrorCode, ack.Error = "unsupported", "Durable input receipts require a supported release-on-completion run." + if _, ok := session.(agent.DurableSteerer); !ok || (!state.releaseOnCompletion && state.preparedHandoff == nil) { + ack.ErrorCode, ack.Error = "unsupported", "Durable input receipts require a supported Turn settlement contract." return &ack } } diff --git a/apps/parsar-daemon/internal/dispatch/suspend.go b/apps/parsar-daemon/internal/dispatch/suspend.go index 924fe237e..6058530f0 100644 --- a/apps/parsar-daemon/internal/dispatch/suspend.go +++ b/apps/parsar-daemon/internal/dispatch/suspend.go @@ -49,6 +49,12 @@ func (r *Router) Quiesce(ctx context.Context, request proto.EnvironmentSuspendPa } } } + for _, owner := range r.executors { + if owner.invalid || owner.preparing || owner.run != nil || owner.admission != nil || owner.environmentID != request.EnvironmentID { + r.mu.Unlock() + return ErrRouterBusy + } + } r.suspension = &request for _, p := range r.preparations { if p.timer != nil { @@ -63,6 +69,12 @@ func (r *Router) Quiesce(ctx context.Context, request proto.EnvironmentSuspendPa } } } + for _, owner := range r.executors { + owner.idleLease++ + if owner.timer != nil { + owner.timer.Stop() + } + } r.mu.Unlock() err := r.shutdownWG.waitContext(ctx) r.mu.Lock() @@ -89,6 +101,9 @@ func (r *Router) Resume(request proto.EnvironmentSuspendPayload, sender Sender) } r.sender = sender r.suspension = nil + for _, owner := range r.executors { + r.scheduleExecutorIdleLocked(owner) + } for _, states := range r.idle { for state := range states { r.scheduleIdleLocked(state) diff --git a/apps/parsar-daemon/internal/dispatch/workspace_directory_test.go b/apps/parsar-daemon/internal/dispatch/workspace_directory_test.go index 49835c073..28a14c02a 100644 --- a/apps/parsar-daemon/internal/dispatch/workspace_directory_test.go +++ b/apps/parsar-daemon/internal/dispatch/workspace_directory_test.go @@ -33,7 +33,7 @@ func TestWorkspaceDirectoryRetainsEnvironmentAndTransferredOwner(t *testing.T) { t.Fatal(got) } if phase == "idle" { - _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "prepare", proto.ExecutionStartPayload{Handle: ready.Handle, RunID: "run", Input: proto.TextInput("start")})) + _ = r.Handle(t.Context(), mustEnv(t, proto.TypeExecutionStart, "prepare", proto.ExecutionStartPayload{Handle: ready.Handle, ExecutorID: ready.ExecutorID, RunID: "run", Input: proto.TextInput("start")})) waitPreparationStatus(t, sender, "prepare", "started", "") _ = r.Handle(t.Context(), mustEnv(t, proto.TypeWorkspaceRead, "stale", request)) if got := waitWorkspaceRead(t, sender, "stale"); got.Outcome != "rejected" { diff --git a/apps/parsar-daemon/internal/dispatch/workspace_read.go b/apps/parsar-daemon/internal/dispatch/workspace_read.go index 77aeb8b67..0932387d2 100644 --- a/apps/parsar-daemon/internal/dispatch/workspace_read.go +++ b/apps/parsar-daemon/internal/dispatch/workspace_read.go @@ -68,6 +68,9 @@ func (r *Router) workspaceResourceLocked(request proto.WorkspaceReadPayload) (an return nil, "resource_unavailable" } resource = p.prepared + if p.executor != nil { + resource = p.executor.native + } } else { s := r.sessions[request.RunID] if s == nil || s.environmentID != request.EnvironmentID || s.session == nil || diff --git a/apps/parsar-daemon/internal/dispatch/workspace_write.go b/apps/parsar-daemon/internal/dispatch/workspace_write.go index 751cdb807..adac13c47 100644 --- a/apps/parsar-daemon/internal/dispatch/workspace_write.go +++ b/apps/parsar-daemon/internal/dispatch/workspace_write.go @@ -58,10 +58,16 @@ func (r *Router) handleWorkspaceWrite(ctx context.Context, env proto.Envelope) e } return r.sendWorkspaceWrite(ctx, env.ID, rejectedWorkspaceWrite("write_capacity")) } - if !r.localWorkspace.AcceptsFileWrite(request.EnvironmentID, request.SessionID) || len(r.sessions) != 0 || len(r.idle) != 0 { + if !r.localWorkspace.AcceptsFileWrite(request.EnvironmentID, request.SessionID) || len(r.sessions) != 0 || len(r.idle) != 0 || len(r.workspaceReads) != 0 { r.mu.Unlock() return r.sendWorkspaceWrite(ctx, env.ID, rejectedWorkspaceWrite("resource_unavailable")) } + for _, owner := range r.executors { + if owner.preparing || owner.admission != nil || owner.run != nil || owner.invalid { + r.mu.Unlock() + return r.sendWorkspaceWrite(ctx, env.ID, rejectedWorkspaceWrite("resource_unavailable")) + } + } for _, p := range r.preparations { if p.owns { r.mu.Unlock() diff --git a/apps/parsar-daemon/internal/localworkspace/binding.go b/apps/parsar-daemon/internal/localworkspace/binding.go index 928db7fdb..825661c78 100644 --- a/apps/parsar-daemon/internal/localworkspace/binding.go +++ b/apps/parsar-daemon/internal/localworkspace/binding.go @@ -90,7 +90,7 @@ func (b *Binding) Configure(r proto.PromptRequestPayload) (proto.PromptRequestPa } if b == nil || r.LocalEnvironment == nil || r.LocalEnvironment.ID != b.environment || r.AgentStateKey != b.stateKey || r.DisableExecutionEnvironment || r.WorkDir != "" || - r.ConversationID != "" || r.WorkspaceAuthoring || !r.StrictResume || !r.ReleaseOnCompletion { + r.ConversationID != "" || r.WorkspaceAuthoring || !r.StrictResume { return r, errors.New("request does not match the dedicated local Environment") } if !r.WorkspaceReadOnly || r.LocalEnvironment.NetworkAccess != "" || len(r.LocalEnvironment.AllowedDomains) > 0 { diff --git a/apps/parsar-daemon/internal/localworkspace/binding_test.go b/apps/parsar-daemon/internal/localworkspace/binding_test.go index 98973300b..8a68555f9 100644 --- a/apps/parsar-daemon/internal/localworkspace/binding_test.go +++ b/apps/parsar-daemon/internal/localworkspace/binding_test.go @@ -107,3 +107,11 @@ func TestDirectoryRejectsMalformedOrIncompleteResponses(t *testing.T) { } } } + +func TestBindingAllowsRetainedExecutor(t *testing.T) { + b, req := testBinding(t) + req.ReleaseOnCompletion = false + if _, err := b.Configure(req); err != nil { + t.Fatal(err) + } +} diff --git a/apps/parsar-daemon/testdata/onboarding/main.go b/apps/parsar-daemon/testdata/onboarding/main.go index 67a29f92b..679f98338 100644 --- a/apps/parsar-daemon/testdata/onboarding/main.go +++ b/apps/parsar-daemon/testdata/onboarding/main.go @@ -28,34 +28,81 @@ func (s *sender) Send(_ context.Context, e proto.Envelope) error { return s.encoder.Encode(e) } -type harness struct{ history map[string]string } +type harness struct { + mu sync.Mutex + history map[string]string +} -func (h *harness) start(_ context.Context, req proto.PromptRequestPayload, out chan<- proto.Envelope) (agent.Session, error) { - if _, err := req.Input.TextOnly(); err != nil { - return nil, err +func (h *harness) prepare(_ context.Context, req proto.PromptRequestPayload) (agent.Executor, error) { + if req.RunID != "" || len(req.Input) != 0 || req.ConversationID != "" { + return nil, errors.New("preparation submitted fixture input") } - if !req.StrictResume || !req.ReleaseOnCompletion || !req.DisableExecutionEnvironment || !req.DisableSubagents || len(req.FunctionTools) > 0 || req.MCPHTTPServers != nil { + if !req.StrictResume || !req.DisableExecutionEnvironment || !req.DisableSubagents || len(req.FunctionTools) > 0 || req.MCPHTTPServers != nil { return nil, errors.New("unsupported fixture operation") } + h.mu.Lock() + defer h.mu.Unlock() previous := h.history[req.AgentStateKey] - if req.AgentSessionID != previous || req.RequireExistingNativeSession { + if req.AgentSessionID != previous || (req.RequireExistingNativeSession && previous == "") { return nil, errors.New("native history mismatch") } - id := previous - if id == "" { - id = "fixture-" + req.AgentStateKey - h.history[req.AgentStateKey] = id + if previous == "" { + previous = "fixture-" + req.AgentStateKey + h.history[req.AgentStateKey] = previous + } + return &executor{native: previous}, nil +} + +type executor struct { + mu sync.Mutex + native string + active *session + closed bool +} + +func (e *executor) StartTurn(ctx context.Context, run string, input proto.MessageInput, out chan<- proto.Envelope) (agent.Turn, error) { + if ctx.Err() != nil || run == "" || out == nil { + return nil, errors.New("invalid fixture Start") + } + if _, err := input.TextOnly(); err != nil { + return nil, err + } + e.mu.Lock() + defer e.mu.Unlock() + if e.closed || e.active != nil { + return nil, errors.New("fixture Executor unavailable") + } + s := &session{out: out, run: run, native: e.native, settled: make(chan struct{})} + s.release = func() { + e.mu.Lock() + if e.active == s { + e.active = nil + } + e.mu.Unlock() } - s := &session{out: out, run: req.RunID, native: id} + e.active = s s.emit(proto.TypeDelta, proto.DeltaPayload{Delta: "ready", Sequence: 1}) return s, nil } +func (e *executor) Close(ctx context.Context) error { + e.mu.Lock() + e.closed = true + current := e.active + e.mu.Unlock() + if current != nil { + return current.Cancel(ctx) + } + return nil +} + type session struct { mu sync.Mutex out chan<- proto.Envelope run, native string closed bool + settled chan struct{} + release func() } func (s *session) emit(kind string, payload any) { @@ -68,6 +115,8 @@ func (s *session) Cancel(context.Context) error { if !s.closed { s.closed = true close(s.out) + s.release() + close(s.settled) } return nil } @@ -92,15 +141,29 @@ func (s *session) SteerWithReceipt(_ context.Context, p proto.PromptSteerPayload s.emit(proto.TypeDone, proto.DonePayload{Content: "ready" + text, Metadata: map[string]any{proto.DoneMetaAgentSessionID: s.native}}) s.closed = true close(s.out) + s.release() + close(s.settled) return nil } +func (s *session) AwaitSettlement(ctx context.Context) (agent.TurnSettlement, error) { + select { + case <-s.settled: + return agent.TurnSettlement{Reusable: true}, nil + case <-ctx.Done(): + return agent.TurnSettlement{}, ctx.Err() + } +} + func run() error { registry := agent.NewRegistry() h := &harness{history: map[string]string{}} registry.RegisterKind(proto.SupportedAgentKind{Kind: "fixture_harness", Available: true, Capabilities: proto.AgentKindCapabilities{ Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: true, ExecutionControls: true, ToolObservations: true, SubagentControl: true, EnvironmentNone: true, - }}, h.start) + }}, func(context.Context, proto.PromptRequestPayload, chan<- proto.Envelope) (agent.Session, error) { + return nil, errors.New("fixture execution requires an Executor") + }) + registry.RegisterExecutor("fixture_harness", h.prepare) sink := &sender{encoder: json.NewEncoder(os.Stdout)} router, err := dispatch.New(dispatch.Config{Registry: registry, Sender: sink, Log: slog.New(slog.NewTextHandler(io.Discard, nil))}) if err != nil { diff --git a/contracts/agents-api/harness-onboarding.md b/contracts/agents-api/harness-onboarding.md index ce98e8e66..c9fdde33e 100644 --- a/contracts/agents-api/harness-onboarding.md +++ b/contracts/agents-api/harness-onboarding.md @@ -1,64 +1,139 @@ # Add a native harness to OpenAgentCore -This reference is for adapter developers using the shared contract on main after -PR #701 (2026-09-19). Start with a working native SDK or machine-readable protocol. -The goal is to register an engine without changing public handlers, storage or the -scheduler. Codex, Claude and additional harnesses have equal architectural status; -each engine qualifies its own supported operations. +Implement one adapter for a native SDK or machine-readable protocol. Core and +Runtime use the common contract; adding a harness should not add engine-name +branches to public handlers, storage, scheduling or environment providers. -The [harness contract](harnesses.md) is the semantic baseline. The +The [harness contract](harnesses.md) defines observable behavior. The [contributor guide](../../CONTRIBUTING.md#harness-qualification-and-onboarding) -owns architecture and delivery rules. The pinned external Agents API in -[upstream.json](upstream.json) remains separate from this private adapter contract. - -## Ownership and implementation locations - -| Component | Responsibility | Existing location | +owns architecture and required checks. The pinned public Agents API in +[upstream.json](upstream.json) is separate from the internal adapter protocol. + +## Ownership + +```text +Core: Session, Turn, immutable configuration, durable events + | + common Runtime protocol + | +Runtime: Executor preparation, reuse, idle expiry, recovery + | + Harness adapter package + | + native SDK, process or connection +``` + +| Component | Responsibility | Location | | --- | --- | --- | -| Core | Public protocol, authentication, resource ownership, durable state and scheduling | `services/agents-api` | -| Runtime | Authenticated connection, dispatch, input receipts and preparation ownership | `apps/parsar-daemon/internal/dispatch` | -| Adapter | Native configuration, process/SDK calls, event translation and native restrictions | `apps/parsar-daemon/internal/agent/` | -| Harness | Model/tool loop and native history | Pinned upstream SDK or executable | -| Service profile | Pure validation of qualified placements and option values | `services/agents-api/internal/engine` | -| Runtime registration | Factory and verified capabilities available in this installation | `apps/parsar-daemon/internal/cli` | - -A service profile authorizes supported combinations. A Runtime advertisement says -what that installation can execute. Neither replaces public schema validation, -tenant authorization or the other boundary. Native limits belong in adapters and -profiles, not engine-name branches in Core. - -## Implement the existing interfaces - -Use `internal/agentdaemon/proto` for requests, events, receipts and errors. Do not -introduce a parallel wire protocol or another model/tool loop. - -| Interface | When required | Observable obligation | +| Core | Public API, authority, durable state, scheduling and configuration snapshots | `services/agents-api` | +| Runtime | Authenticated connection and common Executor/Turn lifecycle | `apps/parsar-daemon/internal/dispatch` | +| Adapter | Native configuration, resources, API calls, event translation and restrictions | `apps/parsar-daemon/internal/agent/` | +| Harness | Native model/tool loop and history | Pinned SDK or executable | +| Service profile | Pure validation of qualified operations and placements | `services/agents-api/internal/engine` | +| Registration | Installed factories and verified capability declarations | `apps/parsar-daemon/internal/cli` | + +An Environment supplies execution resources. Managed Docker, E2B and user-managed +machines differ in provisioning and connection; their connected Runtime uses this +same contract. Operating-system support belongs in the implementation and its +qualification. The current implementation is Linux; a platform-neutral interface +alone does not qualify another platform. Resource management owns machine selection, allocation and Environment creation +and reclamation. Executor close does not release that allocation or delete the +workspace. Environment reclamation is an explicit resource-management action that +coordinates with execution. Runtime connection, installed capability snapshot, +Session Executor and Turn each have their own lifetime. Model providers supply +model communication configuration, not Turn scheduling or native process ownership. + +## Required adapter interfaces + +The canonical signatures and ownership comments are in +[`agent/executor.go`](../../apps/parsar-daemon/internal/agent/executor.go): + +```go +type ExecutorFactory func(context.Context, proto.PromptRequestPayload) (Executor, error) + +type Executor interface { + StartTurn(context.Context, string, proto.MessageInput, chan<- proto.Envelope) (Turn, error) + Close(context.Context) error +} + +type Turn interface { + Cancel(context.Context) error + CancellationOutcome() proto.DonePayload + AwaitSettlement(context.Context) (TurnSettlement, error) +} + +type TurnSettlement struct { + Reusable bool + Reason string +} +``` + +The factory prepares a fixed configuration without sending model input. Executor +owns the native process or connection, native Session, capability configuration and +cleanup resources. If preparation fails while cleanup remains unconfirmed, return +that non-nil Executor together with the error so Runtime retains its cleanup owner. +`StartTurn` creates a fresh Turn wrapper and event stream; +normal completion retains the Executor. Never reset a completed Turn object or +restart the process merely to implement the next Turn. + +A nil Turn from `StartTurn` guarantees no input was submitted and no output writer +was retained. Runtime closes the output channel in that case. Once input may have +been submitted, return a non-nil Turn even with an error: that Turn owns exactly +one output-channel close, and uncertain input must not be replayed. + +`Cancel` addresses that exact Turn. `AwaitSettlement` confirms that native events, +inputs, function results, interactions, child work and output writes have settled. +Only then may `Reusable` be true. False requires a reason and confirmed Executor +close before replacement. An error means settlement is unconfirmed. A caller +cancelling its wait cannot discard resource ownership or redirect a late callback +to the next Turn. Close must retain its exact cleanup target after failure so the +Runtime can retry serially. Successful resource close alone does not prove that a +cancelled input was applied or that its original outcome is known. + +The Runtime owns admission handles, idle deadlines and resource limits. Adapters +own native translation. For example, a Codex adapter retains its app-server and +thread, a Claude adapter retains one streaming Query, and a MiniMax adapter retains +its ACP connection and native Session. Core sees the same Executor and Turn +semantics in each case. + +## Events, inputs and optional capabilities + +Use [`internal/agentdaemon/proto`](../../internal/agentdaemon/proto) for neutral +requests, events and receipts. Each Turn emits only its own events with its Run ID, +in order, and one terminal outcome. Native IDs and usage must be observed rather +than invented. Capture the originating Turn before asynchronous native callbacks; +never attribute a late result to whichever Turn is currently active. + +Initial input and steering use ordered `proto.MessageInput`. Preserve user-message +and content order. Text-only adapters reject images through `TextOnly()` instead +of dropping them. A successful transport write is distinct from confirmed native +application. Resume only the exact history bound to the Session; missing or +ambiguous required history fails before new model input. + +| Interface or contract | When required | Obligation | | --- | --- | --- | -| `agent.Factory` and `agent.Session` | Every harness | Start a run; cancel idempotently; own output closure and native process lifetime | -| `agent.DurableSteerer` | Current public text execution contract | Distinguish a complete native write from confirmed application; return rejection/inactive/not-ready accurately | -| `agent.PreparationFactory` and `agent.Prepared` | Placements requiring preparation | Prepare without consuming model input; transfer ownership once in Start; retain failed cleanup ownership | -| `agent.PreparedCancellation` | Executable preparations | Cancel the same resource across Start; settle only after native effects and output writes stop | -| `agent.PermissionResponder`, `agent.UserChoiceResponder` | Only when emitting those interactions | Route exact interaction identities and preserve application receipts | -| Other optional interfaces | Only for declared operations | Implement the existing operation semantics and validate native support | - -Read the source comments in `registry.go`, `steering.go`, `preparation.go` and -`interactions.go` before implementing. Base `Session.Cancel` is a cancellation -signal; its return alone is not proof that all effects stopped. Use the existing -router's settlement path and adapter outcome interfaces. Do not give both the -adapter and router ownership of closing the same output channel. - -Preserve event order and native call identities. Emit one run terminal outcome; -never manufacture applied input, usage counters or successful cleanup. Resume only -the native history bound to the execution Session. Missing or ambiguous history -fails before new model input. A disconnected observer does not authorize replay. +| `agent.DurableSteerer` | Current public text execution | Distinguish write and application receipts; preserve retry identity | +| `agent.FunctionResultSubmitter` | Public function tools | Match call/result identity and acknowledge native application | +| `agent.PermissionResponder`, `agent.UserChoiceResponder` | When emitting these interactions | Route exact identities and settle receipts | +| `agent.WorkspaceReader`, `agent.WorkspaceDirectoryLister` | Qualified workspace operations | Use the fixed authorized workspace and retain accepted operations through close | +| Neutral message, image, MCP, structured-output and Subagent observations | Only when qualified and advertised | Preserve the operation-specific contract and reject unsupported combinations | + +Optional features need not match another harness. The service profile qualifies +public combinations; the Runtime advertises this installation's available support. +Neither replaces schema validation or tenant authorization. Declaring a capability +without implementing its semantics is an error. Workspace reads may use separate +read-only preparations; those do not start model work or provide another execution +lifecycle. ## Register a supported operation set 1. Pin the upstream source/package version and document the native entry point. 2. Implement the adapter using its SDK or native protocol. Reuse shared process, credential/configuration and local workspace helpers where applicable. -3. Register its `SupportedAgentKind` and factory with `RegisterKind`; register - preparation afterward with `RegisterPreparation` when supported. Runtime +3. Register its `SupportedAgentKind` and direct-call factory with `RegisterKind`, + then register the shared `ExecutorFactory` with `RegisterExecutor`. A direct-call + factory should delegate to the same Executor implementation. Register separate + read-only preparations only when required by qualified workspace operations. Runtime advertisements must describe behavior verified for that installation. 4. Add a profile to the existing static catalog, or supply an immutable catalog through service composition. Custom composition supplies the same @@ -90,7 +165,7 @@ confirmed native output through the existing Message contract. Register public qualification separately from the Runtime capability; see the [structured-output boundary](structured-output.md). No Core engine-name branch is required. -Initial requests, `Prepared.Start` and steering consume the same ordered +Initial requests, `Executor.StartTurn` and steering consume the same ordered `proto.MessageInput`. Text-only adapters use `TextOnly()` to reject images without discarding content. Image adapters translate each part natively and acknowledge an active batch only after all its messages are applied. Register @@ -109,14 +184,18 @@ Before implementation, record the operation set, expected results, exclusions an stopping conditions in the board. A batch ends when its declared operations pass; it does not expand to match another harness's feature list. -- Adapter tests: native failures, ordered events, input write/application receipts, - cancellation settlement, strict continuation and unknown-outcome handling. +- Adapter tests: two ordinary Turns share one native process/connection and history; + cancellation followed by another Turn; stale cancellation and late events; native + exit, cleanup failure, input write/application receipts and unknown outcomes. + Verify fresh per-Turn usage, function, input and child-observation state. - Shared integration: public admission, actual Worker/device selection, gateway, daemon registration/dispatch and durable terminal projection. See `TestThirdHarnessPublicOnboarding` for a synthetic example, not native evidence. - Real acceptance: pinned official Python SDK and raw HTTP against our Agents API, real provider API, native harness and independent execution database. Verify - initial execution, follow-up input, cancellation and restart/continuation. + initial execution, warm follow-up, cancellation and restart/continuation. + Record native owner identity and same-condition cold/warm timing; mock results + cannot establish native reuse or performance gains. For hosted qualification also verify Files/Artifacts, workspace identity, credential protection and foreign-history rejection. - Regression: existing qualified engines keep working. Run targeted tests during @@ -132,31 +211,24 @@ Feishu. Failed or synthetic runs cannot be counted as real acceptance. Merge the bounded PR after required checks/review, update its board child and reassess the full board. Acceptance of one engine is not complete public protocol compatibility. -## MiniMax Code application - -The upstream project is [MiniMax-AI/minimax-code](https://github.com/MiniMax-AI/minimax-code). -The repository already contains an ACP stdio adapter under `agent/mcode`, including -native session creation/loading, events and human interactions. Its existing -product integration is not Agents API qualification. The current service catalog -registers Codex, Claude Code and MiniMax Code. Text qualification in #702 and -workspace qualification in #703 are separate recorded milestones. - -The implementation reuses that adapter with an explicit native 0.4.12 opt-in for -`environment:none` text execution. It adds native active-input receipts and -cancellation settlement, a pure service profile and verified registration. No -public handler, store schema or scheduler engine branch is needed. Existing -product behavior and accepted Codex/Claude features remain protected. - -See [the MiniMax Code deployment guide](../../services/agents-api/deploy/mcode/README.md) -for setup, real-provider acceptance and limits. Hosted workspace, Files/Artifacts, -public functions and MCP are not qualified by the text profile. Native differences -remain separately tracked work; they do not require feature equality for onboarding. - -The requested MiniMax Code workspace Runtime is qualified separately in -[MCODE-WORKSPACE-V1-001](mcode-workspace-v1.md). Its explicit scope includes -workspace execution and shared Files/Artifacts, cancellation/recovery and independent -Docker deployment. Text-only qualification is an intermediate milestone for that -scope, not completion of the requested Runtime integration. +## Minimal runnable example and native references + +[`testdata/onboarding/main.go`](../../apps/parsar-daemon/testdata/onboarding/main.go) +is a runnable synthetic adapter. It demonstrates registration, a Session-owned +Executor, fresh Turns, durable steering, cancellation and history binding through +the actual gateway and Router. Run `TestThirdHarnessPublicOnboarding` against a +dedicated test database to exercise the complete integration path. The fixture is +not shipped and is not evidence for a real SDK or sandbox. + +Use the native adapters as implementation references after choosing a native API: + +- [Codex](../../apps/parsar-daemon/internal/agent/codex/executor.go): app-server transport. +- [Claude](../../apps/parsar-daemon/internal/agent/claudesdk/executor.go): Go ownership and a [TypeScript SDK bridge](../../packages/claude-sdk-adapter). +- [MiniMax](../../apps/parsar-daemon/internal/agent/mcode): ACP and native workspace companion. + +Start with the mandatory text lifecycle, then qualify optional operations using +[harness coverage](harnesses.md) and each operation's contract. Do not copy an +adapter's native limitations into the shared Core protocol. ## Optional Subagent observations diff --git a/contracts/agents-api/harnesses.md b/contracts/agents-api/harnesses.md index 0aaeda64a..5c8976f84 100644 --- a/contracts/agents-api/harnesses.md +++ b/contracts/agents-api/harnesses.md @@ -13,15 +13,15 @@ For implementation steps and interface obligations, see ## Integration surface -1. Implement the existing daemon `agent.Factory`/`Session` and, for prepared - environments, `PreparationFactory`/`Prepared` interfaces. `Session` requires - cancellation; adapters that emit permission or user-choice requests additionally - implement `PermissionResponder` or `UserChoiceResponder`. Other optional - interfaces, such as function results, follow their declared operations. Reuse - `internal/agentdaemon/proto` requests, neutral events, input receipts and errors. -2. Register the factory, preparation factory and verified Runtime capabilities in - the daemon registry. Keep native translation inside the adapter. Dedicated local - environments reuse shared Files/write/export helpers and binding checks. +1. Implement `agent.ExecutorFactory`, `Executor` and `Turn`. Prepare fixed native + resources once, start fresh Turns on that owner, cancel an exact Turn, confirm + settlement and close the Executor. Implement optional interfaces only for + declared operations. Reuse `internal/agentdaemon/proto` requests, neutral events, + input receipts and errors. +2. Register the Executor factory and verified Runtime capabilities in the daemon + registry. Keep native translation in the adapter. Managed and user-managed + environments use the same execution lifecycle after connection; dedicated local + environments retain shared Files/write/export helpers and binding checks. 3. Add a pure qualified profile to `services/agents-api/internal/engine` and its static catalog (or supply an immutable catalog at service composition). Declare supported placements, public configuration/result limits @@ -49,9 +49,10 @@ not a different contract or authority level. The documented extension remains se ## Shared behavioral obligations -- Preparation holds resources without consuming input; start transfers ownership - once. Unused preparation releases through `Close`; cancellation remains valid - across the transfer and reports settlement only after native effects stop. +- Preparation holds resources without consuming input. Normal Turn completion + retains a settled healthy Executor. Cancellation targets one Turn; idle expiry, + environment shutdown and invalidation close the Executor. Failed cleanup retains + ownership and capacity, and never authorizes replay of uncertain input. - Confirm accepted/applied inputs separately. Preserve ordered public Items and events, stable call identity and one terminal outcome. Never replay uncertain work merely because a connection closed. diff --git a/contracts/agents-api/message-input.md b/contracts/agents-api/message-input.md index 165584d93..cfe784f89 100644 --- a/contracts/agents-api/message-input.md +++ b/contracts/agents-api/message-input.md @@ -74,7 +74,7 @@ before. ## Runtime boundary -Private wire 0.5.0 uses `MessageInput` for initial requests, prepared start and +Private wire 0.6.0 uses `MessageInput` for initial requests, prepared start and active steering. Each message contains ordered `InputContent` parts, also reused by function-result content. Core does not download, transcode or repair media. The separate Claude bridge reports protocol 2; daemon readiness rejects protocol 1. diff --git a/internal/agentdaemon/proto/envelope_test.go b/internal/agentdaemon/proto/envelope_test.go index 8aace4d66..5c3cd96f6 100644 --- a/internal/agentdaemon/proto/envelope_test.go +++ b/internal/agentdaemon/proto/envelope_test.go @@ -88,8 +88,8 @@ func TestVersionCompatible(t *testing.T) { ok bool }{ {Version, true}, // exact match - {"0.5.99", true}, // patch drift OK - {"0.4.99", false}, // minor drift NOT OK + {"0.6.99", true}, // patch drift OK + {"0.5.99", false}, // minor drift NOT OK {"1.0.0", false}, // major drift NOT OK {"", false}, // missing {"garbage", false}, // unparseable diff --git a/internal/agentdaemon/proto/preparation.go b/internal/agentdaemon/proto/preparation.go index f26c9eb7e..b03ba0905 100644 --- a/internal/agentdaemon/proto/preparation.go +++ b/internal/agentdaemon/proto/preparation.go @@ -10,16 +10,18 @@ const ( ) // ExecutionPreparePayload reuses execution configuration without accepting input -// or product authoring. The initial private profile requires a bound local environment, -// stable state key, strict resume and completion release. +// or product authoring. SessionID identifies the immutable configuration owner; +// each request reserves a separate Turn admission on its Runtime Executor. type ExecutionPreparePayload struct { + SessionID string `json:"session_id"` Configuration PromptRequestPayload `json:"configuration"` } type ExecutionStartPayload struct { - Handle string `json:"handle"` - RunID string `json:"run_id"` - Input MessageInput `json:"input"` + ExecutorID string `json:"executor_id"` + Handle string `json:"handle"` + RunID string `json:"run_id"` + Input MessageInput `json:"input"` } type ExecutionReleasePayload struct { @@ -31,6 +33,10 @@ type ExecutionReleasePayload struct { // order. ExpiresAt is Unix milliseconds. Repeated requests do not extend it. Retired request IDs // may allocate a fresh handle, but an old handle can never start its replacement. type PreparationStatusPayload struct { + // ExecutorID identifies the native resource owner, independently of this admission handle. + ExecutorID string `json:"executor_id,omitempty"` + // Reused is true when readiness reused an existing settled Executor. + Reused bool `json:"reused,omitempty"` Handle string `json:"handle,omitempty"` Revision uint64 `json:"revision"` State string `json:"state"` diff --git a/internal/agentdaemon/proto/version.go b/internal/agentdaemon/proto/version.go index a030854b8..134ead26e 100644 --- a/internal/agentdaemon/proto/version.go +++ b/internal/agentdaemon/proto/version.go @@ -5,7 +5,7 @@ package proto // upgrade query (`version=`) and in the bootstrap HTTP // response; mismatches fail closed at WS upgrade. const ( - Version = "0.5.0" + Version = "0.6.0" ) // VersionCompatible returns true when clientVersion's "X.Y" prefix diff --git a/packages/claude-sdk-adapter/src/adapter.ts b/packages/claude-sdk-adapter/src/adapter.ts index e4b86b070..3ab8b88d7 100644 --- a/packages/claude-sdk-adapter/src/adapter.ts +++ b/packages/claude-sdk-adapter/src/adapter.ts @@ -1,3 +1,4 @@ +import { ExecutorTurns, type ExecutorEvent } from "./executor_protocol.js"; import { StructuredOutput } from "./structured_output.js"; import { toolSearchEnvironment } from "./tool_search.js"; import { Subagents } from "./subagents.js"; @@ -6,7 +7,7 @@ import { WorkspaceDirectories, type WorkspaceDirectoryEvent } from "./workspace_ import { getSessionInfo, query, startup, type McpServerConfig, type Options, type WarmQuery } from "@anthropic-ai/claude-agent-sdk"; import { WorkspaceReads, type WorkspaceReadEvent } from "./workspace_reads.js"; import { Inputs, type InputEvent } from "./inputs.js"; -import { resultUsage, type NativeUsage } from "./usage.js"; +import { executorResultUsage, resultUsage, type NativeUsage } from "./usage.js"; import { spawnNative } from "./native.js"; import { MessageObserver, type MessageEvent } from "./messages.js"; import { createFunctionServer } from "./functions.js"; @@ -15,11 +16,12 @@ import { MCPProfile } from "./mcp.js"; import { MCPObserver, type MCPEvent } from "./mcp_observer.js"; import { CommandObserver, type CommandEvent } from "./command_observer.js"; import { WorkspaceProfile } from "./workspace.js"; -import { immediateInput, type Prepare, type Start } from "./request.js"; +import { immediateInput, type Prepare, type Start, type ExecutorPrepare } from "./request.js"; import { recoverSession } from "./recovery.js"; export { parseStart, type Start } from "./request.js"; export type Event = + | ExecutorEvent | Fact | WorkspaceDirectoryEvent | WorkspaceReadEvent @@ -34,7 +36,9 @@ export type Event = | { type: "result"; session_id: string; text: string } | { type: "error"; code: "invalid_request" | "history_unavailable" | "execution_failed" | "cancelled" }; -export async function execute(request: Start | Prepare, emit: (event: Event) => Promise, abort: AbortController, functions = new FunctionBridge(emit), inputs = new Inputs(immediateInput(request)), reads = new WorkspaceReads(emit, abort), directories = new WorkspaceDirectories(emit, abort)): Promise { +export async function execute(request: Start | Prepare | ExecutorPrepare, emit: (event: Event) => Promise, abort: AbortController, functions = new FunctionBridge(emit), inputs = new Inputs(immediateInput(request)), reads = new WorkspaceReads(emit, abort), directories = new WorkspaceDirectories(emit, abort), turns?: ExecutorTurns): Promise { + const ownerEmit=emit; + if(turns) emit=event=>turns.emit(event); const definitions = (request.functions ?? []).map(tool => ({ name: tool.name, description: tool.description, inputSchema: tool.parameters, deferLoading: tool.defer_loading })); const names = definitions.map(tool => `mcp__functions__${tool.name}`); const allowed = [...names, ...(request.tool_search ? ["ToolSearch"] : [])]; @@ -42,7 +46,7 @@ export async function execute(request: Start | Prepare, emit: (event: Event) => const profile = declarations === undefined ? undefined : new MCPProfile(declarations, names); const subagents = request.subagents ? new Subagents(request.cwd, request.subagents.max_concurrent, request.resume) : undefined; const workspace = request.workspace === undefined ? undefined : new WorkspaceProfile(request.cwd, request.workspace, names, profile, subagents, !!request.output_format); - const commands = workspace ? new CommandObserver() : undefined; + let commands = workspace ? new CommandObserver() : undefined; if (request.type === "prepare" && !workspace) throw new Error("invalid_request"); if (workspace && "mcp_http_servers" in request) throw new Error("invalid_request"); if (request.require_history && !request.resume) { @@ -59,8 +63,8 @@ export async function execute(request: Start | Prepare, emit: (event: Event) => } subagents?.expectSession(request.resume); const mcpServers: Record = Object.create(null); - if (definitions.length) mcpServers.functions = createFunctionServer(definitions, functions.invoke); - const mcp = profile ? new MCPObserver(profile.identities) : undefined; + if (definitions.length) mcpServers.functions = createFunctionServer(definitions, (call,signal) => { const target=functions; return turns ? turns.track(()=>target.invoke(call,signal)) : target.invoke(call,signal); }); + let mcp = profile ? new MCPObserver(profile.identities) : undefined; if (profile) Object.assign(mcpServers, profile.servers); const children: Promise[] = []; let result: Extract | undefined; @@ -68,12 +72,12 @@ export async function execute(request: Start | Prepare, emit: (event: Event) => const resultIDs = new Set(); let failed = false; let cancellationFactsFailed = false; - const structured = request.output_format ? new StructuredOutput() : undefined; - const messages = request.observe_messages ? new MessageObserver() : undefined; + let structured = request.output_format ? new StructuredOutput() : undefined; + let messages = request.observe_messages ? new MessageObserver() : undefined; let stream: ReturnType | undefined; let warm: WarmQuery | undefined; let nativeAlive = false; - const closeInputs = () => inputs.close(); + const closeInputs = () => { inputs.close(); turns?.close(); }; abort.signal.addEventListener("abort", closeInputs, { once: true }); try { if (abort.signal.aborted) throw new Error("cancelled"); @@ -111,20 +115,46 @@ export async function execute(request: Start | Prepare, emit: (event: Event) => return child; }, }; - if (request.type === "prepare") { + if(turns) { + for(const matchers of Object.values(options.hooks ?? {})) for(const matcher of matchers ?? []) { + matcher.hooks=matcher.hooks.map(hook=>(...args)=>turns.track(()=>hook(...args))); + } + const canUseTool=options.canUseTool; + if(canUseTool) options.canUseTool=(...args)=>turns.track(()=>canUseTool(...args)); + } + if (request.type === "prepare" || request.type === "executor_prepare") { warm = await startup({ options, initializeTimeoutMs: 15000 }); - stream = warm.query(inputs); + stream = warm.query(turns ?? inputs); const initialized = await stream.initializationResult(); - if (initialized.hooks_applied !== true || children.length !== 1 || !nativeAlive || abort.signal.aborted) { + const hooksRequested = Object.values(options.hooks ?? {}).some(matchers => matchers?.some(matcher => matcher.hooks.length > 0)); + if ((hooksRequested && initialized.hooks_applied !== true) || children.length !== 1 || !nativeAlive || abort.signal.aborted) { throw new Error("preparation unavailable"); } - reads.bind(stream, request.cwd); - if (process.platform === "linux") await directories.bind(request.cwd); - await emit({ type: "prepared" }); + if(workspace) { + reads.bind(stream, request.cwd); + if (process.platform === "linux") await directories.bind(request.cwd); + } + if (request.mcp_http_servers?.some(server=>server.required)) profile?.verifyRequired(await stream.mcpServerStatus()); + if(turns) { + turns.configure(stream,(input,output)=>{ + inputs=new Inputs(input); + functions=new FunctionBridge(output); + messages=request.observe_messages ? new MessageObserver() : undefined; + structured=request.output_format ? new StructuredOutput() : undefined; + commands=workspace ? new CommandObserver() : undefined; + mcp=profile ? new MCPObserver(profile.identities) : undefined; + subagents?.beginTurn(); + return inputs; + },async value=>{ + if(value.type==="steer") for(const event of inputs.submit(value)) await emit(event); + else functions.submit(JSON.stringify(value)); + }); + await ownerEmit({type:"executor_ready",protocol:3}); + } else await emit({ type: "prepared" }); } else if (profile) { if (inputs.hasInput) throw new Error("MCP input released before initialization"); warm = await startup({ options, initializeTimeoutMs: 15000 }); - stream = warm.query(inputs); + stream = warm.query(turns ?? inputs); const initialized = await stream.initializationResult(); if (initialized.hooks_applied !== true || children.length !== 1) throw new Error("MCP initialization unavailable"); if (request.mcp_http_servers?.some(server => server.required)) profile.verifyRequired(await stream.mcpServerStatus()); @@ -132,6 +162,10 @@ export async function execute(request: Start | Prepare, emit: (event: Event) => inputs.release(request.input); } else stream = query({ prompt: inputs, options }); for await (const message of stream) { + // Native command queue telemetry can follow a result. It is not model + // output or a consumed-input receipt, and has no authority over a Turn. + if (isCommandLifecycle(message)) continue; + if(turns && !turns.id && !(message.type==="system" && message.subtype==="init")) throw new Error("unbound native event"); subagents?.consume(message); structured?.consume(message, nativeID); await functions.consume(message, nativeID); @@ -148,22 +182,33 @@ export async function execute(request: Start | Prepare, emit: (event: Event) => message.mcp_servers.some(server => server.name !== "functions" || server.status !== "connected")) { throw new Error("unexpected native configuration"); } - for (const event of inputs.start(nativeID)) await emit(event); + turns?.identify(nativeID); + if(!turns || turns.id) for (const event of inputs.start(nativeID)) await emit(event); } else if (!messages && message.type === "stream_event" && message.parent_tool_use_id === null && message.event.type === "content_block_delta" && message.event.delta.type === "text_delta") { await emit({ type: "delta", delta: message.event.delta.text }); } else if (message.type === "result") { if (!message.uuid || resultIDs.has(message.uuid) || !nativeID || message.session_id !== nativeID) throw new Error("invalid native result identity"); resultIDs.add(message.uuid); - await emit({ type: "usage", session_id: nativeID, result_id: message.uuid, usage: resultUsage(message) }); + await emit({ type: "usage", session_id: nativeID, result_id: message.uuid, usage: turns ? executorResultUsage(message) : resultUsage(message) }); for (const event of inputs.consume(message)) await emit(event); - if (message.subtype !== "success" || message.is_error) throw new Error("unsuccessful native result"); - if (structured) await emit(structured.complete(message)); - result = { type: "result", session_id: nativeID, text: message.result }; + if ((message.subtype !== "success" || message.is_error) && !turns?.cancelled) throw new Error("unsuccessful native result"); + if (structured && !turns?.cancelled) await emit(structured.complete(message)); + result = { type: "result", session_id: nativeID, text: message.subtype === "success" ? message.result : "" }; + if(turns?.cancelled && !inputs.complete) throw new Error("unconfirmed cancelled inputs"); + } + if (message.type !== "result" && (!turns || turns.id)) for (const event of inputs.consume(message)) await emit(event); + if(turns && result && inputs.complete && (!subagents || subagents.complete)) { + if(!await turns.quiescent() || abort.signal.aborted) throw new Error("unconfirmed interrupt"); + functions.assertComplete(); mcp?.assertComplete(); commands?.assertComplete(); + if(subagents) for(const event of await subagents.facts()) await emit(event); + await emit(turns.cancelled ? {type:"error",code:"cancelled"} : result); + functions.close(); + await turns.settled(true,""); + result=undefined; } - if (message.type !== "result") for (const event of inputs.consume(message)) await emit(event); } - if (subagents) for (const event of await subagents.facts()) await emit(event); + if (subagents && (!turns || turns.id)) for (const event of await subagents.facts()) await emit(event); functions.assertComplete(); mcp?.assertComplete(); commands?.assertComplete(); @@ -172,6 +217,7 @@ export async function execute(request: Start | Prepare, emit: (event: Event) => } finally { profile?.close(); inputs.close(); + turns?.close(); functions.close(); try { await directories.close(); } catch { failed = true; } await reads.close(); @@ -179,16 +225,29 @@ export async function execute(request: Start | Prepare, emit: (event: Event) => warm?.close(); abort.signal.removeEventListener("abort", closeInputs); const exits = await Promise.all(children); + await turns?.drain(); if (!exits.length || exits.some(code => code !== 0)) failed = true; - if (subagents && abort.signal.aborted && exits.length === 1) { + if (subagents && (!turns || turns.id) && abort.signal.aborted && exits.length === 1) { try { for (const event of await subagents.facts(Date.now())) await emit(event); } catch { cancellationFactsFailed = true; } } if (mcp) for (const event of mcp.close()) await emit(event); if (commands) for (const event of commands.close()) await emit(event); } + if(turns) { + if(turns.id) { + await emit({type:"error",code: turns.cancelled && !cancellationFactsFailed ? "cancelled" : "execution_failed"}); + await turns.settled(false, turns.cancelled ? "cancellation_unconfirmed" : "native_execution_unavailable"); + } else if(failed && !abort.signal.aborted) await ownerEmit({type:"error",code:"execution_failed"}); + return; + } if (cancellationFactsFailed) await emit({ type: "error", code: "execution_failed" }); else if (abort.signal.aborted) await emit({ type: "error", code: "cancelled" }); else if (failed || !result || !inputs.complete) await emit({ type: "error", code: "execution_failed" }); else await emit(result); } + +function isCommandLifecycle(message: unknown): boolean { + return typeof message === "object" && message !== null && + "type" in message && message.type === "command_lifecycle"; +} diff --git a/packages/claude-sdk-adapter/src/executor_protocol.ts b/packages/claude-sdk-adapter/src/executor_protocol.ts new file mode 100644 index 000000000..ba4e4c162 --- /dev/null +++ b/packages/claude-sdk-adapter/src/executor_protocol.ts @@ -0,0 +1,129 @@ +import type { Query, SDKUserMessage } from "@anthropic-ai/claude-agent-sdk"; +import { Inputs } from "./inputs.js"; +import { parseMessageInput, type MessageInput } from "./message_input.js"; + +export type ExecutorEvent = + | { type: "executor_ready"; protocol: 3 } + | { type: "turn_started"; turn_id: string } + | { type: "turn_settled"; turn_id: string; reusable: boolean; reason: string }; +type WireEvent = { type: string; [key: string]: unknown }; +type Turn = { id: string; inputs: Inputs; cancelled: boolean; interrupt?: Promise; callbacks: Set> }; + +// The SDK sees one iterator for the Executor lifetime. Each yielded batch belongs +// to a fresh receipt ledger; finishing a Turn never closes this outer iterator. +export class ExecutorTurns implements AsyncIterable { + private active?: Turn; + private readonly used = new Set(); + private readonly queue: Inputs[] = []; + private wake?: () => void; + private ended = false; + private stream?: Query; + private begin?: (input: MessageInput, emit: (event: WireEvent) => Promise) => Inputs; + private submitInput?: (value: Record) => Promise; + private nativeID = ""; + constructor(private readonly output: (event: WireEvent) => Promise, private readonly abort: AbortController) {} + + configure(stream: Query, begin: NonNullable, submit: (value: Record)=>Promise): void { this.stream = stream; this.begin = begin; this.submitInput=submit; } + async submit(value: Record): Promise { + const payload=this.assertCurrent(value); + if(!this.submitInput || (payload.type!=="steer" && payload.type!=="function_result")) throw new Error("invalid turn input"); + await this.track(()=>this.submitInput!(payload)); + } + identify(id: string): void { + if (!id || this.nativeID && this.nativeID !== id) throw new Error("invalid native identity"); + this.nativeID = id; + } + get id(): string | undefined { return this.active?.id; } + get cancelled(): boolean { return this.active?.cancelled ?? false; } + get inputs(): Inputs { if (!this.active) throw new Error("no active turn"); return this.active.inputs; } + + async start(value: Record): Promise { + const id = value.turn_id; + if (this.ended || this.active || !this.stream || !this.begin || typeof id !== "string" || !id.trim() || id.length > 256 || + this.used.has(id) || Object.keys(value).some(key => !["type", "turn_id", "input"].includes(key))) throw new Error("invalid turn start"); + const input = parseMessageInput(value.input); + const output = async (event: WireEvent) => { + if (this.active?.id !== id) throw new Error("late turn event"); + await this.output({ ...event, turn_id: id }); + }; + const inputs = this.begin(input, output); + this.active = { id, inputs, cancelled: false, callbacks: new Set() }; + this.used.add(id); + await this.output({ type: "turn_started", turn_id: id }); + if (this.nativeID) for (const event of inputs.start(this.nativeID)) await output(event); + this.queue.push(inputs); + this.wake?.(); + } + + async cancel(value: Record): Promise { + if (typeof value.turn_id !== "string" || Object.keys(value).some(key => !["type", "turn_id"].includes(key))) throw new Error("invalid cancellation"); + const turn = this.active; + // A retired identity has no authority over its successor. + if (!turn || value.turn_id !== turn.id) { + if (this.used.has(value.turn_id)) return; + throw new Error("unknown turn"); + } + if (turn.interrupt) return; + turn.cancelled = true; + const discarded=turn.inputs.cancelQueued(); + turn.interrupt = this.stream!.interrupt().then(receipt => { + const confirmed = !discarded && receipt !== undefined && Array.isArray(receipt.still_queued) && receipt.still_queued.length === 0; + if (!confirmed) this.abort.abort(); + return confirmed; + }, () => { this.abort.abort(); return false; }); + await turn.interrupt; + } + + assertCurrent(value: Record): Record { + if (!this.active || this.active.cancelled || value.turn_id !== this.active.id) throw new Error("inactive turn input"); + const { turn_id: _id, ...payload } = value; + return payload; + } + async emit(event: WireEvent): Promise { + if (!this.active) throw new Error("unbound native event"); + await this.output({ ...event, turn_id: this.active.id }); + } + track(operation: () => Promise): Promise { + const turn = this.active; + if (!turn) return Promise.reject(new Error("unbound native callback")); + const promise = Promise.resolve().then(operation); + turn.callbacks.add(promise); + void promise.then(() => turn.callbacks.delete(promise), () => turn.callbacks.delete(promise)); + return promise; + } + async quiescent(): Promise { + const turn = this.active; + if (!turn) return false; + if (this.abort.signal.aborted) return false; + let stop: () => void = () => {}; + const aborted = new Promise(resolve => { + stop = () => resolve(false); + this.abort.signal.addEventListener("abort", stop, { once: true }); + }); + try { + return await Promise.race([ + this.drain().then(async () => !turn.cancelled || await turn.interrupt === true), + aborted, + ]); + } finally { this.abort.signal.removeEventListener("abort", stop); } + } + async drain(): Promise { + const turn = this.active; + while (turn?.callbacks.size) await Promise.allSettled([...turn.callbacks]); + } + async settled(reusable: boolean, reason: string): Promise { + const turn = this.active; + if (!turn) return; + turn.inputs.close(); + this.active = undefined; + await this.output({ type: "turn_settled", turn_id: turn.id, reusable, reason }); + } + close(): void { this.ended = true; this.active?.inputs.close(); this.wake?.(); } + async *[Symbol.asyncIterator](): AsyncIterator { + while (!this.ended) { + const inputs = this.queue.shift(); + if (inputs) yield* inputs; + else await new Promise(resolve => { this.wake = resolve; }); + } + } +} diff --git a/packages/claude-sdk-adapter/src/inputs.ts b/packages/claude-sdk-adapter/src/inputs.ts index 67b37d768..c11ed7867 100644 --- a/packages/claude-sdk-adapter/src/inputs.ts +++ b/packages/claude-sdk-adapter/src/inputs.ts @@ -96,6 +96,11 @@ export class Inputs implements AsyncIterable { get hasInput(): boolean { return this.submitted.size > 0; } + cancelQueued(): boolean { + const discarded = this.queue.length > 0; + this.queue.length = 0; this.close(); return discarded; + } + close(): void { this.ended = true; this.wake?.(); } async *[Symbol.asyncIterator](): AsyncIterator { diff --git a/packages/claude-sdk-adapter/src/main.ts b/packages/claude-sdk-adapter/src/main.ts index 496c9ab0a..86b12aa09 100644 --- a/packages/claude-sdk-adapter/src/main.ts +++ b/packages/claude-sdk-adapter/src/main.ts @@ -1,3 +1,4 @@ +import { ExecutorTurns } from "./executor_protocol.js"; import { WorkspaceDirectories } from "./workspace_directories.js"; import { WorkspaceReads } from "./workspace_reads.js"; import { Inputs } from "./inputs.js"; @@ -12,7 +13,7 @@ const stop = () => abort.abort(); process.once("SIGTERM", stop); process.once("SIGINT", stop); lines.once("close", stop); -const emit = (event: Event): Promise => new Promise((resolve, reject) => { +const emit = (event: Event | {type:string;[key:string]:unknown}): Promise => new Promise((resolve, reject) => { process.stdout.write(JSON.stringify(event) + "\n", error => error ? reject(error) : resolve()); }); try { @@ -30,6 +31,7 @@ try { const reads = new WorkspaceReads(output, abort); const directories = new WorkspaceDirectories(output, abort); const prompts = new Inputs(immediateInput(request)); + const turns=request.type==="executor_prepare" ? new ExecutorTurns(emit,abort) : undefined; const incoming = (async () => { try { for await (const line of { [Symbol.asyncIterator]: () => input }) { @@ -39,6 +41,12 @@ try { reads.submit(value as Record); } else if (value && typeof value === "object" && "type" in value && value.type === "workspace_directory") { directories.submit(value as Record); + } else if(turns) { + if(!value || typeof value!=="object" || Array.isArray(value)) throw new Error("invalid_request"); + const control=value as Record; + if(control.type==="turn_start") await turns.start(control); + else if(control.type==="turn_cancel") await turns.cancel(control); + else await turns.submit(control); } else if (request.type === "prepare" && phase !== "running") { if (phase !== "prepared" || abort.signal.aborted) throw new Error("invalid_request"); prompts.release(preparedInput(value)); @@ -50,7 +58,7 @@ try { } catch { invalid = request.type === "prepare"; abort.abort(); } })(); - try { await execute(request, output, abort, functions, prompts, reads, directories); } + try { await execute(request, output, abort, functions, prompts, reads, directories, turns); } catch { await output({ type: "error", code: "execution_failed" }); } finally { prompts.close(); diff --git a/packages/claude-sdk-adapter/src/request.ts b/packages/claude-sdk-adapter/src/request.ts index dbf353008..ca9636ca0 100644 --- a/packages/claude-sdk-adapter/src/request.ts +++ b/packages/claude-sdk-adapter/src/request.ts @@ -20,20 +20,21 @@ export type Start = { mcp_http_servers?: HTTPServer[]; workspace?: Workspace; }; +export type ExecutorPrepare = Omit & { type: "executor_prepare" }; export type Prepare = Omit & { type: "prepare"; workspace: Workspace }; // MCP startup confirms its hooks before the native input iterator yields. -export function immediateInput(request: Start | Prepare): MessageInput | undefined { +export function immediateInput(request: Start | Prepare | ExecutorPrepare): MessageInput | undefined { return request.type === "start" && request.mcp_http_servers === undefined && !request.workspace?.mcp?.length ? request.input : undefined; } -export function parseRequest(line: string): Start | Prepare { +export function parseRequest(line: string): Start | Prepare | ExecutorPrepare { const value: unknown = JSON.parse(line); if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("invalid_request"); const request = value as Record; const allowed = new Set(["type", "input", "model", "system_prompt", "cwd", "resume", "require_history", "observe_messages", "output_format", "subagents", "functions", "tool_search", "mcp_http_servers", "workspace"]); if (Object.keys(request).some(key => !allowed.has(key)) || - (request.type !== "start" && request.type !== "prepare") || + (request.type !== "start" && request.type !== "prepare" && request.type !== "executor_prepare") || (request.type === "start" ? !Array.isArray(request.input) : "input" in request) || typeof request.model !== "string" || !request.model.trim() || typeof request.system_prompt !== "string" || @@ -68,7 +69,7 @@ export function parseRequest(line: string): Start | Prepare { if (request.require_history && !workspace) throw new Error("invalid_request"); if ((workspace && "mcp_http_servers" in request) || (request.type === "prepare" && !workspace)) throw new Error("invalid_request"); - return request as Start | Prepare; + return request as Start | Prepare | ExecutorPrepare; } export function parseStart(line: string): Start { diff --git a/packages/claude-sdk-adapter/src/runtime_check.ts b/packages/claude-sdk-adapter/src/runtime_check.ts index e221c67fd..dc6caec94 100644 --- a/packages/claude-sdk-adapter/src/runtime_check.ts +++ b/packages/claude-sdk-adapter/src/runtime_check.ts @@ -45,7 +45,7 @@ try { assert.equal(smoke.error, undefined, "bridge_unavailable"); assert.equal(smoke.status, 0, "bridge_unavailable"); assert.deepEqual(JSON.parse(smoke.stdout), { type: "error", code: "invalid_request" }); - process.stdout.write(JSON.stringify({ type: "runtime_ready", protocol: 2, features: [...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions", "workspace_structured_output"] : []), "message_images", "function_result_images", "tool_search", "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"], node: process.versions.node, sdk: sdk.version, mcp: mcp.version, native: nativeVersion }) + "\n"); + process.stdout.write(JSON.stringify({ type: "runtime_ready", protocol: 3, features: ["executor_reuse", ...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions", "workspace_structured_output"] : []), "message_images", "function_result_images", "tool_search", "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"], node: process.versions.node, sdk: sdk.version, mcp: mcp.version, native: nativeVersion }) + "\n"); } catch { // Native diagnostics can include operator environment; never forward them. process.stdout.write(JSON.stringify({ type: "runtime_unavailable" }) + "\n"); diff --git a/packages/claude-sdk-adapter/src/subagents.ts b/packages/claude-sdk-adapter/src/subagents.ts index 0441d0f1c..add5e689d 100644 --- a/packages/claude-sdk-adapter/src/subagents.ts +++ b/packages/claude-sdk-adapter/src/subagents.ts @@ -14,6 +14,11 @@ export class Subagents { constructor(private readonly cwd: string, private readonly limit: number, private resume: string | undefined) { if (!Number.isSafeInteger(limit) || limit < 1) throw new Error("invalid_request"); } + get complete(): boolean { return this.pending.size === 0 && this.running.size === 0; } + beginTurn(): void { + if (!this.complete) throw new Error("native child work is unsettled"); + this.observedCalls.clear(); this.admitted.clear(); + } expectSession(id: string | undefined): void { this.resume = id; } permitsActor(id?: string): boolean { return id === undefined || this.known.has(id); } isCoordination(name: string): boolean { return ["Agent", "Task", "SendMessage"].includes(name); } diff --git a/packages/claude-sdk-adapter/src/usage.ts b/packages/claude-sdk-adapter/src/usage.ts index e7ad54077..03baf1b57 100644 --- a/packages/claude-sdk-adapter/src/usage.ts +++ b/packages/claude-sdk-adapter/src/usage.ts @@ -7,3 +7,14 @@ export function resultUsage(message: SDKResultMessage): NativeUsage { return structuredClone({ usage: message.usage, modelUsage: message.modelUsage, total_cost_usd: message.total_cost_usd, subtype: message.subtype, is_error: message.is_error }); } + +// Streaming-input results mix two scopes. Keep that distinction alongside the +// untouched values so a Turn consumer cannot sum query totals across Turns. +export function executorResultUsage(message: SDKResultMessage): NativeUsage & { + scopes: { usage: "native_turn_main_loop"; modelUsage: "query_cumulative"; total_cost_usd: "query_cumulative_estimate" }; +} { + return { ...resultUsage(message), scopes: { + usage: "native_turn_main_loop", modelUsage: "query_cumulative", + total_cost_usd: "query_cumulative_estimate", + } }; +} diff --git a/packages/claude-sdk-adapter/tests/executor.test.mjs b/packages/claude-sdk-adapter/tests/executor.test.mjs new file mode 100644 index 000000000..79ef95c4c --- /dev/null +++ b/packages/claude-sdk-adapter/tests/executor.test.mjs @@ -0,0 +1,143 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import test from "node:test"; + +const fixture = ` +import assert from "node:assert/strict"; +import { registerHooks } from "node:module"; +import { Client } from "@modelcontextprotocol/sdk/client/index.js"; +import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js"; +const sdk = "export async function startup(args){return globalThis.startupFixture(args)} export function query(){throw new Error('one warm query required')} export async function getSessionInfo(){return {sessionId:'native'}}"; +registerHooks({resolve(specifier,context,next){if(specifier==='@anthropic-ai/claude-agent-sdk') return {url:'data:text/javascript,'+encodeURIComponent(sdk),shortCircuit:true};return next(specifier,context)}}); +globalThis.startupFixture=async({options})=>{ + const child=options.spawnClaudeCodeProcess({command:process.execPath,args:['-e',"process.stdin.resume();process.stdin.on('end',()=>process.exit(0))"],env:options.env,signal:options.abortController.signal}); + process.send({type:'native',pid:child.pid}); + const exited=new Promise(resolve=>child.once('close',()=>{process.send({type:'native_closed'});resolve()})); + let client; + if(options.mcpServers.functions){ + client=new Client({name:'fixture',version:'1'}); + const [a,b]=InMemoryTransport.createLinkedPair(); + await Promise.all([client.connect(a),options.mcpServers.functions.instance.connect(b)]); + } + let queried=false,interrupt,number=0; + const close=()=>{child.stdin.end();interrupt?.()}; + options.abortController.signal.addEventListener('abort',close); + return {close,query(prompt){assert.equal(queried,false);queried=true;process.send({type:'query'});return { + close,async initializationResult(){return process.argv[1]==="no-hook-report" ? {} : process.argv[1]==="false-hook-report" ? {hooks_applied:false} : {hooks_applied:true}}, + async interrupt(){process.send({type:'interrupt'});interrupt?.();return process.argv[1]==='unknown' ? undefined : {still_queued:process.argv[1]==='queued' ? ['not-consumed'] : []}}, + async *[Symbol.asyncIterator](){ + for await(const user of prompt){ + number++; + const text=user.message.content[0].text; + process.send({type:'input',text}); + yield {type:'system',subtype:'init',session_id:'native',tools:client ? ['mcp__functions__lookup'] : [],mcp_servers:client ? [{name:'functions',status:'connected'}] : []}; + if(client){ + if(text==='features'){ + const called=await client.callTool({name:'lookup',arguments:{text},_meta:{'claudecode/toolUseId':'same-native-call'}}); + yield {type:'user',session_id:'native',parent_tool_use_id:null,message:{content:[{type:'tool_result',tool_use_id:'same-native-call',content:called.content,is_error:called.isError}]}}; + } + for(const event of [ + {type:'message_start',message:{id:'same-native-message'}}, + {type:'content_block_start',index:0,content_block:{type:'text',text}}, + {type:'content_block_stop',index:0},{type:'message_stop'} + ])yield {type:'stream_event',parent_tool_use_id:null,session_id:'native',event}; + } + if(text==='hold') await Promise.race([new Promise(resolve=>{interrupt=resolve}),exited]); + if(options.abortController.signal.aborted)return; + yield {type:'result',uuid:'result-'+number,session_id:'native',user_message_uuids:[user.uuid],subtype:'success',is_error:false,result:'answer-'+number,usage:{input_tokens:1,output_tokens:1},modelUsage:{fixture:{inputTokens:number,outputTokens:number,costUSD:number/100}},total_cost_usd:number/100}; + interrupt=undefined; + yield {type:'command_lifecycle',uuid:user.uuid,state:'completed'}; + } + } + }}}; +}; +await import(${JSON.stringify(new URL("../dist/main.js", import.meta.url).href)}); +process.disconnect(); + +`; + +async function launch(t,mode="normal") { + const child=spawn(process.execPath,["--input-type=module","-e",fixture,mode],{stdio:["pipe","pipe","pipe","ipc"]}); + const events=[],observations=[];let buffer="",stderr=""; + child.stdout.setEncoding("utf8");child.stderr.setEncoding("utf8"); + child.stdout.on("data",data=>{buffer+=data;while(buffer.includes("\n")){const end=buffer.indexOf("\n");events.push(JSON.parse(buffer.slice(0,end)));buffer=buffer.slice(end+1)}}); + child.stderr.on("data",data=>{stderr+=data});child.on("message",event=>observations.push(event)); + const closed=new Promise(resolve=>child.once("close",(code,signal)=>resolve({code,signal}))); + t.after(async()=>{child.kill("SIGKILL");await closed}); + const wait=async predicate=>{const until=Date.now()+5000;while(!predicate()){assert.ok(Date.now()setTimeout(resolve,5))}}; + const send=value=>child.stdin.write(JSON.stringify(value)+"\n"); + const start=(id,text)=>send({type:"turn_start",turn_id:id,input:[{content:[{type:"input_text",text}]}]}); + send({type:"executor_prepare",cwd:"/tmp",model:"fixture",system_prompt:"", + ...(mode==="features" ? {observe_messages:true,functions:[{name:"lookup",description:"lookup",parameters:{type:"object",properties:{text:{type:"string"}}}}]} : {})}); + await wait(()=>events.some(event=>event.type==="executor_ready")); + assert.equal(observations.filter(event=>event.type==="input").length,0); + return {child,events,observations,closed,wait,send,start}; +} + +test("executor retains one native process and one Query over two settled Turns",{timeout:10000},async t=>{ + const {child,events,observations,closed,wait,start}=await launch(t); + for(const id of ["first","second"]){start(id,"answer");await wait(()=>events.some(event=>event.type==="turn_settled"&&event.turn_id===id));assert.equal(events.find(event=>event.type==="turn_settled"&&event.turn_id===id).reusable,true)} + assert.equal(observations.filter(event=>event.type==="native").length,1); + assert.equal(observations.filter(event=>event.type==="query").length,1); + for(const event of events.filter(event=>event.type!=="executor_ready")) assert.ok(["first","second"].includes(event.turn_id)); + assert.equal(events.filter(event=>event.type==="input_ready").length,2); + const usage=events.filter(event=>event.type==="usage"); + assert.deepEqual(usage.map(event=>[event.turn_id,event.usage.usage.input_tokens]),[["first",1],["second",1]]); + assert.deepEqual(usage.map(event=>event.usage.modelUsage.fixture.inputTokens),[1,2]); + assert.deepEqual(usage.map(event=>event.usage.total_cost_usd),[0.01,0.02]); + for(const event of usage) assert.deepEqual(event.usage.scopes,{usage:"native_turn_main_loop",modelUsage:"query_cumulative",total_cost_usd:"query_cumulative_estimate"}); + child.stdin.end();assert.deepEqual(await closed,{code:0,signal:null}); + assert.equal(observations.filter(event=>event.type==="native_closed").length,1); +}); + +test("public interrupt settles cancellation and stale cancellation cannot stop the next Turn",{timeout:10000},async t=>{ + const {child,events,observations,closed,wait,send,start}=await launch(t); + start("first","hold");await wait(()=>events.some(event=>event.type==="input_ready")); + send({type:"turn_cancel",turn_id:"first"}); + await wait(()=>events.some(event=>event.type==="turn_settled")); + assert.equal(events.find(event=>event.type==="turn_settled").reusable,true); + assert.ok(events.some(event=>event.type==="error"&&event.code==="cancelled"&&event.turn_id==="first")); + start("second","hold");await wait(()=>events.some(event=>event.type==="input_ready"&&event.turn_id==="second")); + send({type:"turn_cancel",turn_id:"first"}); + await new Promise(resolve=>setTimeout(resolve,20)); + assert.equal(observations.filter(event=>event.type==="interrupt").length,1); + send({type:"turn_cancel",turn_id:"second"}); + await wait(()=>events.some(event=>event.type==="turn_settled"&&event.turn_id==="second")); + child.stdin.end();assert.deepEqual(await closed,{code:0,signal:null}); +}); + +for(const mode of ["queued","unknown"])test(`interrupt ${mode} receipt retires native ownership before non-reusable settlement`,{timeout:10000},async t=>{ + const {events,observations,closed,wait,send,start}=await launch(t,mode); + start("first","hold");await wait(()=>events.some(event=>event.type==="input_ready")); + send({type:"turn_cancel",turn_id:"first"}); + await wait(()=>events.some(event=>event.type==="turn_settled")); + assert.equal(events.find(event=>event.type==="turn_settled").reusable,false); + assert.equal(observations.filter(event=>event.type==="native_closed").length,1); + assert.deepEqual(await closed,{code:0,signal:null}); +}); + +test("reused Turns retain functions, native receipts, steering and message boundaries", {timeout:10000}, async t => { + const {child,events,closed,wait,send,start}=await launch(t,"features"); + for(const id of ["first","second"]){ + start(id,"features"); + await wait(()=>events.some(event=>event.type==="function_call"&&event.turn_id===id)); + send({type:"steer",turn_id:id,input_id:"same-input-id",input:[{content:[{type:"input_text",text:"extra"}]}]}); + send({type:"function_result",turn_id:id,call_id:"same-native-call",delivery_id:"same-delivery-id",success:true,content:[{type:"input_text",text:"function result"}]}); + await wait(()=>events.some(event=>event.type==="turn_settled"&&event.turn_id===id)); + const own=events.filter(event=>event.turn_id===id); + assert.equal(own.find(event=>event.type==="turn_settled").reusable,true); + assert.equal(own.filter(event=>event.type==="function_applied").length,1); + assert.equal(own.find(event=>event.type==="input_applied").input_id,"same-input-id"); + assert.deepEqual(own.filter(event=>event.type==="output_message"&&event.message.status==="completed").map(event=>event.message.text),["features","extra"]); + assert.equal(own.filter(event=>event.type==="usage").length,2); + } + child.stdin.end();assert.deepEqual(await closed,{code:0,signal:null}); +}); + +for(const mode of ["no-hook-report","false-hook-report"])test("none Executor without requested hooks accepts "+mode,{timeout:10000},async t=>{ + const {child,events,closed,wait,start}=await launch(t,mode); + start("first","answer"); + await wait(()=>events.some(event=>event.type==="turn_settled")); + assert.equal(events.find(event=>event.type==="turn_settled").reusable,true); + child.stdin.end();assert.deepEqual(await closed,{code:0,signal:null}); +}); diff --git a/packages/claude-sdk-adapter/tests/usage.test.mjs b/packages/claude-sdk-adapter/tests/usage.test.mjs index 39e65a059..e1f4365cc 100644 --- a/packages/claude-sdk-adapter/tests/usage.test.mjs +++ b/packages/claude-sdk-adapter/tests/usage.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { resultUsage } from "../dist/usage.js"; +import { executorResultUsage, resultUsage } from "../dist/usage.js"; test("preserve separate SDK scopes and price provenance without recalculating", () => { const first = { type: "result", subtype: "success", is_error: false, @@ -25,3 +25,17 @@ test("reported error usage survives and missing fields are not manufactured", () const missing = JSON.parse(JSON.stringify(resultUsage({ subtype: "error_during_execution", is_error: true }))); assert.deepEqual(missing, { subtype: "error_during_execution", is_error: true }); }); + +test("executor preserves cumulative counter resets without inventing per-Turn accounting", () => { + const values = [8, 15, 0, 2].map(total => executorResultUsage({ + subtype: "success", is_error: false, usage: { input_tokens: 2 }, + modelUsage: { primary: { inputTokens: total, costUSD: total / 100 } }, + total_cost_usd: total / 100, + })); + assert.deepEqual(values.map(value => value.modelUsage.primary.inputTokens), [8, 15, 0, 2]); + assert.deepEqual(values.map(value => value.usage.input_tokens), [2, 2, 2, 2]); + for (const value of values) { + assert.equal(value.scopes.modelUsage, "query_cumulative"); + assert.equal(value.scopes.usage, "native_turn_main_loop"); + } +}); diff --git a/scripts/check-claude-sdk-runtime.mjs b/scripts/check-claude-sdk-runtime.mjs index 156f79983..960bb36de 100644 --- a/scripts/check-claude-sdk-runtime.mjs +++ b/scripts/check-claude-sdk-runtime.mjs @@ -22,8 +22,8 @@ const probe = spawnSync(process.execPath, [join(root, "dist/runtime_check.js"), assert.equal(probe.status, 0, "Exported runtime is unavailable"); const report = JSON.parse(probe.stdout); assert.equal(report.type, "runtime_ready"); -assert.equal(report.protocol, 2); -assert.deepEqual(report.features, [...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions", "workspace_structured_output"] : []), "message_images", "function_result_images", "tool_search", "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"]); +assert.equal(report.protocol, 3); +assert.deepEqual(report.features, ["executor_reuse", ...(process.platform === "linux" ? ["workspace_directory", "local_runtime_v1", "workspace_functions", "workspace_structured_output"] : []), "message_images", "function_result_images", "tool_search", "structured_output", "subagent_resources", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required", "workspace_tools", "workspace_prepare", "workspace_read", "workspace_command_observations"]); assert.equal(report.sdk, source.dependencies["@anthropic-ai/claude-agent-sdk"]); assert.equal(report.mcp, source.dependencies["@modelcontextprotocol/sdk"]); console.log(`Verified exported SDK ${report.sdk}, MCP ${report.mcp}, ${report.native}`); diff --git a/services/agents-api/internal/execution/delivery.go b/services/agents-api/internal/execution/delivery.go index 52b6d59dd..054b1bf29 100644 --- a/services/agents-api/internal/execution/delivery.go +++ b/services/agents-api/internal/execution/delivery.go @@ -3,6 +3,7 @@ package execution import ( "context" "encoding/json" + "errors" "strconv" "time" @@ -50,6 +51,8 @@ func abort(peer *gateway.Session, runID string) { } func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, peer *gateway.Session, request proto.PromptRequestPayload, first int64, prepared *preparedStart) (result Result, status string) { + changed, unsubscribeChanges := d.notifications.subscribe(tenantID, sessionID) + defer unsubscribeChanges() status = store.TurnFailed result.AppliedThrough = first subscription, err := peer.SubscribeDurable(request.RunID) @@ -80,6 +83,9 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe } }() var preparationEvents <-chan proto.Envelope + var executorRetried, nativeObserved bool + inputStarted := time.Now() + firstTextObserved := false if prepared != nil { preparationEvents = prepared.sub.Events err = prepared.start(ctx, request) @@ -92,6 +98,8 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe } ticker := time.NewTicker(250 * time.Millisecond) defer ticker.Stop() + workReady := make(chan struct{}, 1) + workReady <- struct{}{} flushTicker := time.NewTicker(100 * time.Millisecond) defer flushTicker.Stop() var pending *pendingInput @@ -118,6 +126,31 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe } started, err := prepared.started(env, request.RunID) if err != nil { + var rejection *preparationRejection + if errors.As(err, &rejection) && rejection.operation == proto.TypeExecutionStart && rejection.code == "executor_unavailable" && !executorRetried && !nativeObserved && cancelReply == nil { + // This receipt guarantees no native input was submitted and + // the previous owner was closed. Unknown delivery is never retried. + executorRetried = true + prepared.close() + currentPeer, peerErr := d.authorizedPeer(ctx, peer.DeviceID) + if peerErr != nil || currentPeer != peer { + result.ErrorCode = "device_disconnected" + return + } + replacement, prepareErr := d.prepareTurnExecutor(ctx, peer, tenantID, sessionID, request.RunID, request, store.TurnInProgress) + if prepareErr != nil { + result.ErrorCode = "executor_recovery_failed" + return + } + prepared = replacement + defer prepared.close() + preparationEvents = prepared.sub.Events + if prepared.start(ctx, request) != nil { + result.ErrorCode = "delivery_unknown" + return + } + continue + } // Cancellation owns the receipt even when it interrupts native Start. if cancelReply != nil { preparationEvents = nil @@ -165,6 +198,11 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe result.ErrorCode = "device_disconnected" return } + nativeObserved = true + if !firstTextObserved && hasText(env) { + firstTextObserved = true + recordFirstText(ctx, sessionID, request.RunID, inputStarted) + } writeErr := journal.observe(ctx, env) if result.mergeObservation(env) != nil { result.ErrorCode = "invalid_executor_result" @@ -223,6 +261,16 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe return } case <-ticker.C: + select { + case workReady <- struct{}{}: + default: + } + case <-changed: + select { + case workReady <- struct{}{}: + default: + } + case <-workReady: if !cancelSent.IsZero() { if time.Since(cancelSent) > 15*time.Second { result.ErrorCode = "cancel_unconfirmed" @@ -233,6 +281,9 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe turn, err := d.Store.GetTurn(ctx, tenantID, sessionID, request.RunID) if err != nil { result.ErrorCode = "execution_state_unavailable" + if ctx.Err() != nil { + result.ErrorCode = "execution_interrupted" + } return } if turn.Status != store.TurnInProgress && turn.Status != store.TurnWaiting { @@ -255,6 +306,9 @@ func (d *Dispatcher) deliver(ctx context.Context, tenantID, sessionID string, pe inputs, err := d.Store.ListTurnInputs(ctx, tenantID, sessionID, request.RunID, result.AppliedThrough, 1) if err != nil { result.ErrorCode = "execution_state_unavailable" + if ctx.Err() != nil { + result.ErrorCode = "execution_interrupted" + } return } if len(inputs) == 0 { diff --git a/services/agents-api/internal/execution/directory_preparation.go b/services/agents-api/internal/execution/directory_preparation.go index 6d4aca014..a5c64a2f9 100644 --- a/services/agents-api/internal/execution/directory_preparation.go +++ b/services/agents-api/internal/execution/directory_preparation.go @@ -42,7 +42,7 @@ func (d *Dispatcher) withPreparedWorkspace(owner context.Context, peer *gateway. } }() prepare, stop := context.WithTimeout(owner, 10*time.Second) - err = send(prepare, peer, proto.TypeExecutionPrepare, prepared.requestID, proto.ExecutionPreparePayload{Configuration: req}) + err = send(prepare, peer, proto.TypeExecutionPrepare, prepared.requestID, proto.ExecutionPreparePayload{SessionID: session.ID, Configuration: req}) if err == nil { err = prepared.awaitDirectoryReady(prepare) } diff --git a/services/agents-api/internal/execution/dispatcher.go b/services/agents-api/internal/execution/dispatcher.go index 11ec43c75..7ca3ab041 100644 --- a/services/agents-api/internal/execution/dispatcher.go +++ b/services/agents-api/internal/execution/dispatcher.go @@ -29,6 +29,7 @@ type DaemonConfig struct { } type Dispatcher struct { + notifications *executionNotifications Policy Store *store.Store Registry *gateway.Registry @@ -89,12 +90,17 @@ func (d *Dispatcher) Run(ctx context.Context, tenantID, sessionID, turnID string if err != nil { return store.Turn{}, err } + req.WorkDir, req.DisableExecutionEnvironment = workDir, noEnvironment + prepared, err := d.prepareTurnExecutor(ctx, peer, tenantID, sessionID, turnID, req, store.TurnQueued) + if err != nil { + return store.Turn{}, err + } + defer prepared.close() if _, err := d.Store.TransitionTurn(ctx, tenantID, sessionID, turnID, store.TurnTransition{ExpectedStatus: store.TurnQueued, Status: store.TurnInProgress}); err != nil { return store.Turn{}, err } req.ConversationID, req.RunID, req.Input = sessionID, turnID, text - req.WorkDir, req.DisableExecutionEnvironment = workDir, noEnvironment - result, status := d.deliver(ctx, tenantID, sessionID, peer, req, through, nil) + result, status := d.deliver(ctx, tenantID, sessionID, peer, req, through, prepared) return d.finishRun(tenantID, sessionID, turnID, snapshot.Agent.Model, result, status) } diff --git a/services/agents-api/internal/execution/environment_admission.go b/services/agents-api/internal/execution/environment_admission.go index b915e37af..da0da4a1c 100644 --- a/services/agents-api/internal/execution/environment_admission.go +++ b/services/agents-api/internal/execution/environment_admission.go @@ -85,7 +85,7 @@ func (w *Worker) submitEnvironmentInputs(ctx context.Context, session store.Sess } if (kind == "cancel" || kind == "tool_result") && !slices.ContainsFunc(inputs, func(input store.Input) bool { return input.Kind != kind }) { // Neither kind creates a Turn. The Session lock preserves target and retry identity. - return w.admission.SubmitInputs(ctx, session.TenantID, session.ID, key, inputs) + return w.admitInputs(ctx, session.TenantID, session.ID, key, inputs) } // Messages start work. A Session from before deployment defaults moved into // Core may have no frozen provider; reject it here instead of queueing work @@ -94,12 +94,15 @@ func (w *Worker) submitEnvironmentInputs(ctx context.Context, session store.Sess if json.Unmarshal(session.Configuration, &snapshot) != nil || !snapshot.ModelProviderConfigured { return nil, store.ErrModelProviderRequired } + changed, unsubscribe := w.dispatcher.notifications.subscribe(session.TenantID, session.ID) + defer unsubscribe() reserve, cancel := context.WithTimeout(ctx, 5*time.Second) reservation, err := w.admission.ReserveEnvironmentInput(reserve, session.TenantID, session.ID, key, inputs) cancel() if err != nil { return nil, err } + w.wakeScheduler() if reservation.State == store.EnvironmentInputPending && !reservation.IsInitial { w.hintRuntimeWake(ctx, session) } @@ -120,6 +123,7 @@ func (w *Worker) submitEnvironmentInputs(ctx context.Context, session store.Sess case <-ctx.Done(): return nil, ctx.Err() case <-ticker.C: + case <-changed: } if err := w.checkAdmissionOwnership(ctx); err != nil { return nil, err diff --git a/services/agents-api/internal/execution/executor_observation.go b/services/agents-api/internal/execution/executor_observation.go new file mode 100644 index 000000000..12eec8293 --- /dev/null +++ b/services/agents-api/internal/execution/executor_observation.go @@ -0,0 +1,41 @@ +package execution + +import ( + "context" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + obslog "github.com/MiniMax-AI-Dev/parsar/internal/obs/log" +) + +// These durations use one Core process monotonic clock. They describe control +// readiness and input-to-first-text latency, not model-only or native tool time. +func recordExecutorReadiness(p *preparedStart, status proto.PreparationStatusPayload) { + obslog.Bg().Info("executor ready", "request_id", p.requestID, + "executor_id", status.ExecutorID, "reused", status.Reused, + "control_ready_ms", time.Since(p.createdAt).Milliseconds()) +} + +// The started receipt confirms adapter ownership, not model consumption. +func recordExecutorStart(p *preparedStart, turn string) { + obslog.Bg().Info("executor start acknowledged", "turn_id", turn, + "request_id", p.requestID, "executor_id", p.executorID, + "start_control_ms", time.Since(p.startSentAt).Milliseconds()) +} + +func recordFirstText(ctx context.Context, session, turn string, submitted time.Time) { + obslog.Bg().InfoContext(ctx, "executor first text", "session_id", session, + "turn_id", turn, "input_to_first_text_ms", time.Since(submitted).Milliseconds()) +} + +func hasText(env proto.Envelope) bool { + switch env.Type { + case proto.TypeDelta: + var text proto.DeltaPayload + return env.DecodePayload(&text) == nil && text.Delta != "" + case proto.TypeOutputMessage: + var text proto.OutputMessagePayload + return env.DecodePayload(&text) == nil && text.Text != nil && *text.Text != "" + } + return false +} diff --git a/services/agents-api/internal/execution/executor_preparation.go b/services/agents-api/internal/execution/executor_preparation.go new file mode 100644 index 000000000..11078598b --- /dev/null +++ b/services/agents-api/internal/execution/executor_preparation.go @@ -0,0 +1,63 @@ +package execution + +import ( + "context" + "errors" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/gateway" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +// prepareTurnExecutor reserves one admission on the Runtime-owned Executor. +// Core does not cache native ownership. A replacement requires the Runtime to +// confirm cleanup and recover the exact Session history before returning ready. +func (d *Dispatcher) prepareTurnExecutor(ctx context.Context, peer *gateway.Session, tenant, session, turn string, request proto.PromptRequestPayload, expectedStatus string) (*preparedStart, error) { + prepared, err := newPreparedStart(peer) + if err != nil { + return nil, err + } + request.RunID, request.ConversationID, request.Input = "", "", nil + if err = send(ctx, peer, proto.TypeExecutionPrepare, prepared.requestID, proto.ExecutionPreparePayload{SessionID: session, Configuration: request}); err == nil { + err = d.awaitTurnExecutor(ctx, tenant, session, turn, expectedStatus, prepared) + } + if err != nil { + prepared.close() + return nil, err + } + return prepared, nil +} + +func (d *Dispatcher) awaitTurnExecutor(ctx context.Context, tenant, session, turn, expectedStatus string, prepared *preparedStart) error { + tick := time.NewTicker(250 * time.Millisecond) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-tick.C: + current, err := d.Store.GetTurn(ctx, tenant, session, turn) + if err != nil { + return err + } + if current.Status != expectedStatus || !current.CancelRequestedAt.IsZero() { + return store.ErrTurnConflict + } + case env, ok := <-prepared.sub.Events: + if !ok { + return errors.New("executor preparation control stream closed") + } + status, err := prepared.observation(env) + if err != nil { + return err + } + if status.State == "ready" && status.RunID == "" && status.ExecutorID != "" { + return nil + } + if status.State != "preparing" { + return errors.New("unexpected executor state before Turn start") + } + } + } +} diff --git a/services/agents-api/internal/execution/notifications.go b/services/agents-api/internal/execution/notifications.go new file mode 100644 index 000000000..830f36859 --- /dev/null +++ b/services/agents-api/internal/execution/notifications.go @@ -0,0 +1,50 @@ +package execution + +import "sync" + +// executionNotifications carries process-local hints after durable commits. +// Every consumer rechecks storage and retains polling for external changes. +// Subscriptions exist only while a caller is waiting or a Turn is active. +type executionNotifications struct { + mu sync.Mutex + waiters map[string]map[chan struct{}]struct{} +} + +func (n *executionNotifications) subscribe(tenant, session string) (<-chan struct{}, func()) { + if n == nil { + return nil, func() {} + } + key := tenant + ":" + session + ch := make(chan struct{}, 1) + n.mu.Lock() + if n.waiters == nil { + n.waiters = make(map[string]map[chan struct{}]struct{}) + } + if n.waiters[key] == nil { + n.waiters[key] = make(map[chan struct{}]struct{}) + } + n.waiters[key][ch] = struct{}{} + n.mu.Unlock() + return ch, func() { + n.mu.Lock() + defer n.mu.Unlock() + delete(n.waiters[key], ch) + if len(n.waiters[key]) == 0 { + delete(n.waiters, key) + } + } +} + +func (n *executionNotifications) notify(tenant, session string) { + if n == nil { + return + } + n.mu.Lock() + defer n.mu.Unlock() + for ch := range n.waiters[tenant+":"+session] { + select { + case ch <- struct{}{}: + default: + } + } +} diff --git a/services/agents-api/internal/execution/notifications_test.go b/services/agents-api/internal/execution/notifications_test.go new file mode 100644 index 000000000..b39588fc9 --- /dev/null +++ b/services/agents-api/internal/execution/notifications_test.go @@ -0,0 +1,50 @@ +package execution + +import ( + "sync" + "testing" +) + +func TestExecutionNotificationsKeepCommitHintsAndIsolateSessions(t *testing.T) { + notifications := &executionNotifications{} + one, leaveOne := notifications.subscribe("tenant", "one") + other, leaveOther := notifications.subscribe("other-tenant", "one") + defer leaveOther() + var writers sync.WaitGroup + for range 20 { + writers.Go(func() { notifications.notify("tenant", "one") }) + } + writers.Wait() + select { + case <-one: + default: + t.Fatal("committed change was lost") + } + select { + case <-other: + t.Fatal("hint crossed tenant ownership") + default: + } + select { + case <-one: + t.Fatal("hints did not coalesce") + default: + } + notifications.notify("tenant", "one") + select { + case <-one: + default: + t.Fatal("next commit was lost") + } + leaveOne() + notifications.notify("tenant", "one") + select { + case <-one: + t.Fatal("detached observer received a hint") + default: + } + leaveOther() + if len(notifications.waiters) != 0 { + t.Fatal("subscriptions leaked") + } +} diff --git a/services/agents-api/internal/execution/preparation.go b/services/agents-api/internal/execution/preparation.go index 8f1706d3d..f33db9a38 100644 --- a/services/agents-api/internal/execution/preparation.go +++ b/services/agents-api/internal/execution/preparation.go @@ -11,11 +11,23 @@ import ( "github.com/google/uuid" ) +type preparationRejection struct { + code string + operation string +} + +func (e *preparationRejection) Error() string { return "preparation control rejected: " + e.code } + type preparedStart struct { - peer *gateway.Session - requestID string - handle string - sub *gateway.Subscription + createdAt time.Time + startSentAt time.Time + startObserved bool + readyObserved bool + peer *gateway.Session + requestID string + handle string + executorID string + sub *gateway.Subscription } func newPreparedStart(peer *gateway.Session) (*preparedStart, error) { @@ -24,7 +36,7 @@ func newPreparedStart(peer *gateway.Session) (*preparedStart, error) { if err != nil { return nil, err } - return &preparedStart{peer: peer, requestID: id, sub: sub}, nil + return &preparedStart{peer: peer, requestID: id, sub: sub, createdAt: time.Now()}, nil } func (p *preparedStart) close() { @@ -45,7 +57,14 @@ func (p *preparedStart) controlStatus(env proto.Envelope) (proto.PreparationStat if status.Handle == "" || status.Revision == 0 || (p.handle != "" && p.handle != status.Handle) { return status, errors.New("preparation identity changed") } - p.handle = status.Handle + if p.executorID != "" && status.ExecutorID != p.executorID { + return status, errors.New("executor identity changed") + } + p.handle, p.executorID = status.Handle, status.ExecutorID + if status.State == "ready" && status.ExecutorID != "" && !p.readyObserved { + p.readyObserved = true + recordExecutorReadiness(p, status) + } return status, nil } @@ -55,7 +74,7 @@ func (p *preparedStart) observation(env proto.Envelope) (proto.PreparationStatus return status, err } if status.State == "rejected" { - return status, errors.New("preparation control rejected") + return status, &preparationRejection{code: status.ErrorCode, operation: status.Operation} } switch status.State { case "preparing", "ready", "starting", "started": @@ -85,7 +104,7 @@ func (d *Dispatcher) awaitPreparation(ctx context.Context, tenant, session strin if err != nil { return pending, err } - if status.State == "ready" && status.RunID == "" { + if status.State == "ready" && status.RunID == "" && status.ExecutorID != "" { return pending, nil } if status.State != "preparing" { @@ -96,7 +115,8 @@ func (d *Dispatcher) awaitPreparation(ctx context.Context, tenant, session strin } func (p *preparedStart) start(ctx context.Context, request proto.PromptRequestPayload) error { - return send(ctx, p.peer, proto.TypeExecutionStart, p.requestID, proto.ExecutionStartPayload{Handle: p.handle, RunID: request.RunID, Input: request.Input}) + p.startSentAt = time.Now() + return send(ctx, p.peer, proto.TypeExecutionStart, p.requestID, proto.ExecutionStartPayload{Handle: p.handle, ExecutorID: p.executorID, RunID: request.RunID, Input: request.Input}) } func (p *preparedStart) started(env proto.Envelope, runID string) (bool, error) { @@ -107,5 +127,9 @@ func (p *preparedStart) started(env proto.Envelope, runID string) (bool, error) if status.RunID != runID || (status.State != "starting" && status.State != "started") { return false, errors.New("unexpected preparation state after admission") } + if status.State == "started" && !p.startObserved { + p.startObserved = true + recordExecutorStart(p, runID) + } return status.State == "started", nil } diff --git a/services/agents-api/internal/execution/prepared_dispatch.go b/services/agents-api/internal/execution/prepared_dispatch.go index 453d85745..c480de5c8 100644 --- a/services/agents-api/internal/execution/prepared_dispatch.go +++ b/services/agents-api/internal/execution/prepared_dispatch.go @@ -16,7 +16,7 @@ type EnvironmentRun struct { Turn store.Turn } -// RunEnvironmentInput owns a private preparation through its first admitted Run. +// RunEnvironmentInput reserves a Turn on the Session-owned Runtime Executor. func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, tenantID, sessionID, reservationID string) (run EnvironmentRun, err error) { if err = d.Store.CheckExecutionOwnership(ctx); err != nil { return run, err @@ -81,7 +81,7 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, tenantID, sessionI return run, err } defer prepared.close() - if err = send(owner, peer, proto.TypeExecutionPrepare, prepared.requestID, proto.ExecutionPreparePayload{Configuration: req}); err != nil { + if err = send(owner, peer, proto.TypeExecutionPrepare, prepared.requestID, proto.ExecutionPreparePayload{SessionID: sessionID, Configuration: req}); err != nil { return run, err } run.Reservation, err = d.awaitPreparation(owner, tenantID, sessionID, run.Reservation, prepared) @@ -96,6 +96,9 @@ func (d *Dispatcher) RunEnvironmentInput(ctx context.Context, tenantID, sessionI // A rejected claim leaves the reservation pending for a later attempt. return run, err } + if err == nil { + d.notifications.notify(tenantID, sessionID) + } run.Reservation = promoted if err != nil || run.Reservation.State != store.EnvironmentInputAdmitted { return run, err diff --git a/services/agents-api/internal/execution/request.go b/services/agents-api/internal/execution/request.go index 8d398e622..000435f17 100644 --- a/services/agents-api/internal/execution/request.go +++ b/services/agents-api/internal/execution/request.go @@ -52,7 +52,7 @@ func (d *Dispatcher) executionRequest(ctx context.Context, session store.Session } request := proto.PromptRequestPayload{AgentKind: session.Engine, FunctionTools: tools.Functions, ToolSearch: tools.Search, AgentOptions: options, ExecutionControls: controls, AgentStateKey: "agents-api-" + session.ID, - AgentSessionID: bound.NativeSessionID, ReleaseOnCompletion: true, StrictResume: true, + AgentSessionID: bound.NativeSessionID, StrictResume: true, RequireExistingNativeSession: recoverNativeSession, ObserveMessages: caps.MessageItems, ObserveToolObservations: true, ObserveSubagentIdentities: snapshot.Agent.MultiAgent.Enabled, diff --git a/services/agents-api/internal/execution/support.go b/services/agents-api/internal/execution/support.go index c00dbc5cf..43798f733 100644 --- a/services/agents-api/internal/execution/support.go +++ b/services/agents-api/internal/execution/support.go @@ -86,6 +86,9 @@ func (p Policy) engineCapabilities(peer *gateway.Session, engine string, snapsho if !known || !found || !info.Available || !caps.Streaming || !caps.Steering || !caps.DurableTurns || !caps.DurableInputReceipts { return fail("device must advertise streaming, steering and durable turns for this engine") } + if !caps.Preparation { + return fail("device must advertise executor preparation") + } if !caps.ExecutionControls { return fail("device must advertise execution_controls") } diff --git a/services/agents-api/internal/execution/worker.go b/services/agents-api/internal/execution/worker.go index cbdd976d7..41eec56b3 100644 --- a/services/agents-api/internal/execution/worker.go +++ b/services/agents-api/internal/execution/worker.go @@ -22,6 +22,7 @@ type Worker struct { lease *store.ExecutionLease directoryReads chan directoryReadRequest fileWrites chan fileWriteRequest + scheduleWake chan struct{} stopped chan struct{} stopOnce sync.Once runtimes *runtimeManager @@ -38,7 +39,8 @@ func StartWorker(ctx context.Context, dispatcher *Dispatcher) (*Worker, error) { } owned := *dispatcher owned.Store = lease.Store() - worker := &Worker{concurrency: dispatcher.MaxConcurrentExecutions, dispatcher: &owned, admission: dispatcher.Store, lease: lease, directoryReads: make(chan directoryReadRequest), fileWrites: make(chan fileWriteRequest), stopped: make(chan struct{}), enrolledConnections: make(map[string]*runtimeConnection)} + owned.notifications = &executionNotifications{} + worker := &Worker{concurrency: dispatcher.MaxConcurrentExecutions, dispatcher: &owned, admission: dispatcher.Store, lease: lease, directoryReads: make(chan directoryReadRequest), fileWrites: make(chan fileWriteRequest), stopped: make(chan struct{}), scheduleWake: make(chan struct{}, 1), enrolledConnections: make(map[string]*runtimeConnection)} worker.runtimes, err = newRuntimeManager(owned.Store, owned.Registry, owned.ManagedRuntimes) if err != nil { _ = lease.Close(context.Background()) @@ -105,7 +107,7 @@ func (w *Worker) SubmitInputs(ctx context.Context, tenant, session, key string, if !w.dispatcher.canAdmitInputs(value.Engine, value.Configuration) { return nil, store.ErrInvalidInput } - return w.admission.SubmitInputs(ctx, tenant, session, key, inputs) + return w.admitInputs(ctx, tenant, session, key, inputs) } // CreateSession validates execution support before reserving or admitting initial work. @@ -113,7 +115,11 @@ func (w *Worker) CreateSession(ctx context.Context, tenant string, input store.C if err := w.validateCreation(ctx, input); err != nil { return store.Session{}, err } - return w.admission.CreateSession(ctx, tenant, input) + session, err := w.admission.CreateSession(ctx, tenant, input) + if err == nil && len(input.InitialInputs) > 0 { + w.wakeScheduler() + } + return session, err } // CreateSessionStream applies the same execution admission before creating a stream. @@ -121,7 +127,11 @@ func (w *Worker) CreateSessionStream(ctx context.Context, tenant string, input s if err := w.validateCreation(ctx, input); err != nil { return store.SessionCreation{}, err } - return w.admission.CreateSessionStream(ctx, tenant, input) + creation, err := w.admission.CreateSessionStream(ctx, tenant, input) + if err == nil && len(input.InitialInputs) > 0 { + w.wakeScheduler() + } + return creation, err } // Run retains queued work across restarts, but never replays an uncertain claim. @@ -174,7 +184,12 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { ticker := time.NewTicker(250 * time.Millisecond) defer ticker.Stop() schedule := workerSchedule{} + // A hint can encounter occupied slots or a preceding execution finishing + // on the same Session. Allow one immediate retry when a slot is released; + // failed preparations still wait for polling instead of spinning. + rescanOnCompletion := false for { + maintenance := false select { case <-ctx.Done(): return ctx.Err() @@ -192,10 +207,15 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { defer running.Done() writesCompleted <- writeCompletion{request: request, result: w.runFileWrite(ctx, request)} }() + continue case write := <-writesCompleted: delete(active, write.request.environment.SessionID) w.observeSlots(len(active)) write.request.result <- write.result + if !rescanOnCompletion { + continue + } + rescanOnCompletion = false case request := <-w.directoryReads: if request.ctx.Err() != nil || reads == w.executionConcurrency() || (!active[request.environment.SessionID] && len(active) == w.executionConcurrency()) { request.reply(directoryReadResult{err: ErrExecutionUnavailable}) @@ -217,6 +237,7 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { } readsCompleted <- readCompletion{id: id, request: request, result: result} }() + continue case read := <-readsCompleted: reads-- if read.id != "" { @@ -224,20 +245,33 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { w.observeSlots(len(active)) } read.request.reply(read.result) + if read.id == "" || !rescanOnCompletion { + continue + } + rescanOnCompletion = false case result := <-completed: delete(active, result.id) w.observeSlots(len(active)) if result.err != nil { return result.err } - case <-ticker.C: - check, stop := context.WithTimeout(ctx, 5*time.Second) - err := w.CheckOwnership(check) - stop() - if err != nil { - w.observeSchedulerPoll(0, err) - return err + if !rescanOnCompletion { + continue } + rescanOnCompletion = false + case <-w.scheduleWake: + rescanOnCompletion = true + case <-ticker.C: + maintenance = true + } + check, stop := context.WithTimeout(ctx, 5*time.Second) + err := w.CheckOwnership(check) + stop() + if err != nil { + w.observeSchedulerPoll(0, err) + return err + } + if maintenance { if _, err := w.dispatcher.Store.ExpireEnvironmentInputs(ctx); err != nil { w.observeSchedulerPoll(0, err) return err @@ -246,35 +280,40 @@ func (w *Worker) Run(ctx context.Context) (runErr error) { w.observeSchedulerPoll(0, err) return err } - if len(active) == w.executionConcurrency() { - w.observeSchedulerPoll(0, nil) - continue - } - devices := w.dispatcher.Registry.Devices() - if len(devices) == 0 { - w.observeSchedulerPoll(0, nil) - continue - } - work, err := schedule.selectWork(ctx, w, devices, active) - w.observeSlots(len(active)) - if err != nil { - w.observeSchedulerPoll(0, err) - return err - } - w.observeSchedulerPoll(len(work), nil) - for _, item := range work { - running.Add(1) - go func() { - defer running.Done() - var err error - if item.reservationID != "" { - err = w.runEnvironmentInput(ctx, item) - } else { - err = w.runClaim(ctx, item.ExecutionWork) - } - completed <- completion{id: item.SessionID, err: err} - }() - } + } + if len(active) == w.executionConcurrency() { + w.observeSchedulerPoll(0, nil) + continue + } + devices := w.dispatcher.Registry.Devices() + if len(devices) == 0 { + w.observeSchedulerPoll(0, nil) + continue + } + if !maintenance { + schedule.nextEnvironmentScan = time.Time{} + } + work, err := schedule.selectWork(ctx, w, devices, active) + w.observeSlots(len(active)) + if err != nil { + w.observeSchedulerPoll(0, err) + return err + } + w.observeSchedulerPoll(len(work), nil) + rescanOnCompletion = rescanOnCompletion && (len(active) > len(work) || len(active) == w.executionConcurrency()) + for _, item := range work { + running.Add(1) + go func() { + defer running.Done() + var err error + if item.reservationID != "" { + err = w.runEnvironmentInput(ctx, item) + } else { + err = w.runClaim(ctx, item.ExecutionWork) + } + w.dispatcher.notifications.notify(item.TenantID, item.SessionID) + completed <- completion{id: item.SessionID, err: err} + }() } } } diff --git a/services/agents-api/internal/execution/worker_schedule.go b/services/agents-api/internal/execution/worker_schedule.go index 3e5321483..fbb932320 100644 --- a/services/agents-api/internal/execution/worker_schedule.go +++ b/services/agents-api/internal/execution/worker_schedule.go @@ -25,8 +25,12 @@ func (s *workerSchedule) selectWork(ctx context.Context, w *Worker, devices []st if err != nil { return nil, err } - if len(turns) == 0 { + if len(turns) == 0 && s.turnCursor != "" { s.turnCursor = "" + turns, err = w.dispatcher.Store.ListExecutionWork(ctx, "", []string{store.TurnQueued}, devices) + if err != nil { + return nil, err + } } var environments []store.EnvironmentInputWork if !time.Now().Before(s.nextEnvironmentScan) { diff --git a/services/agents-api/internal/execution/worker_wakeup.go b/services/agents-api/internal/execution/worker_wakeup.go new file mode 100644 index 000000000..fa5539189 --- /dev/null +++ b/services/agents-api/internal/execution/worker_wakeup.go @@ -0,0 +1,25 @@ +package execution + +import ( + "context" + + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +// wakeScheduler only hints at committed work. The existing loop retains lease, +// capacity and Session ownership; polling recovers absent or coalesced hints. +func (w *Worker) wakeScheduler() { + select { + case w.scheduleWake <- struct{}{}: + default: + } +} + +func (w *Worker) admitInputs(ctx context.Context, tenant, session, key string, inputs []store.Input) ([]store.InputReceipt, error) { + receipts, err := w.admission.SubmitInputs(ctx, tenant, session, key, inputs) + if err == nil { + w.wakeScheduler() + w.dispatcher.notifications.notify(tenant, session) + } + return receipts, err +} diff --git a/services/agents-api/internal/execution/worker_wakeup_test.go b/services/agents-api/internal/execution/worker_wakeup_test.go new file mode 100644 index 000000000..8a0a48f15 --- /dev/null +++ b/services/agents-api/internal/execution/worker_wakeup_test.go @@ -0,0 +1,35 @@ +package execution + +import ( + "sync" + "testing" +) + +func TestSchedulerWakeCoalescesConcurrentAdmissionsAndKeepsNextHint(t *testing.T) { + worker := &Worker{scheduleWake: make(chan struct{}, 1)} + var callers sync.WaitGroup + for range 100 { + callers.Go(func() { + for range 100 { + worker.wakeScheduler() + } + }) + } + callers.Wait() + if got := len(worker.scheduleWake); got != 1 { + t.Fatalf("queued wakeups = %d, want one", got) + } + <-worker.scheduleWake + // A commit while a previous scan is running needs a subsequent scan. + worker.wakeScheduler() + select { + case <-worker.scheduleWake: + default: + t.Fatal("admission during a scan lost its wakeup") + } + select { + case <-worker.scheduleWake: + t.Fatal("coalesced submissions caused an extra scan") + default: + } +} diff --git a/services/agents-api/internal/store/claude_execution_test.go b/services/agents-api/internal/store/claude_execution_test.go index f6eb75005..deca0c94c 100644 --- a/services/agents-api/internal/store/claude_execution_test.go +++ b/services/agents-api/internal/store/claude_execution_test.go @@ -30,7 +30,7 @@ func claudeSession(t *testing.T, h *dispatchHarness, configuration string, prebo func claudeHeartbeat(t *testing.T, h *dispatchHarness, ready bool) { t.Helper() - caps := proto.AgentKindCapabilities{Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: ready, ExecutionControls: true, EnvironmentNone: true, SubagentControl: true, FunctionTools: true, ToolObservations: true} + caps := proto.AgentKindCapabilities{Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: ready, ExecutionControls: true, EnvironmentNone: true, SubagentControl: true, FunctionTools: true, ToolObservations: true, Preparation: true} h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "claude_sdk", Available: true, Capabilities: caps}}}) deadline := time.Now().Add(3 * time.Second) for { @@ -85,7 +85,7 @@ func TestClaudeWorkerSelectsStoredEngineAndRestrictiveCapabilities(t *testing.T) queued() // Durable application receipts are required for this engine too. claudeHeartbeat(t, h, true) var prompt proto.PromptRequestPayload - if h.read(proto.TypePromptRequest).DecodePayload(&prompt) != nil { + if h.read(testExecutionRequest).DecodePayload(&prompt) != nil { t.Fatal("invalid prompt") } if prompt.AgentKind != "claude_sdk" || len(prompt.FunctionTools) != 1 || !prompt.DisableExecutionEnvironment || !prompt.DisableSubagents || prompt.ExecutionControls == nil || *prompt.ExecutionControls != (proto.ExecutionControls{WebSearch: "disabled", TextVerbosity: "medium"}) { diff --git a/services/agents-api/internal/store/claude_mcp_test.go b/services/agents-api/internal/store/claude_mcp_test.go index 87a7bbc7d..c09fed51c 100644 --- a/services/agents-api/internal/store/claude_mcp_test.go +++ b/services/agents-api/internal/store/claude_mcp_test.go @@ -28,7 +28,7 @@ func TestClaudeMCPWaitsForCapableRuntime(t *testing.T) { } claudeSession(t, h, configuration, prebound) // Base MCP support does not imply authentication or required initialization. - caps := proto.AgentKindCapabilities{Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: true, ExecutionControls: true, EnvironmentNone: true, SubagentControl: true, ToolObservations: true, MCPHTTPTools: authenticated || required} + caps := proto.AgentKindCapabilities{Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: true, ExecutionControls: true, EnvironmentNone: true, SubagentControl: true, ToolObservations: true, MCPHTTPTools: authenticated || required, Preparation: true} h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "claude_sdk", Available: true, Capabilities: caps}}}) awaitDaemonRemoteCondition(t, t.Context(), 3*time.Second, "Claude MCP heartbeat", func() bool { peer, _ := h.registry.LookupDevice(h.device.ID) @@ -71,7 +71,7 @@ func TestClaudeMCPWaitsForCapableRuntime(t *testing.T) { caps.MCPHTTPRequired = required h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "claude_sdk", Available: true, Capabilities: caps}}}) var prompt proto.PromptRequestPayload - if h.read(proto.TypePromptRequest).DecodePayload(&prompt) != nil || prompt.AgentKind != "claude_sdk" || prompt.MCPHTTPServers == nil || len(*prompt.MCPHTTPServers) != 1 { + if h.read(testExecutionRequest).DecodePayload(&prompt) != nil || prompt.AgentKind != "claude_sdk" || prompt.MCPHTTPServers == nil || len(*prompt.MCPHTTPServers) != 1 { t.Fatal("missing typed MCP dispatch") } server := (*prompt.MCPHTTPServers)[0] @@ -104,7 +104,7 @@ func TestClaudeMCPUnsupportedSnapshotRejectedBeforeClaim(t *testing.T) { configuration = strings.Replace(configuration, `"tickets"`, `"functions"`, 1) } claudeSession(t, h, configuration, true) - caps := proto.AgentKindCapabilities{Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: true, ExecutionControls: true, EnvironmentNone: true, SubagentControl: true, ToolObservations: true, MCPHTTPTools: true, MCPHTTPRequired: true, MCPHTTPBearerAuth: true} + caps := proto.AgentKindCapabilities{Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: true, ExecutionControls: true, EnvironmentNone: true, SubagentControl: true, ToolObservations: true, MCPHTTPTools: true, MCPHTTPRequired: true, MCPHTTPBearerAuth: true, Preparation: true} if profile == "missing required capability" { caps.MCPHTTPRequired = false } diff --git a/services/agents-api/internal/store/command_output_test.go b/services/agents-api/internal/store/command_output_test.go index ff5d27964..34b7889bb 100644 --- a/services/agents-api/internal/store/command_output_test.go +++ b/services/agents-api/internal/store/command_output_test.go @@ -131,7 +131,7 @@ func TestExecutionJournalsCommandOutputBeforeCancellation(t *testing.T) { ctx := context.Background() input := h.message("command", "run a command") result := h.run(ctx, input.TurnID) - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) h.write(input.TurnID, proto.TypeToolCall, proto.ToolCallPayload{ID: "cmd", Stage: "before", Observation: &proto.ToolObservation{Kind: "command", Command: "wait", Status: "in_progress"}}) h.write(input.TurnID, proto.TypeCommandOutput, proto.CommandOutputPayload{ID: "cmd", Delta: "partial"}) if _, err := h.s.RequestCancel(ctx, h.tenant, h.session.ID, "cancel"); err != nil { diff --git a/services/agents-api/internal/store/dispatch_test.go b/services/agents-api/internal/store/dispatch_test.go index ed371e923..e68ff6213 100644 --- a/services/agents-api/internal/store/dispatch_test.go +++ b/services/agents-api/internal/store/dispatch_test.go @@ -22,6 +22,8 @@ import ( ) type dispatchHarness struct { + writeMu sync.Mutex + admissions map[string]fixtureAdmission t *testing.T s *store.Store d *execution.Dispatcher @@ -92,7 +94,7 @@ func newDispatchHarnessForSession(t *testing.T, configuration []byte, local bool t.Fatal("device connection failed") } t.Cleanup(func() { h.conn.Close() }) - h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{Streaming: true, Steering: true, Resume: true, DurableTurns: true, DurableInputReceipts: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: true, SubagentControl: true, SubagentObservations: true, ToolObservations: true, NativeSessionRecovery: true}}}}) + h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{Streaming: true, Steering: true, Resume: true, DurableTurns: true, DurableInputReceipts: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: true, SubagentControl: true, SubagentObservations: true, ToolObservations: true, NativeSessionRecovery: true, Preparation: true}}}}) deadline := time.Now().Add(3 * time.Second) for { peer, e := h.registry.LookupDevice(h.device.ID) @@ -122,6 +124,12 @@ func (h *dispatchHarness) message(key, text string) store.InputReceipt { func (h *dispatchHarness) write(run, kind string, payload any) { h.t.Helper() + h.writeMu.Lock() + defer h.writeMu.Unlock() + if status, ok := payload.(proto.PreparationStatusPayload); ok && status.Handle != "" && status.ExecutorID == "" { + status.ExecutorID = "executor-" + status.Handle + payload = status + } env, err := proto.NewEnvelope(kind, run, payload) if err != nil { h.t.Fatal(err) @@ -139,6 +147,13 @@ func (h *dispatchHarness) read(kind string) proto.Envelope { if err := h.conn.ReadJSON(&env); err != nil { h.t.Fatal(err) } + if kind == testExecutionRequest { + var keep bool + env, keep = h.executionFrame(env) + if !keep { + continue + } + } if env.Type == kind { return env } @@ -175,7 +190,7 @@ func TestExecutionDispatchSteeringAndNativeContinuity(t *testing.T) { ctx := context.Background() first := h.message("first", "Initial input") result := h.run(ctx, first.TurnID) - request := h.read(proto.TypePromptRequest) + request := h.read(testExecutionRequest) var prompt proto.PromptRequestPayload _ = request.DecodePayload(&prompt) if inputTextForTest(t, prompt.Input) != "Initial input" || prompt.ConversationID != h.session.ID || prompt.AgentOptions["model"] != "test-model" || prompt.AgentOptions["system_prompt"] != "Keep this instruction." { @@ -217,7 +232,7 @@ func TestExecutionDispatchSteeringAndNativeContinuity(t *testing.T) { } next := h.message("third", "Continue the session") result = h.run(ctx, next.TurnID) - request = h.read(proto.TypePromptRequest) + request = h.read(testExecutionRequest) _ = request.DecodePayload(&prompt) if prompt.AgentSessionID != "native-thread-1" || prompt.AgentStateKey != "agents-api-"+h.session.ID { t.Fatal("native continuity lost") @@ -232,7 +247,7 @@ func TestExecutionCancellationRequiresReceiptAndSurvivesContextEnd(t *testing.T) h := newDispatchHarness(t) first := h.message("first", "Run") result := h.run(context.Background(), first.TurnID) - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) _, err := h.s.RequestCancel(context.Background(), h.tenant, h.session.ID, "cancel") if err != nil { t.Fatal(err) @@ -255,7 +270,7 @@ func TestExecutionCancellationRequiresReceiptAndSurvivesContextEnd(t *testing.T) h.finished(result, store.TurnCancelled) next := h.message("next", "Continue after cancellation") result = h.run(context.Background(), next.TurnID) - env = h.read(proto.TypePromptRequest) + env = h.read(testExecutionRequest) var prompt proto.PromptRequestPayload _ = env.DecodePayload(&prompt) if prompt.AgentSessionID != "cancelled-native" { @@ -269,7 +284,7 @@ func TestExecutionCancellationRequiresReceiptAndSurvivesContextEnd(t *testing.T) first := h.message("first", "Run") ctx, cancel := context.WithCancel(context.Background()) result := h.run(ctx, first.TurnID) - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) cancel() done := h.finished(result, store.TurnFailed) var outcome execution.Result @@ -285,7 +300,7 @@ func TestExecutionFailureDoesNotBecomeSuccessOrReplay(t *testing.T) { h := newDispatchHarness(t) first := h.message("first", "Run") result := h.run(context.Background(), first.TurnID) - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) switch kind { case "disconnect": h.conn.Close() diff --git a/services/agents-api/internal/store/environment_expiry_dispatch_test.go b/services/agents-api/internal/store/environment_expiry_dispatch_test.go index 17cc9c360..fc2cfc462 100644 --- a/services/agents-api/internal/store/environment_expiry_dispatch_test.go +++ b/services/agents-api/internal/store/environment_expiry_dispatch_test.go @@ -11,7 +11,7 @@ import ( func enableEnvironmentExpiryDispatch(h *dispatchHarness) { h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{ Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: true, EnvironmentNone: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: true, SubagentControl: true, ToolObservations: true, - }}}}) + Preparation: true}}}}) } func TestWorkerEnvironmentExpiryAtFullExecutionCapacity(t *testing.T) { @@ -26,7 +26,7 @@ func TestWorkerEnvironmentExpiryAtFullExecutionCapacity(t *testing.T) { if _, err := worker.SubmitInputs(t.Context(), h.tenant, session.ID, key, []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"remain active"}`)}}); err != nil { t.Fatal(err) } - requests = append(requests, h.read(proto.TypePromptRequest)) + requests = append(requests, h.read(testExecutionRequest)) sessions = append(sessions, session) } tenant, due := newEnvironmentExpiryReservation(t, h.s) @@ -70,7 +70,7 @@ func TestWorkerEnvironmentExpirySkipsBusySessionAndAllowsDispatch(t *testing.T) t.Fatal(err) } waitEnvironmentExpiry(t, h.s, otherTenant, other) - request := h.read(proto.TypePromptRequest) + request := h.read(testExecutionRequest) if request.ID != receipt[0].TurnID { t.Fatal("unrelated dispatch mismatch", request.ID) } diff --git a/services/agents-api/internal/store/environment_worker_helpers_test.go b/services/agents-api/internal/store/environment_worker_helpers_test.go index 3687d4637..168df067a 100644 --- a/services/agents-api/internal/store/environment_worker_helpers_test.go +++ b/services/agents-api/internal/store/environment_worker_helpers_test.go @@ -63,6 +63,11 @@ func workerFrames(t *testing.T, runtimes ...*dispatchHarness) <-chan proto.Envel if h.conn.ReadJSON(&env) != nil { return } + var keep bool + env, keep = h.executionFrame(env) + if !keep { + continue + } select { case frames <- env: case <-t.Context().Done(): diff --git a/services/agents-api/internal/store/environment_worker_scan_test.go b/services/agents-api/internal/store/environment_worker_scan_test.go index 7627e29e0..bca895242 100644 --- a/services/agents-api/internal/store/environment_worker_scan_test.go +++ b/services/agents-api/internal/store/environment_worker_scan_test.go @@ -24,7 +24,7 @@ func TestWorkerEnvironmentRetriesNewlyReadyAtNextScan(t *testing.T) { _, stop := startEnvironmentExpiryWorker(t, h.d) // Dispatch starts only after selectWork has examined the pending input on // the same pass, while its exact Runtime is still incapable of preparation. - barrier := nextWorkerFrame(t, frames, proto.TypePromptRequest) + barrier := nextWorkerFrame(t, frames, testExecutionRequest) if barrier.ID != receipt.TurnID { t.Fatal("unexpected scan barrier") } @@ -70,7 +70,7 @@ func TestWorkerEnvironmentPaginationReachesReadyTail(t *testing.T) { receipt := h.message("barrier", "ordinary work") scanned := time.Now() _, stop := startEnvironmentExpiryWorker(t, h.d) - barrier := nextWorkerFrame(t, frames, proto.TypePromptRequest) + barrier := nextWorkerFrame(t, frames, testExecutionRequest) if barrier.ID != receipt.TurnID { t.Fatal("unexpected page barrier") } diff --git a/services/agents-api/internal/store/environment_worker_test.go b/services/agents-api/internal/store/environment_worker_test.go index 64615d1b4..313cf2cb6 100644 --- a/services/agents-api/internal/store/environment_worker_test.go +++ b/services/agents-api/internal/store/environment_worker_test.go @@ -38,7 +38,7 @@ func TestWorkerEnvironmentSharesCapacityThroughClaimAndCleanup(t *testing.T) { select { case frame := <-frames: switch frame.Type { - case proto.TypePromptRequest: + case testExecutionRequest: normal = append(normal, frame) case proto.TypeExecutionPrepare: preparing = append(preparing, frame) @@ -96,7 +96,7 @@ func TestWorkerEnvironmentSharesCapacityThroughClaimAndCleanup(t *testing.T) { t.Fatal("duplicate preparation release") } release = frame - case proto.TypePromptRequest: + case testExecutionRequest: if resumed.ID != "" { t.Fatal("cleanup freed more than one capacity slot") } @@ -140,7 +140,7 @@ func TestWorkerEnvironmentRetriesPendingWithoutExtendingDeadline(t *testing.T) { nextWorkerFrame(t, frames, proto.TypeExecutionRelease) h.session = publicSession(t, h, "unrelated") receipt := h.message("ordinary", "make progress after preparation failure") - request := nextWorkerFrame(t, frames, proto.TypePromptRequest) + request := nextWorkerFrame(t, frames, testExecutionRequest) if request.ID != receipt.TurnID { t.Fatal("preparation failure blocked ordinary work") } diff --git a/services/agents-api/internal/store/execution_events_test.go b/services/agents-api/internal/store/execution_events_test.go index ba1404bb0..31bef15ff 100644 --- a/services/agents-api/internal/store/execution_events_test.go +++ b/services/agents-api/internal/store/execution_events_test.go @@ -17,7 +17,7 @@ func TestExecutionPersistsLiveAndCancelledPartialOutput(t *testing.T) { ctx := context.Background() input := h.message("start", "Stream then cancel") result := h.run(ctx, input.TurnID) - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) h.write(input.TurnID, proto.TypeDelta, proto.DeltaPayload{Delta: "已输出", Sequence: 1}) h.write(input.TurnID, proto.TypeToolCall, proto.ToolCallPayload{ID: "tool-1", Name: "Bash", Stage: "before", Observation: &proto.ToolObservation{Kind: "command", Command: "pwd", Status: "in_progress"}}) h.write(input.TurnID, proto.TypeToolCall, proto.ToolCallPayload{ID: "tool-1", Name: "Bash", Stage: "after", Observation: &proto.ToolObservation{Kind: "command", Command: "pwd", Status: "completed"}}) @@ -79,7 +79,7 @@ func TestExecutionDoesNotCompleteAfterEventPersistenceFailure(t *testing.T) { ctx := context.Background() input := h.message("start", "Output beyond storage budget") result := h.run(ctx, input.TurnID) - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) _, pool := store.NewTestStore(t) defer pool.Close() if _, err := pool.Exec(ctx, "UPDATE turns SET event_bytes=33554432 WHERE id=$1", input.TurnID); err != nil { diff --git a/services/agents-api/internal/store/execution_messages_test.go b/services/agents-api/internal/store/execution_messages_test.go index c5d622d49..cbda2d87e 100644 --- a/services/agents-api/internal/store/execution_messages_test.go +++ b/services/agents-api/internal/store/execution_messages_test.go @@ -16,14 +16,14 @@ func TestExecutionNegotiatesAndPersistsMessageObservations(t *testing.T) { first := h.message("legacy", "legacy observation policy") result := h.run(ctx, first.TurnID) var request proto.PromptRequestPayload - env := h.read(proto.TypePromptRequest) + env := h.read(testExecutionRequest) _ = env.DecodePayload(&request) if request.ObserveMessages { t.Fatal("unadvertised observation capability requested") } h.write(first.TurnID, proto.TypeDone, proto.DonePayload{}) h.finished(result, store.TurnCompleted) - h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{Streaming: true, Steering: true, Resume: true, DurableTurns: true, DurableInputReceipts: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: true, SubagentControl: true, ToolObservations: true, MessageItems: true, NativeSessionRecovery: true}}}}) + h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{Streaming: true, Steering: true, Resume: true, DurableTurns: true, DurableInputReceipts: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: true, SubagentControl: true, ToolObservations: true, MessageItems: true, NativeSessionRecovery: true, Preparation: true}}}}) deadline := time.Now().Add(3 * time.Second) for { peer, err := h.registry.LookupDevice(h.device.ID) @@ -41,7 +41,7 @@ func TestExecutionNegotiatesAndPersistsMessageObservations(t *testing.T) { } input := h.message("observed", "stream separate messages") result = h.run(ctx, input.TurnID) - env = h.read(proto.TypePromptRequest) + env = h.read(testExecutionRequest) _ = env.DecodePayload(&request) if !request.ObserveMessages { t.Fatal("advertised capability was not requested") diff --git a/services/agents-api/internal/store/execution_tools_test.go b/services/agents-api/internal/store/execution_tools_test.go index 607fb324a..347aec57b 100644 --- a/services/agents-api/internal/store/execution_tools_test.go +++ b/services/agents-api/internal/store/execution_tools_test.go @@ -16,7 +16,7 @@ func TestExecutionNegotiatesAndPersistsToolObservations(t *testing.T) { ctx := context.Background() input := h.message("observed", "run tools") result := h.run(ctx, input.TurnID) - env := h.read(proto.TypePromptRequest) + env := h.read(testExecutionRequest) var request proto.PromptRequestPayload _ = env.DecodePayload(&request) if !request.ObserveToolObservations || request.ObserveTools || request.ObserveMessages { diff --git a/services/agents-api/internal/store/executor_fixture_test.go b/services/agents-api/internal/store/executor_fixture_test.go new file mode 100644 index 000000000..ef8712387 --- /dev/null +++ b/services/agents-api/internal/store/executor_fixture_test.go @@ -0,0 +1,63 @@ +package store_test + +import ( + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/google/uuid" +) + +// testExecutionRequest is a test projection of configuration plus a confirmed +// Start. It is never sent over the Runtime connection. Tests which exercise +// preparation failures consume the actual control frames directly. +const testExecutionRequest = "test_execution_request" + +type fixtureAdmission struct { + prepare proto.ExecutionPreparePayload + handle string + executor string +} + +func (h *dispatchHarness) executionFrame(env proto.Envelope) (proto.Envelope, bool) { + if h.admissions == nil { + h.admissions = make(map[string]fixtureAdmission) + } + switch env.Type { + case proto.TypeExecutionPrepare: + var prepare proto.ExecutionPreparePayload + if env.DecodePayload(&prepare) != nil { + h.t.Fatal("invalid execution preparation") + } + if prepare.Configuration.LocalEnvironment != nil || prepare.Configuration.WorkspaceReadOnly { + return env, true + } + if prepare.SessionID == "" || prepare.Configuration.RunID != "" || len(prepare.Configuration.Input) != 0 { + h.t.Fatal("preparation changed Session identity or submitted input early") + } + admission := fixtureAdmission{prepare: prepare, handle: uuid.NewString(), executor: "executor-" + prepare.SessionID} + h.admissions[env.ID] = admission + h.write(env.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: admission.handle, ExecutorID: admission.executor, Revision: 1, State: "ready"}) + return proto.Envelope{}, false + case proto.TypeExecutionStart: + admission, ok := h.admissions[env.ID] + if !ok { + return env, true + } + var start proto.ExecutionStartPayload + if env.DecodePayload(&start) != nil || start.Handle != admission.handle || start.ExecutorID != admission.executor || start.RunID == "" || start.Input.Validate() != nil { + h.t.Fatal("Start changed admission, Executor or input identity") + } + h.write(env.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: admission.handle, ExecutorID: admission.executor, Revision: 2, State: "started", RunID: start.RunID}) + request := admission.prepare.Configuration + request.RunID, request.ConversationID, request.Input = start.RunID, admission.prepare.SessionID, start.Input + projected, err := proto.NewEnvelope(testExecutionRequest, start.RunID, request) + if err != nil { + h.t.Fatal(err) + } + return projected, true + case proto.TypeExecutionRelease: + if _, ok := h.admissions[env.ID]; ok { + delete(h.admissions, env.ID) + return proto.Envelope{}, false + } + } + return env, true +} diff --git a/services/agents-api/internal/store/executor_recovery_test.go b/services/agents-api/internal/store/executor_recovery_test.go new file mode 100644 index 000000000..eba3602cb --- /dev/null +++ b/services/agents-api/internal/store/executor_recovery_test.go @@ -0,0 +1,58 @@ +package store_test + +import ( + "testing" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/google/uuid" +) + +func readyExecutorAttempt(t *testing.T, h *dispatchHarness) (proto.Envelope, proto.ExecutionStartPayload) { + t.Helper() + prepare := h.read(proto.TypeExecutionPrepare) + var configuration proto.ExecutionPreparePayload + if prepare.DecodePayload(&configuration) != nil || configuration.SessionID != h.session.ID || len(configuration.Configuration.Input) != 0 { + t.Fatal("invalid Executor preparation identity") + } + handle, executor := uuid.NewString(), uuid.NewString() + h.write(prepare.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, ExecutorID: executor, Revision: 1, State: "ready"}) + frame := h.read(proto.TypeExecutionStart) + var start proto.ExecutionStartPayload + if frame.ID != prepare.ID || frame.DecodePayload(&start) != nil || start.Handle != handle || start.ExecutorID != executor { + t.Fatal("Start lost its admission or Executor identity") + } + return frame, start +} + +func TestExecutorRecoveryRetriesOnlyConfirmedUnsubmittedInput(t *testing.T) { + for _, code := range []string{"executor_unavailable", "executor_cleanup_unconfirmed", "start_failed"} { + t.Run(code, func(t *testing.T) { + h := newDispatchHarness(t) + receipt := h.message("input", "execute once") + result := h.run(t.Context(), receipt.TurnID) + first, start := readyExecutorAttempt(t, h) + h.write(first.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{State: "rejected", Operation: proto.TypeExecutionStart, ErrorCode: code}) + if code != "executor_unavailable" { + h.finished(result, store.TurnFailed) + return + } + next, replacement := readyExecutorAttempt(t, h) + if next.ID == first.ID || replacement.ExecutorID == start.ExecutorID || replacement.RunID != start.RunID || inputTextForTest(t, replacement.Input) != "execute once" { + t.Fatal("recovery changed input or reused retired control ownership") + } + h.write(next.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: replacement.Handle, ExecutorID: replacement.ExecutorID, Revision: 2, State: "started", RunID: replacement.RunID}) + h.write(replacement.RunID, proto.TypeDone, proto.DonePayload{Content: "once"}) + h.finished(result, store.TurnCompleted) + }) + } +} + +func TestExecutorReadinessRejectsChangedOwner(t *testing.T) { + h := newDispatchHarness(t) + receipt := h.message("input", "execute once") + result := h.run(t.Context(), receipt.TurnID) + frame, start := readyExecutorAttempt(t, h) + h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: start.Handle, ExecutorID: uuid.NewString(), Revision: 2, State: "started", RunID: start.RunID}) + h.finished(result, store.TurnFailed) +} diff --git a/services/agents-api/internal/store/function_execution_test.go b/services/agents-api/internal/store/function_execution_test.go index 19416593d..0e138b18c 100644 --- a/services/agents-api/internal/store/function_execution_test.go +++ b/services/agents-api/internal/store/function_execution_test.go @@ -26,7 +26,7 @@ func newFunctionHarness(t *testing.T) *dispatchHarness { if err := h.s.BindSessionDevice(t.Context(), h.tenant, h.session.ID, h.device.ID); err != nil { t.Fatal(err) } - h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{Streaming: true, Steering: true, Resume: true, DurableTurns: true, DurableInputReceipts: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: true, SubagentControl: true, ToolObservations: true, EnvironmentNone: true, FunctionTools: true, FunctionResultImages: true}}}}) + h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{Streaming: true, Steering: true, Resume: true, DurableTurns: true, DurableInputReceipts: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: true, SubagentControl: true, ToolObservations: true, EnvironmentNone: true, FunctionTools: true, FunctionResultImages: true, Preparation: true}}}}) deadline := time.Now().Add(time.Second) for { peer, _ := h.registry.LookupDevice(h.device.ID) @@ -64,7 +64,7 @@ func TestExecutionFunctionsWaitForEveryApplicationReceipt(t *testing.T) { input := h.message("start", "Run functions") result := h.run(t.Context(), input.TurnID) var prompt proto.PromptRequestPayload - _ = h.read(proto.TypePromptRequest).DecodePayload(&prompt) + _ = h.read(testExecutionRequest).DecodePayload(&prompt) if len(prompt.FunctionTools) != 1 || prompt.FunctionTools[0].Name != "lookup_ticket" { t.Fatal(prompt.FunctionTools) } @@ -107,7 +107,7 @@ func TestExecutionFunctionsWaitForEveryApplicationReceipt(t *testing.T) { functionState(t, h, 0) next := h.message("next", "Resume") result = h.run(t.Context(), next.TurnID) - _ = h.read(proto.TypePromptRequest).DecodePayload(&prompt) + _ = h.read(testExecutionRequest).DecodePayload(&prompt) if prompt.AgentSessionID != "native-functions" || len(prompt.FunctionTools) != 1 { t.Fatal(prompt) } @@ -121,7 +121,7 @@ func TestExecutionFunctionsCancellationAndUnconfirmedResults(t *testing.T) { h := newFunctionHarness(t) input := h.message("start", "Run") result := h.run(t.Context(), input.TurnID) - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) h.write(input.TurnID, proto.TypeFunctionCall, proto.FunctionCallPayload{CallID: "a", Name: "lookup_ticket", Arguments: json.RawMessage(`{}`)}) state := functionState(t, h, 1) id := state.RequiredActions[0].CallID @@ -178,7 +178,7 @@ func TestExecutionFunctionsRejectUndeclaredCallsAndPrematureDone(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) defer cancel() result := h.run(ctx, input.TurnID) - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) h.write(input.TurnID, proto.TypeFunctionCall, proto.FunctionCallPayload{CallID: "a", Name: name, Arguments: json.RawMessage(`{}`)}) h.write(input.TurnID, proto.TypeDone, proto.DonePayload{}) h.finished(result, store.TurnFailed) diff --git a/services/agents-api/internal/store/function_input_execution_test.go b/services/agents-api/internal/store/function_input_execution_test.go index 99e763047..a932a988c 100644 --- a/services/agents-api/internal/store/function_input_execution_test.go +++ b/services/agents-api/internal/store/function_input_execution_test.go @@ -14,7 +14,7 @@ func TestExecutionFunctionInputBatchStillSteersMessages(t *testing.T) { h := newFunctionHarness(t) input := h.message("start", "Run") running := h.run(t.Context(), input.TurnID) - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) h.write(input.TurnID, proto.TypeFunctionCall, proto.FunctionCallPayload{CallID: "a", Name: "lookup_ticket", Arguments: json.RawMessage(`{}`)}) state := functionState(t, h, 1) raw, _ := json.Marshal(store.FunctionResultInput{TurnID: input.TurnID, CallID: state.RequiredActions[0].CallID, Result: json.RawMessage(`{"success":true,"output":"answer"}`)}) diff --git a/services/agents-api/internal/store/function_worker_test.go b/services/agents-api/internal/store/function_worker_test.go index 3fed06915..f3965da77 100644 --- a/services/agents-api/internal/store/function_worker_test.go +++ b/services/agents-api/internal/store/function_worker_test.go @@ -16,7 +16,7 @@ import ( ) func TestWorkerWaitsForToolCapabilities(t *testing.T) { - for _, missing := range []string{"durable_input_receipts", "execution_controls", "function_tools", "tool_observations", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required"} { + for _, missing := range []string{"preparation", "durable_input_receipts", "execution_controls", "function_tools", "tool_observations", "mcp_http_tools", "mcp_http_bearer_auth", "mcp_http_required"} { for _, prebound := range []bool{false, true} { t.Run(missing+"/"+map[bool]string{false: "select", true: "bound"}[prebound], func(t *testing.T) { h := newFunctionHarness(t) @@ -44,7 +44,7 @@ func TestWorkerWaitsForToolCapabilities(t *testing.T) { t.Fatal(err) } } - caps := proto.AgentKindCapabilities{Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: missing != "durable_input_receipts", EnvironmentNone: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: missing != "execution_controls", SubagentControl: true, ToolObservations: missing != "tool_observations", MCPHTTPTools: missing != "mcp_http_tools", MCPHTTPRequired: missing != "mcp_http_required", MCPHTTPBearerAuth: missing != "mcp_http_bearer_auth", FunctionTools: missing != "function_tools" && !isMCP} + caps := proto.AgentKindCapabilities{Preparation: missing != "preparation", Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: missing != "durable_input_receipts", EnvironmentNone: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: missing != "execution_controls", SubagentControl: true, ToolObservations: missing != "tool_observations", MCPHTTPTools: missing != "mcp_http_tools", MCPHTTPRequired: missing != "mcp_http_required", MCPHTTPBearerAuth: missing != "mcp_http_bearer_auth", FunctionTools: missing != "function_tools" && !isMCP} heartbeat := func() { h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: caps}}}) } @@ -53,7 +53,7 @@ func TestWorkerWaitsForToolCapabilities(t *testing.T) { for { peer, _ := h.registry.LookupDevice(h.device.ID) info, _, _ := peer.AgentKindStatus("codex") - if info.Capabilities.ExecutionControls == caps.ExecutionControls && info.Capabilities.FunctionTools == caps.FunctionTools && info.Capabilities.ToolObservations == caps.ToolObservations && info.Capabilities.MCPHTTPTools == caps.MCPHTTPTools && info.Capabilities.MCPHTTPBearerAuth == caps.MCPHTTPBearerAuth && info.Capabilities.MCPHTTPRequired == caps.MCPHTTPRequired { + if info.Capabilities.Preparation == caps.Preparation && info.Capabilities.DurableInputReceipts == caps.DurableInputReceipts && info.Capabilities.ExecutionControls == caps.ExecutionControls && info.Capabilities.FunctionTools == caps.FunctionTools && info.Capabilities.ToolObservations == caps.ToolObservations && info.Capabilities.MCPHTTPTools == caps.MCPHTTPTools && info.Capabilities.MCPHTTPBearerAuth == caps.MCPHTTPBearerAuth && info.Capabilities.MCPHTTPRequired == caps.MCPHTTPRequired { break } if time.Now().After(deadline) { @@ -88,10 +88,10 @@ func TestWorkerWaitsForToolCapabilities(t *testing.T) { t.Fatal("bound an incapable device", err) } } - caps.DurableInputReceipts, caps.ExecutionControls, caps.ToolObservations, caps.MCPHTTPTools = true, true, true, true + caps.Preparation, caps.DurableInputReceipts, caps.ExecutionControls, caps.ToolObservations, caps.MCPHTTPTools = true, true, true, true, true caps.MCPHTTPBearerAuth, caps.FunctionTools, caps.MCPHTTPRequired = true, !isMCP, true heartbeat() - request := h.read(proto.TypePromptRequest) + request := h.read(testExecutionRequest) var prompt proto.PromptRequestPayload if request.DecodePayload(&prompt) != nil { t.Fatal("invalid prompt") diff --git a/services/agents-api/internal/store/harness_onboarding_test.go b/services/agents-api/internal/store/harness_onboarding_test.go index 9b723d412..c3655ddde 100644 --- a/services/agents-api/internal/store/harness_onboarding_test.go +++ b/services/agents-api/internal/store/harness_onboarding_test.go @@ -221,19 +221,42 @@ func startOnboardingPeer(t *testing.T, h *dispatchHarness) (<-chan proto.PromptR }() go func() { enc := json.NewEncoder(in) + preparations := make(map[string]proto.ExecutionPreparePayload) for { var e proto.Envelope if err := h.conn.ReadJSON(&e); err != nil { down <- err return } - if e.Type == proto.TypePromptRequest { - var p proto.PromptRequestPayload + if e.Type == proto.TypeExecutionPrepare { + var p proto.ExecutionPreparePayload if err := e.DecodePayload(&p); err != nil { down <- err return } - started <- p + if p.SessionID == "" || p.Configuration.RunID != "" || len(p.Configuration.Input) != 0 { + down <- fmt.Errorf("fixture preparation submitted input or lost Session identity") + return + } + preparations[e.ID] = p + } + if e.Type == proto.TypeExecutionStart { + var start proto.ExecutionStartPayload + if err := e.DecodePayload(&start); err != nil { + down <- err + return + } + p, ok := preparations[e.ID] + if !ok || start.ExecutorID == "" || start.Handle == "" { + down <- fmt.Errorf("fixture Start lacks prepared Executor ownership") + return + } + request := p.Configuration + request.RunID, request.ConversationID, request.Input = start.RunID, p.SessionID, start.Input + started <- request + } + if e.Type == proto.TypeExecutionRelease { + delete(preparations, e.ID) } if err := enc.Encode(e); err != nil { down <- err diff --git a/services/agents-api/internal/store/public_execution_test.go b/services/agents-api/internal/store/public_execution_test.go index 5f7adea85..6d9b2c28b 100644 --- a/services/agents-api/internal/store/public_execution_test.go +++ b/services/agents-api/internal/store/public_execution_test.go @@ -23,7 +23,7 @@ func publicSession(t *testing.T, h *dispatchHarness, key string) store.Session { func TestExecutionWorkerAdmissionBindingAndRecovery(t *testing.T) { h := newDispatchHarness(t) - h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: true, SubagentControl: true, ToolObservations: true, EnvironmentNone: true}}}}) + h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: true, SubagentControl: true, ToolObservations: true, EnvironmentNone: true, Preparation: true}}}}) h.session = publicSession(t, h, "public") ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -58,7 +58,7 @@ func TestExecutionWorkerAdmissionBindingAndRecovery(t *testing.T) { if err != nil || !retry[0].Replayed || retry[0].TurnID != receipts[0].TurnID { t.Fatal(retry, err) } - request := h.read(proto.TypePromptRequest) + request := h.read(testExecutionRequest) var prompt proto.PromptRequestPayload if err := request.DecodePayload(&prompt); err != nil { t.Fatal(err) @@ -84,7 +84,7 @@ func TestExecutionWorkerAdmissionBindingAndRecovery(t *testing.T) { if err != nil { t.Fatal(err) } - request = h.read(proto.TypePromptRequest) + request = h.read(testExecutionRequest) _ = request.DecodePayload(&prompt) if prompt.AgentSessionID != "worker-native" { t.Fatal(prompt) diff --git a/services/agents-api/internal/store/session_deletion_execution_test.go b/services/agents-api/internal/store/session_deletion_execution_test.go index 11ef5efa6..d33489c8a 100644 --- a/services/agents-api/internal/store/session_deletion_execution_test.go +++ b/services/agents-api/internal/store/session_deletion_execution_test.go @@ -31,7 +31,7 @@ func TestDeletedSessionWaitingTurnSettlesWithoutStoppingWorker(t *testing.T) { } }() input := h.message("start", "Run") - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) h.write(input.TurnID, proto.TypeFunctionCall, proto.FunctionCallPayload{CallID: "pending", Name: "lookup_ticket", Arguments: json.RawMessage(`{}`)}) state := functionState(t, h, 1) if err := h.s.DeleteSession(ctx, h.tenant, h.session.ID); !errors.Is(err, store.ErrSessionNotIdle) { @@ -63,7 +63,7 @@ func TestDeletedSessionWaitingTurnSettlesWithoutStoppingWorker(t *testing.T) { } h.session = publicSession(t, h, "unrelated") next := h.message("next", "Unrelated work") - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) h.write(next.TurnID, proto.TypeDone, proto.DonePayload{Content: "unaffected"}) waitTurn(t, h, next.TurnID, store.TurnCompleted) } @@ -120,7 +120,7 @@ func TestWaitingSessionCancelsThenDeletesThroughWorker(t *testing.T) { } }() input := h.message("start", "Run") - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) h.write(input.TurnID, proto.TypeFunctionCall, proto.FunctionCallPayload{CallID: "pending", Name: "lookup_ticket", Arguments: json.RawMessage(`{}`)}) state := functionState(t, h, 1) if err := h.s.DeleteSession(ctx, h.tenant, h.session.ID); !errors.Is(err, store.ErrSessionNotIdle) { @@ -165,7 +165,7 @@ func TestWaitingSessionCancelsThenDeletesThroughWorker(t *testing.T) { } h.session = publicSession(t, h, "unrelated") next := h.message("next", "Unrelated work") - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) h.write(next.TurnID, proto.TypeDone, proto.DonePayload{Content: "unaffected"}) waitTurn(t, h, next.TurnID, store.TurnCompleted) } diff --git a/services/agents-api/internal/store/steering_receipts_test.go b/services/agents-api/internal/store/steering_receipts_test.go index 9b330c3d0..df620a347 100644 --- a/services/agents-api/internal/store/steering_receipts_test.go +++ b/services/agents-api/internal/store/steering_receipts_test.go @@ -19,7 +19,7 @@ func TestExecutionDurableInputReceiptLifetime(t *testing.T) { defer cancel() first := h.message("first", "initial") result := h.run(ctx, first.TurnID) - h.read(proto.TypePromptRequest) + h.read(testExecutionRequest) extra := h.message("extra", "additional") var input proto.PromptSteerPayload if err := h.read(proto.TypePromptSteer).DecodePayload(&input); err != nil || !input.DurableReceipt { diff --git a/services/agents-api/internal/store/structured_output_dispatch_test.go b/services/agents-api/internal/store/structured_output_dispatch_test.go index 91f2c1109..78d26936e 100644 --- a/services/agents-api/internal/store/structured_output_dispatch_test.go +++ b/services/agents-api/internal/store/structured_output_dispatch_test.go @@ -28,7 +28,7 @@ func TestStructuredOutputDispatchRechecksOperationQualification(t *testing.T) { if err = h.s.BindSessionDevice(t.Context(), h.tenant, h.session.ID, h.device.ID); err != nil { t.Fatal(err) } - caps := proto.AgentKindCapabilities{Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: true, ExecutionControls: true, EnvironmentNone: true, SubagentControl: true, ToolObservations: true, StructuredOutput: true, MessageItems: true} + caps := proto.AgentKindCapabilities{Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: true, ExecutionControls: true, EnvironmentNone: true, SubagentControl: true, ToolObservations: true, StructuredOutput: true, MessageItems: true, Preparation: true} h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "fixture_harness", Available: true, Capabilities: caps}}}) peer, _ := h.registry.LookupDevice(h.device.ID) for deadline := time.Now().Add(3 * time.Second); ; { diff --git a/services/agents-api/internal/store/subagent_dispatch_test.go b/services/agents-api/internal/store/subagent_dispatch_test.go index a032b0b3a..146b8510b 100644 --- a/services/agents-api/internal/store/subagent_dispatch_test.go +++ b/services/agents-api/internal/store/subagent_dispatch_test.go @@ -36,7 +36,7 @@ func TestSubagentIdentityUsesLeasedDispatchJournal(t *testing.T) { input := h.message("first", "root message") running := h.run(ctx, input.TurnID) var request proto.PromptRequestPayload - if err = h.read(proto.TypePromptRequest).DecodePayload(&request); err != nil || request.ObserveSubagentIdentities != enabled || request.MaxConcurrentSubagents == nil || *request.MaxConcurrentSubagents != 3 { + if err = h.read(testExecutionRequest).DecodePayload(&request); err != nil || request.ObserveSubagentIdentities != enabled || request.MaxConcurrentSubagents == nil || *request.MaxConcurrentSubagents != 3 { t.Fatal("private observation policy not carried", err) } identity := proto.SubagentIdentityPayload{NativeID: "child", ParentNativeID: "root", NativeCreatedAt: 100, ParentTurnID: "native-turn", SourceItemID: "spawn-item"} diff --git a/services/agents-api/internal/store/worker_input_race_test.go b/services/agents-api/internal/store/worker_input_race_test.go index 8e87c1277..d983e037b 100644 --- a/services/agents-api/internal/store/worker_input_race_test.go +++ b/services/agents-api/internal/store/worker_input_race_test.go @@ -100,7 +100,7 @@ func TestWorkerInputReadSkipsConcurrentlyCancelledCandidate(t *testing.T) { t.Fatal("a stale candidate terminated the global Worker", err) case <-time.After(650 * time.Millisecond): } - frame := h.read(proto.TypePromptRequest) + frame := h.read(testExecutionRequest) if frame.ID != healthy.TurnID { t.Fatal("cancelled candidate reached the Runtime", frame.ID) } diff --git a/services/agents-api/internal/store/worker_lease_loss_test.go b/services/agents-api/internal/store/worker_lease_loss_test.go index d54918a11..f1dbf42cc 100644 --- a/services/agents-api/internal/store/worker_lease_loss_test.go +++ b/services/agents-api/internal/store/worker_lease_loss_test.go @@ -14,7 +14,7 @@ import ( func TestWorkerLeaseLossLeavesUncertainWorkForSuccessor(t *testing.T) { h := newDispatchHarness(t) _, pool := store.NewTestStore(t) - h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: true, SubagentControl: true, ToolObservations: true, EnvironmentNone: true}}}}) + h.write("", proto.TypeHeartbeat, proto.HeartbeatPayload{SupportedAgentKinds: []proto.SupportedAgentKind{{Kind: "codex", Available: true, Capabilities: proto.AgentKindCapabilities{Streaming: true, Steering: true, DurableTurns: true, DurableInputReceipts: true, WebSearchControl: true, TextVerbosity: true, ExecutionControls: true, SubagentControl: true, ToolObservations: true, EnvironmentNone: true, Preparation: true}}}}) h.session = publicSession(t, h, "active") queued := publicSession(t, h, "queued") worker, err := execution.StartWorker(t.Context(), h.d) @@ -37,7 +37,7 @@ func TestWorkerLeaseLossLeavesUncertainWorkForSuccessor(t *testing.T) { if err != nil { t.Fatal(err) } - request := h.read(proto.TypePromptRequest) + request := h.read(testExecutionRequest) if request.ID != receipts[0].TurnID { t.Fatal("unexpected dispatch", request.ID) } diff --git a/services/agents-api/internal/store/worker_wakeup_test.go b/services/agents-api/internal/store/worker_wakeup_test.go new file mode 100644 index 000000000..d53516f3d --- /dev/null +++ b/services/agents-api/internal/store/worker_wakeup_test.go @@ -0,0 +1,141 @@ +package store_test + +import ( + "context" + "encoding/json" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/execution" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +type schedulerQueryOrder struct { + armed atomic.Bool + first chan string +} + +func (trace *schedulerQueryOrder) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + for _, name := range []string{"ListExecutionWork", "ListDueEnvironmentInputs"} { + if strings.HasPrefix(data.SQL, "-- name: "+name+" ") && trace.armed.CompareAndSwap(true, false) { + trace.first <- name + } + } + return ctx +} +func (*schedulerQueryOrder) TraceQueryEnd(context.Context, *pgx.Conn, pgx.TraceQueryEndData) {} + +func TestWorkerSchedulerCommittedAdmissionWakesBeforeMaintenance(t *testing.T) { + for _, operation := range []string{"submit", "create", "stream", "rejected"} { + t.Run(operation, func(t *testing.T) { + h := newDispatchHarness(t) + enableWorkerEnvironment(t, h) + h.session = publicSession(t, h, "wakeup") + _, pool := store.NewTestStore(t) + trace := &schedulerQueryOrder{first: make(chan string, 1)} + config := pool.Config() + config.ConnConfig.Tracer = trace + instrumented, err := pgxpool.NewWithConfig(t.Context(), config) + if err != nil { + t.Fatal(err) + } + defer instrumented.Close() + h.d.Store = store.NewWithCredentialCipher(instrumented, store.FixtureCipher()) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + worker, err := execution.StartWorker(ctx, h.d) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + started := false + defer func() { + cancel() + if !started { + go func() { done <- worker.Run(ctx) }() + } + select { + case <-done: + case <-time.After(10 * time.Second): + t.Error("worker did not stop") + } + }() + input := []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"wake"}`)}} + switch operation { + case "submit": + _, err = worker.SubmitInputs(ctx, h.tenant, h.session.ID, "wake", input) + case "create", "stream": + creation := store.CreateSessionInput{Creator: store.FixtureCreator(), Engine: "codex", IdempotencyKey: "wake", Configuration: h.session.Configuration, InitialInputs: input} + if operation == "create" { + _, err = worker.CreateSession(ctx, h.tenant, creation) + } else { + _, err = worker.CreateSessionStream(ctx, h.tenant, creation) + } + case "rejected": + h.message("wake", "different") + _, err = worker.SubmitInputs(ctx, h.tenant, h.session.ID, "wake", input) + if err == nil { + t.Fatal("changed retry was accepted") + } + err = nil + } + if err != nil { + t.Fatal(err) + } + trace.armed.Store(true) + started = true + go func() { done <- worker.Run(ctx) }() + // Maintenance always starts with expiry. A committed admission must + // select work first, without depending on a short timing threshold. + select { + case first := <-trace.first: + want := "ListExecutionWork" + if operation == "rejected" { + want = "ListDueEnvironmentInputs" + } + if first != want { + t.Fatalf("first scheduler query = %s, want %s", first, want) + } + case <-time.After(5 * time.Second): + t.Fatal("worker did not schedule") + } + }) + } +} + +func TestWorkerSchedulerHintBypassesEnvironmentScanThrottle(t *testing.T) { + h := newDispatchHarnessForSession(t, []byte(`{"agent":{"model":"test-model"},"environment":{"type":"self_hosted","workspace_directory":"/workspace"}}`), false) + enableWorkerEnvironment(t, h) + frames := workerFrames(t, h) + worker, stop := startEnvironmentExpiryWorker(t, h.d) + defer stop() + awaitDaemonRemoteCondition(t, t.Context(), 5*time.Second, "initial empty scheduler scan", func() bool { + return worker.MetricsSnapshot().Scheduler.LastRunAt != nil + }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + admitted := make(chan error, 1) + started := time.Now() + go func() { + _, err := worker.SubmitInputs(ctx, h.tenant, h.session.ID, "wake", []store.Input{{Kind: "message", Payload: json.RawMessage(`{"text":"wake"}`)}}) + admitted <- err + }() + prepare := nextWorkerFrame(t, frames, proto.TypeExecutionPrepare) + if elapsed := time.Since(started); elapsed >= 750*time.Millisecond { + t.Fatalf("committed input waited for the environment scan interval: %s", elapsed) + } + handle := acknowledgePreparation(h, prepare.ID) + h.write(prepare.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: handle, Revision: 2, State: "failed"}) + nextWorkerFrame(t, frames, proto.TypeExecutionRelease) + cancel() + select { + case <-admitted: + case <-time.After(5 * time.Second): + t.Fatal("detached admission waiter did not stop") + } +} diff --git a/services/agents-api/tests/official_user_runtime.py b/services/agents-api/tests/official_user_runtime.py index c006fce52..43b3b7631 100644 --- a/services/agents-api/tests/official_user_runtime.py +++ b/services/agents-api/tests/official_user_runtime.py @@ -43,7 +43,9 @@ def expired(signum, frame): signal.signal(signal.SIGALRM, previous) -def run_acceptance(client, foreign, http, session, runtime, evidence_path, secret_markers=()): +def run_acceptance(client, foreign, http, session, runtime, evidence_path, secret_markers=(), + expected_environment_type="self_hosted", + protected_credential_path="/home/runtime/.oac/daemon/executor-key.json", turn_observer=None): """Return a safe report after real execution; raise on any failed assertion.""" assert threading.current_thread() is threading.main_thread(), "Run acceptance on the main thread" pin = json.loads((Path(__file__).resolve().parents[3] / "contracts/agents-api/upstream.json").read_text()) @@ -71,7 +73,7 @@ def run_acceptance(client, foreign, http, session, runtime, evidence_path, secre outputs = {"/workspace/outputs/a.bin": bytes(range(256)), "/workspace/outputs/b.txt": ("native-user-runtime-" + nonce + "\n").encode()} artifacts = {} - private_paths = ["/home/runtime/.oac/daemon/executor-key.json", + private_paths = [protected_credential_path, "/home/runtime/.oac/daemon/.user-runtime-isolation-canary", "/environment/staging/.user-runtime-isolation-canary"] @@ -149,6 +151,7 @@ def snapshot_items(name): def run_turn(text, number, cancellation=False): observed, stop, requested = [], threading.Event(), threading.Event() + submitted_at, first_text_at, terminal_at = None, None, None cancel_key = nonce + "-cancel" def cancel_running(): @@ -165,6 +168,7 @@ def cancel_running(): raise AssertionError("Cancellation ended before observing native effects") def consume(stream, journal, cancelling=None): + nonlocal first_text_at, terminal_at for event in stream: value = event.to_dict() journal.write(json.dumps(redact(value)) + "\n") @@ -172,6 +176,10 @@ def consume(stream, journal, cancelling=None): safe(value) observed.append(value) kind = value["type"] + if kind == "agent.session.turn.output_text.delta" and first_text_at is None and value.get("delta"): + first_text_at = time.monotonic() + if kind in ("agent.session.turn.completed", "agent.session.turn.cancelled", "agent.session.turn.failed"): + terminal_at = time.monotonic() assert kind not in ("error", "agent.session.failed", "agent.session.turn.failed"), "Native execution failed" if cancelling is not None and cancelling.done(): cancelling.result() @@ -194,6 +202,7 @@ def consume(stream, journal, cancelling=None): with _prompt_deadline(), path.open("w") as journal: path.chmod(0o600) with sessions.events.stream(sid, timeout=240) as stream: + submitted_at = time.monotonic() sessions.events.create(sid, events=[{"type": "agent.session.input.message", "input": [ {"role": "user", "content": [{"type": "input_text", "text": text}]}]}], idempotency_key=nonce + "-turn-" + str(number)) if cancellation: @@ -216,7 +225,11 @@ def consume(stream, journal, cancelling=None): assert current, "Native Turn has no public Items" if not cancellation: assert all(item.get("status") not in ("failed", "incomplete", "in_progress") for item in current), "Completed Turn contains failed native work" - report["turns"].append({"id": turn.id, "status": turn.status, "event_count": len(observed)}) + report["turns"].append({"id": turn.id, "status": turn.status, "event_count": len(observed), + "input_to_first_text_ms": round((first_text_at - submitted_at) * 1000, 3) if first_text_at is not None else None, + "input_to_terminal_ms": round((terminal_at - submitted_at) * 1000, 3) if terminal_at is not None else None}) + if turn_observer is not None: + report["turns"][-1]["runtime_observation"] = safe(turn_observer(number, turn.id)) return turn, current except BaseException: # No model request is retried; an already started tool is cancelled for cleanup. @@ -236,7 +249,8 @@ def check_artifacts(): safe([item.to_dict() for item in values]) try: - assert session.environment.type == "self_hosted" and session.environment.workspace_directory == "/workspace" + assert expected_environment_type in ("self_hosted", "openai_hosted") + assert session.environment.type == expected_environment_type and session.environment.workspace_directory == "/workspace" assert client.beta.agents.environments.retrieve(eid).status == "connected", "Runtime is not connected" assert not list(sessions.turns.list(sid)) and not list(sessions.items.list(sid)), "Use an unused Session" assert not list(sessions.artifacts.list(sid)), "Use a Session without existing Artifacts" @@ -299,6 +313,12 @@ def isolation_proof(phase): pass else: raise AssertionError("Foreign tenant accessed SDK Items") + warm, items = run_turn("Return the entire exact conversation-only remember- token from our previous turn, including its prefix. " + "Do not use tools, read files, or repeat any earlier command.", 2) + assert memory in answer(items), "Warm continuation lost native conversation history" + assert runtime.read(prefix + "-published") == b"published\n", "Warm continuation repeated side effects" + check_artifacts() + report["checks"].append("ordinary_warm_turn_preserves_history_and_effect_counts") committed = snapshot_items("before-restart") report["checks"].append("native_outputs_files_artifacts_items_and_tenant_isolation") runtime.restart() @@ -307,7 +327,7 @@ def isolation_proof(phase): assert snapshot_items("after-restart") == committed, "Restart changed committed Items" second, items = run_turn("Run exactly `python3 " + prefix + "-recover.py` once with your native shell tool; do not edit or retry it. " "Then return the entire exact conversation-only remember- token from our previous turn, including its prefix. " - "Do not look for the token in files or rerun any earlier command.", 2) + "Do not look for the token in files or rerun any earlier command.", 3) assert any(item.get("type") == "command_execution" and item.get("status") == "completed" and prefix + "-recover.py" in item.get("command", "") for item in items), "Missing recovered native isolation command" isolation_proof("recovered") assert memory in answer(items), "Cold continuation lost native conversation history" @@ -316,18 +336,18 @@ def isolation_proof(phase): check_artifacts() report["checks"].append("runtime_and_core_restart_preserve_history_and_outputs") third, items = run_turn("Use your native shell tool to run exactly `python3 " + prefix + "-hold.py` once and wait in the foreground. " - "It intentionally runs until cancelled. Do not background, delegate, retry, edit the script, or restart the command.", 3, cancellation=True) + "It intentionally runs until cancelled. Do not background, delegate, retry, edit the script, or restart the command.", 4, cancellation=True) commands = [item for item in items if item.get("type") == "command_execution" and prefix + "-hold.py" in item.get("command", "")] assert len(commands) == 1 and commands[0].get("status") in ("failed", "incomplete"), "Cancelled command lost its native outcome" stopped = runtime.read(ticks) time.sleep(2) assert runtime.read(ticks) == stopped and runtime.read(starts) == b"started\n", "Effects continued after cancellation settled" sessions.events.create(sid, events=[{"type": "agent.session.input.cancel"}], idempotency_key=nonce + "-cancel") - assert len(list(sessions.turns.list(sid))) == 3, "Duplicate cancellation created work" + assert len(list(sessions.turns.list(sid))) == 4, "Duplicate cancellation created work" check_artifacts() fourth, items = run_turn("Continue after the cancelled command. Never restart it or rerun the publish script. " "Run exactly `python3 " + prefix + "-resume.py` once with your native shell tool, without retries. " - "Then return the entire original conversation-only remember- token, including its prefix.", 4) + "Then return the entire original conversation-only remember- token, including its prefix.", 5) assert any(item.get("type") == "command_execution" and item.get("status") == "completed" and prefix + "-resume.py" in item.get("command", "") for item in items), "Missing resumed native command observation" assert memory in answer(items) and runtime.read(prefix + "-resumed") == b"resumed\n", "Post-cancel continuation failed" assert runtime.read(starts) == b"started\n" and runtime.read(ticks) == stopped, "Continuation repeated cancelled effects" From ea1591af702901202c3559fb48a72715e9ff2fd3 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Mon, 28 Sep 2026 15:15:46 +0000 Subject: [PATCH 2/8] Settle executor cleanup and terminal handoff independently --- CONTRIBUTING.md | 11 +- .../internal/agent/codex/executor.go | 18 +- .../internal/agent/codex/executor_test.go | 60 +++++ .../agent/codex/preparation_helpers_test.go | 8 + .../dispatch/executor_handoff_test.go | 228 ++++++++++++++++++ .../internal/dispatch/prepared_handoff.go | 31 ++- 6 files changed, 336 insertions(+), 20 deletions(-) create mode 100644 apps/parsar-daemon/internal/dispatch/executor_handoff_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 769981bcf..ff5d92c26 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1606,7 +1606,10 @@ confirms that the native owner can accept the next Turn. `Reusable=false` requir a reason and subsequent confirmed Executor close. An error means settlement is unconfirmed; it cannot free ownership or capacity. Caller deadlines stop waiting, not tracked cleanup. Retry the same cleanup target serially. Failed cleanup -blocks replacement and retains its resource slot. +blocks replacement and retains its resource slot. Executor Close confirms resource +retirement independently of the Turn outcome: an immutable Turn error must not +prevent closing the native transport and releasing resources once their work and +output have stopped. One output consumer starts before native Start, drains the bounded 64-frame channel, and retains the terminal observation until Start publication, Turn @@ -1614,7 +1617,11 @@ settlement and admitted operation receipts finish. Natural Done never calls Cancel. Input, function and interaction admission close before settlement, and operations already admitted hold their barrier through native receipts and outbound acknowledgement. Only then forward Done or an applied cancellation -receipt. Preserve the ten-second settlement wait and separate five-second receipt +receipt. Commit native continuity and release the old Run admission before +publishing Done, since the receiver may immediately start another Turn. A late +terminal-send failure belongs to the old Run; it cannot invalidate a successor +that already owns the Executor. Connection shutdown owns transport-loss cleanup. +Preserve the ten-second settlement wait and separate five-second receipt send budget; timeout is not proof of quiescence. The observed cancellation outcome retains native identity, Usage and output without fabricating missing evidence. diff --git a/apps/parsar-daemon/internal/agent/codex/executor.go b/apps/parsar-daemon/internal/agent/codex/executor.go index 954a29d0f..8692b3ce2 100644 --- a/apps/parsar-daemon/internal/agent/codex/executor.go +++ b/apps/parsar-daemon/internal/agent/codex/executor.go @@ -5,6 +5,7 @@ import ( "errors" "strings" "sync" + "time" "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" @@ -121,10 +122,11 @@ func (e *Executor) Close(ctx context.Context) error { select { case <-running.waitDone: default: - if err := running.Cancel(ctx); err != nil { - done <- err - return - } + // Try native cancellation before closing its transport, but never + // let an absent terminal receipt prevent resource retirement. + cancelCtx, stop := context.WithTimeout(ctx, 10*time.Second) + _ = running.Cancel(cancelCtx) + stop() } } base.cancelFn() @@ -133,7 +135,13 @@ func (e *Executor) Close(ctx context.Context) error { active := e.active e.mu.Unlock() if err == nil && active != nil { - _, err = active.AwaitSettlement(ctx) + // A Turn's permanent outcome error survives Close. Cleanup succeeds + // when its work and output have stopped, not when that outcome changes. + select { + case <-active.waitDone: + case <-ctx.Done(): + err = ctx.Err() + } } if err == nil { err = base.rpc.awaitReaders(ctx) diff --git a/apps/parsar-daemon/internal/agent/codex/executor_test.go b/apps/parsar-daemon/internal/agent/codex/executor_test.go index 39413ecfa..c991dd626 100644 --- a/apps/parsar-daemon/internal/agent/codex/executor_test.go +++ b/apps/parsar-daemon/internal/agent/codex/executor_test.go @@ -185,3 +185,63 @@ func TestExecutorCloseRetainsPlanUntilReaped(t *testing.T) { t.Fatal("cleanup retry did not release plan") } } + +func TestExecutorCloseSeparatesTurnFailureFromResourceCleanup(t *testing.T) { + e, root := executorFixture(t, "interrupt-error") + out := make(chan proto.Envelope, 20) + turn, err := e.StartTurn(t.Context(), "cancel-failure", proto.TextInput("hold"), out) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + if err := turn.Cancel(ctx); err == nil { + t.Fatal("native interrupt failure was hidden") + } + if _, err := turn.AwaitSettlement(ctx); err == nil { + t.Fatal("failed Turn unexpectedly settled successfully") + } + if err := e.Close(ctx); err != nil { + t.Fatal("failed Turn prevented confirmed resource cleanup", err) + } + if e.prepared.session.rpc.Alive() || len(preparedCatalogs(t, root)) != 0 { + t.Fatal("Close did not release native process and plan") + } + if _, err := turn.AwaitSettlement(ctx); err == nil { + t.Fatal("resource cleanup fabricated successful Turn settlement") + } +} + +func TestExecutorCloseTerminatesAfterMissingCancellationTerminal(t *testing.T) { + e, root := executorFixture(t, "interrupt-no-terminal") + out := make(chan proto.Envelope, 20) + turn, err := e.StartTurn(t.Context(), "missing-terminal", proto.TextInput("hold"), out) + if err != nil { + t.Fatal(err) + } + cancelCtx, stopCancel := context.WithTimeout(t.Context(), 5*time.Second) + defer stopCancel() + cancelled := make(chan error, 1) + go func() { cancelled <- turn.Cancel(cancelCtx) }() + waitPreparationMethod(t, root, "turn/interrupt") + closeCtx, stopClose := context.WithTimeout(t.Context(), 20*time.Millisecond) + defer stopClose() + if err := e.Close(closeCtx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatal("missing receipt did not reach the cleanup deadline", err) + } + // Retrying waits on the same owner; the expired observation must not skip + // the transport close that can actually stop an unresponsive native Turn. + retry, stopRetry := context.WithTimeout(t.Context(), 2*time.Second) + defer stopRetry() + if err := e.Close(retry); err != nil { + t.Fatal("Close never retired the native process", err) + } + if e.prepared.session.rpc.Alive() || len(preparedCatalogs(t, root)) != 0 { + t.Fatal("unresponsive native process or plan still owned after Close") + } + select { + case <-cancelled: + case <-retry.Done(): + t.Fatal("original cancellation waiter was abandoned") + } +} diff --git a/apps/parsar-daemon/internal/agent/codex/preparation_helpers_test.go b/apps/parsar-daemon/internal/agent/codex/preparation_helpers_test.go index 237d4f329..857e0082d 100644 --- a/apps/parsar-daemon/internal/agent/codex/preparation_helpers_test.go +++ b/apps/parsar-daemon/internal/agent/codex/preparation_helpers_test.go @@ -185,6 +185,11 @@ func TestPreparationFakeCodexProcess(t *testing.T) { } } result = map[string]any{"thread": map[string]string{"id": "fixture-native-thread"}, "model": "fixture-model"} + case "turn/interrupt": + if executorMode == "interrupt-error" { + _ = output.Encode(map[string]any{"id": frame.ID, "error": map[string]any{"code": -32603, "message": "interrupt rejected"}}) + continue + } case "turn/start": if executorMode != "" { turnNumber++ @@ -209,6 +214,9 @@ func TestPreparationFakeCodexProcess(t *testing.T) { } if executorMode != "" && (frame.Method == "turn/start" || frame.Method == "turn/interrupt") { + if frame.Method == "turn/interrupt" && executorMode == "interrupt-no-terminal" { + continue + } if frame.Method == "turn/start" { _ = output.Encode(map[string]any{"method": "turn/started", "params": map[string]any{"threadId": "fixture-native-thread", "turn": map[string]string{"id": currentTurn}}}) } diff --git a/apps/parsar-daemon/internal/dispatch/executor_handoff_test.go b/apps/parsar-daemon/internal/dispatch/executor_handoff_test.go new file mode 100644 index 000000000..730698ef6 --- /dev/null +++ b/apps/parsar-daemon/internal/dispatch/executor_handoff_test.go @@ -0,0 +1,228 @@ +package dispatch + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +type terminalHandoffSender struct { + frames chan proto.Envelope + visible chan struct{} + release chan struct{} + failure error +} + +func (s *terminalHandoffSender) Send(ctx context.Context, env proto.Envelope) error { + select { + case s.frames <- env: + case <-ctx.Done(): + return ctx.Err() + } + if env.Type == proto.TypeDone && env.ID == "first" { + close(s.visible) + select { + case <-s.release: + return s.failure + case <-ctx.Done(): + return ctx.Err() + } + } + return nil +} + +type terminalHandoffExecutor struct { + turns chan *terminalHandoffTurn + closes atomic.Int32 +} + +func (e *terminalHandoffExecutor) Close(context.Context) error { e.closes.Add(1); return nil } +func (e *terminalHandoffExecutor) StartTurn(_ context.Context, id string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Turn, error) { + turn := &terminalHandoffTurn{id: id, out: out, settled: make(chan struct{})} + e.turns <- turn + return turn, nil +} + +type terminalHandoffTurn struct { + id string + out chan<- proto.Envelope + settled chan struct{} + once sync.Once +} + +func (t *terminalHandoffTurn) finish() { + t.once.Do(func() { + env, _ := proto.NewEnvelope(proto.TypeDone, t.id, t.CancellationOutcome()) + t.out <- env + close(t.out) + close(t.settled) + }) +} +func (t *terminalHandoffTurn) Cancel(context.Context) error { t.finish(); return nil } +func (t *terminalHandoffTurn) CancellationOutcome() proto.DonePayload { + return proto.DonePayload{Content: t.id, Metadata: map[string]any{proto.DoneMetaAgentSessionID: "native"}} +} +func (t *terminalHandoffTurn) AwaitSettlement(ctx context.Context) (agent.TurnSettlement, error) { + select { + case <-t.settled: + return agent.TurnSettlement{Reusable: true}, nil + case <-ctx.Done(): + return agent.TurnSettlement{}, ctx.Err() + } +} + +func TestPreparedDonePublishesAfterExecutorHandoff(t *testing.T) { + for _, fail := range []bool{false, true} { + name := "delivered" + if fail { + name = "late_delivery_error" + } + t.Run(name, func(t *testing.T) { + sender := &terminalHandoffSender{frames: make(chan proto.Envelope, 32), visible: make(chan struct{}), release: make(chan struct{})} + if fail { + sender.failure = errors.New("old terminal delivery failed") + } + owner := &terminalHandoffExecutor{turns: make(chan *terminalHandoffTurn, 3)} + registry := agent.NewRegistry() + registry.RegisterKind(proto.SupportedAgentKind{Kind: "handoff", Available: true, Capabilities: proto.AgentKindCapabilities{EnvironmentNone: true}}, + func(context.Context, proto.PromptRequestPayload, chan<- proto.Envelope) (agent.Session, error) { + return nil, errors.New("legacy path forbidden") + }) + var creates atomic.Int32 + registry.RegisterExecutor("handoff", func(context.Context, proto.PromptRequestPayload) (agent.Executor, error) { + creates.Add(1) + return owner, nil + }) + r, err := New(Config{Registry: registry, Sender: sender, IdleTimeout: time.Hour}) + if err != nil { + t.Fatal(err) + } + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(sender.release) }) } + t.Cleanup(func() { + unblock() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := r.Shutdown(ctx); err != nil { + t.Error(err) + } + }) + handle := func(kind, id string, payload any) { + t.Helper() + env, err := proto.NewEnvelope(kind, id, payload) + if err != nil { + t.Fatal(err) + } + if err = r.Handle(t.Context(), env); err != nil { + t.Fatal(err) + } + } + status := func(id, state string) proto.PreparationStatusPayload { + t.Helper() + deadline := time.After(time.Second) + for { + select { + case env := <-sender.frames: + if env.Type != proto.TypePreparationStatus || env.ID != id { + continue + } + var value proto.PreparationStatusPayload + if err := env.DecodePayload(&value); err != nil { + t.Fatal(err) + } + if value.State == state { + return value + } + if value.State == "rejected" || value.State == "failed" { + t.Fatalf("next admission failed: %+v", value) + } + case <-deadline: + t.Fatal("preparation status missing") + } + } + } + request := proto.ExecutionPreparePayload{SessionID: "session", Configuration: proto.PromptRequestPayload{AgentKind: "handoff", AgentStateKey: "agents-api-session", StrictResume: true, DisableExecutionEnvironment: true}} + admit := func(id string) proto.PreparationStatusPayload { + t.Helper() + handle(proto.TypeExecutionPrepare, id, request) + return status(id, "ready") + } + start := func(id string, p proto.PreparationStatusPayload) *terminalHandoffTurn { + t.Helper() + handle(proto.TypeExecutionStart, id, proto.ExecutionStartPayload{Handle: p.Handle, ExecutorID: p.ExecutorID, RunID: id, Input: proto.TextInput(id)}) + status(id, "started") + select { + case turn := <-owner.turns: + return turn + case <-time.After(time.Second): + t.Fatal("native Turn missing") + return nil + } + } + first := admit("first") + old := start("first", first) + r.mu.Lock() + handoff := r.sessions["first"].preparedHandoff + r.mu.Unlock() + old.finish() + select { + case <-sender.visible: + case <-time.After(time.Second): + t.Fatal("Done not visible") + } + // Simulate Core admitting its successor after seeing Done, while the old + // Send is still blocked. Exact native continuity must already be committed. + request.Configuration.AgentSessionID = "native" + request.Configuration.RequireExistingNativeSession = true + second := admit("second") + if second.ExecutorID != first.ExecutorID || !second.Reused || creates.Load() != 1 { + t.Fatal("settled owner was replaced") + } + next := start("second", second) + unblock() + r.mu.Lock() + settled := handoff.release.settled + r.mu.Unlock() + select { + case <-settled: + case <-time.After(time.Second): + t.Fatal("old publication did not finish") + } + r.mu.Lock() + current := r.executors["session"] + valid := current != nil && !current.invalid && current.run == r.sessions["second"] && current.admission == r.preparations[second.Handle] + deliveryErr := handoff.outputErr + r.mu.Unlock() + if !valid || r.ActiveRuns() != 1 || owner.closes.Load() != 0 { + t.Fatal("old terminal cleanup damaged its successor") + } + if (deliveryErr != nil) != fail { + t.Fatalf("delivery error=%v", deliveryErr) + } + next.finish() + // The second terminal does not block: observing it must likewise permit an + // immediate third admission without losing or recreating the retained owner. + for { + select { + case env := <-sender.frames: + if env.Type == proto.TypeDone && env.ID == "second" { + goto completed + } + case <-time.After(time.Second): + t.Fatal("second Done missing") + } + } + completed: + third := admit("third") + if third.ExecutorID != first.ExecutorID || !third.Reused || creates.Load() != 1 { + t.Fatal("successor failed to retain owner") + } + }) + } +} diff --git a/apps/parsar-daemon/internal/dispatch/prepared_handoff.go b/apps/parsar-daemon/internal/dispatch/prepared_handoff.go index 3df8b2934..1b845b300 100644 --- a/apps/parsar-daemon/internal/dispatch/prepared_handoff.go +++ b/apps/parsar-daemon/internal/dispatch/prepared_handoff.go @@ -240,18 +240,17 @@ func (r *Router) runPreparedRelease(state *sessionState, handoff *preparedHandof r.mu.Lock() outputErr, terminal, closed := handoff.outputErr, handoff.terminal, r.closed r.mu.Unlock() - var terminalErr error - if outputErr == nil && !closed && turn != nil { - terminalErr = r.forwardPreparedTerminal(state, release.failure, terminal, terminal == nil) - } + // Done can become visible before Sender.Send returns. Commit the settled + // owner and retire this Run before publication so an immediate successor + // observes both the new native identity and an available Executor. r.cleanupSession(state) r.mu.Lock() - handoff.outputErr = errors.Join(handoff.outputErr, terminalErr) p := handoff.preparation p.busy, p.owns, p.handoff = false, false, nil - p.closeErr = terminalErr p.cancel() - owner.run = nil + if owner.run == state { + owner.run = nil + } if owner.invalid && owner.closeDone != nil && owner.closeErr == nil && r.executors[owner.sessionID] == owner { delete(r.executors, owner.sessionID) } @@ -263,17 +262,23 @@ func (r *Router) runPreparedRelease(state *sessionState, handoff *preparedHandof owner.nativeID = id } } - if terminalErr != nil { - owner.invalid = true - } if !owner.invalid { r.scheduleExecutorIdleLocked(owner) } - close(release.settled) r.mu.Unlock() - if terminalErr != nil { - _ = r.closeExecutor(owner) + + var terminalErr error + if outputErr == nil && !closed && turn != nil { + terminalErr = r.forwardPreparedTerminal(state, release.failure, terminal, terminal == nil) } + r.mu.Lock() + // Delivery failure belongs to this Run, not to a successor that may have + // already acquired the settled owner. Connection shutdown owns transport + // failure cleanup; terminal publication does not change native settlement. + handoff.outputErr = errors.Join(handoff.outputErr, terminalErr) + p.closeErr = terminalErr + close(release.settled) + r.mu.Unlock() } func (r *Router) forwardPreparedOutput(state *sessionState) { From 5b5c10fc4afeb20a1f5ac52207a9810ece85dbed Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Mon, 28 Sep 2026 15:26:26 +0000 Subject: [PATCH 3/8] Preserve native settlement evidence through executor cleanup --- CONTRIBUTING.md | 14 ++- .../agent/claudesdk/commands_session_test.go | 4 +- .../claudesdk/executor_confirmation_test.go | 70 +++++++++++++ .../agent/claudesdk/executor_fixture_test.go | 3 +- .../internal/agent/claudesdk/executor_test.go | 20 +++- .../internal/agent/claudesdk/executor_turn.go | 12 ++- .../agent/claudesdk/functions_test.go | 4 +- .../internal/agent/claudesdk/session.go | 9 +- .../claudesdk/workspace_directory_test.go | 3 +- .../agent/claudesdk/workspace_launch_test.go | 2 +- .../agent/claudesdk/workspace_read_test.go | 3 +- .../internal/agent/codex/executor_test.go | 19 +++- .../internal/agent/codex/executor_turn.go | 11 ++- .../agent/mcode/environment_mcp_test.go | 4 +- .../agent/mcode/executor_backpressure_test.go | 99 +++++++++++++++++++ .../agent/mcode/executor_native_test.go | 10 +- .../internal/agent/mcode/executor_test.go | 8 +- .../internal/agent/mcode/executor_turn.go | 18 ++-- .../internal/agent/mcode/session.go | 8 +- .../internal/agent/mcode/session_test.go | 9 ++ packages/claude-sdk-adapter/src/adapter.ts | 4 +- .../src/executor_protocol.ts | 6 +- .../tests/executor.test.mjs | 14 ++- scripts/name-allowlist.json | 20 ++++ 24 files changed, 315 insertions(+), 59 deletions(-) create mode 100644 apps/parsar-daemon/internal/agent/claudesdk/executor_confirmation_test.go create mode 100644 apps/parsar-daemon/internal/agent/mcode/executor_backpressure_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ff5d92c26..584f24c6f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1601,7 +1601,10 @@ input was submitted. Recheck the same physical peer and current authorization. `Turn.Cancel` targets only that Turn and does not close a healthy Executor. `AwaitSettlement` applies after both natural completion and cancellation. Success means output can no longer be written and the Turn's native events, input, -functions, interactions and child work have settled. `Reusable=true` additionally +functions, interactions and child work have settled. Native completion or +cancellation confirmation is independent of resource retirement: closing a +transport cannot supply missing native terminal or operation receipts. +`Reusable=true` additionally confirms that the native owner can accept the next Turn. `Reusable=false` requires a reason and subsequent confirmed Executor close. An error means settlement is unconfirmed; it cannot free ownership or capacity. Caller deadlines stop waiting, @@ -3213,7 +3216,14 @@ The SDK's per-block assistant snapshots replace draft block text; only native Thinking/tool-only messages produce no text Items; interrupted messages retain their streamed partial text. No phase is inferred from the final result. Turn-owned native work and output draining precede reuse. Executor close releases -the SDK Query and native process. +the SDK Query and native process. The private `turn_settled` frame requires +`confirmed` independently of `reusable`: confirmed native Turn/cancellation +settlement, confirmed resource cleanup, and reuse eligibility are separate facts. +Unknown or nonempty interrupt receipts and unsettled input/function/tool work +remain unconfirmed even after successful teardown. Go rejects cancellation and +AwaitSettlement when native confirmation is missing or false, or its own receipt +ledger remains unsettled. A confirmed Turn may be non-reusable after cleanup; +that state alone does not turn a verified cancellation into an error. `claudesdk.Config.Workspace` is a private, trusted operator binding for one qualified placement. It enables native Bash/Read/Edit and declared host functions diff --git a/apps/parsar-daemon/internal/agent/claudesdk/commands_session_test.go b/apps/parsar-daemon/internal/agent/claudesdk/commands_session_test.go index 5a2d6e41b..ebed0792c 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/commands_session_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/commands_session_test.go @@ -119,8 +119,8 @@ func TestWorkspaceCommandCancellationAndBridgeFailuresCloseOnlyPendingCalls(t *t t.Fatal(err) } } else if mode == "commands-cancel" || mode == "commands-drained" { - if err := s.Cancel(ctx); err != nil { - t.Fatal(err) + if err := s.Cancel(ctx); (err == nil) != (mode == "commands-drained") { + t.Fatalf("command cancellation confirmation: %v", err) } } } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/executor_confirmation_test.go b/apps/parsar-daemon/internal/agent/claudesdk/executor_confirmation_test.go new file mode 100644 index 000000000..bf74b120b --- /dev/null +++ b/apps/parsar-daemon/internal/agent/claudesdk/executor_confirmation_test.go @@ -0,0 +1,70 @@ +//go:build unix + +package claudesdk + +import ( + "context" + "encoding/json" + "testing" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +func TestExecutorNativeConfirmationSurvivesCleanup(t *testing.T) { + for _, mode := range []string{"unknown_cancel", "queued_cancel", "missing_confirmation", "pending_function", "pending_input", "confirmed_closed"} { + t.Run(mode, func(t *testing.T) { + config, req := persistentConfig(t, mode) + if mode == "pending_function" { + req.FunctionTools = []proto.FunctionTool{{Name: "lookup", Parameters: json.RawMessage(`{"type":"object"}`)}} + } + owner, err := NewExecutorFactory(config)(t.Context(), req) + if err != nil { + t.Fatal(err) + } + defer owner.Close(context.Background()) + turn, out := consumeExecutorTurn(t, owner, "cancel", "wait") + if event := <-out; event.Type != proto.TypeDelta { + t.Fatal("initial output missing") + } + if mode == "pending_function" { + if event := <-out; event.Type != proto.TypeFunctionCall { + t.Fatal("function obligation missing") + } + } + var inputReceipt chan error + if mode == "pending_input" { + inputReceipt = make(chan error, 1) + go func() { + inputReceipt <- turn.(*session).SteerWithReceipt(t.Context(), proto.PromptSteerPayload{InputID: "input", Input: proto.TextInput("extra")}, nil) + }() + if event := <-out; event.Type != proto.TypeDelta { + t.Fatal("input write barrier missing") + } + } + cancelErr := turn.Cancel(t.Context()) + for range out { + } + settlement, settlementErr := turn.AwaitSettlement(t.Context()) + confirmed := mode == "confirmed_closed" + if (cancelErr == nil) != confirmed || (settlementErr == nil) != confirmed || settlement.Reusable { + t.Fatalf("cancel=%v settlement=%+v err=%v", cancelErr, settlement, settlementErr) + } + if inputReceipt != nil { + if err := <-inputReceipt; err == nil { + t.Fatal("missing native input receipt was confirmed") + } + } + // Resource cleanup succeeds independently. It must not erase the immutable + // native outcome used by repeat cancellation and Runtime acknowledgement. + if err := owner.Close(t.Context()); err != nil { + t.Fatal(err) + } + if (turn.Cancel(t.Context()) == nil) != confirmed { + t.Fatal("Close changed cancellation confirmation") + } + if _, err := turn.AwaitSettlement(t.Context()); (err == nil) != confirmed { + t.Fatal("Close changed Turn confirmation") + } + }) + } +} diff --git a/apps/parsar-daemon/internal/agent/claudesdk/executor_fixture_test.go b/apps/parsar-daemon/internal/agent/claudesdk/executor_fixture_test.go index 5b2c9244d..55768099f 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/executor_fixture_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/executor_fixture_test.go @@ -46,7 +46,8 @@ func helperTurnOutput(scanner *bufio.Scanner, id string) (func(bridgeEvent), fun if !reusable { reason = "native_error" } - encode(bridgeEvent{Type: "turn_settled", Reusable: &reusable, Reason: reason}) + confirmed := true + encode(bridgeEvent{Type: "turn_settled", Confirmed: &confirmed, Reusable: &reusable, Reason: reason}) for scanner.Scan() { } } diff --git a/apps/parsar-daemon/internal/agent/claudesdk/executor_test.go b/apps/parsar-daemon/internal/agent/claudesdk/executor_test.go index 755a0bc53..c699b63ae 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/executor_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/executor_test.go @@ -63,8 +63,18 @@ func runPersistentExecutorHelper() { } else { encode(bridgeEvent{Type: "result", TurnID: active, SessionID: "native-persistent", Text: active}) } - yes := true - encode(bridgeEvent{Type: "turn_settled", TurnID: active, Reusable: &yes}) + confirmed, reusable, reason := true, true, "" + switch os.Getenv("SDK_EXECUTOR_MODE") { + case "unknown_cancel", "queued_cancel": + confirmed, reusable, reason = false, false, "cancellation_unconfirmed" + case "confirmed_closed": + reusable, reason = false, "native_closed" + } + event := bridgeEvent{Type: "turn_settled", TurnID: active, Confirmed: &confirmed, Reusable: &reusable, Reason: reason} + if os.Getenv("SDK_EXECUTOR_MODE") == "missing_confirmation" { + event.Confirmed = nil + } + encode(event) old, active = active, "" } for scanner.Scan() { @@ -89,10 +99,16 @@ func runPersistentExecutorHelper() { encode(bridgeEvent{Type: "turn_started", TurnID: active}) encode(bridgeEvent{Type: "input_ready", TurnID: active, SessionID: "native-persistent"}) encode(bridgeEvent{Type: "delta", TurnID: active, Delta: "partial"}) + if os.Getenv("SDK_EXECUTOR_MODE") == "pending_function" { + encode(bridgeEvent{Type: "function_call", TurnID: active, Call: &proto.FunctionCallPayload{CallID: "call", Name: "lookup", Arguments: json.RawMessage("{}")}}) + } if len(command.Input) > 0 && len(command.Input[0].Content) > 0 && command.Input[0].Content[0].Text != nil && *command.Input[0].Content[0].Text == "wait" { continue } settle(false) + case "steer": + // A full bridge write has happened, but no native input receipt exists. + encode(bridgeEvent{Type: "delta", TurnID: active, Delta: "steer-written"}) case "turn_cancel": if active == command.TurnID { settle(true) diff --git a/apps/parsar-daemon/internal/agent/claudesdk/executor_turn.go b/apps/parsar-daemon/internal/agent/claudesdk/executor_turn.go index 14f5067ac..285f4caf1 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/executor_turn.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/executor_turn.go @@ -56,6 +56,7 @@ func (s *session) runTurn(start startRequest, out chan<- proto.Envelope) { mcp := mcpState{calls: map[string]proto.ToolObservation{}} commands := commandState{calls: map[string]proto.ToolObservation{}} settlementReceived := false + settlementConfirmed := false reusable := false reason := "bridge_interrupted" for raw := range s.frames { @@ -69,12 +70,12 @@ func (s *session) runTurn(start startRequest, out chan<- proto.Envelope) { continue } if event.Type == "turn_settled" { - if !terminal || event.Reusable == nil || (!*event.Reusable && event.Reason == "") { + if !terminal || event.Reusable == nil || event.Confirmed == nil || (*event.Reusable && !*event.Confirmed) || (!*event.Reusable && event.Reason == "") { failure = fmt.Errorf("claudesdk: invalid Turn settlement") s.invalidate() break } - settlementReceived, reusable, reason = true, *event.Reusable, event.Reason + settlementReceived, settlementConfirmed, reusable, reason = true, *event.Confirmed, *event.Reusable, event.Reason break } if terminal { @@ -149,6 +150,7 @@ func (s *session) runTurn(start startRequest, out chan<- proto.Envelope) { case "result": if !s.matchesInputSession(event.SessionID) || event.SessionID == "" || start.Resume != "" && event.SessionID != start.Resume || usageSession != "" && event.SessionID != usageSession || !s.functionsComplete() || !s.steeringComplete() || !mcp.complete() || !commands.complete() { failure = fmt.Errorf("claudesdk: invalid native completion or unconfirmed input/result") + s.settlementErr = failure s.invalidate() } else { result = &event @@ -172,7 +174,7 @@ func (s *session) runTurn(start startRequest, out chan<- proto.Envelope) { failure = fmt.Errorf("claudesdk: SDK result is missing") } if !s.functionsComplete() || !s.steeringComplete() || !mcp.complete() || !commands.complete() { - reusable, reason = false, "unsettled_native_operations" + settlementConfirmed, reusable, reason = false, false, "unsettled_native_operations" } mcp.close(start, emit) commands.close(start, emit) @@ -189,14 +191,14 @@ func (s *session) runTurn(start startRequest, out chan<- proto.Envelope) { } s.outcome = proto.DonePayload{Content: content.String(), Usage: usage, Metadata: metadata} // Publish the observed cancellation outcome before terminal delivery. - if !settlementReceived || !reusable || outputLost || s.process.Context().Err() != nil { + if s.settlementErr != nil || !settlementReceived || !settlementConfirmed || !reusable || outputLost || s.process.Context().Err() != nil { s.owner.retire() reusable = false if reason == "" { reason = "bridge_interrupted" } } - if !settlementReceived { + if !settlementReceived || !settlementConfirmed { s.settlementErr = fmt.Errorf("claudesdk: native Turn settlement is unconfirmed") } close(s.settled) diff --git a/apps/parsar-daemon/internal/agent/claudesdk/functions_test.go b/apps/parsar-daemon/internal/agent/claudesdk/functions_test.go index f64954b70..09045a612 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/functions_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/functions_test.go @@ -75,8 +75,8 @@ func TestFunctionFactoryNativeReceipts(t *testing.T) { } case proto.TypeDelta: if mode == "functions-cancel" { - if err := running.Cancel(ctx); err != nil { - t.Fatal(err) + if err := running.Cancel(ctx); err == nil { + t.Fatal("unconfirmed function receipts became successful cancellation") } } case proto.TypeError: diff --git a/apps/parsar-daemon/internal/agent/claudesdk/session.go b/apps/parsar-daemon/internal/agent/claudesdk/session.go index dc612355f..f84ff79c0 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/session.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/session.go @@ -55,10 +55,11 @@ func NewFactory(config Config) agent.Factory { } type bridgeEvent struct { - TurnID string `json:"turn_id"` - Reusable *bool `json:"reusable"` - Reason string `json:"reason"` - Protocol int `json:"protocol"` + TurnID string `json:"turn_id"` + Reusable *bool `json:"reusable"` + Confirmed *bool `json:"confirmed"` + Reason string `json:"reason"` + Protocol int `json:"protocol"` Fact json.RawMessage `json:"fact"` InputID string `json:"input_id"` diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory_test.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory_test.go index 2cb969cf2..bc4822ce4 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_directory_test.go @@ -32,7 +32,8 @@ func runWorkspaceDirectoryHelper(scanner *bufio.Scanner, state, mode string) { if req.Type == "turn_cancel" { reusable := false _ = json.NewEncoder(os.Stdout).Encode(bridgeEvent{Type: "error", TurnID: turnID, Code: "cancelled"}) - _ = json.NewEncoder(os.Stdout).Encode(bridgeEvent{Type: "turn_settled", TurnID: turnID, Reusable: &reusable, Reason: "fixture_closed"}) + confirmed := true + _ = json.NewEncoder(os.Stdout).Encode(bridgeEvent{Type: "turn_settled", Confirmed: &confirmed, TurnID: turnID, Reusable: &reusable, Reason: "fixture_closed"}) return } if req.Type == "turn_start" { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_launch_test.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_launch_test.go index 556db3e7f..5a3c4ad98 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_launch_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_launch_test.go @@ -30,7 +30,7 @@ case "$1" in printf '%s\n' '{"type":"executor_ready","protocol":3}' IFS= read -r request printf '%s\n' '{"type":"result","turn_id":"run","session_id":"native","text":"completed"}' - printf '%s\n' '{"type":"turn_settled","turn_id":"run","reusable":true,"reason":""}' ;; + printf '%s\n' '{"type":"turn_settled","turn_id":"run","confirmed":true,"reusable":true,"reason":""}' ;; esac ` if err := os.WriteFile(config.Node, []byte(script), 0o700); err != nil { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/workspace_read_test.go b/apps/parsar-daemon/internal/agent/claudesdk/workspace_read_test.go index ca46342ef..d9bfa2221 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/workspace_read_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/workspace_read_test.go @@ -31,7 +31,8 @@ func runWorkspaceReadHelper(scanner *bufio.Scanner, state, mode string) { if req.Type == "turn_cancel" { reusable := false _ = json.NewEncoder(os.Stdout).Encode(bridgeEvent{Type: "error", TurnID: turnID, Code: "cancelled"}) - _ = json.NewEncoder(os.Stdout).Encode(bridgeEvent{Type: "turn_settled", TurnID: turnID, Reusable: &reusable, Reason: "fixture_closed"}) + confirmed := true + _ = json.NewEncoder(os.Stdout).Encode(bridgeEvent{Type: "turn_settled", Confirmed: &confirmed, TurnID: turnID, Reusable: &reusable, Reason: "fixture_closed"}) return } if req.Type == "turn_start" { diff --git a/apps/parsar-daemon/internal/agent/codex/executor_test.go b/apps/parsar-daemon/internal/agent/codex/executor_test.go index c991dd626..6cd32f60a 100644 --- a/apps/parsar-daemon/internal/agent/codex/executor_test.go +++ b/apps/parsar-daemon/internal/agent/codex/executor_test.go @@ -145,8 +145,18 @@ func TestExecutorStartErrorsRetainExactOwnership(t *testing.T) { if turn == nil || err == nil { t.Fatal("uncertain submission lost owner", err) } - if awaitExecutorTurn(t, turn, out).Reusable { - t.Fatal("failed submission was reusable") + settlement, settleErr := turn.AwaitSettlement(t.Context()) + if settleErr == nil || settlement.Reusable { + t.Fatal("failed submission fabricated native settlement", settlement, settleErr) + } + terminalCount := 0 + for frame := range out { + if frame.Type == proto.TypeDone { + terminalCount++ + } + } + if terminalCount != 1 { + t.Fatalf("failed submission terminal count = %d", terminalCount) } }) } @@ -240,7 +250,10 @@ func TestExecutorCloseTerminatesAfterMissingCancellationTerminal(t *testing.T) { t.Fatal("unresponsive native process or plan still owned after Close") } select { - case <-cancelled: + case cancelErr := <-cancelled: + if cancelErr == nil { + t.Fatal("resource cleanup fabricated native cancellation confirmation") + } case <-retry.Done(): t.Fatal("original cancellation waiter was abandoned") } diff --git a/apps/parsar-daemon/internal/agent/codex/executor_turn.go b/apps/parsar-daemon/internal/agent/codex/executor_turn.go index 216d26bbd..a356f30f7 100644 --- a/apps/parsar-daemon/internal/agent/codex/executor_turn.go +++ b/apps/parsar-daemon/internal/agent/codex/executor_turn.go @@ -33,7 +33,10 @@ func (s *Session) settleExecutorTurn(startErr error) { // replies. Later frames cannot acquire this Turn or redirect to a successor. if !s.nativeSettled.Load() || startErr != nil || !s.rpc.Alive() { s.cancelFn() - s.settlementErr = s.rpc.Close() + s.settlementErr = errors.Join(s.settlementErr, s.rpc.Close()) + } + if !s.nativeSettled.Load() { + s.settlementErr = errors.Join(s.settlementErr, errors.New("codex: native Turn settlement is unconfirmed")) } s.rpc.detachHandlers() s.operationMu.Lock() @@ -44,7 +47,7 @@ func (s *Session) settleExecutorTurn(startErr error) { s.stopCodexInteractionTimers() if !s.nativeSettled.Load() || startErr != nil || !s.rpc.Alive() { s.cancelFn() - s.settlementErr = s.rpc.Close() + s.settlementErr = errors.Join(s.settlementErr, s.rpc.Close()) s.settlement = agent.TurnSettlement{Reason: "native_execution_unavailable"} } else { s.settlement = agent.TurnSettlement{Reusable: true} @@ -57,7 +60,7 @@ func (s *Session) settleExecutorTurn(startErr error) { s.subagents.mu.Unlock() if err != nil { s.settlement = agent.TurnSettlement{Reason: "child_settlement_unconfirmed"} - s.settlementErr = errors.Join(err, s.rpc.Close()) + s.settlementErr = errors.Join(s.settlementErr, err, s.rpc.Close()) } } // The terminal callback has returned before detachHandlers completes. A @@ -70,7 +73,7 @@ func (s *Session) settleExecutorTurn(startErr error) { if s.cancelErr != nil { s.settlement.Reusable = false s.settlement.Reason = "cancellation_unconfirmed" - s.settlementErr = errors.Join(s.cancelErr, s.rpc.Close()) + s.settlementErr = errors.Join(s.settlementErr, s.cancelErr, s.rpc.Close()) } } s.cancelFn() diff --git a/apps/parsar-daemon/internal/agent/mcode/environment_mcp_test.go b/apps/parsar-daemon/internal/agent/mcode/environment_mcp_test.go index e9cb6f18c..20d73b99c 100644 --- a/apps/parsar-daemon/internal/agent/mcode/environment_mcp_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/environment_mcp_test.go @@ -124,8 +124,8 @@ func TestEnvironmentMCPCancelSettlesPendingObservationBeforeDone(t *testing.T) { case <-ctx.Done(): t.Fatal("MCP start was not observed") } - if err := session.Cancel(ctx); err == nil { - t.Fatal("unconfirmed MCP completion authorized reusable cancellation") + if err := session.Cancel(ctx); err != nil { + t.Fatal("stopped nonreusable MCP owner reported cancellation failure", err) } settlement, err := session.(*Session).AwaitSettlement(ctx) if err != nil || settlement.Reusable { diff --git a/apps/parsar-daemon/internal/agent/mcode/executor_backpressure_test.go b/apps/parsar-daemon/internal/agent/mcode/executor_backpressure_test.go new file mode 100644 index 000000000..f2f78c6df --- /dev/null +++ b/apps/parsar-daemon/internal/agent/mcode/executor_backpressure_test.go @@ -0,0 +1,99 @@ +package mcode + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +func TestExecutorBlockedPromptWriteCanBeCancelledOrClosed(t *testing.T) { + for _, operation := range []string{"cancel", "close"} { + t.Run(operation, func(t *testing.T) { + e, _ := executorFixture(t, "executor-backpressure", true) + out := make(chan proto.Envelope, 32) + turn, err := e.StartTurn(t.Context(), "blocked", proto.TextInput(strings.Repeat("x", 4*1024*1024)), out) + if err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(3 * time.Second) + for e.connection.writeMu.TryLock() { + e.connection.writeMu.Unlock() + if time.Now().After(deadline) { + t.Fatal("prompt did not reach native pipe") + } + time.Sleep(time.Millisecond) + } + // The native fixture acknowledged readiness but does not read stdin. + // A multi-megabyte prompt therefore fills the actual child process pipe. + result := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(t.Context(), 200*time.Millisecond) + defer cancel() + if operation == "cancel" { + result <- turn.Cancel(ctx) + } else { + result <- e.Close(ctx) + } + }() + select { + case err := <-result: + if operation == "cancel" && err == nil { + t.Fatal("unacknowledged cancellation was reported applied") + } + case <-time.After(3 * time.Second): + // Emergency cleanup avoids leaking the intentionally blocked fixture. + e.connection.process.Cancel() + t.Fatal(operation + " could not interrupt a blocked native write") + } + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + if err := e.Close(ctx); err != nil { + t.Fatal("Close did not retain cleanup ownership", err) + } + settlement, err := turn.AwaitSettlement(ctx) + if err == nil || settlement.Reusable { + t.Fatal("lost native prompt produced successful settlement", settlement, err) + } + for range out { + } + }) + } +} + +func TestExecutorUnknownSteeringOutcomeCannotBecomeAppliedCancellation(t *testing.T) { + e, _ := executorFixture(t, "executor-steer-unknown", true) + out := make(chan proto.Envelope, 32) + turn, err := e.StartTurn(t.Context(), "unknown-input", proto.TextInput("wait"), out) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + select { + case <-out: + case <-ctx.Done(): + t.Fatal("native prompt did not start") + } + session := turn.(*Session) + if err := session.Steer(ctx, proto.PromptSteerPayload{InputID: "input-unknown", Input: proto.TextInput("continue")}); err == nil { + t.Fatal("unknown native receipt was accepted") + } + if err := turn.Cancel(ctx); err == nil { + t.Fatal("unknown input became applied cancellation") + } + settlement, err := turn.AwaitSettlement(ctx) + if err == nil || settlement.Reusable { + t.Fatal("unknown input lost its settlement error", settlement, err) + } + if err := turn.Cancel(ctx); err == nil { + t.Fatal("repeated cancel lost original uncertainty") + } + if err := e.Close(ctx); err != nil { + t.Fatal("settlement failure prevented resource cleanup", err) + } + for range out { + } +} diff --git a/apps/parsar-daemon/internal/agent/mcode/executor_native_test.go b/apps/parsar-daemon/internal/agent/mcode/executor_native_test.go index 4a1e23949..5340c3f9f 100644 --- a/apps/parsar-daemon/internal/agent/mcode/executor_native_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/executor_native_test.go @@ -136,10 +136,10 @@ func TestNativeMCodeExecutorReuse(t *testing.T) { cancelErr := interrupted.Cancel(stopCtx) settlement, settleErr := interrupted.AwaitSettlement(stopCtx) stop() - if settleErr != nil || !settlement.Reusable { - if cancelErr == nil { - t.Fatal("uncertain cancellation reported success") - } + if settleErr != nil || cancelErr != nil { + t.Fatal("native cancellation did not settle", cancelErr, settleErr) + } + if !settlement.Reusable { t.Log("Native cancellation cannot establish reusable settlement; successor must use recovery") successor := make(chan proto.Envelope, 1) next, nextErr := e.StartTurn(ctx, "unsafe-successor", proto.TextInput("must not execute"), successor) @@ -164,7 +164,7 @@ func TestNativeMCodeExecutorReuse(t *testing.T) { } pid = e.connection.process.Cmd.Process.Pid run("recovery", "What exact marker did I ask you to remember at the beginning? Reply with only that marker.") - t.Log("Verified exact history continuation in a replacement native owner after uncertain cancellation") + t.Log("Verified exact history continuation in a replacement native owner after nonreusable cancellation") return } if cancelErr != nil { diff --git a/apps/parsar-daemon/internal/agent/mcode/executor_test.go b/apps/parsar-daemon/internal/agent/mcode/executor_test.go index 17f532041..f0e44f72c 100644 --- a/apps/parsar-daemon/internal/agent/mcode/executor_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/executor_test.go @@ -140,8 +140,8 @@ func TestExecutorCancellationWithoutRootProofInvalidatesAndLateCancelCannotRetar } ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) defer cancel() - if err = second.Cancel(ctx); err == nil { - t.Fatal("unproven root settlement reported reusable cancellation") + if err = second.Cancel(ctx); err != nil { + t.Fatal("stopped nonreusable owner reported cancellation failure", err) } settlement, err := second.AwaitSettlement(ctx) if err != nil || settlement.Reusable { @@ -209,8 +209,8 @@ func TestExecutorStartFailureOwnership(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) defer cancel() settlement, err := turn.AwaitSettlement(ctx) - if err != nil || settlement.Reusable { - t.Fatal("exited process remained reusable", settlement, err) + if err == nil || settlement.Reusable { + t.Fatal("native exit lost its settlement error", settlement, err) } raw, _ := os.ReadFile(record) if strings.Count(string(raw), "session/prompt\n") != 1 { diff --git a/apps/parsar-daemon/internal/agent/mcode/executor_turn.go b/apps/parsar-daemon/internal/agent/mcode/executor_turn.go index 540a6202b..6a43459f5 100644 --- a/apps/parsar-daemon/internal/agent/mcode/executor_turn.go +++ b/apps/parsar-daemon/internal/agent/mcode/executor_turn.go @@ -45,6 +45,9 @@ func (s *Session) runExecutorTurn() { if (s.cancelled && s.req.DisableSubagents) || s.inputUncertain || len(s.permissions) != 0 || len(s.questions) != 0 { reusable = false } + if s.inputUncertain && err == nil { + err = fmt.Errorf("mcode: native input outcome is unknown") + } metadata := map[string]any{proto.DoneMetaAgentSessionType: "mcode", proto.DoneMetaAgentSessionID: s.sessionID} s.outcome = proto.DonePayload{Content: s.content.String(), Metadata: metadata, SourceCompletedAtMS: s.rootCompletedAtMS} outcome := s.outcome @@ -69,6 +72,7 @@ func (s *Session) runExecutorTurn() { } s.executor.active = nil s.settlement = agent.TurnSettlement{Reusable: reusable} + s.settlementErr = err if !reusable { s.settlement.Reason = "native Turn did not establish reusable settlement" } @@ -100,7 +104,7 @@ func (s *Session) AwaitSettlement(ctx context.Context) (agent.TurnSettlement, er } select { case <-s.settled: - return s.settlement, nil + return s.settlement, s.settlementErr case <-ctx.Done(): return agent.TurnSettlement{}, ctx.Err() } @@ -124,12 +128,12 @@ func (s *Session) cancelTurn(ctx context.Context) error { // Cancellation releases event backpressure but does not cancel native owner // context. ACP prompt completion and child/tool settlement decide reuse. s.outputCancel() + e.mu.Unlock() var err error if first { raw, _ := json.Marshal(map[string]string{"sessionId": s.sessionID}) err = s.writeContext(ctx, rpcFrame{JSONRPC: "2.0", Method: "session/cancel", Params: raw}) } - e.mu.Unlock() if first { if err == nil && s.req.StrictResume && !s.req.DisableSubagents { err = s.stopSubagents(ctx) @@ -142,12 +146,6 @@ func (s *Session) cancelTurn(ctx context.Context) error { if err != nil { return err } - settlement, err := s.AwaitSettlement(ctx) - if err != nil { - return err - } - if !settlement.Reusable { - return fmt.Errorf("mcode: cancellation did not establish reusable settlement") - } - return nil + _, err = s.AwaitSettlement(ctx) + return err } diff --git a/apps/parsar-daemon/internal/agent/mcode/session.go b/apps/parsar-daemon/internal/agent/mcode/session.go index 89f51251f..e65948f08 100644 --- a/apps/parsar-daemon/internal/agent/mcode/session.go +++ b/apps/parsar-daemon/internal/agent/mcode/session.go @@ -23,6 +23,7 @@ type Session struct { *connection executor *executor settlement agent.TurnSettlement + settlementErr error settled chan struct{} inputDone chan struct{} outputCancel context.CancelFunc @@ -275,16 +276,21 @@ func (s *Session) call(method string, params any, result any, prompt bool) error return err } if prompt && s.executor != nil { + // Serialize the admission check with the wire, but release the owner + // mutex before a pipe write so cancellation and Close can stop it. + s.connection.writeMu.Lock() s.executor.mu.Lock() s.mu.Lock() cancelled := s.cancelled s.mu.Unlock() if cancelled || s.executor.closed { s.executor.mu.Unlock() + s.connection.writeMu.Unlock() return errTurnCancelled } - err = s.write(rpcFrame{JSONRPC: "2.0", ID: json.RawMessage(id), Method: method, Params: raw}) s.executor.mu.Unlock() + err = json.NewEncoder(s.process.Stdin).Encode(rpcFrame{JSONRPC: "2.0", ID: json.RawMessage(id), Method: method, Params: raw}) + s.connection.writeMu.Unlock() } else { err = s.write(rpcFrame{JSONRPC: "2.0", ID: json.RawMessage(id), Method: method, Params: raw}) } diff --git a/apps/parsar-daemon/internal/agent/mcode/session_test.go b/apps/parsar-daemon/internal/agent/mcode/session_test.go index 641fe93a0..8d0fb9537 100644 --- a/apps/parsar-daemon/internal/agent/mcode/session_test.go +++ b/apps/parsar-daemon/internal/agent/mcode/session_test.go @@ -250,6 +250,11 @@ func TestMCodeProcess(t *testing.T) { } case "session/set_config_option": configurations++ + if scenario == "executor-backpressure" && configurations > 1 { + send(rpcFrame{JSONRPC: "2.0", ID: frame.ID, Result: json.RawMessage("{}")}) + time.Sleep(time.Minute) + os.Exit(0) + } if scenario == "executor-preexit" && configurations > 1 { os.Exit(0) } @@ -327,6 +332,10 @@ func TestMCodeProcess(t *testing.T) { send(rpcFrame{JSONRPC: "2.0", ID: promptID, Result: raw}) continue case "mcode/session/steer": + if scenario == "executor-steer-unknown" { + send(rpcFrame{JSONRPC: "2.0", ID: frame.ID, Error: &rpcError{Code: -32000, Message: "Unknown input outcome"}}) + continue + } if scenario == "steer-lost" { os.Exit(0) } diff --git a/packages/claude-sdk-adapter/src/adapter.ts b/packages/claude-sdk-adapter/src/adapter.ts index 3ab8b88d7..7f63e0d6c 100644 --- a/packages/claude-sdk-adapter/src/adapter.ts +++ b/packages/claude-sdk-adapter/src/adapter.ts @@ -204,7 +204,7 @@ export async function execute(request: Start | Prepare | ExecutorPrepare, emit: if(subagents) for(const event of await subagents.facts()) await emit(event); await emit(turns.cancelled ? {type:"error",code:"cancelled"} : result); functions.close(); - await turns.settled(true,""); + await turns.settled(true,true,""); result=undefined; } } @@ -237,7 +237,7 @@ export async function execute(request: Start | Prepare | ExecutorPrepare, emit: if(turns) { if(turns.id) { await emit({type:"error",code: turns.cancelled && !cancellationFactsFailed ? "cancelled" : "execution_failed"}); - await turns.settled(false, turns.cancelled ? "cancellation_unconfirmed" : "native_execution_unavailable"); + await turns.settled(false, false, turns.cancelled ? "cancellation_unconfirmed" : "native_execution_unavailable"); } else if(failed && !abort.signal.aborted) await ownerEmit({type:"error",code:"execution_failed"}); return; } diff --git a/packages/claude-sdk-adapter/src/executor_protocol.ts b/packages/claude-sdk-adapter/src/executor_protocol.ts index ba4e4c162..b8e715adf 100644 --- a/packages/claude-sdk-adapter/src/executor_protocol.ts +++ b/packages/claude-sdk-adapter/src/executor_protocol.ts @@ -5,7 +5,7 @@ import { parseMessageInput, type MessageInput } from "./message_input.js"; export type ExecutorEvent = | { type: "executor_ready"; protocol: 3 } | { type: "turn_started"; turn_id: string } - | { type: "turn_settled"; turn_id: string; reusable: boolean; reason: string }; + | { type: "turn_settled"; turn_id: string; confirmed: boolean; reusable: boolean; reason: string }; type WireEvent = { type: string; [key: string]: unknown }; type Turn = { id: string; inputs: Inputs; cancelled: boolean; interrupt?: Promise; callbacks: Set> }; @@ -111,12 +111,12 @@ export class ExecutorTurns implements AsyncIterable { const turn = this.active; while (turn?.callbacks.size) await Promise.allSettled([...turn.callbacks]); } - async settled(reusable: boolean, reason: string): Promise { + async settled(confirmed: boolean, reusable: boolean, reason: string): Promise { const turn = this.active; if (!turn) return; turn.inputs.close(); this.active = undefined; - await this.output({ type: "turn_settled", turn_id: turn.id, reusable, reason }); + await this.output({ type: "turn_settled", turn_id: turn.id, confirmed, reusable, reason }); } close(): void { this.ended = true; this.active?.inputs.close(); this.wake?.(); } async *[Symbol.asyncIterator](): AsyncIterator { diff --git a/packages/claude-sdk-adapter/tests/executor.test.mjs b/packages/claude-sdk-adapter/tests/executor.test.mjs index 79ef95c4c..39c6f1e90 100644 --- a/packages/claude-sdk-adapter/tests/executor.test.mjs +++ b/packages/claude-sdk-adapter/tests/executor.test.mjs @@ -24,7 +24,7 @@ globalThis.startupFixture=async({options})=>{ options.abortController.signal.addEventListener('abort',close); return {close,query(prompt){assert.equal(queried,false);queried=true;process.send({type:'query'});return { close,async initializationResult(){return process.argv[1]==="no-hook-report" ? {} : process.argv[1]==="false-hook-report" ? {hooks_applied:false} : {hooks_applied:true}}, - async interrupt(){process.send({type:'interrupt'});interrupt?.();return process.argv[1]==='unknown' ? undefined : {still_queued:process.argv[1]==='queued' ? ['not-consumed'] : []}}, + async interrupt(){process.send({type:'interrupt'});interrupt?.();if(process.argv[1]==='rejected')throw new Error('interrupt failed');return process.argv[1]==='unknown' ? undefined : {still_queued:process.argv[1]==='queued' ? ['not-consumed'] : []}}, async *[Symbol.asyncIterator](){ for await(const user of prompt){ number++; @@ -76,7 +76,7 @@ async function launch(t,mode="normal") { test("executor retains one native process and one Query over two settled Turns",{timeout:10000},async t=>{ const {child,events,observations,closed,wait,start}=await launch(t); - for(const id of ["first","second"]){start(id,"answer");await wait(()=>events.some(event=>event.type==="turn_settled"&&event.turn_id===id));assert.equal(events.find(event=>event.type==="turn_settled"&&event.turn_id===id).reusable,true)} + for(const id of ["first","second"]){start(id,"answer");await wait(()=>events.some(event=>event.type==="turn_settled"&&event.turn_id===id));assert.equal(events.find(event=>event.type==="turn_settled"&&event.turn_id===id).confirmed,true);assert.equal(events.find(event=>event.type==="turn_settled"&&event.turn_id===id).reusable,true)} assert.equal(observations.filter(event=>event.type==="native").length,1); assert.equal(observations.filter(event=>event.type==="query").length,1); for(const event of events.filter(event=>event.type!=="executor_ready")) assert.ok(["first","second"].includes(event.turn_id)); @@ -95,6 +95,7 @@ test("public interrupt settles cancellation and stale cancellation cannot stop t start("first","hold");await wait(()=>events.some(event=>event.type==="input_ready")); send({type:"turn_cancel",turn_id:"first"}); await wait(()=>events.some(event=>event.type==="turn_settled")); + assert.equal(events.find(event=>event.type==="turn_settled").confirmed,true); assert.equal(events.find(event=>event.type==="turn_settled").reusable,true); assert.ok(events.some(event=>event.type==="error"&&event.code==="cancelled"&&event.turn_id==="first")); start("second","hold");await wait(()=>events.some(event=>event.type==="input_ready"&&event.turn_id==="second")); @@ -106,12 +107,15 @@ test("public interrupt settles cancellation and stale cancellation cannot stop t child.stdin.end();assert.deepEqual(await closed,{code:0,signal:null}); }); -for(const mode of ["queued","unknown"])test(`interrupt ${mode} receipt retires native ownership before non-reusable settlement`,{timeout:10000},async t=>{ +for(const mode of ["queued","unknown","rejected"])test(`interrupt ${mode} receipt retires native ownership before non-reusable settlement`,{timeout:10000},async t=>{ const {events,observations,closed,wait,send,start}=await launch(t,mode); start("first","hold");await wait(()=>events.some(event=>event.type==="input_ready")); send({type:"turn_cancel",turn_id:"first"}); await wait(()=>events.some(event=>event.type==="turn_settled")); - assert.equal(events.find(event=>event.type==="turn_settled").reusable,false); + const settlement=events.find(event=>event.type==="turn_settled"); + assert.equal(settlement.confirmed,false); + assert.equal(settlement.reusable,false); + assert.equal(settlement.reason,"cancellation_unconfirmed"); assert.equal(observations.filter(event=>event.type==="native_closed").length,1); assert.deepEqual(await closed,{code:0,signal:null}); }); @@ -125,6 +129,7 @@ test("reused Turns retain functions, native receipts, steering and message bound send({type:"function_result",turn_id:id,call_id:"same-native-call",delivery_id:"same-delivery-id",success:true,content:[{type:"input_text",text:"function result"}]}); await wait(()=>events.some(event=>event.type==="turn_settled"&&event.turn_id===id)); const own=events.filter(event=>event.turn_id===id); + assert.equal(own.find(event=>event.type==="turn_settled").confirmed,true); assert.equal(own.find(event=>event.type==="turn_settled").reusable,true); assert.equal(own.filter(event=>event.type==="function_applied").length,1); assert.equal(own.find(event=>event.type==="input_applied").input_id,"same-input-id"); @@ -138,6 +143,7 @@ for(const mode of ["no-hook-report","false-hook-report"])test("none Executor wit const {child,events,closed,wait,start}=await launch(t,mode); start("first","answer"); await wait(()=>events.some(event=>event.type==="turn_settled")); + assert.equal(events.find(event=>event.type==="turn_settled").confirmed,true); assert.equal(events.find(event=>event.type==="turn_settled").reusable,true); child.stdin.end();assert.deepEqual(await closed,{code:0,signal:null}); }); diff --git a/scripts/name-allowlist.json b/scripts/name-allowlist.json index a8c53ae8f..37e679122 100644 --- a/scripts/name-allowlist.json +++ b/scripts/name-allowlist.json @@ -823,5 +823,25 @@ "path": "apps/docs/lib/legacy-redirects.ts", "regex": "\"agents-api(?:-sessions|-errors)?\"", "reason": "Retired documentation URL keys redirect to the current public API guide; these are not current commands or API namespaces." + }, + { + "path": "apps/parsar-daemon/internal/cli/connect_cleanup_test.go", + "regex": "\"agents-api-cleanup\"", + "reason": "Executor ownership validates the current Core and local-workspace AgentStateKey namespace; this change does not introduce an alternate identity or compatibility path." + }, + { + "path": "apps/parsar-daemon/internal/dispatch/executor.go", + "regex": "\"agents-api-(?:session)?\"", + "reason": "Executor ownership validates the current Core and local-workspace AgentStateKey namespace; this change does not introduce an alternate identity or compatibility path." + }, + { + "path": "apps/parsar-daemon/internal/dispatch/executor_handoff_test.go", + "regex": "\"agents-api-(?:session)?\"", + "reason": "Executor ownership validates the current Core and local-workspace AgentStateKey namespace; this change does not introduce an alternate identity or compatibility path." + }, + { + "path": "apps/parsar-daemon/internal/dispatch/executor_test.go", + "regex": "\"agents-api-(?:session)?\"", + "reason": "Executor ownership validates the current Core and local-workspace AgentStateKey namespace; this change does not introduce an alternate identity or compatibility path." } ] From 2f4bca6e024cb8887c62e756751bed4d83d52364 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Mon, 28 Sep 2026 15:38:59 +0000 Subject: [PATCH 4/8] Deliver cancellation before joining pending native receipts --- CONTRIBUTING.md | 7 +- .../dispatch/executor_cancel_receipt_test.go | 249 ++++++++++++++++++ .../internal/dispatch/prepared_handoff.go | 18 +- .../prepared_handoff_mutation_test.go | 30 +-- .../agents-api/tests/official_user_runtime.py | 4 +- 5 files changed, 284 insertions(+), 24 deletions(-) create mode 100644 apps/parsar-daemon/internal/dispatch/executor_cancel_receipt_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 584f24c6f..34ef00758 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1619,7 +1619,12 @@ channel, and retains the terminal observation until Start publication, Turn settlement and admitted operation receipts finish. Natural Done never calls Cancel. Input, function and interaction admission close before settlement, and operations already admitted hold their barrier through native receipts and -outbound acknowledgement. Only then forward Done or an applied cancellation +outbound acknowledgement. Send cancellation to the fixed Turn before waiting +for that barrier: a written input may need native interruption to produce its +receipt. Join native settlement, any required confirmed Executor close, output +drain and all admitted operations before an applied acknowledgement or reuse. +A failed Close may report failure while retaining the same Run and outstanding +operations for retry. Closing a caller wait cannot manufacture an applied input receipt. Only then forward Done or an applied cancellation receipt. Commit native continuity and release the old Run admission before publishing Done, since the receiver may immediately start another Turn. A late terminal-send failure belongs to the old Run; it cannot invalidate a successor diff --git a/apps/parsar-daemon/internal/dispatch/executor_cancel_receipt_test.go b/apps/parsar-daemon/internal/dispatch/executor_cancel_receipt_test.go new file mode 100644 index 000000000..8ea3b6ee2 --- /dev/null +++ b/apps/parsar-daemon/internal/dispatch/executor_cancel_receipt_test.go @@ -0,0 +1,249 @@ +package dispatch_test + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/agent" + "github.com/MiniMax-AI-Dev/parsar/apps/parsar-daemon/internal/dispatch" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +type receiptCancelSender struct { + *recSender + entered, release chan struct{} + once sync.Once +} + +func (s *receiptCancelSender) Send(ctx context.Context, env proto.Envelope) error { + if env.Type == proto.TypePromptSteerAck { + var ack proto.PromptSteerAckPayload + _ = env.DecodePayload(&ack) + if ack.InputID == "pending" && !ack.Written { + s.once.Do(func() { close(s.entered) }) + select { + case <-s.release: + case <-ctx.Done(): + return ctx.Err() + } + } + } + return s.recSender.Send(ctx, env) +} + +type receiptCancelExecutor struct { + turn chan *receiptCancelTurn + current *receiptCancelTurn + cancelFails, closeFailsFirst bool + closeEntered, closeRelease chan struct{} + closes atomic.Int32 + closeOnce sync.Once +} + +func (e *receiptCancelExecutor) StartTurn(_ context.Context, id string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Turn, error) { + turn := &receiptCancelTurn{id: id, out: out, owner: e, written: make(chan struct{}), cancelled: make(chan struct{}), closed: make(chan struct{}), settled: make(chan struct{}), awaiting: make(chan struct{})} + e.current = turn + e.turn <- turn + return turn, nil +} +func (e *receiptCancelExecutor) Close(context.Context) error { + attempt := e.closes.Add(1) + if e.closeFailsFirst && attempt == 1 { + return errors.New("native close unconfirmed") + } + e.closeOnce.Do(func() { close(e.closeEntered) }) + <-e.closeRelease + if e.current != nil { + e.current.closeOnce.Do(func() { close(e.current.closed) }) + e.current.finish() + } + return nil +} + +type receiptCancelTurn struct { + id string + out chan<- proto.Envelope + owner *receiptCancelExecutor + written, cancelled, closed, settled, awaiting chan struct{} + cancelOnce, closeOnce, finishOnce, terminalOnce, awaitOnce sync.Once + cancels atomic.Int32 +} + +func (t *receiptCancelTurn) terminal() { + t.terminalOnce.Do(func() { event, _ := proto.NewEnvelope(proto.TypeDone, t.id, t.CancellationOutcome()); t.out <- event }) +} +func (t *receiptCancelTurn) finish() { + t.finishOnce.Do(func() { t.terminal(); close(t.out); close(t.settled) }) +} +func (t *receiptCancelTurn) Cancel(context.Context) error { + t.cancels.Add(1) + t.cancelOnce.Do(func() { close(t.cancelled) }) + if t.owner.cancelFails { + return errors.New("native cancellation unconfirmed") + } + t.finish() + return nil +} +func (t *receiptCancelTurn) AwaitSettlement(ctx context.Context) (agent.TurnSettlement, error) { + t.awaitOnce.Do(func() { close(t.awaiting) }) + select { + case <-t.settled: + return agent.TurnSettlement{Reusable: !t.owner.cancelFails}, nil + case <-ctx.Done(): + return agent.TurnSettlement{}, ctx.Err() + } +} +func (t *receiptCancelTurn) CancellationOutcome() proto.DonePayload { + return proto.DonePayload{Content: "observed", Metadata: map[string]any{proto.DoneMetaAgentSessionID: "native-session"}} +} +func (t *receiptCancelTurn) Steer(ctx context.Context, input proto.PromptSteerPayload) error { + return t.SteerWithReceipt(ctx, input, nil) +} +func (t *receiptCancelTurn) SteerWithReceipt(ctx context.Context, _ proto.PromptSteerPayload, written func()) error { + written() + close(t.written) + native := t.cancelled + if t.owner.cancelFails { + native = t.closed + } + select { + case <-native: + case <-ctx.Done(): + return ctx.Err() + } + return errors.New("native input consumption is unknown") +} + +func TestExecutorCancellationReachesNativeBeforeDurableReceiptJoin(t *testing.T) { + for _, mode := range []string{"cancel", "done_then_cancel", "cancel_failure", "close_failure_retry"} { + t.Run(mode, func(t *testing.T) { + sender := &receiptCancelSender{recSender: &recSender{}, entered: make(chan struct{}), release: make(chan struct{})} + owner := &receiptCancelExecutor{turn: make(chan *receiptCancelTurn, 2), cancelFails: mode == "cancel_failure" || mode == "close_failure_retry", closeFailsFirst: mode == "close_failure_retry", closeEntered: make(chan struct{}), closeRelease: make(chan struct{})} + reg := agent.NewRegistry() + reg.RegisterKind(proto.SupportedAgentKind{Kind: "reusable", Available: true, Capabilities: proto.AgentKindCapabilities{EnvironmentNone: true}}, func(context.Context, proto.PromptRequestPayload, chan<- proto.Envelope) (agent.Session, error) { + return nil, errors.New("legacy factory forbidden") + }) + reg.RegisterExecutor("reusable", func(context.Context, proto.PromptRequestPayload) (agent.Executor, error) { return owner, nil }) + r, err := dispatch.New(dispatch.Config{Registry: reg, Sender: sender, IdleTimeout: time.Hour}) + if err != nil { + t.Fatal(err) + } + var receiptOnce, closeOnce sync.Once + releaseReceipt := func() { receiptOnce.Do(func() { close(sender.release) }) } + releaseClose := func() { closeOnce.Do(func() { close(owner.closeRelease) }) } + t.Cleanup(func() { + releaseReceipt() + releaseClose() + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if err := r.Shutdown(ctx); err != nil { + t.Error(err) + } + }) + wait := func(ch <-chan struct{}, reason string) { + t.Helper() + select { + case <-ch: + case <-time.After(time.Second): + t.Fatal(reason) + } + } + p := executorAdmission(t, r, sender.recSender, "first", executorRequest()) + startExecutorTurn(t, r, sender.recSender, "first", "run", p) + turn := <-owner.turn + input := proto.PromptSteerPayload{InputID: "pending", Input: proto.TextInput("second input"), DurableReceipt: true} + if err := r.Handle(t.Context(), mustEnv(t, proto.TypePromptSteer, "run", input)); err != nil { + t.Fatal(err) + } + wait(turn.written, "input did not reach native write") + if mode == "done_then_cancel" { + turn.terminal() + wait(turn.awaiting, "natural Done did not begin settlement") + } + cancel := func(id string) { + t.Helper() + if err := r.Handle(t.Context(), mustEnv(t, proto.TypePromptCancel, "run", proto.PromptCancelPayload{DeliveryID: id})); err != nil { + t.Fatal(err) + } + } + cancel("cancel") + wait(turn.cancelled, "receipt barrier prevented native cancellation") + acknowledgements := 0 + if mode == "close_failure_retry" { + waitFor(t, func() bool { return len(cancellationAcks(sender.recSender)) == 1 }, "failed Close acknowledgement") + ack := cancellationAcks(sender.recSender)[0] + if ack.Applied || ack.Outcome != nil || r.ActiveRuns() != 1 { + t.Fatal("failed Close released ownership or fabricated cancellation") + } + acknowledgements = 1 + cancel("retry") + } + if owner.cancelFails { + wait(owner.closeEntered, "failed cancellation did not reach resource cleanup") + if r.ActiveRuns() != 1 || len(cancellationAcks(sender.recSender)) != acknowledgements { + t.Fatal("cleanup was acknowledged before resource close") + } + releaseClose() + } + wait(sender.entered, "native cancellation did not unblock the written input") + if r.ActiveRuns() != 1 || len(cancellationAcks(sender.recSender)) != acknowledgements { + t.Fatal("cancellation passed an uncommitted input receipt") + } + for _, env := range sender.snapshot() { + if env.Type == proto.TypeDone { + t.Fatal("Done preceded the admitted input receipt") + } + } + releaseReceipt() + waitFor(t, func() bool { + return len(cancellationAcks(sender.recSender)) == acknowledgements+1 && r.ActiveRuns() == 0 && hasFrame(sender.recSender, proto.TypeDone, "run") + }, "joined cancellation") + frames := sender.snapshot() + inputAt, doneAt, cancelAt := -1, -1, -1 + for i, env := range frames { + switch env.Type { + case proto.TypePromptSteerAck: + var ack proto.PromptSteerAckPayload + _ = env.DecodePayload(&ack) + if !ack.Written { + if ack.Accepted || ack.ErrorCode != "outcome_unknown" { + t.Fatalf("unknown input fabricated receipt: %+v", ack) + } + inputAt = i + } + case proto.TypeDone: + doneAt = i + case proto.TypeInteractionDecisionAck: + cancelAt = i + } + } + if inputAt < 0 || doneAt < inputAt || cancelAt < inputAt || !owner.cancelFails && cancelAt < doneAt { + t.Fatalf("receipt/Done/cancel order=%d/%d/%d", inputAt, doneAt, cancelAt) + } + ack := cancellationAcks(sender.recSender)[acknowledgements] + if ack.Applied == owner.cancelFails || (ack.Outcome != nil) == owner.cancelFails { + t.Fatalf("cancellation evidence changed during cleanup: %+v", ack) + } + if !owner.cancelFails { + req := executorRequest() + req.Configuration.AgentSessionID = "native-session" + req.Configuration.RequireExistingNativeSession = true + next := executorAdmission(t, r, sender.recSender, "next", req) + if next.ExecutorID != p.ExecutorID || !next.Reused { + t.Fatal("healthy cancellation replaced owner") + } + startExecutorTurn(t, r, sender.recSender, "next", "next-run", next) + successor := <-owner.turn + cancel("late-old-run") + if successor.cancels.Load() != 0 || owner.closes.Load() != 0 { + t.Fatal("old cancellation reached successor") + } + successor.finish() + } + }) + } +} diff --git a/apps/parsar-daemon/internal/dispatch/prepared_handoff.go b/apps/parsar-daemon/internal/dispatch/prepared_handoff.go index 1b845b300..89c3d1862 100644 --- a/apps/parsar-daemon/internal/dispatch/prepared_handoff.go +++ b/apps/parsar-daemon/internal/dispatch/prepared_handoff.go @@ -53,7 +53,8 @@ type preparedHandoff struct { outputDone chan struct{} // Every admitted native mutation holds a read lock through its receipt. - // A release attempt takes the write lock before native cancellation. + // Release joins these receipts after native cancellation/settlement, since + // an admitted receipt may itself require cancellation to finish. operations sync.RWMutex published bool // started status committed; protected by Router.mu @@ -164,9 +165,8 @@ func (r *Router) runPreparedRelease(state *sessionState, handoff *preparedHandof } <-handoff.startDone } - // Admission is closed; all earlier mutations retain their native receipt wait. - handoff.operations.Lock() - handoff.operations.Unlock() + // Admission is closed, but native cancellation must be able to release an + // already-written input or tool operation that is waiting for its receipt. turn := handoff.turn var settlement agent.TurnSettlement var nativeErr error @@ -221,6 +221,15 @@ func (r *Router) runPreparedRelease(state *sessionState, handoff *preparedHandof // A failed Turn settlement is not repaired into a cancellation receipt by Close. nativeErr = errors.Join(nativeErr, closeErr) } + if closeErr == nil { + // Native settlement (or confirmed resource close after failure) comes + // first. Keep every admitted operation and its outbound receipt owned + // until it returns; neither cancellation acknowledgement nor reuse can + // pass this join. Failed Close retains the same owner for a later retry. + handoff.operations.Lock() + handoff.operations.Unlock() + <-handoff.outputDone + } r.mu.Lock() attempt.err = nativeErr if closeErr != nil { @@ -236,7 +245,6 @@ func (r *Router) runPreparedRelease(state *sessionState, handoff *preparedHandof release.succeeded, release.outcome = true, outcome close(attempt.done) r.mu.Unlock() - <-handoff.outputDone r.mu.Lock() outputErr, terminal, closed := handoff.outputErr, handoff.terminal, r.closed r.mu.Unlock() diff --git a/apps/parsar-daemon/internal/dispatch/prepared_handoff_mutation_test.go b/apps/parsar-daemon/internal/dispatch/prepared_handoff_mutation_test.go index fa0717b4f..fce4c8640 100644 --- a/apps/parsar-daemon/internal/dispatch/prepared_handoff_mutation_test.go +++ b/apps/parsar-daemon/internal/dispatch/prepared_handoff_mutation_test.go @@ -15,16 +15,12 @@ type preparedMutationSession struct { *fakeSession cancelEntered chan struct{} cancelOnce sync.Once - beforeCancel func() functions atomic.Int32 steers atomic.Int32 reads atomic.Int32 } func (s *preparedMutationSession) Cancel(ctx context.Context) error { - if s.beforeCancel != nil { - s.beforeCancel() - } s.cancelOnce.Do(func() { close(s.cancelEntered) }) return s.fakeSession.Cancel(ctx) } @@ -162,8 +158,11 @@ func TestPreparedHandoffReleaseWaitsForMutationReceipt(t *testing.T) { } select { case <-session.cancelEntered: - t.Fatal("native release overtook an admitted receipt") - case <-time.After(50 * time.Millisecond): + case <-time.After(2 * time.Second): + t.Fatal("native cleanup was blocked by the outbound receipt") + } + if r.ActiveRuns() != 1 || hasFrame(sender.recSender, proto.TypeDone, "run") { + t.Fatal("Run ownership or Done overtook the admitted receipt") } close(sender.release) select { @@ -179,7 +178,7 @@ func TestPreparedHandoffReleaseWaitsForMutationReceipt(t *testing.T) { case <-time.After(2 * time.Second): t.Fatal("release did not follow the completed receipt") } - waitFor(t, func() bool { return r.ActiveRuns() == 0 }, "mutation release cleanup") + waitFor(t, func() bool { return r.ActiveRuns() == 0 && hasFrame(sender.recSender, proto.TypeDone, "run") }, "mutation release cleanup") if session.cancels() != 1 { t.Fatalf("Session release calls = %d, want 1", session.cancels()) } @@ -239,7 +238,6 @@ func TestPreparedHandoffRouterShutdownWaitsForReceiptAttempt(t *testing.T) { t.Run(operation, func(t *testing.T) { sender := &blockingPreparedReceiptSender{recSender: &recSender{}, deliveryID: operation + "-delivery", inputID: operation + "-input", entered: make(chan struct{}), release: make(chan struct{}), exited: make(chan struct{})} defer close(sender.release) - var cancelBeforeReceipt atomic.Bool session := &preparedMutationSession{fakeSession: &fakeSession{closeOutOnCancel: true}, cancelEntered: make(chan struct{})} p := &controlledPreparation{closed: make(chan struct{})} p.start = func(_ context.Context, _ string, _ proto.MessageInput, out chan<- proto.Envelope) (agent.Session, error) { @@ -250,13 +248,6 @@ func TestPreparedHandoffRouterShutdownWaitsForReceiptAttempt(t *testing.T) { startCancellationPreparation(t, r, sender.recSender) waitPreparationStatus(t, sender.recSender, "request", "started", "") - session.beforeCancel = func() { - select { - case <-sender.exited: - default: - cancelBeforeReceipt.Store(true) - } - } var mutation proto.Envelope switch operation { case "function": @@ -283,8 +274,13 @@ func TestPreparedHandoffRouterShutdownWaitsForReceiptAttempt(t *testing.T) { if err := r.Shutdown(ctx); err != nil { t.Fatal(err) } - if cancelBeforeReceipt.Load() || session.cancels() != 1 || r.ActiveRuns() != 0 { - t.Fatalf("shutdown crossed receipt barrier: early=%t cancels=%d active=%d", cancelBeforeReceipt.Load(), session.cancels(), r.ActiveRuns()) + select { + case <-sender.exited: + default: + t.Fatal("shutdown returned before the receipt attempt ended") + } + if session.cancels() != 1 || r.ActiveRuns() != 0 { + t.Fatalf("shutdown did not join native cleanup: cancels=%d active=%d", session.cancels(), r.ActiveRuns()) } }) } diff --git a/services/agents-api/tests/official_user_runtime.py b/services/agents-api/tests/official_user_runtime.py index 43b3b7631..9101634b0 100644 --- a/services/agents-api/tests/official_user_runtime.py +++ b/services/agents-api/tests/official_user_runtime.py @@ -250,7 +250,9 @@ def check_artifacts(): try: assert expected_environment_type in ("self_hosted", "openai_hosted") - assert session.environment.type == expected_environment_type and session.environment.workspace_directory == "/workspace" + assert session.environment.type == expected_environment_type + if expected_environment_type == "self_hosted": + assert session.environment.workspace_directory == "/workspace" assert client.beta.agents.environments.retrieve(eid).status == "connected", "Runtime is not connected" assert not list(sessions.turns.list(sid)) and not list(sessions.items.list(sid)), "Use an unused Session" assert not list(sessions.artifacts.list(sid)), "Use a Session without existing Artifacts" From 13c22c3112d92d52af678572da9e01a80f7139f9 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Mon, 28 Sep 2026 15:55:32 +0000 Subject: [PATCH 5/8] Retry queued Turns after Runtime preparation capacity rejection --- CONTRIBUTING.md | 8 +- .../agents-api/internal/execution/worker.go | 8 + .../internal/store/worker_capacity_test.go | 189 ++++++++++++++++++ 3 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 services/agents-api/internal/store/worker_capacity_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 34ef00758..60c96d603 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1646,8 +1646,12 @@ later Turn. Cancellation uses the exact Run identity. Preparations and Start execute outside the receive loop and Router lock. Admission expires after five minutes; retries do not extend that deadline. Bound active preparation and execution separately from idle retained resources, and count -closing or uncertain resources until cleanup succeeds. At most 64 admission -records are retained; old handles never consume replacement admissions. Idle expiry +closing or uncertain resources until cleanup succeeds. A definite +`execution_prepare` rejection with `preparation_capacity` leaves an unclaimed +queued Turn for the existing Worker scheduler to retry, including capacity held +by cleanup. Other errors and uncertain input delivery do not authorize replay. +At most 64 admission records are retained; old handles never consume replacement +admissions. Idle expiry is a Runtime resource policy, not Core active-Turn concurrency. Shutdown tracks and closes active and idle Executors, retains failed close targets, and allows a later serialized retry. Ordinary disconnection closes the failed transport and keeps diff --git a/services/agents-api/internal/execution/worker.go b/services/agents-api/internal/execution/worker.go index 41eec56b3..c8ec77167 100644 --- a/services/agents-api/internal/execution/worker.go +++ b/services/agents-api/internal/execution/worker.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" "github.com/MiniMax-AI-Dev/parsar/internal/obs/log" "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" ) @@ -343,6 +344,8 @@ func (w *Worker) runClaim(ctx context.Context, item store.ExecutionWork) error { if err == nil || errors.Is(err, store.ErrTurnConflict) { return nil } + var rejection *preparationRejection + capacityRejected := errors.As(err, &rejection) && rejection.operation == proto.TypeExecutionPrepare && rejection.code == "preparation_capacity" outcome := json.RawMessage(`{"error_code":"execution_unavailable"}`) if errors.Is(err, store.ErrModelProviderRequired) { outcome = json.RawMessage(`{"error_code":"model_provider_required"}`) @@ -353,6 +356,11 @@ func (w *Worker) runClaim(ctx context.Context, item store.ExecutionWork) error { if err != nil { return err } + if turn.Status == store.TurnQueued && capacityRejected { + // No input was sent. Leave durable work for the existing scheduler tick; + // active and cleanup-held Runtime capacity have the same rejection. + return nil + } if turn.Status == store.TurnCompleted || turn.Status == store.TurnFailed || turn.Status == store.TurnCancelled { return nil } diff --git a/services/agents-api/internal/store/worker_capacity_test.go b/services/agents-api/internal/store/worker_capacity_test.go new file mode 100644 index 000000000..ccb884df2 --- /dev/null +++ b/services/agents-api/internal/store/worker_capacity_test.go @@ -0,0 +1,189 @@ +package store_test + +import ( + "fmt" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" + "github.com/MiniMax-AI-Dev/parsar/services/agents-api/internal/store" +) + +// Consume the actual controls so capacity rejection is exercised before any +// input, without the ordinary fixture's automatic preparation acknowledgement. +func capacityWorkerFrames(t *testing.T, h *dispatchHarness) <-chan proto.Envelope { + t.Helper() + frames := make(chan proto.Envelope, 64) + go func() { + defer close(frames) + for { + var frame proto.Envelope + if h.conn.ReadJSON(&frame) != nil { + return + } + select { + case frames <- frame: + case <-t.Context().Done(): + return + } + } + }() + return frames +} + +func TestWorkerDefersPreparationCapacityUntilCleanupReleasesSlot(t *testing.T) { + h := newDispatchHarness(t) + enableWorkerEnvironment(t, h) + h.d.MaxConcurrentExecutions = 5 + sessions := make(map[string]string) + for i := range 5 { + h.session = publicSession(t, h, fmt.Sprintf("capacity-%d", i)) + receipt := h.message("work", "once") + sessions[h.session.ID] = receipt.TurnID + } + frames := capacityWorkerFrames(t, h) + _, stop := startEnvironmentExpiryWorker(t, h.d) + defer stop() + admissions := make(map[string]string) + started := make(map[string]int) + var blocked, first string + capacityReleased := false + rejections := 0 + process := func(frame proto.Envelope) { + switch frame.Type { + case proto.TypeExecutionPrepare: + var prepare proto.ExecutionPreparePayload + if frame.DecodePayload(&prepare) != nil || sessions[prepare.SessionID] == "" || len(prepare.Configuration.Input) != 0 || prepare.Configuration.RunID != "" { + t.Fatal("invalid input-free preparation", prepare) + } + if blocked == "" && len(admissions) == 4 { + blocked = prepare.SessionID + } + if prepare.SessionID == blocked && !capacityReleased { + rejections++ + h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{State: "rejected", Operation: proto.TypeExecutionPrepare, ErrorCode: "preparation_capacity"}) + return + } + admissions[frame.ID] = prepare.SessionID + h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: frame.ID, ExecutorID: "executor-" + prepare.SessionID, Revision: 1, State: "ready"}) + case proto.TypeExecutionStart: + var start proto.ExecutionStartPayload + session := admissions[frame.ID] + if frame.DecodePayload(&start) != nil || session == "" || start.RunID != sessions[session] || inputTextForTest(t, start.Input) != "once" { + t.Fatal("Start changed the original input or Turn", start) + } + started[session]++ + if started[session] != 1 || (session == blocked && !capacityReleased) { + t.Fatal("capacity rejection replayed or prematurely started input") + } + if first == "" { + first = session + } + h.write(frame.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: start.Handle, ExecutorID: start.ExecutorID, Revision: 2, State: "started", RunID: start.RunID}) + case proto.TypeExecutionRelease: + default: + t.Fatal("unexpected worker frame", frame.Type) + } + } + next := func() { + t.Helper() + select { + case frame, ok := <-frames: + if !ok { + t.Fatal("Runtime connection closed") + } + process(frame) + case <-time.After(5 * time.Second): + t.Fatal("worker did not send preparation or Start") + } + } + for len(started) != 4 || rejections == 0 { + next() + } + assertQueued := func() { + t.Helper() + turn, err := h.s.GetTurn(t.Context(), h.tenant, blocked, sessions[blocked]) + if err != nil || turn.Status != store.TurnQueued { + t.Fatal("capacity rejection failed queued work", turn, err) + } + } + // Observe multiple scheduler ticks with four active native owners. The + // bounded attempt count detects a completion-triggered busy retry loop. + observeCapacity := func() { + t.Helper() + before := rejections + timer := time.NewTimer(800 * time.Millisecond) + defer timer.Stop() + for { + select { + case frame := <-frames: + process(frame) + if rejections-before > 6 { + t.Fatal("capacity rejection caused a busy retry loop") + } + case <-timer.C: + if rejections == before { + t.Fatal("queued work was not retried by scheduler") + } + assertQueued() + return + } + } + } + observeCapacity() + // After completion, Runtime can retire the idle Executor. Its cleanup + // still owns a slot and reports the same capacity rejection. + h.write(sessions[first], proto.TypeDone, proto.DonePayload{Content: "done"}) + awaitDaemonRemoteCondition(t, t.Context(), 5*time.Second, "first completed Turn", func() bool { + turn, err := h.s.GetTurn(t.Context(), h.tenant, first, sessions[first]) + return err == nil && turn.Status == store.TurnCompleted + }) + observeCapacity() + capacityReleased = true + for started[blocked] == 0 { + next() + } + for session, turn := range sessions { + if session != first { + h.write(turn, proto.TypeDone, proto.DonePayload{Content: "done"}) + } + } + awaitDaemonRemoteCondition(t, t.Context(), 5*time.Second, "all five Turns completed", func() bool { + for session, id := range sessions { + turn, err := h.s.GetTurn(t.Context(), h.tenant, session, id) + if err != nil || turn.Status != store.TurnCompleted { + return false + } + } + return true + }) + if len(started) != 5 || started[blocked] != 1 { + t.Fatal("queued Turn did not start exactly once", started) + } +} + +func TestWorkerDoesNotDeferOtherPreparationOrStartRejections(t *testing.T) { + for _, test := range []struct{ name, operation, code string }{ + {"invalid_configuration", proto.TypeExecutionPrepare, "invalid_configuration"}, + {"resource_unavailable", proto.TypeExecutionPrepare, "resource_unavailable"}, + {"unknown_operation", "", "preparation_capacity"}, + {"start_capacity", proto.TypeExecutionStart, "preparation_capacity"}, + } { + t.Run(test.name, func(t *testing.T) { + h := newDispatchHarness(t) + enableWorkerEnvironment(t, h) + h.session = publicSession(t, h, test.name) + receipt := h.message("work", "once") + frames := capacityWorkerFrames(t, h) + _, stop := startEnvironmentExpiryWorker(t, h.d) + defer stop() + prepare := nextWorkerFrame(t, frames, proto.TypeExecutionPrepare) + if test.operation == proto.TypeExecutionStart { + h.write(prepare.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{Handle: prepare.ID, Revision: 1, State: "ready"}) + nextWorkerFrame(t, frames, proto.TypeExecutionStart) + } + h.write(prepare.ID, proto.TypePreparationStatus, proto.PreparationStatusPayload{State: "rejected", Operation: test.operation, ErrorCode: test.code}) + waitTurn(t, h, receipt.TurnID, store.TurnFailed) + }) + } +} From a5ff4796c44b5c1134fd3fd87727e12023c07c23 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Mon, 28 Sep 2026 16:04:39 +0000 Subject: [PATCH 6/8] Settle unanswered Claude functions after confirmed cancellation --- CONTRIBUTING.md | 3 ++ .../claudesdk/executor_confirmation_test.go | 11 +++---- .../internal/agent/claudesdk/executor_test.go | 4 +-- .../internal/agent/claudesdk/executor_turn.go | 9 ++++-- .../internal/agent/claudesdk/functions.go | 4 +-- packages/claude-sdk-adapter/src/adapter.ts | 3 +- .../claude-sdk-adapter/src/function_bridge.ts | 26 +++++++++++++--- .../tests/executor.test.mjs | 30 +++++++++++++++++-- .../tests/function_bridge.test.mjs | 30 ++++++++++++++++++- .../tests/official_hosted_functions_native.py | 2 +- 10 files changed, 101 insertions(+), 21 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 60c96d603..e2491b2e5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3233,6 +3233,9 @@ remain unconfirmed even after successful teardown. Go rejects cancellation and AwaitSettlement when native confirmation is missing or false, or its own receipt ledger remains unsettled. A confirmed Turn may be non-reusable after cleanup; that state alone does not turn a verified cancellation into an error. +Confirmed native cancellation may settle unanswered function calls after result +admission closes and callbacks drain. A submitted function result still requires +its native application receipt, including when the MCP request aborts. `claudesdk.Config.Workspace` is a private, trusted operator binding for one qualified placement. It enables native Bash/Read/Edit and declared host functions diff --git a/apps/parsar-daemon/internal/agent/claudesdk/executor_confirmation_test.go b/apps/parsar-daemon/internal/agent/claudesdk/executor_confirmation_test.go index bf74b120b..631f52793 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/executor_confirmation_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/executor_confirmation_test.go @@ -11,10 +11,10 @@ import ( ) func TestExecutorNativeConfirmationSurvivesCleanup(t *testing.T) { - for _, mode := range []string{"unknown_cancel", "queued_cancel", "missing_confirmation", "pending_function", "pending_input", "confirmed_closed"} { + for _, mode := range []string{"unknown_cancel", "queued_cancel", "missing_confirmation", "pending_function", "pending_function_unconfirmed", "pending_input", "confirmed_closed"} { t.Run(mode, func(t *testing.T) { config, req := persistentConfig(t, mode) - if mode == "pending_function" { + if mode == "pending_function" || mode == "pending_function_unconfirmed" { req.FunctionTools = []proto.FunctionTool{{Name: "lookup", Parameters: json.RawMessage(`{"type":"object"}`)}} } owner, err := NewExecutorFactory(config)(t.Context(), req) @@ -26,7 +26,7 @@ func TestExecutorNativeConfirmationSurvivesCleanup(t *testing.T) { if event := <-out; event.Type != proto.TypeDelta { t.Fatal("initial output missing") } - if mode == "pending_function" { + if mode == "pending_function" || mode == "pending_function_unconfirmed" { if event := <-out; event.Type != proto.TypeFunctionCall { t.Fatal("function obligation missing") } @@ -45,8 +45,9 @@ func TestExecutorNativeConfirmationSurvivesCleanup(t *testing.T) { for range out { } settlement, settlementErr := turn.AwaitSettlement(t.Context()) - confirmed := mode == "confirmed_closed" - if (cancelErr == nil) != confirmed || (settlementErr == nil) != confirmed || settlement.Reusable { + confirmed := mode == "confirmed_closed" || mode == "pending_function" + reusable := mode == "pending_function" + if (cancelErr == nil) != confirmed || (settlementErr == nil) != confirmed || settlement.Reusable != reusable { t.Fatalf("cancel=%v settlement=%+v err=%v", cancelErr, settlement, settlementErr) } if inputReceipt != nil { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/executor_test.go b/apps/parsar-daemon/internal/agent/claudesdk/executor_test.go index c699b63ae..39d18f03a 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/executor_test.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/executor_test.go @@ -65,7 +65,7 @@ func runPersistentExecutorHelper() { } confirmed, reusable, reason := true, true, "" switch os.Getenv("SDK_EXECUTOR_MODE") { - case "unknown_cancel", "queued_cancel": + case "unknown_cancel", "queued_cancel", "pending_function_unconfirmed": confirmed, reusable, reason = false, false, "cancellation_unconfirmed" case "confirmed_closed": reusable, reason = false, "native_closed" @@ -99,7 +99,7 @@ func runPersistentExecutorHelper() { encode(bridgeEvent{Type: "turn_started", TurnID: active}) encode(bridgeEvent{Type: "input_ready", TurnID: active, SessionID: "native-persistent"}) encode(bridgeEvent{Type: "delta", TurnID: active, Delta: "partial"}) - if os.Getenv("SDK_EXECUTOR_MODE") == "pending_function" { + if strings.HasPrefix(os.Getenv("SDK_EXECUTOR_MODE"), "pending_function") { encode(bridgeEvent{Type: "function_call", TurnID: active, Call: &proto.FunctionCallPayload{CallID: "call", Name: "lookup", Arguments: json.RawMessage("{}")}}) } if len(command.Input) > 0 && len(command.Input[0].Content) > 0 && command.Input[0].Content[0].Text != nil && *command.Input[0].Content[0].Text == "wait" { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/executor_turn.go b/apps/parsar-daemon/internal/agent/claudesdk/executor_turn.go index 285f4caf1..47a587a41 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/executor_turn.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/executor_turn.go @@ -57,6 +57,7 @@ func (s *session) runTurn(start startRequest, out chan<- proto.Envelope) { commands := commandState{calls: map[string]proto.ToolObservation{}} settlementReceived := false settlementConfirmed := false + cancelled := false reusable := false reason := "bridge_interrupted" for raw := range s.frames { @@ -148,7 +149,7 @@ func (s *session) runTurn(start startRequest, out chan<- proto.Envelope) { usageSession = event.SessionID emit(proto.TypeUsage, proto.UsagePayload{Usage: usage}) case "result": - if !s.matchesInputSession(event.SessionID) || event.SessionID == "" || start.Resume != "" && event.SessionID != start.Resume || usageSession != "" && event.SessionID != usageSession || !s.functionsComplete() || !s.steeringComplete() || !mcp.complete() || !commands.complete() { + if !s.matchesInputSession(event.SessionID) || event.SessionID == "" || start.Resume != "" && event.SessionID != start.Resume || usageSession != "" && event.SessionID != usageSession || !s.functionsComplete(false) || !s.steeringComplete() || !mcp.complete() || !commands.complete() { failure = fmt.Errorf("claudesdk: invalid native completion or unconfirmed input/result") s.settlementErr = failure s.invalidate() @@ -157,6 +158,7 @@ func (s *session) runTurn(start startRequest, out chan<- proto.Envelope) { } terminal = true case "error": + cancelled = event.Code == "cancelled" failure = bridgeFailure(event.Code) terminal = true default: @@ -173,12 +175,13 @@ func (s *session) runTurn(start startRequest, out chan<- proto.Envelope) { if result == nil && failure == nil { failure = fmt.Errorf("claudesdk: SDK result is missing") } - if !s.functionsComplete() || !s.steeringComplete() || !mcp.complete() || !commands.complete() { + // Close result admission before deciding which unanswered calls cancellation settled. + s.stopFunctions() + if !s.functionsComplete(cancelled && settlementConfirmed) || !s.steeringComplete() || !mcp.complete() || !commands.complete() { settlementConfirmed, reusable, reason = false, false, "unsettled_native_operations" } mcp.close(start, emit) commands.close(start, emit) - s.stopFunctions() s.stopSteering() metadata := map[string]any{proto.DoneMetaAgentSessionType: "claude_session"} if id := s.inputSessionID(); id != "" { diff --git a/apps/parsar-daemon/internal/agent/claudesdk/functions.go b/apps/parsar-daemon/internal/agent/claudesdk/functions.go index 74df35648..aaa56e61c 100644 --- a/apps/parsar-daemon/internal/agent/claudesdk/functions.go +++ b/apps/parsar-daemon/internal/agent/claudesdk/functions.go @@ -159,11 +159,11 @@ func (s *session) SubmitFunctionResult(ctx context.Context, result proto.Functio } } -func (s *session) functionsComplete() bool { +func (s *session) functionsComplete(cancelled bool) bool { s.functions.mu.Lock() defer s.functions.mu.Unlock() for _, pending := range s.functions.calls { - if !pending.applied { + if !pending.applied && !(cancelled && pending.result == nil) { return false } } diff --git a/packages/claude-sdk-adapter/src/adapter.ts b/packages/claude-sdk-adapter/src/adapter.ts index 7f63e0d6c..e6520541b 100644 --- a/packages/claude-sdk-adapter/src/adapter.ts +++ b/packages/claude-sdk-adapter/src/adapter.ts @@ -168,7 +168,7 @@ export async function execute(request: Start | Prepare | ExecutorPrepare, emit: if(turns && !turns.id && !(message.type==="system" && message.subtype==="init")) throw new Error("unbound native event"); subagents?.consume(message); structured?.consume(message, nativeID); - await functions.consume(message, nativeID); + await functions.consume(message, nativeID, turns?.cancelled); if (mcp) for (const event of mcp.consume(message, nativeID)) await emit(event); if (commands) for (const event of commands.consume(message, nativeID, inputs.hasInput)) await emit(event); if (messages) for (const event of messages.consume(message)) await emit(event); @@ -199,6 +199,7 @@ export async function execute(request: Start | Prepare | ExecutorPrepare, emit: } if (message.type !== "result" && (!turns || turns.id)) for (const event of inputs.consume(message)) await emit(event); if(turns && result && inputs.complete && (!subagents || subagents.complete)) { + if (turns.cancelled) functions.cancelUnanswered(); if(!await turns.quiescent() || abort.signal.aborted) throw new Error("unconfirmed interrupt"); functions.assertComplete(); mcp?.assertComplete(); commands?.assertComplete(); if(subagents) for(const event of await subagents.facts()) await emit(event); diff --git a/packages/claude-sdk-adapter/src/function_bridge.ts b/packages/claude-sdk-adapter/src/function_bridge.ts index d849288c6..777d9ea15 100644 --- a/packages/claude-sdk-adapter/src/function_bridge.ts +++ b/packages/claude-sdk-adapter/src/function_bridge.ts @@ -19,6 +19,7 @@ type Pending = { reject: (error: Error) => void; cleanup: () => void; result?: FunctionResult; + aborted?: boolean; }; export class FunctionBridge { @@ -32,7 +33,9 @@ export class FunctionBridge { this.seen.add(call.id); const waiting = new Promise((resolve, reject) => { const stop = () => { - this.pending.delete(call.id); + const pending = this.pending.get(call.id); + if (pending?.result) pending.aborted = true; + else this.pending.delete(call.id); reject(new Error("Native function call aborted.")); }; signal.addEventListener("abort", stop, { once: true }); @@ -69,15 +72,19 @@ export class FunctionBridge { }), isError: !result.success }); } - async consume(message: SDKMessage, sessionID: string): Promise { + async consume(message: SDKMessage, sessionID: string, cancelled = false): Promise { if (message.type !== "user" || message.parent_tool_use_id !== null || message.session_id !== sessionID || message.isSynthetic || ("isReplay" in message && message.isReplay) || !Array.isArray(message.message.content)) return; for (const block of message.message.content) { if (block.type !== "tool_result") continue; const pending = this.pending.get(block.tool_use_id); - if (!pending) continue; + if (!pending || pending.aborted) continue; const result = pending.result; - if (!result) throw new Error("Native response preceded host result."); + if (!result) { + if (!cancelled || !block.is_error) throw new Error("Native response preceded host result."); + this.cancelUnanswered(block.tool_use_id); + continue; + } // Native error results join MCP text blocks; retain the original parts in the host. const matches = result.success ? matchesNativeContent(result.content, block.content) : block.content === result.content.map(part => part.type === "input_text" ? part.text : "").join("\n"); @@ -90,6 +97,17 @@ export class FunctionBridge { } } + // Only a native cancellation result authorizes abandoning unanswered calls. + // Submitted results retain their receipt obligation, including after abort. + cancelUnanswered(callID?: string): void { + for (const [id, pending] of this.pending) { + if (pending.result || (callID !== undefined && id !== callID)) continue; + pending.cleanup(); + this.pending.delete(id); + pending.reject(new Error("Native function call cancelled.")); + } + } + assertComplete(): void { if (this.pending.size) throw new Error("Native function results are unconfirmed."); } diff --git a/packages/claude-sdk-adapter/tests/executor.test.mjs b/packages/claude-sdk-adapter/tests/executor.test.mjs index 39c6f1e90..894221ed4 100644 --- a/packages/claude-sdk-adapter/tests/executor.test.mjs +++ b/packages/claude-sdk-adapter/tests/executor.test.mjs @@ -24,7 +24,7 @@ globalThis.startupFixture=async({options})=>{ options.abortController.signal.addEventListener('abort',close); return {close,query(prompt){assert.equal(queried,false);queried=true;process.send({type:'query'});return { close,async initializationResult(){return process.argv[1]==="no-hook-report" ? {} : process.argv[1]==="false-hook-report" ? {hooks_applied:false} : {hooks_applied:true}}, - async interrupt(){process.send({type:'interrupt'});interrupt?.();if(process.argv[1]==='rejected')throw new Error('interrupt failed');return process.argv[1]==='unknown' ? undefined : {still_queued:process.argv[1]==='queued' ? ['not-consumed'] : []}}, + async interrupt(){process.send({type:'interrupt'});interrupt?.();if(process.argv[1]==='rejected')throw new Error('interrupt failed');return ['unknown','pending-function-unknown'].includes(process.argv[1]) ? undefined : {still_queued:process.argv[1]==='queued' ? ['not-consumed'] : []}}, async *[Symbol.asyncIterator](){ for await(const user of prompt){ number++; @@ -32,6 +32,11 @@ globalThis.startupFixture=async({options})=>{ process.send({type:'input',text}); yield {type:'system',subtype:'init',session_id:'native',tools:client ? ['mcp__functions__lookup'] : [],mcp_servers:client ? [{name:'functions',status:'connected'}] : []}; if(client){ + if(text==='pending-function'){ + void client.callTool({name:'lookup',arguments:{text},_meta:{'claudecode/toolUseId':'pending-call'}}).catch(()=>{}); + await Promise.race([new Promise(resolve=>{interrupt=resolve}),exited]); + if(process.argv[1]==='pending-function-result') yield {type:'user',session_id:'native',parent_tool_use_id:null,message:{content:[{type:'tool_result',tool_use_id:'pending-call',content:'Interrupted',is_error:true}]}}; + } if(text==='features'){ const called=await client.callTool({name:'lookup',arguments:{text},_meta:{'claudecode/toolUseId':'same-native-call'}}); yield {type:'user',session_id:'native',parent_tool_use_id:null,message:{content:[{type:'tool_result',tool_use_id:'same-native-call',content:called.content,is_error:called.isError}]}}; @@ -68,7 +73,7 @@ async function launch(t,mode="normal") { const send=value=>child.stdin.write(JSON.stringify(value)+"\n"); const start=(id,text)=>send({type:"turn_start",turn_id:id,input:[{content:[{type:"input_text",text}]}]}); send({type:"executor_prepare",cwd:"/tmp",model:"fixture",system_prompt:"", - ...(mode==="features" ? {observe_messages:true,functions:[{name:"lookup",description:"lookup",parameters:{type:"object",properties:{text:{type:"string"}}}}]} : {})}); + ...(mode==="features" || mode.startsWith("pending-function") ? {observe_messages:true,functions:[{name:"lookup",description:"lookup",parameters:{type:"object",properties:{text:{type:"string"}}}}]} : {})}); await wait(()=>events.some(event=>event.type==="executor_ready")); assert.equal(observations.filter(event=>event.type==="input").length,0); return {child,events,observations,closed,wait,send,start}; @@ -147,3 +152,24 @@ for(const mode of ["no-hook-report","false-hook-report"])test("none Executor wit assert.equal(events.find(event=>event.type==="turn_settled").reusable,true); child.stdin.end();assert.deepEqual(await closed,{code:0,signal:null}); }); + +for (const mode of ["pending-function-result", "pending-function-terminal", "pending-function-unknown"]) test(`unanswered function cancellation ${mode} retains native confirmation requirements`, {timeout:10000}, async t => { + const {child,events,closed,wait,send,start}=await launch(t,mode); + start("first","pending-function"); + await wait(()=>events.some(event=>event.type==="function_call")); + send({type:"turn_cancel",turn_id:"first"}); + await wait(()=>events.some(event=>event.type==="turn_settled")); + const settled=events.find(event=>event.type==="turn_settled"); + const confirmed=mode!=="pending-function-unknown"; + assert.equal(settled.confirmed,confirmed); + assert.equal(settled.reusable,confirmed); + assert.ok(events.some(event=>event.type==="error"&&event.code==="cancelled")); + assert.equal(events.filter(event=>event.type==="function_applied").length,0); + if(confirmed){ + start("second","answer"); + await wait(()=>events.some(event=>event.type==="turn_settled"&&event.turn_id==="second")); + assert.equal(events.find(event=>event.type==="turn_settled"&&event.turn_id==="second").confirmed,true); + child.stdin.end(); + } + assert.deepEqual(await closed,{code:0,signal:null}); +}); diff --git a/packages/claude-sdk-adapter/tests/function_bridge.test.mjs b/packages/claude-sdk-adapter/tests/function_bridge.test.mjs index fac8458d9..bd78b0e27 100644 --- a/packages/claude-sdk-adapter/tests/function_bridge.test.mjs +++ b/packages/claude-sdk-adapter/tests/function_bridge.test.mjs @@ -62,7 +62,10 @@ test("an aborted submitted result never becomes a native application receipt", { bridge.submit(JSON.stringify(result("a", false))); await waiting; abort.abort(); - await bridge.consume(native(result("a", false)), "native"); + bridge.cancelUnanswered(); + assert.throws(() => bridge.assertComplete(), /unconfirmed/); + await bridge.consume(native(result("a", false)), "native", true); + assert.throws(() => bridge.assertComplete(), /unconfirmed/); assert.deepEqual(events.map(event => event.type), ["function_call"]); }); @@ -112,3 +115,28 @@ test("native error-image rejection leaves the pending call available for a suppo await bridge.consume(native(result("error", false)), "native"); bridge.assertComplete(); }); + +for (const terminal of ["tool_result", "result"]) test(`native cancellation ${terminal} settles unanswered callbacks without application`, async () => { + const events = []; + const bridge = new FunctionBridge(async event => { events.push(event); }); + const waiting = bridge.invoke(call("a"), new AbortController().signal); + const rejected = assert.rejects(waiting, /cancelled/); + if (terminal === "tool_result") { + const cancellation = native(result("a", false), "Interrupted"); + await assert.rejects(bridge.consume(cancellation, "native"), /preceded/); + await bridge.consume(cancellation, "native", true); + } else bridge.cancelUnanswered(); + await rejected; + bridge.assertComplete(); + assert.deepEqual(events.map(event => event.type), ["function_call"]); +}); + +test("native cancellation preserves a submitted result's missing receipt", async () => { + const bridge = new FunctionBridge(async () => {}); + const waiting = bridge.invoke(call("a"), new AbortController().signal); + bridge.submit(JSON.stringify(result("a"))); + await waiting; + bridge.cancelUnanswered(); + assert.throws(() => bridge.assertComplete(), /unconfirmed/); + bridge.close(); +}); diff --git a/services/agents-api/tests/official_hosted_functions_native.py b/services/agents-api/tests/official_hosted_functions_native.py index 9f57c183f..a573d89ee 100644 --- a/services/agents-api/tests/official_hosted_functions_native.py +++ b/services/agents-api/tests/official_hosted_functions_native.py @@ -83,7 +83,7 @@ def failure(arguments): raise RuntimeError(private_error) _, result = invoke("Call lookup exactly once with key failure. If it fails, do not retry. Then reply with the exact string returned by lookup in our first turn, using conversation history and no file tools.", "function-failure", failure) - assert result["error"] == "Tool handler failed." and "output" not in result + assert result["error"] == "Tool handler failed." and "output" in result and result["output"] is None messages = [i for i in items() if i["type"] == "message" and i["role"] == "assistant"] assert marker in "\n".join(p.get("text", "") for p in messages[-1]["content"]) assert len(calls) == 2 From 541499bf63a6e55d98035b62e84692ee4544e278 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Mon, 28 Sep 2026 16:15:20 +0000 Subject: [PATCH 7/8] Confirm native Codex terminal cleanup before cancellation settles --- CONTRIBUTING.md | 5 + .../internal/agent/codex/executor.go | 12 ++ .../internal/agent/codex/executor_test.go | 4 +- .../internal/agent/codex/executor_turn.go | 9 ++ .../agent/codex/preparation_helpers_test.go | 20 ++++ .../internal/agent/codex/session.go | 1 + .../internal/agent/codex/terminal_cleanup.go | 58 +++++++++ .../agent/codex/terminal_cleanup_test.go | 113 ++++++++++++++++++ 8 files changed, 221 insertions(+), 1 deletion(-) create mode 100644 apps/parsar-daemon/internal/agent/codex/terminal_cleanup.go create mode 100644 apps/parsar-daemon/internal/agent/codex/terminal_cleanup_test.go diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e2491b2e5..70cd94af7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1632,6 +1632,11 @@ that already owns the Executor. Connection shutdown owns transport-loss cleanup. Preserve the ten-second settlement wait and separate five-second receipt send budget; timeout is not proof of quiescence. The observed cancellation outcome retains native identity, Usage and output without fabricating missing evidence. +Codex native Turn interruption can leave background terminals alive. Its adapter +uses the exact thread-owned terminal list and confirmed per-terminal termination +before settling cancellation, and repeats this cleanup during Executor close. +The bulk clean acknowledgement does not prove termination. Failed cleanup keeps +the native owner available for a later close attempt. Private preparation controls reserve a per-Turn admission, not a new Executor. They carry an explicit Session identity and immutable configuration without model diff --git a/apps/parsar-daemon/internal/agent/codex/executor.go b/apps/parsar-daemon/internal/agent/codex/executor.go index 8692b3ce2..2e437c20e 100644 --- a/apps/parsar-daemon/internal/agent/codex/executor.go +++ b/apps/parsar-daemon/internal/agent/codex/executor.go @@ -129,6 +129,18 @@ func (e *Executor) Close(ctx context.Context) error { stop() } } + if running != nil && base.rpc.Alive() { + // A caller deadline only stops waiting. Cleanup retains a bounded + // opportunity to release native terminals before closing their owner. + cleanupCtx, stop := context.WithTimeout(context.Background(), rpcKillTimeout) + err := running.cleanupNativeTerminals(cleanupCtx) + stop() + if err != nil { + // Preserve the native owner and exact handles for a later Close. + done <- err + return + } + } base.cancelFn() err := base.rpc.Close() e.mu.Lock() diff --git a/apps/parsar-daemon/internal/agent/codex/executor_test.go b/apps/parsar-daemon/internal/agent/codex/executor_test.go index 6cd32f60a..f6c9d56fa 100644 --- a/apps/parsar-daemon/internal/agent/codex/executor_test.go +++ b/apps/parsar-daemon/internal/agent/codex/executor_test.go @@ -18,7 +18,9 @@ func executorFixture(t *testing.T, mode string) (*Executor, string) { t.Helper() req, cfg, root := preparationFixture(t) t.Setenv("OAC_TEST_EXECUTOR_MODE", mode) - e, err := newExecutor(t.Context(), req, cfg) + ownerCtx, cancelOwner := context.WithCancel(context.Background()) + t.Cleanup(cancelOwner) + e, err := newExecutor(ownerCtx, req, cfg) if err != nil { t.Fatal(err) } diff --git a/apps/parsar-daemon/internal/agent/codex/executor_turn.go b/apps/parsar-daemon/internal/agent/codex/executor_turn.go index a356f30f7..0579e81fa 100644 --- a/apps/parsar-daemon/internal/agent/codex/executor_turn.go +++ b/apps/parsar-daemon/internal/agent/codex/executor_turn.go @@ -76,6 +76,15 @@ func (s *Session) settleExecutorTurn(startErr error) { s.settlementErr = errors.Join(s.settlementErr, s.cancelErr, s.rpc.Close()) } } + if s.cancelled.Load() && s.nativeSettled.Load() && s.cancelErr == nil && s.rpc.Alive() { + cleanupCtx, stop := context.WithTimeout(context.Background(), 10*time.Second) + err := s.cleanupNativeTerminals(cleanupCtx) + stop() + if err != nil { + s.settlement = agent.TurnSettlement{Reason: "native_terminal_cleanup_unconfirmed"} + s.settlementErr = errors.Join(s.settlementErr, err) + } + } s.cancelFn() } diff --git a/apps/parsar-daemon/internal/agent/codex/preparation_helpers_test.go b/apps/parsar-daemon/internal/agent/codex/preparation_helpers_test.go index 857e0082d..2957a4718 100644 --- a/apps/parsar-daemon/internal/agent/codex/preparation_helpers_test.go +++ b/apps/parsar-daemon/internal/agent/codex/preparation_helpers_test.go @@ -185,6 +185,26 @@ func TestPreparationFakeCodexProcess(t *testing.T) { } } result = map[string]any{"thread": map[string]string{"id": "fixture-native-thread"}, "model": "fixture-model"} + case "thread/backgroundTerminals/list": + result = map[string]any{"data": []any{}} + if executorMode == "terminal-cleanup" { + if _, err := os.Stat(os.Getenv("OAC_TEST_PREPARATION_FRAMES") + ".terminated"); os.IsNotExist(err) { + result = map[string]any{"data": []any{map[string]string{"processId": "owned-terminal"}}} + } + } + case "thread/backgroundTerminals/terminate": + var target struct { + ThreadID string `json:"threadId"` + ProcessID string `json:"processId"` + } + if json.Unmarshal(frame.Params, &target) != nil || target.ThreadID != "fixture-native-thread" || target.ProcessID != "owned-terminal" { + os.Exit(8) + } + _, allowed := os.Stat(os.Getenv("OAC_TEST_PREPARATION_FRAMES") + ".allow-cleanup") + result = map[string]any{"terminated": allowed == nil} + if allowed == nil { + _ = os.WriteFile(os.Getenv("OAC_TEST_PREPARATION_FRAMES")+".terminated", nil, 0600) + } case "turn/interrupt": if executorMode == "interrupt-error" { _ = output.Encode(map[string]any{"id": frame.ID, "error": map[string]any{"code": -32603, "message": "interrupt rejected"}}) diff --git a/apps/parsar-daemon/internal/agent/codex/session.go b/apps/parsar-daemon/internal/agent/codex/session.go index bbd808693..bed20d0d6 100644 --- a/apps/parsar-daemon/internal/agent/codex/session.go +++ b/apps/parsar-daemon/internal/agent/codex/session.go @@ -69,6 +69,7 @@ type Session struct { nativeSettled atomic.Bool settlement agent.TurnSettlement settlementErr error + terminalCleanupMu sync.Mutex operationMu sync.Mutex operations sync.WaitGroup operationsClosed bool diff --git a/apps/parsar-daemon/internal/agent/codex/terminal_cleanup.go b/apps/parsar-daemon/internal/agent/codex/terminal_cleanup.go new file mode 100644 index 000000000..a6f11c050 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/codex/terminal_cleanup.go @@ -0,0 +1,58 @@ +package codex + +import ( + "context" + "encoding/json" + "errors" +) + +// A native interrupted Turn can leave unified-exec terminals running. Retire +// their exact thread-owned handles; the bulk clean RPC only queues cleanup. +func (s *Session) cleanupNativeTerminals(ctx context.Context) error { + s.terminalCleanupMu.Lock() + defer s.terminalCleanupMu.Unlock() + thread := s.currentThreadID() + if thread == "" { + return nil + } + seen := map[string]bool{} + for { + raw, err := s.rpc.Request(ctx, "thread/backgroundTerminals/list", map[string]any{"threadId": thread, "limit": 100}) + if err != nil { + return err + } + var page struct { + Data *[]struct { + ProcessID string `json:"processId"` + } `json:"data"` + NextCursor *string `json:"nextCursor"` + } + if json.Unmarshal(raw, &page) != nil || page.Data == nil { + return errors.New("codex: invalid native terminal list") + } + if len(*page.Data) == 0 { + if page.NextCursor != nil { + return errors.New("codex: incomplete native terminal list") + } + return nil + } + for _, terminal := range *page.Data { + if terminal.ProcessID == "" || seen[terminal.ProcessID] { + return errors.New("codex: invalid or retained native terminal identity") + } + seen[terminal.ProcessID] = true + raw, err = s.rpc.Request(ctx, "thread/backgroundTerminals/terminate", map[string]string{"threadId": thread, "processId": terminal.ProcessID}) + if err != nil { + return err + } + var receipt struct { + Terminated bool `json:"terminated"` + } + if json.Unmarshal(raw, &receipt) != nil || !receipt.Terminated { + return errors.New("codex: native terminal termination unconfirmed") + } + } + // Terminating the first page removes its handles. Read the first page + // again to confirm absence and cover any remaining native terminals. + } +} diff --git a/apps/parsar-daemon/internal/agent/codex/terminal_cleanup_test.go b/apps/parsar-daemon/internal/agent/codex/terminal_cleanup_test.go new file mode 100644 index 000000000..ace5c1a17 --- /dev/null +++ b/apps/parsar-daemon/internal/agent/codex/terminal_cleanup_test.go @@ -0,0 +1,113 @@ +package codex + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/MiniMax-AI-Dev/parsar/internal/agentdaemon/proto" +) + +func TestExecutorCancellationRequiresNativeTerminalCleanup(t *testing.T) { + for _, confirmed := range []bool{false, true} { + t.Run(map[bool]string{false: "retry", true: "confirmed"}[confirmed], func(t *testing.T) { + e, root := executorFixture(t, "terminal-cleanup") + allow := func() { + t.Helper() + if err := os.WriteFile(filepath.Join(root, "frames.jsonl.allow-cleanup"), nil, 0600); err != nil { + t.Fatal(err) + } + } + if confirmed { + allow() + } + out := make(chan proto.Envelope, 20) + turn, err := e.StartTurn(t.Context(), "cancel", proto.TextInput("hold"), out) + if err != nil { + t.Fatal(err) + } + ctx, stop := context.WithTimeout(t.Context(), 5*time.Second) + defer stop() + if err := turn.Cancel(ctx); (err == nil) != confirmed { + t.Fatalf("cancellation confirmed=%t: %v", confirmed, err) + } + settlement, err := turn.AwaitSettlement(ctx) + if (err == nil) != confirmed || settlement.Reusable != confirmed { + t.Fatal("cleanup changed cancellation confirmation", settlement, err) + } + for range out { + } + if !e.prepared.session.rpc.Alive() { + t.Fatal("native owner lost before cleanup/reuse") + } + frames := preparationFrames(t, root) + interrupted, terminated := false, false + for _, frame := range frames { + if frame.Method == "turn/interrupt" { + interrupted = true + } + if frame.Method == "thread/backgroundTerminals/terminate" { + if !interrupted { + t.Fatal("cleanup preceded native interrupt") + } + terminated = true + } + } + if !terminated { + t.Fatal("native interrupted receipt skipped terminal cleanup") + } + if confirmed { + nextOut := make(chan proto.Envelope, 20) + next, err := e.StartTurn(ctx, "next", proto.TextInput("answer"), nextOut) + if err != nil || !awaitExecutorTurn(t, next, nextOut).Reusable { + t.Fatal("confirmed cleanup prevented reuse", err) + } + } else { + if err := e.Close(ctx); err == nil || !e.prepared.session.rpc.Alive() { + t.Fatal("failed cleanup released the owned native process", err) + } + allow() + if err := e.Close(ctx); err != nil { + t.Fatal("cleanup retry failed", err) + } + if err := turn.Cancel(ctx); err == nil { + t.Fatal("resource cleanup fabricated native cancellation confirmation") + } + } + }) + } +} + +func TestExecutorCloseCleansNativeTerminalsAfterNormalCompletion(t *testing.T) { + e, root := executorFixture(t, "terminal-cleanup") + out := make(chan proto.Envelope, 20) + turn, err := e.StartTurn(t.Context(), "normal", proto.TextInput("answer"), out) + if err != nil { + t.Fatal(err) + } + if !awaitExecutorTurn(t, turn, out).Reusable { + t.Fatal("normal Turn did not retain its Executor") + } + for _, frame := range preparationFrames(t, root) { + if frame.Method == "thread/backgroundTerminals/terminate" { + t.Fatal("normal Turn prematurely killed its background terminal") + } + } + if err := os.WriteFile(filepath.Join(root, "frames.jsonl.allow-cleanup"), nil, 0600); err != nil { + t.Fatal(err) + } + if err := e.Close(t.Context()); err != nil { + t.Fatal(err) + } + terminated := false + for _, frame := range preparationFrames(t, root) { + if frame.Method == "thread/backgroundTerminals/terminate" { + terminated = true + } + } + if !terminated || e.prepared.session.rpc.Alive() { + t.Fatal("Executor Close left native terminals or owner alive") + } +} From fecf58da94074ab77b2db6cc14684ae6eb0a1c72 Mon Sep 17 00:00:00 2001 From: saladday <1203511142@qq.com> Date: Mon, 28 Sep 2026 16:16:37 +0000 Subject: [PATCH 8/8] Bound Codex terminal cleanup control writes --- .../parsar-daemon/internal/agent/codex/terminal_cleanup.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apps/parsar-daemon/internal/agent/codex/terminal_cleanup.go b/apps/parsar-daemon/internal/agent/codex/terminal_cleanup.go index a6f11c050..49d409b2a 100644 --- a/apps/parsar-daemon/internal/agent/codex/terminal_cleanup.go +++ b/apps/parsar-daemon/internal/agent/codex/terminal_cleanup.go @@ -15,9 +15,12 @@ func (s *Session) cleanupNativeTerminals(ctx context.Context) error { if thread == "" { return nil } + request := func(method string, params any) (json.RawMessage, error) { + return s.rpc.request(ctx, method, params, func(frame any) error { return s.rpc.writeFrameContext(ctx, frame) }) + } seen := map[string]bool{} for { - raw, err := s.rpc.Request(ctx, "thread/backgroundTerminals/list", map[string]any{"threadId": thread, "limit": 100}) + raw, err := request("thread/backgroundTerminals/list", map[string]any{"threadId": thread, "limit": 100}) if err != nil { return err } @@ -41,7 +44,7 @@ func (s *Session) cleanupNativeTerminals(ctx context.Context) error { return errors.New("codex: invalid or retained native terminal identity") } seen[terminal.ProcessID] = true - raw, err = s.rpc.Request(ctx, "thread/backgroundTerminals/terminate", map[string]string{"threadId": thread, "processId": terminal.ProcessID}) + raw, err = request("thread/backgroundTerminals/terminate", map[string]string{"threadId": thread, "processId": terminal.ProcessID}) if err != nil { return err }