From acb8ab358c8f25748f0c7261ceefc6e69341d441 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 12 Nov 2026 11:19:20 -0700 Subject: [PATCH 1/7] fix: interrupt busy runtimes and confirm cleanup before replacement Signed-off-by: Sara Tadayon --- crates/fabric-core/src/runtime.rs | 247 +++++++++++++++--- .../nemo_fabric.runtime.md | 4 +- docs/sdk/python.mdx | 2 + .../src/nemo_fabric/runtime.py | 18 +- .../src/nemo_fabric/runtime_server/README.md | 2 +- .../src/nemo_fabric/runtime_server/state.py | 27 +- tests/python/runtime_server/test_sdk.py | 42 +++ tests/python/runtime_server/test_state.py | 66 ++++- tests/python/test_runtime.py | 53 +++- 9 files changed, 394 insertions(+), 67 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d084b6126..d7d7cd458 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -947,6 +947,7 @@ struct LocalAdapterHost { execution: Option, adapter_execution: Option, child: Child, + pending_stop: Vec, #[cfg(windows)] job: Option, stdin: ChildStdin, @@ -1958,67 +1959,100 @@ impl RuntimeAdapter for LocalHostAdapter { } fn stop(&self, runtime: &RuntimeHandle) -> Result> { - let host = { local_hosts().get(&runtime.runtime_id).cloned() }; - let Some(host) = host else { + let Some(host) = local_hosts().get(&runtime.runtime_id).cloned() else { + let process = local_host_processes().get(&runtime.runtime_id).cloned(); + if process + .as_ref() + .is_some_and(|process| !owned_process_has_exited(process)) + { + return Err(cleanup_unconfirmed(runtime)); + } + let pending = STOPPING_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .get(&runtime.runtime_id) + .cloned() + .unwrap_or_default(); + confirm_stopped_descendants(runtime, &pending)?; local_host_processes().remove(&runtime.runtime_id); + STOPPING_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .remove(&runtime.runtime_id); return Ok(vec![local_host_stop_event(runtime, true, false)]); }; - let mut host = host.lock().unwrap_or_else(|error| error.into_inner()); - host.cleanup_pending = true; + let deadline = Instant::now() + LOCAL_HOST_STOP_TIMEOUT; + let mut interrupted = None; + let mut guard = loop { + match host.try_lock() { + Ok(guard) => break guard, + Err(std::sync::TryLockError::Poisoned(error)) => break error.into_inner(), + Err(std::sync::TryLockError::WouldBlock) => { + if interrupted.is_none() { + interrupted = Some(interrupt_local_host(runtime)?); + } + if Instant::now() >= deadline { + return Err(cleanup_unconfirmed(runtime)); + } + thread::sleep(Duration::from_millis(10)); + } + } + }; + guard.cleanup_pending = true; + PENDING_LOCAL_CLEANUP.lock().unwrap_or_else(|error| error.into_inner()) + .insert(runtime.runtime_id.clone(), guard.cleanup_scope.clone()); + if let Some(processes) = interrupted { + guard.pending_stop.extend(processes); + } + if let Some(processes) = STOPPING_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .get(&runtime.runtime_id) + { + guard.pending_stop.extend(processes.clone()); + } #[cfg(unix)] - remember_host_descendants(&mut host); + if guard + .child + .try_wait() + .map_err(|_| cleanup_unconfirmed(runtime))? + .is_none() + { + let pid = guard.child.id(); + guard.pending_stop.extend( + host_descendants(pid) + .into_iter() + .map(|pid| LocalHostProcess::new(pid as u32, SystemTime::now())), + ); + } let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Stop(AdapterLifecycleStop { runtime_id: runtime.runtime_id.clone(), })); let result = exchange_lifecycle_message( - &mut host, + &mut guard, &runtime.runtime_id, &request, LOCAL_HOST_STOP_TIMEOUT, ); - let termination = terminate_local_host(&mut host); - let diagnostics = local_host_diagnostics(&host); - let host_crashed = matches!( - &result, - Err(FabricError::AdapterLifecycleOperation { code, .. }) - if code == "host_crashed" - ); - termination.map_err(|source| { - PENDING_LOCAL_CLEANUP - .lock() - .unwrap_or_else(|error| error.into_inner()) - .insert(runtime.runtime_id.clone(), host.cleanup_scope.clone()); - lifecycle_error( - AdapterLifecycleOperation::Stop, - &runtime.runtime_id, - "cleanup_unconfirmed", - format!("persistent local adapter host could not be terminated: {source}"), - diagnostics, - ) - })?; - // Keep the host and its lock available for a retry until termination - // has been confirmed. A failed adapter stop may still be safely killed. + terminate_local_host(&mut guard).map_err(|_| cleanup_unconfirmed(runtime))?; + confirm_stopped_descendants(runtime, &guard.pending_stop)?; + remove_local_host_files(&guard).map_err(|_| cleanup_unconfirmed(runtime))?; local_hosts().remove(&runtime.runtime_id); local_host_processes().remove(&runtime.runtime_id); - PENDING_LOCAL_CLEANUP + STOPPING_PROCESSES .lock() .unwrap_or_else(|error| error.into_inner()) .remove(&runtime.runtime_id); - let removal = remove_local_host_files(&host); + PENDING_LOCAL_CLEANUP.lock().unwrap_or_else(|error| error.into_inner()) + .remove(&runtime.runtime_id); + let host_crashed = matches!( + &result, + Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "host_crashed" + ); if !host_crashed { result?; } - removal.map_err(|source| { - lifecycle_error( - AdapterLifecycleOperation::Stop, - &runtime.runtime_id, - "host_cleanup_failed", - format!("persistent local adapter host files could not be removed: {source}"), - "", - ) - })?; - #[cfg(test)] TEST_STOPPED_AGENTS .lock() @@ -2028,6 +2062,110 @@ impl RuntimeAdapter for LocalHostAdapter { } } +fn cleanup_unconfirmed(runtime: &RuntimeHandle) -> FabricError { + lifecycle_error( + AdapterLifecycleOperation::Stop, + &runtime.runtime_id, + "cleanup_unconfirmed", + "Owned process cleanup is unconfirmed; retry stop before replacement.", + "", + ) +} + +fn owned_process_has_exited(process: &LocalHostProcess) -> bool { + if process.has_exited() == Some(true) { + return true; + } + #[cfg(unix)] + { + let Ok(pid) = libc::pid_t::try_from(process.pid()) else { + return false; + }; + if pid <= 0 { + return false; + } + // SAFETY: signal zero only probes existence. An unidentified process is + // never signalled; ESRCH can still prove that its PID is now absent. + (unsafe { libc::kill(pid, 0) } != 0) + && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + } + #[cfg(not(unix))] + false +} + +#[cfg(unix)] +fn signal_owned_process(process: &LocalHostProcess) -> std::io::Result<()> { + if owned_process_has_exited(process) { + return Ok(()); + } + if !process.is_current() { + return Err(std::io::Error::other("process ownership is unconfirmed")); + } + // SAFETY: the process identity was checked immediately before signalling. + if unsafe { libc::kill(process.pid() as libc::pid_t, libc::SIGKILL) } != 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::ESRCH) { + return Err(error); + } + } + Ok(()) +} + +fn interrupt_local_host(runtime: &RuntimeHandle) -> Result> { + #[cfg(unix)] + { + let process = local_host_processes() + .get(&runtime.runtime_id) + .cloned() + .ok_or_else(|| cleanup_unconfirmed(runtime))?; + if process.has_exited() == Some(true) { + return Ok(Vec::new()); + } + if !process.is_current() { + return Err(cleanup_unconfirmed(runtime)); + } + let descendants: Vec<_> = host_descendants(process.pid()) + .into_iter() + .map(|pid| LocalHostProcess::new(pid as u32, SystemTime::now())) + .collect(); + let descendants = { + let mut retained = STOPPING_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()); + let previous = retained.entry(runtime.runtime_id.clone()).or_default(); + previous.extend(descendants); + previous.clone() + }; + for child in descendants.iter().rev() { + signal_owned_process(child).map_err(|_| cleanup_unconfirmed(runtime))?; + } + signal_owned_process(&process).map_err(|_| cleanup_unconfirmed(runtime))?; + Ok(descendants) + } + #[cfg(not(unix))] + Err(cleanup_unconfirmed(runtime)) +} + +fn confirm_stopped_descendants( + runtime: &RuntimeHandle, + processes: &[LocalHostProcess], +) -> Result<()> { + let deadline = Instant::now() + LOCAL_HOST_EXIT_GRACE; + loop { + if processes.iter().all(owned_process_has_exited) { + return Ok(()); + } + #[cfg(unix)] + for process in processes { + signal_owned_process(process).map_err(|_| cleanup_unconfirmed(runtime))?; + } + if Instant::now() >= deadline { + return Err(cleanup_unconfirmed(runtime)); + } + thread::sleep(Duration::from_millis(10)); + } +} + fn start_local_runtime( plan: &RunPlan, environment: EnvironmentHandle, @@ -3007,6 +3145,10 @@ fn service_release_event(service: &ServiceHandle, already_released: bool) -> Fab ) } +// Keep identity-checked snapshots even if interruption fails before the host lock is released. +static STOPPING_PROCESSES: LazyLock>>> = + LazyLock::new(|| Mutex::new(BTreeMap::new())); + fn local_hosts() -> std::sync::MutexGuard<'static, BTreeMap>>> { LOCAL_HOSTS .lock() @@ -3206,6 +3348,7 @@ fn spawn_local_host( execution: crate::provenance::core_snapshot(&plan.config.provenance), adapter_execution: None, child, + pending_stop: Vec::new(), #[cfg(windows)] job, stdin, @@ -5410,6 +5553,30 @@ else: TEST_STOPPED_AGENTS.lock().expect("stop tracker").clone() } + #[cfg(unix)] + #[test] + fn stop_retries_can_confirm_a_child_that_exited_before_identity_capture() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let mut child = Command::new("sh").args(["-c", "exit 0"]).spawn().unwrap(); + let pid = child.id(); + child.wait().unwrap(); + let gone = LocalHostProcess::new(pid, SystemTime::now()); + assert!(!gone.is_current()); + local_hosts() + .get(&runtime.runtime_id) + .unwrap() + .lock() + .unwrap() + .pending_stop + .push(gone); + stop_runtime(&plan, &runtime) + .expect("a confirmed-gone child cannot permanently block cleanup"); + assert!(!local_hosts().contains_key(&runtime.runtime_id)); + assert!(local_host_has_exited(&runtime.runtime_id).is_none()); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn local_host_reuses_one_process_and_stops_idempotently() { let (root, plan) = local_host_plan("success"); diff --git a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md index a030b498e..4306d1d01 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md @@ -223,7 +223,7 @@ The snapshot covers the local adapter host process and its live descendants. It async def stop() -> None ``` -Destroy an idle runtime exactly once. +Stop the runtime, interrupting an active buffered invocation if needed. Repeated calls after a successful stop are no-ops. A failed runtime may still be stopped so its resources are released. @@ -231,7 +231,7 @@ Repeated calls after a successful stop are no-ops. A failed runtime may still be **Raises:** - - `FabricStateError`: If the runtime is already stopping or has an invocation in flight. + - `FabricStateError`: If the runtime is already stopping or has an streaming invocation in flight. - `FabricNativeUnavailableError`: If the native extension is missing. - `FabricRuntimeError`: If native runtime shutdown fails. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index f5320750f..d805df988 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1695,3 +1695,5 @@ OpenClaw service startup reports retain their original source and capture time w Calling applications and environment owners supply typed `ProvenanceConfig.reports` snapshots through the configuration already sent to NeMo Fabric. Capture them at application initialization and preserve their original `source`, `origin`, availability and `observed_at`. A guest never inspects its caller. Harbor implements this seam with `fabric_provenance`, whose shape matches the SDK control; `--ak 'fabric_provenance={"enabled":false}'` disables collection before host initialization and propagates to the guest. The [software evidence example](https://github.com/NVIDIA/NeMo-Fabric/tree/main/examples/provenance) provides six execution-path checks, separate-venv runs, saved RunResults and private comparison fixtures. Its local provider/service fixtures are labeled separately from live deployments. Safe configuration capture is the P0 follow-up and is outside this software-version contract. + +Buffered runtime shutdown can interrupt an active invocation on supported Unix hosts. `Runtime.stop()` waits for the interrupted invocation to settle before reporting stopped. Cleanup errors retain ownership for retry; the runtime server blocks replacement for the same agent until cleanup succeeds. Streaming shutdown continues to use the stream lifecycle. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 3438a90d2..4e3df3362 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -309,10 +309,12 @@ def stop_after_cancel() -> Any: self._status = RuntimeStatus.STOPPED raise except FabricError: - self._status = RuntimeStatus.FAILED + if self._status is not RuntimeStatus.STOPPED: + self._status = RuntimeStatus.FAILED raise except Exception as error: - self._status = RuntimeStatus.FAILED + if self._status is not RuntimeStatus.STOPPED: + self._status = RuntimeStatus.FAILED raise _runtime_error(error, stage="invoke") from error if absorb_result: self._absorb(typed_result) @@ -794,14 +796,14 @@ def snapshot() -> Any: raise _resource_snapshot_error(native, error) from error async def stop(self) -> None: - """Destroy an idle runtime exactly once. + """Stop the runtime, interrupting an active buffered invocation if needed. Repeated calls after a successful stop are no-ops. A failed runtime may still be stopped so its resources are released. Raises: FabricStateError: If the runtime is already stopping or has an - invocation in flight. + streaming invocation in flight. FabricNativeUnavailableError: If the native extension is missing. FabricRuntimeError: If native runtime shutdown fails. """ @@ -816,8 +818,7 @@ async def stop(self) -> None: ) if self._current_stream is not None and not self._current_stream._finalized: await self._current_stream.aclose() - if self._current_task is not None: - raise FabricStateError("cannot stop while a turn is in flight") + active_task = self._current_task if self._closing: raise FabricStateError("runtime shutdown is already in progress") self._closing = True @@ -838,6 +839,11 @@ def stop() -> Any: return result await _call_blocking(stop) + if active_task is not None and active_task is not asyncio.current_task(): + # Observe the interrupted invocation before publishing STOPPED; + # its error path must not overwrite the final lifecycle state. + while self._current_task is active_task: + await asyncio.sleep(0.01) except asyncio.CancelledError as error: self._status = RuntimeStatus.STOPPED if stopped else RuntimeStatus.FAILED stop_error = error diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md index 3d4558d0c..d1865f8c0 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md @@ -79,7 +79,7 @@ flowchart TD Caller --> Bridge --> Socket --> Host --> Runtime ``` -The outer server is separate from the SDK's adapter process and the selected harness. It serializes starts and serializes invoke/stop operations for each runtime ID, while separate runtimes can progress independently. A busy stop returns `runtime_busy` promptly and retains ownership. Retry stop after the invocation ends. Direct SDK consumers can also use streaming or shared services. +The outer server is separate from the SDK's adapter process and the selected harness. It serializes starts and invocations for each runtime ID, while separate runtimes can progress independently. Stop uses a separate path so an active buffered invocation cannot hold shutdown behind its lock. On supported Unix hosts, busy stop interrupts the owned adapter process tree and confirms exit before releasing ownership. Unconfirmed cleanup retains the runtime and blocks replacement for that agent; independent agents remain available. Direct SDK consumers can also use streaming or shared services. Ordinary SDK stop does not establish that all harness descendants exited. A successful stop response reports completion of the SDK operation. The caller must use the sandbox supervisor to terminate remaining descendants. The server does not interrupt adapters through a new forced-stop SDK API. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py index 9c345c237..3a0444cdd 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py @@ -58,6 +58,7 @@ class OwnedRuntime: state: str = "running" lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False) generation: int = 0 + stop_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False) class RuntimeServer: @@ -89,10 +90,14 @@ def _owned(self, request: dict[str, Any]) -> OwnedRuntime: async def _stop(self, owned: OwnedRuntime) -> None: owned.generation += 1 - owned.state = "unknown" + owned.state = "stopping" runtime = owned.runtime - if runtime is not None: - await runtime.stop() + try: + if runtime is not None: + await runtime.stop() + except BaseException: + owned.state = "unknown" + raise owned.runtime = None owned.state = "stopped" runtime_id = runtime.runtime_id if runtime is not None else None @@ -154,6 +159,12 @@ async def handle(self, request: Any) -> dict[str, Any]: async with self._start_lock: if self.shutting_down: raise ProtocolError("host_stopping") + if any( + owned.agent == request["agent"] + and owned.state in ("stopping", "unknown") + for owned in self.runtimes.values() + ): + raise ProtocolError("cleanup_unconfirmed", "start") if self.start_uncertain: raise ProtocolError("start_uncertain", "start") stage = "validation" @@ -219,16 +230,16 @@ async def handle(self, request: Any) -> dict[str, Any]: raise ProtocolError("runtime_busy", "stop") owned.generation += 1 owned.state = "stopping" - async with owned.lock: - if self.shutting_down: - raise ProtocolError("host_stopping") - if operation == "stop": + if operation == "stop": + async with owned.stop_lock: if owned.state == "stopped": return envelope(operation, result) changed, effects, stage = None, "unknown", "stop" await self._stop(owned) return envelope(operation, result, changed=True) - + async with owned.lock: + if self.shutting_down: + raise ProtocolError("host_stopping") stage = "invoke" if ( owned.state != "running" diff --git a/tests/python/runtime_server/test_sdk.py b/tests/python/runtime_server/test_sdk.py index 8176b9bdc..b1d5956b4 100644 --- a/tests/python/runtime_server/test_sdk.py +++ b/tests/python/runtime_server/test_sdk.py @@ -278,3 +278,45 @@ async def test_health_workers_recover_while_provider_descendants_hold_pipes( with suppress(Exception): await asyncio.wait_for(invoked, 5) await owned.stop() + + +async def test_real_sdk_busy_stop_exits_host_and_native_child( + blocking_adapter_config, tmp_path, process_is_running +): + (tmp_path / "spawn-harness").touch() + host = RuntimeServer(directory=tmp_path) + command = {"interface_version": 2, "agent": "blocking"} + started = await host.handle( + { + **command, + "operation": "start", + "config": blocking_adapter_config.to_mapping(), + } + ) + assert started["status"] == "succeeded", started + runtime_id = started["result"]["runtime_id"] + command["runtime_id"] = runtime_id + owned = host.runtimes[runtime_id].runtime + task = asyncio.create_task( + host.handle({**command, "operation": "invoke", "input": "slow"}) + ) + try: + async with asyncio.timeout(5): + while not (tmp_path / "entered").exists(): + await asyncio.sleep(0.01) + pids = [ + int((tmp_path / name).read_text()) + for name in ("adapter.pid", "harness.pid") + ] + response = await asyncio.wait_for( + host.handle({**command, "operation": "stop"}), 5 + ) + assert response["status"] == "succeeded", response + await asyncio.wait_for(task, 1) + assert all(not process_is_running(pid) for pid in pids) + assert host.runtimes[runtime_id].state == "stopped" + finally: + (tmp_path / "release").touch() + with suppress(Exception): + await task + await owned.stop() diff --git a/tests/python/runtime_server/test_state.py b/tests/python/runtime_server/test_state.py index b77f9be39..2424ecc08 100644 --- a/tests/python/runtime_server/test_state.py +++ b/tests/python/runtime_server/test_state.py @@ -294,9 +294,15 @@ async def test_failed_stop_keeps_ownership_and_allows_distinct_runtime( assert "PRIVATE" not in str(response) assert host.runtimes["runtime-1"].runtime is mock_runtime assert host.runtimes["runtime-1"].state == "unknown" - replacement = runtime_factory("runtime-2") + replacement = runtime_factory("runtime-2", agent="independent") mock_fabric.start_runtime.return_value = replacement - assert (await host.handle(host_request("start")))["status"] == "succeeded" + config = { + "metadata": {"name": "independent"}, + "harness": {"adapter_id": "test.host"}, + } + assert ( + await host.handle(host_request("start", agent="independent", config=config)) + )["status"] == "succeeded" assert mock_fabric.start_runtime.await_count == 2 @@ -389,7 +395,7 @@ async def test_non_active_returned_runtime_is_owned_for_stop( assert next_response["result"]["runtime_id"] == "runtime-2" -async def test_stop_can_be_retried_after_a_busy_invocation( +async def test_stop_does_not_wait_for_the_invocation_lock( host, host_request, mock_runtime ): await host.handle(host_request("start")) @@ -406,11 +412,10 @@ async def invoke(**kwargs): await entered.wait() second = asyncio.create_task(host.handle(host_request("stop"))) await asyncio.sleep(0) - mock_runtime.stop.assert_not_awaited() - assert (await second)["error"]["code"] == "runtime_busy" + mock_runtime.stop.assert_awaited_once() release.set() assert (await first)["status"] == "succeeded" - assert (await host.handle(host_request("stop")))["status"] == "succeeded" + assert (await second)["status"] == "succeeded" mock_runtime.stop.assert_awaited_once_with() @@ -492,3 +497,52 @@ async def test_stopped_runtime_records_are_bounded( assert (await host.handle(host_request("stop", runtime_id="runtime-129")))[ "status" ] == "succeeded" + + +async def test_busy_stop_reaches_sdk_before_invocation_finishes( + host, host_request, mock_runtime +): + await host.handle(host_request("start")) + entered, release = asyncio.Event(), asyncio.Event() + native = mock_runtime.invoke.return_value + + async def invoke(**kwargs): + entered.set() + await release.wait() + return native + + async def stop(): + release.set() + + mock_runtime.invoke.side_effect = invoke + mock_runtime.stop.side_effect = stop + task = asyncio.create_task(host.handle(host_request("invoke", input={}))) + await entered.wait() + try: + response = await asyncio.wait_for(host.handle(host_request("stop")), 0.5) + assert response["status"] == "succeeded" + mock_runtime.stop.assert_awaited_once() + finally: + release.set() + await task + + +async def test_unconfirmed_stop_blocks_replacement_but_not_other_agents( + host, host_request, mock_runtime, mock_fabric, runtime_factory, config +): + await host.handle(host_request("start")) + mock_runtime.stop.side_effect = RuntimeError("PRIVATE shutdown detail") + assert (await host.handle(host_request("stop")))["error"]["code"] == "stop_failed" + rejected = await host.handle(host_request("start")) + assert rejected["error"]["code"] == "cleanup_unconfirmed" + assert "PRIVATE" not in str(rejected) + mock_fabric.start_runtime.assert_awaited_once() + mock_fabric.start_runtime.return_value = runtime_factory( + "independent", agent="other" + ) + other_config = {**config, "metadata": {"name": "other"}} + assert ( + await host.handle(host_request("start", agent="other", config=other_config)) + )["status"] == "succeeded" + mock_runtime.stop.side_effect = None + assert (await host.handle(host_request("stop")))["status"] == "succeeded" diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index f7f730867..a6b9ec1b1 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -906,7 +906,7 @@ async def test_stop_closes_runtime_owned_collector(mock_native: MagicMock): mock_collector.aclose.assert_awaited_once() -async def test_stop_rejects_in_flight_turn( +async def test_stop_interrupts_in_flight_turn( mock_native: MagicMock, ): started = threading.Event() @@ -923,11 +923,16 @@ def blocking_invoke(*args: Any) -> str: turn = asyncio.create_task(runtime.invoke(input="hello")) assert await _wait_for(started) - with pytest.raises(FabricStateError, match="in flight"): - await runtime.stop() + stop_result = mock_native.stop_runtime.return_value - release.set() + def interrupt(*args): + release.set() + return stop_result + + mock_native.stop_runtime.side_effect = interrupt + await asyncio.wait_for(runtime.stop(), 2) await turn + assert runtime.status is RuntimeStatus.STOPPED async def test_concurrent_invokes_are_rejected( @@ -1702,3 +1707,43 @@ async def test_runtime_status_handle_mutation_cannot_change_shared_report( assert report.handle.environment == original["handle"]["environment"] assert report.to_mapping() == original await runtime.stop() + + +async def test_stop_waits_for_invocation_not_the_callers_remaining_work(mock_native): + entered, release = threading.Event(), threading.Event() + later_work = asyncio.Event() + completed = asyncio.Event() + invoke = mock_native.invoke_runtime.side_effect + stop_result = mock_native.stop_runtime.return_value + + def blocking(*args): + entered.set() + assert release.wait(5) + return invoke(*args) + + def interrupt(*args): + release.set() + return stop_result + + mock_native.invoke_runtime.side_effect = blocking + mock_native.stop_runtime.side_effect = interrupt + runtime = _runtime_wrapper(mock_native) + + async def caller(): + await runtime.invoke(input="hello") + completed.set() + await later_work.wait() + + task = asyncio.create_task(caller()) + assert await _wait_for(entered) + stopping = asyncio.create_task(runtime.stop()) + try: + await asyncio.wait_for(completed.wait(), 2) + await asyncio.wait_for(asyncio.shield(stopping), 0.5) + assert not task.done() + assert runtime.status is RuntimeStatus.STOPPED + finally: + release.set() + later_work.set() + await task + await stopping From 2b2a5a0402aa83bd0b24136b1c7fb0c66c722693 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 12 Nov 2026 12:45:15 -0700 Subject: [PATCH 2/7] fix: stop Windows runtime jobs and confirm cleanup Signed-off-by: Sara Tadayon --- crates/fabric-core/src/runtime.rs | 234 +++++++++++++++++++++++++++++- docs/sdk/python.mdx | 2 +- 2 files changed, 231 insertions(+), 5 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d7d7cd458..73d74d930 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -131,6 +131,9 @@ static LOCAL_HOSTS: LazyLock> // whole invocation, so a resource snapshot reads the PID from here instead. static LOCAL_HOST_PROCESSES: LazyLock>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); +#[cfg(windows)] +static WINDOWS_HOST_CONTROLS: LazyLock>>> = + LazyLock::new(|| Mutex::new(BTreeMap::new())); static LOCAL_SERVICES: LazyLock>>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); @@ -1960,7 +1963,9 @@ impl RuntimeAdapter for LocalHostAdapter { fn stop(&self, runtime: &RuntimeHandle) -> Result> { let Some(host) = local_hosts().get(&runtime.runtime_id).cloned() else { + #[cfg(not(windows))] let process = local_host_processes().get(&runtime.runtime_id).cloned(); + #[cfg(not(windows))] if process .as_ref() .is_some_and(|process| !owned_process_has_exited(process)) @@ -1975,6 +1980,8 @@ impl RuntimeAdapter for LocalHostAdapter { .unwrap_or_default(); confirm_stopped_descendants(runtime, &pending)?; local_host_processes().remove(&runtime.runtime_id); + #[cfg(windows)] + windows_host_controls().remove(&runtime.runtime_id); STOPPING_PROCESSES .lock() .unwrap_or_else(|error| error.into_inner()) @@ -2040,6 +2047,8 @@ impl RuntimeAdapter for LocalHostAdapter { remove_local_host_files(&guard).map_err(|_| cleanup_unconfirmed(runtime))?; local_hosts().remove(&runtime.runtime_id); local_host_processes().remove(&runtime.runtime_id); + #[cfg(windows)] + windows_host_controls().remove(&runtime.runtime_id); STOPPING_PROCESSES .lock() .unwrap_or_else(|error| error.into_inner()) @@ -2142,7 +2151,18 @@ fn interrupt_local_host(runtime: &RuntimeHandle) -> Result signal_owned_process(&process).map_err(|_| cleanup_unconfirmed(runtime))?; Ok(descendants) } - #[cfg(not(unix))] + #[cfg(windows)] + { + let control = windows_host_controls() + .get(&runtime.runtime_id) + .cloned() + .ok_or_else(|| cleanup_unconfirmed(runtime))?; + control + .interrupt() + .map_err(|_| cleanup_unconfirmed(runtime))?; + Ok(Vec::new()) + } + #[cfg(not(any(unix, windows)))] Err(cleanup_unconfirmed(runtime)) } @@ -2150,15 +2170,31 @@ fn confirm_stopped_descendants( runtime: &RuntimeHandle, processes: &[LocalHostProcess], ) -> Result<()> { + #[cfg(windows)] + let control = windows_host_controls().get(&runtime.runtime_id).cloned(); let deadline = Instant::now() + LOCAL_HOST_EXIT_GRACE; loop { - if processes.iter().all(owned_process_has_exited) { + let exited = processes.iter().all(owned_process_has_exited); + #[cfg(windows)] + let exited = exited + && control + .as_ref() + .is_none_or(|control| control.cleanup_confirmed() == Some(true)); + if exited { return Ok(()); } #[cfg(unix)] for process in processes { signal_owned_process(process).map_err(|_| cleanup_unconfirmed(runtime))?; } + #[cfg(windows)] + if let Some(control) = &control + && control.job.is_some() + { + control + .interrupt() + .map_err(|_| cleanup_unconfirmed(runtime))?; + } if Instant::now() >= deadline { return Err(cleanup_unconfirmed(runtime)); } @@ -2301,6 +2337,21 @@ fn start_local_runtime( let _ = remove_local_host_files(&host); return Err(error); } + #[cfg(windows)] + match WindowsHostControl::capture(&host) { + Ok(control) => { + windows_host_controls().insert(runtime.runtime_id.clone(), Arc::new(control)); + } + Err(source) => { + crate::check::invalidate_runtime(&runtime); + let _ = terminate_local_host(&mut host); + let _ = remove_local_host_files(&host); + return Err(FabricError::ProcessRunner { + command: host.command.clone(), + source, + }); + } + } local_host_processes().insert( runtime.runtime_id.clone(), LocalHostProcess::new(host.child.id(), started_at), @@ -3164,8 +3215,103 @@ fn local_host_processes() -> std::sync::MutexGuard<'static, BTreeMap Option { - let process = local_host_processes().get(runtime_id).cloned()?; - process.has_exited() + #[cfg(windows)] + { + let control = windows_host_controls().get(runtime_id).cloned()?; + control.has_exited() + } + #[cfg(not(windows))] + { + let process = local_host_processes().get(runtime_id).cloned()?; + process.has_exited() + } +} + +// Keep owned handles outside the invocation lock and after timeout eviction. +// Handles continue to identify the same kernel objects even if PIDs are reused. +#[cfg(windows)] +struct WindowsHostControl { + process: std::os::windows::io::OwnedHandle, + job: Option, +} + +#[cfg(windows)] +impl WindowsHostControl { + fn capture(host: &LocalAdapterHost) -> std::io::Result { + use std::os::windows::io::AsHandle; + Ok(Self { + process: host.child.as_handle().try_clone_to_owned()?, + job: host.job.as_ref().map(|job| job.try_clone()).transpose()?, + }) + } + + fn has_exited(&self) -> Option { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Foundation::{WAIT_OBJECT_0, WAIT_TIMEOUT}; + // SAFETY: this owned process handle stays valid throughout the probe. + match unsafe { + windows_sys::Win32::System::Threading::WaitForSingleObject( + self.process.as_raw_handle(), + 0, + ) + } { + WAIT_OBJECT_0 => Some(true), + WAIT_TIMEOUT => Some(false), + _ => None, + } + } + + fn cleanup_confirmed(&self) -> Option { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::System::JobObjects::{ + JOBOBJECT_BASIC_ACCOUNTING_INFORMATION, JobObjectBasicAccountingInformation, + QueryInformationJobObject, + }; + match self.has_exited() { + Some(true) => {} + other => return other, + } + let Some(job) = &self.job else { + return None; + }; + // SAFETY: this is a plain output structure with no invalid bit patterns. + let mut accounting: JOBOBJECT_BASIC_ACCOUNTING_INFORMATION = unsafe { std::mem::zeroed() }; + // SAFETY: the job handle is owned; the writable buffer matches the class and size. + let queried = unsafe { + QueryInformationJobObject( + job.as_raw_handle(), + JobObjectBasicAccountingInformation, + (&mut accounting as *mut JOBOBJECT_BASIC_ACCOUNTING_INFORMATION).cast(), + std::mem::size_of_val(&accounting) as u32, + std::ptr::null_mut(), + ) + }; + (queried != 0).then_some(accounting.ActiveProcesses == 0) + } + + fn interrupt(&self) -> std::io::Result<()> { + use std::os::windows::io::AsRawHandle; + let job = self + .job + .as_ref() + .ok_or_else(|| std::io::Error::other("owned Windows process tree is unavailable"))?; + // SAFETY: this job handle belongs to the runtime, including its descendants. + if unsafe { + windows_sys::Win32::System::JobObjects::TerminateJobObject(job.as_raw_handle(), 1) + } == 0 + { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + } +} + +#[cfg(windows)] +fn windows_host_controls() +-> std::sync::MutexGuard<'static, BTreeMap>> { + WINDOWS_HOST_CONTROLS + .lock() + .unwrap_or_else(|error| error.into_inner()) } fn local_services() @@ -5140,6 +5286,11 @@ for line in sys.stdin: root = os.path.dirname(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"]) with open(os.path.join(root, "child.pid"), "w", encoding="utf-8") as stream: stream.write(str(child.pid)) + if MODE == "child_after_start": + subprocess.Popen( + [sys.executable, "-c", "import time; time.sleep(30)"], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) service = message["payload"].get("service") if MODE == "connect_failure_once" and service and service["operation"] == "connect": failure_marker = os.environ["FABRIC_FAKE_HOST_STOP_MARKER"] + ".connect-failed" @@ -5553,6 +5704,78 @@ else: TEST_STOPPED_AGENTS.lock().expect("stop tracker").clone() } + #[cfg(windows)] + #[test] + fn windows_busy_stop_retains_ownership_without_a_job() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let control = windows_host_controls() + .get(&runtime.runtime_id) + .unwrap() + .clone(); + let process_only = Arc::new(WindowsHostControl { + process: control.process.try_clone().unwrap(), + job: None, + }); + windows_host_controls().insert(runtime.runtime_id.clone(), process_only.clone()); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let guard = host.lock().unwrap(); + let stopped = stop_runtime(&plan, &runtime); + drop(guard); + let retained = windows_host_controls().contains_key(&runtime.runtime_id); + let still_running = process_only.has_exited(); + windows_host_controls().insert(runtime.runtime_id.clone(), control); + stop_runtime(&plan, &runtime).unwrap(); + assert!( + matches!(stopped, Err(FabricError::AdapterLifecycleOperation { code, .. }) + if code == "cleanup_unconfirmed") + ); + assert!(retained); + assert_eq!(still_running, Some(false)); + assert_eq!(process_only.cleanup_confirmed(), None); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(windows)] + #[test] + fn windows_stop_confirms_the_job_after_the_host_exits() { + let (root, plan) = local_host_plan("child_after_start"); + let runtime = start_runtime(&plan).unwrap(); + let control = windows_host_controls() + .get(&runtime.runtime_id) + .unwrap() + .clone(); + assert_eq!(control.has_exited(), Some(false)); + { + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let mut host = host.lock().unwrap(); + host.child.kill().unwrap(); + host.child.wait().unwrap(); + } + assert_eq!(control.has_exited(), Some(true)); + assert_eq!(control.cleanup_confirmed(), Some(false)); + stop_runtime(&plan, &runtime).expect("stop children after the host already exited"); + assert_eq!(control.cleanup_confirmed(), Some(true)); + assert!(!windows_host_controls().contains_key(&runtime.runtime_id)); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn busy_stop_interrupts_the_owned_host_without_the_invocation_lock() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let guard = host.lock().unwrap(); + let interrupted = interrupt_local_host(&runtime); + drop(guard); + let stopped = stop_runtime(&plan, &runtime); + interrupted.expect("interrupt without taking the busy host lock"); + stopped.expect("confirm cleanup after interruption"); + assert!(!local_hosts().contains_key(&runtime.runtime_id)); + assert!(local_host_has_exited(&runtime.runtime_id).is_none()); + fs::remove_dir_all(root).unwrap(); + } + #[cfg(unix)] #[test] fn stop_retries_can_confirm_a_child_that_exited_before_identity_capture() { @@ -7349,6 +7572,9 @@ else: ); assert!(!runtime_dir.exists()); + #[cfg(any(target_os = "linux", target_os = "macos", windows))] + assert_eq!(local_host_has_exited(&runtime.runtime_id), Some(true)); + #[cfg(any(target_os = "linux", target_os = "macos"))] for level in ["live", "active", "ready"] { let request = serde_json::from_value(serde_json::json!({ diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index d805df988..b017758be 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -1696,4 +1696,4 @@ Calling applications and environment owners supply typed `ProvenanceConfig.repor The [software evidence example](https://github.com/NVIDIA/NeMo-Fabric/tree/main/examples/provenance) provides six execution-path checks, separate-venv runs, saved RunResults and private comparison fixtures. Its local provider/service fixtures are labeled separately from live deployments. Safe configuration capture is the P0 follow-up and is outside this software-version contract. -Buffered runtime shutdown can interrupt an active invocation on supported Unix hosts. `Runtime.stop()` waits for the interrupted invocation to settle before reporting stopped. Cleanup errors retain ownership for retry; the runtime server blocks replacement for the same agent until cleanup succeeds. Streaming shutdown continues to use the stream lifecycle. +Buffered runtime shutdown can interrupt an active invocation on supported Unix and Windows hosts. Windows shutdown requires an owned process job to confirm descendant cleanup. `Runtime.stop()` waits for the interrupted invocation to settle before reporting stopped. Cleanup errors retain ownership for retry; the runtime server blocks replacement for the same agent until cleanup succeeds. Streaming shutdown continues to use the stream lifecycle. From bd1b33a2e48e0d05a8cd4bf5bf895693543f563f Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 12 Nov 2026 14:46:16 -0700 Subject: [PATCH 3/7] fix: retain cleanup ownership through busy runtime shutdown Signed-off-by: Sara Tadayon --- crates/fabric-core/src/error.rs | 58 ++++ crates/fabric-core/src/lib.rs | 2 +- crates/fabric-core/src/resource_snapshot.rs | 38 +++ crates/fabric-core/src/runtime.rs | 286 ++++++++++++++++-- .../error/struct-runtimecleanuppending.mdx | 60 ++++ .../nemo-fabric-core/index.mdx | 1 + .../nemo-fabric-core/runtime/fn-run-plan.mdx | 2 + docs/sdk/python.mdx | 4 +- .../src/nemo_fabric/runtime.py | 17 +- .../src/nemo_fabric/runtime_server/state.py | 2 +- tests/python/runtime_server/test_state.py | 25 ++ tests/python/test_runtime.py | 33 ++ 12 files changed, 490 insertions(+), 38 deletions(-) create mode 100644 docs/reference/api/rust-library-reference/nemo-fabric-core/error/struct-runtimecleanuppending.mdx diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index f816691d4..e4c984b8e 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -235,6 +235,9 @@ pub enum FabricError { /// Bounded adapter-host diagnostics. diagnostics: String, }, + /// A one-shot run could not confirm cleanup; the original runtime remains owned. + #[error("{0}")] + RuntimeCleanupPending(Box), /// A runtime handle was used with a different run plan than the one that created it. #[error( "runtime handle does not match run plan for `{field}`: expected `{expected}` but found `{actual}` (runtime `{runtime_id}`)" @@ -436,3 +439,58 @@ pub enum FabricError { source: serde_json::Error, }, } + +/// Cleanup context retained when [`crate::run_plan`] cannot finish stopping a runtime. +/// +/// Retry [`crate::stop_runtime`] with the original plan and [`Self::runtime`]. +/// The handle and completed output are deliberately excluded from diagnostic formatting. +pub struct RuntimeCleanupPending { + runtime: crate::RuntimeHandle, + result: Option, + cause: FabricError, +} + +impl RuntimeCleanupPending { + pub(crate) fn new( + runtime: crate::RuntimeHandle, + result: Option, + cause: FabricError, + ) -> Self { + Self { + runtime, + result, + cause, + } + } + + /// The original handle required for a cleanup retry in this process. + pub fn runtime(&self) -> &crate::RuntimeHandle { + &self.runtime + } + + /// The completed invocation result, if invocation succeeded before cleanup failed. + pub fn result(&self) -> Option<&crate::RunResult> { + self.result.as_ref() + } + + /// The invocation or stop error that preceded the retained cleanup context. + pub fn cause(&self) -> &FabricError { + &self.cause + } +} + +impl std::fmt::Debug for RuntimeCleanupPending { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RuntimeCleanupPending") + .field("result_available", &self.result.is_some()) + .finish_non_exhaustive() + } +} + +impl std::fmt::Display for RuntimeCleanupPending { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str( + "runtime cleanup is unconfirmed; retry stop with RuntimeCleanupPending::runtime()", + ) + } +} diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index 6e5b02350..88433c8d3 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -52,7 +52,7 @@ pub use config::{ resolve_run_plan_from_config_with_adapter_directories, }; pub use doctor::{DoctorCheck, DoctorReport, DoctorStatus, doctor_plan}; -pub use error::{FabricError, Result}; +pub use error::{FabricError, Result, RuntimeCleanupPending}; pub use provenance::{ExecutionEvidence, ProvenanceConfig}; pub use resource_snapshot::RuntimeResourceSnapshot; pub use runtime::{ diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 219154d6d..9d9b71ce7 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -89,6 +89,12 @@ impl LocalHostProcess { platform::has_exited(self.pid, self.identity) } + /// Whether the same process is suspended and cannot create children. + #[cfg(unix)] + pub(crate) fn is_stopped(&self) -> Option { + platform::is_stopped(self.pid, self.identity?) + } + /// Record the host `pid`, started by NeMo Fabric at `started_at`. pub(crate) fn new(pid: u32, started_at: SystemTime) -> Self { Self { @@ -328,6 +334,7 @@ mod platform { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) struct Stat { pub zombie: bool, + pub stopped: bool, pub ppid: u32, /// `utime + stime + cutime + cstime`, in clock ticks. pub cpu_ticks: u64, @@ -348,6 +355,7 @@ mod platform { .sum::>()?; Some(Stat { zombie: fields.first() == Some(&"Z"), + stopped: matches!(fields.first(), Some(&"T" | &"t")), ppid: u32::try_from(number(1)?).ok()?, cpu_ticks: ticks, start: number(19)?, @@ -540,6 +548,11 @@ mod platform { } } + pub(super) fn is_stopped(pid: u32, identity: u64) -> Option { + let stat = stat(pid).ok()?; + Some(stat.start == identity && stat.stopped) + } + fn sysconf(name: libc::c_int) -> u64 { // SAFETY: sysconf has no memory-safety preconditions. u64::try_from(unsafe { libc::sysconf(name) }).unwrap_or(0) @@ -731,6 +744,26 @@ mod platform { } } + pub(super) fn is_stopped(pid: u32, identity: u64) -> Option { + // SAFETY: proc_bsdinfo is an output structure with a valid zero representation. + let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; + let size = std::mem::size_of_val(&info) as libc::c_int; + // SAFETY: the writable buffer matches PROC_PIDTBSDINFO and its size. + let read = unsafe { + libc::proc_pidinfo( + pid as libc::c_int, + libc::PROC_PIDTBSDINFO, + 0, + (&mut info as *mut libc::proc_bsdinfo).cast(), + size, + ) + }; + if read != size || start_identity(pid) != Some(identity) { + return None; + } + Some(info.pbi_status == libc::SSTOP) + } + pub(super) fn read_tree(root: u32, identity: u64) -> Result { let timebase = timebase().ok_or(PROCESS_UNREADABLE)?; read_tree_with(&Live { timebase }, root, identity) @@ -751,6 +784,11 @@ mod platform { None } + #[cfg(unix)] + pub(super) fn is_stopped(_pid: u32, _identity: u64) -> Option { + None + } + pub(super) fn read_tree(_root: u32, _identity: u64) -> Result { Err("no process reader for this operating system") } diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 73d74d930..72c881db0 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -952,7 +952,7 @@ struct LocalAdapterHost { child: Child, pending_stop: Vec, #[cfg(windows)] - job: Option, + control: Arc, stdin: ChildStdin, responses: Receiver>, command: String, @@ -1001,16 +1001,30 @@ enum LocalServiceState { } /// Invoke a NeMo Fabric run plan. +/// +/// If stopping leaves owned cleanup pending, returns +/// [`FabricError::RuntimeCleanupPending`] with the original runtime handle and +/// any completed result. Retry [`stop_runtime`] with that handle and this plan. pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { let runtime = start_runtime(plan)?; let mut result = match invoke_runtime(plan, &runtime, request) { Ok(result) => result, Err(error) => { let _ = stop_runtime(plan, &runtime); + if local_runtime_is_retained(&runtime.runtime_id) { + return Err(FabricError::RuntimeCleanupPending(Box::new( + crate::RuntimeCleanupPending::new(runtime, None, error), + ))); + } return Err(error); } }; match stop_runtime(plan, &runtime) { + Err(error) if local_runtime_is_retained(&runtime.runtime_id) => { + return Err(FabricError::RuntimeCleanupPending(Box::new( + crate::RuntimeCleanupPending::new(runtime, Some(result), error), + ))); + } Ok(events) => result.events.extend(events), Err(error) if result.status == RunStatus::Succeeded => { result.status = RunStatus::Failed; @@ -1527,7 +1541,7 @@ pub fn stop_runtime(plan: &RunPlan, runtime: &RuntimeHandle) -> Result bool { + let retained = + local_hosts().contains_key(runtime_id) || local_host_processes().contains_key(runtime_id); + #[cfg(windows)] + let retained = retained || windows_host_controls().contains_key(runtime_id); + retained +} + fn cleanup_unconfirmed(runtime: &RuntimeHandle) -> FabricError { lifecycle_error( AdapterLifecycleOperation::Stop, @@ -2120,6 +2142,29 @@ fn signal_owned_process(process: &LocalHostProcess) -> std::io::Result<()> { Ok(()) } +#[cfg(unix)] +fn freeze_owned_process(process: &LocalHostProcess, deadline: Instant) -> std::io::Result<()> { + if owned_process_has_exited(process) { + return Ok(()); + } + if !process.is_current() { + return Err(std::io::Error::other("process ownership is unconfirmed")); + } + // SAFETY: the process identity was checked immediately before signalling. + if unsafe { libc::kill(process.pid() as libc::pid_t, libc::SIGSTOP) } != 0 { + return Err(std::io::Error::last_os_error()); + } + loop { + if process.is_stopped() == Some(true) || owned_process_has_exited(process) { + return Ok(()); + } + if !process.is_current() || Instant::now() >= deadline { + return Err(std::io::Error::other("process suspension is unconfirmed")); + } + thread::sleep(Duration::from_millis(10)); + } +} + fn interrupt_local_host(runtime: &RuntimeHandle) -> Result> { #[cfg(unix)] { @@ -2133,16 +2178,33 @@ fn interrupt_local_host(runtime: &RuntimeHandle) -> Result if !process.is_current() { return Err(cleanup_unconfirmed(runtime)); } - let descendants: Vec<_> = host_descendants(process.pid()) - .into_iter() - .map(|pid| LocalHostProcess::new(pid as u32, SystemTime::now())) - .collect(); + let deadline = Instant::now() + LOCAL_HOST_STOP_TIMEOUT; + // Freeze each parent before listing its children. A frozen parent cannot + // create another child after that list was read. Retain every discovered + // identity before suspension so a partial failure remains retryable. + let mut parents = vec![process.clone()]; + while let Some(parent) = parents.pop() { + freeze_owned_process(&parent, deadline).map_err(|_| cleanup_unconfirmed(runtime))?; + if owned_process_has_exited(&parent) { + continue; + } + let children: Vec<_> = child_pids(parent.pid() as libc::pid_t) + .into_iter() + .map(|pid| LocalHostProcess::new(pid as u32, SystemTime::now())) + .collect(); + STOPPING_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .entry(runtime.runtime_id.clone()) + .or_default() + .extend(children.clone()); + parents.extend(children); + } let descendants = { let mut retained = STOPPING_PROCESSES .lock() .unwrap_or_else(|error| error.into_inner()); let previous = retained.entry(runtime.runtime_id.clone()).or_default(); - previous.extend(descendants); previous.clone() }; for child in descendants.iter().rev() { @@ -2338,20 +2400,7 @@ fn start_local_runtime( return Err(error); } #[cfg(windows)] - match WindowsHostControl::capture(&host) { - Ok(control) => { - windows_host_controls().insert(runtime.runtime_id.clone(), Arc::new(control)); - } - Err(source) => { - crate::check::invalidate_runtime(&runtime); - let _ = terminate_local_host(&mut host); - let _ = remove_local_host_files(&host); - return Err(FabricError::ProcessRunner { - command: host.command.clone(), - source, - }); - } - } + windows_host_controls().insert(runtime.runtime_id.clone(), host.control.clone()); local_host_processes().insert( runtime.runtime_id.clone(), LocalHostProcess::new(host.child.id(), started_at), @@ -3237,11 +3286,16 @@ struct WindowsHostControl { #[cfg(windows)] impl WindowsHostControl { - fn capture(host: &LocalAdapterHost) -> std::io::Result { + fn capture( + child: &Child, + job: Option, + ) -> std::io::Result { use std::os::windows::io::AsHandle; + let job = + job.ok_or_else(|| std::io::Error::other("owned Windows process job is unavailable"))?; Ok(Self { - process: host.child.as_handle().try_clone_to_owned()?, - job: host.job.as_ref().map(|job| job.try_clone()).transpose()?, + process: child.as_handle().try_clone_to_owned()?, + job: Some(job), }) } @@ -3415,7 +3469,18 @@ fn spawn_local_host( } }; #[cfg(windows)] - let job = host_job(&child); + let control = match WindowsHostControl::capture(&child, host_job(&child)) { + Ok(control) => Arc::new(control), + Err(source) => { + let _ = child.kill(); + let _ = child.wait(); + let _ = std::fs::remove_dir_all(&runtime_dir); + return Err(FabricError::ProcessRunner { + command: command_display, + source, + }); + } + }; #[cfg(windows)] if let Err(source) = resume_host(child.id()) { let _ = child.kill(); @@ -3496,7 +3561,7 @@ fn spawn_local_host( child, pending_stop: Vec::new(), #[cfg(windows)] - job, + control, stdin, responses, command: command_display, @@ -3807,7 +3872,7 @@ fn terminate_local_host(host: &mut LocalAdapterHost) -> std::io::Result<()> { } } #[cfg(windows)] - if let Some(job) = &host.job { + if let Some(job) = &host.control.job { // SAFETY: the host owns this job handle until it is dropped. if unsafe { windows_sys::Win32::System::JobObjects::TerminateJobObject( @@ -3938,12 +4003,15 @@ fn child_pids(_parent: libc::pid_t) -> Vec { /// Puts the suspended host in a job object so a forced kill reaches every /// process it starts, even through launchers or other intermediates that -/// already exited. Like Cargo, fall back to killing only the host if the job -/// is unavailable. +/// already exited. Closing the last job handle also terminates its processes. #[cfg(windows)] fn host_job(child: &Child) -> Option { use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle}; - use windows_sys::Win32::System::JobObjects::{AssignProcessToJobObject, CreateJobObjectW}; + use windows_sys::Win32::System::JobObjects::{ + AssignProcessToJobObject, CreateJobObjectW, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, + JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JobObjectExtendedLimitInformation, + SetInformationJobObject, + }; // SAFETY: null security attributes and name are allowed; null means failure. let job = unsafe { CreateJobObjectW(std::ptr::null(), std::ptr::null()) }; @@ -3952,6 +4020,21 @@ fn host_job(child: &Child) -> Option { } // SAFETY: CreateJobObjectW returned a new handle that nothing else owns. let job = unsafe { OwnedHandle::from_raw_handle(job) }; + // SAFETY: this job information structure accepts an all-zero initialization. + let mut limits: JOBOBJECT_EXTENDED_LIMIT_INFORMATION = unsafe { std::mem::zeroed() }; + limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + // SAFETY: the owned job and readable buffer match the information class and size. + if unsafe { + SetInformationJobObject( + job.as_raw_handle(), + JobObjectExtendedLimitInformation, + (&limits as *const JOBOBJECT_EXTENDED_LIMIT_INFORMATION).cast(), + std::mem::size_of_val(&limits) as u32, + ) + } == 0 + { + return None; + } // SAFETY: both handles stay valid for the duration of the call. let assigned = unsafe { AssignProcessToJobObject(job.as_raw_handle(), child.as_raw_handle()) }; (assigned != 0).then_some(job) @@ -5329,7 +5412,7 @@ for line in sys.stdin: # Exit during Fabric's grace wait and leave the child running. time.sleep(1) os._exit(0) - if MODE == "invoke_failure": + if MODE in {"invoke_failure", "invoke_failure_cleanup_failure"}: response("invoke", error=failure("invoke", "fake_invoke", "invoke rejected")) continue invocation = message["payload"] @@ -5459,6 +5542,12 @@ for line in sys.stdin: child.stdin.write(json.dumps({"operation": "stop", "outcome": {"status": "succeeded", "output": None}}) + "\n") child.stdin.flush() os._exit(0) + if MODE in {"cleanup_failure", "invoke_failure_cleanup_failure"}: + import shutil + directory = os.environ["FABRIC_HOME"] + shutil.rmtree(directory) + with open(directory, "w", encoding="utf-8") as stream: + stream.write("cleanup requires a retry") if MODE == "stop_delay": with open(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"], "w", encoding="utf-8") as stream: stream.write("started") @@ -5704,6 +5793,141 @@ else: TEST_STOPPED_AGENTS.lock().expect("stop tracker").clone() } + #[test] + fn unconfirmed_timeout_cleanup_retains_service_membership() { + let (root, plan) = local_host_plan("success"); + let service = prepare_service(&plan).unwrap(); + let runtime = start_runtime_with_service(&plan, &service).unwrap(); + let host = local_hosts().remove(&runtime.runtime_id).unwrap(); + let stopped = stop_runtime(&plan, &runtime); + let retained = local_services() + .get(&service.service_id) + .unwrap() + .lock() + .unwrap() + .active_runtime_ids + .contains(&runtime.runtime_id); + local_hosts().insert(runtime.runtime_id.clone(), host); + stop_runtime(&plan, &runtime).unwrap(); + release_service(&plan, &service).unwrap(); + assert!( + matches!(stopped, Err(FabricError::AdapterLifecycleOperation { code, .. }) + if code == "cleanup_unconfirmed") + ); + assert!(retained); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn one_shot_cleanup_failures_keep_a_safe_retry_handle() { + for mode in ["cleanup_failure", "invoke_failure_cleanup_failure"] { + let (root, plan) = local_host_plan(mode); + let error = run_plan(&plan, RunRequest::text("TEST_ONLY_PRIVATE_RESULT")) + .expect_err("unconfirmed cleanup must retain a retry handle"); + let debug = format!("{error:?}"); + let FabricError::RuntimeCleanupPending(pending) = error else { + panic!("expected retained one-shot cleanup context"); + }; + assert_eq!(pending.result().is_some(), mode == "cleanup_failure"); + let runtime = pending.runtime(); + let directory = local_host_directory(&runtime.runtime_id); + fs::remove_file(&directory).unwrap(); + fs::create_dir_all(&directory).unwrap(); + stop_runtime(&plan, runtime).expect("retry cleanup with the original handle"); + assert!(!local_hosts().contains_key(&runtime.runtime_id)); + assert!(!debug.contains("TEST_ONLY_PRIVATE_RESULT")); + assert!(!debug.contains(&runtime.runtime_binding)); + fs::remove_dir_all(root).unwrap(); + } + } + + #[cfg(unix)] + #[test] + fn freezing_an_owned_process_prevents_new_child_creation() { + let (root, _) = local_host_plan("success"); + let ready = root.join("ready-to-fork"); + let release = root.join("release-fork"); + let marker = root.join("forked"); + let script = format!( + "import pathlib, subprocess, sys, time\n\ + ready=pathlib.Path({})\n\ + release=pathlib.Path({})\n\ + marker=pathlib.Path({})\n\ + ready.touch()\n\ + while not release.exists(): time.sleep(0.01)\n\ + subprocess.Popen([sys.executable,'-c','import time; time.sleep(1)'])\n\ + marker.touch()\n\ + time.sleep(1)\n", + serde_json::to_string(&ready).unwrap(), + serde_json::to_string(&release).unwrap(), + serde_json::to_string(&marker).unwrap(), + ); + let mut child = Command::new(DEFAULT_PYTHON) + .args(["-c", &script]) + .spawn() + .unwrap(); + // Match a started adapter: wait until the interpreter has finished + // launching before exercising suspension of active application work. + let deadline = Instant::now() + LOCAL_HOST_STOP_TIMEOUT; + while !ready.exists() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + let started = ready.exists(); + let process = LocalHostProcess::new(child.id(), SystemTime::now()); + let frozen = freeze_owned_process(&process, Instant::now() + Duration::from_secs(3)); + let stopped = process.is_stopped(); + fs::write(&release, "").unwrap(); + thread::sleep(Duration::from_millis(50)); + let forked = marker.exists(); + child.kill().unwrap(); + child.wait().unwrap(); + assert!(started, "child must initialize before freezing"); + frozen.unwrap(); + assert_eq!(stopped, Some(true)); + assert!(!forked); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(windows)] + #[test] + fn windows_capture_rejects_a_missing_job() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let capture = WindowsHostControl::capture(&host.lock().unwrap().child, None); + stop_runtime(&plan, &runtime).unwrap(); + assert!(capture.is_err()); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(windows)] + #[test] + fn windows_job_closure_terminates_its_processes() { + use std::os::windows::process::CommandExt; + let mut child = Command::new(DEFAULT_PYTHON) + .args(["-c", "import time; time.sleep(30)"]) + .creation_flags(windows_sys::Win32::System::Threading::CREATE_SUSPENDED) + .spawn() + .unwrap(); + let job = host_job(&child).unwrap(); + resume_host(child.id()).unwrap(); + drop(job); + let deadline = Instant::now() + Duration::from_secs(3); + while child.try_wait().unwrap().is_none() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + let terminated = child.try_wait().unwrap().is_some(); + if !terminated { + child.kill().unwrap(); + child.wait().unwrap(); + } + assert!( + terminated, + "closing the last job owner must terminate the owned tree" + ); + } + #[cfg(windows)] #[test] fn windows_busy_stop_retains_ownership_without_a_job() { diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/struct-runtimecleanuppending.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/struct-runtimecleanuppending.mdx new file mode 100644 index 000000000..64a2af2eb --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/struct-runtimecleanuppending.mdx @@ -0,0 +1,60 @@ +--- +title: "Struct Runtime Cleanup Pending" +sidebar-title: "RuntimeCleanupPending" +description: "Cleanup context retained when `crate::run_plan` cannot finish stopping a runtime." +position: 99 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub struct RuntimeCleanupPending { /* private fields */ } +``` + +Cleanup context retained when [`crate::run_plan`](../runtime/fn-run-plan.mdx) cannot finish stopping a runtime. + +Retry [`crate::stop_runtime`](../runtime/fn-stop-runtime.mdx) with the original plan and [`Self::runtime`](struct-runtimecleanuppending.mdx#method.runtime). The handle and completed output are deliberately excluded from diagnostic formatting. + +## Implementations + +### `impl RuntimeCleanupPending` + +
RuntimeCleanupPending"}} />
+ +#### `runtime` + +
RuntimeHandle"}} />
+ +The original handle required for a cleanup retry in this process. + +#### `result` + +
Option<&RunResult>"}} />
+ +The completed invocation result, if invocation succeeded before cleanup failed. + +#### `cause` + +
FabricError"}} />
+ +The invocation or stop error that preceded the retained cleanup context. + +## Trait Implementations + +### `impl Debug for RuntimeCleanupPending` + +
Debug for RuntimeCleanupPending"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Display for RuntimeCleanupPending` + +
Display for RuntimeCleanupPending"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 07728f2b2..437c2120f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -111,6 +111,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use doctor::doctor_plan;` - `pub use error::FabricError;` - `pub use error::Result;` +- `pub use error::RuntimeCleanupPending;` - `pub use provenance::ExecutionEvidence;` - `pub use provenance::ProvenanceConfig;` - `pub use resource_snapshot::RuntimeResourceSnapshot;` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index 331c08b3c..8353c2e48 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -12,3 +12,5 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`.
RunPlan, request: RunRequest) -> Result<RunResult>"}} />
Invoke a NeMo Fabric run plan. + +If stopping leaves owned cleanup pending, returns [`FabricError::RuntimeCleanupPending`](../error/enum-fabricerror.mdx#variant.RuntimeCleanupPending) with the original runtime handle and any completed result. Retry [`stop_runtime`](fn-stop-runtime.mdx) with that handle and this plan. diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index b017758be..232261afb 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -890,6 +890,8 @@ Hermes Agent reuses one agent and session database, and Claude keeps one connect client. Durable Hermes conversations expose their native identifier inside an opaque `SessionRef`. NeMo Fabric does not interpret that identifier. +Buffered runtime shutdown can interrupt an active invocation on supported Unix and Windows hosts. Windows shutdown requires an owned process job to confirm descendant cleanup. `Runtime.stop()` waits for the interrupted invocation to settle before reporting stopped. Cleanup errors retain ownership for retry; the runtime server blocks replacement for the same agent until cleanup succeeds. Streaming shutdown continues to use the stream lifecycle. + ### Keep Agent State Across Runtimes By default, each runtime keeps its native state in its own folder, so a new @@ -1695,5 +1697,3 @@ OpenClaw service startup reports retain their original source and capture time w Calling applications and environment owners supply typed `ProvenanceConfig.reports` snapshots through the configuration already sent to NeMo Fabric. Capture them at application initialization and preserve their original `source`, `origin`, availability and `observed_at`. A guest never inspects its caller. Harbor implements this seam with `fabric_provenance`, whose shape matches the SDK control; `--ak 'fabric_provenance={"enabled":false}'` disables collection before host initialization and propagates to the guest. The [software evidence example](https://github.com/NVIDIA/NeMo-Fabric/tree/main/examples/provenance) provides six execution-path checks, separate-venv runs, saved RunResults and private comparison fixtures. Its local provider/service fixtures are labeled separately from live deployments. Safe configuration capture is the P0 follow-up and is outside this software-version contract. - -Buffered runtime shutdown can interrupt an active invocation on supported Unix and Windows hosts. Windows shutdown requires an owned process job to confirm descendant cleanup. `Runtime.stop()` waits for the interrupted invocation to settle before reporting stopped. Cleanup errors retain ownership for retry; the runtime server blocks replacement for the same agent until cleanup succeeds. Streaming shutdown continues to use the stream lifecycle. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 4e3df3362..ea187b862 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +from concurrent.futures import ThreadPoolExecutor import json import logging from collections.abc import Mapping, Sequence @@ -838,7 +839,10 @@ def stop() -> Any: stopped = True return result - await _call_blocking(stop) + with ThreadPoolExecutor( + max_workers=1, thread_name_prefix="fabric-stop" + ) as executor: + await _call_blocking(stop, executor=executor) if active_task is not None and active_task is not asyncio.current_task(): # Observe the interrupted invocation before publishing STOPPED; # its error path must not overwrite the final lifecycle state. @@ -1107,8 +1111,15 @@ def _resource_snapshot_error(native: Any, error: Exception) -> FabricRuntimeErro return _runtime_error(error, stage="resource_snapshot") -async def _call_blocking(func: Any) -> Any: - worker = asyncio.create_task(asyncio.to_thread(func)) +async def _call_blocking( + func: Any, *, executor: ThreadPoolExecutor | None = None +) -> Any: + if executor is None: + worker = asyncio.create_task(asyncio.to_thread(func)) + else: + worker = asyncio.ensure_future( + asyncio.get_running_loop().run_in_executor(executor, func) + ) try: return await asyncio.shield(worker) except asyncio.CancelledError as cancelled: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py index 3a0444cdd..865396b82 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py @@ -355,7 +355,7 @@ async def shutdown(self) -> None: cleanup_failures: list[Exception] = [] async with self._start_lock: for owned in tuple(self.runtimes.values()): - async with owned.lock: + async with owned.stop_lock, owned.lock: if owned.state != "stopped": try: await self._stop(owned) diff --git a/tests/python/runtime_server/test_state.py b/tests/python/runtime_server/test_state.py index 2424ecc08..b64f84a43 100644 --- a/tests/python/runtime_server/test_state.py +++ b/tests/python/runtime_server/test_state.py @@ -527,6 +527,31 @@ async def stop(): await task +async def test_shutdown_waits_for_a_pending_stop(host, host_request, mock_runtime): + await host.handle(host_request("start")) + entered, release = asyncio.Event(), asyncio.Event() + + async def stop(): + if entered.is_set(): + raise FabricStateError("shutdown is already in progress") + entered.set() + await release.wait() + + mock_runtime.stop.side_effect = stop + stopping = asyncio.create_task(host.handle(host_request("stop"))) + await entered.wait() + shutdown = asyncio.create_task(host.shutdown()) + try: + await asyncio.sleep(0.05) + release.set() + assert (await stopping)["status"] == "succeeded" + await shutdown + mock_runtime.stop.assert_awaited_once() + finally: + release.set() + await asyncio.gather(stopping, shutdown, return_exceptions=True) + + async def test_unconfirmed_stop_blocks_replacement_but_not_other_agents( host, host_request, mock_runtime, mock_fabric, runtime_factory, config ): diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index a6b9ec1b1..f787f7f6f 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -10,6 +10,7 @@ import logging import threading from contextlib import AsyncExitStack +from concurrent.futures import ThreadPoolExecutor from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -159,6 +160,38 @@ def _runtime_wrapper( ) +async def test_stop_does_not_wait_for_a_saturated_default_executor(mock_native): + runtime = _runtime_wrapper(mock_native) + loop = asyncio.get_running_loop() + await asyncio.to_thread(lambda: None) + previous_executor = loop._default_executor + entered, release = threading.Event(), threading.Event() + + def occupy_worker(): + entered.set() + release.wait(5) + + with ThreadPoolExecutor(max_workers=1) as executor: + loop.set_default_executor(executor) + occupied = asyncio.create_task(asyncio.to_thread(occupy_worker)) + try: + async with asyncio.timeout(3): + while not entered.is_set(): + await asyncio.sleep(0.01) + stopping = asyncio.create_task(runtime.stop()) + try: + completed, _ = await asyncio.wait({stopping}, timeout=0.5) + finally: + release.set() + await stopping + finally: + release.set() + await occupied + loop.set_default_executor(previous_executor) + assert stopping in completed + mock_native.stop_runtime.assert_called_once() + + @pytest.mark.parametrize("stage", ["start", "invoke", "stop"]) @pytest.mark.parametrize( ("error_type", "expected_code"), From 1089bfde03a580ac0a8d3373fe11a3a9a0bbf2fc Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 12 Nov 2026 17:10:25 -0700 Subject: [PATCH 4/7] fix: retain cleanup evidence and recovery handles Signed-off-by: Sara Tadayon --- .../nemo_fabric_adapter_contract/models.py | 2 + .../schemas/adapter-check-result.schema.json | 10 + .../src/generated/adapter-check-result.ts | 2 + crates/fabric-core/src/check.rs | 4 + crates/fabric-core/src/runtime.rs | 252 +++++++++++++----- .../nemo_fabric.client.md | 2 +- .../nemo_fabric.errors.md | 67 +++++ .../check/enum-checkreason.mdx | 14 + docs/sdk/python.mdx | 4 + environment-providers/openshell/src/lib.rs | 23 ++ .../adapter-check-result.schema.json | 10 + schemas/sdk/check-report.schema.json | 10 + .../src/nemo_fabric/check.py | 2 + .../src/nemo_fabric/client.py | 11 + .../src/nemo_fabric/errors.py | 14 +- .../src/nemo_fabric/runtime.py | 37 ++- .../src/nemo_fabric/runtime_server/state.py | 10 + .../runtime_server/test_runtime_checks.py | 40 +++ tests/python/runtime_server/test_state.py | 5 +- tests/python/test_runtime.py | 116 ++++++++ 20 files changed, 559 insertions(+), 76 deletions(-) diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py index 3537f8a09..85667a739 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py @@ -71,6 +71,8 @@ def _bounded_int(value: int | None, field_name: str, maximum: int) -> None: CheckReason = Literal[ "server_stopping", "capabilities_unverified", + "runtime_busy", + "cleanup_unconfirmed", "runtime_unavailable", "service_unavailable", "agent_exited", diff --git a/adapter-contract/typescript/schemas/adapter-check-result.schema.json b/adapter-contract/typescript/schemas/adapter-check-result.schema.json index eb405c683..1d3814a12 100644 --- a/adapter-contract/typescript/schemas/adapter-check-result.schema.json +++ b/adapter-contract/typescript/schemas/adapter-check-result.schema.json @@ -38,6 +38,16 @@ "description": "Advertised capability evidence is unavailable.", "type": "string" }, + { + "const": "runtime_busy", + "description": "Runtime cleanup is in progress.", + "type": "string" + }, + { + "const": "cleanup_unconfirmed", + "description": "Owned runtime cleanup could not be confirmed.", + "type": "string" + }, { "const": "runtime_unavailable", "description": "No matching runtime binding exists.", diff --git a/adapter-contract/typescript/src/generated/adapter-check-result.ts b/adapter-contract/typescript/src/generated/adapter-check-result.ts index dd28ad262..aa15f8cc2 100644 --- a/adapter-contract/typescript/src/generated/adapter-check-result.ts +++ b/adapter-contract/typescript/src/generated/adapter-check-result.ts @@ -83,6 +83,8 @@ export type AdapterCheckResult = { export type CheckReason = | "server_stopping" | "capabilities_unverified" + | "runtime_busy" + | "cleanup_unconfirmed" | "runtime_unavailable" | "service_unavailable" | "agent_exited" diff --git a/crates/fabric-core/src/check.rs b/crates/fabric-core/src/check.rs index de4fb1ae0..da1a572b3 100644 --- a/crates/fabric-core/src/check.rs +++ b/crates/fabric-core/src/check.rs @@ -94,6 +94,10 @@ pub enum CheckReason { ServerStopping, /// Advertised capability evidence is unavailable. CapabilitiesUnverified, + /// Runtime cleanup is in progress. + RuntimeBusy, + /// Owned runtime cleanup could not be confirmed. + CleanupUnconfirmed, /// No matching runtime binding exists. RuntimeUnavailable, /// No matching active service binding exists. diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 72c881db0..d22735a48 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1994,6 +1994,8 @@ impl RuntimeAdapter for LocalHostAdapter { .unwrap_or_default(); confirm_stopped_descendants(runtime, &pending)?; local_host_processes().remove(&runtime.runtime_id); + #[cfg(unix)] + frozen_process_trees().remove(&runtime.runtime_id); #[cfg(windows)] windows_host_controls().remove(&runtime.runtime_id); STOPPING_PROCESSES @@ -2022,6 +2024,11 @@ impl RuntimeAdapter for LocalHostAdapter { guard.cleanup_pending = true; PENDING_LOCAL_CLEANUP.lock().unwrap_or_else(|error| error.into_inner()) .insert(runtime.runtime_id.clone(), guard.cleanup_scope.clone()); + #[cfg(unix)] + if frozen_tree_is_pending(&runtime.runtime_id) { + interrupted = Some(interrupt_local_host(runtime)?); + } + let was_interrupted = interrupted.is_some(); if let Some(processes) = interrupted { guard.pending_stop.extend(processes); } @@ -2061,6 +2068,8 @@ impl RuntimeAdapter for LocalHostAdapter { remove_local_host_files(&guard).map_err(|_| cleanup_unconfirmed(runtime))?; local_hosts().remove(&runtime.runtime_id); local_host_processes().remove(&runtime.runtime_id); + #[cfg(unix)] + frozen_process_trees().remove(&runtime.runtime_id); #[cfg(windows)] windows_host_controls().remove(&runtime.runtime_id); STOPPING_PROCESSES @@ -2069,10 +2078,11 @@ impl RuntimeAdapter for LocalHostAdapter { .remove(&runtime.runtime_id); PENDING_LOCAL_CLEANUP.lock().unwrap_or_else(|error| error.into_inner()) .remove(&runtime.runtime_id); - let host_crashed = matches!( - &result, - Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "host_crashed" - ); + let host_crashed = was_interrupted + || matches!( + &result, + Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "host_crashed" + ); if !host_crashed { result?; } @@ -2165,6 +2175,83 @@ fn freeze_owned_process(process: &LocalHostProcess, deadline: Instant) -> std::i } } +#[cfg(unix)] +struct FrozenProcessTree { + pending: Vec, + processes: Vec, + lost_parent: bool, +} + +#[cfg(unix)] +static FROZEN_PROCESS_TREES: LazyLock>>>> = + LazyLock::new(|| Mutex::new(BTreeMap::new())); + +#[cfg(unix)] +fn frozen_process_trees() +-> std::sync::MutexGuard<'static, BTreeMap>>> { + FROZEN_PROCESS_TREES + .lock() + .unwrap_or_else(|error| error.into_inner()) +} + +#[cfg(unix)] +fn frozen_tree_is_pending(runtime_id: &str) -> bool { + let tree = frozen_process_trees().get(runtime_id).cloned(); + tree.is_some_and(|tree| { + let tree = tree.lock().unwrap_or_else(|error| error.into_inner()); + tree.lost_parent || !tree.pending.is_empty() + }) +} + +#[cfg(unix)] +fn collect_frozen_host_with( + runtime: &RuntimeHandle, + process: &LocalHostProcess, + mut children: impl FnMut(libc::pid_t) -> std::io::Result>, +) -> Result> { + let tree = frozen_process_trees() + .entry(runtime.runtime_id.clone()) + .or_insert_with(|| { + Arc::new(Mutex::new(FrozenProcessTree { + pending: vec![process.clone()], + processes: vec![process.clone()], + lost_parent: false, + })) + }) + .clone(); + let mut tree = tree.lock().unwrap_or_else(|error| error.into_inner()); + if tree.lost_parent { + return Err(cleanup_unconfirmed(runtime)); + } + let deadline = Instant::now() + LOCAL_HOST_STOP_TIMEOUT; + while let Some(parent) = tree.pending.last().cloned() { + if freeze_owned_process(&parent, deadline).is_err() { + tree.lost_parent |= owned_process_has_exited(&parent); + return Err(cleanup_unconfirmed(runtime)); + } + if owned_process_has_exited(&parent) { + tree.lost_parent = true; + return Err(cleanup_unconfirmed(runtime)); + } + let listed = + children(parent.pid() as libc::pid_t).map_err(|_| cleanup_unconfirmed(runtime))?; + // An exited parent can have already lost its children to reparenting. + // Keep that missing evidence across retries rather than guessing empty. + if owned_process_has_exited(&parent) { + tree.lost_parent = true; + return Err(cleanup_unconfirmed(runtime)); + } + let listed: Vec<_> = listed + .into_iter() + .map(|pid| LocalHostProcess::new(pid as u32, SystemTime::now())) + .collect(); + tree.pending.pop(); + tree.processes.extend(listed.clone()); + tree.pending.extend(listed); + } + Ok(tree.processes.clone()) +} + fn interrupt_local_host(runtime: &RuntimeHandle) -> Result> { #[cfg(unix)] { @@ -2172,41 +2259,11 @@ fn interrupt_local_host(runtime: &RuntimeHandle) -> Result .get(&runtime.runtime_id) .cloned() .ok_or_else(|| cleanup_unconfirmed(runtime))?; - if process.has_exited() == Some(true) { - return Ok(Vec::new()); - } - if !process.is_current() { - return Err(cleanup_unconfirmed(runtime)); - } - let deadline = Instant::now() + LOCAL_HOST_STOP_TIMEOUT; - // Freeze each parent before listing its children. A frozen parent cannot - // create another child after that list was read. Retain every discovered - // identity before suspension so a partial failure remains retryable. - let mut parents = vec![process.clone()]; - while let Some(parent) = parents.pop() { - freeze_owned_process(&parent, deadline).map_err(|_| cleanup_unconfirmed(runtime))?; - if owned_process_has_exited(&parent) { - continue; - } - let children: Vec<_> = child_pids(parent.pid() as libc::pid_t) - .into_iter() - .map(|pid| LocalHostProcess::new(pid as u32, SystemTime::now())) - .collect(); - STOPPING_PROCESSES - .lock() - .unwrap_or_else(|error| error.into_inner()) - .entry(runtime.runtime_id.clone()) - .or_default() - .extend(children.clone()); - parents.extend(children); - } - let descendants = { - let mut retained = STOPPING_PROCESSES - .lock() - .unwrap_or_else(|error| error.into_inner()); - let previous = retained.entry(runtime.runtime_id.clone()).or_default(); - previous.clone() - }; + let descendants = collect_frozen_host_with(runtime, &process, child_pids)?; + STOPPING_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(runtime.runtime_id.clone(), descendants.clone()); for child in descendants.iter().rev() { signal_owned_process(child).map_err(|_| cleanup_unconfirmed(runtime))?; } @@ -2232,6 +2289,10 @@ fn confirm_stopped_descendants( runtime: &RuntimeHandle, processes: &[LocalHostProcess], ) -> Result<()> { + #[cfg(unix)] + if frozen_tree_is_pending(&runtime.runtime_id) { + return Err(cleanup_unconfirmed(runtime)); + } #[cfg(windows)] let control = windows_host_controls().get(&runtime.runtime_id).cloned(); let deadline = Instant::now() + LOCAL_HOST_EXIT_GRACE; @@ -3944,7 +4005,7 @@ fn host_descendants(host_pid: u32) -> Vec { let mut descendants = Vec::new(); let mut parents = vec![host_pid]; while let Some(parent) = parents.pop() { - let children = child_pids(parent); + let children = child_pids(parent).unwrap_or_default(); parents.extend(&children); descendants.extend(children); } @@ -3952,53 +4013,55 @@ fn host_descendants(host_pid: u32) -> Vec { } #[cfg(target_os = "linux")] -fn child_pids(parent: libc::pid_t) -> Vec { - let Ok(entries) = std::fs::read_dir("/proc") else { - return Vec::new(); - }; - entries - .flatten() - .filter_map(|entry| { - let pid = entry.file_name().to_str()?.parse().ok()?; - let stat = std::fs::read_to_string(entry.path().join("stat")).ok()?; - // The command name may contain spaces or parentheses; the parent PID - // is the second field after it. - let ppid: libc::pid_t = stat - .rsplit_once(')')? - .1 - .split_whitespace() - .nth(1)? - .parse() - .ok()?; - (ppid == parent).then_some(pid) - }) - .collect() +fn child_pids(parent: libc::pid_t) -> std::io::Result> { + let mut children = Vec::new(); + for entry in std::fs::read_dir("/proc")? { + let entry = entry?; + let Some(pid) = entry + .file_name() + .to_str() + .and_then(|value| value.parse().ok()) + else { + continue; + }; + let stat = match std::fs::read_to_string(entry.path().join("stat")) { + Ok(stat) => stat, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; + let ppid = stat + .rsplit_once(')') + .and_then(|(_, fields)| fields.split_whitespace().nth(1)) + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| std::io::Error::other("process parent is unreadable"))?; + if ppid == parent { + children.push(pid); + } + } + Ok(children) } #[cfg(target_os = "macos")] -fn child_pids(parent: libc::pid_t) -> Vec { +fn child_pids(parent: libc::pid_t) -> std::io::Result> { let mut children = vec![0; 64]; loop { - let Ok(bytes) = i32::try_from(std::mem::size_of_val(children.as_slice())) else { - return Vec::new(); - }; + let bytes = i32::try_from(std::mem::size_of_val(children.as_slice())) + .map_err(|_| std::io::Error::other("process child list is too large"))?; // SAFETY: the buffer is writable and sized in bytes as libproc requires. let count = unsafe { libc::proc_listchildpids(parent, children.as_mut_ptr().cast(), bytes) }; - let Ok(count) = usize::try_from(count) else { - return Vec::new(); - }; + let count = usize::try_from(count).map_err(|_| std::io::Error::last_os_error())?; if count < children.len() { children.truncate(count); - return children; + return Ok(children); } children.resize(children.len() * 2, 0); } } #[cfg(all(unix, not(any(target_os = "linux", target_os = "macos"))))] -fn child_pids(_parent: libc::pid_t) -> Vec { - Vec::new() +fn child_pids(_parent: libc::pid_t) -> std::io::Result> { + Err(std::io::Error::from(std::io::ErrorKind::Unsupported)) } /// Puts the suspended host in a job object so a forced kill reaches every @@ -5793,6 +5856,53 @@ else: TEST_STOPPED_AGENTS.lock().expect("stop tracker").clone() } + #[cfg(unix)] + #[test] + fn an_unlisted_parent_exit_remains_unconfirmed_on_stop_retry() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let mut guard = host.lock().unwrap(); + guard.child.kill().unwrap(); + guard.child.wait().unwrap(); + let first = stop_runtime(&plan, &runtime); + drop(guard); + let retried = stop_runtime(&plan, &runtime); + let retained = local_hosts().contains_key(&runtime.runtime_id); + // This fixture has no descendants; remove only its injected uncertainty + // after checking that production retry does not erase missing evidence. + frozen_process_trees().remove(&runtime.runtime_id); + stop_runtime(&plan, &runtime).unwrap(); + assert!( + matches!(first, Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "cleanup_unconfirmed") + ); + assert!( + matches!(retried, Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "cleanup_unconfirmed") + ); + assert!(retained); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn failed_child_listing_retains_a_retryable_frozen_parent() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let process = local_host_processes() + .get(&runtime.runtime_id) + .unwrap() + .clone(); + let failed = collect_frozen_host_with(&runtime, &process, |_| { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + }); + let retained = frozen_process_trees().contains_key(&runtime.runtime_id); + stop_runtime(&plan, &runtime).expect("retry with readable process evidence"); + assert!(failed.is_err()); + assert!(retained); + assert!(!local_hosts().contains_key(&runtime.runtime_id)); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn unconfirmed_timeout_cleanup_retains_service_membership() { let (root, plan) = local_host_plan("success"); diff --git a/docs/reference/api/python-library-reference/nemo_fabric.client.md b/docs/reference/api/python-library-reference/nemo_fabric.client.md index 6848b98d3..815b74b20 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.client.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.client.md @@ -350,7 +350,7 @@ async def run( Execute one complete start, invoke, and stop lifecycle. -``input`` and ``request`` are mutually exclusive. Omitting both produces an empty text input. Use ``RunRequest`` when the invocation needs a caller-owned request ID, context, or overrides. NeMo Fabric attempts to stop a started runtime even when invocation fails. +``input`` and ``request`` are mutually exclusive. Omitting both produces an empty text input. Use ``RunRequest`` when the invocation needs a caller-owned request ID, context, or overrides. NeMo Fabric attempts to stop a started runtime even when invocation fails. Unconfirmed cleanup raises an error with ``cleanup_runtime`` for an explicit stop retry in the same process. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.errors.md b/docs/reference/api/python-library-reference/nemo_fabric.errors.md index 50e2dff6e..dd37f3637 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.errors.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.errors.md @@ -10,6 +10,9 @@ SPDX-License-Identifier: Apache-2.0 --> Public exception hierarchy for the NeMo Fabric Python SDK. +**Global Variables** +--------------- +- **TYPE_CHECKING** --- @@ -62,6 +65,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Original runtime for a failed one-shot cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -104,6 +115,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Original runtime for a failed one-shot cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -146,6 +165,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Original runtime for a failed one-shot cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -188,6 +215,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Original runtime for a failed one-shot cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -230,6 +265,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Original runtime for a failed one-shot cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -272,6 +315,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Original runtime for a failed one-shot cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -316,6 +367,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Original runtime for a failed one-shot cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -360,6 +419,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Original runtime for a failed one-shot cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/check/enum-checkreason.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/check/enum-checkreason.mdx index 79e8235c1..51bb22039 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/check/enum-checkreason.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/check/enum-checkreason.mdx @@ -13,6 +13,8 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. pub enum CheckReason { ServerStopping, CapabilitiesUnverified, + RuntimeBusy, + CleanupUnconfirmed, RuntimeUnavailable, ServiceUnavailable, AgentExited, @@ -44,6 +46,18 @@ The server is stopping. Advertised capability evidence is unavailable. +### `RuntimeBusy` + +
+ +Runtime cleanup is in progress. + +### `CleanupUnconfirmed` + +
+ +Owned runtime cleanup could not be confirmed. + ### `RuntimeUnavailable`
diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 232261afb..98e5c43b2 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -892,6 +892,10 @@ opaque `SessionRef`. NeMo Fabric does not interpret that identifier. Buffered runtime shutdown can interrupt an active invocation on supported Unix and Windows hosts. Windows shutdown requires an owned process job to confirm descendant cleanup. `Runtime.stop()` waits for the interrupted invocation to settle before reporting stopped. Cleanup errors retain ownership for retry; the runtime server blocks replacement for the same agent until cleanup succeeds. Streaming shutdown continues to use the stream lifecycle. +If one-shot `Fabric.run()` cannot confirm cleanup, the raised error provides `cleanup_runtime`. Keep that object and retry `await error.cleanup_runtime.stop()` in the same process. Cancellation can carry the same recovery object. The recovery handle is excluded from error text and structured details. When an exited parent leaves undiscoverable children, cleanup remains unconfirmed and the owning process or sandbox must recover. + +Runtime health checks report pending shutdown as `runtime_busy`, with state `unknown` and activity `stopping`. After failed cleanup, checks report `cleanup_unconfirmed`, with state `failed` and activity `unknown`. These reports describe recorded lifecycle state and do not probe the harness during cleanup. + ### Keep Agent State Across Runtimes By default, each runtime keeps its native state in its own folder, so a new diff --git a/environment-providers/openshell/src/lib.rs b/environment-providers/openshell/src/lib.rs index a22109178..f3e2a6529 100644 --- a/environment-providers/openshell/src/lib.rs +++ b/environment-providers/openshell/src/lib.rs @@ -564,6 +564,7 @@ fn valid_diagnostic_code(code: &str) -> bool { | "session_resume_unsupported" | "runtime_busy" | "cleanup_unconfirmed" + | "runtime_busy" | "invalid_request" | "interface_unsupported" | "wrong_agent" @@ -2562,6 +2563,28 @@ mod tests { assert_eq!(arguments.last().unwrap(), "--ready"); } + #[test] + fn cleanup_health_evidence_survives_the_provider_decoder() { + for (code, state, activity) in [ + ("runtime_busy", "unknown", "stopping"), + ("cleanup_unconfirmed", "failed", "unknown"), + ] { + let report = json!({ + "schema_version": "fabric.check/v1alpha1", "scope": "runtime", + "runtime_id": "runtime-1", "requested_level": "live", + "outcome": "unknown", "reason_code": code, + "completed_at": "2026-11-12T12:00:00Z", "elapsed_ms": 1, + "server": null, "runtime": {"state": state, "activity": activity}, "checks": [] + }); + let response = json!({"operation": "check", "status": "failed", "changed": false, + "result": report, "error": {"code": code, "stage": "check", "effects": "none", "message": "PRIVATE"}}); + let decoded = + decode_sdk_command_response("check", 1, response.to_string().as_bytes()).unwrap(); + assert_eq!(decoded, json!({"status": "returned", "value": report})); + assert!(!decoded.to_string().contains("PRIVATE")); + } + } + #[test] fn failed_health_evidence_is_returned_to_the_sdk() { let report = json!({ diff --git a/schemas/adapter-contract/adapter-check-result.schema.json b/schemas/adapter-contract/adapter-check-result.schema.json index eb405c683..1d3814a12 100644 --- a/schemas/adapter-contract/adapter-check-result.schema.json +++ b/schemas/adapter-contract/adapter-check-result.schema.json @@ -38,6 +38,16 @@ "description": "Advertised capability evidence is unavailable.", "type": "string" }, + { + "const": "runtime_busy", + "description": "Runtime cleanup is in progress.", + "type": "string" + }, + { + "const": "cleanup_unconfirmed", + "description": "Owned runtime cleanup could not be confirmed.", + "type": "string" + }, { "const": "runtime_unavailable", "description": "No matching runtime binding exists.", diff --git a/schemas/sdk/check-report.schema.json b/schemas/sdk/check-report.schema.json index 2ad6011d9..29e92c831 100644 --- a/schemas/sdk/check-report.schema.json +++ b/schemas/sdk/check-report.schema.json @@ -102,6 +102,16 @@ "description": "Advertised capability evidence is unavailable.", "type": "string" }, + { + "const": "runtime_busy", + "description": "Runtime cleanup is in progress.", + "type": "string" + }, + { + "const": "cleanup_unconfirmed", + "description": "Owned runtime cleanup could not be confirmed.", + "type": "string" + }, { "const": "runtime_unavailable", "description": "No matching runtime binding exists.", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/check.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/check.py index 3cb546d92..9add70cb9 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/check.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/check.py @@ -29,6 +29,8 @@ CheckReason = Literal[ "server_stopping", "capabilities_unverified", + "runtime_busy", + "cleanup_unconfirmed", "runtime_unavailable", "service_unavailable", "agent_exited", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py index 1b3f5fc6e..44aacac9d 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py @@ -527,6 +527,8 @@ async def run( an empty text input. Use ``RunRequest`` when the invocation needs a caller-owned request ID, context, or overrides. NeMo Fabric attempts to stop a started runtime even when invocation fails. + Unconfirmed cleanup raises an error with ``cleanup_runtime`` for an + explicit stop retry in the same process. Args: config: Complete typed ``FabricConfig``. @@ -567,6 +569,15 @@ async def run( await _run_native_lifecycle(native, plan.to_mapping(), request_payload) ) except (FabricError, asyncio.CancelledError) as error: + cleanup_handle = getattr(error, "_fabric_cleanup_runtime_handle", None) + if cleanup_handle is not None: + recovery = Runtime(client=self, plan=plan, runtime=cleanup_handle) + recovery._status = RuntimeStatus.FAILED + if isinstance(error, FabricError): + error._cleanup_runtime = recovery + else: + error.cleanup_runtime = recovery + del error._fabric_cleanup_runtime_handle # The selected report already includes the caller and guest SDK facts. # Optional evidence must not change the lifecycle exception or cause. if isinstance(error, FabricError): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py index 54851e9af..c46641c0b 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py @@ -7,7 +7,10 @@ from collections.abc import Mapping from copy import deepcopy -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .runtime import Runtime class FabricError(RuntimeError): @@ -48,6 +51,15 @@ def __init__( self.code = code self.retryable = retryable self.details = deepcopy(dict(details or {})) + self._cleanup_runtime: Runtime | None = None + + @property + def cleanup_runtime(self) -> Runtime | None: + """Original runtime for a failed one-shot cleanup retry, or ``None``. + + This in-memory capability is excluded from error text and structured details. + """ + return self._cleanup_runtime class FabricConfigError(FabricError): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index ea187b862..459a651b5 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -276,7 +276,16 @@ def invoke() -> dict[str, Any]: native_result = json.loads(encoded) return native_result - result = await _call_blocking(invoke) + if ( + self._current_stream is not None + and not self._current_stream._finalized + ): + result = await _call_blocking(invoke) + else: + with ThreadPoolExecutor( + max_workers=1, thread_name_prefix="fabric-invoke" + ) as executor: + result = await _call_blocking(invoke, executor=executor) typed_result = RunResult.from_mapping(result) except asyncio.CancelledError: if absorb_result and native_result is not None: @@ -298,7 +307,10 @@ def stop_after_cancel() -> Any: return result try: - await _call_blocking(stop_after_cancel) + with ThreadPoolExecutor( + max_workers=1, thread_name_prefix="fabric-stop" + ) as executor: + await _call_blocking(stop_after_cancel, executor=executor) except asyncio.CancelledError: self._status = ( RuntimeStatus.STOPPED if stopped else RuntimeStatus.FAILED @@ -1017,8 +1029,10 @@ async def _run_native_lifecycle( request: Mapping[str, Any], ) -> dict[str, Any]: observations = {"request_id": request.get("request_id")} + cleanup_runtime_handle: dict[str, Any] | None = None def run() -> dict[str, Any]: + nonlocal cleanup_runtime_handle plan_json = json.dumps(dict(plan)) try: runtime = json.loads(native.start_runtime(plan_json)) @@ -1049,9 +1063,11 @@ def run() -> dict[str, Any]: return result finally: cleanup = "completed" + stop_error: Exception | None = None try: stop_events = json.loads(native.stop_runtime(plan_json, runtime_json)) except Exception as error: + stop_error = error cleanup = "failed" if invoke_error is None: if result is None: @@ -1074,6 +1090,21 @@ def run() -> dict[str, Any]: ) if result is not None: observations["partial_result"] = result + if ( + stop_error is not None + and _runtime_error(stop_error, stage="stop").code + == "cleanup_unconfirmed" + ): + cleanup_runtime_handle = runtime + failure = invoke_error or FabricRuntimeError( + "Runtime cleanup is unconfirmed; retry cleanup_runtime.stop().", + stage="stop", + code="cleanup_unconfirmed", + ) + failure._fabric_cleanup_runtime_handle = runtime + _retain_runtime_evidence(failure, observations) + if invoke_error is None: + raise failure from stop_error if invoke_error is not None: # Keep obtained identity and cleanup evidence beside the original # exception. A protocol failure is not a normalized invocation. @@ -1084,6 +1115,8 @@ def run() -> dict[str, Any]: except asyncio.CancelledError as error: # _call_blocking waits for native cleanup before propagating cancellation. error._fabric_runtime_evidence = dict(observations) + if cleanup_runtime_handle is not None: + error._fabric_cleanup_runtime_handle = cleanup_runtime_handle raise except FabricError: raise diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py index 865396b82..ff81125c0 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py @@ -303,6 +303,16 @@ async def _check_runtime( if owned.agent != command["agent"]: raise ProtocolError("wrong_agent", "check") runtime = owned.runtime + if owned.state in {"stopping", "unknown"}: + pending = owned.state == "stopping" + report = runtime_report( + request, "runtime_busy" if pending else "cleanup_unconfirmed", started + ) + report["runtime"] = { + "state": "unknown" if pending else "failed", + "activity": "stopping" if pending else "unknown", + } + return check_response(report) if ( self.shutting_down or owned.state != "running" diff --git a/tests/python/runtime_server/test_runtime_checks.py b/tests/python/runtime_server/test_runtime_checks.py index 2201c0099..2939e9c4b 100644 --- a/tests/python/runtime_server/test_runtime_checks.py +++ b/tests/python/runtime_server/test_runtime_checks.py @@ -263,3 +263,43 @@ async def check(**kwargs): finally: release.set() await checking + + +async def test_runtime_check_reports_pending_and_unconfirmed_cleanup( + host, host_request, mock_fabric, mock_runtime +): + from nemo_fabric import FabricRuntimeError + + await host.handle(host_request("start")) + entered, release = asyncio.Event(), asyncio.Event() + mock_fabric.check = AsyncMock() + + async def stop(): + entered.set() + await release.wait() + raise FabricRuntimeError( + "test cleanup failure", stage="stop", code="cleanup_unconfirmed" + ) + + mock_runtime.stop.side_effect = stop + stopping = asyncio.create_task(host.handle(host_request("stop"))) + try: + async with asyncio.timeout(1): + await entered.wait() + pending = await host.handle(host_request("check", runtime_id="runtime-1")) + assert pending["result"]["reason_code"] == "runtime_busy" + assert pending["result"]["runtime"] == { + "state": "unknown", + "activity": "stopping", + } + protocol.validate_response(pending, "check") + release.set() + assert (await stopping)["status"] == "failed" + failed = await host.handle(host_request("check", runtime_id="runtime-1")) + assert failed["result"]["reason_code"] == "cleanup_unconfirmed" + assert failed["result"]["runtime"] == {"state": "failed", "activity": "unknown"} + protocol.validate_response(failed, "check") + mock_fabric.check.assert_not_awaited() + finally: + release.set() + await asyncio.gather(stopping, return_exceptions=True) diff --git a/tests/python/runtime_server/test_state.py b/tests/python/runtime_server/test_state.py index b64f84a43..408d60e95 100644 --- a/tests/python/runtime_server/test_state.py +++ b/tests/python/runtime_server/test_state.py @@ -542,7 +542,10 @@ async def stop(): await entered.wait() shutdown = asyncio.create_task(host.shutdown()) try: - await asyncio.sleep(0.05) + async with asyncio.timeout(1): + while not host.shutting_down: + await asyncio.sleep(0) + assert not shutdown.done() release.set() assert (await stopping)["status"] == "succeeded" await shutdown diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index f787f7f6f..8017b689e 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -160,6 +160,122 @@ def _runtime_wrapper( ) +@pytest.mark.parametrize("cancel_invocation", [False, True]) +async def test_stop_settles_an_invocation_when_the_default_executor_is_full( + mock_native, cancel_invocation +): + runtime = _runtime_wrapper(mock_native) + loop = asyncio.get_running_loop() + await asyncio.to_thread(lambda: None) + previous = loop._default_executor + occupied, release, stopped = threading.Event(), threading.Event(), threading.Event() + original_invoke = mock_native.invoke_runtime.side_effect + + def invoke(*args): + assert stopped.wait(5) + return original_invoke(*args) + + def stop(*args): + stopped.set() + return json.dumps([]) + + mock_native.invoke_runtime.side_effect = invoke + mock_native.stop_runtime.side_effect = stop + with ThreadPoolExecutor(max_workers=1) as executor: + loop.set_default_executor(executor) + worker = asyncio.create_task( + asyncio.to_thread(lambda: (occupied.set(), release.wait(5))) + ) + invoking = None + stopping = None + try: + async with asyncio.timeout(1): + while not occupied.is_set(): + await asyncio.sleep(0) + invoking = asyncio.create_task(runtime.invoke(input="queued turn")) + async with asyncio.timeout(1): + while runtime._current_task is not invoking: + await asyncio.sleep(0) + if cancel_invocation: + invoking.cancel() + stopping = asyncio.create_task(runtime.stop()) + done, _ = await asyncio.wait({stopping}, timeout=0.5) + completed_before_release = stopping in done + finally: + release.set() + stopped.set() + await worker + if invoking is not None: + outcome = (await asyncio.gather(invoking, return_exceptions=True))[0] + if stopping is not None: + await stopping + loop.set_default_executor(previous) + assert isinstance(outcome, asyncio.CancelledError) == cancel_invocation + assert completed_before_release + assert runtime.status is RuntimeStatus.STOPPED + + +@pytest.mark.parametrize("invoke_fails", [False, True]) +async def test_one_shot_retains_a_cleanup_runtime_without_exposing_its_handle( + native_client, mock_native, invoke_fails +): + handle = _runtime() + handle["environment"]["env"] = {"RECOVERY_TEST_SECRET": "private-cleanup-value"} + mock_native.start_runtime.return_value = json.dumps(handle) + original = FabricRuntimeError( + "test invocation failure", stage="invoke", code="test_failure" + ) + if invoke_fails: + mock_native.invoke_runtime.side_effect = original + mock_native.stop_runtime.side_effect = [ + FabricRuntimeError( + "test cleanup failure", stage="stop", code="cleanup_unconfirmed" + ), + json.dumps([]), + ] + with pytest.raises(FabricRuntimeError) as caught: + await native_client.run(_config(), input="hello") + if invoke_fails: + assert caught.value is original + cleanup_runtime = caught.value.cleanup_runtime + assert isinstance(cleanup_runtime, Runtime) + assert cleanup_runtime.handle.runtime_id == "runtime-1" + assert "private-cleanup-value" not in repr(caught.value) + assert "fabric-runtime-binding-test" not in json.dumps(caught.value.details) + await cleanup_runtime.stop() + assert cleanup_runtime.status is RuntimeStatus.STOPPED + assert mock_native.stop_runtime.call_count == 2 + + +async def test_cancelled_one_shot_retains_a_cleanup_runtime(native_client, mock_native): + entered, release = threading.Event(), threading.Event() + + def stop(*args): + if mock_native.stop_runtime.call_count == 1: + entered.set() + assert release.wait(5) + raise FabricRuntimeError( + "test cleanup failure", stage="stop", code="cleanup_unconfirmed" + ) + return json.dumps([]) + + mock_native.stop_runtime.side_effect = stop + running = asyncio.create_task(native_client.run(_config(), input="hello")) + try: + async with asyncio.timeout(2): + while not entered.is_set(): + await asyncio.sleep(0.01) + running.cancel() + release.set() + with pytest.raises(asyncio.CancelledError) as caught: + await running + await caught.value.cleanup_runtime.stop() + assert mock_native.stop_runtime.call_count == 2 + finally: + release.set() + await asyncio.gather(running, return_exceptions=True) + + async def test_stop_does_not_wait_for_a_saturated_default_executor(mock_native): runtime = _runtime_wrapper(mock_native) loop = asyncio.get_running_loop() From 9ba031f6e89a9c2d68392485945afbcdebced87d Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 12 Nov 2026 21:08:25 -0700 Subject: [PATCH 5/7] fix: preserve cleanup failures and terminate known owned hosts Signed-off-by: Sara Tadayon --- crates/fabric-core/src/runtime.rs | 274 ++++++++++++++++-- docs/sdk/python.mdx | 4 +- .../src/nemo_fabric/runtime_server/README.md | 2 +- 3 files changed, 252 insertions(+), 28 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d22735a48..38a6e6238 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1301,13 +1301,13 @@ fn start_service( ) { Ok(output) => output, Err(error) => { - let _ = terminate_local_host(&mut host); + let _ = terminate_failed_start(&mut host); let _ = remove_local_host_files(&host); return Err(error); } }; let info: AdapterServiceInfo = serde_json::from_value(output).map_err(|source| { - let _ = terminate_local_host(&mut host); + let _ = terminate_failed_start(&mut host); let _ = remove_local_host_files(&host); lifecycle_error( AdapterLifecycleOperation::Start, @@ -2049,6 +2049,7 @@ impl RuntimeAdapter for LocalHostAdapter { let pid = guard.child.id(); guard.pending_stop.extend( host_descendants(pid) + .map_err(|_| cleanup_unconfirmed(runtime))? .into_iter() .map(|pid| LocalHostProcess::new(pid as u32, SystemTime::now())), ); @@ -2203,6 +2204,18 @@ fn frozen_tree_is_pending(runtime_id: &str) -> bool { }) } +#[cfg(unix)] +fn frozen_tree_error(runtime: &RuntimeHandle, tree: &FrozenProcessTree) -> FabricError { + if tree.lost_parent { + // Missing descendants still prevent confirmation, but known frozen work + // must not be left suspended. Keep every identity for later inspection. + for process in tree.processes.iter().rev() { + let _ = signal_owned_process(process); + } + } + cleanup_unconfirmed(runtime) +} + #[cfg(unix)] fn collect_frozen_host_with( runtime: &RuntimeHandle, @@ -2221,17 +2234,17 @@ fn collect_frozen_host_with( .clone(); let mut tree = tree.lock().unwrap_or_else(|error| error.into_inner()); if tree.lost_parent { - return Err(cleanup_unconfirmed(runtime)); + return Err(frozen_tree_error(runtime, &tree)); } let deadline = Instant::now() + LOCAL_HOST_STOP_TIMEOUT; while let Some(parent) = tree.pending.last().cloned() { if freeze_owned_process(&parent, deadline).is_err() { tree.lost_parent |= owned_process_has_exited(&parent); - return Err(cleanup_unconfirmed(runtime)); + return Err(frozen_tree_error(runtime, &tree)); } if owned_process_has_exited(&parent) { tree.lost_parent = true; - return Err(cleanup_unconfirmed(runtime)); + return Err(frozen_tree_error(runtime, &tree)); } let listed = children(parent.pid() as libc::pid_t).map_err(|_| cleanup_unconfirmed(runtime))?; @@ -2239,7 +2252,7 @@ fn collect_frozen_host_with( // Keep that missing evidence across retries rather than guessing empty. if owned_process_has_exited(&parent) { tree.lost_parent = true; - return Err(cleanup_unconfirmed(runtime)); + return Err(frozen_tree_error(runtime, &tree)); } let listed: Vec<_> = listed .into_iter() @@ -3142,9 +3155,18 @@ fn invalidate_timed_out_local_host( runtime_id: &str, expected_host: &Arc>, host: &mut LocalAdapterHost, +) { + invalidate_timed_out_local_host_with(runtime_id, expected_host, host, terminate_local_host); +} + +fn invalidate_timed_out_local_host_with( + runtime_id: &str, + expected_host: &Arc>, + host: &mut LocalAdapterHost, + terminate: impl FnOnce(&mut LocalAdapterHost) -> std::io::Result<()>, ) { host.cleanup_pending = true; - if terminate_local_host(host).is_err() { + if terminate(host).is_err() { return; } host.cleanup_pending = false; @@ -3238,7 +3260,10 @@ fn local_host_stop_event( fn stop_service_host(record: &mut LocalServiceRecord) -> Result<()> { record.host.cleanup_pending = true; #[cfg(unix)] - remember_host_descendants(&mut record.host); + remember_host_descendants(&mut record.host).map_err(|_| lifecycle_error( + AdapterLifecycleOperation::Stop, &record.runtime.runtime_id, "cleanup_unconfirmed", + "Owned service descendants could not be inspected.", "", + ))?; let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Stop(AdapterLifecycleStop { runtime_id: record.runtime.runtime_id.clone(), @@ -3897,20 +3922,42 @@ fn take_local_host_stderr(host: &mut LocalAdapterHost) -> String { String::from_utf8_lossy(&bytes[start..]).into_owned() } +fn terminate_failed_start(host: &mut LocalAdapterHost) -> std::io::Result<()> { + terminate_failed_start_with(host, terminate_local_host) +} + +fn terminate_failed_start_with( + host: &mut LocalAdapterHost, + terminate: impl FnOnce(&mut LocalAdapterHost) -> std::io::Result<()>, +) -> std::io::Result<()> { + let outcome = terminate(host); + if outcome.is_err() { + // A failed start has no published cleanup handle. Always attempt to + // terminate its directly owned child before dropping the host record. + // Keep the error: this fallback does not confirm descendant cleanup. + let _ = host.child.kill(); + let deadline = Instant::now() + LOCAL_HOST_EXIT_GRACE; + while matches!(host.child.try_wait(), Ok(None)) && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + } + outcome +} + fn terminate_local_host(host: &mut LocalAdapterHost) -> std::io::Result<()> { #[cfg(test)] if tests::FAIL_NEXT_HOST_TERMINATION.with(|fail| fail.replace(false)) { return Err(std::io::Error::other("simulated host termination failure")); } #[cfg(unix)] - remember_host_descendants(host); + remember_host_descendants(host)?; let deadline = Instant::now() + LOCAL_HOST_EXIT_GRACE; while host.child.try_wait()?.is_none() && Instant::now() < deadline { thread::sleep(Duration::from_millis(10)); } #[cfg(unix)] { - remember_host_descendants(host); + remember_host_descendants(host)?; for process in &host.descendants { if process.has_exited() == Some(true) { continue; @@ -3974,14 +4021,14 @@ fn terminate_local_host(host: &mut LocalAdapterHost) -> std::io::Result<()> { } #[cfg(unix)] -fn remember_host_descendants(host: &mut LocalAdapterHost) { +fn remember_host_descendants(host: &mut LocalAdapterHost) -> std::io::Result<()> { // An exited host's PID can be reused. Never discover children from that PID. if !matches!(host.child.try_wait(), Ok(None)) { - return; + return Ok(()); } host.descendants .retain(|process| process.has_exited() != Some(true)); - for pid in host_descendants(host.child.id()) { + for pid in host_descendants(host.child.id())? { if !host .descendants .iter() @@ -3991,25 +4038,33 @@ fn remember_host_descendants(host: &mut LocalAdapterHost) { .push(LocalHostProcess::new(pid as u32, SystemTime::now())); } } + Ok(()) } /// Lists every live descendant of a host, including children that started /// their own session. Processes that already detached from the host's process -/// tree cannot be found this way, so cleanup stays best-effort. +/// tree cannot be found this way. Listing failures must remain unconfirmed. #[cfg(unix)] -fn host_descendants(host_pid: u32) -> Vec { - let Ok(host_pid) = libc::pid_t::try_from(host_pid) else { - return Vec::new(); - }; +fn host_descendants(host_pid: u32) -> std::io::Result> { + host_descendants_with(host_pid, child_pids) +} + +#[cfg(unix)] +fn host_descendants_with( + host_pid: u32, + mut children: impl FnMut(libc::pid_t) -> std::io::Result>, +) -> std::io::Result> { + let host_pid = libc::pid_t::try_from(host_pid) + .map_err(|_| std::io::Error::other("process identity is invalid"))?; // Snapshot the whole tree first: killed processes' children are re-parented. let mut descendants = Vec::new(); let mut parents = vec![host_pid]; while let Some(parent) = parents.pop() { - let children = child_pids(parent).unwrap_or_default(); - parents.extend(&children); - descendants.extend(children); + let listed = children(parent)?; + parents.extend(&listed); + descendants.extend(listed); } - descendants + Ok(descendants) } #[cfg(target_os = "linux")] @@ -5433,10 +5488,19 @@ for line in sys.stdin: with open(os.path.join(root, "child.pid"), "w", encoding="utf-8") as stream: stream.write(str(child.pid)) if MODE == "child_after_start": - subprocess.Popen( - [sys.executable, "-c", "import time; time.sleep(30)"], + root = os.path.dirname(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"]) + ready = os.path.join(root, "child.ready") + child = subprocess.Popen( + [sys.executable, "-c", "import pathlib,sys,time; pathlib.Path(sys.argv[1]).touch(); time.sleep(30)", ready], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) + deadline = time.monotonic() + 5 + while not os.path.exists(ready): + if child.poll() is not None or time.monotonic() >= deadline: + raise RuntimeError("fixture child did not become ready") + time.sleep(0.01) + with open(os.path.join(root, "child.pid"), "w", encoding="utf-8") as stream: + stream.write(str(child.pid)) service = message["payload"].get("service") if MODE == "connect_failure_once" and service and service["operation"] == "connect": failure_marker = os.environ["FABRIC_FAKE_HOST_STOP_MARKER"] + ".connect-failed" @@ -5883,6 +5947,134 @@ else: fs::remove_dir_all(root).unwrap(); } + #[test] + fn failed_start_cleanup_kills_owned_runtime_and_service_hosts() { + for service in [false, true] { + let (root, plan) = local_host_plan("success"); + if service { + let handle = prepare_service(&plan).unwrap(); + let record = local_services().get(&handle.service_id).unwrap().clone(); + let mut record = record.lock().unwrap(); + let outcome = terminate_failed_start_with(&mut record.host, |_| { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + }); + let exited = record.host.child.try_wait().unwrap().is_some(); + drop(record); + release_service(&plan, &handle).unwrap(); + assert_eq!( + outcome.unwrap_err().kind(), + std::io::ErrorKind::PermissionDenied + ); + assert!(exited, "a failed service start cannot abandon its host"); + } else { + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let mut host = host.lock().unwrap(); + let outcome = terminate_failed_start_with(&mut host, |_| { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + }); + let exited = host.child.try_wait().unwrap().is_some(); + drop(host); + stop_runtime(&plan, &runtime).unwrap(); + assert_eq!( + outcome.unwrap_err().kind(), + std::io::ErrorKind::PermissionDenied + ); + assert!(exited, "a failed runtime start cannot abandon its host"); + } + fs::remove_dir_all(root).unwrap(); + } + } + + #[test] + fn failed_timeout_termination_retains_the_host_for_stop_retry() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let mut guard = host.lock().unwrap(); + invalidate_timed_out_local_host_with(&runtime.runtime_id, &host, &mut guard, |_| { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + }); + let retained = local_hosts().contains_key(&runtime.runtime_id); + let files_retained = guard.stderr_path.exists(); + drop(guard); + stop_runtime(&plan, &runtime).unwrap(); + fs::remove_dir_all(root).unwrap(); + assert!(retained); + assert!(files_retained); + } + + #[cfg(unix)] + #[test] + fn incomplete_descendant_listing_cannot_be_an_empty_snapshot() { + let error = host_descendants_with(100, |parent| { + if parent == 100 { + Ok(vec![101]) + } else { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + } + }) + .expect_err("a descendant listing failure must propagate"); + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + } + + #[cfg(unix)] + #[test] + fn lost_parent_cleanup_terminates_known_frozen_processes() { + let (root, plan) = local_host_plan("child_after_start"); + let runtime = start_runtime(&plan).unwrap(); + let host_process = local_host_processes() + .get(&runtime.runtime_id) + .unwrap() + .clone(); + let child_pid: u32 = fs::read_to_string(root.join("child.pid")) + .unwrap() + .parse() + .unwrap(); + let child = LocalHostProcess::new(child_pid, SystemTime::now()); + let deadline = Instant::now() + Duration::from_secs(2); + freeze_owned_process(&host_process, deadline).unwrap(); + freeze_owned_process(&child, deadline).unwrap(); + let mut departed = Command::new("sh").args(["-c", "exit 0"]).spawn().unwrap(); + let departed_pid = departed.id(); + departed.wait().unwrap(); + let departed = LocalHostProcess::new(departed_pid, SystemTime::now()); + frozen_process_trees().insert( + runtime.runtime_id.clone(), + Arc::new(Mutex::new(FrozenProcessTree { + pending: vec![departed.clone()], + processes: vec![host_process.clone(), child.clone(), departed], + lost_parent: false, + })), + ); + let outcome = collect_frozen_host_with(&runtime, &host_process, |_| unreachable!()); + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline + && !(owned_process_has_exited(&host_process) && owned_process_has_exited(&child)) + { + thread::sleep(Duration::from_millis(10)); + } + let survivors_exited = + owned_process_has_exited(&host_process) && owned_process_has_exited(&child); + let retained = frozen_tree_is_pending(&runtime.runtime_id); + // Always release the real fixture processes, including on the red run. + signal_owned_process(&child).unwrap(); + signal_owned_process(&host_process).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + host.lock().unwrap().child.wait().unwrap(); + frozen_process_trees().remove(&runtime.runtime_id); + stop_runtime(&plan, &runtime).unwrap(); + fs::remove_dir_all(root).unwrap(); + assert!( + matches!(outcome, Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "cleanup_unconfirmed") + ); + assert!(retained); + assert!( + survivors_exited, + "known frozen work must not survive lost-parent cleanup" + ); + } + #[cfg(unix)] #[test] fn failed_child_listing_retains_a_retryable_frozen_parent() { @@ -6096,8 +6288,24 @@ else: #[test] fn busy_stop_interrupts_the_owned_host_without_the_invocation_lock() { - let (root, plan) = local_host_plan("success"); + let (root, plan) = local_host_plan("child_after_start"); let runtime = start_runtime(&plan).unwrap(); + let child_pid: u32 = fs::read_to_string(root.join("child.pid")) + .unwrap() + .parse() + .unwrap(); + #[cfg(unix)] + let child = LocalHostProcess::new(child_pid, SystemTime::now()); + #[cfg(windows)] + let child = { + use std::os::windows::io::{FromRawHandle, OwnedHandle}; + // SAFETY: OpenProcess returns a new owned handle with synchronization access. + let raw = unsafe { + windows_sys::Win32::System::Threading::OpenProcess(0x0010_0000, 0, child_pid) + }; + assert!(!raw.is_null()); + unsafe { OwnedHandle::from_raw_handle(raw) } + }; let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); let guard = host.lock().unwrap(); let interrupted = interrupt_local_host(&runtime); @@ -6105,6 +6313,22 @@ else: let stopped = stop_runtime(&plan, &runtime); interrupted.expect("interrupt without taking the busy host lock"); stopped.expect("confirm cleanup after interruption"); + #[cfg(unix)] + assert!(owned_process_has_exited(&child)); + #[cfg(windows)] + { + use std::os::windows::io::AsRawHandle; + // SAFETY: the captured child handle remains owned until this assertion completes. + assert_eq!( + unsafe { + windows_sys::Win32::System::Threading::WaitForSingleObject( + child.as_raw_handle(), + 1000, + ) + }, + windows_sys::Win32::Foundation::WAIT_OBJECT_0 + ); + } assert!(!local_hosts().contains_key(&runtime.runtime_id)); assert!(local_host_has_exited(&runtime.runtime_id).is_none()); fs::remove_dir_all(root).unwrap(); diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 98e5c43b2..3fba28580 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -365,11 +365,11 @@ Service checks evaluate earlier levels in order under the same three-second defa `fabric.check/v1alpha1` identifies the shared report schema returned by `Fabric.check()` and `fabric-agent-ctl check`. The executable ships with -`nemo-fabric-runtime` and follows the SDK package release. +`nemo-fabric-runtime` and follows the SDK package release. Health readers validate reason codes strictly. Use the same `nemo-fabric-runtime` release for the caller, sandbox server, and `fabric-agent-ctl`; mixed-release health reports are not supported. The schema identifier does not negotiate SDK releases. `fabric-agent-ctl serve` starts one runtime server per sandbox. The `start`, `invoke`, `stop`, and `status` commands call `Fabric.start_runtime()`, `Runtime.invoke()`, `Runtime.stop()`, and `Fabric.runtime_status()`. Start accepts `FabricConfig` and returns the SDK `RuntimeHandle` mapping. Invoke accepts a full `RunRequest` through `--request` or any JSON SDK input through `--input`, and returns the SDK `RunResult` mapping. Stop returns JSON null. These payloads occupy the response envelope's `result` field; command diagnostics remain separate. Targeted commands require the returned runtime ID and exact agent name. The caller waits for the private socket before start. Caller-side preflight uses `Fabric.plan()` and `Fabric.doctor()`; startup validates configuration against installed descriptors. -One sandbox can own multiple runtimes, each addressed by its returned runtime ID. Invoke and stop operations are independent across IDs; stopping one runtime leaves the others usable, and another runtime can be started after a stop. An uncertain start blocks further starts because the server cannot safely determine whether an unaddressable runtime exists. Ordinary SDK stop does not confirm descendant cleanup, so the runtime server retains `fabric.unclean` after runtime use and reports `cleanup_unconfirmed` if previous sandbox process cleanup cannot be confirmed. Refer to the [runtime server command guide](https://github.com/NVIDIA/NeMo-Fabric/blob/main/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md). +One sandbox can own multiple runtimes, each addressed by its returned runtime ID. Invoke and stop operations are independent across IDs; stopping one runtime leaves the others usable, and another runtime can be started after a stop. An uncertain start blocks further starts because the server cannot safely determine whether an unaddressable runtime exists. Buffered SDK stop confirms owned process cleanup before reporting success and retains ownership when cleanup is unconfirmed. The runtime server also retains `fabric.unclean` after runtime use or an uncertain start until previous sandbox process cleanup can be confirmed. Refer to the [runtime server command guide](https://github.com/NVIDIA/NeMo-Fabric/blob/main/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md). ## Configure Agents In Code diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md index d1865f8c0..211ea0ed5 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md @@ -51,7 +51,7 @@ Each completed check preserves an immutable `fabric.check/v1alpha1` report in `r An unavailable runtime returns `unknown/runtime_unavailable` with no agent observations. If the server knows the runtime is stopped or the SDK records a stopped or failed lifecycle, `runtime.state` preserves that state. A failed lifecycle leaves activity unknown. These lifecycle records do not establish that the agent's child processes exited. -`fabric.check/v1alpha1` identifies the shared health request and report format so readers can reject formats they do not support. The identifier stays the same across SDK releases while the health format remains compatible. `fabric-agent-ctl` ships with `nemo-fabric-runtime`; the executable and all subcommands follow the SDK package release. +`fabric.check/v1alpha1` identifies the shared health request and report format so readers can reject formats they do not support. The identifier stays the same across SDK releases while the health format remains compatible. `fabric-agent-ctl` ships with `nemo-fabric-runtime`; the executable and all subcommands follow the SDK package release. Health readers validate reason codes strictly. The caller, server, and `fabric-agent-ctl` must use the same `nemo-fabric-runtime` release; mixed-release health reports are not supported. The format identifier does not negotiate SDK releases. Report timing covers the client call, including connection, response decoding, and validation. `completed_at` records client completion; each `observed_at` records when the server collected the evidence. Successful checks, confirmed failures, and timeouts use the same timing interval. From 8edb987e5a849b47e248a5eea8f1d7a73c73a7d2 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Thu, 12 Nov 2026 22:05:49 -0700 Subject: [PATCH 6/7] fix: block retained host calls and preserve startup ownership Signed-off-by: Sara Tadayon --- crates/fabric-core/src/error.rs | 6 +- crates/fabric-core/src/runtime.rs | 270 +++++++++++------- crates/fabric-python/src/lib.rs | 21 ++ .../nemo_fabric.errors.md | 16 +- .../error/struct-runtimecleanuppending.mdx | 6 +- docs/sdk/python.mdx | 4 +- .../src/nemo_fabric/client.py | 57 +++- .../src/nemo_fabric/errors.py | 8 +- .../src/nemo_fabric/runtime.py | 6 + tests/python/test_native_sdk.py | 38 +++ tests/python/test_runtime.py | 43 +++ 11 files changed, 347 insertions(+), 128 deletions(-) diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index e4c984b8e..424dc5cab 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -235,7 +235,7 @@ pub enum FabricError { /// Bounded adapter-host diagnostics. diagnostics: String, }, - /// A one-shot run could not confirm cleanup; the original runtime remains owned. + /// Owned lifecycle cleanup is unconfirmed; the original runtime remains owned. #[error("{0}")] RuntimeCleanupPending(Box), /// A runtime handle was used with a different run plan than the one that created it. @@ -440,7 +440,7 @@ pub enum FabricError { }, } -/// Cleanup context retained when [`crate::run_plan`] cannot finish stopping a runtime. +/// Cleanup context retained when startup or invocation cannot finish owned cleanup. /// /// Retry [`crate::stop_runtime`] with the original plan and [`Self::runtime`]. /// The handle and completed output are deliberately excluded from diagnostic formatting. @@ -473,7 +473,7 @@ impl RuntimeCleanupPending { self.result.as_ref() } - /// The invocation or stop error that preceded the retained cleanup context. + /// The startup, invocation, or stop error that preceded the retained cleanup context. pub fn cause(&self) -> &FabricError { &self.cause } diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 38a6e6238..45d9a1942 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -950,6 +950,8 @@ struct LocalAdapterHost { execution: Option, adapter_execution: Option, child: Child, + invocation_unavailable: bool, + cleanup_discovery_pending: bool, pending_stop: Vec, #[cfg(windows)] control: Arc, @@ -1174,7 +1176,11 @@ pub fn start_runtime_with_service( let runtime = match started { Ok(runtime) => runtime, Err(error) => { - if let FabricError::AdapterLifecycleOperation { + if let FabricError::RuntimeCleanupPending(context) = &error { + record + .active_runtime_ids + .insert(context.runtime().runtime_id.clone()); + } else let FabricError::AdapterLifecycleOperation { code, runtime_id, .. } = &error && code == "cleanup_unconfirmed" @@ -1300,23 +1306,21 @@ fn start_service( LOCAL_HOST_START_TIMEOUT, ) { Ok(output) => output, - Err(error) => { - let _ = terminate_failed_start(&mut host); - let _ = remove_local_host_files(&host); - return Err(error); + Err(error) => return Err(retain_failed_start(&runtime, host, error)), + }; + let info: AdapterServiceInfo = match serde_json::from_value(output) { + Ok(info) => info, + Err(source) => { + let cause = lifecycle_error( + AdapterLifecycleOperation::Start, + &service_id, + "invalid_service_info", + format!("adapter returned invalid service information: {source}"), + local_host_diagnostics(&host), + ); + return Err(retain_failed_start(&runtime, host, cause)); } }; - let info: AdapterServiceInfo = serde_json::from_value(output).map_err(|source| { - let _ = terminate_failed_start(&mut host); - let _ = remove_local_host_files(&host); - lifecycle_error( - AdapterLifecycleOperation::Start, - &service_id, - "invalid_service_info", - format!("adapter returned invalid service information: {source}"), - local_host_diagnostics(&host), - ) - })?; let software_report = if plan.config.provenance.enabled { host.adapter_execution .as_ref() @@ -2024,6 +2028,9 @@ impl RuntimeAdapter for LocalHostAdapter { guard.cleanup_pending = true; PENDING_LOCAL_CLEANUP.lock().unwrap_or_else(|error| error.into_inner()) .insert(runtime.runtime_id.clone(), guard.cleanup_scope.clone()); + if guard.cleanup_discovery_pending { + interrupted = Some(interrupt_local_host(runtime)?); + } #[cfg(unix)] if frozen_tree_is_pending(&runtime.runtime_id) { interrupted = Some(interrupt_local_host(runtime)?); @@ -2436,42 +2443,7 @@ fn start_local_runtime( .and_then(|output| confirm_native_session(plan, &mut runtime, &mut host, output)); if let Err(error) = started { crate::check::invalidate_runtime(&runtime); - if terminate_local_host(&mut host).is_err() { - host.cleanup_pending = true; - // Startup errors can contain credentials. Keep details private and - // return only the path while cleanup ownership remains retained. - let diagnostic_path = host.runtime_dir.join("startup-error.log"); - let mut options = File::options(); - options.create_new(true).write(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600); - } - let diagnostics = options - .open(&diagnostic_path) - .and_then(|mut file| { - file.write_all(error.to_string().as_bytes())?; - file.sync_all() - }) - .map(|()| diagnostic_path.display().to_string()) - .unwrap_or_else(|_| "Startup diagnostics could not be saved".to_string()); - let scope = host.cleanup_scope.clone(); - local_hosts().insert(runtime.runtime_id.clone(), Arc::new(Mutex::new(host))); - PENDING_LOCAL_CLEANUP - .lock() - .unwrap_or_else(|error| error.into_inner()) - .insert(runtime.runtime_id.clone(), scope); - return Err(lifecycle_error( - AdapterLifecycleOperation::Start, - &runtime.runtime_id, - "cleanup_unconfirmed", - "failed start left native cleanup unconfirmed", - diagnostics, - )); - } - let _ = remove_local_host_files(&host); - return Err(error); + return Err(retain_failed_start(&runtime, host, error)); } #[cfg(windows)] windows_host_controls().insert(runtime.runtime_id.clone(), host.control.clone()); @@ -2772,6 +2744,15 @@ fn run_local_host_invocation_with_timeout( "runtime handle does not match its bound native session", )); } + if host_guard.invocation_unavailable { + return Err(lifecycle_error( + operation, + &runtime.runtime_id, + "host_unavailable", + "The retained host is available only for cleanup.", + "", + )); + } let artifacts = host_guard.artifacts.clone(); let relay_config = host_guard.relay_config.clone(); let fabric_home = prepare_fabric_home(&artifacts, runtime, &invocation)?; @@ -3166,7 +3147,9 @@ fn invalidate_timed_out_local_host_with( terminate: impl FnOnce(&mut LocalAdapterHost) -> std::io::Result<()>, ) { host.cleanup_pending = true; + host.invocation_unavailable = true; if terminate(host).is_err() { + host.cleanup_discovery_pending = true; return; } host.cleanup_pending = false; @@ -3645,6 +3628,8 @@ fn spawn_local_host( execution: crate::provenance::core_snapshot(&plan.config.provenance), adapter_execution: None, child, + invocation_unavailable: false, + cleanup_discovery_pending: false, pending_stop: Vec::new(), #[cfg(windows)] control, @@ -3922,26 +3907,36 @@ fn take_local_host_stderr(host: &mut LocalAdapterHost) -> String { String::from_utf8_lossy(&bytes[start..]).into_owned() } -fn terminate_failed_start(host: &mut LocalAdapterHost) -> std::io::Result<()> { - terminate_failed_start_with(host, terminate_local_host) +fn retain_failed_start( + runtime: &RuntimeHandle, + host: LocalAdapterHost, + cause: FabricError, +) -> FabricError { + retain_failed_start_with(runtime, host, cause, terminate_local_host) } -fn terminate_failed_start_with( - host: &mut LocalAdapterHost, +fn retain_failed_start_with( + runtime: &RuntimeHandle, + mut host: LocalAdapterHost, + cause: FabricError, terminate: impl FnOnce(&mut LocalAdapterHost) -> std::io::Result<()>, -) -> std::io::Result<()> { - let outcome = terminate(host); - if outcome.is_err() { - // A failed start has no published cleanup handle. Always attempt to - // terminate its directly owned child before dropping the host record. - // Keep the error: this fallback does not confirm descendant cleanup. - let _ = host.child.kill(); - let deadline = Instant::now() + LOCAL_HOST_EXIT_GRACE; - while matches!(host.child.try_wait(), Ok(None)) && Instant::now() < deadline { - thread::sleep(Duration::from_millis(10)); - } +) -> FabricError { + let process = LocalHostProcess::new(host.child.id(), SystemTime::now()); + let terminated = terminate(&mut host).is_ok(); + if terminated && remove_local_host_files(&host).is_ok() { + return cause; } - outcome + host.invocation_unavailable = true; + host.cleanup_discovery_pending |= !terminated; + #[cfg(windows)] + windows_host_controls().insert(runtime.runtime_id.clone(), host.control.clone()); + local_host_processes().insert(runtime.runtime_id.clone(), process); + local_hosts().insert(runtime.runtime_id.clone(), Arc::new(Mutex::new(host))); + FabricError::RuntimeCleanupPending(Box::new(crate::RuntimeCleanupPending::new( + runtime.clone(), + None, + cause, + ))) } fn terminate_local_host(host: &mut LocalAdapterHost) -> std::io::Result<()> { @@ -5475,6 +5470,14 @@ for line in sys.stdin: message = json.loads(line) operation = message["operation"] if operation == "start": + if MODE == "connected_start_cleanup_failure" and message["payload"].get("service", {}).get("operation") == "connect": + import shutil + directory = os.environ["FABRIC_HOME"] + shutil.rmtree(directory) + with open(directory, "w", encoding="utf-8") as stream: + stream.write("startup cleanup requires a retry") + response("start", error=failure("start", "fixture_start_failure", "start rejected")) + continue if MODE == "start_failure": print("start diagnostic", file=sys.stderr, flush=True) response("start", error=failure("start", "fake_start", "start rejected")) @@ -5948,44 +5951,113 @@ else: } #[test] - fn failed_start_cleanup_kills_owned_runtime_and_service_hosts() { + fn failed_start_discovery_error_retains_runtime_and_service_owners() { for service in [false, true] { - let (root, plan) = local_host_plan("success"); - if service { + let (root, plan) = local_host_plan("child_after_start"); + let (runtime, host) = if service { let handle = prepare_service(&plan).unwrap(); - let record = local_services().get(&handle.service_id).unwrap().clone(); - let mut record = record.lock().unwrap(); - let outcome = terminate_failed_start_with(&mut record.host, |_| { - Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) - }); - let exited = record.host.child.try_wait().unwrap().is_some(); - drop(record); - release_service(&plan, &handle).unwrap(); - assert_eq!( - outcome.unwrap_err().kind(), - std::io::ErrorKind::PermissionDenied - ); - assert!(exited, "a failed service start cannot abandon its host"); + let record = local_services().remove(&handle.service_id).unwrap(); + let record = Arc::try_unwrap(record).ok().unwrap().into_inner().unwrap(); + (record.runtime, record.host) } else { let runtime = start_runtime(&plan).unwrap(); - let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); - let mut host = host.lock().unwrap(); - let outcome = terminate_failed_start_with(&mut host, |_| { - Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) - }); - let exited = host.child.try_wait().unwrap().is_some(); - drop(host); - stop_runtime(&plan, &runtime).unwrap(); - assert_eq!( - outcome.unwrap_err().kind(), - std::io::ErrorKind::PermissionDenied - ); - assert!(exited, "a failed runtime start cannot abandon its host"); - } + let host = local_hosts().remove(&runtime.runtime_id).unwrap(); + let host = Arc::try_unwrap(host).ok().unwrap().into_inner().unwrap(); + (runtime, host) + }; + let child_pid: u32 = fs::read_to_string(root.join("child.pid")) + .unwrap() + .parse() + .unwrap(); + let cause = lifecycle_error( + AdapterLifecycleOperation::Start, + &runtime.runtime_id, + "fixture_start_failure", + "PRIVATE_START_CAUSE", + "", + ); + let error = retain_failed_start_with(&runtime, host, cause, |_| { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + }); + assert!(!format!("{error:?}").contains("PRIVATE_START_CAUSE")); + assert!(!error.to_string().contains(&runtime.runtime_binding)); + let FabricError::RuntimeCleanupPending(pending) = error else { + panic!("failed startup must retain cleanup context"); + }; + assert_eq!(pending.runtime(), &runtime); + assert!(local_hosts().contains_key(&runtime.runtime_id)); + assert!(local_host_directory(&runtime.runtime_id).exists()); + let denied = invoke_runtime(&plan, &runtime, RunRequest::text("must not invoke")); + stop_runtime(&plan, pending.runtime()).unwrap(); + #[cfg(unix)] + assert!(owned_process_has_exited(&LocalHostProcess::new( + child_pid, + SystemTime::now() + ))); + #[cfg(windows)] + let _ = child_pid; + assert!(!local_hosts().contains_key(&runtime.runtime_id)); + assert!( + matches!(denied, Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "host_unavailable") + ); fs::remove_dir_all(root).unwrap(); } } + #[cfg(unix)] + #[test] + fn retained_connected_start_blocks_service_release_until_cleanup() { + let (root, plan) = local_host_plan("connected_start_cleanup_failure"); + let service = prepare_service(&plan).unwrap(); + let error = start_runtime_with_service(&plan, &service).unwrap_err(); + let FabricError::RuntimeCleanupPending(context) = error else { + panic!("connected startup must retain cleanup context"); + }; + let release = release_service(&plan, &service); + assert!(matches!(release, Err(FabricError::ServiceInUse { .. }))); + let directory = local_host_directory(&context.runtime().runtime_id); + assert!(directory.is_file()); + fs::remove_file(&directory).unwrap(); + fs::create_dir(&directory).unwrap(); + stop_runtime(&plan, context.runtime()).unwrap(); + release_service(&plan, &service).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn retained_timeout_host_rejects_queued_and_later_invocations() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let mut guard = host.lock().unwrap(); + let queued_plan = plan.clone(); + let queued_runtime = runtime.clone(); + let queued = thread::spawn(move || { + invoke_runtime(&queued_plan, &queued_runtime, RunRequest::text("queued")) + }); + let deadline = Instant::now() + Duration::from_secs(2); + while Arc::strong_count(&host) < 3 { + assert!( + Instant::now() < deadline, + "queued invocation did not acquire the host record" + ); + thread::sleep(Duration::from_millis(10)); + } + invalidate_timed_out_local_host_with(&runtime.runtime_id, &host, &mut guard, |_| { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + }); + drop(guard); + let queued = queued.join().unwrap(); + let later = invoke_runtime(&plan, &runtime, RunRequest::text("later")); + stop_runtime(&plan, &runtime).unwrap(); + fs::remove_dir_all(root).unwrap(); + for outcome in [queued, later] { + assert!( + matches!(outcome, Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "host_unavailable") + ); + } + } + #[test] fn failed_timeout_termination_retains_the_host_for_stop_retry() { let (root, plan) = local_host_plan("success"); @@ -8455,7 +8527,7 @@ else: )); assert!(matches!( second_error, - FabricError::AdapterLifecycleOperation { code, .. } if code == "host_crashed" + FabricError::AdapterLifecycleOperation { code, .. } if code == "host_unavailable" )); assert!(!local_hosts().contains_key(&runtime.runtime_id)); let stopped = stop_runtime(&plan, &runtime).expect("stop after timeout is idempotent"); diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 5636d1da2..44e9d8f64 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -464,6 +464,27 @@ fn to_py_error(error: nemo_fabric_core::FabricError) -> PyErr { Err(setattr_error) => setattr_error, } } + FabricError::RuntimeCleanupPending(context) => { + let failure = PyRuntimeError::new_err( + "Runtime cleanup is unconfirmed; retry cleanup_runtime.stop().", + ); + let encoded = match serde_json::to_string(context.runtime()) { + Ok(encoded) => encoded, + Err(_) => { + return PyRuntimeError::new_err("Runtime cleanup context is unavailable."); + } + }; + match Python::attach(|py| -> PyResult<()> { + failure.value(py).setattr("code", "cleanup_unconfirmed")?; + let handle = py.import("json")?.call_method1("loads", (encoded,))?; + failure + .value(py) + .setattr("_fabric_cleanup_runtime_handle", handle) + }) { + Ok(()) => failure, + Err(error) => error, + } + } _ => PyRuntimeError::new_err(error.to_string()), } } diff --git a/docs/reference/api/python-library-reference/nemo_fabric.errors.md b/docs/reference/api/python-library-reference/nemo_fabric.errors.md index dd37f3637..94cc6f920 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.errors.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.errors.md @@ -69,7 +69,7 @@ Initialize a structured NeMo Fabric exception. ### property cleanup_runtime -Original runtime for a failed one-shot cleanup retry, or ``None``. +Owned runtime for a failed lifecycle cleanup retry, or ``None``. This in-memory capability is excluded from error text and structured details. @@ -119,7 +119,7 @@ Initialize a structured NeMo Fabric exception. ### property cleanup_runtime -Original runtime for a failed one-shot cleanup retry, or ``None``. +Owned runtime for a failed lifecycle cleanup retry, or ``None``. This in-memory capability is excluded from error text and structured details. @@ -169,7 +169,7 @@ Initialize a structured NeMo Fabric exception. ### property cleanup_runtime -Original runtime for a failed one-shot cleanup retry, or ``None``. +Owned runtime for a failed lifecycle cleanup retry, or ``None``. This in-memory capability is excluded from error text and structured details. @@ -219,7 +219,7 @@ Initialize a structured NeMo Fabric exception. ### property cleanup_runtime -Original runtime for a failed one-shot cleanup retry, or ``None``. +Owned runtime for a failed lifecycle cleanup retry, or ``None``. This in-memory capability is excluded from error text and structured details. @@ -269,7 +269,7 @@ Initialize a structured NeMo Fabric exception. ### property cleanup_runtime -Original runtime for a failed one-shot cleanup retry, or ``None``. +Owned runtime for a failed lifecycle cleanup retry, or ``None``. This in-memory capability is excluded from error text and structured details. @@ -319,7 +319,7 @@ Initialize a structured NeMo Fabric exception. ### property cleanup_runtime -Original runtime for a failed one-shot cleanup retry, or ``None``. +Owned runtime for a failed lifecycle cleanup retry, or ``None``. This in-memory capability is excluded from error text and structured details. @@ -371,7 +371,7 @@ Initialize a structured NeMo Fabric exception. ### property cleanup_runtime -Original runtime for a failed one-shot cleanup retry, or ``None``. +Owned runtime for a failed lifecycle cleanup retry, or ``None``. This in-memory capability is excluded from error text and structured details. @@ -423,7 +423,7 @@ Initialize a structured NeMo Fabric exception. ### property cleanup_runtime -Original runtime for a failed one-shot cleanup retry, or ``None``. +Owned runtime for a failed lifecycle cleanup retry, or ``None``. This in-memory capability is excluded from error text and structured details. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/struct-runtimecleanuppending.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/struct-runtimecleanuppending.mdx index 64a2af2eb..73211d9a4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/struct-runtimecleanuppending.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/struct-runtimecleanuppending.mdx @@ -1,7 +1,7 @@ --- title: "Struct Runtime Cleanup Pending" sidebar-title: "RuntimeCleanupPending" -description: "Cleanup context retained when `crate::run_plan` cannot finish stopping a runtime." +description: "Cleanup context retained when startup or invocation cannot finish owned cleanup." position: 99 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. @@ -13,7 +13,7 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. pub struct RuntimeCleanupPending { /* private fields */ } ``` -Cleanup context retained when [`crate::run_plan`](../runtime/fn-run-plan.mdx) cannot finish stopping a runtime. +Cleanup context retained when startup or invocation cannot finish owned cleanup. Retry [`crate::stop_runtime`](../runtime/fn-stop-runtime.mdx) with the original plan and [`Self::runtime`](struct-runtimecleanuppending.mdx#method.runtime). The handle and completed output are deliberately excluded from diagnostic formatting. @@ -39,7 +39,7 @@ The completed invocation result, if invocation succeeded before cleanup failed.
FabricError"}} />
-The invocation or stop error that preceded the retained cleanup context. +The startup, invocation, or stop error that preceded the retained cleanup context. ## Trait Implementations diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index 3fba28580..c81c232e1 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -892,7 +892,9 @@ opaque `SessionRef`. NeMo Fabric does not interpret that identifier. Buffered runtime shutdown can interrupt an active invocation on supported Unix and Windows hosts. Windows shutdown requires an owned process job to confirm descendant cleanup. `Runtime.stop()` waits for the interrupted invocation to settle before reporting stopped. Cleanup errors retain ownership for retry; the runtime server blocks replacement for the same agent until cleanup succeeds. Streaming shutdown continues to use the stream lifecycle. -If one-shot `Fabric.run()` cannot confirm cleanup, the raised error provides `cleanup_runtime`. Keep that object and retry `await error.cleanup_runtime.stop()` in the same process. Cancellation can carry the same recovery object. The recovery handle is excluded from error text and structured details. When an exited parent leaves undiscoverable children, cleanup remains unconfirmed and the owning process or sandbox must recover. +If `Fabric.run()`, `Fabric.start_runtime()`, `Fabric.prepare_service()`, or `Fabric.attach_service()` cannot confirm cleanup after a failed lifecycle operation, the raised error provides `cleanup_runtime`. Keep that object and retry `await error.cleanup_runtime.stop()` in the same process. Cancellation can carry the same recovery object. The recovery handle is excluded from error text and structured details. When an exited parent leaves undiscoverable children, cleanup remains unconfirmed and the owning process or sandbox must recover. + +Retained hosts from a timed-out invocation or failed start accept cleanup only. New and queued invocations return `host_unavailable` after acquiring the host lock. Failed connected starts keep their service membership until cleanup succeeds, so the service cannot be released while that runtime remains owned. Runtime health checks report pending shutdown as `runtime_busy`, with state `unknown` and activity `stopping`. After failed cleanup, checks report `cleanup_unconfirmed`, with state `failed` and activity `unknown`. These reports describe recorded lifecycle state and do not probe the harness during cleanup. diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py index 44aacac9d..c50068e25 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py @@ -569,15 +569,7 @@ async def run( await _run_native_lifecycle(native, plan.to_mapping(), request_payload) ) except (FabricError, asyncio.CancelledError) as error: - cleanup_handle = getattr(error, "_fabric_cleanup_runtime_handle", None) - if cleanup_handle is not None: - recovery = Runtime(client=self, plan=plan, runtime=cleanup_handle) - recovery._status = RuntimeStatus.FAILED - if isinstance(error, FabricError): - error._cleanup_runtime = recovery - else: - error.cleanup_runtime = recovery - del error._fabric_cleanup_runtime_handle + self._attach_cleanup_runtime(error, plan) # The selected report already includes the caller and guest SDK facts. # Optional evidence must not change the lifecycle exception or cause. if isinstance(error, FabricError): @@ -592,6 +584,43 @@ async def run( ) raise + def _attach_cleanup_runtime(self, error: BaseException, plan: RunPlan) -> None: + cleanup = getattr(error, "_fabric_cleanup_runtime_handle", None) + if cleanup is not None: + recovery = Runtime(client=self, plan=plan, runtime=cleanup) + recovery._status = RuntimeStatus.FAILED + if isinstance(error, FabricError): + error._cleanup_runtime = recovery + else: + error.cleanup_runtime = recovery + del error._fabric_cleanup_runtime_handle + + async def _call_start_with_cleanup(self, start: Any, plan: RunPlan) -> Any: + cleanup = None + + def call(): + nonlocal cleanup + try: + return start() + except Exception as error: + cleanup = getattr(error, "_fabric_cleanup_runtime_handle", None) + raise + + try: + return await _call_blocking(call) + except BaseException as error: + if cleanup is not None: + failure = ( + error + if isinstance(error, (FabricError, asyncio.CancelledError)) + else _runtime_error(error, stage="start") + ) + failure._fabric_cleanup_runtime_handle = cleanup + self._attach_cleanup_runtime(failure, plan) + if failure is not error: + raise failure from error + raise + async def start_runtime( self, config: FabricConfig, @@ -1097,7 +1126,7 @@ def start() -> dict[str, Any]: return started_runtime try: - runtime = await _call_blocking(start) + runtime = await self._call_start_with_cleanup(start, plan) except asyncio.CancelledError: if started_runtime is not None: try: @@ -1192,7 +1221,9 @@ def prepare() -> dict[str, Any]: return started_service try: - handle = ServiceHandle.from_mapping(await _call_blocking(prepare)) + handle = ServiceHandle.from_mapping( + await self._call_start_with_cleanup(prepare, plan) + ) except BaseException as error: await _release_registered_service(native, plan, started_service) if isinstance(error, (asyncio.CancelledError, FabricError)): @@ -1253,7 +1284,9 @@ def attach() -> dict[str, Any]: return attached_service try: - handle = ServiceHandle.from_mapping(await _call_blocking(attach)) + handle = ServiceHandle.from_mapping( + await self._call_start_with_cleanup(attach, plan) + ) except BaseException as error: await _release_registered_service(native, plan, attached_service) if isinstance(error, (asyncio.CancelledError, FabricError)): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py index c46641c0b..8464c1edd 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py @@ -55,7 +55,7 @@ def __init__( @property def cleanup_runtime(self) -> Runtime | None: - """Original runtime for a failed one-shot cleanup retry, or ``None``. + """Owned runtime for a failed lifecycle cleanup retry, or ``None``. This in-memory capability is excluded from error text and structured details. """ @@ -78,12 +78,16 @@ def _runtime_error(error: Exception, *, stage: str) -> FabricRuntimeError: "timeout" if isinstance(error, TimeoutError) else getattr(error, "code", None) ) retryable = getattr(error, "retryable", False) - return FabricRuntimeError( + failure = FabricRuntimeError( str(error), stage=stage, code=code if isinstance(code, str) else None, retryable=retryable if isinstance(retryable, bool) else False, ) + cleanup = getattr(error, "_fabric_cleanup_runtime_handle", None) + if cleanup is not None: + failure._fabric_cleanup_runtime_handle = cleanup + return failure class FabricStateError(FabricRuntimeError): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 459a651b5..1d2e6e634 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -1037,9 +1037,15 @@ def run() -> dict[str, Any]: try: runtime = json.loads(native.start_runtime(plan_json)) except FabricError as error: + cleanup_runtime_handle = getattr( + error, "_fabric_cleanup_runtime_handle", None + ) _retain_runtime_evidence(error, observations) raise except Exception as error: + cleanup_runtime_handle = getattr( + error, "_fabric_cleanup_runtime_handle", None + ) failure = _runtime_error(error, stage="start") _retain_runtime_evidence(failure, observations) raise failure from error diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index ab1c93b41..716be5aa6 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -461,3 +461,41 @@ async def smoke(client: Fabric, fixture_agent: Path) -> None: assert empty_chunks == [] assert empty_result.status == "succeeded" assert empty_result.output["openai_stream_invocation_count"] == 2 + + +@pytest.mark.skipif( + sys.platform == "win32", reason="Unix host directory repair fixture" +) +@pytest.mark.parametrize("entry", ["start_runtime", "run"]) +async def test_native_failed_start_exposes_a_private_cleanup_retry( + blocking_adapter_config, tmp_path, entry +): + from nemo_fabric import RuntimeStatus + + script = tmp_path / "adapter.py" + source = script.read_text() + anchor = " if failure.exists():\n" + replacement = """ if failure.exists(): + if operation == 'start': + import shutil + directory = Path(os.environ['FABRIC_HOME']) + (root/'cleanup.path').write_text(str(directory)) + shutil.rmtree(directory) + directory.write_text('cleanup requires repair') +""" + assert source.count(anchor) == 1 + script.write_text(source.replace(anchor, replacement)) + (tmp_path / "fail-start").write_text("fixture_start_failure") + with pytest.raises(FabricRuntimeError) as caught: + await getattr(Fabric(), entry)(blocking_adapter_config, base_dir=tmp_path) + recovery = caught.value.cleanup_runtime + assert recovery is not None + assert caught.value.code == "cleanup_unconfirmed" + assert "runtime_binding" not in repr(caught.value) + assert "PRIVATE detail" not in str(caught.value) + directory = Path((tmp_path / "cleanup.path").read_text()) + assert directory.is_file() + directory.unlink() + directory.mkdir(mode=0o700) + await recovery.stop() + assert recovery.status is RuntimeStatus.STOPPED diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index 8017b689e..8499a2ce0 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -1896,3 +1896,46 @@ async def caller(): later_work.set() await task await stopping + + +@pytest.mark.parametrize("entry", ["start_runtime", "run", "prepare_service"]) +@pytest.mark.parametrize("cancel_start", [False, True]) +async def test_failed_start_preserves_private_cleanup_runtime( + native_client, mock_native, entry, cancel_start +): + entered, release = threading.Event(), threading.Event() + handle = _runtime() + handle["environment"]["env"] = {"TEST_STARTUP_SECRET": "private-startup-value"} + + def start(*args): + entered.set() + assert release.wait(5) + error = RuntimeError("cleanup unconfirmed") + error.code = "cleanup_unconfirmed" + error._fabric_cleanup_runtime_handle = handle + raise error + + method = "prepare_service" if entry == "prepare_service" else "start_runtime" + getattr(mock_native, method).side_effect = start + task = asyncio.create_task(getattr(native_client, entry)(_config())) + try: + async with asyncio.timeout(2): + while not entered.is_set(): + await asyncio.sleep(0.01) + if cancel_start: + task.cancel() + release.set() + expected = asyncio.CancelledError if cancel_start else FabricRuntimeError + with pytest.raises(expected) as caught: + await task + recovery = caught.value.cleanup_runtime + assert isinstance(recovery, Runtime) + assert recovery.runtime_id == handle["runtime_id"] + assert "private-startup-value" not in repr(caught.value) + if isinstance(caught.value, FabricRuntimeError): + assert "private-startup-value" not in json.dumps(caught.value.details) + await recovery.stop() + assert recovery.status is RuntimeStatus.STOPPED + finally: + release.set() + await asyncio.gather(task, return_exceptions=True) From 053a0077e6481d96c40f2002eef1c2730d0e5424 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Fri, 13 Nov 2026 13:17:44 -0700 Subject: [PATCH 7/7] fix: distinguish requested runtime stops from host crashes Signed-off-by: Sara Tadayon --- crates/fabric-core/src/runtime.rs | 284 +++++++++++++++--- .../adapter-contract/index.mdx | 2 +- .../nemo-fabric-core/agent-config/index.mdx | 2 +- .../agent-execution/index.mdx | 2 +- .../nemo-fabric-core/check/index.mdx | 2 +- .../nemo-fabric-core/config/index.mdx | 2 +- .../nemo-fabric-core/doctor/index.mdx | 2 +- .../error/enum-fabricerror.mdx | 8 +- .../nemo-fabric-core/error/index.mdx | 6 +- .../nemo-fabric-core/fn-version.mdx | 2 +- .../nemo-fabric-core/provenance/index.mdx | 2 +- .../resource-snapshot/index.mdx | 2 +- .../nemo-fabric-core/runtime/index.mdx | 2 +- .../nemo-fabric-core/schema/index.mdx | 2 +- .../nemo-fabric-core/session/index.mdx | 2 +- .../nemo-fabric-core/system-stats/index.mdx | 2 +- docs/sdk/python.mdx | 2 +- environment-providers/openshell/src/lib.rs | 1 - .../src/nemo_fabric/runtime_server/README.md | 2 +- .../src/nemo_fabric/runtime_server/state.py | 2 - tests/python/runtime_server/test_sdk.py | 4 +- .../runtime_server/test_session_commands.py | 15 +- 22 files changed, 289 insertions(+), 61 deletions(-) diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index 45d9a1942..eaa6d4134 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -1180,7 +1180,7 @@ pub fn start_runtime_with_service( record .active_runtime_ids .insert(context.runtime().runtime_id.clone()); - } else let FabricError::AdapterLifecycleOperation { + } else if let FabricError::AdapterLifecycleOperation { code, runtime_id, .. } = &error && code == "cleanup_unconfirmed" @@ -2006,7 +2006,15 @@ impl RuntimeAdapter for LocalHostAdapter { .lock() .unwrap_or_else(|error| error.into_inner()) .remove(&runtime.runtime_id); - return Ok(vec![local_host_stop_event(runtime, true, false)]); + return Ok(vec![local_host_stop_event( + runtime, + true, + false, + INTERRUPTED_RUNTIMES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .remove(&runtime.runtime_id), + )]); }; let deadline = Instant::now() + LOCAL_HOST_STOP_TIMEOUT; let mut interrupted = None; @@ -2026,7 +2034,9 @@ impl RuntimeAdapter for LocalHostAdapter { } }; guard.cleanup_pending = true; - PENDING_LOCAL_CLEANUP.lock().unwrap_or_else(|error| error.into_inner()) + PENDING_LOCAL_CLEANUP + .lock() + .unwrap_or_else(|error| error.into_inner()) .insert(runtime.runtime_id.clone(), guard.cleanup_scope.clone()); if guard.cleanup_discovery_pending { interrupted = Some(interrupt_local_host(runtime)?); @@ -2035,7 +2045,6 @@ impl RuntimeAdapter for LocalHostAdapter { if frozen_tree_is_pending(&runtime.runtime_id) { interrupted = Some(interrupt_local_host(runtime)?); } - let was_interrupted = interrupted.is_some(); if let Some(processes) = interrupted { guard.pending_stop.extend(processes); } @@ -2084,14 +2093,20 @@ impl RuntimeAdapter for LocalHostAdapter { .lock() .unwrap_or_else(|error| error.into_inner()) .remove(&runtime.runtime_id); - PENDING_LOCAL_CLEANUP.lock().unwrap_or_else(|error| error.into_inner()) + PENDING_LOCAL_CLEANUP + .lock() + .unwrap_or_else(|error| error.into_inner()) .remove(&runtime.runtime_id); - let host_crashed = was_interrupted - || matches!( + let was_interrupted = INTERRUPTED_RUNTIMES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .remove(&runtime.runtime_id); + let host_crashed = !was_interrupted + && matches!( &result, Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "host_crashed" ); - if !host_crashed { + if !host_crashed && !was_interrupted { result?; } #[cfg(test)] @@ -2099,7 +2114,12 @@ impl RuntimeAdapter for LocalHostAdapter { .lock() .expect("stop tracker") .push(runtime.agent_name.clone()); - Ok(vec![local_host_stop_event(runtime, false, host_crashed)]) + Ok(vec![local_host_stop_event( + runtime, + false, + host_crashed, + was_interrupted, + )]) } } @@ -2284,6 +2304,12 @@ fn interrupt_local_host(runtime: &RuntimeHandle) -> Result .lock() .unwrap_or_else(|error| error.into_inner()) .insert(runtime.runtime_id.clone(), descendants.clone()); + if !owned_process_has_exited(&process) { + INTERRUPTED_RUNTIMES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(runtime.runtime_id.clone()); + } for child in descendants.iter().rev() { signal_owned_process(child).map_err(|_| cleanup_unconfirmed(runtime))?; } @@ -2296,6 +2322,12 @@ fn interrupt_local_host(runtime: &RuntimeHandle) -> Result .get(&runtime.runtime_id) .cloned() .ok_or_else(|| cleanup_unconfirmed(runtime))?; + if control.has_exited() == Some(false) { + INTERRUPTED_RUNTIMES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(runtime.runtime_id.clone()); + } control .interrupt() .map_err(|_| cleanup_unconfirmed(runtime))?; @@ -2505,10 +2537,44 @@ fn retry_pending_local_cleanup( ) }; terminate_local_host(&mut host).map_err(unconfirmed)?; + #[cfg(unix)] + let discovery_pending = frozen_tree_is_pending(&runtime_id); + #[cfg(not(unix))] + let discovery_pending = false; + #[cfg(windows)] + let job_pending = host.control.cleanup_confirmed() != Some(true); + #[cfg(not(windows))] + let job_pending = false; + if discovery_pending + || job_pending + || host + .pending_stop + .iter() + .any(|process| !owned_process_has_exited(process)) + { + return Err(unconfirmed(std::io::Error::other( + "owned cleanup remains unconfirmed", + ))); + } + // Termination and every retained ownership check succeeded. A stale + // pre-termination flag adds no further evidence of live owned work. + host.cleanup_discovery_pending = false; remove_local_host_files(&host).map_err(unconfirmed)?; host._state_lock.take(); local_hosts().remove(&runtime_id); local_host_processes().remove(&runtime_id); + #[cfg(unix)] + frozen_process_trees().remove(&runtime_id); + #[cfg(windows)] + windows_host_controls().remove(&runtime_id); + STOPPING_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .remove(&runtime_id); + INTERRUPTED_RUNTIMES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .remove(&runtime_id); } PENDING_LOCAL_CLEANUP .lock() @@ -2729,6 +2795,15 @@ fn run_local_host_invocation_with_timeout( "native session identity is no longer confirmed", )); } + if host_guard.invocation_unavailable { + return Err(lifecycle_error( + operation, + &runtime.runtime_id, + "host_unavailable", + "The retained host is available only for cleanup.", + "", + )); + } if host_guard.cleanup_pending { return Err(lifecycle_error( operation, @@ -2744,15 +2819,6 @@ fn run_local_host_invocation_with_timeout( "runtime handle does not match its bound native session", )); } - if host_guard.invocation_unavailable { - return Err(lifecycle_error( - operation, - &runtime.runtime_id, - "host_unavailable", - "The retained host is available only for cleanup.", - "", - )); - } let artifacts = host_guard.artifacts.clone(); let relay_config = host_guard.relay_config.clone(); let fabric_home = prepare_fabric_home(&artifacts, runtime, &invocation)?; @@ -2932,6 +2998,22 @@ fn run_local_host_invocation_with_timeout( )) } Err(error) => { + let error = if matches!(&error, FabricError::AdapterLifecycleOperation {code, ..} if code == "host_crashed" || code == "host_timeout") + && INTERRUPTED_RUNTIMES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .contains(&runtime.runtime_id) + { + lifecycle_error( + operation, + &runtime.runtime_id, + "interrupted", + "The invocation was interrupted by a requested runtime stop.", + "", + ) + } else { + error + }; if matches!( &error, FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" @@ -3225,6 +3307,7 @@ fn local_host_stop_event( runtime: &RuntimeHandle, already_stopped: bool, host_crashed: bool, + interrupted: bool, ) -> FabricEvent { event_with_metadata( "runtime_stop", @@ -3236,6 +3319,14 @@ fn local_host_stop_event( ), ("already_stopped".to_string(), Value::Bool(already_stopped)), ("host_crashed".to_string(), Value::Bool(host_crashed)), + ( + "code".to_string(), + if interrupted { + Value::String("interrupted".into()) + } else { + Value::Null + }, + ), ]), ) } @@ -3243,10 +3334,15 @@ fn local_host_stop_event( fn stop_service_host(record: &mut LocalServiceRecord) -> Result<()> { record.host.cleanup_pending = true; #[cfg(unix)] - remember_host_descendants(&mut record.host).map_err(|_| lifecycle_error( - AdapterLifecycleOperation::Stop, &record.runtime.runtime_id, "cleanup_unconfirmed", - "Owned service descendants could not be inspected.", "", - ))?; + remember_host_descendants(&mut record.host).map_err(|_| { + lifecycle_error( + AdapterLifecycleOperation::Stop, + &record.runtime.runtime_id, + "cleanup_unconfirmed", + "Owned service descendants could not be inspected.", + "", + ) + })?; let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Stop(AdapterLifecycleStop { runtime_id: record.runtime.runtime_id.clone(), @@ -3315,6 +3411,9 @@ fn service_release_event(service: &ServiceHandle, already_released: bool) -> Fab } // Keep identity-checked snapshots even if interruption fails before the host lock is released. +static INTERRUPTED_RUNTIMES: LazyLock>> = + LazyLock::new(|| Mutex::new(BTreeSet::new())); + static STOPPING_PROCESSES: LazyLock>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); @@ -3926,6 +4025,34 @@ fn retain_failed_start_with( if terminated && remove_local_host_files(&host).is_ok() { return cause; } + let diagnostic_path = host.runtime_dir.join("startup-error.log"); + let mut options = File::options(); + options.create_new(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let diagnostics = options + .open(&diagnostic_path) + .and_then(|mut file| { + file.write_all(cause.to_string().as_bytes())?; + file.sync_all() + }) + .map(|()| diagnostic_path.display().to_string()) + .unwrap_or_else(|_| "Startup diagnostics could not be saved".to_string()); + let cause = lifecycle_error( + AdapterLifecycleOperation::Start, + &runtime.runtime_id, + "cleanup_unconfirmed", + "failed start left native cleanup unconfirmed", + diagnostics, + ); + host.cleanup_pending = true; + PENDING_LOCAL_CLEANUP + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(runtime.runtime_id.clone(), host.cleanup_scope.clone()); host.invocation_unavailable = true; host.cleanup_discovery_pending |= !terminated; #[cfg(windows)] @@ -4033,6 +4160,7 @@ fn remember_host_descendants(host: &mut LocalAdapterHost) -> std::io::Result<()> .push(LocalHostProcess::new(pid as u32, SystemTime::now())); } } + host.cleanup_discovery_pending = false; Ok(()) } @@ -6014,7 +6142,10 @@ else: panic!("connected startup must retain cleanup context"); }; let release = release_service(&plan, &service); - assert!(matches!(release, Err(FabricError::ServiceInUse { .. }))); + assert!( + matches!(release, Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "cleanup_unconfirmed") + ); + assert!(local_hosts().contains_key(&context.runtime().runtime_id)); let directory = local_host_directory(&context.runtime().runtime_id); assert!(directory.is_file()); fs::remove_file(&directory).unwrap(); @@ -6358,6 +6489,39 @@ else: fs::remove_dir_all(root).unwrap(); } + #[test] + fn requested_busy_stop_is_interrupted_instead_of_crashed() { + let (root, plan) = local_host_plan("invoke_timeout"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let stderr = host.lock().unwrap().stderr_path.clone(); + let invocation_plan = plan.clone(); + let invocation_runtime = runtime.clone(); + let invocation = thread::spawn(move || { + invoke_runtime( + &invocation_plan, + &invocation_runtime, + RunRequest::text("wait"), + ) + }); + let deadline = Instant::now() + Duration::from_secs(5); + while !fs::read_to_string(&stderr) + .unwrap() + .contains("invoke accepted") + { + assert!(Instant::now() < deadline, "invocation did not start"); + thread::sleep(Duration::from_millis(10)); + } + let events = stop_runtime(&plan, &runtime).unwrap(); + let error = invocation.join().unwrap().unwrap_err(); + fs::remove_dir_all(root).unwrap(); + assert!( + matches!(error, FabricError::AdapterLifecycleOperation {code, ..} if code == "interrupted") + ); + assert_eq!(events[0].metadata["host_crashed"], false); + assert_eq!(events[0].metadata["code"], "interrupted"); + } + #[test] fn busy_stop_interrupts_the_owned_host_without_the_invocation_lock() { let (root, plan) = local_host_plan("child_after_start"); @@ -6623,6 +6787,44 @@ else: let _ = fs::remove_dir_all(root); } + #[cfg(unix)] + #[test] + fn automatic_scope_cleanup_retains_unfinished_frozen_discovery() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let scope = host.lock().unwrap().cleanup_scope.clone(); + PENDING_LOCAL_CLEANUP + .lock() + .unwrap() + .insert(runtime.runtime_id.clone(), scope.clone()); + let process = local_host_processes() + .get(&runtime.runtime_id) + .unwrap() + .clone(); + let discovery = collect_frozen_host_with(&runtime, &process, |_| { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + }); + assert!(discovery.is_err()); + signal_owned_process(&process).unwrap(); + host.lock().unwrap().child.wait().unwrap(); + let retry = retry_pending_local_cleanup(&scope, AdapterLifecycleOperation::Start); + assert!( + matches!(retry, Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "cleanup_unconfirmed") + ); + assert!(local_hosts().contains_key(&runtime.runtime_id)); + assert!( + PENDING_LOCAL_CLEANUP + .lock() + .unwrap() + .contains_key(&runtime.runtime_id) + ); + // This fixture had no children. Clear only its injected incomplete discovery. + frozen_process_trees().remove(&runtime.runtime_id); + stop_runtime(&plan, &runtime).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn failed_one_shot_stop_cleanup_is_retried_before_replacement() { for (keep_state, prepare_replacement) in @@ -6634,9 +6836,18 @@ else: local_host_plan("success") }; FAIL_NEXT_HOST_TERMINATION.with(|fail| fail.set(true)); - let result = run_plan(&plan, RunRequest::text("completed")).unwrap(); - assert_eq!(result.status, RunStatus::Failed); - assert_eq!(result.error.as_ref().unwrap().stage, ErrorStage::Stop); + let error = run_plan(&plan, RunRequest::text("completed")).unwrap_err(); + let FabricError::RuntimeCleanupPending(pending) = error else { + panic!("expected retained cleanup context") + }; + let result = pending + .result() + .expect("completed invocation remains available"); + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.runtime_id, pending.runtime().runtime_id); + assert!( + matches!(pending.cause(), FabricError::AdapterLifecycleOperation {operation: "stop", code, ..} if code == "cleanup_unconfirmed") + ); assert!(local_hosts().contains_key(&result.runtime_id)); FAIL_NEXT_HOST_TERMINATION.with(|fail| fail.set(true)); @@ -6715,13 +6926,14 @@ else: FAIL_NEXT_HOST_TERMINATION.with(|fail| fail.set(true)); let error = start_runtime(&plan).unwrap_err(); let message = error.to_string(); - let (failed_id, diagnostics) = match error { + let FabricError::RuntimeCleanupPending(pending) = error else { + panic!("expected retained startup cleanup context") + }; + let failed_id = pending.runtime().runtime_id.clone(); + let diagnostics = match pending.cause() { FabricError::AdapterLifecycleOperation { - code, - runtime_id, - diagnostics, - .. - } if code == "cleanup_unconfirmed" => (runtime_id, diagnostics), + code, diagnostics, .. + } if code == "cleanup_unconfirmed" => diagnostics.clone(), other => panic!("unexpected startup failure: {other}"), }; let diagnostic_path = PathBuf::from(diagnostics); @@ -6767,12 +6979,10 @@ else: let service = prepare_service(&plan).unwrap(); FAIL_NEXT_HOST_TERMINATION.with(|fail| fail.set(true)); let error = start_runtime_with_service(&plan, &service).unwrap_err(); - let failed_id = match error { - FabricError::AdapterLifecycleOperation { - code, runtime_id, .. - } if code == "cleanup_unconfirmed" => runtime_id, - other => panic!("unexpected startup failure: {other}"), + let FabricError::RuntimeCleanupPending(pending) = error else { + panic!("expected retained connected startup context") }; + let failed_id = pending.runtime().runtime_id.clone(); let record = local_services().get(&service.service_id).cloned().unwrap(); assert!( record diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index fe950a53e..a613e4f64 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 165 +position: 166 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 71104c85b..48f092f67 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 166 +position: 167 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index ba24f43ed..6194d6aac 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 167 +position: 168 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/check/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/check/index.mdx index 56b4d0dad..1a17c31b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/check/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/check/index.mdx @@ -2,7 +2,7 @@ title: "Module check" sidebar-title: "check" description: "Runtime health checks and shared contracts." -position: 168 +position: 169 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index c685b867f..2542912d3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 169 +position: 170 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index c71073030..db748d3db 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 170 +position: 171 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 23376c571..926a51ed3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n        message: String,\n    },\n    ResolveBaseDirectory {\n        path: PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    StateInUse {\n        path: PathBuf,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
String,\n        message: String,\n    },\n    ResolveBaseDirectory {\n        path: PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeCleanupPending(Box<RuntimeCleanupPending>),\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    StateInUse {\n        path: PathBuf,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -421,6 +421,12 @@ Human-readable failure message. Bounded adapter-host diagnostics. +### `RuntimeCleanupPending(Box)` + +
Box<RuntimeCleanupPending>)"}} />
+ +Owned lifecycle cleanup is unconfirmed; the original runtime remains owned. + ### `RuntimeHandleMismatch`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index d2862663b..9670e4008 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 171 +position: 172 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -11,6 +11,10 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Error types for NeMo Fabric core. +## Structs + +- [RuntimeCleanupPending](struct-runtimecleanuppending.mdx): Cleanup context retained when startup or invocation cannot finish owned cleanup. + ## Enums - [FabricError](enum-fabricerror.mdx): Errors raised by NeMo Fabric config loading and validation. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 763e6e69c..c1bd3a056 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 178 +position: 179 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/provenance/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/provenance/index.mdx index 92aa1c299..4909b0a85 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/provenance/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/provenance/index.mdx @@ -2,7 +2,7 @@ title: "Module provenance" sidebar-title: "provenance" description: "Selected software evidence. Optional evidence never controls invocation success." -position: 172 +position: 173 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index c1deacf18..e784207c4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -3,7 +3,7 @@ title: "Module resource_snapshot" sidebar-title: "resource_snapshot" slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" description: "Resource snapshots of a runtime's local adapter host and its descendants." -position: 173 +position: 174 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index af0211128..7683d4dd7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 174 +position: 175 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 4268f3f13..d5f5661cf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 175 +position: 176 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/session/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/session/index.mdx index 34a05fadb..6ccad6fd2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/session/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/session/index.mdx @@ -2,7 +2,7 @@ title: "Module session" sidebar-title: "session" description: "Explicit selection of durable harness-native sessions." -position: 176 +position: 177 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx index cc4e9639a..f7c93c8bb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx @@ -3,7 +3,7 @@ title: "Module system_stats" sidebar-title: "system_stats" slug: "/reference/api/rust-library-reference/nemo-fabric-core/system_stats" description: "Invocation measurements collected by NeMo Fabric in the executing runtime." -position: 177 +position: 178 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index c81c232e1..7c9aa27f8 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -890,7 +890,7 @@ Hermes Agent reuses one agent and session database, and Claude keeps one connect client. Durable Hermes conversations expose their native identifier inside an opaque `SessionRef`. NeMo Fabric does not interpret that identifier. -Buffered runtime shutdown can interrupt an active invocation on supported Unix and Windows hosts. Windows shutdown requires an owned process job to confirm descendant cleanup. `Runtime.stop()` waits for the interrupted invocation to settle before reporting stopped. Cleanup errors retain ownership for retry; the runtime server blocks replacement for the same agent until cleanup succeeds. Streaming shutdown continues to use the stream lifecycle. +Buffered runtime shutdown can interrupt an active invocation on supported Unix and Windows hosts. Windows shutdown requires an owned process job to confirm descendant cleanup. `Runtime.stop()` waits for the interrupted invocation to settle before reporting stopped. Cleanup errors retain ownership for retry; the runtime server blocks replacement for the same agent until cleanup succeeds. Streaming shutdown continues to use the stream lifecycle. An invocation interrupted by a requested stop reports `interrupted`; genuine host crashes still report `host_crashed`. Cleanup confirms processes that remain in Fabric’s owned host tree. The sandbox supervisor must terminate processes that detached before Fabric captured that tree. If `Fabric.run()`, `Fabric.start_runtime()`, `Fabric.prepare_service()`, or `Fabric.attach_service()` cannot confirm cleanup after a failed lifecycle operation, the raised error provides `cleanup_runtime`. Keep that object and retry `await error.cleanup_runtime.stop()` in the same process. Cancellation can carry the same recovery object. The recovery handle is excluded from error text and structured details. When an exited parent leaves undiscoverable children, cleanup remains unconfirmed and the owning process or sandbox must recover. diff --git a/environment-providers/openshell/src/lib.rs b/environment-providers/openshell/src/lib.rs index f3e2a6529..5e7525e6d 100644 --- a/environment-providers/openshell/src/lib.rs +++ b/environment-providers/openshell/src/lib.rs @@ -564,7 +564,6 @@ fn valid_diagnostic_code(code: &str) -> bool { | "session_resume_unsupported" | "runtime_busy" | "cleanup_unconfirmed" - | "runtime_busy" | "invalid_request" | "interface_unsupported" | "wrong_agent" diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md index 211ea0ed5..624226d08 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md @@ -81,7 +81,7 @@ flowchart TD The outer server is separate from the SDK's adapter process and the selected harness. It serializes starts and invocations for each runtime ID, while separate runtimes can progress independently. Stop uses a separate path so an active buffered invocation cannot hold shutdown behind its lock. On supported Unix hosts, busy stop interrupts the owned adapter process tree and confirms exit before releasing ownership. Unconfirmed cleanup retains the runtime and blocks replacement for that agent; independent agents remain available. Direct SDK consumers can also use streaming or shared services. -Ordinary SDK stop does not establish that all harness descendants exited. A successful stop response reports completion of the SDK operation. The caller must use the sandbox supervisor to terminate remaining descendants. The server does not interrupt adapters through a new forced-stop SDK API. +A confirmed stop covers processes that remain in Fabric’s owned host tree. Processes detached before Fabric captures that tree can remain alive; the caller must use the sandbox supervisor to terminate those processes. An invocation interrupted by a requested stop reports `interrupted`, and the stop event does not report a host crash. ## Transport And Failure Behavior diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py index ff81125c0..b46e31bf3 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py @@ -226,8 +226,6 @@ async def handle(self, request: Any) -> dict[str, Any]: owned = self._owned(request) if operation == "stop" and owned.state != "stopped": - if owned.lock.locked(): - raise ProtocolError("runtime_busy", "stop") owned.generation += 1 owned.state = "stopping" if operation == "stop": diff --git a/tests/python/runtime_server/test_sdk.py b/tests/python/runtime_server/test_sdk.py index b1d5956b4..3dc8d2811 100644 --- a/tests/python/runtime_server/test_sdk.py +++ b/tests/python/runtime_server/test_sdk.py @@ -312,7 +312,9 @@ async def test_real_sdk_busy_stop_exits_host_and_native_child( host.handle({**command, "operation": "stop"}), 5 ) assert response["status"] == "succeeded", response - await asyncio.wait_for(task, 1) + invoked = await asyncio.wait_for(task, 1) + assert invoked["error"]["code"] == "invoke_failed" + assert "Cause: interrupted." in invoked["error"]["message"] assert all(not process_is_running(pid) for pid in pids) assert host.runtimes[runtime_id].state == "stopped" finally: diff --git a/tests/python/runtime_server/test_session_commands.py b/tests/python/runtime_server/test_session_commands.py index 83e201ac4..bb2bb7280 100644 --- a/tests/python/runtime_server/test_session_commands.py +++ b/tests/python/runtime_server/test_session_commands.py @@ -118,7 +118,7 @@ async def test_known_start_rejection_can_be_corrected( assert (await host.handle(host_request("start")))["status"] == "succeeded" -async def test_busy_stop_returns_promptly_and_retains_ownership( +async def test_busy_stop_failure_returns_promptly_and_retains_ownership( host, host_request, mock_runtime ): await host.handle(host_request("start")) @@ -130,16 +130,25 @@ async def invoke(**kwargs): return mock_runtime.invoke.return_value mock_runtime.invoke.side_effect = invoke + mock_runtime.stop.side_effect = FabricRuntimeError( + "PRIVATE cleanup detail", code="cleanup_unconfirmed" + ) task = asyncio.create_task(host.handle(host_request("invoke", input={}))) await entered.wait() try: response = await asyncio.wait_for(host.handle(host_request("stop")), 0.5) - assert response["error"]["code"] == "runtime_busy" + assert response["error"]["code"] == "stop_failed" + assert response["error"]["effects"] == "unknown" + assert "PRIVATE" not in str(response) + assert (await host.handle(host_request("start")))["error"][ + "code" + ] == "cleanup_unconfirmed" assert host.runtimes["runtime-1"].runtime is mock_runtime - mock_runtime.stop.assert_not_awaited() + mock_runtime.stop.assert_awaited_once() finally: release.set() await task + mock_runtime.stop.side_effect = None assert (await host.handle(host_request("stop")))["status"] == "succeeded"