diff --git a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py index 3537f8a09..85667a739 100644 --- a/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py +++ b/adapter-contract/python/src/nemo_fabric_adapter_contract/models.py @@ -71,6 +71,8 @@ def _bounded_int(value: int | None, field_name: str, maximum: int) -> None: CheckReason = Literal[ "server_stopping", "capabilities_unverified", + "runtime_busy", + "cleanup_unconfirmed", "runtime_unavailable", "service_unavailable", "agent_exited", diff --git a/adapter-contract/typescript/schemas/adapter-check-result.schema.json b/adapter-contract/typescript/schemas/adapter-check-result.schema.json index eb405c683..1d3814a12 100644 --- a/adapter-contract/typescript/schemas/adapter-check-result.schema.json +++ b/adapter-contract/typescript/schemas/adapter-check-result.schema.json @@ -38,6 +38,16 @@ "description": "Advertised capability evidence is unavailable.", "type": "string" }, + { + "const": "runtime_busy", + "description": "Runtime cleanup is in progress.", + "type": "string" + }, + { + "const": "cleanup_unconfirmed", + "description": "Owned runtime cleanup could not be confirmed.", + "type": "string" + }, { "const": "runtime_unavailable", "description": "No matching runtime binding exists.", diff --git a/adapter-contract/typescript/src/generated/adapter-check-result.ts b/adapter-contract/typescript/src/generated/adapter-check-result.ts index dd28ad262..aa15f8cc2 100644 --- a/adapter-contract/typescript/src/generated/adapter-check-result.ts +++ b/adapter-contract/typescript/src/generated/adapter-check-result.ts @@ -83,6 +83,8 @@ export type AdapterCheckResult = { export type CheckReason = | "server_stopping" | "capabilities_unverified" + | "runtime_busy" + | "cleanup_unconfirmed" | "runtime_unavailable" | "service_unavailable" | "agent_exited" diff --git a/crates/fabric-core/src/check.rs b/crates/fabric-core/src/check.rs index de4fb1ae0..da1a572b3 100644 --- a/crates/fabric-core/src/check.rs +++ b/crates/fabric-core/src/check.rs @@ -94,6 +94,10 @@ pub enum CheckReason { ServerStopping, /// Advertised capability evidence is unavailable. CapabilitiesUnverified, + /// Runtime cleanup is in progress. + RuntimeBusy, + /// Owned runtime cleanup could not be confirmed. + CleanupUnconfirmed, /// No matching runtime binding exists. RuntimeUnavailable, /// No matching active service binding exists. diff --git a/crates/fabric-core/src/error.rs b/crates/fabric-core/src/error.rs index f816691d4..424dc5cab 100644 --- a/crates/fabric-core/src/error.rs +++ b/crates/fabric-core/src/error.rs @@ -235,6 +235,9 @@ pub enum FabricError { /// Bounded adapter-host diagnostics. diagnostics: String, }, + /// Owned lifecycle cleanup is unconfirmed; the original runtime remains owned. + #[error("{0}")] + RuntimeCleanupPending(Box), /// A runtime handle was used with a different run plan than the one that created it. #[error( "runtime handle does not match run plan for `{field}`: expected `{expected}` but found `{actual}` (runtime `{runtime_id}`)" @@ -436,3 +439,58 @@ pub enum FabricError { source: serde_json::Error, }, } + +/// Cleanup context retained when startup or invocation cannot finish owned cleanup. +/// +/// Retry [`crate::stop_runtime`] with the original plan and [`Self::runtime`]. +/// The handle and completed output are deliberately excluded from diagnostic formatting. +pub struct RuntimeCleanupPending { + runtime: crate::RuntimeHandle, + result: Option, + cause: FabricError, +} + +impl RuntimeCleanupPending { + pub(crate) fn new( + runtime: crate::RuntimeHandle, + result: Option, + cause: FabricError, + ) -> Self { + Self { + runtime, + result, + cause, + } + } + + /// The original handle required for a cleanup retry in this process. + pub fn runtime(&self) -> &crate::RuntimeHandle { + &self.runtime + } + + /// The completed invocation result, if invocation succeeded before cleanup failed. + pub fn result(&self) -> Option<&crate::RunResult> { + self.result.as_ref() + } + + /// The startup, invocation, or stop error that preceded the retained cleanup context. + pub fn cause(&self) -> &FabricError { + &self.cause + } +} + +impl std::fmt::Debug for RuntimeCleanupPending { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RuntimeCleanupPending") + .field("result_available", &self.result.is_some()) + .finish_non_exhaustive() + } +} + +impl std::fmt::Display for RuntimeCleanupPending { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str( + "runtime cleanup is unconfirmed; retry stop with RuntimeCleanupPending::runtime()", + ) + } +} diff --git a/crates/fabric-core/src/lib.rs b/crates/fabric-core/src/lib.rs index 6e5b02350..88433c8d3 100644 --- a/crates/fabric-core/src/lib.rs +++ b/crates/fabric-core/src/lib.rs @@ -52,7 +52,7 @@ pub use config::{ resolve_run_plan_from_config_with_adapter_directories, }; pub use doctor::{DoctorCheck, DoctorReport, DoctorStatus, doctor_plan}; -pub use error::{FabricError, Result}; +pub use error::{FabricError, Result, RuntimeCleanupPending}; pub use provenance::{ExecutionEvidence, ProvenanceConfig}; pub use resource_snapshot::RuntimeResourceSnapshot; pub use runtime::{ diff --git a/crates/fabric-core/src/resource_snapshot.rs b/crates/fabric-core/src/resource_snapshot.rs index 219154d6d..9d9b71ce7 100644 --- a/crates/fabric-core/src/resource_snapshot.rs +++ b/crates/fabric-core/src/resource_snapshot.rs @@ -89,6 +89,12 @@ impl LocalHostProcess { platform::has_exited(self.pid, self.identity) } + /// Whether the same process is suspended and cannot create children. + #[cfg(unix)] + pub(crate) fn is_stopped(&self) -> Option { + platform::is_stopped(self.pid, self.identity?) + } + /// Record the host `pid`, started by NeMo Fabric at `started_at`. pub(crate) fn new(pid: u32, started_at: SystemTime) -> Self { Self { @@ -328,6 +334,7 @@ mod platform { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(super) struct Stat { pub zombie: bool, + pub stopped: bool, pub ppid: u32, /// `utime + stime + cutime + cstime`, in clock ticks. pub cpu_ticks: u64, @@ -348,6 +355,7 @@ mod platform { .sum::>()?; Some(Stat { zombie: fields.first() == Some(&"Z"), + stopped: matches!(fields.first(), Some(&"T" | &"t")), ppid: u32::try_from(number(1)?).ok()?, cpu_ticks: ticks, start: number(19)?, @@ -540,6 +548,11 @@ mod platform { } } + pub(super) fn is_stopped(pid: u32, identity: u64) -> Option { + let stat = stat(pid).ok()?; + Some(stat.start == identity && stat.stopped) + } + fn sysconf(name: libc::c_int) -> u64 { // SAFETY: sysconf has no memory-safety preconditions. u64::try_from(unsafe { libc::sysconf(name) }).unwrap_or(0) @@ -731,6 +744,26 @@ mod platform { } } + pub(super) fn is_stopped(pid: u32, identity: u64) -> Option { + // SAFETY: proc_bsdinfo is an output structure with a valid zero representation. + let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() }; + let size = std::mem::size_of_val(&info) as libc::c_int; + // SAFETY: the writable buffer matches PROC_PIDTBSDINFO and its size. + let read = unsafe { + libc::proc_pidinfo( + pid as libc::c_int, + libc::PROC_PIDTBSDINFO, + 0, + (&mut info as *mut libc::proc_bsdinfo).cast(), + size, + ) + }; + if read != size || start_identity(pid) != Some(identity) { + return None; + } + Some(info.pbi_status == libc::SSTOP) + } + pub(super) fn read_tree(root: u32, identity: u64) -> Result { let timebase = timebase().ok_or(PROCESS_UNREADABLE)?; read_tree_with(&Live { timebase }, root, identity) @@ -751,6 +784,11 @@ mod platform { None } + #[cfg(unix)] + pub(super) fn is_stopped(_pid: u32, _identity: u64) -> Option { + None + } + pub(super) fn read_tree(_root: u32, _identity: u64) -> Result { Err("no process reader for this operating system") } diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index d084b6126..eaa6d4134 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -131,6 +131,9 @@ static LOCAL_HOSTS: LazyLock> // whole invocation, so a resource snapshot reads the PID from here instead. static LOCAL_HOST_PROCESSES: LazyLock>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); +#[cfg(windows)] +static WINDOWS_HOST_CONTROLS: LazyLock>>> = + LazyLock::new(|| Mutex::new(BTreeMap::new())); static LOCAL_SERVICES: LazyLock>>>> = LazyLock::new(|| Mutex::new(BTreeMap::new())); @@ -947,8 +950,11 @@ struct LocalAdapterHost { execution: Option, adapter_execution: Option, child: Child, + invocation_unavailable: bool, + cleanup_discovery_pending: bool, + pending_stop: Vec, #[cfg(windows)] - job: Option, + control: Arc, stdin: ChildStdin, responses: Receiver>, command: String, @@ -997,16 +1003,30 @@ enum LocalServiceState { } /// Invoke a NeMo Fabric run plan. +/// +/// If stopping leaves owned cleanup pending, returns +/// [`FabricError::RuntimeCleanupPending`] with the original runtime handle and +/// any completed result. Retry [`stop_runtime`] with that handle and this plan. pub fn run_plan(plan: &RunPlan, request: RunRequest) -> Result { let runtime = start_runtime(plan)?; let mut result = match invoke_runtime(plan, &runtime, request) { Ok(result) => result, Err(error) => { let _ = stop_runtime(plan, &runtime); + if local_runtime_is_retained(&runtime.runtime_id) { + return Err(FabricError::RuntimeCleanupPending(Box::new( + crate::RuntimeCleanupPending::new(runtime, None, error), + ))); + } return Err(error); } }; match stop_runtime(plan, &runtime) { + Err(error) if local_runtime_is_retained(&runtime.runtime_id) => { + return Err(FabricError::RuntimeCleanupPending(Box::new( + crate::RuntimeCleanupPending::new(runtime, Some(result), error), + ))); + } Ok(events) => result.events.extend(events), Err(error) if result.status == RunStatus::Succeeded => { result.status = RunStatus::Failed; @@ -1156,7 +1176,11 @@ pub fn start_runtime_with_service( let runtime = match started { Ok(runtime) => runtime, Err(error) => { - if let FabricError::AdapterLifecycleOperation { + if let FabricError::RuntimeCleanupPending(context) = &error { + record + .active_runtime_ids + .insert(context.runtime().runtime_id.clone()); + } else if let FabricError::AdapterLifecycleOperation { code, runtime_id, .. } = &error && code == "cleanup_unconfirmed" @@ -1282,23 +1306,21 @@ fn start_service( LOCAL_HOST_START_TIMEOUT, ) { Ok(output) => output, - Err(error) => { - let _ = terminate_local_host(&mut host); - let _ = remove_local_host_files(&host); - return Err(error); + Err(error) => return Err(retain_failed_start(&runtime, host, error)), + }; + let info: AdapterServiceInfo = match serde_json::from_value(output) { + Ok(info) => info, + Err(source) => { + let cause = lifecycle_error( + AdapterLifecycleOperation::Start, + &service_id, + "invalid_service_info", + format!("adapter returned invalid service information: {source}"), + local_host_diagnostics(&host), + ); + return Err(retain_failed_start(&runtime, host, cause)); } }; - let info: AdapterServiceInfo = serde_json::from_value(output).map_err(|source| { - let _ = terminate_local_host(&mut host); - let _ = remove_local_host_files(&host); - lifecycle_error( - AdapterLifecycleOperation::Start, - &service_id, - "invalid_service_info", - format!("adapter returned invalid service information: {source}"), - local_host_diagnostics(&host), - ) - })?; let software_report = if plan.config.provenance.enabled { host.adapter_execution .as_ref() @@ -1523,7 +1545,7 @@ pub fn stop_runtime(plan: &RunPlan, runtime: &RuntimeHandle) -> Result Result> { - let host = { local_hosts().get(&runtime.runtime_id).cloned() }; - let Some(host) = host else { + let Some(host) = local_hosts().get(&runtime.runtime_id).cloned() else { + #[cfg(not(windows))] + let process = local_host_processes().get(&runtime.runtime_id).cloned(); + #[cfg(not(windows))] + if process + .as_ref() + .is_some_and(|process| !owned_process_has_exited(process)) + { + return Err(cleanup_unconfirmed(runtime)); + } + let pending = STOPPING_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .get(&runtime.runtime_id) + .cloned() + .unwrap_or_default(); + confirm_stopped_descendants(runtime, &pending)?; local_host_processes().remove(&runtime.runtime_id); - return Ok(vec![local_host_stop_event(runtime, true, false)]); + #[cfg(unix)] + frozen_process_trees().remove(&runtime.runtime_id); + #[cfg(windows)] + windows_host_controls().remove(&runtime.runtime_id); + STOPPING_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .remove(&runtime.runtime_id); + return Ok(vec![local_host_stop_event( + runtime, + true, + false, + INTERRUPTED_RUNTIMES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .remove(&runtime.runtime_id), + )]); + }; + let deadline = Instant::now() + LOCAL_HOST_STOP_TIMEOUT; + let mut interrupted = None; + let mut guard = loop { + match host.try_lock() { + Ok(guard) => break guard, + Err(std::sync::TryLockError::Poisoned(error)) => break error.into_inner(), + Err(std::sync::TryLockError::WouldBlock) => { + if interrupted.is_none() { + interrupted = Some(interrupt_local_host(runtime)?); + } + if Instant::now() >= deadline { + return Err(cleanup_unconfirmed(runtime)); + } + thread::sleep(Duration::from_millis(10)); + } + } }; - let mut host = host.lock().unwrap_or_else(|error| error.into_inner()); - host.cleanup_pending = true; + guard.cleanup_pending = true; + PENDING_LOCAL_CLEANUP + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(runtime.runtime_id.clone(), guard.cleanup_scope.clone()); + if guard.cleanup_discovery_pending { + interrupted = Some(interrupt_local_host(runtime)?); + } + #[cfg(unix)] + if frozen_tree_is_pending(&runtime.runtime_id) { + interrupted = Some(interrupt_local_host(runtime)?); + } + if let Some(processes) = interrupted { + guard.pending_stop.extend(processes); + } + if let Some(processes) = STOPPING_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .get(&runtime.runtime_id) + { + guard.pending_stop.extend(processes.clone()); + } #[cfg(unix)] - remember_host_descendants(&mut host); + if guard + .child + .try_wait() + .map_err(|_| cleanup_unconfirmed(runtime))? + .is_none() + { + let pid = guard.child.id(); + guard.pending_stop.extend( + host_descendants(pid) + .map_err(|_| cleanup_unconfirmed(runtime))? + .into_iter() + .map(|pid| LocalHostProcess::new(pid as u32, SystemTime::now())), + ); + } let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Stop(AdapterLifecycleStop { runtime_id: runtime.runtime_id.clone(), })); let result = exchange_lifecycle_message( - &mut host, + &mut guard, &runtime.runtime_id, &request, LOCAL_HOST_STOP_TIMEOUT, ); - let termination = terminate_local_host(&mut host); - let diagnostics = local_host_diagnostics(&host); - let host_crashed = matches!( - &result, - Err(FabricError::AdapterLifecycleOperation { code, .. }) - if code == "host_crashed" - ); - termination.map_err(|source| { - PENDING_LOCAL_CLEANUP - .lock() - .unwrap_or_else(|error| error.into_inner()) - .insert(runtime.runtime_id.clone(), host.cleanup_scope.clone()); - lifecycle_error( - AdapterLifecycleOperation::Stop, - &runtime.runtime_id, - "cleanup_unconfirmed", - format!("persistent local adapter host could not be terminated: {source}"), - diagnostics, - ) - })?; - // Keep the host and its lock available for a retry until termination - // has been confirmed. A failed adapter stop may still be safely killed. + terminate_local_host(&mut guard).map_err(|_| cleanup_unconfirmed(runtime))?; + confirm_stopped_descendants(runtime, &guard.pending_stop)?; + remove_local_host_files(&guard).map_err(|_| cleanup_unconfirmed(runtime))?; local_hosts().remove(&runtime.runtime_id); local_host_processes().remove(&runtime.runtime_id); + #[cfg(unix)] + frozen_process_trees().remove(&runtime.runtime_id); + #[cfg(windows)] + windows_host_controls().remove(&runtime.runtime_id); + STOPPING_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .remove(&runtime.runtime_id); PENDING_LOCAL_CLEANUP .lock() .unwrap_or_else(|error| error.into_inner()) .remove(&runtime.runtime_id); - let removal = remove_local_host_files(&host); - if !host_crashed { + let was_interrupted = INTERRUPTED_RUNTIMES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .remove(&runtime.runtime_id); + let host_crashed = !was_interrupted + && matches!( + &result, + Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "host_crashed" + ); + if !host_crashed && !was_interrupted { result?; } - removal.map_err(|source| { - lifecycle_error( - AdapterLifecycleOperation::Stop, - &runtime.runtime_id, - "host_cleanup_failed", - format!("persistent local adapter host files could not be removed: {source}"), - "", - ) - })?; - #[cfg(test)] TEST_STOPPED_AGENTS .lock() .expect("stop tracker") .push(runtime.agent_name.clone()); - Ok(vec![local_host_stop_event(runtime, false, host_crashed)]) + Ok(vec![local_host_stop_event( + runtime, + false, + host_crashed, + was_interrupted, + )]) + } +} + +fn local_runtime_is_retained(runtime_id: &str) -> bool { + let retained = + local_hosts().contains_key(runtime_id) || local_host_processes().contains_key(runtime_id); + #[cfg(windows)] + let retained = retained || windows_host_controls().contains_key(runtime_id); + retained +} + +fn cleanup_unconfirmed(runtime: &RuntimeHandle) -> FabricError { + lifecycle_error( + AdapterLifecycleOperation::Stop, + &runtime.runtime_id, + "cleanup_unconfirmed", + "Owned process cleanup is unconfirmed; retry stop before replacement.", + "", + ) +} + +fn owned_process_has_exited(process: &LocalHostProcess) -> bool { + if process.has_exited() == Some(true) { + return true; + } + #[cfg(unix)] + { + let Ok(pid) = libc::pid_t::try_from(process.pid()) else { + return false; + }; + if pid <= 0 { + return false; + } + // SAFETY: signal zero only probes existence. An unidentified process is + // never signalled; ESRCH can still prove that its PID is now absent. + (unsafe { libc::kill(pid, 0) } != 0) + && std::io::Error::last_os_error().raw_os_error() == Some(libc::ESRCH) + } + #[cfg(not(unix))] + false +} + +#[cfg(unix)] +fn signal_owned_process(process: &LocalHostProcess) -> std::io::Result<()> { + if owned_process_has_exited(process) { + return Ok(()); + } + if !process.is_current() { + return Err(std::io::Error::other("process ownership is unconfirmed")); + } + // SAFETY: the process identity was checked immediately before signalling. + if unsafe { libc::kill(process.pid() as libc::pid_t, libc::SIGKILL) } != 0 { + let error = std::io::Error::last_os_error(); + if error.raw_os_error() != Some(libc::ESRCH) { + return Err(error); + } + } + Ok(()) +} + +#[cfg(unix)] +fn freeze_owned_process(process: &LocalHostProcess, deadline: Instant) -> std::io::Result<()> { + if owned_process_has_exited(process) { + return Ok(()); + } + if !process.is_current() { + return Err(std::io::Error::other("process ownership is unconfirmed")); + } + // SAFETY: the process identity was checked immediately before signalling. + if unsafe { libc::kill(process.pid() as libc::pid_t, libc::SIGSTOP) } != 0 { + return Err(std::io::Error::last_os_error()); + } + loop { + if process.is_stopped() == Some(true) || owned_process_has_exited(process) { + return Ok(()); + } + if !process.is_current() || Instant::now() >= deadline { + return Err(std::io::Error::other("process suspension is unconfirmed")); + } + thread::sleep(Duration::from_millis(10)); + } +} + +#[cfg(unix)] +struct FrozenProcessTree { + pending: Vec, + processes: Vec, + lost_parent: bool, +} + +#[cfg(unix)] +static FROZEN_PROCESS_TREES: LazyLock>>>> = + LazyLock::new(|| Mutex::new(BTreeMap::new())); + +#[cfg(unix)] +fn frozen_process_trees() +-> std::sync::MutexGuard<'static, BTreeMap>>> { + FROZEN_PROCESS_TREES + .lock() + .unwrap_or_else(|error| error.into_inner()) +} + +#[cfg(unix)] +fn frozen_tree_is_pending(runtime_id: &str) -> bool { + let tree = frozen_process_trees().get(runtime_id).cloned(); + tree.is_some_and(|tree| { + let tree = tree.lock().unwrap_or_else(|error| error.into_inner()); + tree.lost_parent || !tree.pending.is_empty() + }) +} + +#[cfg(unix)] +fn frozen_tree_error(runtime: &RuntimeHandle, tree: &FrozenProcessTree) -> FabricError { + if tree.lost_parent { + // Missing descendants still prevent confirmation, but known frozen work + // must not be left suspended. Keep every identity for later inspection. + for process in tree.processes.iter().rev() { + let _ = signal_owned_process(process); + } + } + cleanup_unconfirmed(runtime) +} + +#[cfg(unix)] +fn collect_frozen_host_with( + runtime: &RuntimeHandle, + process: &LocalHostProcess, + mut children: impl FnMut(libc::pid_t) -> std::io::Result>, +) -> Result> { + let tree = frozen_process_trees() + .entry(runtime.runtime_id.clone()) + .or_insert_with(|| { + Arc::new(Mutex::new(FrozenProcessTree { + pending: vec![process.clone()], + processes: vec![process.clone()], + lost_parent: false, + })) + }) + .clone(); + let mut tree = tree.lock().unwrap_or_else(|error| error.into_inner()); + if tree.lost_parent { + return Err(frozen_tree_error(runtime, &tree)); + } + let deadline = Instant::now() + LOCAL_HOST_STOP_TIMEOUT; + while let Some(parent) = tree.pending.last().cloned() { + if freeze_owned_process(&parent, deadline).is_err() { + tree.lost_parent |= owned_process_has_exited(&parent); + return Err(frozen_tree_error(runtime, &tree)); + } + if owned_process_has_exited(&parent) { + tree.lost_parent = true; + return Err(frozen_tree_error(runtime, &tree)); + } + let listed = + children(parent.pid() as libc::pid_t).map_err(|_| cleanup_unconfirmed(runtime))?; + // An exited parent can have already lost its children to reparenting. + // Keep that missing evidence across retries rather than guessing empty. + if owned_process_has_exited(&parent) { + tree.lost_parent = true; + return Err(frozen_tree_error(runtime, &tree)); + } + let listed: Vec<_> = listed + .into_iter() + .map(|pid| LocalHostProcess::new(pid as u32, SystemTime::now())) + .collect(); + tree.pending.pop(); + tree.processes.extend(listed.clone()); + tree.pending.extend(listed); + } + Ok(tree.processes.clone()) +} + +fn interrupt_local_host(runtime: &RuntimeHandle) -> Result> { + #[cfg(unix)] + { + let process = local_host_processes() + .get(&runtime.runtime_id) + .cloned() + .ok_or_else(|| cleanup_unconfirmed(runtime))?; + let descendants = collect_frozen_host_with(runtime, &process, child_pids)?; + STOPPING_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(runtime.runtime_id.clone(), descendants.clone()); + if !owned_process_has_exited(&process) { + INTERRUPTED_RUNTIMES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(runtime.runtime_id.clone()); + } + for child in descendants.iter().rev() { + signal_owned_process(child).map_err(|_| cleanup_unconfirmed(runtime))?; + } + signal_owned_process(&process).map_err(|_| cleanup_unconfirmed(runtime))?; + Ok(descendants) + } + #[cfg(windows)] + { + let control = windows_host_controls() + .get(&runtime.runtime_id) + .cloned() + .ok_or_else(|| cleanup_unconfirmed(runtime))?; + if control.has_exited() == Some(false) { + INTERRUPTED_RUNTIMES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(runtime.runtime_id.clone()); + } + control + .interrupt() + .map_err(|_| cleanup_unconfirmed(runtime))?; + Ok(Vec::new()) + } + #[cfg(not(any(unix, windows)))] + Err(cleanup_unconfirmed(runtime)) +} + +fn confirm_stopped_descendants( + runtime: &RuntimeHandle, + processes: &[LocalHostProcess], +) -> Result<()> { + #[cfg(unix)] + if frozen_tree_is_pending(&runtime.runtime_id) { + return Err(cleanup_unconfirmed(runtime)); + } + #[cfg(windows)] + let control = windows_host_controls().get(&runtime.runtime_id).cloned(); + let deadline = Instant::now() + LOCAL_HOST_EXIT_GRACE; + loop { + let exited = processes.iter().all(owned_process_has_exited); + #[cfg(windows)] + let exited = exited + && control + .as_ref() + .is_none_or(|control| control.cleanup_confirmed() == Some(true)); + if exited { + return Ok(()); + } + #[cfg(unix)] + for process in processes { + signal_owned_process(process).map_err(|_| cleanup_unconfirmed(runtime))?; + } + #[cfg(windows)] + if let Some(control) = &control + && control.job.is_some() + { + control + .interrupt() + .map_err(|_| cleanup_unconfirmed(runtime))?; + } + if Instant::now() >= deadline { + return Err(cleanup_unconfirmed(runtime)); + } + thread::sleep(Duration::from_millis(10)); } } @@ -2126,43 +2475,10 @@ fn start_local_runtime( .and_then(|output| confirm_native_session(plan, &mut runtime, &mut host, output)); if let Err(error) = started { crate::check::invalidate_runtime(&runtime); - if terminate_local_host(&mut host).is_err() { - host.cleanup_pending = true; - // Startup errors can contain credentials. Keep details private and - // return only the path while cleanup ownership remains retained. - let diagnostic_path = host.runtime_dir.join("startup-error.log"); - let mut options = File::options(); - options.create_new(true).write(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600); - } - let diagnostics = options - .open(&diagnostic_path) - .and_then(|mut file| { - file.write_all(error.to_string().as_bytes())?; - file.sync_all() - }) - .map(|()| diagnostic_path.display().to_string()) - .unwrap_or_else(|_| "Startup diagnostics could not be saved".to_string()); - let scope = host.cleanup_scope.clone(); - local_hosts().insert(runtime.runtime_id.clone(), Arc::new(Mutex::new(host))); - PENDING_LOCAL_CLEANUP - .lock() - .unwrap_or_else(|error| error.into_inner()) - .insert(runtime.runtime_id.clone(), scope); - return Err(lifecycle_error( - AdapterLifecycleOperation::Start, - &runtime.runtime_id, - "cleanup_unconfirmed", - "failed start left native cleanup unconfirmed", - diagnostics, - )); - } - let _ = remove_local_host_files(&host); - return Err(error); + return Err(retain_failed_start(&runtime, host, error)); } + #[cfg(windows)] + windows_host_controls().insert(runtime.runtime_id.clone(), host.control.clone()); local_host_processes().insert( runtime.runtime_id.clone(), LocalHostProcess::new(host.child.id(), started_at), @@ -2221,10 +2537,44 @@ fn retry_pending_local_cleanup( ) }; terminate_local_host(&mut host).map_err(unconfirmed)?; + #[cfg(unix)] + let discovery_pending = frozen_tree_is_pending(&runtime_id); + #[cfg(not(unix))] + let discovery_pending = false; + #[cfg(windows)] + let job_pending = host.control.cleanup_confirmed() != Some(true); + #[cfg(not(windows))] + let job_pending = false; + if discovery_pending + || job_pending + || host + .pending_stop + .iter() + .any(|process| !owned_process_has_exited(process)) + { + return Err(unconfirmed(std::io::Error::other( + "owned cleanup remains unconfirmed", + ))); + } + // Termination and every retained ownership check succeeded. A stale + // pre-termination flag adds no further evidence of live owned work. + host.cleanup_discovery_pending = false; remove_local_host_files(&host).map_err(unconfirmed)?; host._state_lock.take(); local_hosts().remove(&runtime_id); local_host_processes().remove(&runtime_id); + #[cfg(unix)] + frozen_process_trees().remove(&runtime_id); + #[cfg(windows)] + windows_host_controls().remove(&runtime_id); + STOPPING_PROCESSES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .remove(&runtime_id); + INTERRUPTED_RUNTIMES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .remove(&runtime_id); } PENDING_LOCAL_CLEANUP .lock() @@ -2445,6 +2795,15 @@ fn run_local_host_invocation_with_timeout( "native session identity is no longer confirmed", )); } + if host_guard.invocation_unavailable { + return Err(lifecycle_error( + operation, + &runtime.runtime_id, + "host_unavailable", + "The retained host is available only for cleanup.", + "", + )); + } if host_guard.cleanup_pending { return Err(lifecycle_error( operation, @@ -2639,6 +2998,22 @@ fn run_local_host_invocation_with_timeout( )) } Err(error) => { + let error = if matches!(&error, FabricError::AdapterLifecycleOperation {code, ..} if code == "host_crashed" || code == "host_timeout") + && INTERRUPTED_RUNTIMES + .lock() + .unwrap_or_else(|error| error.into_inner()) + .contains(&runtime.runtime_id) + { + lifecycle_error( + operation, + &runtime.runtime_id, + "interrupted", + "The invocation was interrupted by a requested runtime stop.", + "", + ) + } else { + error + }; if matches!( &error, FabricError::AdapterLifecycleOperation { code, .. } if code == "host_timeout" @@ -2843,9 +3218,20 @@ fn invalidate_timed_out_local_host( runtime_id: &str, expected_host: &Arc>, host: &mut LocalAdapterHost, +) { + invalidate_timed_out_local_host_with(runtime_id, expected_host, host, terminate_local_host); +} + +fn invalidate_timed_out_local_host_with( + runtime_id: &str, + expected_host: &Arc>, + host: &mut LocalAdapterHost, + terminate: impl FnOnce(&mut LocalAdapterHost) -> std::io::Result<()>, ) { host.cleanup_pending = true; - if terminate_local_host(host).is_err() { + host.invocation_unavailable = true; + if terminate(host).is_err() { + host.cleanup_discovery_pending = true; return; } host.cleanup_pending = false; @@ -2921,6 +3307,7 @@ fn local_host_stop_event( runtime: &RuntimeHandle, already_stopped: bool, host_crashed: bool, + interrupted: bool, ) -> FabricEvent { event_with_metadata( "runtime_stop", @@ -2932,6 +3319,14 @@ fn local_host_stop_event( ), ("already_stopped".to_string(), Value::Bool(already_stopped)), ("host_crashed".to_string(), Value::Bool(host_crashed)), + ( + "code".to_string(), + if interrupted { + Value::String("interrupted".into()) + } else { + Value::Null + }, + ), ]), ) } @@ -2939,7 +3334,15 @@ fn local_host_stop_event( fn stop_service_host(record: &mut LocalServiceRecord) -> Result<()> { record.host.cleanup_pending = true; #[cfg(unix)] - remember_host_descendants(&mut record.host); + remember_host_descendants(&mut record.host).map_err(|_| { + lifecycle_error( + AdapterLifecycleOperation::Stop, + &record.runtime.runtime_id, + "cleanup_unconfirmed", + "Owned service descendants could not be inspected.", + "", + ) + })?; let request = AdapterLifecycleRequest::new(AdapterLifecycleRequestKind::Stop(AdapterLifecycleStop { runtime_id: record.runtime.runtime_id.clone(), @@ -3007,6 +3410,13 @@ fn service_release_event(service: &ServiceHandle, already_released: bool) -> Fab ) } +// Keep identity-checked snapshots even if interruption fails before the host lock is released. +static INTERRUPTED_RUNTIMES: LazyLock>> = + LazyLock::new(|| Mutex::new(BTreeSet::new())); + +static STOPPING_PROCESSES: LazyLock>>> = + LazyLock::new(|| Mutex::new(BTreeMap::new())); + fn local_hosts() -> std::sync::MutexGuard<'static, BTreeMap>>> { LOCAL_HOSTS .lock() @@ -3022,16 +3432,116 @@ fn local_host_processes() -> std::sync::MutexGuard<'static, BTreeMap Option { - let process = local_host_processes().get(runtime_id).cloned()?; - process.has_exited() + #[cfg(windows)] + { + let control = windows_host_controls().get(runtime_id).cloned()?; + control.has_exited() + } + #[cfg(not(windows))] + { + let process = local_host_processes().get(runtime_id).cloned()?; + process.has_exited() + } } -fn local_services() --> std::sync::MutexGuard<'static, BTreeMap>>> { - LOCAL_SERVICES - .lock() - .unwrap_or_else(|error| error.into_inner()) -} +// Keep owned handles outside the invocation lock and after timeout eviction. +// Handles continue to identify the same kernel objects even if PIDs are reused. +#[cfg(windows)] +struct WindowsHostControl { + process: std::os::windows::io::OwnedHandle, + job: Option, +} + +#[cfg(windows)] +impl WindowsHostControl { + fn capture( + child: &Child, + job: Option, + ) -> std::io::Result { + use std::os::windows::io::AsHandle; + let job = + job.ok_or_else(|| std::io::Error::other("owned Windows process job is unavailable"))?; + Ok(Self { + process: child.as_handle().try_clone_to_owned()?, + job: Some(job), + }) + } + + fn has_exited(&self) -> Option { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::Foundation::{WAIT_OBJECT_0, WAIT_TIMEOUT}; + // SAFETY: this owned process handle stays valid throughout the probe. + match unsafe { + windows_sys::Win32::System::Threading::WaitForSingleObject( + self.process.as_raw_handle(), + 0, + ) + } { + WAIT_OBJECT_0 => Some(true), + WAIT_TIMEOUT => Some(false), + _ => None, + } + } + + fn cleanup_confirmed(&self) -> Option { + use std::os::windows::io::AsRawHandle; + use windows_sys::Win32::System::JobObjects::{ + JOBOBJECT_BASIC_ACCOUNTING_INFORMATION, JobObjectBasicAccountingInformation, + QueryInformationJobObject, + }; + match self.has_exited() { + Some(true) => {} + other => return other, + } + let Some(job) = &self.job else { + return None; + }; + // SAFETY: this is a plain output structure with no invalid bit patterns. + let mut accounting: JOBOBJECT_BASIC_ACCOUNTING_INFORMATION = unsafe { std::mem::zeroed() }; + // SAFETY: the job handle is owned; the writable buffer matches the class and size. + let queried = unsafe { + QueryInformationJobObject( + job.as_raw_handle(), + JobObjectBasicAccountingInformation, + (&mut accounting as *mut JOBOBJECT_BASIC_ACCOUNTING_INFORMATION).cast(), + std::mem::size_of_val(&accounting) as u32, + std::ptr::null_mut(), + ) + }; + (queried != 0).then_some(accounting.ActiveProcesses == 0) + } + + fn interrupt(&self) -> std::io::Result<()> { + use std::os::windows::io::AsRawHandle; + let job = self + .job + .as_ref() + .ok_or_else(|| std::io::Error::other("owned Windows process tree is unavailable"))?; + // SAFETY: this job handle belongs to the runtime, including its descendants. + if unsafe { + windows_sys::Win32::System::JobObjects::TerminateJobObject(job.as_raw_handle(), 1) + } == 0 + { + return Err(std::io::Error::last_os_error()); + } + Ok(()) + } +} + +#[cfg(windows)] +fn windows_host_controls() +-> std::sync::MutexGuard<'static, BTreeMap>> { + WINDOWS_HOST_CONTROLS + .lock() + .unwrap_or_else(|error| error.into_inner()) +} + +fn local_services() +-> std::sync::MutexGuard<'static, BTreeMap>>> { + LOCAL_SERVICES + .lock() + .unwrap_or_else(|error| error.into_inner()) +} pub(crate) fn local_host_directory(id: &str) -> PathBuf { std::env::temp_dir() @@ -3127,7 +3637,18 @@ fn spawn_local_host( } }; #[cfg(windows)] - let job = host_job(&child); + let control = match WindowsHostControl::capture(&child, host_job(&child)) { + Ok(control) => Arc::new(control), + Err(source) => { + let _ = child.kill(); + let _ = child.wait(); + let _ = std::fs::remove_dir_all(&runtime_dir); + return Err(FabricError::ProcessRunner { + command: command_display, + source, + }); + } + }; #[cfg(windows)] if let Err(source) = resume_host(child.id()) { let _ = child.kill(); @@ -3206,8 +3727,11 @@ fn spawn_local_host( execution: crate::provenance::core_snapshot(&plan.config.provenance), adapter_execution: None, child, + invocation_unavailable: false, + cleanup_discovery_pending: false, + pending_stop: Vec::new(), #[cfg(windows)] - job, + control, stdin, responses, command: command_display, @@ -3482,20 +4006,80 @@ fn take_local_host_stderr(host: &mut LocalAdapterHost) -> String { String::from_utf8_lossy(&bytes[start..]).into_owned() } +fn retain_failed_start( + runtime: &RuntimeHandle, + host: LocalAdapterHost, + cause: FabricError, +) -> FabricError { + retain_failed_start_with(runtime, host, cause, terminate_local_host) +} + +fn retain_failed_start_with( + runtime: &RuntimeHandle, + mut host: LocalAdapterHost, + cause: FabricError, + terminate: impl FnOnce(&mut LocalAdapterHost) -> std::io::Result<()>, +) -> FabricError { + let process = LocalHostProcess::new(host.child.id(), SystemTime::now()); + let terminated = terminate(&mut host).is_ok(); + if terminated && remove_local_host_files(&host).is_ok() { + return cause; + } + let diagnostic_path = host.runtime_dir.join("startup-error.log"); + let mut options = File::options(); + options.create_new(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let diagnostics = options + .open(&diagnostic_path) + .and_then(|mut file| { + file.write_all(cause.to_string().as_bytes())?; + file.sync_all() + }) + .map(|()| diagnostic_path.display().to_string()) + .unwrap_or_else(|_| "Startup diagnostics could not be saved".to_string()); + let cause = lifecycle_error( + AdapterLifecycleOperation::Start, + &runtime.runtime_id, + "cleanup_unconfirmed", + "failed start left native cleanup unconfirmed", + diagnostics, + ); + host.cleanup_pending = true; + PENDING_LOCAL_CLEANUP + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(runtime.runtime_id.clone(), host.cleanup_scope.clone()); + host.invocation_unavailable = true; + host.cleanup_discovery_pending |= !terminated; + #[cfg(windows)] + windows_host_controls().insert(runtime.runtime_id.clone(), host.control.clone()); + local_host_processes().insert(runtime.runtime_id.clone(), process); + local_hosts().insert(runtime.runtime_id.clone(), Arc::new(Mutex::new(host))); + FabricError::RuntimeCleanupPending(Box::new(crate::RuntimeCleanupPending::new( + runtime.clone(), + None, + cause, + ))) +} + fn terminate_local_host(host: &mut LocalAdapterHost) -> std::io::Result<()> { #[cfg(test)] if tests::FAIL_NEXT_HOST_TERMINATION.with(|fail| fail.replace(false)) { return Err(std::io::Error::other("simulated host termination failure")); } #[cfg(unix)] - remember_host_descendants(host); + remember_host_descendants(host)?; let deadline = Instant::now() + LOCAL_HOST_EXIT_GRACE; while host.child.try_wait()?.is_none() && Instant::now() < deadline { thread::sleep(Duration::from_millis(10)); } #[cfg(unix)] { - remember_host_descendants(host); + remember_host_descendants(host)?; for process in &host.descendants { if process.has_exited() == Some(true) { continue; @@ -3518,7 +4102,7 @@ fn terminate_local_host(host: &mut LocalAdapterHost) -> std::io::Result<()> { } } #[cfg(windows)] - if let Some(job) = &host.job { + if let Some(job) = &host.control.job { // SAFETY: the host owns this job handle until it is dropped. if unsafe { windows_sys::Win32::System::JobObjects::TerminateJobObject( @@ -3559,14 +4143,14 @@ fn terminate_local_host(host: &mut LocalAdapterHost) -> std::io::Result<()> { } #[cfg(unix)] -fn remember_host_descendants(host: &mut LocalAdapterHost) { +fn remember_host_descendants(host: &mut LocalAdapterHost) -> std::io::Result<()> { // An exited host's PID can be reused. Never discover children from that PID. if !matches!(host.child.try_wait(), Ok(None)) { - return; + return Ok(()); } host.descendants .retain(|process| process.has_exited() != Some(true)); - for pid in host_descendants(host.child.id()) { + for pid in host_descendants(host.child.id())? { if !host .descendants .iter() @@ -3576,85 +4160,99 @@ fn remember_host_descendants(host: &mut LocalAdapterHost) { .push(LocalHostProcess::new(pid as u32, SystemTime::now())); } } + host.cleanup_discovery_pending = false; + Ok(()) } /// Lists every live descendant of a host, including children that started /// their own session. Processes that already detached from the host's process -/// tree cannot be found this way, so cleanup stays best-effort. +/// tree cannot be found this way. Listing failures must remain unconfirmed. #[cfg(unix)] -fn host_descendants(host_pid: u32) -> Vec { - let Ok(host_pid) = libc::pid_t::try_from(host_pid) else { - return Vec::new(); - }; +fn host_descendants(host_pid: u32) -> std::io::Result> { + host_descendants_with(host_pid, child_pids) +} + +#[cfg(unix)] +fn host_descendants_with( + host_pid: u32, + mut children: impl FnMut(libc::pid_t) -> std::io::Result>, +) -> std::io::Result> { + let host_pid = libc::pid_t::try_from(host_pid) + .map_err(|_| std::io::Error::other("process identity is invalid"))?; // Snapshot the whole tree first: killed processes' children are re-parented. let mut descendants = Vec::new(); let mut parents = vec![host_pid]; while let Some(parent) = parents.pop() { - let children = child_pids(parent); - parents.extend(&children); - descendants.extend(children); + let listed = children(parent)?; + parents.extend(&listed); + descendants.extend(listed); } - descendants + Ok(descendants) } #[cfg(target_os = "linux")] -fn child_pids(parent: libc::pid_t) -> Vec { - let Ok(entries) = std::fs::read_dir("/proc") else { - return Vec::new(); - }; - entries - .flatten() - .filter_map(|entry| { - let pid = entry.file_name().to_str()?.parse().ok()?; - let stat = std::fs::read_to_string(entry.path().join("stat")).ok()?; - // The command name may contain spaces or parentheses; the parent PID - // is the second field after it. - let ppid: libc::pid_t = stat - .rsplit_once(')')? - .1 - .split_whitespace() - .nth(1)? - .parse() - .ok()?; - (ppid == parent).then_some(pid) - }) - .collect() +fn child_pids(parent: libc::pid_t) -> std::io::Result> { + let mut children = Vec::new(); + for entry in std::fs::read_dir("/proc")? { + let entry = entry?; + let Some(pid) = entry + .file_name() + .to_str() + .and_then(|value| value.parse().ok()) + else { + continue; + }; + let stat = match std::fs::read_to_string(entry.path().join("stat")) { + Ok(stat) => stat, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(error), + }; + let ppid = stat + .rsplit_once(')') + .and_then(|(_, fields)| fields.split_whitespace().nth(1)) + .and_then(|value| value.parse::().ok()) + .ok_or_else(|| std::io::Error::other("process parent is unreadable"))?; + if ppid == parent { + children.push(pid); + } + } + Ok(children) } #[cfg(target_os = "macos")] -fn child_pids(parent: libc::pid_t) -> Vec { +fn child_pids(parent: libc::pid_t) -> std::io::Result> { let mut children = vec![0; 64]; loop { - let Ok(bytes) = i32::try_from(std::mem::size_of_val(children.as_slice())) else { - return Vec::new(); - }; + let bytes = i32::try_from(std::mem::size_of_val(children.as_slice())) + .map_err(|_| std::io::Error::other("process child list is too large"))?; // SAFETY: the buffer is writable and sized in bytes as libproc requires. let count = unsafe { libc::proc_listchildpids(parent, children.as_mut_ptr().cast(), bytes) }; - let Ok(count) = usize::try_from(count) else { - return Vec::new(); - }; + let count = usize::try_from(count).map_err(|_| std::io::Error::last_os_error())?; if count < children.len() { children.truncate(count); - return children; + return Ok(children); } children.resize(children.len() * 2, 0); } } #[cfg(all(unix, not(any(target_os = "linux", target_os = "macos"))))] -fn child_pids(_parent: libc::pid_t) -> Vec { - Vec::new() +fn child_pids(_parent: libc::pid_t) -> std::io::Result> { + Err(std::io::Error::from(std::io::ErrorKind::Unsupported)) } /// Puts the suspended host in a job object so a forced kill reaches every /// process it starts, even through launchers or other intermediates that -/// already exited. Like Cargo, fall back to killing only the host if the job -/// is unavailable. +/// already exited. Closing the last job handle also terminates its processes. #[cfg(windows)] fn host_job(child: &Child) -> Option { use std::os::windows::io::{AsRawHandle, FromRawHandle, OwnedHandle}; - use windows_sys::Win32::System::JobObjects::{AssignProcessToJobObject, CreateJobObjectW}; + use windows_sys::Win32::System::JobObjects::{ + AssignProcessToJobObject, CreateJobObjectW, JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, + JOBOBJECT_EXTENDED_LIMIT_INFORMATION, JobObjectExtendedLimitInformation, + SetInformationJobObject, + }; // SAFETY: null security attributes and name are allowed; null means failure. let job = unsafe { CreateJobObjectW(std::ptr::null(), std::ptr::null()) }; @@ -3663,6 +4261,21 @@ fn host_job(child: &Child) -> Option { } // SAFETY: CreateJobObjectW returned a new handle that nothing else owns. let job = unsafe { OwnedHandle::from_raw_handle(job) }; + // SAFETY: this job information structure accepts an all-zero initialization. + let mut limits: JOBOBJECT_EXTENDED_LIMIT_INFORMATION = unsafe { std::mem::zeroed() }; + limits.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + // SAFETY: the owned job and readable buffer match the information class and size. + if unsafe { + SetInformationJobObject( + job.as_raw_handle(), + JobObjectExtendedLimitInformation, + (&limits as *const JOBOBJECT_EXTENDED_LIMIT_INFORMATION).cast(), + std::mem::size_of_val(&limits) as u32, + ) + } == 0 + { + return None; + } // SAFETY: both handles stay valid for the duration of the call. let assigned = unsafe { AssignProcessToJobObject(job.as_raw_handle(), child.as_raw_handle()) }; (assigned != 0).then_some(job) @@ -4985,6 +5598,14 @@ for line in sys.stdin: message = json.loads(line) operation = message["operation"] if operation == "start": + if MODE == "connected_start_cleanup_failure" and message["payload"].get("service", {}).get("operation") == "connect": + import shutil + directory = os.environ["FABRIC_HOME"] + shutil.rmtree(directory) + with open(directory, "w", encoding="utf-8") as stream: + stream.write("startup cleanup requires a retry") + response("start", error=failure("start", "fixture_start_failure", "start rejected")) + continue if MODE == "start_failure": print("start diagnostic", file=sys.stderr, flush=True) response("start", error=failure("start", "fake_start", "start rejected")) @@ -4997,6 +5618,20 @@ for line in sys.stdin: root = os.path.dirname(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"]) with open(os.path.join(root, "child.pid"), "w", encoding="utf-8") as stream: stream.write(str(child.pid)) + if MODE == "child_after_start": + root = os.path.dirname(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"]) + ready = os.path.join(root, "child.ready") + child = subprocess.Popen( + [sys.executable, "-c", "import pathlib,sys,time; pathlib.Path(sys.argv[1]).touch(); time.sleep(30)", ready], + stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + deadline = time.monotonic() + 5 + while not os.path.exists(ready): + if child.poll() is not None or time.monotonic() >= deadline: + raise RuntimeError("fixture child did not become ready") + time.sleep(0.01) + with open(os.path.join(root, "child.pid"), "w", encoding="utf-8") as stream: + stream.write(str(child.pid)) service = message["payload"].get("service") if MODE == "connect_failure_once" and service and service["operation"] == "connect": failure_marker = os.environ["FABRIC_FAKE_HOST_STOP_MARKER"] + ".connect-failed" @@ -5035,7 +5670,7 @@ for line in sys.stdin: # Exit during Fabric's grace wait and leave the child running. time.sleep(1) os._exit(0) - if MODE == "invoke_failure": + if MODE in {"invoke_failure", "invoke_failure_cleanup_failure"}: response("invoke", error=failure("invoke", "fake_invoke", "invoke rejected")) continue invocation = message["payload"] @@ -5165,6 +5800,12 @@ for line in sys.stdin: child.stdin.write(json.dumps({"operation": "stop", "outcome": {"status": "succeeded", "output": None}}) + "\n") child.stdin.flush() os._exit(0) + if MODE in {"cleanup_failure", "invoke_failure_cleanup_failure"}: + import shutil + directory = os.environ["FABRIC_HOME"] + shutil.rmtree(directory) + with open(directory, "w", encoding="utf-8") as stream: + stream.write("cleanup requires a retry") if MODE == "stop_delay": with open(os.environ["FABRIC_FAKE_HOST_STOP_MARKER"], "w", encoding="utf-8") as stream: stream.write("started") @@ -5410,6 +6051,549 @@ else: TEST_STOPPED_AGENTS.lock().expect("stop tracker").clone() } + #[cfg(unix)] + #[test] + fn an_unlisted_parent_exit_remains_unconfirmed_on_stop_retry() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let mut guard = host.lock().unwrap(); + guard.child.kill().unwrap(); + guard.child.wait().unwrap(); + let first = stop_runtime(&plan, &runtime); + drop(guard); + let retried = stop_runtime(&plan, &runtime); + let retained = local_hosts().contains_key(&runtime.runtime_id); + // This fixture has no descendants; remove only its injected uncertainty + // after checking that production retry does not erase missing evidence. + frozen_process_trees().remove(&runtime.runtime_id); + stop_runtime(&plan, &runtime).unwrap(); + assert!( + matches!(first, Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "cleanup_unconfirmed") + ); + assert!( + matches!(retried, Err(FabricError::AdapterLifecycleOperation { code, .. }) if code == "cleanup_unconfirmed") + ); + assert!(retained); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn failed_start_discovery_error_retains_runtime_and_service_owners() { + for service in [false, true] { + let (root, plan) = local_host_plan("child_after_start"); + let (runtime, host) = if service { + let handle = prepare_service(&plan).unwrap(); + let record = local_services().remove(&handle.service_id).unwrap(); + let record = Arc::try_unwrap(record).ok().unwrap().into_inner().unwrap(); + (record.runtime, record.host) + } else { + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().remove(&runtime.runtime_id).unwrap(); + let host = Arc::try_unwrap(host).ok().unwrap().into_inner().unwrap(); + (runtime, host) + }; + let child_pid: u32 = fs::read_to_string(root.join("child.pid")) + .unwrap() + .parse() + .unwrap(); + let cause = lifecycle_error( + AdapterLifecycleOperation::Start, + &runtime.runtime_id, + "fixture_start_failure", + "PRIVATE_START_CAUSE", + "", + ); + let error = retain_failed_start_with(&runtime, host, cause, |_| { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + }); + assert!(!format!("{error:?}").contains("PRIVATE_START_CAUSE")); + assert!(!error.to_string().contains(&runtime.runtime_binding)); + let FabricError::RuntimeCleanupPending(pending) = error else { + panic!("failed startup must retain cleanup context"); + }; + assert_eq!(pending.runtime(), &runtime); + assert!(local_hosts().contains_key(&runtime.runtime_id)); + assert!(local_host_directory(&runtime.runtime_id).exists()); + let denied = invoke_runtime(&plan, &runtime, RunRequest::text("must not invoke")); + stop_runtime(&plan, pending.runtime()).unwrap(); + #[cfg(unix)] + assert!(owned_process_has_exited(&LocalHostProcess::new( + child_pid, + SystemTime::now() + ))); + #[cfg(windows)] + let _ = child_pid; + assert!(!local_hosts().contains_key(&runtime.runtime_id)); + assert!( + matches!(denied, Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "host_unavailable") + ); + fs::remove_dir_all(root).unwrap(); + } + } + + #[cfg(unix)] + #[test] + fn retained_connected_start_blocks_service_release_until_cleanup() { + let (root, plan) = local_host_plan("connected_start_cleanup_failure"); + let service = prepare_service(&plan).unwrap(); + let error = start_runtime_with_service(&plan, &service).unwrap_err(); + let FabricError::RuntimeCleanupPending(context) = error else { + panic!("connected startup must retain cleanup context"); + }; + let release = release_service(&plan, &service); + assert!( + matches!(release, Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "cleanup_unconfirmed") + ); + assert!(local_hosts().contains_key(&context.runtime().runtime_id)); + let directory = local_host_directory(&context.runtime().runtime_id); + assert!(directory.is_file()); + fs::remove_file(&directory).unwrap(); + fs::create_dir(&directory).unwrap(); + stop_runtime(&plan, context.runtime()).unwrap(); + release_service(&plan, &service).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn retained_timeout_host_rejects_queued_and_later_invocations() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let mut guard = host.lock().unwrap(); + let queued_plan = plan.clone(); + let queued_runtime = runtime.clone(); + let queued = thread::spawn(move || { + invoke_runtime(&queued_plan, &queued_runtime, RunRequest::text("queued")) + }); + let deadline = Instant::now() + Duration::from_secs(2); + while Arc::strong_count(&host) < 3 { + assert!( + Instant::now() < deadline, + "queued invocation did not acquire the host record" + ); + thread::sleep(Duration::from_millis(10)); + } + invalidate_timed_out_local_host_with(&runtime.runtime_id, &host, &mut guard, |_| { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + }); + drop(guard); + let queued = queued.join().unwrap(); + let later = invoke_runtime(&plan, &runtime, RunRequest::text("later")); + stop_runtime(&plan, &runtime).unwrap(); + fs::remove_dir_all(root).unwrap(); + for outcome in [queued, later] { + assert!( + matches!(outcome, Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "host_unavailable") + ); + } + } + + #[test] + fn failed_timeout_termination_retains_the_host_for_stop_retry() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let mut guard = host.lock().unwrap(); + invalidate_timed_out_local_host_with(&runtime.runtime_id, &host, &mut guard, |_| { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + }); + let retained = local_hosts().contains_key(&runtime.runtime_id); + let files_retained = guard.stderr_path.exists(); + drop(guard); + stop_runtime(&plan, &runtime).unwrap(); + fs::remove_dir_all(root).unwrap(); + assert!(retained); + assert!(files_retained); + } + + #[cfg(unix)] + #[test] + fn incomplete_descendant_listing_cannot_be_an_empty_snapshot() { + let error = host_descendants_with(100, |parent| { + if parent == 100 { + Ok(vec![101]) + } else { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + } + }) + .expect_err("a descendant listing failure must propagate"); + assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied); + } + + #[cfg(unix)] + #[test] + fn lost_parent_cleanup_terminates_known_frozen_processes() { + let (root, plan) = local_host_plan("child_after_start"); + let runtime = start_runtime(&plan).unwrap(); + let host_process = local_host_processes() + .get(&runtime.runtime_id) + .unwrap() + .clone(); + let child_pid: u32 = fs::read_to_string(root.join("child.pid")) + .unwrap() + .parse() + .unwrap(); + let child = LocalHostProcess::new(child_pid, SystemTime::now()); + let deadline = Instant::now() + Duration::from_secs(2); + freeze_owned_process(&host_process, deadline).unwrap(); + freeze_owned_process(&child, deadline).unwrap(); + let mut departed = Command::new("sh").args(["-c", "exit 0"]).spawn().unwrap(); + let departed_pid = departed.id(); + departed.wait().unwrap(); + let departed = LocalHostProcess::new(departed_pid, SystemTime::now()); + frozen_process_trees().insert( + runtime.runtime_id.clone(), + Arc::new(Mutex::new(FrozenProcessTree { + pending: vec![departed.clone()], + processes: vec![host_process.clone(), child.clone(), departed], + lost_parent: false, + })), + ); + let outcome = collect_frozen_host_with(&runtime, &host_process, |_| unreachable!()); + let deadline = Instant::now() + Duration::from_secs(2); + while Instant::now() < deadline + && !(owned_process_has_exited(&host_process) && owned_process_has_exited(&child)) + { + thread::sleep(Duration::from_millis(10)); + } + let survivors_exited = + owned_process_has_exited(&host_process) && owned_process_has_exited(&child); + let retained = frozen_tree_is_pending(&runtime.runtime_id); + // Always release the real fixture processes, including on the red run. + signal_owned_process(&child).unwrap(); + signal_owned_process(&host_process).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + host.lock().unwrap().child.wait().unwrap(); + frozen_process_trees().remove(&runtime.runtime_id); + stop_runtime(&plan, &runtime).unwrap(); + fs::remove_dir_all(root).unwrap(); + assert!( + matches!(outcome, Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "cleanup_unconfirmed") + ); + assert!(retained); + assert!( + survivors_exited, + "known frozen work must not survive lost-parent cleanup" + ); + } + + #[cfg(unix)] + #[test] + fn failed_child_listing_retains_a_retryable_frozen_parent() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let process = local_host_processes() + .get(&runtime.runtime_id) + .unwrap() + .clone(); + let failed = collect_frozen_host_with(&runtime, &process, |_| { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + }); + let retained = frozen_process_trees().contains_key(&runtime.runtime_id); + stop_runtime(&plan, &runtime).expect("retry with readable process evidence"); + assert!(failed.is_err()); + assert!(retained); + assert!(!local_hosts().contains_key(&runtime.runtime_id)); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn unconfirmed_timeout_cleanup_retains_service_membership() { + let (root, plan) = local_host_plan("success"); + let service = prepare_service(&plan).unwrap(); + let runtime = start_runtime_with_service(&plan, &service).unwrap(); + let host = local_hosts().remove(&runtime.runtime_id).unwrap(); + let stopped = stop_runtime(&plan, &runtime); + let retained = local_services() + .get(&service.service_id) + .unwrap() + .lock() + .unwrap() + .active_runtime_ids + .contains(&runtime.runtime_id); + local_hosts().insert(runtime.runtime_id.clone(), host); + stop_runtime(&plan, &runtime).unwrap(); + release_service(&plan, &service).unwrap(); + assert!( + matches!(stopped, Err(FabricError::AdapterLifecycleOperation { code, .. }) + if code == "cleanup_unconfirmed") + ); + assert!(retained); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn one_shot_cleanup_failures_keep_a_safe_retry_handle() { + for mode in ["cleanup_failure", "invoke_failure_cleanup_failure"] { + let (root, plan) = local_host_plan(mode); + let error = run_plan(&plan, RunRequest::text("TEST_ONLY_PRIVATE_RESULT")) + .expect_err("unconfirmed cleanup must retain a retry handle"); + let debug = format!("{error:?}"); + let FabricError::RuntimeCleanupPending(pending) = error else { + panic!("expected retained one-shot cleanup context"); + }; + assert_eq!(pending.result().is_some(), mode == "cleanup_failure"); + let runtime = pending.runtime(); + let directory = local_host_directory(&runtime.runtime_id); + fs::remove_file(&directory).unwrap(); + fs::create_dir_all(&directory).unwrap(); + stop_runtime(&plan, runtime).expect("retry cleanup with the original handle"); + assert!(!local_hosts().contains_key(&runtime.runtime_id)); + assert!(!debug.contains("TEST_ONLY_PRIVATE_RESULT")); + assert!(!debug.contains(&runtime.runtime_binding)); + fs::remove_dir_all(root).unwrap(); + } + } + + #[cfg(unix)] + #[test] + fn freezing_an_owned_process_prevents_new_child_creation() { + let (root, _) = local_host_plan("success"); + let ready = root.join("ready-to-fork"); + let release = root.join("release-fork"); + let marker = root.join("forked"); + let script = format!( + "import pathlib, subprocess, sys, time\n\ + ready=pathlib.Path({})\n\ + release=pathlib.Path({})\n\ + marker=pathlib.Path({})\n\ + ready.touch()\n\ + while not release.exists(): time.sleep(0.01)\n\ + subprocess.Popen([sys.executable,'-c','import time; time.sleep(1)'])\n\ + marker.touch()\n\ + time.sleep(1)\n", + serde_json::to_string(&ready).unwrap(), + serde_json::to_string(&release).unwrap(), + serde_json::to_string(&marker).unwrap(), + ); + let mut child = Command::new(DEFAULT_PYTHON) + .args(["-c", &script]) + .spawn() + .unwrap(); + // Match a started adapter: wait until the interpreter has finished + // launching before exercising suspension of active application work. + let deadline = Instant::now() + LOCAL_HOST_STOP_TIMEOUT; + while !ready.exists() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + let started = ready.exists(); + let process = LocalHostProcess::new(child.id(), SystemTime::now()); + let frozen = freeze_owned_process(&process, Instant::now() + Duration::from_secs(3)); + let stopped = process.is_stopped(); + fs::write(&release, "").unwrap(); + thread::sleep(Duration::from_millis(50)); + let forked = marker.exists(); + child.kill().unwrap(); + child.wait().unwrap(); + assert!(started, "child must initialize before freezing"); + frozen.unwrap(); + assert_eq!(stopped, Some(true)); + assert!(!forked); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(windows)] + #[test] + fn windows_capture_rejects_a_missing_job() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let capture = WindowsHostControl::capture(&host.lock().unwrap().child, None); + stop_runtime(&plan, &runtime).unwrap(); + assert!(capture.is_err()); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(windows)] + #[test] + fn windows_job_closure_terminates_its_processes() { + use std::os::windows::process::CommandExt; + let mut child = Command::new(DEFAULT_PYTHON) + .args(["-c", "import time; time.sleep(30)"]) + .creation_flags(windows_sys::Win32::System::Threading::CREATE_SUSPENDED) + .spawn() + .unwrap(); + let job = host_job(&child).unwrap(); + resume_host(child.id()).unwrap(); + drop(job); + let deadline = Instant::now() + Duration::from_secs(3); + while child.try_wait().unwrap().is_none() && Instant::now() < deadline { + thread::sleep(Duration::from_millis(10)); + } + let terminated = child.try_wait().unwrap().is_some(); + if !terminated { + child.kill().unwrap(); + child.wait().unwrap(); + } + assert!( + terminated, + "closing the last job owner must terminate the owned tree" + ); + } + + #[cfg(windows)] + #[test] + fn windows_busy_stop_retains_ownership_without_a_job() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let control = windows_host_controls() + .get(&runtime.runtime_id) + .unwrap() + .clone(); + let process_only = Arc::new(WindowsHostControl { + process: control.process.try_clone().unwrap(), + job: None, + }); + windows_host_controls().insert(runtime.runtime_id.clone(), process_only.clone()); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let guard = host.lock().unwrap(); + let stopped = stop_runtime(&plan, &runtime); + drop(guard); + let retained = windows_host_controls().contains_key(&runtime.runtime_id); + let still_running = process_only.has_exited(); + windows_host_controls().insert(runtime.runtime_id.clone(), control); + stop_runtime(&plan, &runtime).unwrap(); + assert!( + matches!(stopped, Err(FabricError::AdapterLifecycleOperation { code, .. }) + if code == "cleanup_unconfirmed") + ); + assert!(retained); + assert_eq!(still_running, Some(false)); + assert_eq!(process_only.cleanup_confirmed(), None); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(windows)] + #[test] + fn windows_stop_confirms_the_job_after_the_host_exits() { + let (root, plan) = local_host_plan("child_after_start"); + let runtime = start_runtime(&plan).unwrap(); + let control = windows_host_controls() + .get(&runtime.runtime_id) + .unwrap() + .clone(); + assert_eq!(control.has_exited(), Some(false)); + { + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let mut host = host.lock().unwrap(); + host.child.kill().unwrap(); + host.child.wait().unwrap(); + } + assert_eq!(control.has_exited(), Some(true)); + assert_eq!(control.cleanup_confirmed(), Some(false)); + stop_runtime(&plan, &runtime).expect("stop children after the host already exited"); + assert_eq!(control.cleanup_confirmed(), Some(true)); + assert!(!windows_host_controls().contains_key(&runtime.runtime_id)); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn requested_busy_stop_is_interrupted_instead_of_crashed() { + let (root, plan) = local_host_plan("invoke_timeout"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let stderr = host.lock().unwrap().stderr_path.clone(); + let invocation_plan = plan.clone(); + let invocation_runtime = runtime.clone(); + let invocation = thread::spawn(move || { + invoke_runtime( + &invocation_plan, + &invocation_runtime, + RunRequest::text("wait"), + ) + }); + let deadline = Instant::now() + Duration::from_secs(5); + while !fs::read_to_string(&stderr) + .unwrap() + .contains("invoke accepted") + { + assert!(Instant::now() < deadline, "invocation did not start"); + thread::sleep(Duration::from_millis(10)); + } + let events = stop_runtime(&plan, &runtime).unwrap(); + let error = invocation.join().unwrap().unwrap_err(); + fs::remove_dir_all(root).unwrap(); + assert!( + matches!(error, FabricError::AdapterLifecycleOperation {code, ..} if code == "interrupted") + ); + assert_eq!(events[0].metadata["host_crashed"], false); + assert_eq!(events[0].metadata["code"], "interrupted"); + } + + #[test] + fn busy_stop_interrupts_the_owned_host_without_the_invocation_lock() { + let (root, plan) = local_host_plan("child_after_start"); + let runtime = start_runtime(&plan).unwrap(); + let child_pid: u32 = fs::read_to_string(root.join("child.pid")) + .unwrap() + .parse() + .unwrap(); + #[cfg(unix)] + let child = LocalHostProcess::new(child_pid, SystemTime::now()); + #[cfg(windows)] + let child = { + use std::os::windows::io::{FromRawHandle, OwnedHandle}; + // SAFETY: OpenProcess returns a new owned handle with synchronization access. + let raw = unsafe { + windows_sys::Win32::System::Threading::OpenProcess(0x0010_0000, 0, child_pid) + }; + assert!(!raw.is_null()); + unsafe { OwnedHandle::from_raw_handle(raw) } + }; + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let guard = host.lock().unwrap(); + let interrupted = interrupt_local_host(&runtime); + drop(guard); + let stopped = stop_runtime(&plan, &runtime); + interrupted.expect("interrupt without taking the busy host lock"); + stopped.expect("confirm cleanup after interruption"); + #[cfg(unix)] + assert!(owned_process_has_exited(&child)); + #[cfg(windows)] + { + use std::os::windows::io::AsRawHandle; + // SAFETY: the captured child handle remains owned until this assertion completes. + assert_eq!( + unsafe { + windows_sys::Win32::System::Threading::WaitForSingleObject( + child.as_raw_handle(), + 1000, + ) + }, + windows_sys::Win32::Foundation::WAIT_OBJECT_0 + ); + } + assert!(!local_hosts().contains_key(&runtime.runtime_id)); + assert!(local_host_has_exited(&runtime.runtime_id).is_none()); + fs::remove_dir_all(root).unwrap(); + } + + #[cfg(unix)] + #[test] + fn stop_retries_can_confirm_a_child_that_exited_before_identity_capture() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let mut child = Command::new("sh").args(["-c", "exit 0"]).spawn().unwrap(); + let pid = child.id(); + child.wait().unwrap(); + let gone = LocalHostProcess::new(pid, SystemTime::now()); + assert!(!gone.is_current()); + local_hosts() + .get(&runtime.runtime_id) + .unwrap() + .lock() + .unwrap() + .pending_stop + .push(gone); + stop_runtime(&plan, &runtime) + .expect("a confirmed-gone child cannot permanently block cleanup"); + assert!(!local_hosts().contains_key(&runtime.runtime_id)); + assert!(local_host_has_exited(&runtime.runtime_id).is_none()); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn local_host_reuses_one_process_and_stops_idempotently() { let (root, plan) = local_host_plan("success"); @@ -5603,6 +6787,44 @@ else: let _ = fs::remove_dir_all(root); } + #[cfg(unix)] + #[test] + fn automatic_scope_cleanup_retains_unfinished_frozen_discovery() { + let (root, plan) = local_host_plan("success"); + let runtime = start_runtime(&plan).unwrap(); + let host = local_hosts().get(&runtime.runtime_id).unwrap().clone(); + let scope = host.lock().unwrap().cleanup_scope.clone(); + PENDING_LOCAL_CLEANUP + .lock() + .unwrap() + .insert(runtime.runtime_id.clone(), scope.clone()); + let process = local_host_processes() + .get(&runtime.runtime_id) + .unwrap() + .clone(); + let discovery = collect_frozen_host_with(&runtime, &process, |_| { + Err(std::io::Error::from(std::io::ErrorKind::PermissionDenied)) + }); + assert!(discovery.is_err()); + signal_owned_process(&process).unwrap(); + host.lock().unwrap().child.wait().unwrap(); + let retry = retry_pending_local_cleanup(&scope, AdapterLifecycleOperation::Start); + assert!( + matches!(retry, Err(FabricError::AdapterLifecycleOperation {code, ..}) if code == "cleanup_unconfirmed") + ); + assert!(local_hosts().contains_key(&runtime.runtime_id)); + assert!( + PENDING_LOCAL_CLEANUP + .lock() + .unwrap() + .contains_key(&runtime.runtime_id) + ); + // This fixture had no children. Clear only its injected incomplete discovery. + frozen_process_trees().remove(&runtime.runtime_id); + stop_runtime(&plan, &runtime).unwrap(); + fs::remove_dir_all(root).unwrap(); + } + #[test] fn failed_one_shot_stop_cleanup_is_retried_before_replacement() { for (keep_state, prepare_replacement) in @@ -5614,9 +6836,18 @@ else: local_host_plan("success") }; FAIL_NEXT_HOST_TERMINATION.with(|fail| fail.set(true)); - let result = run_plan(&plan, RunRequest::text("completed")).unwrap(); - assert_eq!(result.status, RunStatus::Failed); - assert_eq!(result.error.as_ref().unwrap().stage, ErrorStage::Stop); + let error = run_plan(&plan, RunRequest::text("completed")).unwrap_err(); + let FabricError::RuntimeCleanupPending(pending) = error else { + panic!("expected retained cleanup context") + }; + let result = pending + .result() + .expect("completed invocation remains available"); + assert_eq!(result.status, RunStatus::Succeeded); + assert_eq!(result.runtime_id, pending.runtime().runtime_id); + assert!( + matches!(pending.cause(), FabricError::AdapterLifecycleOperation {operation: "stop", code, ..} if code == "cleanup_unconfirmed") + ); assert!(local_hosts().contains_key(&result.runtime_id)); FAIL_NEXT_HOST_TERMINATION.with(|fail| fail.set(true)); @@ -5695,13 +6926,14 @@ else: FAIL_NEXT_HOST_TERMINATION.with(|fail| fail.set(true)); let error = start_runtime(&plan).unwrap_err(); let message = error.to_string(); - let (failed_id, diagnostics) = match error { + let FabricError::RuntimeCleanupPending(pending) = error else { + panic!("expected retained startup cleanup context") + }; + let failed_id = pending.runtime().runtime_id.clone(); + let diagnostics = match pending.cause() { FabricError::AdapterLifecycleOperation { - code, - runtime_id, - diagnostics, - .. - } if code == "cleanup_unconfirmed" => (runtime_id, diagnostics), + code, diagnostics, .. + } if code == "cleanup_unconfirmed" => diagnostics.clone(), other => panic!("unexpected startup failure: {other}"), }; let diagnostic_path = PathBuf::from(diagnostics); @@ -5747,12 +6979,10 @@ else: let service = prepare_service(&plan).unwrap(); FAIL_NEXT_HOST_TERMINATION.with(|fail| fail.set(true)); let error = start_runtime_with_service(&plan, &service).unwrap_err(); - let failed_id = match error { - FabricError::AdapterLifecycleOperation { - code, runtime_id, .. - } if code == "cleanup_unconfirmed" => runtime_id, - other => panic!("unexpected startup failure: {other}"), + let FabricError::RuntimeCleanupPending(pending) = error else { + panic!("expected retained connected startup context") }; + let failed_id = pending.runtime().runtime_id.clone(); let record = local_services().get(&service.service_id).cloned().unwrap(); assert!( record @@ -7182,6 +8412,9 @@ else: ); assert!(!runtime_dir.exists()); + #[cfg(any(target_os = "linux", target_os = "macos", windows))] + assert_eq!(local_host_has_exited(&runtime.runtime_id), Some(true)); + #[cfg(any(target_os = "linux", target_os = "macos"))] for level in ["live", "active", "ready"] { let request = serde_json::from_value(serde_json::json!({ @@ -7504,7 +8737,7 @@ else: )); assert!(matches!( second_error, - FabricError::AdapterLifecycleOperation { code, .. } if code == "host_crashed" + FabricError::AdapterLifecycleOperation { code, .. } if code == "host_unavailable" )); assert!(!local_hosts().contains_key(&runtime.runtime_id)); let stopped = stop_runtime(&plan, &runtime).expect("stop after timeout is idempotent"); diff --git a/crates/fabric-python/src/lib.rs b/crates/fabric-python/src/lib.rs index 5636d1da2..44e9d8f64 100644 --- a/crates/fabric-python/src/lib.rs +++ b/crates/fabric-python/src/lib.rs @@ -464,6 +464,27 @@ fn to_py_error(error: nemo_fabric_core::FabricError) -> PyErr { Err(setattr_error) => setattr_error, } } + FabricError::RuntimeCleanupPending(context) => { + let failure = PyRuntimeError::new_err( + "Runtime cleanup is unconfirmed; retry cleanup_runtime.stop().", + ); + let encoded = match serde_json::to_string(context.runtime()) { + Ok(encoded) => encoded, + Err(_) => { + return PyRuntimeError::new_err("Runtime cleanup context is unavailable."); + } + }; + match Python::attach(|py| -> PyResult<()> { + failure.value(py).setattr("code", "cleanup_unconfirmed")?; + let handle = py.import("json")?.call_method1("loads", (encoded,))?; + failure + .value(py) + .setattr("_fabric_cleanup_runtime_handle", handle) + }) { + Ok(()) => failure, + Err(error) => error, + } + } _ => PyRuntimeError::new_err(error.to_string()), } } diff --git a/docs/reference/api/python-library-reference/nemo_fabric.client.md b/docs/reference/api/python-library-reference/nemo_fabric.client.md index 6848b98d3..815b74b20 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.client.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.client.md @@ -350,7 +350,7 @@ async def run( Execute one complete start, invoke, and stop lifecycle. -``input`` and ``request`` are mutually exclusive. Omitting both produces an empty text input. Use ``RunRequest`` when the invocation needs a caller-owned request ID, context, or overrides. NeMo Fabric attempts to stop a started runtime even when invocation fails. +``input`` and ``request`` are mutually exclusive. Omitting both produces an empty text input. Use ``RunRequest`` when the invocation needs a caller-owned request ID, context, or overrides. NeMo Fabric attempts to stop a started runtime even when invocation fails. Unconfirmed cleanup raises an error with ``cleanup_runtime`` for an explicit stop retry in the same process. diff --git a/docs/reference/api/python-library-reference/nemo_fabric.errors.md b/docs/reference/api/python-library-reference/nemo_fabric.errors.md index 50e2dff6e..94cc6f920 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.errors.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.errors.md @@ -10,6 +10,9 @@ SPDX-License-Identifier: Apache-2.0 --> Public exception hierarchy for the NeMo Fabric Python SDK. +**Global Variables** +--------------- +- **TYPE_CHECKING** --- @@ -62,6 +65,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Owned runtime for a failed lifecycle cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -104,6 +115,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Owned runtime for a failed lifecycle cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -146,6 +165,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Owned runtime for a failed lifecycle cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -188,6 +215,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Owned runtime for a failed lifecycle cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -230,6 +265,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Owned runtime for a failed lifecycle cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -272,6 +315,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Owned runtime for a failed lifecycle cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -316,6 +367,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Owned runtime for a failed lifecycle cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + @@ -360,6 +419,14 @@ Initialize a structured NeMo Fabric exception. - `details`: Optional structured diagnostic context. The exception stores a deep copy. +--- + +### property cleanup_runtime + +Owned runtime for a failed lifecycle cleanup retry, or ``None``. + +This in-memory capability is excluded from error text and structured details. + diff --git a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md index a030b498e..4306d1d01 100644 --- a/docs/reference/api/python-library-reference/nemo_fabric.runtime.md +++ b/docs/reference/api/python-library-reference/nemo_fabric.runtime.md @@ -223,7 +223,7 @@ The snapshot covers the local adapter host process and its live descendants. It async def stop() -> None ``` -Destroy an idle runtime exactly once. +Stop the runtime, interrupting an active buffered invocation if needed. Repeated calls after a successful stop are no-ops. A failed runtime may still be stopped so its resources are released. @@ -231,7 +231,7 @@ Repeated calls after a successful stop are no-ops. A failed runtime may still be **Raises:** - - `FabricStateError`: If the runtime is already stopping or has an invocation in flight. + - `FabricStateError`: If the runtime is already stopping or has an streaming invocation in flight. - `FabricNativeUnavailableError`: If the native extension is missing. - `FabricRuntimeError`: If native runtime shutdown fails. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx index fe950a53e..a613e4f64 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/adapter-contract/index.mdx @@ -3,7 +3,7 @@ title: "Module adapter_contract" sidebar-title: "adapter_contract" slug: "/reference/api/rust-library-reference/nemo-fabric-core/adapter_contract" description: "Shared southbound adapter contract metadata." -position: 165 +position: 166 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx index 71104c85b..48f092f67 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-config/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_config" sidebar-title: "agent_config" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_config" description: "Configuration projected southbound to an adapter target." -position: 166 +position: 167 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx index ba24f43ed..6194d6aac 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/agent-execution/index.mdx @@ -3,7 +3,7 @@ title: "Module agent_execution" sidebar-title: "agent_execution" slug: "/reference/api/rust-library-reference/nemo-fabric-core/agent_execution" description: "Request and result structures exchanged with an adapter target." -position: 167 +position: 168 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/check/enum-checkreason.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/check/enum-checkreason.mdx index 79e8235c1..51bb22039 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/check/enum-checkreason.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/check/enum-checkreason.mdx @@ -13,6 +13,8 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. pub enum CheckReason { ServerStopping, CapabilitiesUnverified, + RuntimeBusy, + CleanupUnconfirmed, RuntimeUnavailable, ServiceUnavailable, AgentExited, @@ -44,6 +46,18 @@ The server is stopping. Advertised capability evidence is unavailable. +### `RuntimeBusy` + +
+ +Runtime cleanup is in progress. + +### `CleanupUnconfirmed` + +
+ +Owned runtime cleanup could not be confirmed. + ### `RuntimeUnavailable`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/check/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/check/index.mdx index 56b4d0dad..1a17c31b7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/check/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/check/index.mdx @@ -2,7 +2,7 @@ title: "Module check" sidebar-title: "check" description: "Runtime health checks and shared contracts." -position: 168 +position: 169 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx index c685b867f..2542912d3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/config/index.mdx @@ -2,7 +2,7 @@ title: "Module config" sidebar-title: "config" description: "NeMo Fabric config models and loading helpers." -position: 169 +position: 170 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx index c71073030..db748d3db 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/doctor/index.mdx @@ -2,7 +2,7 @@ title: "Module doctor" sidebar-title: "doctor" description: "Plan diagnostics for NeMo Fabric." -position: 170 +position: 171 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx index 23376c571..926a51ed3 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/enum-fabricerror.mdx @@ -9,7 +9,7 @@ SPDX-License-Identifier: Apache-2.0 */} Generated from `cargo doc --no-deps -p nemo-fabric-core`. -
String,\n        message: String,\n    },\n    ResolveBaseDirectory {\n        path: PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    StateInUse {\n        path: PathBuf,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
+
String,\n        message: String,\n    },\n    ResolveBaseDirectory {\n        path: PathBuf,\n        source: Error,\n    },\n    PathNotFound(PathBuf),\n    UnknownAdapter {\n        adapter_id: String,\n        available: Vec<String>,\n    },\n    UnknownAdapterTarget {\n        target_id: String,\n        available: Vec<String>,\n    },\n    AmbiguousDescriptor {\n        descriptor_kind: &'static str,\n        id: String,\n        paths: Vec<PathBuf>,\n    },\n    AdapterDescriptorMismatch {\n        path: PathBuf,\n        field: &'static str,\n        expected: String,\n        actual: String,\n    },\n    AdapterDescriptorUnsupported {\n        adapter_id: String,\n        field: &'static str,\n        value: String,\n    },\n    InvalidAdapterDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidAdapterTargetDescriptor {\n        path: PathBuf,\n        message: String,\n    },\n    InvalidHarnessSettings {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        settings_path: String,\n        reason: String,\n    },\n    InvalidWorkflow {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        workflow_path: String,\n        reason: String,\n    },\n    InvalidToolDefinition {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        definition_path: String,\n        reason: String,\n    },\n    InvalidAdapterExtension {\n        adapter_id: String,\n        descriptor_source: DescriptorSource,\n        descriptor_path: PathBuf,\n        extension_path: String,\n        reason: String,\n    },\n    InvalidConfig {\n        field: String,\n        reason: String,\n    },\n    AdapterCompatibility {\n        adapter_id: String,\n        field: String,\n        reason: String,\n    },\n    UnknownSchema {\n        schema: String,\n        available: Vec<String>,\n    },\n    UnsupportedRuntimeAdapter {\n        harness: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedRuntimeCapability {\n        adapter_id: String,\n        capability: &'static str,\n    },\n    InvalidOpenAiStreamTransport {\n        field: &'static str,\n        reason: &'static str,\n    },\n    AdapterLifecycleOperation {\n        operation: &'static str,\n        runtime_id: String,\n        code: String,\n        message: String,\n        diagnostics: String,\n    },\n    RuntimeCleanupPending(Box<RuntimeCleanupPending>),\n    RuntimeHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        runtime_id: String,\n    },\n    ServiceHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        service_id: String,\n    },\n    StateInUse {\n        path: PathBuf,\n    },\n    ServiceInUse {\n        service_id: String,\n        runtime_ids: Vec<String>,\n    },\n    ServiceNotActive {\n        service_id: String,\n    },\n    UnsupportedEnvironmentProvider {\n        provider: String,\n        adapter_kind: AdapterKind,\n    },\n    UnsupportedEnvironmentRelease {\n        provider: String,\n    },\n    EnvironmentProviderOperation {\n        provider: String,\n        operation: String,\n        code: String,\n        message: String,\n    },\n    EnvironmentInUse {\n        environment_id: String,\n        runtime_id: String,\n    },\n    EnvironmentReleaseInProgress {\n        environment_id: String,\n    },\n    EnvironmentHandleRequired {\n        provider: String,\n    },\n    EnvironmentHandleMismatch {\n        field: &'static str,\n        expected: String,\n        actual: String,\n        environment_id: String,\n    },\n    ResourceObservationUnavailable {\n        runtime_id: String,\n        reason: String,\n    },\n    ResourceObservationUnsupported {\n        runtime_id: String,\n        reason: String,\n    },\n    InvalidProcessSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    InvalidPythonSettings {\n        path: PathBuf,\n        source: Error,\n    },\n    PythonInterpreterUnavailable {\n        path: PathBuf,\n        origin: String,\n        reason: String,\n    },\n    ProcessRunner {\n        command: String,\n        source: Error,\n    },\n    SerializeJson(Error),\n    Read {\n        path: PathBuf,\n        source: Error,\n    },\n    Write {\n        path: PathBuf,\n        source: Error,\n    },\n    ParseJson {\n        path: PathBuf,\n        source: Error,\n    },\n}"}} />
Errors raised by NeMo Fabric config loading and validation. @@ -421,6 +421,12 @@ Human-readable failure message. Bounded adapter-host diagnostics. +### `RuntimeCleanupPending(Box)` + +
Box<RuntimeCleanupPending>)"}} />
+ +Owned lifecycle cleanup is unconfirmed; the original runtime remains owned. + ### `RuntimeHandleMismatch`
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx index d2862663b..9670e4008 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/index.mdx @@ -2,7 +2,7 @@ title: "Module error" sidebar-title: "error" description: "Error types for NeMo Fabric core." -position: 171 +position: 172 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} @@ -11,6 +11,10 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`. Error types for NeMo Fabric core. +## Structs + +- [RuntimeCleanupPending](struct-runtimecleanuppending.mdx): Cleanup context retained when startup or invocation cannot finish owned cleanup. + ## Enums - [FabricError](enum-fabricerror.mdx): Errors raised by NeMo Fabric config loading and validation. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/error/struct-runtimecleanuppending.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/struct-runtimecleanuppending.mdx new file mode 100644 index 000000000..73211d9a4 --- /dev/null +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/error/struct-runtimecleanuppending.mdx @@ -0,0 +1,60 @@ +--- +title: "Struct Runtime Cleanup Pending" +sidebar-title: "RuntimeCleanupPending" +description: "Cleanup context retained when startup or invocation cannot finish owned cleanup." +position: 99 +--- +{/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 */} + +Generated from `cargo doc --no-deps -p nemo-fabric-core`. + +```rust +pub struct RuntimeCleanupPending { /* private fields */ } +``` + +Cleanup context retained when startup or invocation cannot finish owned cleanup. + +Retry [`crate::stop_runtime`](../runtime/fn-stop-runtime.mdx) with the original plan and [`Self::runtime`](struct-runtimecleanuppending.mdx#method.runtime). The handle and completed output are deliberately excluded from diagnostic formatting. + +## Implementations + +### `impl RuntimeCleanupPending` + +
RuntimeCleanupPending"}} />
+ +#### `runtime` + +
RuntimeHandle"}} />
+ +The original handle required for a cleanup retry in this process. + +#### `result` + +
Option<&RunResult>"}} />
+ +The completed invocation result, if invocation succeeded before cleanup failed. + +#### `cause` + +
FabricError"}} />
+ +The startup, invocation, or stop error that preceded the retained cleanup context. + +## Trait Implementations + +### `impl Debug for RuntimeCleanupPending` + +
Debug for RuntimeCleanupPending"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
+ +### `impl Display for RuntimeCleanupPending` + +
Display for RuntimeCleanupPending"}} />
+ +#### `fmt` + +
fmt(&self, f: &mut Formatter<'_>) -> Result"}} />
diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx index 763e6e69c..c1bd3a056 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/fn-version.mdx @@ -2,7 +2,7 @@ title: "Function version" sidebar-title: "version" description: "Returns the crate version compiled into this build." -position: 178 +position: 179 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx index 07728f2b2..437c2120f 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/index.mdx @@ -111,6 +111,7 @@ Core config and runtime contract for NeMo Fabric. - `pub use doctor::doctor_plan;` - `pub use error::FabricError;` - `pub use error::Result;` +- `pub use error::RuntimeCleanupPending;` - `pub use provenance::ExecutionEvidence;` - `pub use provenance::ProvenanceConfig;` - `pub use resource_snapshot::RuntimeResourceSnapshot;` diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/provenance/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/provenance/index.mdx index 92aa1c299..4909b0a85 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/provenance/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/provenance/index.mdx @@ -2,7 +2,7 @@ title: "Module provenance" sidebar-title: "provenance" description: "Selected software evidence. Optional evidence never controls invocation success." -position: 172 +position: 173 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx index c1deacf18..e784207c4 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/resource-snapshot/index.mdx @@ -3,7 +3,7 @@ title: "Module resource_snapshot" sidebar-title: "resource_snapshot" slug: "/reference/api/rust-library-reference/nemo-fabric-core/resource_snapshot" description: "Resource snapshots of a runtime's local adapter host and its descendants." -position: 173 +position: 174 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx index 331c08b3c..8353c2e48 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/fn-run-plan.mdx @@ -12,3 +12,5 @@ Generated from `cargo doc --no-deps -p nemo-fabric-core`.
RunPlan, request: RunRequest) -> Result<RunResult>"}} />
Invoke a NeMo Fabric run plan. + +If stopping leaves owned cleanup pending, returns [`FabricError::RuntimeCleanupPending`](../error/enum-fabricerror.mdx#variant.RuntimeCleanupPending) with the original runtime handle and any completed result. Retry [`stop_runtime`](fn-stop-runtime.mdx) with that handle and this plan. diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx index af0211128..7683d4dd7 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/runtime/index.mdx @@ -2,7 +2,7 @@ title: "Module runtime" sidebar-title: "runtime" description: "Runtime invocation helpers." -position: 174 +position: 175 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx index 4268f3f13..d5f5661cf 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/schema/index.mdx @@ -2,7 +2,7 @@ title: "Module schema" sidebar-title: "schema" description: "JSON Schema generation for the public NeMo Fabric contract." -position: 175 +position: 176 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/session/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/session/index.mdx index 34a05fadb..6ccad6fd2 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/session/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/session/index.mdx @@ -2,7 +2,7 @@ title: "Module session" sidebar-title: "session" description: "Explicit selection of durable harness-native sessions." -position: 176 +position: 177 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx index cc4e9639a..f7c93c8bb 100644 --- a/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx +++ b/docs/reference/api/rust-library-reference/nemo-fabric-core/system-stats/index.mdx @@ -3,7 +3,7 @@ title: "Module system_stats" sidebar-title: "system_stats" slug: "/reference/api/rust-library-reference/nemo-fabric-core/system_stats" description: "Invocation measurements collected by NeMo Fabric in the executing runtime." -position: 177 +position: 178 --- {/* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 */} diff --git a/docs/sdk/python.mdx b/docs/sdk/python.mdx index f5320750f..7c9aa27f8 100644 --- a/docs/sdk/python.mdx +++ b/docs/sdk/python.mdx @@ -365,11 +365,11 @@ Service checks evaluate earlier levels in order under the same three-second defa `fabric.check/v1alpha1` identifies the shared report schema returned by `Fabric.check()` and `fabric-agent-ctl check`. The executable ships with -`nemo-fabric-runtime` and follows the SDK package release. +`nemo-fabric-runtime` and follows the SDK package release. Health readers validate reason codes strictly. Use the same `nemo-fabric-runtime` release for the caller, sandbox server, and `fabric-agent-ctl`; mixed-release health reports are not supported. The schema identifier does not negotiate SDK releases. `fabric-agent-ctl serve` starts one runtime server per sandbox. The `start`, `invoke`, `stop`, and `status` commands call `Fabric.start_runtime()`, `Runtime.invoke()`, `Runtime.stop()`, and `Fabric.runtime_status()`. Start accepts `FabricConfig` and returns the SDK `RuntimeHandle` mapping. Invoke accepts a full `RunRequest` through `--request` or any JSON SDK input through `--input`, and returns the SDK `RunResult` mapping. Stop returns JSON null. These payloads occupy the response envelope's `result` field; command diagnostics remain separate. Targeted commands require the returned runtime ID and exact agent name. The caller waits for the private socket before start. Caller-side preflight uses `Fabric.plan()` and `Fabric.doctor()`; startup validates configuration against installed descriptors. -One sandbox can own multiple runtimes, each addressed by its returned runtime ID. Invoke and stop operations are independent across IDs; stopping one runtime leaves the others usable, and another runtime can be started after a stop. An uncertain start blocks further starts because the server cannot safely determine whether an unaddressable runtime exists. Ordinary SDK stop does not confirm descendant cleanup, so the runtime server retains `fabric.unclean` after runtime use and reports `cleanup_unconfirmed` if previous sandbox process cleanup cannot be confirmed. Refer to the [runtime server command guide](https://github.com/NVIDIA/NeMo-Fabric/blob/main/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md). +One sandbox can own multiple runtimes, each addressed by its returned runtime ID. Invoke and stop operations are independent across IDs; stopping one runtime leaves the others usable, and another runtime can be started after a stop. An uncertain start blocks further starts because the server cannot safely determine whether an unaddressable runtime exists. Buffered SDK stop confirms owned process cleanup before reporting success and retains ownership when cleanup is unconfirmed. The runtime server also retains `fabric.unclean` after runtime use or an uncertain start until previous sandbox process cleanup can be confirmed. Refer to the [runtime server command guide](https://github.com/NVIDIA/NeMo-Fabric/blob/main/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md). ## Configure Agents In Code @@ -890,6 +890,14 @@ Hermes Agent reuses one agent and session database, and Claude keeps one connect client. Durable Hermes conversations expose their native identifier inside an opaque `SessionRef`. NeMo Fabric does not interpret that identifier. +Buffered runtime shutdown can interrupt an active invocation on supported Unix and Windows hosts. Windows shutdown requires an owned process job to confirm descendant cleanup. `Runtime.stop()` waits for the interrupted invocation to settle before reporting stopped. Cleanup errors retain ownership for retry; the runtime server blocks replacement for the same agent until cleanup succeeds. Streaming shutdown continues to use the stream lifecycle. An invocation interrupted by a requested stop reports `interrupted`; genuine host crashes still report `host_crashed`. Cleanup confirms processes that remain in Fabric’s owned host tree. The sandbox supervisor must terminate processes that detached before Fabric captured that tree. + +If `Fabric.run()`, `Fabric.start_runtime()`, `Fabric.prepare_service()`, or `Fabric.attach_service()` cannot confirm cleanup after a failed lifecycle operation, the raised error provides `cleanup_runtime`. Keep that object and retry `await error.cleanup_runtime.stop()` in the same process. Cancellation can carry the same recovery object. The recovery handle is excluded from error text and structured details. When an exited parent leaves undiscoverable children, cleanup remains unconfirmed and the owning process or sandbox must recover. + +Retained hosts from a timed-out invocation or failed start accept cleanup only. New and queued invocations return `host_unavailable` after acquiring the host lock. Failed connected starts keep their service membership until cleanup succeeds, so the service cannot be released while that runtime remains owned. + +Runtime health checks report pending shutdown as `runtime_busy`, with state `unknown` and activity `stopping`. After failed cleanup, checks report `cleanup_unconfirmed`, with state `failed` and activity `unknown`. These reports describe recorded lifecycle state and do not probe the harness during cleanup. + ### Keep Agent State Across Runtimes By default, each runtime keeps its native state in its own folder, so a new diff --git a/environment-providers/openshell/src/lib.rs b/environment-providers/openshell/src/lib.rs index a22109178..5e7525e6d 100644 --- a/environment-providers/openshell/src/lib.rs +++ b/environment-providers/openshell/src/lib.rs @@ -2562,6 +2562,28 @@ mod tests { assert_eq!(arguments.last().unwrap(), "--ready"); } + #[test] + fn cleanup_health_evidence_survives_the_provider_decoder() { + for (code, state, activity) in [ + ("runtime_busy", "unknown", "stopping"), + ("cleanup_unconfirmed", "failed", "unknown"), + ] { + let report = json!({ + "schema_version": "fabric.check/v1alpha1", "scope": "runtime", + "runtime_id": "runtime-1", "requested_level": "live", + "outcome": "unknown", "reason_code": code, + "completed_at": "2026-11-12T12:00:00Z", "elapsed_ms": 1, + "server": null, "runtime": {"state": state, "activity": activity}, "checks": [] + }); + let response = json!({"operation": "check", "status": "failed", "changed": false, + "result": report, "error": {"code": code, "stage": "check", "effects": "none", "message": "PRIVATE"}}); + let decoded = + decode_sdk_command_response("check", 1, response.to_string().as_bytes()).unwrap(); + assert_eq!(decoded, json!({"status": "returned", "value": report})); + assert!(!decoded.to_string().contains("PRIVATE")); + } + } + #[test] fn failed_health_evidence_is_returned_to_the_sdk() { let report = json!({ diff --git a/schemas/adapter-contract/adapter-check-result.schema.json b/schemas/adapter-contract/adapter-check-result.schema.json index eb405c683..1d3814a12 100644 --- a/schemas/adapter-contract/adapter-check-result.schema.json +++ b/schemas/adapter-contract/adapter-check-result.schema.json @@ -38,6 +38,16 @@ "description": "Advertised capability evidence is unavailable.", "type": "string" }, + { + "const": "runtime_busy", + "description": "Runtime cleanup is in progress.", + "type": "string" + }, + { + "const": "cleanup_unconfirmed", + "description": "Owned runtime cleanup could not be confirmed.", + "type": "string" + }, { "const": "runtime_unavailable", "description": "No matching runtime binding exists.", diff --git a/schemas/sdk/check-report.schema.json b/schemas/sdk/check-report.schema.json index 2ad6011d9..29e92c831 100644 --- a/schemas/sdk/check-report.schema.json +++ b/schemas/sdk/check-report.schema.json @@ -102,6 +102,16 @@ "description": "Advertised capability evidence is unavailable.", "type": "string" }, + { + "const": "runtime_busy", + "description": "Runtime cleanup is in progress.", + "type": "string" + }, + { + "const": "cleanup_unconfirmed", + "description": "Owned runtime cleanup could not be confirmed.", + "type": "string" + }, { "const": "runtime_unavailable", "description": "No matching runtime binding exists.", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/check.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/check.py index 3cb546d92..9add70cb9 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/check.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/check.py @@ -29,6 +29,8 @@ CheckReason = Literal[ "server_stopping", "capabilities_unverified", + "runtime_busy", + "cleanup_unconfirmed", "runtime_unavailable", "service_unavailable", "agent_exited", diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py index 1b3f5fc6e..c50068e25 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/client.py @@ -527,6 +527,8 @@ async def run( an empty text input. Use ``RunRequest`` when the invocation needs a caller-owned request ID, context, or overrides. NeMo Fabric attempts to stop a started runtime even when invocation fails. + Unconfirmed cleanup raises an error with ``cleanup_runtime`` for an + explicit stop retry in the same process. Args: config: Complete typed ``FabricConfig``. @@ -567,6 +569,7 @@ async def run( await _run_native_lifecycle(native, plan.to_mapping(), request_payload) ) except (FabricError, asyncio.CancelledError) as error: + self._attach_cleanup_runtime(error, plan) # The selected report already includes the caller and guest SDK facts. # Optional evidence must not change the lifecycle exception or cause. if isinstance(error, FabricError): @@ -581,6 +584,43 @@ async def run( ) raise + def _attach_cleanup_runtime(self, error: BaseException, plan: RunPlan) -> None: + cleanup = getattr(error, "_fabric_cleanup_runtime_handle", None) + if cleanup is not None: + recovery = Runtime(client=self, plan=plan, runtime=cleanup) + recovery._status = RuntimeStatus.FAILED + if isinstance(error, FabricError): + error._cleanup_runtime = recovery + else: + error.cleanup_runtime = recovery + del error._fabric_cleanup_runtime_handle + + async def _call_start_with_cleanup(self, start: Any, plan: RunPlan) -> Any: + cleanup = None + + def call(): + nonlocal cleanup + try: + return start() + except Exception as error: + cleanup = getattr(error, "_fabric_cleanup_runtime_handle", None) + raise + + try: + return await _call_blocking(call) + except BaseException as error: + if cleanup is not None: + failure = ( + error + if isinstance(error, (FabricError, asyncio.CancelledError)) + else _runtime_error(error, stage="start") + ) + failure._fabric_cleanup_runtime_handle = cleanup + self._attach_cleanup_runtime(failure, plan) + if failure is not error: + raise failure from error + raise + async def start_runtime( self, config: FabricConfig, @@ -1086,7 +1126,7 @@ def start() -> dict[str, Any]: return started_runtime try: - runtime = await _call_blocking(start) + runtime = await self._call_start_with_cleanup(start, plan) except asyncio.CancelledError: if started_runtime is not None: try: @@ -1181,7 +1221,9 @@ def prepare() -> dict[str, Any]: return started_service try: - handle = ServiceHandle.from_mapping(await _call_blocking(prepare)) + handle = ServiceHandle.from_mapping( + await self._call_start_with_cleanup(prepare, plan) + ) except BaseException as error: await _release_registered_service(native, plan, started_service) if isinstance(error, (asyncio.CancelledError, FabricError)): @@ -1242,7 +1284,9 @@ def attach() -> dict[str, Any]: return attached_service try: - handle = ServiceHandle.from_mapping(await _call_blocking(attach)) + handle = ServiceHandle.from_mapping( + await self._call_start_with_cleanup(attach, plan) + ) except BaseException as error: await _release_registered_service(native, plan, attached_service) if isinstance(error, (asyncio.CancelledError, FabricError)): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py index 54851e9af..8464c1edd 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/errors.py @@ -7,7 +7,10 @@ from collections.abc import Mapping from copy import deepcopy -from typing import Any +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from .runtime import Runtime class FabricError(RuntimeError): @@ -48,6 +51,15 @@ def __init__( self.code = code self.retryable = retryable self.details = deepcopy(dict(details or {})) + self._cleanup_runtime: Runtime | None = None + + @property + def cleanup_runtime(self) -> Runtime | None: + """Owned runtime for a failed lifecycle cleanup retry, or ``None``. + + This in-memory capability is excluded from error text and structured details. + """ + return self._cleanup_runtime class FabricConfigError(FabricError): @@ -66,12 +78,16 @@ def _runtime_error(error: Exception, *, stage: str) -> FabricRuntimeError: "timeout" if isinstance(error, TimeoutError) else getattr(error, "code", None) ) retryable = getattr(error, "retryable", False) - return FabricRuntimeError( + failure = FabricRuntimeError( str(error), stage=stage, code=code if isinstance(code, str) else None, retryable=retryable if isinstance(retryable, bool) else False, ) + cleanup = getattr(error, "_fabric_cleanup_runtime_handle", None) + if cleanup is not None: + failure._fabric_cleanup_runtime_handle = cleanup + return failure class FabricStateError(FabricRuntimeError): diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py index 3438a90d2..1d2e6e634 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +from concurrent.futures import ThreadPoolExecutor import json import logging from collections.abc import Mapping, Sequence @@ -275,7 +276,16 @@ def invoke() -> dict[str, Any]: native_result = json.loads(encoded) return native_result - result = await _call_blocking(invoke) + if ( + self._current_stream is not None + and not self._current_stream._finalized + ): + result = await _call_blocking(invoke) + else: + with ThreadPoolExecutor( + max_workers=1, thread_name_prefix="fabric-invoke" + ) as executor: + result = await _call_blocking(invoke, executor=executor) typed_result = RunResult.from_mapping(result) except asyncio.CancelledError: if absorb_result and native_result is not None: @@ -297,7 +307,10 @@ def stop_after_cancel() -> Any: return result try: - await _call_blocking(stop_after_cancel) + with ThreadPoolExecutor( + max_workers=1, thread_name_prefix="fabric-stop" + ) as executor: + await _call_blocking(stop_after_cancel, executor=executor) except asyncio.CancelledError: self._status = ( RuntimeStatus.STOPPED if stopped else RuntimeStatus.FAILED @@ -309,10 +322,12 @@ def stop_after_cancel() -> Any: self._status = RuntimeStatus.STOPPED raise except FabricError: - self._status = RuntimeStatus.FAILED + if self._status is not RuntimeStatus.STOPPED: + self._status = RuntimeStatus.FAILED raise except Exception as error: - self._status = RuntimeStatus.FAILED + if self._status is not RuntimeStatus.STOPPED: + self._status = RuntimeStatus.FAILED raise _runtime_error(error, stage="invoke") from error if absorb_result: self._absorb(typed_result) @@ -794,14 +809,14 @@ def snapshot() -> Any: raise _resource_snapshot_error(native, error) from error async def stop(self) -> None: - """Destroy an idle runtime exactly once. + """Stop the runtime, interrupting an active buffered invocation if needed. Repeated calls after a successful stop are no-ops. A failed runtime may still be stopped so its resources are released. Raises: FabricStateError: If the runtime is already stopping or has an - invocation in flight. + streaming invocation in flight. FabricNativeUnavailableError: If the native extension is missing. FabricRuntimeError: If native runtime shutdown fails. """ @@ -816,8 +831,7 @@ async def stop(self) -> None: ) if self._current_stream is not None and not self._current_stream._finalized: await self._current_stream.aclose() - if self._current_task is not None: - raise FabricStateError("cannot stop while a turn is in flight") + active_task = self._current_task if self._closing: raise FabricStateError("runtime shutdown is already in progress") self._closing = True @@ -837,7 +851,15 @@ def stop() -> Any: stopped = True return result - await _call_blocking(stop) + with ThreadPoolExecutor( + max_workers=1, thread_name_prefix="fabric-stop" + ) as executor: + await _call_blocking(stop, executor=executor) + if active_task is not None and active_task is not asyncio.current_task(): + # Observe the interrupted invocation before publishing STOPPED; + # its error path must not overwrite the final lifecycle state. + while self._current_task is active_task: + await asyncio.sleep(0.01) except asyncio.CancelledError as error: self._status = RuntimeStatus.STOPPED if stopped else RuntimeStatus.FAILED stop_error = error @@ -1007,15 +1029,23 @@ async def _run_native_lifecycle( request: Mapping[str, Any], ) -> dict[str, Any]: observations = {"request_id": request.get("request_id")} + cleanup_runtime_handle: dict[str, Any] | None = None def run() -> dict[str, Any]: + nonlocal cleanup_runtime_handle plan_json = json.dumps(dict(plan)) try: runtime = json.loads(native.start_runtime(plan_json)) except FabricError as error: + cleanup_runtime_handle = getattr( + error, "_fabric_cleanup_runtime_handle", None + ) _retain_runtime_evidence(error, observations) raise except Exception as error: + cleanup_runtime_handle = getattr( + error, "_fabric_cleanup_runtime_handle", None + ) failure = _runtime_error(error, stage="start") _retain_runtime_evidence(failure, observations) raise failure from error @@ -1039,9 +1069,11 @@ def run() -> dict[str, Any]: return result finally: cleanup = "completed" + stop_error: Exception | None = None try: stop_events = json.loads(native.stop_runtime(plan_json, runtime_json)) except Exception as error: + stop_error = error cleanup = "failed" if invoke_error is None: if result is None: @@ -1064,6 +1096,21 @@ def run() -> dict[str, Any]: ) if result is not None: observations["partial_result"] = result + if ( + stop_error is not None + and _runtime_error(stop_error, stage="stop").code + == "cleanup_unconfirmed" + ): + cleanup_runtime_handle = runtime + failure = invoke_error or FabricRuntimeError( + "Runtime cleanup is unconfirmed; retry cleanup_runtime.stop().", + stage="stop", + code="cleanup_unconfirmed", + ) + failure._fabric_cleanup_runtime_handle = runtime + _retain_runtime_evidence(failure, observations) + if invoke_error is None: + raise failure from stop_error if invoke_error is not None: # Keep obtained identity and cleanup evidence beside the original # exception. A protocol failure is not a normalized invocation. @@ -1074,6 +1121,8 @@ def run() -> dict[str, Any]: except asyncio.CancelledError as error: # _call_blocking waits for native cleanup before propagating cancellation. error._fabric_runtime_evidence = dict(observations) + if cleanup_runtime_handle is not None: + error._fabric_cleanup_runtime_handle = cleanup_runtime_handle raise except FabricError: raise @@ -1101,8 +1150,15 @@ def _resource_snapshot_error(native: Any, error: Exception) -> FabricRuntimeErro return _runtime_error(error, stage="resource_snapshot") -async def _call_blocking(func: Any) -> Any: - worker = asyncio.create_task(asyncio.to_thread(func)) +async def _call_blocking( + func: Any, *, executor: ThreadPoolExecutor | None = None +) -> Any: + if executor is None: + worker = asyncio.create_task(asyncio.to_thread(func)) + else: + worker = asyncio.ensure_future( + asyncio.get_running_loop().run_in_executor(executor, func) + ) try: return await asyncio.shield(worker) except asyncio.CancelledError as cancelled: diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md index 3d4558d0c..624226d08 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/README.md @@ -51,7 +51,7 @@ Each completed check preserves an immutable `fabric.check/v1alpha1` report in `r An unavailable runtime returns `unknown/runtime_unavailable` with no agent observations. If the server knows the runtime is stopped or the SDK records a stopped or failed lifecycle, `runtime.state` preserves that state. A failed lifecycle leaves activity unknown. These lifecycle records do not establish that the agent's child processes exited. -`fabric.check/v1alpha1` identifies the shared health request and report format so readers can reject formats they do not support. The identifier stays the same across SDK releases while the health format remains compatible. `fabric-agent-ctl` ships with `nemo-fabric-runtime`; the executable and all subcommands follow the SDK package release. +`fabric.check/v1alpha1` identifies the shared health request and report format so readers can reject formats they do not support. The identifier stays the same across SDK releases while the health format remains compatible. `fabric-agent-ctl` ships with `nemo-fabric-runtime`; the executable and all subcommands follow the SDK package release. Health readers validate reason codes strictly. The caller, server, and `fabric-agent-ctl` must use the same `nemo-fabric-runtime` release; mixed-release health reports are not supported. The format identifier does not negotiate SDK releases. Report timing covers the client call, including connection, response decoding, and validation. `completed_at` records client completion; each `observed_at` records when the server collected the evidence. Successful checks, confirmed failures, and timeouts use the same timing interval. @@ -79,9 +79,9 @@ flowchart TD Caller --> Bridge --> Socket --> Host --> Runtime ``` -The outer server is separate from the SDK's adapter process and the selected harness. It serializes starts and serializes invoke/stop operations for each runtime ID, while separate runtimes can progress independently. A busy stop returns `runtime_busy` promptly and retains ownership. Retry stop after the invocation ends. Direct SDK consumers can also use streaming or shared services. +The outer server is separate from the SDK's adapter process and the selected harness. It serializes starts and invocations for each runtime ID, while separate runtimes can progress independently. Stop uses a separate path so an active buffered invocation cannot hold shutdown behind its lock. On supported Unix hosts, busy stop interrupts the owned adapter process tree and confirms exit before releasing ownership. Unconfirmed cleanup retains the runtime and blocks replacement for that agent; independent agents remain available. Direct SDK consumers can also use streaming or shared services. -Ordinary SDK stop does not establish that all harness descendants exited. A successful stop response reports completion of the SDK operation. The caller must use the sandbox supervisor to terminate remaining descendants. The server does not interrupt adapters through a new forced-stop SDK API. +A confirmed stop covers processes that remain in Fabric’s owned host tree. Processes detached before Fabric captures that tree can remain alive; the caller must use the sandbox supervisor to terminate those processes. An invocation interrupted by a requested stop reports `interrupted`, and the stop event does not report a host crash. ## Transport And Failure Behavior diff --git a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py index 9c345c237..b46e31bf3 100644 --- a/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py +++ b/sdk/python/nemo-fabric-runtime/src/nemo_fabric/runtime_server/state.py @@ -58,6 +58,7 @@ class OwnedRuntime: state: str = "running" lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False) generation: int = 0 + stop_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False) class RuntimeServer: @@ -89,10 +90,14 @@ def _owned(self, request: dict[str, Any]) -> OwnedRuntime: async def _stop(self, owned: OwnedRuntime) -> None: owned.generation += 1 - owned.state = "unknown" + owned.state = "stopping" runtime = owned.runtime - if runtime is not None: - await runtime.stop() + try: + if runtime is not None: + await runtime.stop() + except BaseException: + owned.state = "unknown" + raise owned.runtime = None owned.state = "stopped" runtime_id = runtime.runtime_id if runtime is not None else None @@ -154,6 +159,12 @@ async def handle(self, request: Any) -> dict[str, Any]: async with self._start_lock: if self.shutting_down: raise ProtocolError("host_stopping") + if any( + owned.agent == request["agent"] + and owned.state in ("stopping", "unknown") + for owned in self.runtimes.values() + ): + raise ProtocolError("cleanup_unconfirmed", "start") if self.start_uncertain: raise ProtocolError("start_uncertain", "start") stage = "validation" @@ -215,20 +226,18 @@ async def handle(self, request: Any) -> dict[str, Any]: owned = self._owned(request) if operation == "stop" and owned.state != "stopped": - if owned.lock.locked(): - raise ProtocolError("runtime_busy", "stop") owned.generation += 1 owned.state = "stopping" - async with owned.lock: - if self.shutting_down: - raise ProtocolError("host_stopping") - if operation == "stop": + if operation == "stop": + async with owned.stop_lock: if owned.state == "stopped": return envelope(operation, result) changed, effects, stage = None, "unknown", "stop" await self._stop(owned) return envelope(operation, result, changed=True) - + async with owned.lock: + if self.shutting_down: + raise ProtocolError("host_stopping") stage = "invoke" if ( owned.state != "running" @@ -292,6 +301,16 @@ async def _check_runtime( if owned.agent != command["agent"]: raise ProtocolError("wrong_agent", "check") runtime = owned.runtime + if owned.state in {"stopping", "unknown"}: + pending = owned.state == "stopping" + report = runtime_report( + request, "runtime_busy" if pending else "cleanup_unconfirmed", started + ) + report["runtime"] = { + "state": "unknown" if pending else "failed", + "activity": "stopping" if pending else "unknown", + } + return check_response(report) if ( self.shutting_down or owned.state != "running" @@ -344,7 +363,7 @@ async def shutdown(self) -> None: cleanup_failures: list[Exception] = [] async with self._start_lock: for owned in tuple(self.runtimes.values()): - async with owned.lock: + async with owned.stop_lock, owned.lock: if owned.state != "stopped": try: await self._stop(owned) diff --git a/tests/python/runtime_server/test_runtime_checks.py b/tests/python/runtime_server/test_runtime_checks.py index 2201c0099..2939e9c4b 100644 --- a/tests/python/runtime_server/test_runtime_checks.py +++ b/tests/python/runtime_server/test_runtime_checks.py @@ -263,3 +263,43 @@ async def check(**kwargs): finally: release.set() await checking + + +async def test_runtime_check_reports_pending_and_unconfirmed_cleanup( + host, host_request, mock_fabric, mock_runtime +): + from nemo_fabric import FabricRuntimeError + + await host.handle(host_request("start")) + entered, release = asyncio.Event(), asyncio.Event() + mock_fabric.check = AsyncMock() + + async def stop(): + entered.set() + await release.wait() + raise FabricRuntimeError( + "test cleanup failure", stage="stop", code="cleanup_unconfirmed" + ) + + mock_runtime.stop.side_effect = stop + stopping = asyncio.create_task(host.handle(host_request("stop"))) + try: + async with asyncio.timeout(1): + await entered.wait() + pending = await host.handle(host_request("check", runtime_id="runtime-1")) + assert pending["result"]["reason_code"] == "runtime_busy" + assert pending["result"]["runtime"] == { + "state": "unknown", + "activity": "stopping", + } + protocol.validate_response(pending, "check") + release.set() + assert (await stopping)["status"] == "failed" + failed = await host.handle(host_request("check", runtime_id="runtime-1")) + assert failed["result"]["reason_code"] == "cleanup_unconfirmed" + assert failed["result"]["runtime"] == {"state": "failed", "activity": "unknown"} + protocol.validate_response(failed, "check") + mock_fabric.check.assert_not_awaited() + finally: + release.set() + await asyncio.gather(stopping, return_exceptions=True) diff --git a/tests/python/runtime_server/test_sdk.py b/tests/python/runtime_server/test_sdk.py index 8176b9bdc..3dc8d2811 100644 --- a/tests/python/runtime_server/test_sdk.py +++ b/tests/python/runtime_server/test_sdk.py @@ -278,3 +278,47 @@ async def test_health_workers_recover_while_provider_descendants_hold_pipes( with suppress(Exception): await asyncio.wait_for(invoked, 5) await owned.stop() + + +async def test_real_sdk_busy_stop_exits_host_and_native_child( + blocking_adapter_config, tmp_path, process_is_running +): + (tmp_path / "spawn-harness").touch() + host = RuntimeServer(directory=tmp_path) + command = {"interface_version": 2, "agent": "blocking"} + started = await host.handle( + { + **command, + "operation": "start", + "config": blocking_adapter_config.to_mapping(), + } + ) + assert started["status"] == "succeeded", started + runtime_id = started["result"]["runtime_id"] + command["runtime_id"] = runtime_id + owned = host.runtimes[runtime_id].runtime + task = asyncio.create_task( + host.handle({**command, "operation": "invoke", "input": "slow"}) + ) + try: + async with asyncio.timeout(5): + while not (tmp_path / "entered").exists(): + await asyncio.sleep(0.01) + pids = [ + int((tmp_path / name).read_text()) + for name in ("adapter.pid", "harness.pid") + ] + response = await asyncio.wait_for( + host.handle({**command, "operation": "stop"}), 5 + ) + assert response["status"] == "succeeded", response + invoked = await asyncio.wait_for(task, 1) + assert invoked["error"]["code"] == "invoke_failed" + assert "Cause: interrupted." in invoked["error"]["message"] + assert all(not process_is_running(pid) for pid in pids) + assert host.runtimes[runtime_id].state == "stopped" + finally: + (tmp_path / "release").touch() + with suppress(Exception): + await task + await owned.stop() diff --git a/tests/python/runtime_server/test_session_commands.py b/tests/python/runtime_server/test_session_commands.py index 83e201ac4..bb2bb7280 100644 --- a/tests/python/runtime_server/test_session_commands.py +++ b/tests/python/runtime_server/test_session_commands.py @@ -118,7 +118,7 @@ async def test_known_start_rejection_can_be_corrected( assert (await host.handle(host_request("start")))["status"] == "succeeded" -async def test_busy_stop_returns_promptly_and_retains_ownership( +async def test_busy_stop_failure_returns_promptly_and_retains_ownership( host, host_request, mock_runtime ): await host.handle(host_request("start")) @@ -130,16 +130,25 @@ async def invoke(**kwargs): return mock_runtime.invoke.return_value mock_runtime.invoke.side_effect = invoke + mock_runtime.stop.side_effect = FabricRuntimeError( + "PRIVATE cleanup detail", code="cleanup_unconfirmed" + ) task = asyncio.create_task(host.handle(host_request("invoke", input={}))) await entered.wait() try: response = await asyncio.wait_for(host.handle(host_request("stop")), 0.5) - assert response["error"]["code"] == "runtime_busy" + assert response["error"]["code"] == "stop_failed" + assert response["error"]["effects"] == "unknown" + assert "PRIVATE" not in str(response) + assert (await host.handle(host_request("start")))["error"][ + "code" + ] == "cleanup_unconfirmed" assert host.runtimes["runtime-1"].runtime is mock_runtime - mock_runtime.stop.assert_not_awaited() + mock_runtime.stop.assert_awaited_once() finally: release.set() await task + mock_runtime.stop.side_effect = None assert (await host.handle(host_request("stop")))["status"] == "succeeded" diff --git a/tests/python/runtime_server/test_state.py b/tests/python/runtime_server/test_state.py index b77f9be39..408d60e95 100644 --- a/tests/python/runtime_server/test_state.py +++ b/tests/python/runtime_server/test_state.py @@ -294,9 +294,15 @@ async def test_failed_stop_keeps_ownership_and_allows_distinct_runtime( assert "PRIVATE" not in str(response) assert host.runtimes["runtime-1"].runtime is mock_runtime assert host.runtimes["runtime-1"].state == "unknown" - replacement = runtime_factory("runtime-2") + replacement = runtime_factory("runtime-2", agent="independent") mock_fabric.start_runtime.return_value = replacement - assert (await host.handle(host_request("start")))["status"] == "succeeded" + config = { + "metadata": {"name": "independent"}, + "harness": {"adapter_id": "test.host"}, + } + assert ( + await host.handle(host_request("start", agent="independent", config=config)) + )["status"] == "succeeded" assert mock_fabric.start_runtime.await_count == 2 @@ -389,7 +395,7 @@ async def test_non_active_returned_runtime_is_owned_for_stop( assert next_response["result"]["runtime_id"] == "runtime-2" -async def test_stop_can_be_retried_after_a_busy_invocation( +async def test_stop_does_not_wait_for_the_invocation_lock( host, host_request, mock_runtime ): await host.handle(host_request("start")) @@ -406,11 +412,10 @@ async def invoke(**kwargs): await entered.wait() second = asyncio.create_task(host.handle(host_request("stop"))) await asyncio.sleep(0) - mock_runtime.stop.assert_not_awaited() - assert (await second)["error"]["code"] == "runtime_busy" + mock_runtime.stop.assert_awaited_once() release.set() assert (await first)["status"] == "succeeded" - assert (await host.handle(host_request("stop")))["status"] == "succeeded" + assert (await second)["status"] == "succeeded" mock_runtime.stop.assert_awaited_once_with() @@ -492,3 +497,80 @@ async def test_stopped_runtime_records_are_bounded( assert (await host.handle(host_request("stop", runtime_id="runtime-129")))[ "status" ] == "succeeded" + + +async def test_busy_stop_reaches_sdk_before_invocation_finishes( + host, host_request, mock_runtime +): + await host.handle(host_request("start")) + entered, release = asyncio.Event(), asyncio.Event() + native = mock_runtime.invoke.return_value + + async def invoke(**kwargs): + entered.set() + await release.wait() + return native + + async def stop(): + release.set() + + mock_runtime.invoke.side_effect = invoke + mock_runtime.stop.side_effect = stop + task = asyncio.create_task(host.handle(host_request("invoke", input={}))) + await entered.wait() + try: + response = await asyncio.wait_for(host.handle(host_request("stop")), 0.5) + assert response["status"] == "succeeded" + mock_runtime.stop.assert_awaited_once() + finally: + release.set() + await task + + +async def test_shutdown_waits_for_a_pending_stop(host, host_request, mock_runtime): + await host.handle(host_request("start")) + entered, release = asyncio.Event(), asyncio.Event() + + async def stop(): + if entered.is_set(): + raise FabricStateError("shutdown is already in progress") + entered.set() + await release.wait() + + mock_runtime.stop.side_effect = stop + stopping = asyncio.create_task(host.handle(host_request("stop"))) + await entered.wait() + shutdown = asyncio.create_task(host.shutdown()) + try: + async with asyncio.timeout(1): + while not host.shutting_down: + await asyncio.sleep(0) + assert not shutdown.done() + release.set() + assert (await stopping)["status"] == "succeeded" + await shutdown + mock_runtime.stop.assert_awaited_once() + finally: + release.set() + await asyncio.gather(stopping, shutdown, return_exceptions=True) + + +async def test_unconfirmed_stop_blocks_replacement_but_not_other_agents( + host, host_request, mock_runtime, mock_fabric, runtime_factory, config +): + await host.handle(host_request("start")) + mock_runtime.stop.side_effect = RuntimeError("PRIVATE shutdown detail") + assert (await host.handle(host_request("stop")))["error"]["code"] == "stop_failed" + rejected = await host.handle(host_request("start")) + assert rejected["error"]["code"] == "cleanup_unconfirmed" + assert "PRIVATE" not in str(rejected) + mock_fabric.start_runtime.assert_awaited_once() + mock_fabric.start_runtime.return_value = runtime_factory( + "independent", agent="other" + ) + other_config = {**config, "metadata": {"name": "other"}} + assert ( + await host.handle(host_request("start", agent="other", config=other_config)) + )["status"] == "succeeded" + mock_runtime.stop.side_effect = None + assert (await host.handle(host_request("stop")))["status"] == "succeeded" diff --git a/tests/python/test_native_sdk.py b/tests/python/test_native_sdk.py index ab1c93b41..716be5aa6 100644 --- a/tests/python/test_native_sdk.py +++ b/tests/python/test_native_sdk.py @@ -461,3 +461,41 @@ async def smoke(client: Fabric, fixture_agent: Path) -> None: assert empty_chunks == [] assert empty_result.status == "succeeded" assert empty_result.output["openai_stream_invocation_count"] == 2 + + +@pytest.mark.skipif( + sys.platform == "win32", reason="Unix host directory repair fixture" +) +@pytest.mark.parametrize("entry", ["start_runtime", "run"]) +async def test_native_failed_start_exposes_a_private_cleanup_retry( + blocking_adapter_config, tmp_path, entry +): + from nemo_fabric import RuntimeStatus + + script = tmp_path / "adapter.py" + source = script.read_text() + anchor = " if failure.exists():\n" + replacement = """ if failure.exists(): + if operation == 'start': + import shutil + directory = Path(os.environ['FABRIC_HOME']) + (root/'cleanup.path').write_text(str(directory)) + shutil.rmtree(directory) + directory.write_text('cleanup requires repair') +""" + assert source.count(anchor) == 1 + script.write_text(source.replace(anchor, replacement)) + (tmp_path / "fail-start").write_text("fixture_start_failure") + with pytest.raises(FabricRuntimeError) as caught: + await getattr(Fabric(), entry)(blocking_adapter_config, base_dir=tmp_path) + recovery = caught.value.cleanup_runtime + assert recovery is not None + assert caught.value.code == "cleanup_unconfirmed" + assert "runtime_binding" not in repr(caught.value) + assert "PRIVATE detail" not in str(caught.value) + directory = Path((tmp_path / "cleanup.path").read_text()) + assert directory.is_file() + directory.unlink() + directory.mkdir(mode=0o700) + await recovery.stop() + assert recovery.status is RuntimeStatus.STOPPED diff --git a/tests/python/test_runtime.py b/tests/python/test_runtime.py index f7f730867..8499a2ce0 100644 --- a/tests/python/test_runtime.py +++ b/tests/python/test_runtime.py @@ -10,6 +10,7 @@ import logging import threading from contextlib import AsyncExitStack +from concurrent.futures import ThreadPoolExecutor from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -159,6 +160,154 @@ def _runtime_wrapper( ) +@pytest.mark.parametrize("cancel_invocation", [False, True]) +async def test_stop_settles_an_invocation_when_the_default_executor_is_full( + mock_native, cancel_invocation +): + runtime = _runtime_wrapper(mock_native) + loop = asyncio.get_running_loop() + await asyncio.to_thread(lambda: None) + previous = loop._default_executor + occupied, release, stopped = threading.Event(), threading.Event(), threading.Event() + original_invoke = mock_native.invoke_runtime.side_effect + + def invoke(*args): + assert stopped.wait(5) + return original_invoke(*args) + + def stop(*args): + stopped.set() + return json.dumps([]) + + mock_native.invoke_runtime.side_effect = invoke + mock_native.stop_runtime.side_effect = stop + with ThreadPoolExecutor(max_workers=1) as executor: + loop.set_default_executor(executor) + worker = asyncio.create_task( + asyncio.to_thread(lambda: (occupied.set(), release.wait(5))) + ) + invoking = None + stopping = None + try: + async with asyncio.timeout(1): + while not occupied.is_set(): + await asyncio.sleep(0) + invoking = asyncio.create_task(runtime.invoke(input="queued turn")) + async with asyncio.timeout(1): + while runtime._current_task is not invoking: + await asyncio.sleep(0) + if cancel_invocation: + invoking.cancel() + stopping = asyncio.create_task(runtime.stop()) + done, _ = await asyncio.wait({stopping}, timeout=0.5) + completed_before_release = stopping in done + finally: + release.set() + stopped.set() + await worker + if invoking is not None: + outcome = (await asyncio.gather(invoking, return_exceptions=True))[0] + if stopping is not None: + await stopping + loop.set_default_executor(previous) + assert isinstance(outcome, asyncio.CancelledError) == cancel_invocation + assert completed_before_release + assert runtime.status is RuntimeStatus.STOPPED + + +@pytest.mark.parametrize("invoke_fails", [False, True]) +async def test_one_shot_retains_a_cleanup_runtime_without_exposing_its_handle( + native_client, mock_native, invoke_fails +): + handle = _runtime() + handle["environment"]["env"] = {"RECOVERY_TEST_SECRET": "private-cleanup-value"} + mock_native.start_runtime.return_value = json.dumps(handle) + original = FabricRuntimeError( + "test invocation failure", stage="invoke", code="test_failure" + ) + if invoke_fails: + mock_native.invoke_runtime.side_effect = original + mock_native.stop_runtime.side_effect = [ + FabricRuntimeError( + "test cleanup failure", stage="stop", code="cleanup_unconfirmed" + ), + json.dumps([]), + ] + with pytest.raises(FabricRuntimeError) as caught: + await native_client.run(_config(), input="hello") + if invoke_fails: + assert caught.value is original + cleanup_runtime = caught.value.cleanup_runtime + assert isinstance(cleanup_runtime, Runtime) + assert cleanup_runtime.handle.runtime_id == "runtime-1" + assert "private-cleanup-value" not in repr(caught.value) + assert "fabric-runtime-binding-test" not in json.dumps(caught.value.details) + await cleanup_runtime.stop() + assert cleanup_runtime.status is RuntimeStatus.STOPPED + assert mock_native.stop_runtime.call_count == 2 + + +async def test_cancelled_one_shot_retains_a_cleanup_runtime(native_client, mock_native): + entered, release = threading.Event(), threading.Event() + + def stop(*args): + if mock_native.stop_runtime.call_count == 1: + entered.set() + assert release.wait(5) + raise FabricRuntimeError( + "test cleanup failure", stage="stop", code="cleanup_unconfirmed" + ) + return json.dumps([]) + + mock_native.stop_runtime.side_effect = stop + running = asyncio.create_task(native_client.run(_config(), input="hello")) + try: + async with asyncio.timeout(2): + while not entered.is_set(): + await asyncio.sleep(0.01) + running.cancel() + release.set() + with pytest.raises(asyncio.CancelledError) as caught: + await running + await caught.value.cleanup_runtime.stop() + assert mock_native.stop_runtime.call_count == 2 + finally: + release.set() + await asyncio.gather(running, return_exceptions=True) + + +async def test_stop_does_not_wait_for_a_saturated_default_executor(mock_native): + runtime = _runtime_wrapper(mock_native) + loop = asyncio.get_running_loop() + await asyncio.to_thread(lambda: None) + previous_executor = loop._default_executor + entered, release = threading.Event(), threading.Event() + + def occupy_worker(): + entered.set() + release.wait(5) + + with ThreadPoolExecutor(max_workers=1) as executor: + loop.set_default_executor(executor) + occupied = asyncio.create_task(asyncio.to_thread(occupy_worker)) + try: + async with asyncio.timeout(3): + while not entered.is_set(): + await asyncio.sleep(0.01) + stopping = asyncio.create_task(runtime.stop()) + try: + completed, _ = await asyncio.wait({stopping}, timeout=0.5) + finally: + release.set() + await stopping + finally: + release.set() + await occupied + loop.set_default_executor(previous_executor) + assert stopping in completed + mock_native.stop_runtime.assert_called_once() + + @pytest.mark.parametrize("stage", ["start", "invoke", "stop"]) @pytest.mark.parametrize( ("error_type", "expected_code"), @@ -906,7 +1055,7 @@ async def test_stop_closes_runtime_owned_collector(mock_native: MagicMock): mock_collector.aclose.assert_awaited_once() -async def test_stop_rejects_in_flight_turn( +async def test_stop_interrupts_in_flight_turn( mock_native: MagicMock, ): started = threading.Event() @@ -923,11 +1072,16 @@ def blocking_invoke(*args: Any) -> str: turn = asyncio.create_task(runtime.invoke(input="hello")) assert await _wait_for(started) - with pytest.raises(FabricStateError, match="in flight"): - await runtime.stop() + stop_result = mock_native.stop_runtime.return_value - release.set() + def interrupt(*args): + release.set() + return stop_result + + mock_native.stop_runtime.side_effect = interrupt + await asyncio.wait_for(runtime.stop(), 2) await turn + assert runtime.status is RuntimeStatus.STOPPED async def test_concurrent_invokes_are_rejected( @@ -1702,3 +1856,86 @@ async def test_runtime_status_handle_mutation_cannot_change_shared_report( assert report.handle.environment == original["handle"]["environment"] assert report.to_mapping() == original await runtime.stop() + + +async def test_stop_waits_for_invocation_not_the_callers_remaining_work(mock_native): + entered, release = threading.Event(), threading.Event() + later_work = asyncio.Event() + completed = asyncio.Event() + invoke = mock_native.invoke_runtime.side_effect + stop_result = mock_native.stop_runtime.return_value + + def blocking(*args): + entered.set() + assert release.wait(5) + return invoke(*args) + + def interrupt(*args): + release.set() + return stop_result + + mock_native.invoke_runtime.side_effect = blocking + mock_native.stop_runtime.side_effect = interrupt + runtime = _runtime_wrapper(mock_native) + + async def caller(): + await runtime.invoke(input="hello") + completed.set() + await later_work.wait() + + task = asyncio.create_task(caller()) + assert await _wait_for(entered) + stopping = asyncio.create_task(runtime.stop()) + try: + await asyncio.wait_for(completed.wait(), 2) + await asyncio.wait_for(asyncio.shield(stopping), 0.5) + assert not task.done() + assert runtime.status is RuntimeStatus.STOPPED + finally: + release.set() + later_work.set() + await task + await stopping + + +@pytest.mark.parametrize("entry", ["start_runtime", "run", "prepare_service"]) +@pytest.mark.parametrize("cancel_start", [False, True]) +async def test_failed_start_preserves_private_cleanup_runtime( + native_client, mock_native, entry, cancel_start +): + entered, release = threading.Event(), threading.Event() + handle = _runtime() + handle["environment"]["env"] = {"TEST_STARTUP_SECRET": "private-startup-value"} + + def start(*args): + entered.set() + assert release.wait(5) + error = RuntimeError("cleanup unconfirmed") + error.code = "cleanup_unconfirmed" + error._fabric_cleanup_runtime_handle = handle + raise error + + method = "prepare_service" if entry == "prepare_service" else "start_runtime" + getattr(mock_native, method).side_effect = start + task = asyncio.create_task(getattr(native_client, entry)(_config())) + try: + async with asyncio.timeout(2): + while not entered.is_set(): + await asyncio.sleep(0.01) + if cancel_start: + task.cancel() + release.set() + expected = asyncio.CancelledError if cancel_start else FabricRuntimeError + with pytest.raises(expected) as caught: + await task + recovery = caught.value.cleanup_runtime + assert isinstance(recovery, Runtime) + assert recovery.runtime_id == handle["runtime_id"] + assert "private-startup-value" not in repr(caught.value) + if isinstance(caught.value, FabricRuntimeError): + assert "private-startup-value" not in json.dumps(caught.value.details) + await recovery.stop() + assert recovery.status is RuntimeStatus.STOPPED + finally: + release.set() + await asyncio.gather(task, return_exceptions=True)