Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ def _bounded_int(value: int | None, field_name: str, maximum: int) -> None:
CheckReason = Literal[
"server_stopping",
"capabilities_unverified",
"runtime_busy",
"cleanup_unconfirmed",
"runtime_unavailable",
"service_unavailable",
"agent_exited",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,16 @@
"description": "Advertised capability evidence is unavailable.",
"type": "string"
},
{
"const": "runtime_busy",
"description": "Runtime cleanup is in progress.",
"type": "string"
},
{
"const": "cleanup_unconfirmed",
"description": "Owned runtime cleanup could not be confirmed.",
"type": "string"
},
{
"const": "runtime_unavailable",
"description": "No matching runtime binding exists.",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,8 @@ export type AdapterCheckResult = {
export type CheckReason =
| "server_stopping"
| "capabilities_unverified"
| "runtime_busy"
| "cleanup_unconfirmed"
| "runtime_unavailable"
| "service_unavailable"
| "agent_exited"
Expand Down
4 changes: 4 additions & 0 deletions crates/fabric-core/src/check.rs
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,10 @@ pub enum CheckReason {
ServerStopping,
/// Advertised capability evidence is unavailable.
CapabilitiesUnverified,
/// Runtime cleanup is in progress.
RuntimeBusy,
/// Owned runtime cleanup could not be confirmed.
CleanupUnconfirmed,
/// No matching runtime binding exists.
RuntimeUnavailable,
/// No matching active service binding exists.
Expand Down
58 changes: 58 additions & 0 deletions crates/fabric-core/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,9 @@ pub enum FabricError {
/// Bounded adapter-host diagnostics.
diagnostics: String,
},
/// Owned lifecycle cleanup is unconfirmed; the original runtime remains owned.
#[error("{0}")]
RuntimeCleanupPending(Box<RuntimeCleanupPending>),
/// A runtime handle was used with a different run plan than the one that created it.
#[error(
"runtime handle does not match run plan for `{field}`: expected `{expected}` but found `{actual}` (runtime `{runtime_id}`)"
Expand Down Expand Up @@ -436,3 +439,58 @@ pub enum FabricError {
source: serde_json::Error,
},
}

/// Cleanup context retained when startup or invocation cannot finish owned cleanup.
///
/// Retry [`crate::stop_runtime`] with the original plan and [`Self::runtime`].
/// The handle and completed output are deliberately excluded from diagnostic formatting.
pub struct RuntimeCleanupPending {
runtime: crate::RuntimeHandle,
result: Option<crate::RunResult>,
cause: FabricError,
}

impl RuntimeCleanupPending {
pub(crate) fn new(
runtime: crate::RuntimeHandle,
result: Option<crate::RunResult>,
cause: FabricError,
) -> Self {
Self {
runtime,
result,
cause,
}
}

/// The original handle required for a cleanup retry in this process.
pub fn runtime(&self) -> &crate::RuntimeHandle {
&self.runtime
}

/// The completed invocation result, if invocation succeeded before cleanup failed.
pub fn result(&self) -> Option<&crate::RunResult> {
self.result.as_ref()
}

/// The startup, invocation, or stop error that preceded the retained cleanup context.
pub fn cause(&self) -> &FabricError {
&self.cause
}
}

impl std::fmt::Debug for RuntimeCleanupPending {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("RuntimeCleanupPending")
.field("result_available", &self.result.is_some())
.finish_non_exhaustive()
}
}

impl std::fmt::Display for RuntimeCleanupPending {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(
"runtime cleanup is unconfirmed; retry stop with RuntimeCleanupPending::runtime()",
)
}
}
2 changes: 1 addition & 1 deletion crates/fabric-core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ pub use config::{
resolve_run_plan_from_config_with_adapter_directories,
};
pub use doctor::{DoctorCheck, DoctorReport, DoctorStatus, doctor_plan};
pub use error::{FabricError, Result};
pub use error::{FabricError, Result, RuntimeCleanupPending};
pub use provenance::{ExecutionEvidence, ProvenanceConfig};
pub use resource_snapshot::RuntimeResourceSnapshot;
pub use runtime::{
Expand Down
38 changes: 38 additions & 0 deletions crates/fabric-core/src/resource_snapshot.rs
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,12 @@ impl LocalHostProcess {
platform::has_exited(self.pid, self.identity)
}

/// Whether the same process is suspended and cannot create children.
#[cfg(unix)]
pub(crate) fn is_stopped(&self) -> Option<bool> {
platform::is_stopped(self.pid, self.identity?)
}

/// Record the host `pid`, started by NeMo Fabric at `started_at`.
pub(crate) fn new(pid: u32, started_at: SystemTime) -> Self {
Self {
Expand Down Expand Up @@ -328,6 +334,7 @@ mod platform {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) struct Stat {
pub zombie: bool,
pub stopped: bool,
pub ppid: u32,
/// `utime + stime + cutime + cstime`, in clock ticks.
pub cpu_ticks: u64,
Expand All @@ -348,6 +355,7 @@ mod platform {
.sum::<Option<u64>>()?;
Some(Stat {
zombie: fields.first() == Some(&"Z"),
stopped: matches!(fields.first(), Some(&"T" | &"t")),
ppid: u32::try_from(number(1)?).ok()?,
cpu_ticks: ticks,
start: number(19)?,
Expand Down Expand Up @@ -540,6 +548,11 @@ mod platform {
}
}

pub(super) fn is_stopped(pid: u32, identity: u64) -> Option<bool> {
let stat = stat(pid).ok()?;
Some(stat.start == identity && stat.stopped)
}

fn sysconf(name: libc::c_int) -> u64 {
// SAFETY: sysconf has no memory-safety preconditions.
u64::try_from(unsafe { libc::sysconf(name) }).unwrap_or(0)
Expand Down Expand Up @@ -731,6 +744,26 @@ mod platform {
}
}

pub(super) fn is_stopped(pid: u32, identity: u64) -> Option<bool> {
// SAFETY: proc_bsdinfo is an output structure with a valid zero representation.
let mut info: libc::proc_bsdinfo = unsafe { std::mem::zeroed() };
let size = std::mem::size_of_val(&info) as libc::c_int;
// SAFETY: the writable buffer matches PROC_PIDTBSDINFO and its size.
let read = unsafe {
libc::proc_pidinfo(
pid as libc::c_int,
libc::PROC_PIDTBSDINFO,
0,
(&mut info as *mut libc::proc_bsdinfo).cast(),
size,
)
};
if read != size || start_identity(pid) != Some(identity) {
return None;
}
Some(info.pbi_status == libc::SSTOP)
}

pub(super) fn read_tree(root: u32, identity: u64) -> Result<TreeRead, &'static str> {
let timebase = timebase().ok_or(PROCESS_UNREADABLE)?;
read_tree_with(&Live { timebase }, root, identity)
Expand All @@ -751,6 +784,11 @@ mod platform {
None
}

#[cfg(unix)]
pub(super) fn is_stopped(_pid: u32, _identity: u64) -> Option<bool> {
None
}

pub(super) fn read_tree(_root: u32, _identity: u64) -> Result<TreeRead, &'static str> {
Err("no process reader for this operating system")
}
Expand Down
Loading
Loading