host: report restore failure facts without lifecycle policy

nvx-rs/public-api/linux.txt

@@ -2123,6 +2123,12 @@ field nvx::host::report::Report::status_code
 field nvx::host::report::Report::teardown_complete
 fn nvx::host::report::read
 mod nvx::host::restore
+enum nvx::host::restore::Attribution
+variant nvx::host::restore::Attribution::Host
+variant nvx::host::restore::Attribution::Repair
+variant nvx::host::restore::Attribution::Request
+variant nvx::host::restore::Attribution::Snapshot
+variant nvx::host::restore::Attribution::Unknown
 struct nvx::host::restore::ReadyListener
 method nvx::host::restore::ReadyListener::into_std
 enum nvx::host::restore::RestoreAction
@@ -2134,17 +2140,42 @@ field nvx::host::restore::RestoreAction::Fail::0
 variant nvx::host::restore::RestoreAction::SendActivate
 field nvx::host::restore::RestoreAction::SendActivate::0
 struct nvx::host::restore::RestoreError
+field nvx::host::restore::RestoreError::attribution
 field nvx::host::restore::RestoreError::evidence
+field nvx::host::restore::RestoreError::guest_may_have_run
+field nvx::host::restore::RestoreError::message
 field nvx::host::restore::RestoreError::phase
 field nvx::host::restore::RestoreError::source
 enum nvx::host::restore::RestoreEvidence
 variant nvx::host::restore::RestoreEvidence::Cancelled
+variant nvx::host::restore::RestoreEvidence::EndpointBind
 variant nvx::host::restore::RestoreEvidence::Exited
 variant nvx::host::restore::RestoreEvidence::Expired
+variant nvx::host::restore::RestoreEvidence::ForcedKill
+variant nvx::host::restore::RestoreEvidence::ForwardBind
+variant nvx::host::restore::RestoreEvidence::HostFailure
+variant nvx::host::restore::RestoreEvidence::IncompatibleHost
+variant nvx::host::restore::RestoreEvidence::InvalidActivate
+variant nvx::host::restore::RestoreEvidence::InvalidMappings
+variant nvx::host::restore::RestoreEvidence::InvalidSettings
+variant nvx::host::restore::RestoreEvidence::InvalidTargets
 variant nvx::host::restore::RestoreEvidence::Io
+variant nvx::host::restore::RestoreEvidence::LostStatus
 variant nvx::host::restore::RestoreEvidence::Malformed
+variant nvx::host::restore::RestoreEvidence::MalformedSnapshot
 variant nvx::host::restore::RestoreEvidence::PeerMismatch
+variant nvx::host::restore::RestoreEvidence::RepairStatus
 variant nvx::host::restore::RestoreEvidence::ReportUnreadable
+variant nvx::host::restore::RestoreEvidence::Reset
+variant nvx::host::restore::RestoreEvidence::ResourceExhaustion
+variant nvx::host::restore::RestoreEvidence::RuntimeMismatch
+variant nvx::host::restore::RestoreEvidence::SnapshotDefect
+variant nvx::host::restore::RestoreEvidence::SuppliedGeometry
+variant nvx::host::restore::RestoreEvidence::SuppliedIdentity
+variant nvx::host::restore::RestoreEvidence::SuppliedLength
+variant nvx::host::restore::RestoreEvidence::UnservableImage
+variant nvx::host::restore::RestoreEvidence::UnusableRoot
+variant nvx::host::restore::RestoreEvidence::UnusableScratch
 enum nvx::host::restore::RestoreInput
 variant nvx::host::restore::RestoreInput::ActivateSent
 variant nvx::host::restore::RestoreInput::Cancelled

nvx-rs/public-api/windows.txt

@@ -1046,6 +1046,12 @@ field nvx::host::report::Report::status_code
 field nvx::host::report::Report::teardown_complete
 fn nvx::host::report::read
 mod nvx::host::restore
+enum nvx::host::restore::Attribution
+variant nvx::host::restore::Attribution::Host
+variant nvx::host::restore::Attribution::Repair
+variant nvx::host::restore::Attribution::Request
+variant nvx::host::restore::Attribution::Snapshot
+variant nvx::host::restore::Attribution::Unknown
 struct nvx::host::restore::ReadyListener
 method nvx::host::restore::ReadyListener::into_std
 enum nvx::host::restore::RestoreAction
@@ -1057,17 +1063,42 @@ field nvx::host::restore::RestoreAction::Fail::0
 variant nvx::host::restore::RestoreAction::SendActivate
 field nvx::host::restore::RestoreAction::SendActivate::0
 struct nvx::host::restore::RestoreError
+field nvx::host::restore::RestoreError::attribution
 field nvx::host::restore::RestoreError::evidence
+field nvx::host::restore::RestoreError::guest_may_have_run
+field nvx::host::restore::RestoreError::message
 field nvx::host::restore::RestoreError::phase
 field nvx::host::restore::RestoreError::source
 enum nvx::host::restore::RestoreEvidence
 variant nvx::host::restore::RestoreEvidence::Cancelled
+variant nvx::host::restore::RestoreEvidence::EndpointBind
 variant nvx::host::restore::RestoreEvidence::Exited
 variant nvx::host::restore::RestoreEvidence::Expired
+variant nvx::host::restore::RestoreEvidence::ForcedKill
+variant nvx::host::restore::RestoreEvidence::ForwardBind
+variant nvx::host::restore::RestoreEvidence::HostFailure
+variant nvx::host::restore::RestoreEvidence::IncompatibleHost
+variant nvx::host::restore::RestoreEvidence::InvalidActivate
+variant nvx::host::restore::RestoreEvidence::InvalidMappings
+variant nvx::host::restore::RestoreEvidence::InvalidSettings
+variant nvx::host::restore::RestoreEvidence::InvalidTargets
 variant nvx::host::restore::RestoreEvidence::Io
+variant nvx::host::restore::RestoreEvidence::LostStatus
 variant nvx::host::restore::RestoreEvidence::Malformed
+variant nvx::host::restore::RestoreEvidence::MalformedSnapshot
 variant nvx::host::restore::RestoreEvidence::PeerMismatch
+variant nvx::host::restore::RestoreEvidence::RepairStatus
 variant nvx::host::restore::RestoreEvidence::ReportUnreadable
+variant nvx::host::restore::RestoreEvidence::Reset
+variant nvx::host::restore::RestoreEvidence::ResourceExhaustion
+variant nvx::host::restore::RestoreEvidence::RuntimeMismatch
+variant nvx::host::restore::RestoreEvidence::SnapshotDefect
+variant nvx::host::restore::RestoreEvidence::SuppliedGeometry
+variant nvx::host::restore::RestoreEvidence::SuppliedIdentity
+variant nvx::host::restore::RestoreEvidence::SuppliedLength
+variant nvx::host::restore::RestoreEvidence::UnservableImage
+variant nvx::host::restore::RestoreEvidence::UnusableRoot
+variant nvx::host::restore::RestoreEvidence::UnusableScratch
 enum nvx::host::restore::RestoreInput
 variant nvx::host::restore::RestoreInput::ActivateSent
 variant nvx::host::restore::RestoreInput::Cancelled

nvx-rs/src/host/restore.rs

@@ -22,6 +22,7 @@ use super::report::Report;
 use super::vm::VmDir;
 use crate::activation::{ActivationRecord, ReadyActivation};
 use crate::protocol::restore_ready::parse_ready_event;
+use crate::protocol::status;
 use crate::{Error, Result};
 
 /// The pending part of a restore.
@@ -50,15 +51,133 @@ pub enum RestoreEvidence {
     Malformed,
     /// The caller could not read the VM's outcome report.
     ReportUnreadable,
-    /// The caller's budget expired.
+    /// The caller's budget or the guest's activation deadline expired.
     Expired,
     /// The caller cancelled the restore.
     Cancelled,
     /// The caller verified that the VM exited before Ready.
     Exited,
+    /// The requested resources are invalid or the guest refused them.
+    InvalidTargets,
+    /// The requested settings cannot be applied.
+    InvalidSettings,
+    /// The activation record is invalid or the guest refused it.
+    InvalidActivate,
+    /// The requested mappings or export cannot be applied.
+    InvalidMappings,
+    /// The snapshot cannot serve the requested image.
+    UnservableImage,
+    /// The requested scratch is not usable.
+    UnusableScratch,
+    /// The requested root cannot be assembled.
+    UnusableRoot,
+    /// A supplied file has the wrong identity.
+    SuppliedIdentity,
+    /// A supplied file has the wrong length.
+    SuppliedLength,
+    /// A supplied file has the wrong device geometry.
+    SuppliedGeometry,
+    /// A restore endpoint could not be bound.
+    EndpointBind,
+    /// A requested network forward could not be bound.
+    ForwardBind,
+    /// The host exhausted storage, memory or descriptors.
+    ResourceExhaustion,
+    /// The host cannot present the snapshot's CPU or time contract.
+    IncompatibleHost,
+    /// The restore uses another runtime identity.
+    RuntimeMismatch,
+    /// Another host operation failed.
+    HostFailure,
+    /// The VM ended without a usable status.
+    LostStatus,
+    /// The control session reset or ended.
+    Reset,
+    /// The caller or another controller forcibly ended the VM.
+    ForcedKill,
+    /// The snapshot is malformed or inconsistent.
+    MalformedSnapshot,
+    /// A diagnosis attributes a capture or repair defect to the snapshot.
+    SnapshotDefect,
+    /// A guest repair or time ABI status, whose provenance is not established.
+    RepairStatus,
 }
 
-/// A restore's failure phase, evidence and original cause.
+impl RestoreEvidence {
+    /// Attributes evidence without prescribing a retry or disposition.
+    fn attribution(self) -> Attribution {
+        match self {
+            Self::InvalidTargets
+            | Self::InvalidSettings
+            | Self::InvalidActivate
+            | Self::InvalidMappings
+            | Self::UnservableImage
+            | Self::UnusableScratch
+            | Self::UnusableRoot
+            | Self::SuppliedIdentity
+            | Self::SuppliedLength
+            | Self::SuppliedGeometry => Attribution::Request,
+            Self::Io
+            | Self::ReportUnreadable
+            | Self::EndpointBind
+            | Self::ForwardBind
+            | Self::ResourceExhaustion
+            | Self::IncompatibleHost
+            | Self::RuntimeMismatch
+            | Self::HostFailure => Attribution::Host,
+            Self::PeerMismatch
+            | Self::Malformed
+            | Self::Expired
+            | Self::Cancelled
+            | Self::Exited
+            | Self::LostStatus
+            | Self::Reset
+            | Self::ForcedKill => Attribution::Unknown,
+            Self::MalformedSnapshot | Self::SnapshotDefect => Attribution::Snapshot,
+            Self::RepairStatus => Attribution::Repair,
+        }
+    }
+
+    /// Classifies only a caller-supplied guest outcome before workload Start.
+    fn reported(report: &Report) -> Option<Self> {
+        if !matches!(report.category.as_deref(), Some("guest-exit" | "success")) {
+            return None;
+        }
+        let code = u8::try_from(report.status_code?).ok()?;
+        if status::TIME_STATUSES.contains(&code) || status::REPAIR_STATUSES.contains(&code) {
+            return Some(Self::RepairStatus);
+        }
+        // The machine never sends Start, so no workload can have supplied an activation status.
+        Some(match code {
+            status::ACTIVATION_DEADLINE => Self::Expired,
+            status::ACTIVATION_PROTOCOL | status::ACTIVATION_ORDER => Self::Malformed,
+            status::ACTIVATION_RECORD => Self::InvalidActivate,
+            status::ACTIVATION_RESOURCES => Self::InvalidTargets,
+            status::ACTIVATION_ROOT => Self::UnusableRoot,
+            status::ACTIVATION_WORKLOAD | status::ACTIVATION_NETWORK => Self::InvalidSettings,
+            status::ACTIVATION_EXPORT => Self::InvalidMappings,
+            _ => return None,
+        })
+    }
+}
+
+/// What a restore failure's evidence attributes it to, without a disposition or retry policy.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+#[non_exhaustive]
+pub enum Attribution {
+    /// The requested image, upper, resources or settings.
+    Request,
+    /// The host or its own operations.
+    Host,
+    /// The snapshot's payload or a diagnosed capture or repair defect.
+    Snapshot,
+    /// The guest's repair or time ABI, whose provenance is not established.
+    Repair,
+    /// The host, protocol or neither: the evidence establishes no provenance.
+    Unknown,
+}
+
+/// A restore's failure facts and original cause.
 ///
 /// This error neither stops the VM nor records a disposition. The caller retains its process,
 /// session and storage and decides what to do with them.
@@ -67,12 +186,49 @@ pub enum RestoreEvidence {
 pub struct RestoreError {
     /// The phase of the failed operation.
     pub phase: RestorePhase,
+    /// What the evidence attributes the failure to, without prescribing a response.
+    pub attribution: Attribution,
     /// What the caller or the machine observed.
     pub evidence: RestoreEvidence,
+    /// Whether guest code could have executed since the caller reported Spawned.
+    ///
+    /// This remains true after session loss or verified exit. It does not prove execution,
+    /// imply that the VM still runs, or mean that a workload Start was sent.
+    pub guest_may_have_run: bool,
+    /// The failure's description, including the messages of its original cause chain.
+    pub message: String,
     /// The original cause, when the failed operation supplied one.
     pub source: Option<Box<dyn StdError + Send + Sync>>,
 }
 
+impl RestoreError {
+    /// Captures failure facts while retaining the original cause and its complete messages.
+    fn new(
+        phase: RestorePhase,
+        evidence: RestoreEvidence,
+        guest_may_have_run: bool,
+        source: Option<Box<dyn StdError + Send + Sync>>,
+    ) -> Self {
+        let mut message = format!("restore failed during {phase:?}: {evidence:?}");
+        let mut cause = source
+            .as_deref()
+            .map(|source| source as &(dyn StdError + 'static));
+        while let Some(source) = cause {
+            message.push_str(": ");
+            message.push_str(&source.to_string());
+            cause = source.source();
+        }
+        Self {
+            phase,
+            attribution: evidence.attribution(),
+            evidence,
+            guest_may_have_run,
+            message,
+            source,
+        }
+    }
+}
+
 impl fmt::Display for RestoreError {
     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(
@@ -186,6 +342,8 @@ pub struct RestoreMachine {
     encoded: Vec<u8>,
     /// The activation that a successful Ready must report.
     expected: ReadyActivation,
+    /// Guest execution cannot be ruled out once the caller reports a successful spawn.
+    guest_may_have_run: bool,
     /// The pending operation or terminal state.
     state: MachineState,
     /// The next action, which an input may invalidate before it is taken.
@@ -201,13 +359,14 @@ impl RestoreMachine {
         let mut machine = Self {
             encoded: record.encode(),
             expected: ReadyActivation::of(record),
+            guest_may_have_run: false,
             state: MachineState::Spawn,
             action: None,
         };
         if record.restore().is_none() {
             machine.fail(
                 RestorePhase::ReadyEvent,
-                RestoreEvidence::Malformed,
+                RestoreEvidence::InvalidActivate,
                 Some(Box::new(Error::unsupported(
                     "a restore needs an activation record with a transaction",
                 ))),
@@ -227,6 +386,7 @@ impl RestoreMachine {
         };
         match (self.state, input) {
             (MachineState::Spawn, RestoreInput::Spawned(_)) => {
+                self.guest_may_have_run = true;
                 self.state = MachineState::Event;
                 self.action = Some(RestoreAction::AwaitReadyEvent);
             }
@@ -247,8 +407,7 @@ impl RestoreMachine {
             }
             (MachineState::Activate | MachineState::Ready, RestoreInput::GuestReady(reported)) => {
                 if reported == self.expected {
-                    self.state = MachineState::Finished;
-                    self.action = Some(RestoreAction::Done);
+                    self.finish(RestoreAction::Done);
                 } else {
                     self.fail(
                         RestorePhase::Ready,
@@ -282,7 +441,20 @@ impl RestoreMachine {
             }
             (_, RestoreInput::Expired) => self.fail(phase, RestoreEvidence::Expired, None),
             (_, RestoreInput::Cancelled) => self.fail(phase, RestoreEvidence::Cancelled, None),
-            (_, RestoreInput::Exited(_)) => self.fail(phase, RestoreEvidence::Exited, None),
+            (_, RestoreInput::Exited(report)) => {
+                let evidence = report
+                    .as_ref()
+                    .and_then(RestoreEvidence::reported)
+                    .unwrap_or(RestoreEvidence::Exited);
+                let mut error = RestoreError::new(phase, evidence, self.guest_may_have_run, None);
+                if let Some(report) = report {
+                    error.message.push_str(&format!(
+                        "; VM outcome: category {:?}, status {:?}",
+                        report.category, report.status_code
+                    ));
+                }
+                self.finish(RestoreAction::Fail(error));
+            }
             _ => self.fail(
                 phase,
                 RestoreEvidence::Malformed,
@@ -312,12 +484,18 @@ impl RestoreMachine {
         evidence: RestoreEvidence,
         source: Option<Box<dyn StdError + Send + Sync>>,
     ) {
-        self.state = MachineState::Finished;
-        self.action = Some(RestoreAction::Fail(RestoreError {
+        self.finish(RestoreAction::Fail(RestoreError::new(
             phase,
             evidence,
+            self.guest_may_have_run,
             source,
-        }));
+        )));
+    }
+
+    /// Latches one terminal action without inspecting or changing caller-owned resources.
+    fn finish(&mut self, action: RestoreAction) {
+        self.state = MachineState::Finished;
+        self.action = Some(action);
     }
 }