diff --git a/nvx-rs/public-api/linux.txt b/nvx-rs/public-api/linux.txt index fd8a79a92..c5bd556d1 100644 --- a/nvx-rs/public-api/linux.txt +++ b/nvx-rs/public-api/linux.txt @@ -2123,6 +2123,12 @@ field nvx::host::report::Report::status_code field nvx::host::report::Report::teardown_complete fn nvx::host::report::read mod nvx::host::restore +enum nvx::host::restore::Attribution +variant nvx::host::restore::Attribution::Host +variant nvx::host::restore::Attribution::Repair +variant nvx::host::restore::Attribution::Request +variant nvx::host::restore::Attribution::Snapshot +variant nvx::host::restore::Attribution::Unknown struct nvx::host::restore::ReadyListener method nvx::host::restore::ReadyListener::into_std enum nvx::host::restore::RestoreAction @@ -2134,17 +2140,42 @@ field nvx::host::restore::RestoreAction::Fail::0 variant nvx::host::restore::RestoreAction::SendActivate field nvx::host::restore::RestoreAction::SendActivate::0 struct nvx::host::restore::RestoreError +field nvx::host::restore::RestoreError::attribution field nvx::host::restore::RestoreError::evidence +field nvx::host::restore::RestoreError::guest_may_have_run +field nvx::host::restore::RestoreError::message field nvx::host::restore::RestoreError::phase field nvx::host::restore::RestoreError::source enum nvx::host::restore::RestoreEvidence variant nvx::host::restore::RestoreEvidence::Cancelled +variant nvx::host::restore::RestoreEvidence::EndpointBind variant nvx::host::restore::RestoreEvidence::Exited variant nvx::host::restore::RestoreEvidence::Expired +variant nvx::host::restore::RestoreEvidence::ForcedKill +variant nvx::host::restore::RestoreEvidence::ForwardBind +variant nvx::host::restore::RestoreEvidence::HostFailure +variant nvx::host::restore::RestoreEvidence::IncompatibleHost +variant nvx::host::restore::RestoreEvidence::InvalidActivate +variant nvx::host::restore::RestoreEvidence::InvalidMappings +variant nvx::host::restore::RestoreEvidence::InvalidSettings +variant nvx::host::restore::RestoreEvidence::InvalidTargets variant nvx::host::restore::RestoreEvidence::Io +variant nvx::host::restore::RestoreEvidence::LostStatus variant nvx::host::restore::RestoreEvidence::Malformed +variant nvx::host::restore::RestoreEvidence::MalformedSnapshot variant nvx::host::restore::RestoreEvidence::PeerMismatch +variant nvx::host::restore::RestoreEvidence::RepairStatus variant nvx::host::restore::RestoreEvidence::ReportUnreadable +variant nvx::host::restore::RestoreEvidence::Reset +variant nvx::host::restore::RestoreEvidence::ResourceExhaustion +variant nvx::host::restore::RestoreEvidence::RuntimeMismatch +variant nvx::host::restore::RestoreEvidence::SnapshotDefect +variant nvx::host::restore::RestoreEvidence::SuppliedGeometry +variant nvx::host::restore::RestoreEvidence::SuppliedIdentity +variant nvx::host::restore::RestoreEvidence::SuppliedLength +variant nvx::host::restore::RestoreEvidence::UnservableImage +variant nvx::host::restore::RestoreEvidence::UnusableRoot +variant nvx::host::restore::RestoreEvidence::UnusableScratch enum nvx::host::restore::RestoreInput variant nvx::host::restore::RestoreInput::ActivateSent variant nvx::host::restore::RestoreInput::Cancelled diff --git a/nvx-rs/public-api/windows.txt b/nvx-rs/public-api/windows.txt index 689025b3b..81c173fe6 100644 --- a/nvx-rs/public-api/windows.txt +++ b/nvx-rs/public-api/windows.txt @@ -1046,6 +1046,12 @@ field nvx::host::report::Report::status_code field nvx::host::report::Report::teardown_complete fn nvx::host::report::read mod nvx::host::restore +enum nvx::host::restore::Attribution +variant nvx::host::restore::Attribution::Host +variant nvx::host::restore::Attribution::Repair +variant nvx::host::restore::Attribution::Request +variant nvx::host::restore::Attribution::Snapshot +variant nvx::host::restore::Attribution::Unknown struct nvx::host::restore::ReadyListener method nvx::host::restore::ReadyListener::into_std enum nvx::host::restore::RestoreAction @@ -1057,17 +1063,42 @@ field nvx::host::restore::RestoreAction::Fail::0 variant nvx::host::restore::RestoreAction::SendActivate field nvx::host::restore::RestoreAction::SendActivate::0 struct nvx::host::restore::RestoreError +field nvx::host::restore::RestoreError::attribution field nvx::host::restore::RestoreError::evidence +field nvx::host::restore::RestoreError::guest_may_have_run +field nvx::host::restore::RestoreError::message field nvx::host::restore::RestoreError::phase field nvx::host::restore::RestoreError::source enum nvx::host::restore::RestoreEvidence variant nvx::host::restore::RestoreEvidence::Cancelled +variant nvx::host::restore::RestoreEvidence::EndpointBind variant nvx::host::restore::RestoreEvidence::Exited variant nvx::host::restore::RestoreEvidence::Expired +variant nvx::host::restore::RestoreEvidence::ForcedKill +variant nvx::host::restore::RestoreEvidence::ForwardBind +variant nvx::host::restore::RestoreEvidence::HostFailure +variant nvx::host::restore::RestoreEvidence::IncompatibleHost +variant nvx::host::restore::RestoreEvidence::InvalidActivate +variant nvx::host::restore::RestoreEvidence::InvalidMappings +variant nvx::host::restore::RestoreEvidence::InvalidSettings +variant nvx::host::restore::RestoreEvidence::InvalidTargets variant nvx::host::restore::RestoreEvidence::Io +variant nvx::host::restore::RestoreEvidence::LostStatus variant nvx::host::restore::RestoreEvidence::Malformed +variant nvx::host::restore::RestoreEvidence::MalformedSnapshot variant nvx::host::restore::RestoreEvidence::PeerMismatch +variant nvx::host::restore::RestoreEvidence::RepairStatus variant nvx::host::restore::RestoreEvidence::ReportUnreadable +variant nvx::host::restore::RestoreEvidence::Reset +variant nvx::host::restore::RestoreEvidence::ResourceExhaustion +variant nvx::host::restore::RestoreEvidence::RuntimeMismatch +variant nvx::host::restore::RestoreEvidence::SnapshotDefect +variant nvx::host::restore::RestoreEvidence::SuppliedGeometry +variant nvx::host::restore::RestoreEvidence::SuppliedIdentity +variant nvx::host::restore::RestoreEvidence::SuppliedLength +variant nvx::host::restore::RestoreEvidence::UnservableImage +variant nvx::host::restore::RestoreEvidence::UnusableRoot +variant nvx::host::restore::RestoreEvidence::UnusableScratch enum nvx::host::restore::RestoreInput variant nvx::host::restore::RestoreInput::ActivateSent variant nvx::host::restore::RestoreInput::Cancelled diff --git a/nvx-rs/src/host/restore.rs b/nvx-rs/src/host/restore.rs index 2fc713a73..244509d94 100644 --- a/nvx-rs/src/host/restore.rs +++ b/nvx-rs/src/host/restore.rs @@ -22,6 +22,7 @@ use super::report::Report; use super::vm::VmDir; use crate::activation::{ActivationRecord, ReadyActivation}; use crate::protocol::restore_ready::parse_ready_event; +use crate::protocol::status; use crate::{Error, Result}; /// The pending part of a restore. @@ -50,15 +51,133 @@ pub enum RestoreEvidence { Malformed, /// The caller could not read the VM's outcome report. ReportUnreadable, - /// The caller's budget expired. + /// The caller's budget or the guest's activation deadline expired. Expired, /// The caller cancelled the restore. Cancelled, /// The caller verified that the VM exited before Ready. Exited, + /// The requested resources are invalid or the guest refused them. + InvalidTargets, + /// The requested settings cannot be applied. + InvalidSettings, + /// The activation record is invalid or the guest refused it. + InvalidActivate, + /// The requested mappings or export cannot be applied. + InvalidMappings, + /// The snapshot cannot serve the requested image. + UnservableImage, + /// The requested scratch is not usable. + UnusableScratch, + /// The requested root cannot be assembled. + UnusableRoot, + /// A supplied file has the wrong identity. + SuppliedIdentity, + /// A supplied file has the wrong length. + SuppliedLength, + /// A supplied file has the wrong device geometry. + SuppliedGeometry, + /// A restore endpoint could not be bound. + EndpointBind, + /// A requested network forward could not be bound. + ForwardBind, + /// The host exhausted storage, memory or descriptors. + ResourceExhaustion, + /// The host cannot present the snapshot's CPU or time contract. + IncompatibleHost, + /// The restore uses another runtime identity. + RuntimeMismatch, + /// Another host operation failed. + HostFailure, + /// The VM ended without a usable status. + LostStatus, + /// The control session reset or ended. + Reset, + /// The caller or another controller forcibly ended the VM. + ForcedKill, + /// The snapshot is malformed or inconsistent. + MalformedSnapshot, + /// A diagnosis attributes a capture or repair defect to the snapshot. + SnapshotDefect, + /// A guest repair or time ABI status, whose provenance is not established. + RepairStatus, } -/// A restore's failure phase, evidence and original cause. +impl RestoreEvidence { + /// Attributes evidence without prescribing a retry or disposition. + fn attribution(self) -> Attribution { + match self { + Self::InvalidTargets + | Self::InvalidSettings + | Self::InvalidActivate + | Self::InvalidMappings + | Self::UnservableImage + | Self::UnusableScratch + | Self::UnusableRoot + | Self::SuppliedIdentity + | Self::SuppliedLength + | Self::SuppliedGeometry => Attribution::Request, + Self::Io + | Self::ReportUnreadable + | Self::EndpointBind + | Self::ForwardBind + | Self::ResourceExhaustion + | Self::IncompatibleHost + | Self::RuntimeMismatch + | Self::HostFailure => Attribution::Host, + Self::PeerMismatch + | Self::Malformed + | Self::Expired + | Self::Cancelled + | Self::Exited + | Self::LostStatus + | Self::Reset + | Self::ForcedKill => Attribution::Unknown, + Self::MalformedSnapshot | Self::SnapshotDefect => Attribution::Snapshot, + Self::RepairStatus => Attribution::Repair, + } + } + + /// Classifies only a caller-supplied guest outcome before workload Start. + fn reported(report: &Report) -> Option { + if !matches!(report.category.as_deref(), Some("guest-exit" | "success")) { + return None; + } + let code = u8::try_from(report.status_code?).ok()?; + if status::TIME_STATUSES.contains(&code) || status::REPAIR_STATUSES.contains(&code) { + return Some(Self::RepairStatus); + } + // The machine never sends Start, so no workload can have supplied an activation status. + Some(match code { + status::ACTIVATION_DEADLINE => Self::Expired, + status::ACTIVATION_PROTOCOL | status::ACTIVATION_ORDER => Self::Malformed, + status::ACTIVATION_RECORD => Self::InvalidActivate, + status::ACTIVATION_RESOURCES => Self::InvalidTargets, + status::ACTIVATION_ROOT => Self::UnusableRoot, + status::ACTIVATION_WORKLOAD | status::ACTIVATION_NETWORK => Self::InvalidSettings, + status::ACTIVATION_EXPORT => Self::InvalidMappings, + _ => return None, + }) + } +} + +/// What a restore failure's evidence attributes it to, without a disposition or retry policy. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum Attribution { + /// The requested image, upper, resources or settings. + Request, + /// The host or its own operations. + Host, + /// The snapshot's payload or a diagnosed capture or repair defect. + Snapshot, + /// The guest's repair or time ABI, whose provenance is not established. + Repair, + /// The host, protocol or neither: the evidence establishes no provenance. + Unknown, +} + +/// A restore's failure facts and original cause. /// /// This error neither stops the VM nor records a disposition. The caller retains its process, /// session and storage and decides what to do with them. @@ -67,12 +186,49 @@ pub enum RestoreEvidence { pub struct RestoreError { /// The phase of the failed operation. pub phase: RestorePhase, + /// What the evidence attributes the failure to, without prescribing a response. + pub attribution: Attribution, /// What the caller or the machine observed. pub evidence: RestoreEvidence, + /// Whether guest code could have executed since the caller reported Spawned. + /// + /// This remains true after session loss or verified exit. It does not prove execution, + /// imply that the VM still runs, or mean that a workload Start was sent. + pub guest_may_have_run: bool, + /// The failure's description, including the messages of its original cause chain. + pub message: String, /// The original cause, when the failed operation supplied one. pub source: Option>, } +impl RestoreError { + /// Captures failure facts while retaining the original cause and its complete messages. + fn new( + phase: RestorePhase, + evidence: RestoreEvidence, + guest_may_have_run: bool, + source: Option>, + ) -> Self { + let mut message = format!("restore failed during {phase:?}: {evidence:?}"); + let mut cause = source + .as_deref() + .map(|source| source as &(dyn StdError + 'static)); + while let Some(source) = cause { + message.push_str(": "); + message.push_str(&source.to_string()); + cause = source.source(); + } + Self { + phase, + attribution: evidence.attribution(), + evidence, + guest_may_have_run, + message, + source, + } + } +} + impl fmt::Display for RestoreError { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { write!( @@ -186,6 +342,8 @@ pub struct RestoreMachine { encoded: Vec, /// The activation that a successful Ready must report. expected: ReadyActivation, + /// Guest execution cannot be ruled out once the caller reports a successful spawn. + guest_may_have_run: bool, /// The pending operation or terminal state. state: MachineState, /// The next action, which an input may invalidate before it is taken. @@ -201,13 +359,14 @@ impl RestoreMachine { let mut machine = Self { encoded: record.encode(), expected: ReadyActivation::of(record), + guest_may_have_run: false, state: MachineState::Spawn, action: None, }; if record.restore().is_none() { machine.fail( RestorePhase::ReadyEvent, - RestoreEvidence::Malformed, + RestoreEvidence::InvalidActivate, Some(Box::new(Error::unsupported( "a restore needs an activation record with a transaction", ))), @@ -227,6 +386,7 @@ impl RestoreMachine { }; match (self.state, input) { (MachineState::Spawn, RestoreInput::Spawned(_)) => { + self.guest_may_have_run = true; self.state = MachineState::Event; self.action = Some(RestoreAction::AwaitReadyEvent); } @@ -247,8 +407,7 @@ impl RestoreMachine { } (MachineState::Activate | MachineState::Ready, RestoreInput::GuestReady(reported)) => { if reported == self.expected { - self.state = MachineState::Finished; - self.action = Some(RestoreAction::Done); + self.finish(RestoreAction::Done); } else { self.fail( RestorePhase::Ready, @@ -282,7 +441,20 @@ impl RestoreMachine { } (_, RestoreInput::Expired) => self.fail(phase, RestoreEvidence::Expired, None), (_, RestoreInput::Cancelled) => self.fail(phase, RestoreEvidence::Cancelled, None), - (_, RestoreInput::Exited(_)) => self.fail(phase, RestoreEvidence::Exited, None), + (_, RestoreInput::Exited(report)) => { + let evidence = report + .as_ref() + .and_then(RestoreEvidence::reported) + .unwrap_or(RestoreEvidence::Exited); + let mut error = RestoreError::new(phase, evidence, self.guest_may_have_run, None); + if let Some(report) = report { + error.message.push_str(&format!( + "; VM outcome: category {:?}, status {:?}", + report.category, report.status_code + )); + } + self.finish(RestoreAction::Fail(error)); + } _ => self.fail( phase, RestoreEvidence::Malformed, @@ -312,12 +484,18 @@ impl RestoreMachine { evidence: RestoreEvidence, source: Option>, ) { - self.state = MachineState::Finished; - self.action = Some(RestoreAction::Fail(RestoreError { + self.finish(RestoreAction::Fail(RestoreError::new( phase, evidence, + self.guest_may_have_run, source, - })); + ))); + } + + /// Latches one terminal action without inspecting or changing caller-owned resources. + fn finish(&mut self, action: RestoreAction) { + self.state = MachineState::Finished; + self.action = Some(action); } } diff --git a/nvx-rs/src/host/restore/tests.rs b/nvx-rs/src/host/restore/tests.rs index 2a5a28fe4..b302590b0 100644 --- a/nvx-rs/src/host/restore/tests.rs +++ b/nvx-rs/src/host/restore/tests.rs @@ -677,12 +677,21 @@ fn restore_machine_checks_the_transaction_and_refuses_a_cold_record() { let error = restore_failure(&mut machine); assert_eq!(error.phase, RestorePhase::Ready); assert_eq!(error.evidence, RestoreEvidence::Malformed); + assert_eq!(error.attribution, Attribution::Unknown); + assert!(error.guest_may_have_run); assert!(error.source.is_some()); } let cold = ActivationRecord::new(None, restore_record().configuration().clone()).unwrap(); let error = restore_failure(&mut RestoreMachine::new(&cold)); assert_eq!(error.phase, RestorePhase::ReadyEvent); - assert_eq!(error.evidence, RestoreEvidence::Malformed); + assert_eq!(error.evidence, RestoreEvidence::InvalidActivate); + assert_eq!(error.attribution, Attribution::Request); + assert!(!error.guest_may_have_run); + assert!( + error + .message + .contains("activation record with a transaction") + ); } #[test] @@ -739,6 +748,12 @@ fn restore_machine_keeps_operation_phase_evidence_and_original_source() { let error = restore_failure(&mut machine); assert_eq!(error.phase, RestorePhase::Activate); assert_eq!(error.evidence, RestoreEvidence::PeerMismatch); + assert_eq!(error.attribution, Attribution::Unknown); + assert!(error.guest_may_have_run); + assert_eq!( + error.message, + "restore failed during Activate: PeerMismatch: original operation failure" + ); let returned = StdError::source(&error) .unwrap() .downcast_ref::() @@ -758,6 +773,8 @@ fn restore_machine_keeps_operation_phase_evidence_and_original_source() { let error = restore_failure(&mut machine); assert_eq!(error.phase, phase); assert_eq!(error.evidence, RestoreEvidence::Io); + assert_eq!(error.attribution, Attribution::Host); + assert!(error.guest_may_have_run); assert!(error.source.is_none()); } } @@ -791,7 +808,292 @@ fn restore_machine_expiry_cancellation_and_exit_fail_every_pending_phase() { let error = restore_failure(&mut machine); assert_eq!(error.phase, phase); assert_eq!(error.evidence, evidence); + assert_eq!(error.attribution, Attribution::Unknown); + assert!(error.guest_may_have_run); + assert!(error.message.contains(&format!("{phase:?}"))); + assert!(error.source.is_none()); + } + } +} + +#[test] +fn restore_failures_attribute_the_existing_policy_free_evidence_classes() { + let classes: [(&[RestoreEvidence], Attribution); 5] = [ + ( + &[ + RestoreEvidence::InvalidTargets, + RestoreEvidence::InvalidSettings, + RestoreEvidence::InvalidActivate, + RestoreEvidence::InvalidMappings, + RestoreEvidence::UnservableImage, + RestoreEvidence::UnusableScratch, + RestoreEvidence::UnusableRoot, + RestoreEvidence::SuppliedIdentity, + RestoreEvidence::SuppliedLength, + RestoreEvidence::SuppliedGeometry, + ], + Attribution::Request, + ), + ( + &[ + RestoreEvidence::Io, + RestoreEvidence::ReportUnreadable, + RestoreEvidence::EndpointBind, + RestoreEvidence::ForwardBind, + RestoreEvidence::ResourceExhaustion, + RestoreEvidence::IncompatibleHost, + RestoreEvidence::RuntimeMismatch, + RestoreEvidence::HostFailure, + ], + Attribution::Host, + ), + ( + &[ + RestoreEvidence::PeerMismatch, + RestoreEvidence::Malformed, + RestoreEvidence::Expired, + RestoreEvidence::Cancelled, + RestoreEvidence::Exited, + RestoreEvidence::LostStatus, + RestoreEvidence::Reset, + RestoreEvidence::ForcedKill, + ], + Attribution::Unknown, + ), + ( + &[ + RestoreEvidence::MalformedSnapshot, + RestoreEvidence::SnapshotDefect, + ], + Attribution::Snapshot, + ), + (&[RestoreEvidence::RepairStatus], Attribution::Repair), + ]; + for phase in [ + RestorePhase::ReadyEvent, + RestorePhase::Connect, + RestorePhase::Activate, + RestorePhase::Ready, + ] { + for (evidence, attribution) in classes { + for &evidence in evidence { + let mut machine = restore_machine_at(phase); + machine.input(RestoreInput::OperationFailed { + phase, + evidence, + source: None, + }); + let error = restore_failure(&mut machine); + assert_eq!(error.phase, phase); + assert_eq!(error.evidence, evidence); + assert_eq!(error.attribution, attribution); + assert!(error.guest_may_have_run); + assert!(!error.message.is_empty()); + assert!(error.source.is_none()); + } + } + } +} + +#[test] +fn restore_exit_reports_keep_guest_status_classification_and_actual_details() { + let classes = [ + ( + status::ACTIVATION_DEADLINE, + RestoreEvidence::Expired, + Attribution::Unknown, + ), + ( + status::ACTIVATION_PROTOCOL, + RestoreEvidence::Malformed, + Attribution::Unknown, + ), + ( + status::ACTIVATION_ORDER, + RestoreEvidence::Malformed, + Attribution::Unknown, + ), + ( + status::ACTIVATION_RECORD, + RestoreEvidence::InvalidActivate, + Attribution::Request, + ), + ( + status::ACTIVATION_RESOURCES, + RestoreEvidence::InvalidTargets, + Attribution::Request, + ), + ( + status::ACTIVATION_ROOT, + RestoreEvidence::UnusableRoot, + Attribution::Request, + ), + ( + status::ACTIVATION_WORKLOAD, + RestoreEvidence::InvalidSettings, + Attribution::Request, + ), + ( + status::ACTIVATION_NETWORK, + RestoreEvidence::InvalidSettings, + Attribution::Request, + ), + ( + status::ACTIVATION_EXPORT, + RestoreEvidence::InvalidMappings, + Attribution::Request, + ), + ]; + for phase in [ + RestorePhase::ReadyEvent, + RestorePhase::Connect, + RestorePhase::Activate, + RestorePhase::Ready, + ] { + for category in ["guest-exit", "success"] { + let repair = status::TIME_STATUSES + .chain(status::REPAIR_STATUSES) + .map(|code| (code, RestoreEvidence::RepairStatus, Attribution::Repair)); + for (code, evidence, attribution) in classes.into_iter().chain(repair) { + let mut machine = restore_machine_at(phase); + machine.input(RestoreInput::Exited(Some(Report { + category: Some(category.to_owned()), + status_code: Some(i64::from(code)), + teardown_complete: Some(true), + }))); + let error = restore_failure(&mut machine); + assert_eq!(error.phase, phase); + assert_eq!(error.evidence, evidence); + assert_eq!(error.attribution, attribution); + assert!(error.guest_may_have_run); + assert!(error.message.contains(category)); + assert!(error.message.contains(&format!("status Some({code})"))); + assert!(error.source.is_none()); + } + } + } + for (category, code) in [ + (None, Some(195)), + (Some("vmm-failure"), Some(195)), + (Some("teardown-failure"), Some(195)), + (Some("guest-exit"), None), + (Some("guest-exit"), Some(-1)), + (Some("guest-exit"), Some(256)), + (Some("guest-exit"), Some(0)), + (Some("guest-exit"), Some(137)), + ] { + let mut machine = restore_machine_at(RestorePhase::Ready); + machine.input(RestoreInput::Exited(Some(Report { + category: category.map(str::to_owned), + status_code: code, + teardown_complete: None, + }))); + let error = restore_failure(&mut machine); + assert_eq!(error.evidence, RestoreEvidence::Exited); + assert_eq!(error.attribution, Attribution::Unknown); + assert!(error.message.contains(&format!("status {code:?}"))); + } +} + +#[test] +fn restore_guest_execution_possibility_is_false_before_spawn_and_sticky_after_it() { + let mut machine = RestoreMachine::new(&restore_record()); + machine.input(RestoreInput::Cancelled); + machine.input(RestoreInput::Spawned( + held_process(std::process::id()).reference().clone(), + )); + assert!(!restore_failure(&mut machine).guest_may_have_run); + + for phase in [ + RestorePhase::Connect, + RestorePhase::Activate, + RestorePhase::Ready, + ] { + let mut machine = restore_machine_at(phase); + machine.input(RestoreInput::SessionLost); + assert!(matches!( + machine.next_action(), + Some(RestoreAction::Connect) + )); + machine.input(RestoreInput::OperationFailed { + phase: RestorePhase::ReadyEvent, + evidence: RestoreEvidence::Io, + source: None, + }); + let error = restore_failure(&mut machine); + assert_eq!(error.phase, RestorePhase::ReadyEvent); + assert!(error.guest_may_have_run); + } +} + +#[test] +fn restore_failure_messages_keep_the_original_cause_chain_without_replacing_it() { + let source = Box::new( + Error::io("activation write failed") + .with_source(Error::io("transport failed").with_source(io::Error::other("root cause"))), + ); + let original: *const Error = &*source; + let mut machine = restore_machine_at(RestorePhase::Ready); + machine.input(RestoreInput::OperationFailed { + phase: RestorePhase::Activate, + evidence: RestoreEvidence::Io, + source: Some(source), + }); + let error = restore_failure(&mut machine); + assert_eq!(error.attribution, Attribution::Host); + assert!(error.message.contains("activation write failed")); + assert!(error.message.contains("transport failed")); + assert!(error.message.ends_with("root cause")); + assert_eq!(error.to_string(), "restore failed during Activate: Io"); + let returned = StdError::source(&error) + .unwrap() + .downcast_ref::() + .unwrap(); + assert!(std::ptr::eq(original, returned)); +} + +#[test] +fn restore_failure_and_drop_leave_real_running_or_exited_processes_and_caller_files_unchanged() { + use std::collections::BTreeSet; + use std::fs; + + for exited in [false, true] { + let (_temporary, directory) = vm_directory(); + let state = directory.path().join("caller-state"); + fs::write(&state, b"caller-owned").unwrap(); + let listener = ReadyListener::bind(&directory).unwrap(); + let mut client = Client::start(listener.endpoint(), b"OPENVMM_RESTORE_READY_V1\n", false); + if exited { + client.stop(); } + let entries = || { + fs::read_dir(directory.path()) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect::>() + }; + let before = entries(); + let mut machine = RestoreMachine::new(&restore_record()); + machine.input(RestoreInput::Spawned(client.process.reference().clone())); + assert!(matches!( + machine.next_action(), + Some(RestoreAction::AwaitReadyEvent) + )); + machine.input(if exited { + RestoreInput::Exited(None) + } else { + RestoreInput::Cancelled + }); + let error = restore_failure(&mut machine); + assert!(error.guest_may_have_run); + drop(machine); + drop(error); + assert_eq!(client.process.has_exited().unwrap(), exited); + assert_eq!(client.child.try_wait().unwrap().is_some(), exited); + assert_eq!(entries(), before); + assert_eq!(fs::read(&state).unwrap(), b"caller-owned"); + client.stop(); + assert!(client.process.has_exited().unwrap()); } }