diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 0bbf4be63..a3900bd43 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -125,7 +125,7 @@ changes only when the module's purpose does. | [`nvx::host::runtime`](#nvxhostruntime) | `Runtime`: what every launch shares, the runtime's files and their identity, the hypervisor and its probe, and the launch options | | [`nvx::host::vm`](#nvxhostvm) | The directory of a VM, private to the current user, with the fixed names of the files and endpoints of its launches | | [`nvx::host::config`](#nvxhostconfig) | `VmConfig`, a VM's configuration as one value, and `CaptureConfig`, what a capture adds and the identity of its snapshots, checked without reading the host | -| [`nvx::host::restore`](#nvxhostrestore) | The restore-ready listener, with native ownership transfer for a caller's own I/O | +| [`nvx::host::restore`](#nvxhostrestore) | A caller-driven restore to Ready without I/O or a clock, and its peer-verified restore-ready listener | | [`nvx::host::artifacts`](#nvxhostartifacts) | Locating the OpenVMM executable and the guest artifacts that a VM runs, and their runtime identity | | [`nvx::host::launch`](#nvxhostlaunch) | `LaunchSpec` and `VmmLaunch`: the OpenVMM command line of a managed microVM, its checks, and the inputs that it shares with an adapter's launch | | [`nvx::host::network`](#nvxhostnetwork) | The one lowering of a network posture into OpenVMM options, for the CLI and external orchestrators | @@ -540,6 +540,22 @@ which the capture's count. ### `nvx::host::restore` +`RestoreMachine::new(&ActivationRecord)` latches one canonical record and its +transaction. After `Spawned`, its actions await the restore-ready event, +connect a verified control session, and send the same activation bytes on +every session. `ActivateSent` marks a completed write before waiting for +Ready; a caller that combines the write and Ready read may supply `GuestReady` +directly. `GuestReady` must echo the transaction. A lost session offers +`Connect` again, with no built-in retry count, delay or deadline; the caller +may instead feed `Expired` or `Cancelled`. + +The caller drives every I/O operation and peer check and supplies verified +`Exited` events. `OperationFailed` keeps its explicit `RestorePhase`, +`RestoreEvidence` and original error source in `RestoreError`; expiry, +cancellation and exit fail every pending phase. Failure stops no process and +writes no record. A matching Ready ends at `Done`, without sending mappings or +Start. Terminal results are offered once and absorb later inputs. + `ReadyListener` owns a restore-ready endpoint: a mode-0600 Unix socket in the private VM directory on Linux, or a fresh first-instance named pipe with an owner-only protected access control list on Windows. The library's receiver diff --git a/nvx-rs/public-api/linux.txt b/nvx-rs/public-api/linux.txt index a8830709f..fd8a79a92 100644 --- a/nvx-rs/public-api/linux.txt +++ b/nvx-rs/public-api/linux.txt @@ -2125,6 +2125,53 @@ fn nvx::host::report::read mod nvx::host::restore struct nvx::host::restore::ReadyListener method nvx::host::restore::ReadyListener::into_std +enum nvx::host::restore::RestoreAction +variant nvx::host::restore::RestoreAction::AwaitReadyEvent +variant nvx::host::restore::RestoreAction::Connect +variant nvx::host::restore::RestoreAction::Done +variant nvx::host::restore::RestoreAction::Fail +field nvx::host::restore::RestoreAction::Fail::0 +variant nvx::host::restore::RestoreAction::SendActivate +field nvx::host::restore::RestoreAction::SendActivate::0 +struct nvx::host::restore::RestoreError +field nvx::host::restore::RestoreError::evidence +field nvx::host::restore::RestoreError::phase +field nvx::host::restore::RestoreError::source +enum nvx::host::restore::RestoreEvidence +variant nvx::host::restore::RestoreEvidence::Cancelled +variant nvx::host::restore::RestoreEvidence::Exited +variant nvx::host::restore::RestoreEvidence::Expired +variant nvx::host::restore::RestoreEvidence::Io +variant nvx::host::restore::RestoreEvidence::Malformed +variant nvx::host::restore::RestoreEvidence::PeerMismatch +variant nvx::host::restore::RestoreEvidence::ReportUnreadable +enum nvx::host::restore::RestoreInput +variant nvx::host::restore::RestoreInput::ActivateSent +variant nvx::host::restore::RestoreInput::Cancelled +variant nvx::host::restore::RestoreInput::Connected +variant nvx::host::restore::RestoreInput::Exited +field nvx::host::restore::RestoreInput::Exited::0 +variant nvx::host::restore::RestoreInput::Expired +variant nvx::host::restore::RestoreInput::GuestReady +field nvx::host::restore::RestoreInput::GuestReady::0 +variant nvx::host::restore::RestoreInput::OperationFailed +field nvx::host::restore::RestoreInput::OperationFailed::evidence +field nvx::host::restore::RestoreInput::OperationFailed::phase +field nvx::host::restore::RestoreInput::OperationFailed::source +variant nvx::host::restore::RestoreInput::ReadyEvent +field nvx::host::restore::RestoreInput::ReadyEvent::0 +variant nvx::host::restore::RestoreInput::SessionLost +variant nvx::host::restore::RestoreInput::Spawned +field nvx::host::restore::RestoreInput::Spawned::0 +struct nvx::host::restore::RestoreMachine +method nvx::host::restore::RestoreMachine::input +method nvx::host::restore::RestoreMachine::new +method nvx::host::restore::RestoreMachine::next_action +enum nvx::host::restore::RestorePhase +variant nvx::host::restore::RestorePhase::Activate +variant nvx::host::restore::RestorePhase::Connect +variant nvx::host::restore::RestorePhase::Ready +variant nvx::host::restore::RestorePhase::ReadyEvent mod nvx::host::runtime struct nvx::host::runtime::ArtifactIdentity method nvx::host::runtime::ArtifactIdentity::as_str diff --git a/nvx-rs/public-api/windows.txt b/nvx-rs/public-api/windows.txt index ab8e7d184..689025b3b 100644 --- a/nvx-rs/public-api/windows.txt +++ b/nvx-rs/public-api/windows.txt @@ -1048,6 +1048,53 @@ fn nvx::host::report::read mod nvx::host::restore struct nvx::host::restore::ReadyListener method nvx::host::restore::ReadyListener::into_std +enum nvx::host::restore::RestoreAction +variant nvx::host::restore::RestoreAction::AwaitReadyEvent +variant nvx::host::restore::RestoreAction::Connect +variant nvx::host::restore::RestoreAction::Done +variant nvx::host::restore::RestoreAction::Fail +field nvx::host::restore::RestoreAction::Fail::0 +variant nvx::host::restore::RestoreAction::SendActivate +field nvx::host::restore::RestoreAction::SendActivate::0 +struct nvx::host::restore::RestoreError +field nvx::host::restore::RestoreError::evidence +field nvx::host::restore::RestoreError::phase +field nvx::host::restore::RestoreError::source +enum nvx::host::restore::RestoreEvidence +variant nvx::host::restore::RestoreEvidence::Cancelled +variant nvx::host::restore::RestoreEvidence::Exited +variant nvx::host::restore::RestoreEvidence::Expired +variant nvx::host::restore::RestoreEvidence::Io +variant nvx::host::restore::RestoreEvidence::Malformed +variant nvx::host::restore::RestoreEvidence::PeerMismatch +variant nvx::host::restore::RestoreEvidence::ReportUnreadable +enum nvx::host::restore::RestoreInput +variant nvx::host::restore::RestoreInput::ActivateSent +variant nvx::host::restore::RestoreInput::Cancelled +variant nvx::host::restore::RestoreInput::Connected +variant nvx::host::restore::RestoreInput::Exited +field nvx::host::restore::RestoreInput::Exited::0 +variant nvx::host::restore::RestoreInput::Expired +variant nvx::host::restore::RestoreInput::GuestReady +field nvx::host::restore::RestoreInput::GuestReady::0 +variant nvx::host::restore::RestoreInput::OperationFailed +field nvx::host::restore::RestoreInput::OperationFailed::evidence +field nvx::host::restore::RestoreInput::OperationFailed::phase +field nvx::host::restore::RestoreInput::OperationFailed::source +variant nvx::host::restore::RestoreInput::ReadyEvent +field nvx::host::restore::RestoreInput::ReadyEvent::0 +variant nvx::host::restore::RestoreInput::SessionLost +variant nvx::host::restore::RestoreInput::Spawned +field nvx::host::restore::RestoreInput::Spawned::0 +struct nvx::host::restore::RestoreMachine +method nvx::host::restore::RestoreMachine::input +method nvx::host::restore::RestoreMachine::new +method nvx::host::restore::RestoreMachine::next_action +enum nvx::host::restore::RestorePhase +variant nvx::host::restore::RestorePhase::Activate +variant nvx::host::restore::RestorePhase::Connect +variant nvx::host::restore::RestorePhase::Ready +variant nvx::host::restore::RestorePhase::ReadyEvent mod nvx::host::runtime struct nvx::host::runtime::ArtifactIdentity method nvx::host::runtime::ArtifactIdentity::as_str diff --git a/nvx-rs/src/host/mod.rs b/nvx-rs/src/host/mod.rs index d18162cd4..1ddd49312 100644 --- a/nvx-rs/src/host/mod.rs +++ b/nvx-rs/src/host/mod.rs @@ -52,8 +52,8 @@ //! opened, which it inherits through the [`supervisor`]'s allowlist. //! - [`vm`] holds a caller-owned VM directory, private to the current user, the fixed names of //! its files and endpoints, and its control credential, which the library never writes. -//! - [`restore`] listens for OpenVMM's restore-ready event and verifies its client against the -//! held VM process. +//! - [`restore`] orders caller-driven activation to Ready without I/O or a clock, and listens for +//! OpenVMM's restore-ready event from the held VM process. //! - `cli` (feature `cli`) is the `nvx-rs` command-line interface, which runs microVMs and //! sandboxes, looks up and downloads artifacts, and compiles egress policies. //! diff --git a/nvx-rs/src/host/restore.rs b/nvx-rs/src/host/restore.rs index 73c898ede..2fc713a73 100644 --- a/nvx-rs/src/host/restore.rs +++ b/nvx-rs/src/host/restore.rs @@ -1,21 +1,326 @@ -//! A restore-ready endpoint whose client must be the held VM process. +//! Caller-driven restores and their peer-verified restore-ready endpoint. +//! +//! [`RestoreMachine`] orders activation without I/O or a clock. The caller supplies every +//! [`RestoreInput`], performs each [`RestoreAction`], and owns retries, timing and the VM process. +//! Success ends at Ready: mappings and workload startup remain the caller's protocol. //! //! A launch binds before spawning the VM and receives through its verified [`Process`]. //! Parse the completed bytes with -//! [`parse_ready_event`](crate::protocol::restore_ready::parse_ready_event). +//! [`parse_ready_event`]. //! External I/O users take the native listener with `ReadyListener::into_std` on Linux or //! Windows and check its client's process identity before reading the event. -use std::io; +use std::error::Error as StdError; #[cfg(not(windows))] use std::path::Path; use std::time::Duration; +use std::{fmt, io}; use super::platform; use super::process::{Process, ProcessRef}; +use super::report::Report; use super::vm::VmDir; +use crate::activation::{ActivationRecord, ReadyActivation}; +use crate::protocol::restore_ready::parse_ready_event; use crate::{Error, Result}; +/// The pending part of a restore. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum RestorePhase { + /// Waiting for the launched VM's restore-ready event. + ReadyEvent, + /// Connecting and preparing the caller's verified control session. + Connect, + /// Sending the canonical activation record. + Activate, + /// Waiting for Ready to echo the activation's transaction. + Ready, +} + +/// An observed fact about a failed restore, without attribution or cleanup policy. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum RestoreEvidence { + /// A caller's peer check refused a process. + PeerMismatch, + /// A caller's I/O operation failed. + Io, + /// An event, response or ordering violated the restore protocol. + Malformed, + /// The caller could not read the VM's outcome report. + ReportUnreadable, + /// The caller's budget expired. + Expired, + /// The caller cancelled the restore. + Cancelled, + /// The caller verified that the VM exited before Ready. + Exited, +} + +/// A restore's failure phase, evidence and original cause. +/// +/// This error neither stops the VM nor records a disposition. The caller retains its process, +/// session and storage and decides what to do with them. +#[derive(Debug)] +#[non_exhaustive] +pub struct RestoreError { + /// The phase of the failed operation. + pub phase: RestorePhase, + /// What the caller or the machine observed. + pub evidence: RestoreEvidence, + /// The original cause, when the failed operation supplied one. + pub source: Option>, +} + +impl fmt::Display for RestoreError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "restore failed during {:?}: {:?}", + self.phase, self.evidence + ) + } +} + +impl StdError for RestoreError { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + self.source + .as_deref() + .map(|source| source as &(dyn StdError + 'static)) + } +} + +/// A caller's result or observation for a restore's pending step. +#[derive(Debug)] +#[non_exhaustive] +pub enum RestoreInput { + /// The caller launched and holds the VM named by this reference. + Spawned(ProcessRef), + /// Completed bytes read from the caller's peer-verified restore-ready listener. + ReadyEvent(Vec), + /// The caller prepared a new, peer-verified control session. + Connected, + /// The activation write completed; Ready is still pending. + /// + /// A caller that combines the write and Ready read may supply GuestReady directly instead. + ActivateSent, + /// The control session was lost; the caller decides whether to reconnect. + SessionLost, + /// The activation reported by the guest's Ready. + GuestReady(ReadyActivation), + /// A final operation failure, preserving its explicit phase, evidence and original source. + OperationFailed { + /// The operation's phase, including a write failure after taking SendActivate. + phase: RestorePhase, + /// What the caller observed. + evidence: RestoreEvidence, + /// The operation's original cause. + source: Option>, + }, + /// A verified VM exit and an already-read outcome report, when available. + Exited(Option), + /// The caller's budget for the pending step expired. + Expired, + /// The caller gave up. + Cancelled, +} + +/// An operation for the caller, or the terminal result of a restore. +#[derive(Debug)] +#[non_exhaustive] +pub enum RestoreAction { + /// Receive the complete, peer-verified restore-ready event. + AwaitReadyEvent, + /// Prepare a new, peer-verified control session. + Connect, + /// Send these canonical record bytes as the first request of the session and await Ready. + SendActivate(Vec), + /// Ready echoed the expected transaction; no mappings or workload Start were sent. + Done, + /// The restore failed; its VM process remains the caller's. + Fail(RestoreError), +} + +/// Progress that distinguishes queued actions from the caller's pending operations. +#[derive(Clone, Copy, Debug)] +enum MachineState { + /// Awaiting the caller's spawn notification. + Spawn, + /// Awaiting the restore-ready event. + Event, + /// Awaiting a verified control session. + Connect, + /// The activation has not yet been offered to the caller. + ActivateQueued, + /// The activation write or combined write-and-Ready operation is pending. + Activate, + /// The write completed and Ready is pending. + Ready, + /// A terminal result was reached. + Finished, +} + +impl MachineState { + /// Returns the pending phase, or None after the terminal result. + fn phase(self) -> Option { + match self { + Self::Spawn | Self::Event => Some(RestorePhase::ReadyEvent), + Self::Connect => Some(RestorePhase::Connect), + Self::ActivateQueued | Self::Activate => Some(RestorePhase::Activate), + Self::Ready => Some(RestorePhase::Ready), + Self::Finished => None, + } + } +} + +/// The ordered rules of one restore, without I/O, a clock or a retry budget. +/// +/// Feed [`RestoreInput::Spawned`] after launching the VM, then perform the offered actions and +/// supply their results. Every session receives the same canonical activation bytes. A lost +/// session offers Connect again; the caller may instead cancel or report an expired budget. +/// Terminal success or failure absorbs later inputs and is offered only once. +#[derive(Debug)] +pub struct RestoreMachine { + /// The one canonical record sent in every session. + encoded: Vec, + /// The activation that a successful Ready must report. + expected: ReadyActivation, + /// The pending operation or terminal state. + state: MachineState, + /// The next action, which an input may invalidate before it is taken. + action: Option, +} + +impl RestoreMachine { + /// Latches the record's canonical encoding and expected transaction. + /// + /// A cold record without a restore transaction queues Fail immediately. A valid restore + /// waits for Spawned; it never reads a record file or changes its record between sessions. + pub fn new(record: &ActivationRecord) -> Self { + let mut machine = Self { + encoded: record.encode(), + expected: ReadyActivation::of(record), + state: MachineState::Spawn, + action: None, + }; + if record.restore().is_none() { + machine.fail( + RestorePhase::ReadyEvent, + RestoreEvidence::Malformed, + Some(Box::new(Error::unsupported( + "a restore needs an activation record with a transaction", + ))), + ); + } + machine + } + + /// Applies an observation, replacing any action invalidated by failure or session loss. + /// + /// OperationFailed keeps the caller's phase even when it differs from the machine's pending + /// phase. Expiry, cancellation and verified exit fail every pending phase. An unexpected + /// progress input fails explicitly; inputs after a terminal result are ignored. + pub fn input(&mut self, input: RestoreInput) { + let Some(phase) = self.state.phase() else { + return; + }; + match (self.state, input) { + (MachineState::Spawn, RestoreInput::Spawned(_)) => { + self.state = MachineState::Event; + self.action = Some(RestoreAction::AwaitReadyEvent); + } + (MachineState::Event, RestoreInput::ReadyEvent(bytes)) => { + if let Err(source) = parse_ready_event(&bytes) { + self.fail(phase, RestoreEvidence::Malformed, Some(Box::new(source))); + } else { + self.state = MachineState::Connect; + self.action = Some(RestoreAction::Connect); + } + } + (MachineState::Connect, RestoreInput::Connected) => { + self.state = MachineState::ActivateQueued; + self.action = Some(RestoreAction::SendActivate(self.encoded.clone())); + } + (MachineState::Activate, RestoreInput::ActivateSent) => { + self.state = MachineState::Ready; + } + (MachineState::Activate | MachineState::Ready, RestoreInput::GuestReady(reported)) => { + if reported == self.expected { + self.state = MachineState::Finished; + self.action = Some(RestoreAction::Done); + } else { + self.fail( + RestorePhase::Ready, + RestoreEvidence::Malformed, + Some(Box::new(Error::protocol(format!( + "Ready echoed transaction {:?}, not {:?}", + reported.transaction, self.expected.transaction + )))), + ); + } + } + ( + MachineState::Connect + | MachineState::ActivateQueued + | MachineState::Activate + | MachineState::Ready, + RestoreInput::SessionLost, + ) => { + self.state = MachineState::Connect; + self.action = Some(RestoreAction::Connect); + } + ( + _, + RestoreInput::OperationFailed { + phase, + evidence, + source, + }, + ) => { + self.fail(phase, evidence, source); + } + (_, RestoreInput::Expired) => self.fail(phase, RestoreEvidence::Expired, None), + (_, RestoreInput::Cancelled) => self.fail(phase, RestoreEvidence::Cancelled, None), + (_, RestoreInput::Exited(_)) => self.fail(phase, RestoreEvidence::Exited, None), + _ => self.fail( + phase, + RestoreEvidence::Malformed, + Some(Box::new(Error::protocol( + "unexpected progress input during restore", + ))), + ), + } + } + + /// Takes the next action once, or returns None while awaiting a caller input. + /// + /// Taking SendActivate starts the pending Activate phase. ActivateSent advances to the Ready + /// wait; a caller that combines the write and Ready read can report GuestReady directly. + pub fn next_action(&mut self) -> Option { + let action = self.action.take()?; + if matches!(&action, RestoreAction::SendActivate(_)) { + self.state = MachineState::Activate; + } + Some(action) + } + + /// Queues the terminal failure and invalidates any pending action without cleanup. + fn fail( + &mut self, + phase: RestorePhase, + evidence: RestoreEvidence, + source: Option>, + ) { + self.state = MachineState::Finished; + self.action = Some(RestoreAction::Fail(RestoreError { + phase, + evidence, + source, + })); + } +} + /// The listener of one restore, bound before its VM starts. /// /// The first receive fixes the expected process identity. Every receive borrows its held diff --git a/nvx-rs/src/host/restore/tests.rs b/nvx-rs/src/host/restore/tests.rs index 782dc3567..2a5a28fe4 100644 --- a/nvx-rs/src/host/restore/tests.rs +++ b/nvx-rs/src/host/restore/tests.rs @@ -7,6 +7,10 @@ use std::time::Instant; use super::*; use crate::ErrorKind; +use crate::activation::{ + ActivationRecord, Configuration, ExpectedResources, Export, ExportKind, Image, MachineIdMode, + OverlayMode, ReadyActivation, RestoreSection, TransactionId, Upper, View, +}; use crate::host::platform::inherit::spawn_command; use crate::host::process::BootId; use crate::host::vm::create_private_dir; @@ -561,3 +565,262 @@ fn read_native_pipe(pipe: std::os::windows::io::OwnedHandle, process: &Process) } data } + +fn restore_record() -> ActivationRecord { + ActivationRecord::new( + Some(RestoreSection { + transaction: TransactionId::from_bytes([1; 16]), + }), + Configuration { + image: Image::Erofs { uuid: None }, + upper: Upper::Tmpfs { + max_memory_percent: 50, + }, + overlay: OverlayMode::TrustedLowers, + export: Some(Export { + kind: ExportKind::Aggregate, + }), + mappings: 1, + network: None, + view: View { + hostname: b"restore-test".to_vec(), + machine_id: MachineIdMode::Generated, + }, + resources: ExpectedResources { + cpus: 2, + memory: 1 << 30, + }, + }, + ) + .unwrap() +} + +fn restore_machine_at(phase: RestorePhase) -> RestoreMachine { + let mut machine = RestoreMachine::new(&restore_record()); + machine.input(RestoreInput::Spawned(ProcessRef { + pid: 7, + start_time: 11, + boot: BootId::from_bytes([3; 16]), + })); + assert!(matches!( + machine.next_action(), + Some(RestoreAction::AwaitReadyEvent) + )); + if phase == RestorePhase::ReadyEvent { + return machine; + } + machine.input(RestoreInput::ReadyEvent( + b"OPENVMM_RESTORE_READY_V1\n".to_vec(), + )); + assert!(matches!( + machine.next_action(), + Some(RestoreAction::Connect) + )); + if phase == RestorePhase::Connect { + return machine; + } + machine.input(RestoreInput::Connected); + assert!(matches!( + machine.next_action(), + Some(RestoreAction::SendActivate(_)) + )); + if phase == RestorePhase::Ready { + machine.input(RestoreInput::ActivateSent); + } + machine +} + +fn restore_failure(machine: &mut RestoreMachine) -> RestoreError { + let Some(RestoreAction::Fail(error)) = machine.next_action() else { + panic!("the restore did not fail"); + }; + assert!(machine.next_action().is_none()); + error +} + +#[test] +fn restore_machine_resends_canonical_bytes_and_stops_at_ready() { + let record = restore_record(); + let mut machine = restore_machine_at(RestorePhase::Connect); + for _ in 0..5 { + machine.input(RestoreInput::Connected); + let Some(RestoreAction::SendActivate(bytes)) = machine.next_action() else { + panic!("the session received no activation"); + }; + assert_eq!(bytes, record.encode()); + assert!(machine.next_action().is_none()); + machine.input(RestoreInput::ActivateSent); + machine.input(RestoreInput::SessionLost); + assert!(matches!( + machine.next_action(), + Some(RestoreAction::Connect) + )); + } + machine.input(RestoreInput::Connected); + assert!(matches!( + machine.next_action(), + Some(RestoreAction::SendActivate(_)) + )); + machine.input(RestoreInput::ActivateSent); + machine.input(RestoreInput::GuestReady(ReadyActivation::of(&record))); + assert!(matches!(machine.next_action(), Some(RestoreAction::Done))); + machine.input(RestoreInput::SessionLost); + machine.input(RestoreInput::Cancelled); + assert!(machine.next_action().is_none()); +} + +#[test] +fn restore_machine_checks_the_transaction_and_refuses_a_cold_record() { + for transaction in [None, Some(TransactionId::from_bytes([2; 16]))] { + let mut machine = restore_machine_at(RestorePhase::Ready); + machine.input(RestoreInput::GuestReady(ReadyActivation { transaction })); + let error = restore_failure(&mut machine); + assert_eq!(error.phase, RestorePhase::Ready); + assert_eq!(error.evidence, RestoreEvidence::Malformed); + assert!(error.source.is_some()); + } + let cold = ActivationRecord::new(None, restore_record().configuration().clone()).unwrap(); + let error = restore_failure(&mut RestoreMachine::new(&cold)); + assert_eq!(error.phase, RestorePhase::ReadyEvent); + assert_eq!(error.evidence, RestoreEvidence::Malformed); +} + +#[test] +fn restore_machine_session_loss_invalidates_each_session_phase() { + for phase in [ + RestorePhase::Connect, + RestorePhase::Activate, + RestorePhase::Ready, + ] { + let mut machine = restore_machine_at(phase); + machine.input(RestoreInput::SessionLost); + assert!(matches!( + machine.next_action(), + Some(RestoreAction::Connect) + )); + assert!(machine.next_action().is_none()); + machine.input(RestoreInput::Connected); + let Some(RestoreAction::SendActivate(bytes)) = machine.next_action() else { + panic!("the replacement session received no activation"); + }; + assert_eq!(bytes, restore_record().encode()); + machine.input(RestoreInput::GuestReady(ReadyActivation::of( + &restore_record(), + ))); + assert!(matches!(machine.next_action(), Some(RestoreAction::Done))); + } + + let mut machine = restore_machine_at(RestorePhase::Connect); + machine.input(RestoreInput::Connected); + machine.input(RestoreInput::SessionLost); + assert!(matches!( + machine.next_action(), + Some(RestoreAction::Connect) + )); + assert!(machine.next_action().is_none()); +} + +#[test] +fn restore_machine_keeps_operation_phase_evidence_and_original_source() { + for phase in [ + RestorePhase::ReadyEvent, + RestorePhase::Connect, + RestorePhase::Activate, + RestorePhase::Ready, + ] { + let mut machine = restore_machine_at(phase); + let source = Box::new(io::Error::other("original operation failure")); + let original: *const io::Error = &*source; + machine.input(RestoreInput::OperationFailed { + phase: RestorePhase::Activate, + evidence: RestoreEvidence::PeerMismatch, + source: Some(source), + }); + let error = restore_failure(&mut machine); + assert_eq!(error.phase, RestorePhase::Activate); + assert_eq!(error.evidence, RestoreEvidence::PeerMismatch); + let returned = StdError::source(&error) + .unwrap() + .downcast_ref::() + .unwrap(); + assert!(std::ptr::eq(original, returned)); + assert_eq!( + error.to_string(), + "restore failed during Activate: PeerMismatch" + ); + + let mut machine = restore_machine_at(phase); + machine.input(RestoreInput::OperationFailed { + phase, + evidence: RestoreEvidence::Io, + source: None, + }); + let error = restore_failure(&mut machine); + assert_eq!(error.phase, phase); + assert_eq!(error.evidence, RestoreEvidence::Io); + assert!(error.source.is_none()); + } +} + +#[test] +fn restore_machine_expiry_cancellation_and_exit_fail_every_pending_phase() { + for phase in [ + RestorePhase::ReadyEvent, + RestorePhase::Connect, + RestorePhase::Activate, + RestorePhase::Ready, + ] { + for (input, evidence) in [ + (RestoreInput::Expired, RestoreEvidence::Expired), + (RestoreInput::Cancelled, RestoreEvidence::Cancelled), + (RestoreInput::Exited(None), RestoreEvidence::Exited), + ( + RestoreInput::Exited(Some(Report { + status_code: Some(0), + ..Report::default() + })), + RestoreEvidence::Exited, + ), + ] { + let mut machine = restore_machine_at(phase); + machine.input(input); + machine.input(RestoreInput::Connected); + machine.input(RestoreInput::GuestReady(ReadyActivation::of( + &restore_record(), + ))); + let error = restore_failure(&mut machine); + assert_eq!(error.phase, phase); + assert_eq!(error.evidence, evidence); + } + } +} + +#[test] +fn restore_machine_refuses_malformed_events_and_out_of_order_progress() { + for event in [ + b"OPENVMM_RESTORE_READY_V1".to_vec(), + b"OPENVMM_RESTORE_READY_V1\nx".to_vec(), + ] { + let mut machine = restore_machine_at(RestorePhase::ReadyEvent); + machine.input(RestoreInput::ReadyEvent(event)); + let error = restore_failure(&mut machine); + assert_eq!(error.phase, RestorePhase::ReadyEvent); + assert_eq!(error.evidence, RestoreEvidence::Malformed); + assert!(error.source.is_some()); + } + let mut machine = RestoreMachine::new(&restore_record()); + machine.input(RestoreInput::Connected); + assert_eq!( + restore_failure(&mut machine).evidence, + RestoreEvidence::Malformed + ); + let mut machine = restore_machine_at(RestorePhase::Connect); + machine.input(RestoreInput::Connected); + machine.input(RestoreInput::GuestReady(ReadyActivation::of( + &restore_record(), + ))); + assert_eq!( + restore_failure(&mut machine).evidence, + RestoreEvidence::Malformed + ); +}