diff --git a/doc/run.md b/doc/run.md index bdb042708..7ead91862 100644 --- a/doc/run.md +++ b/doc/run.md @@ -1185,7 +1185,9 @@ otherwise read or change: `provision` refuses a share whose resolved host path equals, contains, or lies inside the sandbox's state directory, its image, its scratch image, or the OpenVMM, kernel, or initramfs of the artifact root, or the OpenVMM that `NVX_OPENVMM` names, before it records the sandbox, and -`start` refuses one again before it launches the VM. +`start` refuses one again before it launches the VM. The `start` of a restored +sandbox also refuses a share that reaches its snapshot store, whose files the +view of each restore links, before it takes the snapshot's lock. `start` resolves the recorded host paths again, so a share that came to reach one of these files through a link since `provision` is refused too. A `provision` that finds no artifact root leaves the artifacts to `start`. A diff --git a/nvx-rs/src/host/cli/sandbox/lifecycle/mod.rs b/nvx-rs/src/host/cli/sandbox/lifecycle/mod.rs index 664647302..c79519624 100644 --- a/nvx-rs/src/host/cli/sandbox/lifecycle/mod.rs +++ b/nvx-rs/src/host/cli/sandbox/lifecycle/mod.rs @@ -5,9 +5,14 @@ //! matching saved VM only after authenticated readiness and mappings. Every lifecycle command //! takes the CLI store's recoverable lock; indeterminate launches retain their evidence. //! +//! A restored sandbox, whose configuration names a named snapshot, restores it at each start in +//! place of a cold boot, under the snapshot's lock, through the blocking restore driver, which +//! readies the guest, maps the shares and runs the recorded workload. +//! //! Exec and stop reattach with runtime and process verification. Stop requests NVXC shutdown, //! waits for exit, then forces only the verified process. Deprovision removes only this stopped -//! sandbox's owned files. No running sandbox from the old runtime store is migrated. +//! sandbox's owned files, and a restored sandbox's view of its snapshot under the snapshot's +//! lock. No running sandbox from the old runtime store is migrated. //! //! A run launches the VM that a provision and a start would launch, in a temporary VM directory, //! without a state directory, executes one workload over NVXC, and stops it the way stop does. @@ -93,7 +98,7 @@ pub(crate) fn provision( let scratch = RecordedScratch::new(launch)?; write_json( &config_path, - &serialize_launch(launch, distro, scratch.as_ref(), config)?, + &serialize_launch(launch, distro, scratch.as_ref(), config, None)?, ) } diff --git a/nvx-rs/src/host/cli/sandbox/lifecycle/start.rs b/nvx-rs/src/host/cli/sandbox/lifecycle/start.rs index daf35c872..eacdf1ce8 100644 --- a/nvx-rs/src/host/cli/sandbox/lifecycle/start.rs +++ b/nvx-rs/src/host/cli/sandbox/lifecycle/start.rs @@ -4,6 +4,9 @@ use std::fs; use std::io; use std::path::Path; +use std::sync::atomic::Ordering; +use std::sync::mpsc::{self, RecvTimeoutError}; +use std::thread; use std::time::Instant; use serde_json::Value; @@ -14,28 +17,38 @@ use super::{ CONFIG_NAME, Found, ProvisionConfig, REQUIRED_FEATURES, connect, find_vm, mismatched, require_config, running_vm, runtime_for, }; +use crate::activation::{ActivationRecord, TransactionId}; use crate::host::cli::artifacts::locate; use crate::host::cli::deadline::{Deadline, RETRY_INTERVAL}; use crate::host::cli::download::Hold; use crate::host::cli::exec_lock::ExecLock; use crate::host::cli::sandbox::lock::LifecycleLock; use crate::host::cli::sandbox::state::{ - Config, RecordedDistro, RecordedScratch, read_config, serialize_launch, + Config, RecordedDistro, RecordedScratch, Source, read_config, serialize_launch, }; use crate::host::cli::sandbox::{ Artifacts, Options, error_with_cause, hypervisor, path_string, refuse_own_files, }; use crate::host::cli::session::{ExecRequest, ExecResult, Session}; use crate::host::cli::shell::format_command; +use crate::host::cli::snapshots::Store; use crate::host::cli::store::StateFile; use crate::host::cli::{Exit, Result, hex, io_error, output, random_bytes}; +use crate::host::config::{RootConfig, UpperConfig, VmConfig}; +use crate::host::platform; use crate::host::process::KILL_TIMEOUT; use crate::host::report::Report; +use crate::host::restore::RestoreEvidence; +use crate::host::restore_driver::{self, Cancel, ControlRestoreError, Workload}; use crate::host::runtime::{ArtifactIdentity, Runtime, RuntimeIdentity}; use crate::host::sandbox::SandboxLaunch; use crate::host::sandbox::distro::DISTRO_POLICY_REVISION; -use crate::host::sandbox::scratch::CLAIM_RELEASE_WAIT; -use crate::host::vm::{self, SavedVm, VmDir}; +use crate::host::sandbox::scratch::{CLAIM_RELEASE_WAIT, CheckedScratch}; +use crate::host::session::{ExecEvent, SessionError}; +use crate::host::snapshot::OPENVMM_DIR; +use crate::host::vm::{self, Launched, Prepared, SavedVm, VmDir}; +use crate::protocol::workload::WorkloadIdentity; +use crate::protocol::{self, ExitCategory, HostMap, WorkloadEnvironment}; use crate::{ErrorCode, ErrorKind}; /// Starts a provisioned sandbox, as `start` does. @@ -77,11 +90,17 @@ fn start_locked( } let config = Config::new(config)?; let launch = config.launch()?; + let source = config.source()?; refuse_own_files( Some(("state directory", state_dir)), &launch, Some(&locate(artifacts.root)?), )?; + // The view of each restore links the snapshot's files, which a share of the store would + // give the guest to change. + if let Some(source) = &source { + refuse_own_files(Some(("snapshot store", &source.store)), &launch, None)?; + } let binding = config.distro_binding(); let scratch_binding = config.scratch_binding()?; let distro = binding @@ -100,21 +119,134 @@ fn start_locked( plan.kernel_command_line(config.cmdline())?; artifacts.require()?; let runtime = runtime_for(artifacts, &config, artifacts.launch_identity()?)?; + let restore = source.as_ref().map(|source| Restore { + source, + identity: launch.settings().workload_identity, + }); let config = config.vm_config(&launch)?; - let dir = VmDir::open(state_dir).map_err(error_with_cause)?; + let started = launch_locked(lock, &runtime, config, restore, scratch, timeout, vm::spawn); + drop(distro); + started +} + +/// What a start restores in place of a cold boot: the named snapshot of a restored sandbox, and +/// the identity of the workload that the restore runs. +struct Restore<'a> { + /// The snapshot and the workload, as the sandbox's configuration records them. + source: &'a Source, + /// The workload's UID and GID. + identity: (u32, u32), +} + +/// Launches the VM of the sandbox of a locked state directory through the launch journal of its +/// store, with `spawn`: a cold boot of `config`, or a restore of `restore`, and commits it once +/// its guest is ready. +/// +/// A restore holds its snapshot's lock from before it creates its view, which links the +/// snapshot's files into the state directory, until the start ends, past the restore-ready event +/// of the verified VM, or until the VM has verifiably exited: the lock first lets it remove the +/// view of an earlier start. It restores the snapshot's online processors and runs the restored +/// workload until it exits. A command that dies before the guest is ready leaves its journal, +/// whose recovery ends the VM, as for an interrupted cold boot. A start that cannot confirm that +/// its VM ended leaves the journal to the next command's recovery as well, and its lock goes with +/// the command. +/// +/// A restore's guest accepts only a fresh upper, so a restore with a scratch image refuses to +/// start once an earlier start left its view, which each start creates before its guest can use +/// the image, even if that start failed before its guest did. +fn launch_locked( + lock: &mut LifecycleLock, + runtime: &Runtime, + mut config: VmConfig, + restore: Option>, + scratch: Option, + timeout: f64, + spawn: impl FnOnce(Prepared) -> crate::Result, +) -> Result { + let dir = VmDir::open(lock.locked().dir()).map_err(error_with_cause)?; + if restore.is_some() + && matches!(&config.root, RootConfig::Layered(root) + if matches!(root.upper, UpperConfig::Scratch { .. })) + && fs::symlink_metadata(dir.path().join(OPENVMM_DIR)).is_ok() + { + return Err(Exit::error( + ErrorCode::Unsupported, + "an earlier start of this restored sandbox may have given its scratch image to a \ + guest, and a restore needs a fresh one; deprovision the sandbox and restore the \ + snapshot again with a fresh scratch image", + )); + } + let store = restore + .as_ref() + .map(|restore| Store::open(&restore.source.store)) + .transpose()?; + let held = match (&store, &restore) { + (Some(store), Some(restore)) => { + Some(store.lock(&restore.source.snapshot, Deadline::after(timeout))?) + } + _ => None, + }; + let mut snapshot = None; + if let Some(held) = &held { + held.remove_view(&dir)?; + let opened = held.open()?; + let key = opened.key(); + config.boot = None; + config.processors = match key.boot_processors() { + 0 => key.processors(), + online => online, + }; + snapshot = Some(opened); + } lock.clear_outputs()?; - let mut prepared = vm::create(&runtime, &config, None, &dir).map_err(error_with_cause)?; + let mut prepared = + vm::create(runtime, &config, snapshot.as_ref(), &dir).map_err(error_with_cause)?; + let activation = match &snapshot { + Some(_) => Some( + prepared + .activation_record(TransactionId::from_bytes(random_bytes()?)) + .map_err(error_with_cause)?, + ), + None => None, + }; + // OpenVMM opens a restore's scratch on Windows only while no other handle may write it, so + // the claim's handle closes there, and the guard that OpenVMM inherits keeps the claim. + #[cfg(windows)] + let (scratch, guard) = match scratch { + Some(scratch) if snapshot.is_some() => (None, Some(scratch.into_file())), + scratch => (scratch, None), + }; + #[cfg(windows)] + for file in scratch + .as_ref() + .map(CheckedScratch::claim) + .into_iter() + .chain(&guard) + { + prepared.inherit(file.try_clone().map_err(Exit::from_os_error)?); + } + #[cfg(not(windows))] if let Some(scratch) = &scratch { - #[cfg(not(windows))] prepared.inherit(scratch.file().try_clone().map_err(Exit::from_os_error)?); - #[cfg(windows)] - prepared.inherit(scratch.claim().try_clone().map_err(Exit::from_os_error)?); } let mut starting = lock .locked() .begin_launch(&prepared, &runtime.identity) .map_err(error_with_cause)?; - let launched = vm::spawn(prepared).map_err(error_with_cause)?; + let mut launched = match spawn(prepared) { + Ok(launched) => launched, + Err(error) => { + drop(starting); + // A spawn that could not confirm the end of its VM leaves it to the journal, whose + // recovery ends the VM of a restore before the snapshot's lock goes. + if held.is_some() { + lock.locked() + .recover(Instant::now() + KILL_TIMEOUT) + .map_err(error_with_cause)?; + } + return Err(error_with_cause(error)); + } + }; let saved = SavedVm { vm_dir: dir, endpoint: launched.outputs.endpoint.clone(), @@ -122,26 +254,30 @@ fn start_locked( process: launched.process.reference().clone(), runtime: runtime.identity.clone(), }; + let maps = config.mapping_table(); let mut stopping = false; let ready = starting .record_process(&saved.process) .map_err(error_with_cause) - .and_then(|()| check_ready(&saved, &runtime, timeout)) - .and_then(|mut session| { - let maps = config.mapping_table(); - if maps.is_empty() { - return Ok(()); + .and_then(|()| match (&activation, &restore) { + (Some(record), Some(restore)) => { + restore_guest(&mut launched, record, restore, &maps, timeout) } - session.map_host_paths(&maps, timeout).inspect_err(|error| { - if !matches!(error, Exit::Interrupted) { - match session.stop(timeout) { - Ok(()) => stopping = true, - Err(error) => output::print_stderr(&format!( - "warning: startup guest shutdown failed: {error:?}" - )), - } + _ => check_ready(&saved, runtime, timeout).and_then(|mut session| { + if maps.is_empty() { + return Ok(()); } - }) + session.map_host_paths(&maps, timeout).inspect_err(|error| { + if !matches!(error, Exit::Interrupted) { + match session.stop(timeout) { + Ok(()) => stopping = true, + Err(error) => output::print_stderr(&format!( + "warning: startup guest shutdown failed: {error:?}" + )), + } + } + }) + }), }); let Err(mut error) = ready else { return starting.commit(&saved).map_err(error_with_cause); @@ -171,7 +307,8 @@ fn start_locked( .recover(Instant::now() + KILL_TIMEOUT) .map_err(error_with_cause)?; lock.clear_runtime()?; - drop((distro, scratch)); + // The VM has exited, which ends its claim of the scratch image and its use of the snapshot. + drop((scratch, held)); let report = stopped.report.map_err(error_with_cause)?; if stopped.self_exited && !stopping && !matches!(error, Exit::Interrupted) { return Err(exited_during_startup(report)); @@ -234,6 +371,109 @@ fn check_ready(saved: &SavedVm, runtime: &Runtime, timeout: f64) -> Result, + maps: &[HostMap], + timeout: f64, +) -> Result { + let (uid, gid) = restore.identity; + let workload = Workload { + identity: WorkloadIdentity { + uid, + gid, + create_account: false, + }, + exec: protocol::Workload { + argv: restore.source.workload.clone(), + timeout_ms: 0, + cwd: None, + environment: WorkloadEnvironment::Default, + }, + request_id: 1, + }; + let deadline = Deadline::after(timeout); + let cancel = Cancel::new(false); + let restored = thread::scope(|scope| { + // Dropping the sender, also as a panic unwinds, ends the watcher. + let (_restoring, ended) = mpsc::channel::<()>(); + scope.spawn(|| { + let ended = ended; + while let Err(RecvTimeoutError::Timeout) = ended.recv_timeout(RETRY_INTERVAL) { + if platform::interrupted() || deadline.passed() { + cancel.store(true, Ordering::Release); + } + } + }); + restore_driver::restore(launched, record, &workload, maps, &cancel) + }); + // Closing the session at once can lose the workload's Start, so the start keeps the session + // until the workload ends, reading its events in slices that let Ctrl-C through. + let error = match restored { + Ok(mut session) => loop { + if platform::interrupted() { + return Err(Exit::Interrupted); + } + let slice = deadline.instant().min(Instant::now() + RETRY_INTERVAL); + match session.next_exec_event(workload.request_id, Some(slice)) { + Ok(ExecEvent::Stdout(data)) => { + output::write_stdout(&data).map_err(Exit::from_os_error)?; + } + Ok(ExecEvent::Stderr(data)) => { + output::write_stderr(&data).map_err(Exit::from_os_error)?; + } + Ok(ExecEvent::Exit { + category: ExitCategory::Exit, + status: 0, + }) => return Ok(()), + Ok(ExecEvent::Exit { category, status }) => { + return Err(Exit::error( + ErrorCode::Other, + format!("the restored workload ended with {category:?} status {status}"), + )); + } + Ok(ExecEvent::Rejected { status, category }) => { + return Err(Exit::error( + ErrorCode::ExecRefused, + format!("the guest refused the restored workload: {category} ({status})"), + )); + } + Err(SessionError::TimedOut) if !deadline.passed() => {} + Err(_) if platform::interrupted() => return Err(Exit::Interrupted), + Err(error) => break ControlRestoreError::Start(error), + } + }, + Err(_) if platform::interrupted() => return Err(Exit::Interrupted), + Err(error) => error, + }; + let code = match &error { + _ if deadline.passed() => ErrorCode::TimedOut, + ControlRestoreError::Machine(error) => match error.evidence { + RestoreEvidence::Expired => ErrorCode::TimedOut, + RestoreEvidence::Exited => ErrorCode::Exited, + RestoreEvidence::Io | RestoreEvidence::HostFailure => ErrorCode::Io, + _ => ErrorCode::Protocol, + }, + ControlRestoreError::Maps(error) | ControlRestoreError::Start(error) => match error { + SessionError::TimedOut => ErrorCode::TimedOut, + // As for a cold start, a closed endpoint proves no exit, and gives OpenVMM its grace + // to exit by itself before the start ends it. + SessionError::ProcessExited | SessionError::Closed => return Err(Exit::EndpointClosed), + SessionError::Io(_) => ErrorCode::Io, + _ => ErrorCode::Protocol, + }, + }; + Err(Exit::error( + code, + format!("snapshot restore failed: {error}"), + )) +} + /// Runs a workload under exec admission. The lifecycle lock is released before connecting, so /// a long workload cannot delay a stop. /// @@ -497,7 +737,7 @@ fn record( config: &ProvisionConfig<'_>, ) -> Result<(Config, SandboxLaunch)> { let scratch = RecordedScratch::new(launch)?; - let record = serialize_launch(launch, distro, scratch.as_ref(), config)?; + let record = serialize_launch(launch, distro, scratch.as_ref(), config, None)?; let config = Config::new( serde_json::from_str(&record) .map_err(|error| Exit::error(ErrorCode::Other, error.to_string()))?, diff --git a/nvx-rs/src/host/cli/sandbox/lifecycle/start/tests.rs b/nvx-rs/src/host/cli/sandbox/lifecycle/start/tests.rs index d585117a1..0bddb5b4d 100644 --- a/nvx-rs/src/host/cli/sandbox/lifecycle/start/tests.rs +++ b/nvx-rs/src/host/cli/sandbox/lifecycle/start/tests.rs @@ -9,6 +9,11 @@ use crate::host::sandbox::mount::{plan_mounts, validate_mounts}; use crate::host::sandbox::options::{sandbox_mounts, validate_sandbox_systemd_policy}; use crate::host::sandbox::{SandboxSettings, SandboxUpper}; +/// The starts of restored sandboxes, with a fake OpenVMM and guest in place of OpenVMM's restore, +/// which the tests fake on Linux and Windows. +#[cfg(any(target_os = "linux", windows))] +mod restore; + #[test] fn run_checks_again_that_its_shares_do_not_overlap() { let directory = tempfile::tempdir().unwrap(); diff --git a/nvx-rs/src/host/cli/sandbox/lifecycle/start/tests/restore.rs b/nvx-rs/src/host/cli/sandbox/lifecycle/start/tests/restore.rs new file mode 100644 index 000000000..3e93d8e2d --- /dev/null +++ b/nvx-rs/src/host/cli/sandbox/lifecycle/start/tests/restore.rs @@ -0,0 +1,729 @@ +use std::ffi::OsString; +use std::fs::{self, OpenOptions}; +use std::io::{Read, Write}; +use std::path::PathBuf; +use std::sync::Mutex; +use std::time::Duration; + +use serde_json::{Value, json}; + +use super::super::*; +use crate::activation::{ActivationRecord, Image, Network, Publication, ReadyActivation, Upper}; +use crate::host::cli::hex; +use crate::host::cli::sandbox::lifecycle::{ProvisionConfig, deprovision}; +use crate::host::cli::sandbox::state::{RecordedDistro, serialize_launch, write_json}; +use crate::host::cli::snapshots::Name; +use crate::host::cli::store::StateDir; +use crate::host::filesystem::{Bind, ExportChild, ExportPlan, Owner}; +use crate::host::network::{Access, NetworkPolicy}; +use crate::host::process::{Opened, Process, ProcessRef}; +use crate::host::sandbox::distro::DistroBinding; +use crate::host::sandbox::layer::{LayerRole, SandboxLayer}; +use crate::host::sandbox::mount::{plan_mounts, validate_mounts}; +use crate::host::sandbox::options::sandbox_mounts; +use crate::host::sandbox::{SandboxLaunch, SandboxSettings, SandboxUpper}; +use crate::host::snapshot::tests::RestoreFixture; +use crate::host::snapshot::{DESCRIPTION_FILE, KEY_FILE, OPENVMM_DIR}; +use crate::protocol::root::RootImage; +use crate::protocol::{ + APP_ACTIVATE, APP_EXEC, APP_EXIT, APP_HEADER_LEN, APP_MAPS, APP_READY, APP_STDOUT, OUTER_DATA, + OUTER_HEADER_LEN, OUTER_HOST_ATTACH, OUTER_MAX_PAYLOAD, OUTER_READY, decode_app, + decode_outer_header, encode_app, encode_exec_payload, encode_maps_payloads, encode_outer, +}; + +/// The fake VMM and guest that the fixture's restores spawn in place of OpenVMM. +const HELPER: &str = "host::cli::sandbox::lifecycle::start::tests::restore::restore_helper"; +/// OpenVMM's restore-ready event. +const READY_EVENT: &[u8] = b"OPENVMM_RESTORE_READY_V1\n"; +/// The workload of the fixture's restored sandboxes. +const WORKLOAD: [&str; 2] = ["/bin/echo", "restored"]; + +/// The name of the fixture's snapshot. +fn name() -> Name { + Name::try_from("source".to_owned()).unwrap() +} + +/// Returns the activation record of an activate request's `payload`, after the workload's UID, +/// GID, and account flag. +fn activation(payload: &[u8]) -> (ActivationRecord, (u32, u32, u8)) { + let id = |at: usize| u32::from_le_bytes(payload[at..at + 4].try_into().unwrap()); + let record = ActivationRecord::decode(&payload[9..]).unwrap(); + (record, (id(0), id(4), payload[8])) +} + +/// A store whose one snapshot, `source`, restores the configuration of a [`RestoreFixture`], +/// and the fake VMs that the fixture's restores spawned, which it ends. +struct Fixture { + /// The store, the state directories, and the fake VMs' recipes and traces. + root: tempfile::TempDir, + /// The configuration and runtime that the snapshot restores. + restore: RestoreFixture, + /// The store. + store: Store, + /// Each fake VM, with the OpenVMM arguments that it replaced, and whether every VM spawned + /// before it had exited when it spawned. + spawned: Mutex, bool)>>, +} + +impl Fixture { + fn new(nic: bool) -> Self { + let restore = RestoreFixture::current(nic); + let root = tempfile::tempdir().unwrap(); + let store = Store::create(&root.path().join("store")).unwrap(); + let snapshot = restore.opened(); + { + let held = store.lock(&name(), Deadline::after(10.0)).unwrap(); + let staged = held.stage().unwrap(); + for entry in [KEY_FILE, DESCRIPTION_FILE, OPENVMM_DIR] { + fs::rename(snapshot.dir.join(entry), staged.dir().join(entry)).unwrap(); + } + held.publish(staged).unwrap(); + } + Self { + root, + restore, + store, + spawned: Mutex::new(Vec::new()), + } + } + + fn state(&self, state: &str) -> PathBuf { + self.root.path().join(state) + } + + fn source(&self) -> Source { + Source { + snapshot: name(), + store: self.root.path().join("store"), + workload: WORKLOAD.map(str::to_owned).to_vec(), + } + } + + /// Restores the snapshot into the state directory `state`, whose fake VM does as `behavior` + /// says, as a start of a sandbox restored from it does. + fn start(&self, state: &str, config: VmConfig, behavior: &str, timeout: f64) -> Result { + let state_dir = self.state(state); + let mut lock = LifecycleLock::acquire(&state_dir, timeout, true).unwrap(); + let source = self.source(); + let restore = Restore { + source: &source, + identity: (1000, 1001), + }; + launch_locked( + &mut lock, + &self.restore.runtime, + config, + Some(restore), + None, + timeout, + |prepared| self.spawn(&state_dir, prepared, behavior), + ) + } + + /// Spawns the fake VM of `behavior` in place of the OpenVMM of `prepared`. + fn spawn( + &self, + state_dir: &Path, + prepared: Prepared, + behavior: &str, + ) -> crate::Result { + let arguments = prepared.arguments().to_vec(); + let ready = arguments + .windows(2) + .find(|pair| pair[0] == "--restore-ready-path") + .map(|pair| pair[1].to_str().unwrap().to_owned()) + .unwrap(); + let recipe = json!({ + "endpoint": prepared.outputs().endpoint, "ready": ready, "behavior": behavior, + "trace": state_dir.with_extension("trace"), + }); + fs::write( + state_dir.with_extension("json"), + serde_json::to_vec(&recipe).unwrap(), + ) + .unwrap(); + let mut spawned = self.spawned.lock().unwrap(); + let exited = spawned + .iter() + .all(|(reference, ..)| matches!(Process::open(reference), Ok(Opened::Exited(_)))); + let launched = vm::spawn( + prepared.run_instead( + std::env::current_exe().unwrap(), + ["--exact", HELPER, "--ignored", "--nocapture"] + .map(OsString::from) + .to_vec(), + ), + )?; + spawned.push((launched.process.reference().clone(), arguments, exited)); + Ok(launched) + } + + /// Returns the kind and payload of each NVXC request of the fake VM of `state`, in order. + fn requests(&self, state: &str) -> Vec<(u8, Vec)> { + fs::read_to_string(self.state(state).with_extension("trace")) + .unwrap_or_default() + .lines() + .map(|line| { + let request: Value = serde_json::from_str(line).unwrap(); + let payload = request["payload"].as_str().unwrap(); + let payload = (0..payload.len()) + .step_by(2) + .map(|at| u8::from_str_radix(&payload[at..at + 2], 16).unwrap()) + .collect(); + (request["kind"].as_u64().unwrap() as u8, payload) + }) + .collect() + } + + fn spawned(&self) -> Vec<(ProcessRef, Vec, bool)> { + self.spawned.lock().unwrap().clone() + } + + /// Returns whether the fake VM `index` has exited, which it waits a few seconds for. + fn exited(&self, index: usize) -> bool { + match Process::open(&self.spawned()[index].0).unwrap() { + Opened::Exited(_) => true, + Opened::Running(process) => process + .wait(Instant::now() + Duration::from_secs(5)) + .unwrap() + .is_some(), + } + } + + /// Returns whether another restore could take the snapshot's lock at once. + fn snapshot_free(&self) -> bool { + self.store.lock(&name(), Deadline::after(0.0)).is_ok() + } + + /// Returns how many entries the view of the snapshot in the state directory `state` has. + fn view(&self, state: &str) -> usize { + fs::read_dir(self.state(state).join(OPENVMM_DIR)).map_or(0, |entries| entries.count()) + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + for (reference, ..) in self.spawned() { + if let Ok(Opened::Running(process)) = Process::open(&reference) { + let _ = process.kill(); + let _ = process.wait(Instant::now() + KILL_TIMEOUT); + } + } + } +} + +/// Returns the configuration of a restore of `fixture`, of a GPT image if `gpt`, with an upper in +/// RAM if `ram`, one read-only share, and a policy that allows egress and publishes the guest's +/// listeners. +fn restored_config(fixture: &Fixture, gpt: bool, ram: bool) -> VmConfig { + let mut config = fixture.restore.config.clone(); + let RootConfig::Layered(root) = &mut config.root else { + unreachable!("the fixture restores a layered root"); + }; + if gpt { + root.format = RootImage::gpt(None); + root.gpt_first_partition_bound = Some(512 * 1024); + } + if ram { + root.upper = UpperConfig::Ram { + max_memory_percent: 33, + }; + } + let shared = fixture.root.path().join("work"); + fs::create_dir_all(&shared).unwrap(); + config.filesystem = Some(ExportPlan { + children: vec![ExportChild { + root: shared, + file: false, + writable: false, + denied: Vec::new(), + denied_identities: Vec::new(), + allowed: Vec::new(), + write: Vec::new(), + }], + binds: vec![Bind { + child: 0, + source: String::new(), + target: "/work".to_owned(), + read_only: true, + identity: None, + }], + owner: Owner::Vmm, + }); + let mut policy = NetworkPolicy::new(Access::Allow); + policy.ingress = Access::Allow; + config.network.as_mut().unwrap().policy = policy; + config +} + +#[test] +fn named_snapshots_restore_each_image_format_and_upper_with_their_own_policy_and_workload() { + for (gpt, ram) in [(false, false), (false, true), (true, false), (true, true)] { + let fixture = Fixture::new(true); + let config = restored_config(&fixture, gpt, ram); + let maps = config.mapping_table(); + fixture.start("state", config, "ready", 30.0).unwrap(); + let requests = fixture.requests("state"); + let kinds: Vec = requests.iter().map(|(kind, _)| *kind).collect(); + assert_eq!(kinds, [APP_ACTIVATE, APP_MAPS, APP_EXEC]); + let (record, identity) = activation(&requests[0].1); + assert_eq!(identity, (1000, 1001, 0)); + let configuration = record.configuration(); + // The restore's own policy gives the guest its DNS server and announces its listeners. + assert_eq!( + configuration.network, + Some(Network { + dns: Some("10.42.0.1".parse().unwrap()), + publication: Publication::Published, + proxy: None, + }) + ); + assert_eq!(matches!(configuration.image, Image::Gpt { .. }), gpt); + assert_eq!(matches!(configuration.upper, Upper::Tmpfs { .. }), ram); + assert_eq!(configuration.mappings, 1); + // The snapshot's two online processors, of four. + assert_eq!(configuration.resources.cpus, 2); + assert_eq!( + requests[1].1, + encode_maps_payloads(&maps, OUTER_MAX_PAYLOAD - APP_HEADER_LEN).unwrap()[0] + ); + let workload = protocol::Workload { + argv: WORKLOAD.map(str::to_owned).to_vec(), + timeout_ms: 0, + cwd: None, + environment: WorkloadEnvironment::Default, + }; + assert_eq!(requests[2].1, encode_exec_payload(&workload).unwrap()); + let [(reference, arguments, _)] = &fixture.spawned()[..] else { + panic!("one VM restores"); + }; + assert!( + arguments + .windows(2) + .any(|pair| pair[0] == "--network-ingress" && pair[1] == "allow") + ); + assert!( + arguments + .iter() + .any(|argument| argument == "--restore-network-policy") + ); + assert_eq!( + arguments + .iter() + .any(|argument| argument == "--restore-absent-scratch"), + ram + ); + // The VM runs on, committed with its view of the snapshot, whose lock is free again. + let locked = StateDir::open(&fixture.state("state")) + .unwrap() + .lock(Deadline::after(5.0)) + .unwrap(); + assert_eq!(&locked.runtime().unwrap().unwrap().process, reference); + assert!(locked.journal().unwrap().is_none()); + assert!(!fixture.exited(0)); + assert_eq!(fixture.view("state"), 3); + assert!(fixture.snapshot_free()); + } +} + +#[test] +fn concurrent_restores_of_one_snapshot_wait_for_its_lock_until_the_first_vm_has_exited() { + let fixture = Fixture::new(false); + let config = &fixture.restore.config; + let (first, second) = thread::scope(|scope| { + let first = scope.spawn(|| fixture.start("first", config.clone(), "stall", 2.0)); + let until = Instant::now() + Duration::from_secs(20); + while fixture.spawned().is_empty() { + assert!(Instant::now() < until, "the first restore spawned no VM"); + thread::sleep(Duration::from_millis(10)); + } + assert!(!fixture.snapshot_free()); + let second = fixture.start("second", config.clone(), "ready", 30.0); + (first.join().unwrap(), second) + }); + // The first VM never became ready, so its restore timed out and ended it, and the second VM + // started only once the first had exited. + assert_eq!(first.unwrap_err().code(), Some(ErrorCode::TimedOut)); + second.unwrap(); + let spawned = fixture.spawned(); + assert_eq!(spawned.len(), 2); + assert!(spawned[1].2); + assert!(fixture.exited(0)); + assert!(fixture.requests("first").is_empty()); + assert_eq!(fixture.requests("second").len(), 2); + assert!(fixture.snapshot_free()); +} + +#[test] +fn a_failed_restore_ends_its_vm_and_its_launch_before_its_snapshot_goes_free() { + for (behavior, code, message) in [ + ("malformed", ErrorCode::Protocol, "snapshot restore failed"), + ("exit", ErrorCode::Exited, "OpenVMM exited during startup"), + ( + "failing", + ErrorCode::Other, + "the restored workload ended with Exit status 3", + ), + // A guest that hangs up proves no exit: as for a cold start, its VM has the grace to exit + // by itself before the start ends it. + ("hangup", ErrorCode::Exited, "OpenVMM exited during startup"), + ] { + let fixture = Fixture::new(false); + let error = fixture + .start("state", fixture.restore.config.clone(), behavior, 30.0) + .unwrap_err(); + assert_eq!(error.code(), Some(code), "{behavior}: {error:?}"); + assert!( + error.message().is_some_and(|text| text.contains(message)), + "{behavior}: {error:?}" + ); + assert!(fixture.exited(0), "{behavior}"); + let locked = StateDir::open(&fixture.state("state")) + .unwrap() + .lock(Deadline::after(5.0)) + .unwrap(); + assert!(locked.journal().unwrap().is_none(), "{behavior}"); + assert!(locked.runtime().unwrap().is_none(), "{behavior}"); + assert!(fixture.snapshot_free(), "{behavior}"); + } +} + +#[test] +fn a_restore_whose_spawn_cannot_confirm_that_its_vm_ended_recovers_it_before_its_snapshot_goes_free() + { + let fixture = Fixture::new(false); + let state_dir = fixture.state("state"); + let mut lock = LifecycleLock::acquire(&state_dir, 30.0, true).unwrap(); + let source = fixture.source(); + let restore = Restore { + source: &source, + identity: (1000, 1001), + }; + let error = launch_locked( + &mut lock, + &fixture.restore.runtime, + fixture.restore.config.clone(), + Some(restore), + None, + 30.0, + |prepared| { + // The VM runs, but the spawn cannot confirm that it ended. + fixture.spawn(&state_dir, prepared, "stall")?; + Err(crate::Error::new( + ErrorKind::StillRunning, + "the spawn cannot confirm that its VM ended", + )) + }, + ) + .unwrap_err(); + assert!( + error + .message() + .is_some_and(|text| text.contains("cannot confirm that its VM ended")), + "{error:?}" + ); + // The journal's recovery ended the VM, which nothing recorded, before the snapshot went free. + assert!(fixture.exited(0)); + assert!(lock.locked().journal().unwrap().is_none()); + assert!(fixture.snapshot_free()); +} + +#[test] +fn a_later_start_restores_again_with_an_upper_in_ram_but_never_with_a_scratch_image_it_gave() { + for ram in [true, false] { + let fixture = Fixture::new(false); + let mut config = fixture.restore.config.clone(); + if ram { + let RootConfig::Layered(root) = &mut config.root else { + unreachable!("the fixture restores a layered root"); + }; + root.upper = UpperConfig::Ram { + max_memory_percent: 33, + }; + } + fixture + .start("state", config.clone(), "ready", 30.0) + .unwrap(); + let Opened::Running(first) = Process::open(&fixture.spawned()[0].0).unwrap() else { + panic!("the restored VM runs"); + }; + first.kill().unwrap(); + assert!(fixture.exited(0)); + LifecycleLock::acquire(&fixture.state("state"), 5.0, false) + .unwrap() + .clear_runtime() + .unwrap(); + let again = fixture.start("state", config, "ready", 30.0); + if ram { + again.unwrap(); + assert_eq!(fixture.spawned().len(), 2); + assert_eq!(fixture.requests("state").len(), 4); + } else { + // The guest of the first start may have written the image, which a restore's guest + // refuses: nothing launches, and the view stays for deprovision. + let error = again.unwrap_err(); + assert_eq!(error.code(), Some(ErrorCode::Unsupported), "{error:?}"); + assert_eq!(fixture.spawned().len(), 1); + assert_eq!(fixture.requests("state").len(), 2); + } + assert_eq!(fixture.view("state"), 3, "{ram}"); + assert!(fixture.snapshot_free()); + } +} + +#[test] +fn a_restore_whose_command_dies_before_ready_never_activates_and_recovery_ends_its_vm() { + let fixture = Fixture::new(false); + let state_dir = fixture.state("state"); + vm::create_private_dir(&state_dir).unwrap(); + let state = StateDir::open(&state_dir).unwrap(); + { + // What a restore's start does before its guest is ready. + let held = fixture.store.lock(&name(), Deadline::after(5.0)).unwrap(); + let snapshot = held.open().unwrap(); + let runtime = &fixture.restore.runtime; + let vm = VmDir::open(&state_dir).unwrap(); + let prepared = vm::create(runtime, &fixture.restore.config, Some(&snapshot), &vm).unwrap(); + let mut locked = state.lock(Deadline::after(5.0)).unwrap(); + let mut launch = locked.begin_launch(&prepared, &runtime.identity).unwrap(); + let launched = fixture.spawn(&state_dir, prepared, "stall").unwrap(); + launch.record_process(launched.process.reference()).unwrap(); + // The command dies here: its locks go, and its journal, VM, and view stay. + } + assert!(fixture.snapshot_free()); + assert!(!fixture.exited(0)); + let recovered = state.lock(Deadline::after(10.0)).unwrap(); + assert!(fixture.exited(0)); + assert!(recovered.journal().unwrap().is_none()); + assert!(recovered.runtime().unwrap().is_none()); + assert!(fixture.requests("state").is_empty()); + assert_eq!(fixture.view("state"), 3); +} + +#[test] +fn deprovision_removes_a_restored_sandboxs_view_only_under_its_snapshots_lock() { + let fixture = Fixture::new(false); + let state_dir = fixture.state("state"); + vm::create_private_dir(&state_dir).unwrap(); + { + let held = fixture.store.lock(&name(), Deadline::after(5.0)).unwrap(); + let snapshot = held.open().unwrap(); + let vm = VmDir::open(&state_dir).unwrap(); + let runtime = &fixture.restore.runtime; + drop(vm::create(runtime, &fixture.restore.config, Some(&snapshot), &vm).unwrap()); + } + let source = fixture.source(); + // The record of a restored sandbox, of an EROFS image with an upper in RAM. + let image = fixture.root.path().join("distro.erofs"); + fs::write(&image, b"distro layer\n").unwrap(); + let layer = SandboxLayer::new( + LayerRole::Distro, + image.to_str().unwrap(), + "11111111-1111-1111-1111-111111111111", + ); + let binding = DistroBinding::verify(&layer).unwrap(); + let launch = + SandboxLaunch::new(vec![layer], SandboxUpper::RAM, SandboxSettings::default()).unwrap(); + let options = ProvisionConfig { + hypervisor: "kvm", + memory_mib: 512, + cmdline: "", + ram_upper_percent: None, + }; + let distro = RecordedDistro::new(&launch, binding).unwrap(); + let text = serialize_launch(&launch, &distro, None, &options, Some(&source)).unwrap(); + write_json(&state_dir.join(CONFIG_NAME), &text).unwrap(); + let state_path = state_dir.to_str().unwrap(); + let artifacts = Artifacts { root: None }; + let held = fixture.store.lock(&name(), Deadline::after(5.0)).unwrap(); + let error = deprovision(state_path, &artifacts, 0.5).unwrap_err(); + assert_eq!(error.code(), Some(ErrorCode::SandboxBusy), "{error:?}"); + assert_eq!(fixture.view("state"), 3); + assert!(state_dir.join(CONFIG_NAME).exists()); + drop(held); + deprovision(state_path, &artifacts, 5.0).unwrap(); + assert!(!state_dir.exists()); + // The snapshot's own files stay whole. + fixture + .store + .lock(&name(), Deadline::after(5.0)) + .unwrap() + .open() + .unwrap(); +} + +/// A start of a restored sandbox refuses a share of its snapshot store, through which the guest +/// could change the snapshot's files that the view of each restore links, before it takes the +/// snapshot's lock or launches anything. +#[test] +fn a_restored_start_refuses_a_share_of_its_snapshot_store() { + let fixture = Fixture::new(false); + let state_dir = fixture.state("state"); + vm::create_private_dir(&state_dir).unwrap(); + let image = fixture.root.path().join("distro.erofs"); + fs::write(&image, b"distro layer\n").unwrap(); + let layer = SandboxLayer::new( + LayerRole::Distro, + image.to_str().unwrap(), + "11111111-1111-1111-1111-111111111111", + ); + let binding = DistroBinding::verify(&layer).unwrap(); + let store = fixture.root.path().join("store"); + let mounts = sandbox_mounts(&[format!("/data,{},rw", store.display())], &[], None).unwrap(); + validate_mounts(&mounts).unwrap(); + let launch = SandboxLaunch::new(vec![layer], SandboxUpper::RAM, SandboxSettings::default()) + .unwrap() + .with_export(plan_mounts(&mounts).unwrap()) + .unwrap(); + let options = ProvisionConfig { + hypervisor: "kvm", + memory_mib: 512, + cmdline: "", + ram_upper_percent: None, + }; + let distro = RecordedDistro::new(&launch, binding).unwrap(); + let source = fixture.source(); + let text = serialize_launch(&launch, &distro, None, &options, Some(&source)).unwrap(); + write_json(&state_dir.join(CONFIG_NAME), &text).unwrap(); + let root = fixture.state("checkout"); + let artifacts = Artifacts { + root: Some(root.to_str().unwrap()), + }; + let error = start(state_dir.to_str().unwrap(), &artifacts, 5.0).unwrap_err(); + assert_eq!(error.code(), Some(ErrorCode::Unsupported), "{error:?}"); + let message = error.message().unwrap_or_default(); + assert!( + message.contains("overlaps the sandbox's snapshot store"), + "{message}" + ); + assert_eq!(fixture.view("state"), 0); + assert!(fixture.spawned().is_empty()); +} + +/// Reads one record of the control endpoint, or `None` once the host has gone. +fn read_record(stream: &mut impl Read) -> Option<(u8, Vec)> { + let mut header = [0; OUTER_HEADER_LEN]; + stream.read_exact(&mut header).ok()?; + let header = decode_outer_header(&header).unwrap(); + let mut payload = vec![0; header.length]; + stream.read_exact(&mut payload).ok()?; + Some((header.record_type, payload)) +} + +/// Serves one session of the host as the guest of the recipe's behavior does, and traces its +/// requests. +fn serve(stream: &mut (impl Read + Write), capability: &[u8], epoch: u64, recipe: &Value) { + let Some((kind, payload)) = read_record(stream) else { + return; + }; + assert_eq!((kind, payload.as_slice()), (OUTER_HOST_ATTACH, capability)); + let ready = encode_outer(OUTER_READY, &[7; 16], epoch, 0, &[]).unwrap(); + if stream.write_all(&ready).is_err() { + return; + } + let trace = recipe["trace"].as_str().unwrap(); + let mut sequence = 1; + while let Some((kind, payload)) = read_record(stream) { + assert_eq!(kind, OUTER_DATA); + let request = decode_app(&payload).unwrap(); + let line = json!({"kind": request.kind, "payload": hex(request.payload)}); + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(trace) + .unwrap(); + writeln!(file, "{line}").unwrap(); + // A guest that hangs up does once it has the workload's Start, which it never answers. + if request.kind == APP_EXEC && recipe["behavior"] == "hangup" { + return; + } + let replies = match request.kind { + APP_ACTIVATE if recipe["behavior"] == "malformed" => vec![(APP_READY, 0, vec![0])], + APP_ACTIVATE => { + let ready = ReadyActivation::of(&activation(request.payload).0); + vec![(APP_READY, 0, ready.encode())] + } + APP_MAPS => vec![(APP_READY, 0, Vec::new())], + // The workload writes its output, then exits, with status 3 if it fails. + _ => vec![ + (APP_STDOUT, 0, b"restored\n".to_vec()), + ( + APP_EXIT, + if recipe["behavior"] == "failing" { + 3 + } else { + 0 + }, + b"exit".to_vec(), + ), + ], + }; + for (kind, status, payload) in replies { + let reply = encode_app(kind, request.request_id, status, &payload); + let record = encode_outer(OUTER_DATA, &[7; 16], epoch, sequence, &reply).unwrap(); + if stream.write_all(&record).is_err() { + return; + } + sequence += 1; + } + } +} + +#[test] +#[ignore = "the fake VMM and guest of a test restore; not real-VM restore proof"] +fn restore_helper() { + let mut capability = Vec::new(); + std::io::stdin().read_to_end(&mut capability).unwrap(); + let recipe = std::env::current_dir().unwrap().with_extension("json"); + let recipe: Value = serde_json::from_slice(&fs::read(recipe).unwrap()).unwrap(); + let field = |name: &str| recipe[name].as_str().unwrap().to_owned(); + if recipe["behavior"] == "exit" { + std::process::exit(3); + } + #[cfg(target_os = "linux")] + let server = std::os::unix::net::UnixListener::bind(field("endpoint")).unwrap(); + #[cfg(windows)] + let server = platform::tests::control_server(&field("endpoint")); + if recipe["behavior"] == "stall" { + // The VM serves its control endpoint, but its guest never becomes ready. + loop { + #[cfg(target_os = "linux")] + thread::sleep(Duration::from_secs(1)); + #[cfg(windows)] + { + platform::tests::accept_control(&server); + // Each client keeps the pipe until it hangs up. + let _ = std::io::copy(&mut &server, &mut std::io::sink()); + platform::tests::disconnect_control(&server); + } + } + } + #[cfg(target_os = "linux")] + let mut event = std::os::unix::net::UnixStream::connect(field("ready")).unwrap(); + #[cfg(windows)] + let mut event = OpenOptions::new() + .write(true) + .open(field("ready").replace('/', "\\")) + .unwrap(); + event.write_all(READY_EVENT).unwrap(); + drop(event); + for epoch in 1.. { + #[cfg(target_os = "linux")] + let mut stream = server.accept().unwrap().0; + #[cfg(windows)] + let mut stream = { + platform::tests::accept_control(&server); + &server + }; + serve(&mut stream, &capability, epoch, &recipe); + #[cfg(windows)] + platform::tests::disconnect_control(&server); + if recipe["behavior"] == "hangup" { + // The VM exits by itself, a while after its guest hung up. + #[cfg(target_os = "linux")] + drop(stream); + thread::sleep(Duration::from_secs(2)); + std::process::exit(4); + } + } +} diff --git a/nvx-rs/src/host/cli/sandbox/lifecycle/stop.rs b/nvx-rs/src/host/cli/sandbox/lifecycle/stop.rs index 4e5010612..c0a3f5460 100644 --- a/nvx-rs/src/host/cli/sandbox/lifecycle/stop.rs +++ b/nvx-rs/src/host/cli/sandbox/lifecycle/stop.rs @@ -1,20 +1,24 @@ //! Stopping a managed sandbox and removing a stopped sandbox's state. +use std::fs; use std::time::{Duration, Instant}; use super::directory::prepare_state_directory; use super::outcome::read_openvmm_outcome; -use super::{Found, Running, connect, differing_parts, found_vm, running}; +use super::{CONFIG_NAME, Found, Running, connect, differing_parts, found_vm, running}; use crate::ErrorCode; use crate::host::cli::deadline::{Deadline, RETRY_INTERVAL}; use crate::host::cli::sandbox::lock::LifecycleLock; +use crate::host::cli::sandbox::state::{Config, read_config}; use crate::host::cli::sandbox::{Artifacts, error_with_cause}; +use crate::host::cli::snapshots::Store; use crate::host::cli::store::StateFile; use crate::host::cli::{Exit, Result, check_interrupt, output}; use crate::host::process::Process; use crate::host::runtime::IdentityPart; +use crate::host::snapshot::OPENVMM_DIR; use crate::host::supervisor::StopOutcome; -use crate::host::vm::{self, SavedVm, VmOutputs}; +use crate::host::vm::{self, SavedVm, VmDir, VmOutputs}; /// Stops a running sandbox, and checks the outcome report that its OpenVMM wrote, unless another /// runtime identity than the artifacts' started the VM. @@ -183,6 +187,9 @@ pub(super) fn wait_for_exit(process: &Process, deadline: Deadline) -> Result, timeout: f64) -> Result { let state_dir = prepare_state_directory(state_path, false)?; let mut lock = LifecycleLock::acquire(&state_dir, timeout, false)?; @@ -195,6 +202,16 @@ pub(crate) fn deprovision(state_path: &str, artifacts: &Artifacts<'_>, timeout: )); } } + // The store accepts a state directory that a VM directory would not, such as an earlier + // release's, so deprovision requires a VM directory only to remove a view from one. + if fs::symlink_metadata(state_dir.join(OPENVMM_DIR)).is_ok() + && let Some(source) = Config::new(read_config(&state_dir.join(CONFIG_NAME))?)?.source()? + { + let vm = VmDir::open(&state_dir).map_err(error_with_cause)?; + Store::open(&source.store)? + .lock(&source.snapshot, Deadline::after(timeout))? + .remove_view(&vm)?; + } // No OpenVMM of the sandbox runs, and a start that was interrupted is over, so the store // removes the sandbox: its files and the configuration, then refuses files that are not // NVX's, and otherwise removes its lock file, while it still holds the lock, and the diff --git a/nvx-rs/src/host/cli/sandbox/lock/tests.rs b/nvx-rs/src/host/cli/sandbox/lock/tests.rs index 0ffb1b80e..a0392e66d 100644 --- a/nvx-rs/src/host/cli/sandbox/lock/tests.rs +++ b/nvx-rs/src/host/cli/sandbox/lock/tests.rs @@ -96,6 +96,23 @@ fn provision_refuses_an_inherited_vm_directory_without_changing_it() { assert_eq!(entries, [std::ffi::OsString::from("foreign.txt")]); } +/// A deprovision requires a VM directory only of a restored sandbox, to remove its view, and +/// removes a cold sandbox whose state directory the store accepts but a VM directory would not. +#[cfg(windows)] +#[test] +fn deprovision_removes_a_cold_sandbox_whose_state_directory_is_no_vm_directory() { + let directory = tempfile::tempdir().unwrap(); + let parent = directory.path().join("private-parent"); + StateDir::create(&parent).unwrap(); + let state = parent.join("inherited"); + std::fs::create_dir(&state).unwrap(); + VmDir::open(&state).unwrap_err(); + std::fs::write(state.join(CONFIG_NAME), "{}").unwrap(); + let artifacts = super::super::Artifacts { root: None }; + super::super::lifecycle::deprovision(state.to_str().unwrap(), &artifacts, 5.0).unwrap(); + assert!(!state.exists()); +} + #[test] fn removals_refuse_live_records_and_keep_foreign_files() { let directory = tempfile::tempdir().unwrap(); diff --git a/nvx-rs/src/host/cli/sandbox/mod.rs b/nvx-rs/src/host/cli/sandbox/mod.rs index a42943898..8fed77427 100644 --- a/nvx-rs/src/host/cli/sandbox/mod.rs +++ b/nvx-rs/src/host/cli/sandbox/mod.rs @@ -346,11 +346,11 @@ pub(super) fn run(root: Option<&str>, options: Options) -> Result { } /// Refuses a share of the sandbox's own files: one whose host path equals, contains, or lies -/// inside `directory`, if the sandbox has one, with its name, its state directory or a run's VM -/// directory; the image or scratch image of `launch`; or, if known, the OpenVMM, kernel, or Alpine -/// initramfs of the artifact root of `artifacts`, or the OpenVMM that `NVX_OPENVMM` names instead, -/// which a run runs. Through it, the guest could read or change the sandbox's records, its -/// storage, or what the host runs. +/// inside `directory`, if the sandbox has one, with its name, its state directory, a run's VM +/// directory, or a restored sandbox's snapshot store; the image or scratch image of `launch`; or, +/// if known, the OpenVMM, kernel, or Alpine initramfs of the artifact root of `artifacts`, or the +/// OpenVMM that `NVX_OPENVMM` names instead, which a run runs. Through it, the guest could read or +/// change the sandbox's records, its storage, its snapshot, or what the host runs. /// /// The paths are compared once [`paths::resolve`] has resolved them, without a Windows verbatim /// prefix. Resolution follows symbolic links and junctions, but not hard links or bind mounts. diff --git a/nvx-rs/src/host/cli/sandbox/state.rs b/nvx-rs/src/host/cli/sandbox/state.rs index f0bcbd98b..c2dbcaf92 100644 --- a/nvx-rs/src/host/cli/sandbox/state.rs +++ b/nvx-rs/src/host/cli/sandbox/state.rs @@ -5,6 +5,12 @@ //! Provision checks the same [`VmConfig`] that start reconstructs. Older records are //! refused, not migrated. Bindings still check the recorded caller-owned paths before //! launch, and the caller keeps those paths stable while the VM uses them. +//! +//! The configuration of a restored sandbox also records its [`Source`]: the name of its snapshot +//! under `source_snapshot`, the absolute path of the snapshot's store under `source_store`, and +//! the program and arguments of the workload that each restore runs under `workload`, which +//! [`Config::source`] reads back for `start` to restore the sandbox and for `deprovision` to +//! remove its view of the snapshot. That of a sandbox that boots cold records none of them. use std::fs::OpenOptions; use std::io::Write; @@ -191,26 +197,31 @@ enum Upper { Ram { max_memory_percent: u8 }, } +/// The named snapshot from which each start of a restored sandbox restores it, and the workload +/// that each restore runs: the only association between a snapshot and the sandboxes that it +/// supplies, which their own configurations record. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Source { + /// The snapshot's name. + pub(crate) snapshot: SnapshotName, + /// The absolute path of the snapshot's store. + pub(crate) store: PathBuf, + /// The program, an absolute guest path, and the arguments of the workload. + pub(crate) workload: Vec, +} + /// Returns the JSON text of the configuration that `provision` records for `launch` and the other /// options, with `distro`, the launch's distro layer at the path that `provision` verified and /// records, and its binding, and `scratch`, the launch's scratch image at the path that -/// `provision` records, and its binding, or `None` for an upper in RAM. +/// `provision` records, and its binding, or `None` for an upper in RAM. The configuration of a +/// restored sandbox also records its `source`, without any index of the sandboxes that a snapshot +/// supplies; that of a sandbox that boots cold, whose `source` is `None`, records none. pub(crate) fn serialize_launch( launch: &SandboxLaunch, distro: &RecordedDistro, scratch: Option<&RecordedScratch>, config: &ProvisionConfig<'_>, -) -> Result { - serialize_launch_with_snapshot(launch, distro, scratch, config, None) -} - -/// Records a restored sandbox's source alongside its own launch, without a snapshot index. -pub(crate) fn serialize_launch_with_snapshot( - launch: &SandboxLaunch, - distro: &RecordedDistro, - scratch: Option<&RecordedScratch>, - config: &ProvisionConfig<'_>, - source_snapshot: Option<&SnapshotName>, + source: Option<&Source>, ) -> Result { let settings = launch.settings(); let (workload_uid, workload_gid) = settings.workload_identity; @@ -237,7 +248,11 @@ pub(crate) fn serialize_launch_with_snapshot( distro_binding: distro.binding.clone(), scratch_binding: scratch.map(|scratch| scratch.binding.clone()), export: launch.export().cloned(), - source_snapshot: source_snapshot.cloned(), + source_snapshot: source.map(|source| source.snapshot.clone()), + source_store: source + .map(|source| path_string(&source.store)) + .transpose()?, + workload: source.map(|source| source.workload.clone()), }; record .vm_config(&record.launch()?)? @@ -283,18 +298,33 @@ pub(crate) struct Config { /// launch. #[serde(default, skip_serializing_if = "Option::is_none")] source_snapshot: Option, + /// The store of the snapshot of a restored sandbox. + #[serde(default, skip_serializing_if = "Option::is_none")] + source_store: Option, + /// The workload that each restore of a restored sandbox runs. + #[serde(default, skip_serializing_if = "Option::is_none")] + workload: Option>, } impl Config { - #[cfg_attr( - not(test), - expect( - dead_code, - reason = "SL3.8a locks a restored sandbox's source snapshot" - ) - )] - pub(crate) fn source_snapshot(&self) -> Option<&SnapshotName> { - self.source_snapshot.as_ref() + /// Returns the [`Source`] of a restored sandbox, or `None` for one that boots cold. + /// + /// A configuration that records only part of a source, a store that is not absolute, or an + /// empty workload fails as `sandbox configuration is malformed`. + pub(crate) fn source(&self) -> Result> { + match (&self.source_snapshot, &self.source_store, &self.workload) { + (None, None, None) => Ok(None), + (Some(snapshot), Some(store), Some(workload)) + if Path::new(store).is_absolute() && !workload.is_empty() => + { + Ok(Some(Source { + snapshot: snapshot.clone(), + store: PathBuf::from(store), + workload: workload.clone(), + })) + } + _ => Err(malformed()), + } } /// Returns the fields of `object`, a configuration that [`read_config`] read, which fails as diff --git a/nvx-rs/src/host/cli/sandbox/state/tests.rs b/nvx-rs/src/host/cli/sandbox/state/tests.rs index 17332890c..89f952492 100644 --- a/nvx-rs/src/host/cli/sandbox/state/tests.rs +++ b/nvx-rs/src/host/cli/sandbox/state/tests.rs @@ -1,8 +1,10 @@ +use serde_json::json; + use super::*; use crate::host::sandbox::layer::LayerRole; #[test] -fn restored_sandbox_records_round_trip_only_their_own_snapshot_name() { +fn restored_sandbox_records_round_trip_only_their_own_source() { let directory = tempfile::tempdir().unwrap(); let image = directory.path().join("distro.erofs"); std::fs::write(&image, b"distro layer\n").unwrap(); @@ -21,16 +23,45 @@ fn restored_sandbox_records_round_trip_only_their_own_snapshot_name() { cmdline: "", ram_upper_percent: None, }; - for source in [None, Some("My.snapshot-1"), Some("other")] { - let name = source.map(|name| SnapshotName::try_from(name.to_owned()).unwrap()); - let text = serialize_launch_with_snapshot(&launch, &distro, None, &options, name.as_ref()) - .unwrap(); + for name in [None, Some("My.snapshot-1"), Some("other")] { + let source = name.map(|name| Source { + snapshot: SnapshotName::try_from(name.to_owned()).unwrap(), + store: directory.path().join("store"), + workload: vec!["/bin/echo".to_owned(), name.to_owned()], + }); + let text = serialize_launch(&launch, &distro, None, &options, source.as_ref()).unwrap(); let object: Object = serde_json::from_str(&text).unwrap(); - assert_eq!(object.contains_key("source_snapshot"), source.is_some()); - let record = Config::new(object.clone()).unwrap(); - assert_eq!(record.source_snapshot(), name.as_ref()); - let mut malformed = object; - malformed.insert("source_snapshot".to_owned(), Value::from("../other")); - assert!(Config::new(malformed).is_err()); + for key in ["source_snapshot", "source_store", "workload"] { + assert_eq!(object.contains_key(key), name.is_some(), "{key}"); + } + assert_eq!( + Config::new(object.clone()).unwrap().source().unwrap(), + source + ); + if source.is_none() { + continue; + } + // A name that escapes its store, a part of a source, a relative store, and an empty + // workload are refused. + for (key, value) in [ + ("source_snapshot", Some(json!("../other"))), + ("source_snapshot", None), + ("source_store", None), + ("source_store", Some(json!("relative"))), + ("workload", None), + ("workload", Some(json!([]))), + ] { + let mut malformed = object.clone(); + match value { + Some(value) => malformed.insert(key.to_owned(), value), + None => malformed.remove(key), + }; + assert!( + Config::new(malformed) + .and_then(|config| config.source()) + .is_err(), + "{key}" + ); + } } } diff --git a/nvx-rs/src/host/cli/snapshots.rs b/nvx-rs/src/host/cli/snapshots.rs index 1125b20fe..9456f727d 100644 --- a/nvx-rs/src/host/cli/snapshots.rs +++ b/nvx-rs/src/host/cli/snapshots.rs @@ -3,9 +3,15 @@ //! Names map to hexadecimal directory names under `snapshots/`. Each published directory //! contains only the strict snapshot's three entries; its lock stays under `locks/`. //! Completed private directories under `staging/` publish with one non-replacing rename. +//! +//! Each start of a restored sandbox links the snapshot's files into the sandbox's view of it +//! under the snapshot's lock, which [`LockedSnapshot::remove_view`] also takes to remove the +//! view. pub(super) mod capture; +use std::fs; +use std::io; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; @@ -13,8 +19,9 @@ use serde::{Deserialize, Serialize}; use super::deadline::Deadline; use super::store::{LockedState, StateDir}; use super::{Exit, Result, hex, io_error, temporary_path}; -use crate::host::snapshot::Snapshot; -use crate::host::{platform, store}; +use crate::host::snapshot::{OPENVMM_DIR, SNAPSHOT_ARTIFACTS, Snapshot}; +use crate::host::vm::VmDir; +use crate::host::{filesystem, paths, platform, store}; use crate::{Error, ErrorCode}; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -59,6 +66,18 @@ impl Store { }) } + /// Opens an existing store, private to the current user, as [`create`](Self::create) makes + /// it, without creating anything. + pub(crate) fn open(root: &Path) -> Result { + let root = StateDir::open(&std::path::absolute(root).map_err(Exit::from_os_error)?)?; + Ok(Self { + entries: StateDir::open(&root.dir().join("snapshots"))?, + locks: StateDir::open(&root.dir().join("locks"))?, + staging: StateDir::open(&root.dir().join("staging"))?, + root, + }) + } + fn check(&self) -> Result { for directory in [&self.root, &self.entries, &self.locks, &self.staging] { StateDir::open(directory.dir())?; @@ -70,7 +89,18 @@ impl Store { self.check()?; // Hex preserves case distinctions and avoids reserved host filenames. let directory = hex(name.0.as_bytes()); - let lock = StateDir::create(&self.locks.dir().join(&directory))?.lock(deadline)?; + let lock = StateDir::create(&self.locks.dir().join(&directory))? + .lock(deadline) + .map_err(|error| match error.code() { + Some(ErrorCode::SandboxBusy) => Exit::error( + ErrorCode::SandboxBusy, + format!( + "another command still holds the lock of the snapshot {}", + name.0 + ), + ), + _ => error, + })?; Ok(LockedSnapshot { store: self, dir: self.entries.dir().join(directory), @@ -97,6 +127,54 @@ impl LockedSnapshot<'_> { Snapshot::open(&self.dir).map_err(Into::into) } + /// Removes the view of this snapshot that a restore created in `vm`, and on Linux the + /// restore-ready socket of that restore, which this lock must cover, since removing the view + /// changes the links of the snapshot's files. Refuses a view that is a link or holds anything + /// else, which it leaves, and anything else in place of the socket, and, before it removes + /// anything, a VM directory in the store, whose view could be a snapshot's own files. + pub(crate) fn remove_view(&self, vm: &VmDir) -> Result { + outside_store(self.store.root.dir(), vm.path())?; + let foreign = |path: &Path| { + Exit::error( + ErrorCode::ForeignFiles, + format!("{} is not a restore's own", path.display()), + ) + }; + #[cfg(not(windows))] + { + use std::os::unix::fs::FileTypeExt; + + let ready = PathBuf::from(vm.restore_ready_endpoint()?); + match fs::symlink_metadata(&ready) { + Ok(metadata) if metadata.file_type().is_socket() => fs::remove_file(&ready) + .map_err(|error| io_error(ErrorCode::Io, &error, &ready))?, + Ok(_) => return Err(foreign(&ready)), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(error) => return Err(io_error(ErrorCode::Io, &error, &ready)), + } + } + let view = vm.path().join(OPENVMM_DIR); + match fs::symlink_metadata(&view) { + Ok(metadata) if metadata.is_dir() && !filesystem::is_reparse_point(&metadata) => {} + Ok(_) => return Err(foreign(&view)), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(io_error(ErrorCode::Io, &error, &view)), + } + for name in SNAPSHOT_ARTIFACTS { + let path = view.join(name); + match fs::remove_file(&path) { + Err(error) if error.kind() != io::ErrorKind::NotFound => { + return Err(io_error(ErrorCode::Io, &error, &path)); + } + _ => {} + } + } + match fs::remove_dir(&view) { + Err(error) if error.kind() == io::ErrorKind::DirectoryNotEmpty => Err(foreign(&view)), + removed => removed.map_err(|error| io_error(ErrorCode::Io, &error, &view)), + } + } + pub(crate) fn publish(&self, staged: StateDir) -> Result { self.store.check()?; if staged.dir().parent() != Some(self.store.staging.dir()) { @@ -115,5 +193,18 @@ impl LockedSnapshot<'_> { } } +/// Refuses `dir` in the snapshot store at `root`, once the links of both are resolved: the view +/// of a restore in that directory could take the files of the store's snapshots for its own. +fn outside_store(root: &Path, dir: &Path) -> Result { + let resolve = |path: &Path| paths::resolve(path).map_err(Exit::from_os_error); + if resolve(dir)?.starts_with(resolve(root)?) { + return Err(Exit::error( + ErrorCode::Unsupported, + "the sandbox's state directory must be outside the snapshot store", + )); + } + Ok(()) +} + #[cfg(test)] mod tests; diff --git a/nvx-rs/src/host/cli/snapshots/tests.rs b/nvx-rs/src/host/cli/snapshots/tests.rs index 4b75fa374..c0b7aae5d 100644 --- a/nvx-rs/src/host/cli/snapshots/tests.rs +++ b/nvx-rs/src/host/cli/snapshots/tests.rs @@ -236,3 +236,57 @@ fn shared_directory_acls_are_refused_before_publication() { assert!(pending.dir().exists()); assert!(!held.dir.exists()); } + +/// Returns a store with the published snapshot `first`, and a VM directory with a view of it. +fn restored_view(root: &Path) -> (Store, VmDir) { + fs::create_dir_all(root).unwrap(); + let store = Store::create(&root.join("store")).unwrap(); + let held = store.lock(&name("first"), Deadline::after(0.0)).unwrap(); + held.publish(staged(&held)).unwrap(); + let path = root.join("vm"); + crate::host::vm::create_private_dir(&path).unwrap(); + let vm = VmDir::open(&path).unwrap(); + let view = path.join(OPENVMM_DIR); + crate::host::vm::create_private_dir(&view).unwrap(); + for name in SNAPSHOT_ARTIFACTS { + fs::hard_link(held.dir.join(OPENVMM_DIR).join(name), view.join(name)).unwrap(); + } + drop(held); + (store, vm) +} + +#[test] +fn removing_a_view_takes_only_the_snapshots_links_and_keeps_the_snapshot() { + let root = tempfile::tempdir().unwrap(); + let (store, vm) = restored_view(root.path()); + let held = store.lock(&name("first"), Deadline::after(0.0)).unwrap(); + held.remove_view(&vm).unwrap(); + assert!(!vm.path().join(OPENVMM_DIR).exists()); + held.remove_view(&vm).unwrap(); + held.open().unwrap(); + // A view that holds anything else keeps it, and one that is a link keeps its target. + let (store, vm) = restored_view(&root.path().join("extra")); + fs::write(vm.path().join(OPENVMM_DIR).join("other"), b"kept").unwrap(); + let held = store.lock(&name("first"), Deadline::after(0.0)).unwrap(); + let error = held.remove_view(&vm).unwrap_err(); + assert_eq!(error.code(), Some(ErrorCode::ForeignFiles), "{error:?}"); + assert_eq!( + fs::read(vm.path().join(OPENVMM_DIR).join("other")).unwrap(), + b"kept" + ); + let (store, vm) = restored_view(&root.path().join("linked")); + let view = vm.path().join(OPENVMM_DIR); + let target = root.path().join("target"); + fs::rename(&view, &target).unwrap(); + link_directory(&view, &target); + let held = store.lock(&name("first"), Deadline::after(0.0)).unwrap(); + let error = held.remove_view(&vm).unwrap_err(); + assert_eq!(error.code(), Some(ErrorCode::ForeignFiles), "{error:?}"); + assert_eq!(fs::read_dir(&target).unwrap().count(), 3); + // A VM directory in the store, here the snapshot's own, whose `openvmm/` holds the snapshot's + // files, is refused before anything goes. + let own = VmDir::open(&held.dir).unwrap(); + let error = held.remove_view(&own).unwrap_err(); + assert_eq!(error.code(), Some(ErrorCode::Unsupported), "{error:?}"); + held.open().unwrap(); +} diff --git a/nvx-rs/src/host/restore_driver.rs b/nvx-rs/src/host/restore_driver.rs index 9b819950e..e35650aa0 100644 --- a/nvx-rs/src/host/restore_driver.rs +++ b/nvx-rs/src/host/restore_driver.rs @@ -108,13 +108,6 @@ impl std::error::Error for ControlRestoreError { /// `maps` is the mapping table that `record` announces. A failure stops nothing: the caller keeps /// the VM's process, its outputs and its restore-ready listener, and decides what to do with /// them. -#[cfg_attr( - not(test), - expect( - dead_code, - reason = "SL3.8a's `snapshot restore` restores sandboxes through it" - ) -)] pub(crate) fn restore( launched: &mut Launched, record: &ActivationRecord, diff --git a/nvx-rs/src/host/vm/tests.rs b/nvx-rs/src/host/vm/tests.rs index c434c6b8f..f70176271 100644 --- a/nvx-rs/src/host/vm/tests.rs +++ b/nvx-rs/src/host/vm/tests.rs @@ -49,6 +49,17 @@ impl Launched { } } +#[cfg(all(feature = "cli", any(target_os = "linux", windows)))] +impl Prepared { + /// Runs `program` with `arguments` in place of OpenVMM, with the rest of the checked launch: + /// its capability, outputs, log, inherited resources, and restore-ready listener. + pub(crate) fn run_instead(mut self, program: PathBuf, arguments: Vec) -> Self { + self.program = program; + self.arguments = arguments; + self + } +} + #[test] fn the_reserved_names_are_the_documented_ones_and_leave_the_openvmm_prefix_to_openvmm() { let names = [ diff --git a/scripts/test-layout-allowlist.json b/scripts/test-layout-allowlist.json index 5c0555d1e..76a11780e 100644 --- a/scripts/test-layout-allowlist.json +++ b/scripts/test-layout-allowlist.json @@ -11,10 +11,8 @@ "nvx-rs/src/guest/sys/entry.rs": 3, "nvx-rs/src/guest/sys/spawn.rs": 7, "nvx-rs/src/guest/time/system.rs": 3, - "nvx-rs/src/host/cli/sandbox/state.rs": 1, "nvx-rs/src/host/network/address.rs": 2, "nvx-rs/src/host/platform/windows/spawn.rs": 1, - "nvx-rs/src/host/restore_driver.rs": 1, "nvx-rs/src/host/session.rs": 1, "nvx-rs/src/host/vm.rs": 2, "nvx-rs/src/protocol/app.rs": 6