diff --git a/.github/workflows/run-nvx-microvm-tests.yml b/.github/workflows/run-nvx-microvm-tests.yml index 29d0da83c..23ca8fa5f 100644 --- a/.github/workflows/run-nvx-microvm-tests.yml +++ b/.github/workflows/run-nvx-microvm-tests.yml @@ -452,25 +452,21 @@ jobs: nvx-rs/target/release/nvx-rs sandbox exec \ --state-dir "${state}" \ --entrypoint /sbin/nvx-sandbox-smoke \ + --arg init \ --arg /workspace \ --arg rw nvx-rs/target/release/nvx-rs sandbox stop --state-dir "${state}" - # A clean ext4 journal shows that stop unmounted the share, overlay, - # and scratch in order instead of powering off with them mounted. - python3 - "${state}/outcome.json" "${scratch}" <<'EOF' + # nvx-init owns the sandbox VM's lifecycle, so OpenVMM, which starts + # it without a lifecycle option, reports a run. + python3 - "${state}/outcome.json" <<'EOF' import json - import struct import sys with open(sys.argv[1], encoding="utf-8") as report: outcome = json.load(report)["outcome"] - expected = {"operation": "managed", "category": "success", "status_code": 0} + expected = {"operation": "run", "category": "success", "status_code": 0} if outcome != expected: sys.exit(f"managed sandbox outcome is invalid: {outcome}") - with open(sys.argv[2], "rb") as image: - image.seek(1024 + 0x60) - if struct.unpack(""${errors}"; then echo "the caller-owned managed sandbox reached its share as the runner" >&2 exit 1 fi + cat "${errors}" >&2 grep -q 'virtio-fs request cannot run as its caller; failing it with EPERM' \ "${state}/openvmm.log" - grep -q 'NVX-MANAGED-ERROR: stage=host-mappings' "${state}/openvmm.log" + # The guest refuses to map the share with EPERM, 1, which start + # reports. + grep -q 'could not mount a mapped host path (error 1)' "${errors}" + rm -f "${errors}" test ! -e "${state}/runtime.json" test "$(ls -A "${share}")" = nvx-host-marker nvx-rs/target/release/nvx-rs sandbox deprovision --state-dir "${state}" @@ -690,24 +691,17 @@ jobs: EOF nvx-rs/target/release/nvx-rs sandbox start --state-dir "${state}" nvx-rs/target/release/nvx-rs sandbox exec --state-dir "${state}" \ - "${smoke_arguments[@]}" + --arg init "${smoke_arguments[@]}" nvx-rs/target/release/nvx-rs sandbox stop --state-dir "${state}" - # A clean ext4 journal shows that stop unmounted every share, the - # overlay, and scratch in order. - python3 - "${state}/outcome.json" "${scratch}" <<'EOF' + python3 - "${state}/outcome.json" <<'EOF' import json - import struct import sys with open(sys.argv[1], encoding="utf-8") as report: outcome = json.load(report)["outcome"] - expected = {"operation": "managed", "category": "success", "status_code": 0} + expected = {"operation": "run", "category": "success", "status_code": 0} if outcome != expected: sys.exit(f"managed sandbox outcome is invalid: {outcome}") - with open(sys.argv[2], "rb") as image: - image.seek(1024 + 0x60) - if struct.unpack("&2 @@ -85,6 +94,13 @@ check_namespaces() { init_uids=$(status_field /proc/1/status Uid) [ "$init_uids" = "$workload_ids" ] || fail "PID 1 of the workload PID namespace runs as $init_uids" + check_hostname +} + +# The workload's host name is the one that the kernel command line names, or +# nvx-sandbox. nvx-init's command line names its bytes in lowercase +# hexadecimal. +check_hostname() { expected_hostname=nvx-sandbox # Read the whole file: a caller's --cmdline keeps its newlines, so tokens # may follow a line break. @@ -99,6 +115,9 @@ check_namespaces() { done set +f read -r workload_hostname Exit { io_error(ErrorCode::Io, error, path) } -/// Removes `path` unless it is missing. -pub(super) fn unlink_missing_ok(path: &Path) -> Result { - match std::fs::remove_file(path) { - Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(os_error(&error, path)), - _ => Ok(()), - } -} - /// Returns a builder of directories of `mode`, which Windows ignores. fn dir_builder(mode: u32) -> std::fs::DirBuilder { #[cfg_attr(not(unix), allow(unused_mut))] diff --git a/nvx-rs/src/host/cli/sandbox/lifecycle/mod.rs b/nvx-rs/src/host/cli/sandbox/lifecycle/mod.rs index 8f582f44e..ef7ebee65 100644 --- a/nvx-rs/src/host/cli/sandbox/lifecycle/mod.rs +++ b/nvx-rs/src/host/cli/sandbox/lifecycle/mod.rs @@ -1,58 +1,13 @@ -//! The lifecycle of a managed sandbox, a port of the former -//! `scripts/nvx_tools/sandbox_lifecycle.py`, which it replaced (microsoft/nvx#359), on the -//! [`store`](crate::host::store), [`supervisor`], and [`session`](crate::host::session) that -//! every host of a sandbox shares. +//! Persistent sandboxes on the CLI's store and caller-owned [`crate::host::vm`] interface. //! -//! A sandbox lives in its state directory, which is the directory of its store: +//! Provision keeps today's configuration, image bindings, network and export. Start journals the +//! prepared capability and outputs before spawn, records the verified process, and commits the +//! matching saved VM only after authenticated readiness and mappings. Every lifecycle command +//! takes the CLI store's recoverable lock; indeterminate launches retain their evidence. //! -//! ```text -//! / -//! config.json the launch that provision records, in format 8, with the export plan of -//! its shares or `null` without shares, its upper, the bindings of its -//! distro layer and of its scratch image, if any, and its network -//! sandbox.lock the store's lock, which serializes the lifecycle transitions -//! launch.json the store's journal of a start in progress, or of an interrupted one -//! runtime.json the store's runtime record of the running VM, in its format 4 -//! control.capability the 32-byte capability of the running OpenVMM -//! control.sock the control endpoint of the running OpenVMM, on POSIX -//! openvmm.log OpenVMM's output, which each start replaces -//! outcome.json OpenVMM's outcome report of the latest start -//! ``` -//! -//! The CLI keeps its own record, the configuration, in its own formats; the store keeps the -//! runtime records, and reads no other format than its own: a sandbox that an earlier build -//! started must be stopped with that build. -//! -//! [`provision`], [`fn@crate::host::cli::sandbox::lifecycle::start`], [`fn@crate::host::cli::sandbox::lifecycle::stop`], and [`deprovision`] hold the [`LifecycleLock`] for their -//! whole transition, and [`exec_workload`] only while it finds the running VM, so a long workload -//! cannot delay a stop. `config.json` also records the binding of the distro layer that -//! `provision` verified, and that of the scratch image of an upper on scratch, which [`fn@crate::host::cli::sandbox::lifecycle::start`] checks from metadata -//! alone, and the network as its typed policy, which [`fn@crate::host::cli::sandbox::lifecycle::start`] checks again, in place of the -//! network options that `sandbox_lifecycle.py` recorded: it refuses to start a sandbox that -//! `nvx.py` or an earlier `nvx-rs` provisioned, which has no bindings or no typed network. It -//! records shares as the export plan that `provision` made, whose host objects [`fn@crate::host::cli::sandbox::lifecycle::start`] checks -//! again. [`fn@crate::host::cli::sandbox::lifecycle::start`] exports the shares, and once the guest agent is ready sends it the plan's -//! mapping table, which it mounts in order before any workload runs; a mapping that fails ends the -//! start, after the guest shuts down as a stop has it do. -//! -//! Every transition but provision finds the sandbox's VM with [`supervisor::reattach`], as an -//! orchestrator does after it restarts: it ends a start that was interrupted, holds a recorded -//! OpenVMM that still runs through its verified identity, and clears the runtime record of one -//! that exited without a stop, which the first transition to find it reports, or which a start -//! replaces. A VM that another runtime identity than the artifacts' started, as after the -//! artifacts were replaced, is never driven, since its guest may speak other contracts: [`fn@crate::host::cli::sandbox::lifecycle::stop`] -//! ends it by identity-verified forced termination, without a control session, and the other -//! transitions refuse it. [`fn@crate::host::cli::sandbox::lifecycle::start`] launches OpenVMM with [`supervisor::launch`], under the -//! runtime identity that the install metadata of the artifacts names, and [`fn@crate::host::cli::sandbox::lifecycle::stop`] asks the guest -//! of a VM of that identity to shut down, waits for OpenVMM to exit, and reports a VM that does -//! not, without ending it otherwise. -//! -//! [`fn@crate::host::cli::sandbox::lifecycle::start`] hands OpenVMM, through a [`crate::host::sandbox::PreparedLaunch`] of the checked files, the objects that -//! the checks found, rather than paths that something could replace meanwhile, and claims the -//! scratch image, if any, for this sandbox while OpenVMM runs. It holds the checked files until -//! the guest is ready, when it releases them, and on Windows OpenVMM's copies of them, or until it -//! has ended an OpenVMM whose start failed. A sandbox whose upper is in RAM starts each time with -//! an empty upper, which its stop discards. +//! Exec and stop reattach with runtime and process verification. Stop requests NVXC shutdown, +//! waits for exit, then forces only the verified process. Deprovision removes only this stopped +//! sandbox's owned files. No running sandbox from the old runtime store is migrated. mod directory; mod outcome; @@ -63,18 +18,22 @@ use std::path::Path; use std::time::Duration; use crate::ErrorCode; +use crate::host::artifacts::{Guest, KERNEL_NAME}; +use crate::host::cli::artifacts::{Located, locate}; use crate::host::cli::deadline::RETRY_INTERVAL; use crate::host::cli::sandbox::lock::LifecycleLock; use crate::host::cli::sandbox::state::{ - RecordedDistro, RecordedScratch, serialize_launch, write_json, + Config, RecordedDistro, RecordedScratch, read_config, serialize_launch, write_json, }; use crate::host::cli::sandbox::{Artifacts, error_with_cause}; use crate::host::cli::session::Session; +use crate::host::cli::store::StateFile; use crate::host::cli::{Exit, Result}; +use crate::host::process::{Opened, Process}; +use crate::host::runtime::{IdentityPart, LaunchOptions, Runtime, RuntimeIdentity}; use crate::host::sandbox::{SandboxLaunch, require_file}; -use crate::host::store::{IdentityPart, LockedStore, RUNTIME_NAME}; -use crate::host::supervisor::{self, Reattached, Vmm}; -use crate::protocol::{CAPABILITY_LEN, GuestFeatures}; +use crate::host::vm::SavedVm; +use crate::protocol::GuestFeatures; pub(crate) use self::directory::{create_state_directory, prepare_state_directory}; pub(crate) use self::outcome::{ @@ -117,7 +76,7 @@ pub(crate) fn provision( ) -> Result { let state_dir = prepare_state_directory(state_path, true)?; let config_path = state_dir.join(CONFIG_NAME); - let runtime_path = state_dir.join(RUNTIME_NAME); + let runtime_path = state_dir.join(StateFile::Runtime.name()); let _lock = LifecycleLock::acquire(&state_dir, timeout, true)?; if config_path.exists() || runtime_path.exists() { return Err(Exit::error( @@ -133,55 +92,40 @@ pub(crate) fn provision( ) } -/// What a transition finds of the sandbox's VM, through [`supervisor::reattach`]. enum Found { /// No VM is recorded, or a start that was interrupted is over. Nothing, /// The recorded OpenVMM exited without a stop, and its runtime record is cleared. Exited, /// The VM runs under the runtime identity of the artifacts. - Live(Vmm), + Live(SavedVm, Process), /// The VM runs under another runtime identity, which differs in these parts. - Mismatched(Vmm, Vec), + Mismatched(SavedVm, Process, Vec), } -/// Finds the sandbox's VM in the locked store, as an orchestrator does after it restarts within -/// the same host boot: with [`supervisor::reattach`], under the runtime identity of `artifacts`. -/// -/// Only a recorded launch needs that identity, so a sandbox that never started, or that stopped, -/// needs no install metadata. Fails for a record that the store does not read, such as one of an -/// earlier format, and when whether the VM runs cannot be determined. -fn find_vm(locked: &mut LockedStore, artifacts: &Artifacts<'_>) -> Result { - if !records_launch(locked)? { +fn find_vm(lock: &mut LifecycleLock, artifacts: &Artifacts<'_>) -> Result { + let Some(saved) = lock.locked().runtime().map_err(error_with_cause)? else { return Ok(Found::Nothing); - } + }; + let process = match Process::open(&saved.process).map_err(|error| { + Exit::error( + ErrorCode::VmmIndeterminate, + format!("cannot determine whether the OpenVMM process of the sandbox runs: {error}"), + ) + })? { + Opened::Running(process) => process, + Opened::Exited(_) => { + lock.clear_runtime()?; + return Ok(Found::Exited); + } + }; let runtime = artifacts.runtime_identity()?; - Ok( - match supervisor::reattach(locked, &runtime).map_err(error_with_cause)? { - Reattached::Live(vmm) => Found::Live(vmm), - Reattached::Mismatched { vmm, differences } => Found::Mismatched(vmm, differences), - Reattached::Exited { - record: Some(_), .. - } => Found::Exited, - Reattached::Exited { record: None, .. } => Found::Nothing, - Reattached::Indeterminate(reason) => { - return Err(Exit::error( - ErrorCode::VmmIndeterminate, - format!( - "cannot determine whether the OpenVMM process of the sandbox runs: \ - {reason}" - ), - )); - } - }, - ) -} - -/// Returns whether the locked store records a launch of the sandbox, in its launch journal or its -/// runtime record, which alone needs the runtime identity of the artifacts. -fn records_launch(locked: &mut LockedStore) -> Result { - Ok(locked.journal().map_err(error_with_cause)?.is_some() - || locked.runtime().map_err(error_with_cause)?.is_some()) + let differences = saved.runtime.differences(&runtime); + Ok(if differences.is_empty() { + Found::Live(saved, process) + } else { + Found::Mismatched(saved, process, differences) + }) } /// Returns the parts of a runtime identity that `differences` names, as messages list them. @@ -193,7 +137,7 @@ fn differing_parts(differences: &[IdentityPart]) -> String { /// Returns the error of a transition other than stop on a VM that another runtime identity than /// the artifacts' started, which differs in `differences`: the CLI does not drive it, since it may /// speak other contracts, and only its stop ends it. -fn mismatched(vmm: &Vmm, differences: &[IdentityPart]) -> Exit { +fn mismatched(saved: &SavedVm, differences: &[IdentityPart]) -> Exit { Exit::error( ErrorCode::RuntimeMismatch, format!( @@ -201,7 +145,7 @@ fn mismatched(vmm: &Vmm, differences: &[IdentityPart]) -> Exit { {}, so it is not driven: stop it to terminate its OpenVMM process {} without a \ guest shutdown", differing_parts(differences), - vmm.record().process().pid + saved.process.pid ), ) } @@ -209,19 +153,19 @@ fn mismatched(vmm: &Vmm, differences: &[IdentityPart]) -> Exit { /// The running VM of a sandbox, and the capability with which its control session attaches. struct Running { /// The VM. - vmm: Vmm, - /// Its control capability. - capability: [u8; CAPABILITY_LEN], + saved: SavedVm, + /// The held, identity-verified process. + process: Process, } /// Returns the running VM of the sandbox in the locked store of `state_dir`, as exec requires it. fn running_vm( state_dir: &Path, - locked: &mut LockedStore, + lock: &mut LifecycleLock, artifacts: &Artifacts<'_>, ) -> Result { - let found = found_vm(locked, artifacts)?; - running(state_dir, locked, found) + let found = found_vm(lock, artifacts)?; + running(state_dir, found) } /// Finds the sandbox's VM as [`find_vm`] does, for an operation other than a start, which holds @@ -229,19 +173,19 @@ fn running_vm( /// recorded launch needs, it holds the lock of their downloads, as [`Artifacts::hold`] says, so /// that no download replaces them meanwhile. A sandbox that records no launch needs neither the /// artifacts nor their root. -fn found_vm(locked: &mut LockedStore, artifacts: &Artifacts<'_>) -> Result { - if !records_launch(locked)? { +fn found_vm(lock: &mut LifecycleLock, artifacts: &Artifacts<'_>) -> Result { + if lock.locked().runtime().map_err(error_with_cause)?.is_none() { return Ok(Found::Nothing); } let _hold = artifacts.hold()?; - find_vm(locked, artifacts) + find_vm(lock, artifacts) } /// Returns the running VM that `found` describes in the locked store of `state_dir`, as exec and /// stop require it, with its control capability. -fn running(state_dir: &Path, locked: &mut LockedStore, found: Found) -> Result { - let vmm = match found { - Found::Live(vmm) => vmm, +fn running(state_dir: &Path, found: Found) -> Result { + let (saved, process) = match found { + Found::Live(saved, process) => (saved, process), Found::Nothing => { // A directory without the configuration of a sandbox holds no provisioned one, which // a start would report too, rather than one that is stopped. @@ -257,20 +201,34 @@ fn running(state_dir: &Path, locked: &mut LockedStore, found: Found) -> Result return Err(mismatched(&vmm, &differences)), + Found::Mismatched(saved, _, differences) => return Err(mismatched(&saved, &differences)), }; - let capability = locked.read_capability().map_err(error_with_cause)?; - Ok(Running { vmm, capability }) + Ok(Running { saved, process }) } -/// Connects to the control endpoint of `vmm`, and attaches with `capability`. -fn connect(vmm: &Vmm, capability: &[u8; CAPABILITY_LEN], timeout: f64) -> Result { - Session::connect( - vmm.endpoint(), - vmm.record().process().pid, - capability, - timeout, - ) +fn runtime_for( + artifacts: &Artifacts<'_>, + config: &Config, + identity: RuntimeIdentity, +) -> Result { + let Located { dir, openvmm } = locate(artifacts.root)?; + Ok(Runtime { + openvmm, + kernel: dir.artifact(KERNEL_NAME), + initramfs: dir.initramfs(Guest::Alpine), + identity, + hypervisor: config.hypervisor().parse().map_err(error_with_cause)?, + cpu_profile: None, + windows_breakaway: false, + options: LaunchOptions::default(), + }) +} + +fn connect(saved: &SavedVm, artifacts: &Artifacts<'_>, timeout: f64) -> Result { + let _hold = artifacts.hold()?; + let config = Config::new(read_config(&saved.vm_dir.path().join(CONFIG_NAME))?)?; + let runtime = runtime_for(artifacts, &config, artifacts.runtime_identity()?)?; + Session::reattach(saved, &runtime, timeout) } /// Checks that `state_dir` holds the configuration of a provisioned sandbox: a directory without diff --git a/nvx-rs/src/host/cli/sandbox/lifecycle/start.rs b/nvx-rs/src/host/cli/sandbox/lifecycle/start.rs index dd3ae5627..e584cf0f6 100644 --- a/nvx-rs/src/host/cli/sandbox/lifecycle/start.rs +++ b/nvx-rs/src/host/cli/sandbox/lifecycle/start.rs @@ -1,42 +1,46 @@ //! Starting a managed sandbox and executing workloads in its running VM. -use std::ffi::OsString; use std::path::Path; -use std::time::{Duration, Instant}; +use std::time::Instant; use serde_json::Value; -use super::directory::{prepare_state_directory, unlink_missing_ok}; -use super::outcome::read_openvmm_outcome; +use super::directory::prepare_state_directory; use super::{ - CONFIG_NAME, Found, REQUIRED_FEATURES, connect, find_vm, mismatched, require_config, running_vm, + CONFIG_NAME, Found, REQUIRED_FEATURES, connect, find_vm, mismatched, require_config, + running_vm, runtime_for, }; -use crate::host::cli::deadline::{Deadline, RETRY_INTERVAL}; +use crate::activation::{Journal, MachineIdMode, OverlayMode, View}; +use crate::host::cli::deadline::Deadline; use crate::host::cli::sandbox::lock::LifecycleLock; -use crate::host::cli::sandbox::state::{Config, random_bytes, read_config}; +use crate::host::cli::sandbox::state::{Config, read_config}; use crate::host::cli::sandbox::{Artifacts, error_with_cause}; use crate::host::cli::session::{ExecRequest, ExecResult, Session}; -use crate::host::cli::{Exit, Result}; -use crate::host::sandbox::PreparedLaunch; +use crate::host::cli::{Exit, Result, output}; +use crate::host::config::{ + Boot, ConsoleAttachment, LayeredRoot, NetworkConfig, RootConfig, UpperConfig, VmConfig, +}; +use crate::host::process::KILL_TIMEOUT; +use crate::host::runtime::Runtime; use crate::host::sandbox::distro::DISTRO_POLICY_REVISION; use crate::host::sandbox::scratch::CLAIM_RELEASE_WAIT; -use crate::host::store::{LaunchId, LockedStore, OUTCOME_NAME, Recovery}; -use crate::host::supervisor::{self, Launched, VmmCommand}; -use crate::protocol::CAPABILITY_LEN; +use crate::host::sandbox::{SandboxLaunch, SandboxPlan}; +use crate::host::vm::{self, SavedVm, VmDir}; +use crate::net::Ipv4Cidr; use crate::{ErrorCode, ErrorKind}; /// Starts a provisioned sandbox, as `start` does. pub(crate) fn start(state_path: &str, artifacts: &Artifacts<'_>, timeout: f64) -> Result { let state_dir = prepare_state_directory(state_path, false)?; let mut lock = LifecycleLock::acquire(&state_dir, timeout, false)?; - start_locked(&state_dir, lock.locked(), artifacts, timeout) + start_locked(&state_dir, &mut lock, artifacts, timeout) } /// Starts the sandbox of a locked state directory, as `_start` does, through the launch journal /// of its store. fn start_locked( state_dir: &Path, - locked: &mut LockedStore, + lock: &mut LifecycleLock, artifacts: &Artifacts<'_>, timeout: f64, ) -> Result { @@ -52,27 +56,18 @@ fn start_locked( Some(hypervisor) => Some(artifacts.ensure(hypervisor)?), None => None, }; - match find_vm(locked, artifacts)? { + match find_vm(lock, artifacts)? { Found::Nothing | Found::Exited => {} - Found::Live(_) => { + Found::Live(..) => { return Err(Exit::error( ErrorCode::SandboxRunning, "sandbox is already running or has stale runtime state", )); } - Found::Mismatched(vmm, differences) => return Err(mismatched(&vmm, &differences)), + Found::Mismatched(saved, _, differences) => return Err(mismatched(&saved, &differences)), } - let outcome_path = state_dir.join(OUTCOME_NAME); - unlink_missing_ok(&outcome_path)?; let config = Config::new(config)?; let launch = config.launch()?; - // The checks read no content of the distro layer, its manifest, or the scratch image, and - // never verify them again. The prepared launch holds the files that they return, and hands - // OpenVMM those objects: on Linux, OpenVMM inherits their descriptors; on Windows, it opens - // them by their final paths, which the files, and OpenVMM's copies of them, keep anything from - // replacing before OpenVMM, whose opens share them, has opened them, even if the start ends - // first. The scratch image's check also claims it for this sandbox, until OpenVMM exits. An - // upper in RAM has no scratch image, and its configuration no binding of one. let binding = config.distro_binding()?; let scratch_binding = config.scratch_binding()?; let distro = binding @@ -88,133 +83,153 @@ fn start_locked( ), _ => None, }; - // The plan takes the format and identity of the distro layer from its binding. let plan = launch.bound_plan(&binding)?; - let [executable, kernel, initrd] = artifacts.require()?; - let runtime = artifacts.launch_identity()?; - let capability = random_bytes::()?; - if capability == [0; CAPABILITY_LEN] { - return Err(Exit::error( - ErrorCode::Other, - "the system returned an all-zero control capability", - )); + // Keep provision's refusal of caller-owned nvx_* tokens, without launching its old plan. + plan.kernel_command_line(config.cmdline())?; + artifacts.require()?; + let runtime = runtime_for(artifacts, &config, artifacts.launch_identity()?)?; + let config = vm_config(&config, &launch, &plan)?; + let dir = VmDir::open(state_dir).map_err(error_with_cause)?; + lock.clear_outputs()?; + let mut prepared = vm::create(&runtime, &config, None, &dir).map_err(error_with_cause)?; + if let Some(scratch) = &scratch { + #[cfg(not(windows))] + prepared.inherit(scratch.file().try_clone().map_err(Exit::from_os_error)?); + #[cfg(windows)] + prepared.inherit(scratch.claim().try_clone().map_err(Exit::from_os_error)?); } - let memory_mib = config.memory_mib(); - let prepared = PreparedLaunch::new(plan, distro, scratch, config.cmdline())?; - // The control endpoint, which the launch chooses, goes between these arguments. - let mut head = vec![executable]; - head.extend(prepared.openvmm_arguments().iter().cloned()); - head.extend( - [ - "--microvm-lifecycle", - "managed", - "--single-process", - "--hypervisor", - ] - .map(str::to_owned), - ); - head.push(config.hypervisor().to_owned()); - head.push("--memory".to_owned()); - head.push(format!("{memory_mib}M")); - head.extend(["--kernel".to_owned(), kernel, "--initrd".to_owned(), initrd]); - head.push("--cmdline".to_owned()); - head.push(prepared.kernel_command_line().to_owned()); - head.extend(["--virtio-console", "none", "--microvm-control-console"].map(str::to_owned)); - let tail = vec![ - "--microvm-control-auth-stdin".to_owned(), - "--microvm-report".to_owned(), - outcome_path.display().to_string(), - ]; - let directory = std::env::current_dir().map_err(Exit::from_os_error)?; - let mut command = Vec::new(); - let launch = LaunchId::generate().map_err(error_with_cause)?; - let mut starting = supervisor::launch(locked, &launch, &runtime, &capability, |endpoint| { - command = head; - command.push(format!("listen={endpoint}")); - command.extend(tail); - let arguments = command[1..].iter().map(OsString::from).collect(); - let mut vmm = VmmCommand::new(&command[0], arguments, directory); - // OpenVMM inherits the checked files that the prepared launch passes, and nothing else. - vmm.inherit = prepared.inherited(); - Ok(vmm) - }) - .map_err(error_with_cause)?; - let endpoint = starting.endpoint().to_owned(); - let maps = prepared.plan().mapping_table(); - // Whether the guest agent acknowledged a request to shut down after a mapping failed. + let mut starting = lock + .locked() + .begin_launch(&prepared, &runtime.identity) + .map_err(error_with_cause)?; + let launched = vm::spawn(prepared).map_err(error_with_cause)?; + let saved = SavedVm { + vm_dir: dir, + endpoint: launched.outputs.endpoint.clone(), + capability: *launched.capability.as_bytes(), + process: launched.process.reference().clone(), + runtime: runtime.identity.clone(), + }; let mut stopping = false; - let ready = - check_ready(&endpoint, starting.child(), &capability, timeout).and_then(|mut session| { - // Workloads run only once the guest agent has mapped every share in order, so a - // mapping that fails ends the start. A sandbox without shares maps nothing. + let ready = starting + .record_process(&saved.process) + .map_err(error_with_cause) + .and_then(|()| check_ready(&saved, &runtime, timeout)) + .and_then(|mut session| { + let maps = config.mapping_table(); if maps.is_empty() { return Ok(()); } session.map_host_paths(&maps, timeout).inspect_err(|error| { - // The agent then refuses workloads, so the guest shuts down as a stop has it do: - // it unmounts what it mapped, the export, the overlay, and the upper before the - // VM powers off. A failed transfer closed the session, so a guest that stopped - // answering is terminated at once. if !matches!(error, Exit::Interrupted) { - stopping = session.stop(timeout).is_ok(); + match session.stop(timeout) { + Ok(()) => stopping = true, + Err(error) => output::print_stderr(&format!( + "warning: startup guest shutdown failed: {error:?}" + )), + } } }) }); - // The start holds the checked files until the guest is ready, or until it has ended an - // OpenVMM whose start failed. - let mut prepared = Some(prepared); - let ready = ready.and_then(|()| { - // The guest is ready, so OpenVMM holds the images: the checked files can go, with - // OpenVMM's copies of them on Windows, which releases their guards, while OpenVMM keeps - // the scratch image's claim. - prepared - .take() - .expect("the start holds the checked files until the guest is ready") - .release(starting.process()) - .map_err(error_with_cause) - }); - let Err(error) = ready else { - return starting.commit().map(drop).map_err(error_with_cause); + let Err(mut error) = ready else { + return starting.commit(&saved).map_err(error_with_cause); }; - // OpenVMM closes the control endpoint as it tears the VM down, before it publishes its - // outcome report and exits, so only a closed endpoint, or a guest that shuts down, warrants - // waiting for it. The session reports an OpenVMM that exited before it served the endpoint - // as a closed endpoint too, whose wait then ends at once. + drop(starting); let grace = if stopping || matches!(error, Exit::EndpointClosed) { - timeout + Deadline::after(timeout) } else { - 0.0 - }; - let status = end_failed_start(starting.child(), grace, &command)?; - // OpenVMM has exited, so the launch ends: its capability, control socket, report staging - // file, and journal go, and its log and outcome report stay. - let unended = match starting.abandon() { - Ok(Recovery::Indeterminate(reason)) => Some(reason), - Ok(_) => None, - Err(failure) => Some(failure.message().to_owned()), - }; - // The checked files can go too, which releases their guards and the scratch image's claim. - drop(prepared); - let error = match status { - // A guest that shut down after a mapping failed exits as a stopped one does, so the - // mapping's failure tells why the start failed. - Some(status) if !stopping && !matches!(error, Exit::Interrupted) => Exit::error( - ErrorCode::Exited, - startup_exit_message(&outcome_path, status)?, - ), - _ => error, + Deadline::after(0.0) }; - if let Some(reason) = unended - && let Some(message) = error.message() - { + if let Err(wait) = super::stop::wait_for_exit(&launched.process, grace) { + if !matches!(wait, Exit::Interrupted) { + return Err(wait); + } + error = wait; + } + let now = Instant::now(); + let stopped = vm::stop( + &launched.process, + &launched.outputs, + |_| false, + now, + now + KILL_TIMEOUT, + ) + .map_err(error_with_cause)?; + lock.locked() + .recover(Instant::now() + KILL_TIMEOUT) + .map_err(error_with_cause)?; + lock.clear_runtime()?; + drop((distro, scratch)); + let report = stopped.report.map_err(error_with_cause)?; + if stopped.self_exited && !stopping && !matches!(error, Exit::Interrupted) { + let detail = match report { + Some(report) => match (report.category, report.status_code) { + (Some(category), Some(status)) => format!(": {category} status {status}"), + _ => String::new(), + }, + None => " without an outcome report".to_owned(), + }; return Err(Exit::error( - error.code().unwrap_or(ErrorCode::Other), - format!("{message}; the launch could not be ended, and remains for recovery: {reason}"), + ErrorCode::Exited, + format!("OpenVMM exited during startup{detail}"), )); } Err(error) } +fn vm_config(config: &Config, launch: &SandboxLaunch, plan: &SandboxPlan) -> Result { + let (uid, gid) = launch.settings().workload_identity; + let mut command_line = format!("nvx_sandbox=1 nvx_mode=managed nvx_identity={uid}:{gid}"); + if !config.cmdline().is_empty() { + command_line.push(' '); + command_line.push_str(config.cmdline()); + } + let network = launch + .network() + .map(|network| -> Result { + Ok(NetworkConfig { + guest_network: Some( + Ipv4Cidr::parse(network.guest_network()) + .map_err(|error| Exit::error(ErrorCode::Unsupported, error.to_string()))?, + ), + policy: network.policy().clone(), + }) + }) + .transpose()?; + Ok(VmConfig { + processors: 1, + memory_mib: config.memory_mib(), + root: RootConfig::Layered(LayeredRoot { + image: launch.distro().path().to_path_buf(), + format: plan.distro().image(), + gpt_first_partition_bound: None, + upper: match launch.scratch() { + Some(path) => UpperConfig::Scratch { + path: path.to_path_buf(), + journal: Journal::Allowed, + }, + None => UpperConfig::Ram { + max_memory_percent: UpperConfig::DEFAULT_MAX_MEMORY_PERCENT, + }, + }, + overlay: OverlayMode::TrustedLowers, + }), + view: View { + hostname: launch.settings().hostname.as_bytes().to_vec(), + // As the sandbox agent did, nvx-init gives the workload a machine ID of the VM's own. + machine_id: MachineIdMode::Generated, + }, + network, + filesystem: launch.export().cloned(), + console: ConsoleAttachment::Disconnected, + boot: Some(Boot { + command_line, + memory_capacity_mib: None, + capture: None, + }), + }) +} + /// Returns the error of a start whose distro layer or scratch image fails the check of its /// binding: a binding that no longer holds needs a new provision, which verifies the layer and /// binds the image again, while a scratch image that another sandbox claims, and a failure of the @@ -236,13 +251,8 @@ fn binding_error(error: crate::Error) -> Exit { /// process of the start, serves, or until OpenVMM exits, checks that the guest provides the /// features of [`REQUIRED_FEATURES`], and returns the session, on which the caller sends the /// guest what it needs before the launch commits. -fn check_ready( - endpoint: &str, - launched: &mut Launched, - capability: &[u8; CAPABILITY_LEN], - timeout: f64, -) -> Result { - let mut session = Session::connect_launched(endpoint, launched, capability, timeout)?; +fn check_ready(saved: &SavedVm, runtime: &Runtime, timeout: f64) -> Result { + let mut session = Session::reattach(saved, runtime, timeout)?; session.ping(timeout)?; let features = session.features(timeout)?; let missing = features.missing(REQUIRED_FEATURES); @@ -259,81 +269,6 @@ fn check_ready( Ok(session) } -/// Ends the OpenVMM process of a failed start, as `_end_failed_start` does: returns OpenVMM's -/// status if it exits on its own within `grace` seconds, or terminates it and returns `None`. -/// Terminating OpenVMM while it publishes its outcome report would lose the report, and leave -/// only its staging file, which the store removes as the launch ends. -fn end_failed_start( - launched: &mut Launched, - grace: f64, - command: &[String], -) -> Result> { - let waited = if grace > 0.0 { - wait_interruptibly(launched, Deadline::after(grace)) - } else { - Ok(()) - }; - let status = launched - .wait_timeout(Duration::ZERO) - .map_err(Exit::from_os_error)?; - if status.is_none() { - launched.terminate().map_err(Exit::from_os_error)?; - let exited = launched - .wait_timeout(Duration::from_secs(5)) - .map_err(Exit::from_os_error)?; - if exited.is_none() { - launched.kill().map_err(Exit::from_os_error)?; - let killed = launched - .wait_timeout(Duration::from_secs(5)) - .map_err(Exit::from_os_error)?; - if killed.is_none() { - return Err(Exit::error( - ErrorCode::TimedOut, - format!("command {command:?} did not exit within 5 seconds of its kill"), - )); - } - } - } - // Ctrl-C during the grace period ends the start as interrupted, after OpenVMM. - waited?; - Ok(status) -} - -/// Waits until `launched` exits or `deadline` passes, failing if Ctrl-C arrives. -fn wait_interruptibly(launched: &mut Launched, deadline: Deadline) -> Result { - loop { - let slice = deadline - .remaining() - .map_or(RETRY_INTERVAL, |left| left.min(RETRY_INTERVAL)); - crate::host::cli::check_interrupt()?; - if launched - .wait_timeout(slice) - .map_err(Exit::from_os_error)? - .is_some() - || deadline.passed() - { - return Ok(()); - } - } -} - -/// Describes how OpenVMM ended a start, preferring its outcome report, which keeps the guest's -/// own status: its category and its integer status, when it holds both. -fn startup_exit_message(outcome_path: &Path, status: i64) -> Result { - let report = read_openvmm_outcome(outcome_path).ok(); - if let Some(outcome) = report.as_ref().and_then(|report| report.get("outcome")) - && let Some(Value::String(category)) = outcome.get("category") - && let Some(status_code) = outcome.get("status_code").and_then(Value::as_i64) - { - return Ok(format!( - "OpenVMM exited during startup: {category} status {status_code}" - )); - } - Ok(format!( - "OpenVMM exited during startup with status {status}" - )) -} - /// Runs a workload in a running sandbox, as `exec_workload` does. The lock is held only while the /// running VM is found, so a long workload cannot delay a stop. pub(crate) fn exec_workload( @@ -345,8 +280,8 @@ pub(crate) fn exec_workload( let state_dir = prepare_state_directory(state_path, false)?; let running = { let mut lock = LifecycleLock::acquire(&state_dir, response_timeout, false)?; - running_vm(&state_dir, lock.locked(), artifacts)? + running_vm(&state_dir, &mut lock, artifacts)? }; - let mut session = connect(&running.vmm, &running.capability, response_timeout)?; + let mut session = connect(&running.saved, artifacts, response_timeout)?; session.exec(request, response_timeout) } diff --git a/nvx-rs/src/host/cli/sandbox/lifecycle/stop.rs b/nvx-rs/src/host/cli/sandbox/lifecycle/stop.rs index f4ae206eb..4e5010612 100644 --- a/nvx-rs/src/host/cli/sandbox/lifecycle/stop.rs +++ b/nvx-rs/src/host/cli/sandbox/lifecycle/stop.rs @@ -9,100 +9,132 @@ use crate::ErrorCode; use crate::host::cli::deadline::{Deadline, RETRY_INTERVAL}; use crate::host::cli::sandbox::lock::LifecycleLock; use crate::host::cli::sandbox::{Artifacts, error_with_cause}; +use crate::host::cli::store::StateFile; use crate::host::cli::{Exit, Result, check_interrupt, output}; -use crate::host::store::{IdentityPart, LockedStore, OUTCOME_NAME}; -use crate::host::supervisor::{StopOutcome, Vmm}; +use crate::host::process::Process; +use crate::host::runtime::IdentityPart; +use crate::host::supervisor::StopOutcome; +use crate::host::vm::{self, SavedVm, VmOutputs}; /// Stops a running sandbox, and checks the outcome report that its OpenVMM wrote, unless another /// runtime identity than the artifacts' started the VM. /// -/// The supervisor's stop runs one graceful step, which asks the guest to shut the VM down and -/// waits up to `timeout` seconds for OpenVMM to exit, so that Ctrl-C ends the wait. A VM whose -/// OpenVMM is still running then is reported, and keeps its runtime record: the stop never takes -/// the forced step. +/// A STOP acknowledgement is not exit proof. After its grace period the stop forces only the +/// verified process; an indeterminate exit retains the runtime record. /// /// A VM of another runtime identity may speak other contracts, so the stop asks it nothing: it /// ends the VM as [`terminate_mismatched`] does. pub(crate) fn stop(state_path: &str, artifacts: &Artifacts<'_>, timeout: f64) -> Result { let state_dir = prepare_state_directory(state_path, false)?; let mut lock = LifecycleLock::acquire(&state_dir, timeout, false)?; - let locked = lock.locked(); - let Running { vmm, capability } = match found_vm(locked, artifacts)? { - Found::Mismatched(vmm, differences) => { - return terminate_mismatched(locked, &vmm, &differences, timeout); + let Running { saved, process } = match found_vm(&mut lock, artifacts)? { + Found::Mismatched(saved, process, differences) => { + return terminate_mismatched(&mut lock, &saved, &process, &differences, timeout); } - found => running(&state_dir, locked, found)?, + found => running(&state_dir, found)?, }; - // The graceful step waits for the exit itself, so the stop's own wait, until a deadline that - // has passed by then, only checks whether OpenVMM has exited. - let stopped = vmm - .stop(locked, Instant::now(), None, |vmm, _| { - { - let mut session = connect(vmm, &capability, timeout)?; - session.stop(timeout)?; - } - wait_for_exit(vmm, Deadline::after(timeout)).map(drop) - }) - .map_err(error_with_cause)?; - if let Some(error) = stopped.graceful_error { - return Err(error); + let grace = Deadline::after(timeout); + let requested = connect(&saved, artifacts, timeout).and_then(|mut session| { + session.stop(grace.remaining().map_or(0.0, |left| left.as_secs_f64())) + }); + if matches!(requested, Err(Exit::Interrupted)) { + return requested; } - if stopped.outcome == StopOutcome::TimedOut { - return Err(Exit::error( - ErrorCode::TimedOut, - "OpenVMM did not terminate after managed stop", + wait_for_exit(&process, grace)?; + let (outcome, stopped) = stop_verified(&saved, &process, timeout)?; + lock.clear_runtime()?; + if let Err(error) = requested { + output::print_stderr(&format!( + "warning: managed guest shutdown failed: {error:?}" )); } - read_openvmm_outcome(&state_dir.join(OUTCOME_NAME)).map(drop) + if outcome == StopOutcome::Forced { + output::print_stderr("warning: OpenVMM was forcibly terminated after managed stop"); + } + let report = stopped.report.map_err(error_with_cause)?; + if report.is_none() && outcome == StopOutcome::Forced { + output::print_stderr("warning: the forced stop produced no outcome report"); + return Ok(()); + } + read_openvmm_outcome(&state_dir.join(StateFile::Outcome.name())).map(drop) } -/// Ends the VM `vmm`, which another runtime identity than the artifacts' started, and which -/// differs from them in `differences`, by identity-verified forced termination, and warns on -/// standard error that it did. -/// -/// Its guest is not asked to shut down, and no control session is opened to it: its OpenVMM is -/// terminated through its verified process identity, as [`Vmm::terminate`] does, and the stop -/// waits up to `timeout` seconds for it to exit, as [`terminate_within`] does, so that Ctrl-C ends -/// the wait. One that does not exit then, whose exit cannot be determined, or whose wait Ctrl-C -/// ends, keeps its runtime record, so that a later stop retries. -fn terminate_mismatched( - locked: &mut LockedStore, - vmm: &Vmm, - differences: &[IdentityPart], +fn stop_verified( + saved: &SavedVm, + process: &Process, timeout: f64, -) -> Result { - let pid = vmm.record().process().pid; - let parts = differing_parts(differences); +) -> Result<(StopOutcome, vm::Stopped)> { + let outputs = VmOutputs { + log: saved.vm_dir.path().join(StateFile::Log.name()), + outcome: saved.vm_dir.path().join(StateFile::Outcome.name()), + endpoint: saved.endpoint.clone(), + console: None, + }; + let (mut stopped, mut failure) = (None, None); let outcome = terminate_within( Deadline::after(timeout), - |bound| vmm.terminate(locked, bound).map_err(error_with_cause), - |deadline| wait_for_exit(vmm, deadline), + |bound| { + let now = Instant::now(); + match vm::stop(process, &outputs, |_| false, now, now + bound) { + Ok(result) => { + let outcome = if result.forced { + StopOutcome::Forced + } else { + StopOutcome::Graceful + }; + stopped = Some(result); + Ok(outcome) + } + Err(error) if error.kind() == crate::ErrorKind::StillRunning => { + failure = Some(error); + Ok(StopOutcome::TimedOut) + } + Err(error) => Err(error_with_cause(error)), + } + }, + |deadline| wait_for_exit(process, deadline), )?; + if outcome == StopOutcome::TimedOut { + return Err(error_with_cause( + failure.expect("an unconfirmed stop retained its failure"), + )); + } + Ok(( + outcome, + stopped.expect("a confirmed stop returned its report result"), + )) +} + +fn terminate_mismatched( + lock: &mut LifecycleLock, + saved: &SavedVm, + process: &Process, + differences: &[IdentityPart], + timeout: f64, +) -> Result { + let (outcome, stopped) = stop_verified(saved, process, timeout)?; + lock.clear_runtime()?; + if let Err(error) = stopped.report { + output::print_stderr(&format!( + "warning: stopped VM outcome could not be read: {error}" + )); + } let ended = match outcome { StopOutcome::Forced => "was terminated without a guest shutdown", StopOutcome::Graceful => "had exited", - StopOutcome::TimedOut => { - return Err(Exit::error( - ErrorCode::TimedOut, - format!( - "OpenVMM process {pid} of the sandbox, which runs under another runtime \ - identity than these artifacts, did not exit after it was terminated" - ), - )); - } + StopOutcome::TimedOut => unreachable!("an unconfirmed stop returns an error"), }; output::print_stderr(&format!( "warning: sandbox ran under another runtime identity than these artifacts, which differ \ - in {parts}: its OpenVMM process {pid} {ended}" + in {}: its OpenVMM process {} {ended}", + differing_parts(differences), + saved.process.pid )); Ok(()) } /// Ends an OpenVMM by identity-verified forced termination, and waits until `deadline` for it to -/// exit: `terminate` terminates it unless it has exited, waits up to the bound that it takes for -/// it to exit, and clears its runtime record once it has, as [`Vmm::terminate`] does, and `wait` -/// waits for its exit until a deadline, fails if Ctrl-C arrives first, and returns whether it -/// exited, as [`wait_for_exit`] does. +/// exit: `terminate` calls [`vm::stop`] in a short slice, and `wait` polls for exit with Ctrl-C. /// /// The termination waits for one retry interval at most, and `wait` for the rest, so that Ctrl-C /// ends the wait for an OpenVMM that does not exit at once, as one in uninterruptible I/O. Once @@ -131,15 +163,16 @@ pub(super) fn terminate_within( /// Waits until the OpenVMM of `vmm` exits or `deadline` passes, and returns whether it exited, /// failing if Ctrl-C arrives first. -fn wait_for_exit(vmm: &Vmm, deadline: Deadline) -> Result { +pub(super) fn wait_for_exit(process: &Process, deadline: Deadline) -> Result { loop { check_interrupt()?; let slice = deadline .remaining() .map_or(RETRY_INTERVAL, |left| left.min(RETRY_INTERVAL)); - if vmm + if process .wait(Instant::now() + slice) .map_err(Exit::from_os_error)? + .is_some() { return Ok(true); } @@ -153,9 +186,9 @@ fn wait_for_exit(vmm: &Vmm, deadline: Deadline) -> Result { pub(crate) fn deprovision(state_path: &str, artifacts: &Artifacts<'_>, timeout: f64) -> Result { let state_dir = prepare_state_directory(state_path, false)?; let mut lock = LifecycleLock::acquire(&state_dir, timeout, false)?; - match found_vm(lock.locked(), artifacts)? { + match found_vm(&mut lock, artifacts)? { Found::Nothing | Found::Exited => {} - Found::Live(_) | Found::Mismatched(..) => { + Found::Live(..) | Found::Mismatched(..) => { return Err(Exit::error( ErrorCode::SandboxRunning, "sandbox must be stopped before deprovision", diff --git a/nvx-rs/src/host/cli/sandbox/lock.rs b/nvx-rs/src/host/cli/sandbox/lock.rs index 80958c0d9..8ba6e4d85 100644 --- a/nvx-rs/src/host/cli/sandbox/lock.rs +++ b/nvx-rs/src/host/cli/sandbox/lock.rs @@ -1,37 +1,15 @@ -//! The lifecycle lock of a state directory: the store's -//! [lifecycle lock](crate::host::store#lifecycle-locks) of the sandbox, whose caller keeps its -//! configuration record, `config.json`, in the directory. -//! -//! - Provision, start, stop, and deprovision hold the lock for their whole transition, and exec -//! while it finds the running VM. A caller that cannot take the lock retries every 25 ms -//! within its timeout, and Ctrl-C ends the wait. -//! - The store takes the lock of the file that the path names when the lock is granted. A -//! provision whose directory deprovision removed meanwhile creates it again; any other -//! transition then fails, as for a path that names no directory. -//! - Deprovision [removes](crate::host::store#removal) the sandbox through the store, which -//! removes the lock file before the directory. A transition that waited for it then finds the -//! directory gone, or takes the lock of a new lock file in the directory and finds no sandbox -//! there. Every transition that leaves the directory without a provisioned sandbox removes its -//! lock file as it releases the lock, so that the directory empties. - -use std::io; -use std::path::{Path, PathBuf}; -use std::time::Instant; +//! The CLI store's recoverable lifecycle lock and removal of its owned files. use super::error_with_cause; use super::lifecycle::{CONFIG_NAME, LOCK_RETRY_INTERVAL, create_state_directory}; +use crate::ErrorCode; use crate::host::cli::deadline::Deadline; +use crate::host::cli::store::{LockedState, StateDir, StateFile}; use crate::host::cli::{Exit, Result, io_error}; -use crate::host::paths; -use crate::host::store::{CallerFiles, LockedStore, RemovalError, SandboxStore}; -use crate::{ErrorCode, ErrorKind}; - -/// Returns the kind of the I/O error that caused the store's `error`, if one did. -fn cause_kind(error: &crate::Error) -> Option { - std::error::Error::source(error)? - .downcast_ref::() - .map(io::Error::kind) -} +use crate::host::platform; +use crate::host::vm::VmDir; +use std::io; +use std::path::{Path, PathBuf}; /// Returns the error of a transition that still waits for another: Python's `TimeoutError`. pub(crate) fn transition_in_progress() -> Exit { @@ -41,12 +19,6 @@ pub(crate) fn transition_in_progress() -> Exit { ) } -/// Returns the files that the CLI keeps in a state directory beside the store's: its -/// configuration record, whose presence means that the sandbox is provisioned. -fn caller_files() -> CallerFiles { - CallerFiles::new(CONFIG_NAME).expect("the configuration record has a plain file name") -} - /// The exclusive lock that serializes the lifecycle transitions of a state directory: the /// lifecycle lock of its sandbox's store, through which the transition changes the store's /// records. Dropping it releases the lock. @@ -54,7 +26,7 @@ pub(crate) struct LifecycleLock { /// The state directory. state_dir: PathBuf, /// The sandbox's locked store. - locked: LockedStore, + locked: LockedState, } impl LifecycleLock { @@ -71,30 +43,35 @@ impl LifecycleLock { )); } let deadline = Deadline::after(timeout); - let files = caller_files(); loop { - // The store tries the lock once for a deadline that has passed, so that this wait - // checks for Ctrl-C between tries. - let now = Some(Instant::now()); let taken = if provision { - SandboxStore::lock_for_provision(state_dir, &files, now) + StateDir::create(state_dir) } else { - SandboxStore::lock_existing(state_dir, &files, now) - }; + StateDir::open(state_dir) + } + .and_then(|state| state.lock(deadline)); let error = match taken { Ok(locked) => { - return Ok(Self { + let lock = Self { state_dir: state_dir.to_path_buf(), locked, - }); + }; + if provision { + VmDir::open(state_dir).map_err(|error| { + error_with_cause( + crate::Error::io( + "sandbox state directory must be a private VM directory", + ) + .with_source(error), + ) + })?; + } + return Ok(lock); } Err(error) => error, }; - match (error.kind(), cause_kind(&error)) { - (ErrorKind::TimedOut, _) => {} - // Deprovision removed the state directory meanwhile, which only a provision - // creates again. - (_, Some(io::ErrorKind::NotFound)) if !provision => { + match error.code() { + Some(ErrorCode::NotFound) if !provision && !state_dir.is_dir() => { return Err(Exit::error( ErrorCode::SandboxNotProvisioned, format!( @@ -103,18 +80,19 @@ impl LifecycleLock { ), )); } - // The store creates the state directory again itself, once its parent exists. - (_, Some(io::ErrorKind::NotFound)) => { + Some(ErrorCode::NotFound) => { if deadline.passed() { return Err(transition_in_progress()); } - create_state_directory(state_dir)?; + if provision { + create_state_directory(state_dir)?; + } continue; } - // Any other failure ends the transition at once. The store waits itself for a - // removed lock file or state directory that Windows keeps until its last handle - // closes, so a denied access is one for good, such as a read-only lock file's. - _ => return Err(error_with_cause(error)), + _ if cfg!(windows) + && (platform::is_deleted(state_dir) + || platform::is_deleted(&state_dir.join(StateFile::Lock.name()))) => {} + _ => return Err(error), } if !deadline.pause(LOCK_RETRY_INTERVAL)? { return Err(transition_in_progress()); @@ -123,73 +101,154 @@ impl LifecycleLock { } /// Returns the sandbox's locked store. - pub(crate) fn locked(&mut self) -> &mut LockedStore { + pub(crate) fn locked(&mut self) -> &mut LockedState { &mut self.locked } + /// Called only after exit proof, retaining the previous launch's log and outcome. + pub(crate) fn clear_runtime(&self) -> Result { + self.remove_files(&[ + StateFile::Capability, + StateFile::Socket, + StateFile::Journal, + StateFile::Runtime, + ]) + } + + pub(crate) fn clear_outputs(&self) -> Result { + self.remove_files(&[StateFile::Socket, StateFile::Log, StateFile::Outcome]) + } + + fn remove_files(&self, names: &[StateFile]) -> Result { + let directory = self.locked.directory()?; + for name in names { + directory.remove_file(name.name()).map_err(|error| { + io_error(ErrorCode::Io, &error, &self.state_dir.join(name.name())) + })?; + } + for entry in std::fs::read_dir(&self.state_dir).map_err(Exit::from_os_error)? { + let name = entry.map_err(Exit::from_os_error)?.file_name(); + let name = name.to_str().ok_or_else(|| { + Exit::error(ErrorCode::Io, "a sandbox file name is not valid Unicode") + })?; + if StateFile::owns_temporary(name) { + directory.remove_file(name).map_err(Exit::from_os_error)?; + } + } + directory.sync().map_err(Exit::from_os_error) + } + /// Removes the sandbox and its state directory through the store, as deprovision does once no /// OpenVMM of the sandbox can run, waiting up to `timeout` seconds for open files, such as the /// lock files of other transitions, that keep the emptied directory, and releases the lock. pub(crate) fn remove(self, timeout: f64) -> Result { - let Self { state_dir, locked } = self; - let dir = locked.dir().to_owned(); - let failed = |error| removal_error(error, &state_dir, &dir); - let Some(mut lingering) = locked.remove().map_err(failed)? else { - return Ok(()); - }; - // A removed file that a handle still keeps, such as a lock file, stays in the directory, - // as do the lock files of transitions that wait meanwhile. Each such transition finds no - // sandbox, and removes its lock file as it releases the lock, so the directory empties - // after the removal released its own, once the other handles close. The removal is tried - // once at a time, so that this wait checks for Ctrl-C between tries. + let directory = self.locked.directory()?; + for name in [StateFile::Journal, StateFile::Runtime] { + if directory + .entry_id(name.name()) + .map_err(Exit::from_os_error)? + .is_some() + { + return Err(Exit::error( + ErrorCode::SandboxRunning, + "sandbox must be stopped before deprovision", + )); + } + } + let state_dir = self.state_dir.clone(); + let identity = platform::file_id(&state_dir).map_err(Exit::from_os_error)?; + let names: Vec<_> = StateFile::ALL + .into_iter() + .filter(|name| !matches!(name, StateFile::Lock | StateFile::Config)) + .chain([StateFile::Config]) + .collect(); + self.remove_files(&names)?; + let foreign = remaining(&state_dir)?; + if !foreign.is_empty() { + return Err(foreign_files(foreign)); + } + directory + .remove_file(StateFile::Lock.name()) + .map_err(Exit::from_os_error)?; + directory.sync().map_err(Exit::from_os_error)?; + drop(self); let deadline = Deadline::after(timeout); loop { - match lingering.wait(Some(Instant::now())) { + match platform::file_id(&state_dir) { + Ok(current) if current == identity => {} + Ok(_) => return Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(Exit::from_os_error(error)), + } + match std::fs::remove_dir(&state_dir) { Ok(()) => return Ok(()), - Err(RemovalError::TimedOut(error)) if !deadline.passed() => { + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(error) => { + if state_dir.join(CONFIG_NAME).exists() { + return Err(Exit::error( + ErrorCode::SandboxProvisioned, + "sandbox was provisioned again while deprovision removed it", + )); + } + let foreign = remaining(&state_dir)?; + if !foreign.is_empty() { + return Err(foreign_files(foreign)); + } if !deadline.pause(LOCK_RETRY_INTERVAL)? { - return Err(failed(RemovalError::TimedOut(error))); + return Err(io_error(ErrorCode::TimedOut, &error, &state_dir)); } } - Err(error) => return Err(failed(error)), } } } } -/// Returns the error of `nvx.py`'s deprovision for the store's removal of the sandbox in -/// `state_dir`, whose store names it `dir`, that `error` ended, with the code that the store's -/// conversion gives it. -fn removal_error(error: RemovalError, state_dir: &Path, dir: &Path) -> Exit { - match error { - // The CLI always declares its configuration record. - error @ RemovalError::Undeclared => Exit::error(ErrorCode::Unsupported, error.to_string()), - RemovalError::Launched => Exit::error( - ErrorCode::SandboxRunning, - "sandbox must be stopped before deprovision", - ), - RemovalError::Foreign(names) => Exit::error( - ErrorCode::ForeignFiles, - format!( - "sandbox state directory contains files not owned by NVX: {}", - names.join(", ") - ), - ), - RemovalError::ProvisionedAgain => Exit::error( - ErrorCode::SandboxProvisioned, - "sandbox was provisioned again while deprovision removed it", - ), - RemovalError::TimedOut(error) => io_error(ErrorCode::TimedOut, &error, state_dir), - RemovalError::Io { path, error } => { - // The store names the directory, or one of its entries, by its own path. - let shown = match path.strip_prefix(dir) { - Ok(name) => state_dir.join(name), - Err(_) => path, - }; - io_error(ErrorCode::Io, &error, &paths::normalized(&shown)) +impl Drop for LifecycleLock { + fn drop(&mut self) { + if [StateFile::Config, StateFile::Journal, StateFile::Runtime] + .into_iter() + .all(|name| { + matches!(std::fs::symlink_metadata(self.state_dir.join(name.name())), + Err(error) if error.kind() == io::ErrorKind::NotFound) + }) + && let Ok(directory) = self.locked.directory() + { + let _ = directory.remove_file(StateFile::Lock.name()); } } } +fn remaining(path: &Path) -> Result> { + std::fs::read_dir(path) + .map_err(Exit::from_os_error)? + .filter_map(|entry| { + let name = match entry { + Ok(entry) => entry.file_name(), + Err(error) => return Some(Err(Exit::from_os_error(error))), + }; + let name = match name.into_string() { + Ok(name) => name, + Err(_) => { + return Some(Err(Exit::error( + ErrorCode::Io, + "a sandbox file name is not valid Unicode", + ))); + } + }; + (name != StateFile::Lock.name() && !name.starts_with(".nfs")).then_some(Ok(name)) + }) + .collect() +} + +fn foreign_files(names: Vec) -> Exit { + Exit::error( + ErrorCode::ForeignFiles, + format!( + "sandbox state directory contains files not owned by NVX: {}", + names.join(", ") + ), + ) +} + #[cfg(test)] mod tests; diff --git a/nvx-rs/src/host/cli/sandbox/lock/tests.rs b/nvx-rs/src/host/cli/sandbox/lock/tests.rs index c7b015c71..7c1c6c6db 100644 --- a/nvx-rs/src/host/cli/sandbox/lock/tests.rs +++ b/nvx-rs/src/host/cli/sandbox/lock/tests.rs @@ -2,7 +2,7 @@ use super::*; fn state_dir(directory: &tempfile::TempDir) -> PathBuf { let state = directory.path().join("state"); - SandboxStore::create(&state).unwrap(); + StateDir::create(&state).unwrap(); state } @@ -62,57 +62,95 @@ fn directories_that_others_may_access_are_refused() { ); } +#[cfg(windows)] #[test] -fn removals_keep_the_messages_of_deprovision() { +fn provision_refuses_an_inherited_vm_directory_without_changing_it() { let directory = tempfile::tempdir().unwrap(); - let state = state_dir(&directory); - let dir = state.clone(); - let message = |error| { - let exit = removal_error(error, &state, &dir); - (exit.code().unwrap(), exit.message().unwrap().to_owned()) - }; - assert_eq!( - message(RemovalError::Launched), - ( - ErrorCode::SandboxRunning, - "sandbox must be stopped before deprovision".to_owned() - ) + let parent = directory.path().join("private-parent"); + StateDir::create(&parent).unwrap(); + let state = parent.join("inherited"); + std::fs::create_dir(&state).unwrap(); + StateDir::open(&state).unwrap(); + let refusal = VmDir::open(&state).unwrap_err().to_string(); + let identity = platform::file_id(&state).unwrap(); + std::fs::write(state.join("foreign.txt"), "keep").unwrap(); + let error = LifecycleLock::acquire(&state, 1.0, true).err().unwrap(); + assert_eq!(error.code(), Some(ErrorCode::Io)); + assert!( + error + .message() + .unwrap() + .contains("sandbox state directory must be a private VM directory") ); + assert_eq!(platform::file_id(&state).unwrap(), identity); + assert_eq!(VmDir::open(&state).unwrap_err().to_string(), refusal); assert_eq!( - message(RemovalError::Foreign(vec![ - "b.txt".to_owned(), - "a.txt".to_owned() - ])), - ( - ErrorCode::ForeignFiles, - "sandbox state directory contains files not owned by NVX: b.txt, a.txt".to_owned() - ) + std::fs::read_to_string(state.join("foreign.txt")).unwrap(), + "keep" ); + assert!(!state.join(CONFIG_NAME).exists()); + let entries: Vec<_> = std::fs::read_dir(&state) + .unwrap() + .map(|entry| entry.unwrap().file_name()) + .collect(); + assert_eq!(entries, [std::ffi::OsString::from("foreign.txt")]); +} + +#[test] +fn removals_refuse_live_records_and_keep_foreign_files() { + let directory = tempfile::tempdir().unwrap(); + let state = state_dir(&directory); + std::fs::write(state.join(CONFIG_NAME), "{}").unwrap(); + for record in [StateFile::Journal, StateFile::Runtime] { + let lock = LifecycleLock::acquire(&state, 10.0, false).unwrap(); + let path = state.join(record.name()); + std::fs::write(&path, "{}").unwrap(); + let error = lock.remove(10.0).unwrap_err(); + assert_eq!(error.code(), Some(ErrorCode::SandboxRunning)); + assert_eq!( + error.message(), + Some("sandbox must be stopped before deprovision") + ); + assert!(path.exists()); + assert!(state.join(CONFIG_NAME).exists()); + std::fs::remove_file(path).unwrap(); + } + std::fs::write(state.join("foreign.txt"), "keep").unwrap(); + let lock = LifecycleLock::acquire(&state, 10.0, false).unwrap(); + let error = lock.remove(10.0).unwrap_err(); + assert_eq!(error.code(), Some(ErrorCode::ForeignFiles)); assert_eq!( - message(RemovalError::ProvisionedAgain), - ( - ErrorCode::SandboxProvisioned, - "sandbox was provisioned again while deprovision removed it".to_owned() - ) + std::fs::read_to_string(state.join("foreign.txt")).unwrap(), + "keep" ); - let denied = || io::Error::from(io::ErrorKind::PermissionDenied); - let config = state.join(CONFIG_NAME); - for (path, shown) in [ - (dir.clone(), state.as_path()), - (dir.join(CONFIG_NAME), config.as_path()), + assert!(!state.join(CONFIG_NAME).exists()); + assert!(!state.join(StateFile::Lock.name()).exists()); +} + +#[test] +fn removal_cleans_owned_temporaries_without_touching_a_neighbour() { + let directory = tempfile::tempdir().unwrap(); + let state = state_dir(&directory); + let neighbour = directory.path().join("neighbour"); + StateDir::create(&neighbour).unwrap(); + std::fs::write(neighbour.join(CONFIG_NAME), "neighbour").unwrap(); + std::fs::write(state.join(CONFIG_NAME), "{}").unwrap(); + for name in [ + ".runtime.json.42.tmp", + ".config.json.0123456789abcdef0123456789abcdef.tmp", + ".tmpAb12Cd", + "openvmm.log", + "outcome.json", + "control.capability", ] { - let expected = format!("{shown:?}: {}", denied()); - assert_eq!( - message(RemovalError::Io { - path, - error: denied() - }), - (ErrorCode::Io, expected) - ); + std::fs::write(state.join(name), "owned").unwrap(); } + let lock = LifecycleLock::acquire(&state, 10.0, false).unwrap(); + lock.remove(10.0).unwrap(); + assert!(!state.exists()); assert_eq!( - message(RemovalError::TimedOut(denied())), - (ErrorCode::TimedOut, format!("{state:?}: {}", denied())) + std::fs::read_to_string(neighbour.join(CONFIG_NAME)).unwrap(), + "neighbour" ); } diff --git a/nvx-rs/src/host/cli/session.rs b/nvx-rs/src/host/cli/session.rs index fc6dddcef..ef819c211 100644 --- a/nvx-rs/src/host/cli/session.rs +++ b/nvx-rs/src/host/cli/session.rs @@ -4,7 +4,7 @@ //! //! It keeps the observable behavior of the Python session: //! -//! - [`Session::connect`] retries the endpoint every 25 ms until its timeout, then attaches with +//! - [`Session::connect_unless_exited`] retries the endpoint every 25 ms until its timeout, then attaches with //! the capability within a fresh timeout. On Windows, it gives up at once on the errors that //! Python does not retry. On Linux, unsupported peer verification fails at once, retaining //! its diagnostic instead of reporting that the endpoint did not appear. @@ -20,10 +20,11 @@ //! - A session whose transfer failed is closed, and refuses further requests. //! //! Unlike the Python session, it also hands the guest agent a host mapping table, as `start` -//! does for the shares of a sandbox: in order, and in as many requests as it takes. And -//! [`Session::connect_launched`], with which `start` waits for the endpoint of the OpenVMM that -//! it launched, stops waiting once that OpenVMM exits, as `nvx::host::session::connect` does, -//! where Python waited until its timeout. +//! does for the shares of a sandbox: in order, and in as many requests as it takes. +//! [`Session::reattach`] verifies a saved VM and observes its transport before authentication; +//! [`Session::connect_launched_cancellable`], with which a cold run waits for the endpoint of the +//! OpenVMM that it launched, also stops waiting once that OpenVMM exits, where Python waited until +//! its timeout. //! //! The transports also wait in slices of at most 250 ms, as Python's do, so that Ctrl-C ends a //! wait within that time, except while a request is being sent, which its timeout bounds. @@ -35,17 +36,20 @@ use std::time::Duration; use super::deadline::{Deadline, RETRY_INTERVAL, sleep}; use super::{Exit, Result, check_interrupt}; -use crate::ErrorCode; use crate::host::platform::{self, Transport}; +use crate::host::runtime::Runtime; use crate::host::sandbox::options::validate_exec_environment; use crate::host::session::{ControlSession, SessionError}; use crate::host::supervisor::Launched; +use crate::host::transport; +use crate::host::vm::{self, SavedVm}; use crate::protocol::{ APP_ERROR, APP_EXEC, APP_EXIT, APP_FEATURES, APP_HEADER_LEN, APP_MAPS, APP_PING, APP_READY, APP_STDERR, APP_STDOUT, APP_STOP, APP_STOPPED, CAPABILITY_LEN, GuestFeatures, HostMap, MAPPING_FAILED, MAX_OUTPUT_BYTES, OUTER_MAX_PAYLOAD, UNSUPPORTED_OPERATION, Workload, WorkloadEnvironment, encode_exec_payload, encode_maps_payloads, }; +use crate::{ErrorCode, ErrorKind}; /// The exit categories that `exec` accepts, `MANAGED_EXIT_CATEGORIES`. A cancellation, which this /// session never requests, is not among them. @@ -282,49 +286,62 @@ pub(crate) struct Session { } impl Session { - /// Connects to `endpoint`, which OpenVMM process `pid` serves, and attaches with - /// `capability`, as `ControlSession.connect(endpoint, capability, timeout)` does: the endpoint - /// gets `timeout` seconds to appear, and the attachment `timeout` seconds more. - /// - /// Like `nvx::host::session`, it refuses an endpoint that another process serves, which it - /// treats as one that has not appeared yet, rather than handing that process the capability. - pub(crate) fn connect( - endpoint: &str, - pid: u32, - capability: &[u8; CAPABILITY_LEN], - timeout: f64, - ) -> Result { - Self::connect_unless_exited(endpoint, pid, capability, timeout, || false, None) + /// Observes the same verified transport through attachment and later control operations. + pub(crate) fn reattach(saved: &SavedVm, runtime: &Runtime, timeout: f64) -> Result { + let probe = Arc::new(Probe::default()); + let attached = vm::reattach_with_transport( + saved, + runtime, + Deadline::after(timeout).instant(), + |endpoint, reference, deadline| loop { + if platform::interrupted() { + probe.interrupted.store(true, Ordering::SeqCst); + return Err(crate::Error::io("control connection interrupted")); + } + let until = (std::time::Instant::now() + POLL_SLICE).min(deadline); + match transport::connect_verified(endpoint, reference, until) { + Ok(inner) => { + return Ok(Box::new(ProbedTransport { + inner, + probe: Arc::clone(&probe), + })); + } + Err(error) + if error.kind() == ErrorKind::TimedOut + && std::time::Instant::now() < deadline => {} + Err(error) => return Err(error), + } + }, + ) + .map_err(|error| { + if probe.interrupted.load(Ordering::SeqCst) { + Exit::Interrupted + } else if probe.closed.load(Ordering::SeqCst) + || error.kind() == ErrorKind::Exited + || std::error::Error::source(&error) + .and_then(|source| source.downcast_ref::()) + .is_some_and(closes) + { + Exit::EndpointClosed + } else { + super::sandbox::error_with_cause(error) + } + })?; + Ok(Self { + inner: Some(ControlSession { + connection: attached.connection, + }), + probe, + }) } - /// Connects to `endpoint`, which `launched`, the OpenVMM process that a start launched, - /// serves, as [`connect`](Self::connect) does, but stops waiting for the endpoint once + /// Connects to `endpoint`, which `launched`, the OpenVMM process that a cold run launched, + /// serves, as [`Self::connect_unless_exited`] does, but stops waiting for the endpoint once /// OpenVMM has exited, as `nvx::host::session::connect` does: an OpenVMM that exits before it /// serves its endpoint, as one that refuses its configuration does, then ends the wait at /// once, rather than at its timeout. A check that fails keeps the wait going until the - /// timeout, as there. - pub(crate) fn connect_launched( - endpoint: &str, - launched: &mut Launched, - capability: &[u8; CAPABILITY_LEN], - timeout: f64, - ) -> Result { - let pid = launched.id(); - Self::connect_unless_exited( - endpoint, - pid, - capability, - timeout, - || { - launched - .wait_timeout(Duration::ZERO) - .is_ok_and(|status| status.is_some()) - }, - None, - ) - } - - /// Connects an owned foreground launch whose output owner can cancel both wait phases. + /// timeout, as there. The foreground's output owner can cancel both wait phases through + /// `cancellation`. pub(super) fn connect_launched_cancellable( endpoint: &str, launched: &mut Launched, @@ -347,7 +364,7 @@ impl Session { ) } - /// Connects as [`connect`](Self::connect) does, but fails as soon as `exited`, which it calls + /// Connects and attaches, but fails as soon as `exited`, which it calls /// after each attempt that finds no endpoint, tells that OpenVMM process `pid` has exited: /// with [`Exit::EndpointClosed`], as the session reports the `ProcessExited` of /// `nvx::host::session`. @@ -399,7 +416,7 @@ impl Session { } /// Attaches over a connected `transport` with `capability` within `timeout` seconds, as the - /// second half of [`connect`](Self::connect). + /// second half of [`Self::connect_unless_exited`]. fn attach( transport: Box, capability: &[u8; CAPABILITY_LEN], diff --git a/nvx-rs/src/host/cli/session/tests.rs b/nvx-rs/src/host/cli/session/tests.rs index 2aadea163..50f017f3d 100644 --- a/nvx-rs/src/host/cli/session/tests.rs +++ b/nvx-rs/src/host/cli/session/tests.rs @@ -8,6 +8,7 @@ use std::thread; use std::time::Instant; use super::*; +use crate::host::transport::HostConnection; use crate::protocol::{ EXEC_CWD_PRESENT, EXEC_ENVIRONMENT_PRESENT, EXEC_EXTENDED, EXEC_INHERIT_DEFAULT_ENV, OUTER_DATA, OUTER_HEADER_LEN, OUTER_HOST_ATTACH, OUTER_READY, OUTER_RESET, decode_app, @@ -17,6 +18,24 @@ use crate::protocol::{ const CAPABILITY: [u8; CAPABILITY_LEN] = [7; CAPABILITY_LEN]; const INSTANCE: [u8; 16] = [0x11; 16]; +impl Session { + fn from_connection(connection: HostConnection) -> Self { + Self { + inner: Some(ControlSession { connection }), + probe: Arc::new(Probe::default()), + } + } + + fn connect( + endpoint: &str, + pid: u32, + capability: &[u8; CAPABILITY_LEN], + timeout: f64, + ) -> Result { + Self::connect_unless_exited(endpoint, pid, capability, timeout, || false, None) + } +} + /// What a [`Recording`] transport records: the direction and timeout of each transfer. type Transfers = Arc)>>>; @@ -168,6 +187,34 @@ fn session_with( (session, worker) } +#[test] +fn an_owned_connection_keeps_its_authenticated_framing_without_another_attach() { + let (host, guest) = stream_pair(64); + let worker = thread::spawn(move || { + let mut guest = Guest::new(guest); + guest.accept(); + for expected in [17, 0] { + let (kind, request_id, payload) = guest.request(); + assert_eq!(kind, APP_PING); + assert!(payload.is_empty()); + if expected != 0 { + assert_eq!(request_id, expected); + } + guest.send(APP_READY, request_id, 0, &[]); + } + }); + let until = Instant::now() + Duration::from_secs(5); + let mut connection = HostConnection::attach(Box::new(host), &CAPABILITY, until).unwrap(); + connection + .send(&encode_app(APP_PING, 17, 0, &[]), Some(until)) + .unwrap(); + let reply = connection.receive(Some(until)).unwrap(); + assert_eq!(decode_app(&reply).unwrap().request_id, 17); + let mut session = Session::from_connection(connection); + session.ping(5.0).unwrap(); + worker.join().unwrap(); +} + fn request(arguments: &[&str]) -> ExecRequest { ExecRequest { arguments: arguments @@ -728,10 +775,11 @@ fn check_writes_to_a_slow_endpoint( assert!(reader.join().unwrap() > 0, "the endpoint read nothing"); } -/// Checks the wait for `endpoint`, which never appears, of a start that checks after each -/// attempt whether the OpenVMM that it launched has exited, as [`Session::connect_launched`] -/// checks it: an OpenVMM that exits ends the wait at once, as a closed endpoint, rather than at -/// the timeout, and one that runs on keeps it going until the timeout. +/// Checks the wait for `endpoint`, which never appears, of a cold run that checks after each +/// attempt whether the OpenVMM that it launched has exited, as +/// [`Session::connect_launched_cancellable`] checks it: an OpenVMM that exits ends the wait at +/// once, as a closed endpoint, rather than at the timeout, and one that runs on keeps it going +/// until the timeout. #[cfg(any(unix, windows))] fn check_waits_for_openvmm_that_exits(endpoint: &str) { let mut checks = 0; diff --git a/nvx-rs/src/host/cli/store/locked.rs b/nvx-rs/src/host/cli/store/locked.rs index 35bf46ffb..389eefa36 100644 --- a/nvx-rs/src/host/cli/store/locked.rs +++ b/nvx-rs/src/host/cli/store/locked.rs @@ -43,6 +43,24 @@ impl Drop for LockedState { } } +impl LockedState { + pub(crate) fn directory(&self) -> Result { + let directory = platform::Directory::open(self.dir()).map_err(Exit::from_os_error)?; + let held = platform::opened_file_id(&self.lock).map_err(Exit::from_os_error)?; + if directory + .entry_id(StateFile::Lock.name()) + .map_err(Exit::from_os_error)? + != Some(held) + { + return Err(Exit::error( + ErrorCode::Io, + "the directory that the path names does not hold the sandbox's lock", + )); + } + Ok(directory) + } +} + impl StateDir { /// Takes the lock of the directory, trying it again every 25 ms until `deadline`. /// diff --git a/nvx-rs/src/host/cli/store/mod.rs b/nvx-rs/src/host/cli/store/mod.rs index 2d5f06ba9..17f392a6c 100644 --- a/nvx-rs/src/host/cli/store/mod.rs +++ b/nvx-rs/src/host/cli/store/mod.rs @@ -36,14 +36,6 @@ //! Taking the next lock recovers an interrupted, uncommitted launch before allowing mutation: //! it ends only a verified process, removes its journal and capability only after proved exit, //! and refuses indeterminate recovery without changing that evidence. -#![cfg_attr( - not(test), - expect( - dead_code, - reason = "SL3.5b switches the sandbox lifecycle commands to the CLI's own store" - ) -)] - mod files; mod launch; mod liveness; diff --git a/nvx-rs/src/host/cli/store/names.rs b/nvx-rs/src/host/cli/store/names.rs index c6e01e876..b91ed43f9 100644 --- a/nvx-rs/src/host/cli/store/names.rs +++ b/nvx-rs/src/host/cli/store/names.rs @@ -23,6 +23,22 @@ pub(crate) enum StateFile { } impl StateFile { + pub(crate) fn owns_temporary(name: &str) -> bool { + name.strip_prefix(".tmp").is_some_and(|suffix| { + suffix.len() == 6 && suffix.bytes().all(|byte| byte.is_ascii_alphanumeric()) + }) || name + .strip_prefix('.') + .and_then(|name| name.strip_suffix(".tmp")) + .and_then(|name| name.rsplit_once('.')) + .is_some_and(|(file, suffix)| { + Self::ALL.iter().any(|name| name.name() == file) + && (!suffix.is_empty() && suffix.bytes().all(|byte| byte.is_ascii_digit()) + || file == Self::Config.name() + && suffix.len() == 32 + && suffix.bytes().all(|byte| byte.is_ascii_hexdigit())) + }) + } + /// Every file that the sandbox commands keep in a state directory. pub(crate) const ALL: [Self; 8] = [ Self::Config, diff --git a/nvx-rs/src/host/platform/windows/spawn.rs b/nvx-rs/src/host/platform/windows/spawn.rs index bcaf571e3..4ceb78880 100644 --- a/nvx-rs/src/host/platform/windows/spawn.rs +++ b/nvx-rs/src/host/platform/windows/spawn.rs @@ -33,8 +33,6 @@ use windows_sys::Win32::System::Threading::{ WaitForSingleObject, }; -#[cfg(feature = "cli")] -use super::process::refused_termination; use super::process::{ProcessHandle, creation_time}; use super::{owned, raw, wide}; use crate::host::platform::check_working_dir; @@ -115,21 +113,6 @@ impl LaunchedProcess { } Ok(Some(status)) } - - /// Asks the process to end with status 1 without waiting for it, as `Popen.terminate` and - /// `Popen.kill` do on Windows. - #[cfg(feature = "cli")] - pub(crate) fn request_termination(&self) -> io::Result<()> { - // SAFETY: CreateProcessW returned the handle with full access. - if unsafe { TerminateProcess(raw(&self.process), 1) } == 0 { - // A process that is exiting or exited cannot be terminated; Python ignores that too. - let refused = refused_termination(io::Error::last_os_error()); - if refused.is_err() && self.wait_timeout(Duration::ZERO)?.is_none() { - return refused; - } - } - Ok(()) - } } /// Owns an initialized process-thread attribute list. diff --git a/nvx-rs/src/host/session.rs b/nvx-rs/src/host/session.rs index 8fb8c1f77..ecfc9b2cc 100644 --- a/nvx-rs/src/host/session.rs +++ b/nvx-rs/src/host/session.rs @@ -145,7 +145,7 @@ pub enum ExecEvent { /// later calls fail. pub struct ControlSession { /// The host's NVXS connection to OpenVMM's control endpoint. - connection: HostConnection, + pub(crate) connection: HostConnection, } impl ControlSession { diff --git a/nvx-rs/src/host/spawn.rs b/nvx-rs/src/host/spawn.rs index 1fc1a3eb8..4056545bb 100644 --- a/nvx-rs/src/host/spawn.rs +++ b/nvx-rs/src/host/spawn.rs @@ -386,40 +386,6 @@ impl Launched { self.reaped |= status.is_some(); Ok(status) } - - /// Asks the process to end, as `Popen.terminate` does: with `SIGTERM` on Unix, and by - /// terminating it with status 1 on Windows. A process that has exited is left alone. - #[cfg(feature = "cli")] - pub(crate) fn terminate(&mut self) -> io::Result<()> { - #[cfg(windows)] - return self.process.request_termination(); - #[cfg(not(windows))] - self.signal(nix::sys::signal::Signal::SIGTERM) - } - - /// Ends the process, as `Popen.kill` does: with `SIGKILL` on Unix, and by terminating it with - /// status 1 on Windows. A process that has exited is left alone. - #[cfg(feature = "cli")] - pub(crate) fn kill(&mut self) -> io::Result<()> { - #[cfg(windows)] - return self.process.request_termination(); - #[cfg(not(windows))] - self.signal(nix::sys::signal::Signal::SIGKILL) - } - - /// Sends `signal` to the process unless it has exited. The child is not reaped until it is - /// waited for, so its process ID cannot name another process meanwhile. - #[cfg(all(feature = "cli", not(windows)))] - fn signal(&mut self, signal: nix::sys::signal::Signal) -> io::Result<()> { - if self.try_reap()?.is_some() { - return Ok(()); - } - let pid = i32::try_from(self.child.id()).map_err(io::Error::other)?; - match nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid), signal) { - Ok(()) | Err(nix::errno::Errno::ESRCH) => Ok(()), - Err(error) => Err(error.into()), - } - } } /// Returns `Popen.returncode` for `status`: the exit code, or the negated signal number. diff --git a/nvx-rs/src/host/supervisor/launch.rs b/nvx-rs/src/host/supervisor/launch.rs index 3186b07bf..26a59691f 100644 --- a/nvx-rs/src/host/supervisor/launch.rs +++ b/nvx-rs/src/host/supervisor/launch.rs @@ -241,17 +241,6 @@ impl<'s> Starting<'s> { &self.inner().process } - /// Returns the VMM's child, through which the CLI waits for a VMM whose start failed, reads - /// its exit status, and ends it as `subprocess` ends it, before it abandons the launch. - #[cfg(feature = "cli")] - pub(crate) fn child(&mut self) -> &mut Launched { - &mut self - .inner - .as_mut() - .expect("a launch is in progress until it is committed or abandoned") - .child - } - /// Commits the launch of a VMM that serves its control endpoint: writes its runtime record, /// as [`Launch::commit`] does, releases the VMM's child, which keeps running, and returns the /// VM. diff --git a/nvx-rs/src/host/vm.rs b/nvx-rs/src/host/vm.rs index 080c301ba..27c66511f 100644 --- a/nvx-rs/src/host/vm.rs +++ b/nvx-rs/src/host/vm.rs @@ -169,9 +169,19 @@ pub fn reattach( saved: &SavedVm, runtime: &Runtime, deadline: std::time::Instant, +) -> Result { + reattach_with_transport(saved, runtime, deadline, transport::connect_verified) +} + +/// Lets the CLI observe its verified transport before the single authenticated attachment. +pub(crate) fn reattach_with_transport( + saved: &SavedVm, + runtime: &Runtime, + deadline: std::time::Instant, + connect: impl FnOnce(&str, &ProcessRef, std::time::Instant) -> Result>, ) -> Result { let process = verify(saved, &runtime.identity)?; - let transport = transport::connect_verified(&saved.endpoint, &saved.process, deadline)?; + let transport = connect(&saved.endpoint, &saved.process, deadline)?; let connection = HostConnection::attach(transport, &saved.capability, deadline)?; Ok(Reattached { process, diff --git a/nvx-rs/src/testing/fake_openvmm.rs b/nvx-rs/src/testing/fake_openvmm.rs index d7cc3b7de..c593ba2e5 100644 --- a/nvx-rs/src/testing/fake_openvmm.rs +++ b/nvx-rs/src/testing/fake_openvmm.rs @@ -103,7 +103,9 @@ //! kernel command line, which such a command line may omit, has `fake_exit_status=STATUS`. A //! restore whose snapshot directory does not exist fails with status 1 and writes no report. A //! restore records its command line next to its snapshot, since it has no kernel. The outcome -//! reports of sandboxes and of these runs take the form of OpenVMM's. +//! reports of sandboxes and of these runs take the form of OpenVMM's. As OpenVMM does, a managed +//! VM's report names the operation `managed` only if `--microvm-lifecycle` selects its lifecycle, +//! and `run` if the CLI's tokens do. //! //! As OpenVMM does, a restore checks its `--virtio-console` against the attachment of the boot //! console, `console:microvm-virtio0`, that the machine contract of the snapshot's manifest holds, @@ -192,7 +194,7 @@ use crate::protocol::{ APP_STDERR, APP_STDOUT, APP_STOP, APP_STOPPED, BUSY, CAPABILITY_LEN, CWD_FAILED, ExitCategory, GuestFeatures, INVALID_REQUEST, LAUNCH_FAILED, MAPPING_FAILED, MAPPINGS_INCOMPLETE, MAPS_TOKEN, MAX_HOST_MAPPINGS, MAX_OUTPUT_BYTES, MapsPayload, OUTER_DATA, OUTER_ERROR, OUTER_HEADER_LEN, - OUTER_HOST_ATTACH, OUTER_READY, OuterHeader, UNSUPPORTED_OPERATION, decode_app, + OUTER_HOST_ATTACH, OUTER_READY, OUTER_WAIT, OuterHeader, UNSUPPORTED_OPERATION, decode_app, decode_exec_payload, decode_maps_payload, decode_outer_header, encode_app, encode_outer, }; @@ -350,6 +352,8 @@ struct Options { program: String, /// The lifecycle. lifecycle: Lifecycle, + /// Whether `--microvm-lifecycle` selected the lifecycle, rather than the CLI's tokens. + lifecycle_option: bool, /// Whether the command line launches the sandbox profile of `nvx-rs sandbox`: a container /// over EROFS layers and an ext4 scratch disk. sandbox: bool, @@ -382,10 +386,14 @@ struct Options { sessions_log: PathBuf, /// Status with which to stop immediately after launch, when requested. exit_on_start: Option, + /// Status published after closing a waiting attachment, when requested. + exit_during_attach: Option, /// Status with which a one-shot workload ends instead of its own, when requested. exit_status: Option, /// Artificial delay before the fake guest becomes ready. boot_delay: Duration, + /// Artificial delay with the host attached but no guest yet. + guest_delay: Duration, /// Whether the boot console emits the binary-output regression fixture. binary_console: bool, /// Number of bytes in the sustained-console regression fixture. @@ -430,6 +438,19 @@ struct Options { capture: Option, } +impl Options { + /// Returns the operation that OpenVMM reports for a managed VM: `managed` only for the + /// lifecycle that `--microvm-lifecycle` selects, and `run` for one whose init owns its + /// lifecycle, as the CLI's tokens select it. + fn managed_operation(&self) -> &'static str { + if self.lifecycle_option { + "managed" + } else { + "run" + } + } +} + /// The network policy of a command line, as the `network_policy` section of OpenVMM's outcome /// report summarizes it. #[derive(Debug, Clone, Default)] @@ -613,7 +634,20 @@ pub fn main() -> ExitCode { }); } match serve(&options, &capability) { - Ok(()) => ExitCode::SUCCESS, + Ok(()) => { + if let Some(code) = options.exit_during_attach { + thread::sleep(options.exit_delay); + write_report( + &options, + options.managed_operation(), + "guest-exit", + i32::from(code), + ); + ExitCode::from(code) + } else { + ExitCode::SUCCESS + } + } Err(error) => { eprintln!("{program}: {error}"); ExitCode::from(1) @@ -1352,16 +1386,19 @@ fn parse(program: &str, arguments: &[String]) -> Result { let mut option_arguments = arguments.iter(); let mut run = true; let mut capture = false; + let mut lifecycle_option = false; while let Some(argument) = option_arguments.next() { let name = argument.as_str(); if name == "--microvm-lifecycle" || name == "--microvm-sandbox-block" { run = false; } capture |= name == "--snapshot-destination"; + lifecycle_option |= name == "--microvm-lifecycle"; if VALUED.contains(&name) || RUN_VALUED.contains(&name) || CAPTURE_VALUED.contains(&name) { let _ = option_arguments.next(); } } + let cold_sandbox = !run && !capture && !lifecycle_option; let mut values: BTreeMap<&str, Vec<&str>> = BTreeMap::new(); let mut flags = BTreeSet::new(); let mut iterator = arguments.iter(); @@ -1373,7 +1410,7 @@ fn parse(program: &str, arguments: &[String]) -> Result { { flags.insert(name); } else if VALUED.contains(&name) - || ((run || capture) && RUN_VALUED.contains(&name)) + || ((run || capture || cold_sandbox) && RUN_VALUED.contains(&name)) || (capture && CAPTURE_VALUED.contains(&name)) { let value = iterator @@ -1402,7 +1439,7 @@ fn parse(program: &str, arguments: &[String]) -> Result { // whatever tokens its command line has. let run_mode = if run && values.contains_key("--restore-snapshot") { Some((Lifecycle::Run, DEFAULT_WORKLOAD_UID)) - } else if run { + } else if run || cold_sandbox { let command_line = values.get("--cmdline").and_then(|given| given.first()); Some(run_lifecycle(command_line.copied().unwrap_or_default())?) } else { @@ -1425,6 +1462,12 @@ fn parse(program: &str, arguments: &[String]) -> Result { if !flags.contains("--single-process") { return Err("--single-process is required".to_owned()); } + if cold_sandbox { + if !matches!(lifecycle, Lifecycle::Managed) { + return Err("a cold persistent sandbox selects nvx_mode=managed".to_owned()); + } + expect("--guest-shutdown-action", "exit")?; + } let boot_console = if run && values.contains_key("--guest-shutdown-action") { if values.contains_key("--restore-snapshot") { return Err("--guest-shutdown-action is only for a cold boot".to_owned()); @@ -1496,7 +1539,7 @@ fn parse(program: &str, arguments: &[String]) -> Result { // A run uses the workload identity that the CLI's tokens name, or else, like a capture, the // default workload identity of the guest, and records its command line next to the kernel of // a cold boot or to the snapshot of a restore. - let (workload_uid, anchor, restore) = if run || capture { + let (workload_uid, anchor, restore) = if run || capture || cold_sandbox { if values.contains_key("--microvm-workload-identity") { return Err(format!( "{} takes no --microvm-workload-identity", @@ -1611,7 +1654,11 @@ fn parse(program: &str, arguments: &[String]) -> Result { } (0, Vec::new(), Vec::new(), Vec::new(), blocks) } else { - let workload = entrypoint_and_arguments()?; + let workload = if cold_sandbox { + Vec::new() + } else { + entrypoint_and_arguments()? + }; if token_values("nvx_hostname=").len() != 1 { return Err("the kernel command line must set one nvx_hostname".to_owned()); } @@ -1791,6 +1838,7 @@ fn parse(program: &str, arguments: &[String]) -> Result { Ok(Options { program: program.to_owned(), lifecycle, + lifecycle_option, sandbox, run, restore, @@ -1806,8 +1854,10 @@ fn parse(program: &str, arguments: &[String]) -> Result { report, hypervisor: hypervisor.to_owned(), exit_on_start: knob("fake_exit_on_start").and_then(|value| value.parse().ok()), + exit_during_attach: knob("fake_exit_during_attach").and_then(|value| value.parse().ok()), exit_status: knob("fake_exit_status").and_then(|value| value.parse().ok()), boot_delay: millis("fake_boot_delay_ms").unwrap_or_default(), + guest_delay: millis("fake_guest_delay_ms").unwrap_or_default(), binary_console: knob("fake_binary_console") == Some("1"), console_bytes: match knob("fake_console_bytes") { None => 0, @@ -2929,6 +2979,18 @@ fn serve_client( eprintln!("foreground handshake gate failed: {error}"); return Flow::Continue; } + if options.exit_during_attach.is_some() || !options.guest_delay.is_zero() { + if stream + .write_all(&control_record(OUTER_WAIT, &[0; 16], 0, &[])) + .is_err() + { + return Flow::Continue; + } + if options.exit_during_attach.is_some() { + return Flow::Exit; + } + thread::sleep(options.guest_delay); + } let mut instance = [0u8; 16]; while instance == [0; 16] { if getrandom::fill(&mut instance).is_err() { @@ -2952,7 +3014,7 @@ fn serve_client( }; match session.run(options, guest) { Ok(Flow::Exit) => { - write_report(options, "managed", "success", 0); + write_report(options, options.managed_operation(), "success", 0); thread::sleep(options.exit_delay); Flow::Exit } diff --git a/nvx-rs/tests/cli.rs b/nvx-rs/tests/cli.rs index 3938ca914..5cc2f83a9 100644 --- a/nvx-rs/tests/cli.rs +++ b/nvx-rs/tests/cli.rs @@ -200,13 +200,24 @@ fn a_mapping_that_the_guest_refuses_leaves_no_usable_start() { start.extend(["--timeout".to_owned(), "2".to_owned()]); let output = run(&root, &start); assert_eq!(output.status.code(), Some(1), "{state}"); - assert_eq!( - text(&output.stderr), - error_line( - "protocol", - "managed guest could not mount a mapped host path (error 13)" - ) + let error = text(&output.stderr); + let primary = error_line( + "protocol", + "managed guest could not mount a mapped host path (error 13)", ); + assert!(error.ends_with(&primary), "{error}"); + if stops { + assert_eq!(error, primary); + } else { + assert!( + error.contains("warning: startup guest shutdown failed:"), + "{error}" + ); + assert!( + error.contains("managed control response timed out"), + "{error}" + ); + } let mut names: Vec = fs::read_dir(root.path.join(state)) .unwrap() .map(|entry| entry.unwrap().file_name().into_string().unwrap()) @@ -221,7 +232,7 @@ fn a_mapping_that_the_guest_refuses_leaves_no_usable_start() { &fs::read_to_string(root.path.join(state).join("outcome.json")).unwrap(), ) .unwrap(); - assert_eq!(report["outcome"]["operation"], "managed"); + assert_eq!(report["outcome"]["operation"], "run"); assert_eq!(report["outcome"]["category"], "success"); } else { assert_eq!(names, ["config.json", "openvmm.log", "sandbox.lock"]); @@ -1084,7 +1095,7 @@ fn a_start_that_was_interrupted_is_ended_under_another_identity_too() { } #[test] -fn runtime_records_of_earlier_formats_fail_and_name_their_format() { +fn runtime_records_of_earlier_stores_are_refused_without_migration() { let root = Root::new(); provisioned(&root, "state"); // The runtime record that the CLI wrote before the store, in its format 1. @@ -1096,16 +1107,17 @@ fn runtime_records_of_earlier_formats_fail_and_name_their_format() { serde_json::to_string(endpoint.to_str().unwrap()).unwrap() ); fs::write(&path, &record).unwrap(); - let unsupported = format!( - "the sandbox runtime record {} has the unsupported format 1; this build reads format 4 \ - only and migrates no other, so stop the sandbox with the build that started it", - path.display() - ); for operation in ["start", "exec", "stop", "deprovision"] { - assert_eq!( - transition(&root, operation, "state", &[]), - failed("format-unsupported", &unsupported), - "{operation}" + let output = run(&root, &lifecycle(operation, &root, "state")); + assert_eq!(output.status.code(), Some(1), "{operation}"); + let error = text(&output.stderr); + assert!( + error.contains("error[io]: cannot parse"), + "{operation}: {error}" + ); + assert!( + error.contains(path.to_str().unwrap()), + "{operation}: {error}" ); } assert_eq!(fs::read_to_string(&path).unwrap(), record); @@ -1141,25 +1153,21 @@ fn a_managed_start_runs_only_the_openvmm_of_the_root() { } #[test] -fn stop_reports_its_timeout_and_leaves_the_vm_running() { +fn stop_forces_the_verified_vm_after_its_grace_period_even_after_an_acknowledgement() { let root = Root::new(); // A guest that never answers the stop. provisioned_with(&root, "unanswered", "fake_ignore_stop=1"); assert_eq!(transition(&root, "start", "unanswered", &[]), succeeded()); - let record = runtime_record(&root, "unanswered").unwrap(); - assert_eq!( - transition(&root, "stop", "unanswered", &["--timeout", "1"]), - failed("timed-out", "managed control response timed out") - ); - assert_eq!(runtime_record(&root, "unanswered").unwrap(), record); - let process = recorded_process(&root, "unanswered").expect("the VM still runs"); - process.kill().unwrap(); + let process = recorded_process(&root, "unanswered").expect("the VM runs"); + let (status, error) = transition(&root, "stop", "unanswered", &["--timeout", "1"]); + assert_eq!(status, Some(0), "{error}"); assert!( - process - .wait(Instant::now() + Duration::from_secs(10)) - .unwrap() - .is_some() + error.contains("managed control response timed out"), + "{error}" ); + assert!(error.contains("forcibly terminated"), "{error}"); + assert!(process.has_exited().unwrap()); + assert_eq!(runtime_record(&root, "unanswered"), None); assert_eq!( transition(&root, "deprovision", "unanswered", &[]), succeeded() @@ -1168,22 +1176,14 @@ fn stop_reports_its_timeout_and_leaves_the_vm_running() { // A guest that shuts down, whose OpenVMM takes longer to exit than the stop waits. provisioned_with(&root, "slow", "fake_exit_delay_ms=4000"); assert_eq!(transition(&root, "start", "slow", &[]), succeeded()); - let record = runtime_record(&root, "slow").unwrap(); - assert_eq!( - transition(&root, "stop", "slow", &["--timeout", "1"]), - failed("timed-out", "OpenVMM did not terminate after managed stop") - ); - assert_eq!(runtime_record(&root, "slow").unwrap(), record); - let process = recorded_process(&root, "slow").expect("the VM still runs"); - assert!( - process - .wait(Instant::now() + Duration::from_secs(30)) - .unwrap() - .is_some() - ); + let process = recorded_process(&root, "slow").expect("the VM runs"); + let (status, error) = transition(&root, "stop", "slow", &["--timeout", "1"]); + assert_eq!(status, Some(0), "{error}"); + assert!(error.contains("forcibly terminated"), "{error}"); + assert!(process.has_exited().unwrap()); assert_eq!( transition(&root, "stop", "slow", &[]), - failed("sandbox-exited", STALE) + failed("sandbox-not-running", "sandbox is not running") ); assert_eq!(runtime_record(&root, "slow"), None); assert_eq!(transition(&root, "deprovision", "slow", &[]), succeeded()); @@ -1917,11 +1917,9 @@ fn wait_for_json(path: &Path) -> Value { } #[test] -fn start_hands_openvmm_the_images_that_it_checked() { - // Images that something replaces after the start checked them and before OpenVMM opens them: - // on Linux, OpenVMM opens the checked files through the descriptors that it inherits, whatever - // has their paths now; on Windows, the checked files keep anything from replacing the images - // until the guest is ready. +fn start_opens_the_stable_caller_paths_that_provision_checked() { + // The caller keeps its image paths stable through start and the VM's lifetime. This no + // longer promises descriptor transfer or a Windows Ready-time guard-release boundary. let root = Root::new(); let output = run( &root, @@ -1934,8 +1932,6 @@ fn start_hands_openvmm_the_images_that_it_checked() { assert!(output.status.success(), "{}", text(&output.stderr)); let distro = root.path.join("distro.erofs"); let scratch = root.path.join("scratch.ext4"); - #[cfg(windows)] - let moved = |path: &Path| path.with_extension("moved"); let start = Command::new(NVX_RS) .arg("--root") .arg(&root.path) @@ -1955,38 +1951,32 @@ fn start_hands_openvmm_the_images_that_it_checked() { .join("build") .join("fake-openvmm-state-blocks.json"); assert!(!blocks.exists()); - for (path, replacement) in [ - (&distro, "replaced distro\n"), - (&scratch, "replaced scratch\n"), - ] { - let new = path.with_extension("new"); - fs::write(&new, replacement).unwrap(); - let replaced = fs::rename(&new, path); - assert_eq!( - replaced.is_ok(), - cfg!(unix), - "{}: {replaced:?}", - path.display() - ); - } - // On Windows, while the start holds the checked files, nothing writes the distro layer, and - // nothing removes or renames either image, even once OpenVMM reads and writes the scratch - // image. - #[cfg(windows)] - for opened in [false, true] { - if opened { - wait_for(&blocks); - } - assert!(fs::OpenOptions::new().append(true).open(&distro).is_err()); - for path in [&distro, &scratch] { - assert!(fs::rename(path, moved(path)).is_err()); - assert!(fs::remove_file(path).is_err()); - } - } + let arguments = root.openvmm_arguments("build/fake-openvmm-state.json"); + let mut options = openvmm::Options::parse(&arguments); + openvmm::assert_stable_blocks(&root, &options.take("--microvm-sandbox-block"), true); + let journal = wait_for_json(&root.below("state/launch.json")); + assert_eq!( + Path::new(journal["vmDir"].as_str().unwrap()), + root.below("state") + ); + assert_eq!( + Path::new(journal["log"].as_str().unwrap()), + root.below("state/openvmm.log") + ); + assert_eq!( + Path::new(journal["outcome"].as_str().unwrap()), + root.below("state/outcome.json") + ); + assert_eq!( + journal["capability"], + json!(fs::read(root.below("state/control.capability")).unwrap()) + ); + assert_eq!(fs::read(&distro).unwrap(), b"distro\n"); + assert_eq!(fs::read(&scratch).unwrap(), b"scratch\n"); let output = start.wait_with_output().unwrap(); assert!(output.status.success(), "{}", text(&output.stderr)); - // OpenVMM opened the objects that the start checked. + // OpenVMM opened the caller-owned images at those unchanged paths. let opened: Value = serde_json::from_str(&fs::read_to_string(&blocks).unwrap()).unwrap(); let opened: Vec<(&str, &str)> = opened .as_array() @@ -2000,13 +1990,6 @@ fn start_hands_openvmm_the_images_that_it_checked() { }) .collect(); assert_eq!(opened, [("distro", "distro\n"), ("scratch", "scratch\n")]); - // Once the guest is ready, the start releases the checked files: on Windows, the images can be - // renamed again, as OpenVMM's own opens share it. - #[cfg(windows)] - for path in [&distro, &scratch] { - fs::rename(path, moved(path)).unwrap(); - fs::rename(moved(path), path).unwrap(); - } for operation in ["stop", "deprovision"] { let output = run(&root, &lifecycle(operation, &root, "state")); assert!( @@ -2015,15 +1998,14 @@ fn start_hands_openvmm_the_images_that_it_checked() { text(&output.stderr) ); } + assert_eq!(fs::read(&distro).unwrap(), b"distro\n"); + assert_eq!(fs::read(&scratch).unwrap(), b"scratch\n"); } #[test] -fn a_start_that_ends_before_openvmm_opens_the_images_leaves_it_the_checked_ones() { - // A start that ends after it spawned OpenVMM and before OpenVMM opened its images, as when the - // CLI crashes, closes its checked files early: on Windows, the copies of them that OpenVMM - // inherited keep anything from replacing the images at their final paths until OpenVMM exits; - // on Linux, OpenVMM opens the checked files through the descriptors that it inherited, - // whatever has their paths. The sandbox's next transition ends that OpenVMM. +fn an_interrupted_start_keeps_its_claim_until_recovery_confirms_exit() { + // Image paths remain caller-owned and stable even after the CLI dies. The inherited scratch + // claim, unlike a path guard, must survive that death until verified recovery ends the VM. let root = Root::new(); // OpenVMM opens its images seconds after it starts, and a guest that answers after a minute // keeps the start waiting. @@ -2070,23 +2052,16 @@ fn a_start_that_ends_before_openvmm_opens_the_images_leaves_it_the_checked_ones( .join("build") .join("fake-openvmm-state-blocks.json"); assert!(!blocks.exists()); - for (path, replacement) in [ - (&distro, "replaced distro\n"), - (&scratch, "replaced scratch\n"), - ] { - let new = path.with_extension("new"); - fs::write(&new, replacement).unwrap(); - let replaced = fs::rename(&new, path); - assert_eq!( - replaced.is_ok(), - cfg!(unix), - "{}: {replaced:?}", - path.display() - ); - } - - // OpenVMM opens the objects that the start checked, and dumps what they held while the start's - // OpenVMM still runs: the dump is whole once it parses, which no prefix of it does. + let scratch_binding: nvx::host::sandbox::scratch::ScratchBinding = + serde_json::from_value(root.json("state/config.json")["scratch_binding"].clone()).unwrap(); + let error = scratch_binding + .check(&scratch, Instant::now() + Duration::from_millis(100)) + .unwrap_err(); + assert_eq!(error.code(), nvx::ErrorCode::ScratchClaimed); + assert_eq!(fs::read(&distro).unwrap(), b"distro\n"); + assert_eq!(fs::read(&scratch).unwrap(), b"scratch\n"); + + // The surviving VM opens the same stable caller paths. let opened = wait_for_json(&blocks); let opened: Vec<(&str, &str)> = opened .as_array() @@ -2101,20 +2076,22 @@ fn a_start_that_ends_before_openvmm_opens_the_images_leaves_it_the_checked_ones( .collect(); assert_eq!(opened, [("distro", "distro\n"), ("scratch", "scratch\n")]); - // The next transition ends the start, whose OpenVMM it verifies through the journal, and the - // copies of the checked files go with OpenVMM: on Windows, the images can be renamed again. + // The next transition verifies the journal's process and proves exit before clearing it. assert_eq!( transition(&root, "stop", "state", &[]), failed("sandbox-not-running", "sandbox is not running") ); assert!(process.has_exited().unwrap()); - #[cfg(windows)] - for path in [&distro, &scratch] { - let moved = path.with_extension("moved"); - fs::rename(path, &moved).unwrap(); - fs::rename(&moved, path).unwrap(); - } + assert!(!journal.exists()); + assert!(!root.below("state/control.capability").exists()); + assert!( + scratch_binding + .check(&scratch, Instant::now() + Duration::from_secs(2)) + .is_ok() + ); assert_eq!(transition(&root, "deprovision", "state", &[]), succeeded()); + assert_eq!(fs::read(&distro).unwrap(), b"distro\n"); + assert_eq!(fs::read(&scratch).unwrap(), b"scratch\n"); } /// Makes `handle` inheritable, or not, as code that spawns without a handle list does around its @@ -2745,6 +2722,37 @@ fn run_shares_relative_roots_with_absolute_policy_paths() { ); } +#[test] +fn an_endpoint_closed_during_start_keeps_the_vms_own_failure_report() { + let root = Root::new(); + let output = run( + &root, + &provision( + &root, + "state", + "fake_exit_during_attach=5 fake_exit_delay_ms=200", + ), + ); + assert!(output.status.success(), "{}", text(&output.stderr)); + let output = run(&root, &lifecycle("start", &root, "state")); + assert_eq!(output.status.code(), Some(1)); + assert_eq!( + text(&output.stderr), + error_line( + "exited", + "OpenVMM exited during startup: guest-exit status 5" + ) + ); + let state = root.path.join("state"); + let report: Value = + serde_json::from_slice(&fs::read(state.join("outcome.json")).unwrap()).unwrap(); + assert_eq!(report["outcome"]["status_code"], 5); + assert!(!state.join("runtime.json").exists()); + assert!(!state.join("launch.json").exists()); + let output = run(&root, &lifecycle("deprovision", &root, "state")); + assert!(output.status.success(), "{}", text(&output.stderr)); +} + #[cfg(unix)] mod interrupts { use std::time::Instant; @@ -2830,6 +2838,75 @@ mod interrupts { assert!(elapsed < Duration::from_secs(10), "{elapsed:?}"); } + #[test] + fn interrupting_start_bounds_connect_attach_and_failure_grace() { + for cmdline in [ + "fake_boot_delay_ms=60000", + "fake_guest_delay_ms=60000", + "fake_exit_during_attach=5 fake_exit_delay_ms=60000", + ] { + let root = Root::new(); + let output = run(&root, &provision(&root, "state", cmdline)); + assert!(output.status.success(), "{}", text(&output.stderr)); + let mut arguments = lifecycle("start", &root, "state"); + arguments.extend(["--timeout".to_owned(), "60".to_owned()]); + let child = Command::new(NVX_RS) + .arg("--root") + .arg(&root.path) + .args(arguments) + .current_dir(&root.path) + .env_remove("NVX_OPENVMM") + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let state = root.path.join("state"); + let until = Instant::now() + Duration::from_secs(30); + let reference: nvx::host::process::ProcessRef = loop { + if let Ok(bytes) = fs::read(state.join("launch.json")) { + let journal: Value = serde_json::from_slice(&bytes).unwrap(); + if !journal["process"].is_null() { + break serde_json::from_value(journal["process"].clone()).unwrap(); + } + } + assert!(Instant::now() < until, "{cmdline}"); + thread::sleep(Duration::from_millis(10)); + }; + let nvx::host::process::Opened::Running(process) = + nvx::host::process::Process::open(&reference).unwrap() + else { + panic!("the journal's VM already exited: {cmdline}"); + }; + thread::sleep(Duration::from_millis(200)); + let started = Instant::now(); + nix::sys::signal::kill( + nix::unistd::Pid::from_raw(i32::try_from(child.id()).unwrap()), + nix::sys::signal::Signal::SIGINT, + ) + .unwrap(); + let output = child.wait_with_output().unwrap(); + assert_eq!( + output.status.code(), + Some(130), + "{cmdline}: {}", + text(&output.stderr) + ); + assert_eq!(text(&output.stderr), "Interrupted\n", "{cmdline}"); + assert!(started.elapsed() < Duration::from_secs(3), "{cmdline}"); + assert!( + process + .wait(Instant::now() + Duration::from_secs(2)) + .unwrap() + .is_some() + ); + assert!(!state.join("runtime.json").exists()); + assert!(!state.join("launch.json").exists()); + let output = run(&root, &lifecycle("deprovision", &root, "state")); + assert!(output.status.success(), "{}", text(&output.stderr)); + } + } + #[test] fn interrupting_a_cold_boot_ends_it_without_guidance() { // The fake OpenVMM delays its boot, which Ctrl-C ends, alone or with OpenVMM, as the diff --git a/nvx-rs/tests/cli/formats.rs b/nvx-rs/tests/cli/formats.rs index bf0178ffd..5c924ce4c 100644 --- a/nvx-rs/tests/cli/formats.rs +++ b/nvx-rs/tests/cli/formats.rs @@ -17,7 +17,7 @@ use serde_json::{Value, json}; use crate::harness::{HYPERVISOR, Root, UUID, entries, in_parallel, json}; use crate::lifecycle::{PROVISIONED, provision, sandbox}; -use crate::openvmm::{Options, WORKLOAD, assert_checked_blocks, assert_opened_images, managed}; +use crate::openvmm::{Options, WORKLOAD, assert_opened_images, assert_stable_blocks, managed}; /// What a start does with a configuration of a format. enum Start { @@ -209,9 +209,9 @@ fn configurations_of_every_format_start_and_exec() { .to_owned(); let mut options = Options::parse(&root.openvmm_arguments("build/fake-openvmm-state.json")); - assert_checked_blocks(&root, &options.take("--microvm-sandbox-block")); + assert_stable_blocks(&root, &options.take("--microvm-sandbox-block"), true); assert_opened_images(&root, "state"); - let mut expected = openvmm(managed("state", &endpoint)); + let mut expected = openvmm(managed("state", &endpoint)).persistent_expected(); expected.take("--microvm-sandbox-block"); expected.assert_matches(&root, &options, name); let mapped = root.below("build/fake-openvmm-state-maps.json"); diff --git a/nvx-rs/tests/cli/lifecycle.rs b/nvx-rs/tests/cli/lifecycle.rs index ce2ac6b58..d3797de27 100644 --- a/nvx-rs/tests/cli/lifecycle.rs +++ b/nvx-rs/tests/cli/lifecycle.rs @@ -24,8 +24,8 @@ use crate::harness::{ in_parallel, is_hex, json, }; use crate::openvmm::{ - NO_NETWORK, NetworkPolicy, Options, WORKLOAD, assert_checked_blocks, assert_checked_distro, - assert_endpoint, assert_opened_distro, assert_opened_images, assert_outcome_report, managed, + NO_NETWORK, NetworkPolicy, Options, WORKLOAD, assert_endpoint, assert_opened_distro, + assert_opened_images, assert_outcome_report, assert_stable_blocks, managed, }; /// The entries of the state directory of a sandbox that never started: its configuration and @@ -134,23 +134,22 @@ pub(crate) fn assert_bindings(root: &Root, mut config: Value) -> Value { config } -/// Asserts that `record`, the runtime record of the sandbox in the state directory `state`, below -/// the root, is the store's format 4 for an OpenVMM that runs: with a launch ID, the OpenVMM -/// process by its ID, start time, and the host's boot, the control endpoint, and the runtime -/// identity that the build provenance of the root names, with this build's contracts. Returns the -/// endpoint and the process. +/// Checks the CLI's exact saved VM, its verified process and its installed runtime identity. fn assert_runtime_record(root: &Root, state: &str, record: &Value) -> (String, Process) { let mut names: Vec<&String> = record.as_object().unwrap().keys().collect(); names.sort(); assert_eq!( names, - ["endpoint", "format", "launch", "process", "runtime"], + ["capability", "endpoint", "process", "runtime", "vmDir"], "{record}" ); - assert_eq!(record["format"], 4, "{record}"); - assert!( - record["launch"].as_str().is_some_and(|id| is_hex(id, 32)), - "{record}" + assert_eq!( + Path::new(record["vmDir"].as_str().unwrap()), + root.below(state) + ); + assert_eq!( + serde_json::from_value::>(record["capability"].clone()).unwrap(), + fs::read(root.below(state).join("control.capability")).unwrap() ); let reference: ProcessRef = serde_json::from_value(record["process"].clone()).unwrap(); let Opened::Running(process) = Process::open(&reference).unwrap() else { @@ -584,9 +583,9 @@ fn check_lifecycle(case: &Configuration) { assert_capability(&state); let mut options = Options::parse(&root.openvmm_arguments(&format!("build/fake-openvmm-{token}.json"))); - assert_checked_blocks(&root, &options.take("--microvm-sandbox-block")); + assert_stable_blocks(&root, &options.take("--microvm-sandbox-block"), true); assert_opened_images(&root, token); - let mut expected = (case.openvmm)(managed(case.state, &endpoint)); + let mut expected = (case.openvmm)(managed(case.state, &endpoint)).persistent_expected(); expected.take("--microvm-sandbox-block"); expected.assert_matches(&root, &options, &context); let maps = root.below(&format!("build/fake-openvmm-{token}-maps.json")); @@ -612,7 +611,7 @@ fn check_lifecycle(case: &Configuration) { assert_outcome_report( &json(&state.join("outcome.json")), HYPERVISOR, - ("managed", "success", 0), + ("run", "success", 0), case.network, ); } @@ -678,9 +677,9 @@ fn lifecycle_of_a_sandbox_whose_upper_is_in_ram() { let (endpoint, process) = assert_runtime_record(&root, "state", &json(&state.join("runtime.json"))); let mut options = Options::parse(&root.openvmm_arguments("build/fake-openvmm-state.json")); - assert_checked_distro(&root, &options.take("--microvm-sandbox-block")); + assert_stable_blocks(&root, &options.take("--microvm-sandbox-block"), false); assert_opened_distro(&root, "state"); - let mut expected = managed("state", &endpoint).in_ram(); + let mut expected = managed("state", &endpoint).in_ram().persistent_expected(); expected.take("--microvm-sandbox-block"); expected.assert_matches(&root, &options, &context); let outcome = operate( @@ -703,7 +702,7 @@ fn lifecycle_of_a_sandbox_whose_upper_is_in_ram() { assert_outcome_report( &json(&state.join("outcome.json")), HYPERVISOR, - ("managed", "success", 0), + ("run", "success", 0), NO_NETWORK, ); } @@ -753,7 +752,7 @@ fn stops_keep_an_outcome_report_that_the_caller_reads_whatever_its_umask() { assert_outcome_report( &json(&report), HYPERVISOR, - ("managed", "success", 0), + ("run", "success", 0), NO_NETWORK, ); step("deprovision", &[]).assert_silent_success(); @@ -1171,7 +1170,7 @@ fn failed_start(root: &Root) { root, &outcome, "exited", - &["OpenVMM exited during startup", "status 5"], + &["OpenVMM exited during startup", "without an outcome report"], ); assert!( elapsed < Duration::from_secs(60), diff --git a/nvx-rs/tests/cli/openvmm.rs b/nvx-rs/tests/cli/openvmm.rs index 48360f4cf..48b0adb9c 100644 --- a/nvx-rs/tests/cli/openvmm.rs +++ b/nvx-rs/tests/cli/openvmm.rs @@ -186,6 +186,67 @@ impl Options { panic!("{context}: the OpenVMM options differ:\n{differences}"); } + /// Projects the old persistent-case inputs onto the CLI-owned managed boot contract. + pub(crate) fn persistent_expected(mut self) -> Self { + let identities = self.take("--microvm-workload-identity"); + let [identity] = identities.as_slice() else { + panic!("a persistent fixture names one workload identity"); + }; + assert_eq!(self.take("--microvm-lifecycle"), ["managed"]); + let line = self.value("--cmdline").to_owned(); + let tokens: Vec<_> = line.split_ascii_whitespace().collect(); + let hostname = tokens + .iter() + .find_map(|token| token.strip_prefix("nvx_hostname=")) + .unwrap(); + let mut kernel: Vec = tokens + .iter() + .filter(|token| { + token.starts_with("nvx_layer=") + || token.starts_with("nvx_scratch=") + || token.starts_with("nvx_overlay_upper=") + }) + .map(|token| (*token).to_owned()) + .collect(); + kernel.push("nvx_overlay_mode=trusted-lowers".to_owned()); + kernel.push( + if tokens.contains(&RAM_UPPER) { + "nvx_ram_upper_percent=50" + } else { + "nvx_scratch_journal=allowed" + } + .to_owned(), + ); + kernel.push(format!( + "nvx_hostname={}", + hostname + .bytes() + .map(|byte| format!("{byte:02x}")) + .collect::() + )); + kernel.push("nvx_machine_id=generated".to_owned()); + kernel.extend( + tokens + .iter() + .filter(|token| token.starts_with("nvx_maps=")) + .map(|token| (*token).to_owned()), + ); + kernel.extend([ + "nvx_sandbox=1".to_owned(), + "nvx_mode=managed".to_owned(), + format!("nvx_identity={identity}"), + ]); + kernel.extend( + tokens + .iter() + .filter(|token| !token.starts_with("nvx_")) + .map(|token| (*token).to_owned()), + ); + self.with("--processors", ["1"]) + .with("--guest-shutdown-action", ["exit"]) + .kernel(&kernel) + } + /// Projects an old cold-run fixture onto the checked public VM contract, independently of /// the CLI parser and its private token renderer. Restore and sandbox fixtures stay old. pub(crate) fn cold_run_expected(&self, root: &Root, mode: &str) -> nvx::Result { @@ -694,6 +755,20 @@ pub(crate) fn share_token(index: usize, target: &str, access: &str, file: bool) ) } +/// Asserts that `blocks`, the `--microvm-sandbox-block` values of a launch, attach the distro +/// and optional scratch at the stable caller-owned paths, not inherited descriptors. +pub(crate) fn assert_stable_blocks(root: &Root, blocks: &[String], scratch: bool) { + let mut expected = vec![root.expand("distro:file:{root/distro.erofs},ro")]; + if scratch { + expected.push(root.expand("scratch:file:{root/scratch.ext4}")); + } + let actual: Vec<_> = blocks + .iter() + .map(|block| block.replace("file:\\\\?\\", "file:")) + .collect(); + assert_eq!(actual, expected); +} + /// Asserts that `blocks`, the `--microvm-sandbox-block` values of a launch, attach the distro /// layer of the root read-only and then its scratch image through files that the launch checked: /// on Linux, descriptors that OpenVMM inherits, which it names as `/proc/self/fd/N`, and on diff --git a/scripts/nvx_tools/managed_exec_tests.py b/scripts/nvx_tools/managed_exec_tests.py index b45baa5c5..a2270c444 100644 --- a/scripts/nvx_tools/managed_exec_tests.py +++ b/scripts/nvx_tools/managed_exec_tests.py @@ -412,6 +412,9 @@ def persist_evidence() -> None: ) expect_rejection(rejected, "argument-invalid") + # The guest agent refuses a workload whose working directory it cannot + # enter before the workload runs, and nvx-rs reports that refusal with + # its `exec-refused` code and the agent's `cwd-failed` category. for cwd in ("/does-not-exist", "/etc/passwd", "/root"): failed = workload( "/bin/sh", @@ -419,11 +422,14 @@ def persist_evidence() -> None: "--arg=printf workload-must-not-run", "--cwd", cwd, - expected=125, + expected=1, ) + refusal = nvx_rs.reported_error(failed.stderr) if ( failed.stdout - or b"cannot use working directory" not in failed.stderr + or refusal is None + or refusal.code != "exec-refused" + or "category=cwd-failed" not in refusal.message ): raise RuntimeError( "invalid CWD did not fail before workload execution" diff --git a/scripts/nvx_tools/sandbox_lifecycle_tests.py b/scripts/nvx_tools/sandbox_lifecycle_tests.py index c6cf3dd34..f0b7fe7b9 100644 --- a/scripts/nvx_tools/sandbox_lifecycle_tests.py +++ b/scripts/nvx_tools/sandbox_lifecycle_tests.py @@ -20,7 +20,6 @@ import os import shutil import signal -import struct import subprocess import sys import tempfile @@ -66,14 +65,14 @@ FOREIGN_CONTENT = "operator data\n" EVIDENCE_PREFIX = "sandbox-lifecycle-" EVIDENCE_LIMIT = 64 * 1024 -# ext4 sets this incompatible feature while its journal needs recovery. -EXT4_INCOMPAT_RECOVER = 0x4 -MANAGED_SUCCESS = {"operation": "managed", "category": "success", "status_code": 0} +# A sandbox's VM starts without an OpenVMM lifecycle option, since nvx-init +# owns its lifecycle, so OpenVMM reports its operation as a run. +MANAGED_SUCCESS = {"operation": "run", "category": "success", "status_code": 0} # The guest powers off with status 125 when it refuses the workload identity, # and a managed start reports the status from OpenVMM's outcome report. REFUSAL_CATEGORY = "guest-exit" MANAGED_REFUSAL = { - "operation": "managed", + "operation": "run", "category": REFUSAL_CATEGORY, "status_code": 125, } @@ -113,7 +112,7 @@ def __str__(self) -> str: OUTCOME = "outcome.json" LOCK = "sandbox.lock" # The fields of the store's runtime record. -RUNTIME_FIELDS = frozenset({"format", "launch", "process", "endpoint", "runtime"}) +RUNTIME_FIELDS = frozenset({"capability", "endpoint", "process", "runtime", "vmDir"}) PROVISIONED_ENTRIES = frozenset({CONFIG, LOCK}) # Windows hosts serve the control endpoint as a named pipe outside the state. RUNNING_ENTRIES = frozenset( @@ -285,15 +284,6 @@ def kill_openvmm(runtime: dict[str, Any], timeout: float, marker: str) -> None: time.sleep(0.05) -def scratch_journal_clean(path: Path) -> bool: - with path.open("rb") as image: - image.seek(1024 + 0x60) - incompatible = image.read(4) - if len(incompatible) != 4: - raise RuntimeError(f"scratch superblock is truncated: {path}") - return not struct.unpack(" list[str] | None: if not state.is_dir(): return None @@ -384,6 +374,33 @@ def read_runtime(state: Path) -> dict[str, Any]: runtime = cast(dict[str, Any], value) if frozenset(runtime) != RUNTIME_FIELDS: raise RuntimeError(f"sandbox runtime record has unexpected fields: {path}") + vm_dir = runtime["vmDir"] + capability = runtime["capability"] + if not isinstance(capability, list): + raise RuntimeError(f"sandbox runtime record has an invalid capability: {path}") + values = cast(list[object], capability) + if len(values) != 32 or any( + type(byte) is not int or not 0 <= byte <= 255 for byte in values + ): + raise RuntimeError(f"sandbox runtime record has an invalid capability: {path}") + try: + if ( + not isinstance(vm_dir, str) + or not Path(vm_dir).is_absolute() + or not Path(vm_dir).samefile(state) + ): + raise RuntimeError( + f"sandbox runtime record names another VM directory: {path}" + ) + installed = (state / CAPABILITY).read_bytes() + except OSError as error: + raise RuntimeError( + f"sandbox runtime record ownership cannot be checked: {path}" + ) from error + if bytes(cast(list[int], values)) != installed: + raise RuntimeError( + f"sandbox runtime record capability differs from its owned file: {path}" + ) return runtime @@ -730,8 +747,14 @@ def start() -> subprocess.CompletedProcess[bytes]: def require_started(self, description: str) -> dict[str, Any]: self.require_entries(self.state, RUNNING_ENTRIES, f"after the {description}") runtime = read_runtime(self.state) + evidence = { + name: value for name, value in runtime.items() if name != "capability" + } + evidence["capabilitySha256"] = hashlib.sha256( + bytes(runtime["capability"]) + ).hexdigest() self.save_evidence( - "runtime.json", (json.dumps(runtime, indent=2) + "\n").encode() + "runtime.json", (json.dumps(evidence, indent=2) + "\n").encode() ) if not openvmm_running(runtime, self.root.name): raise RuntimeError(f"OpenVMM is not running after the {description}") @@ -777,10 +800,10 @@ def check_warm_state(self) -> None: self.require_output(result, STATE_CONTENT, "the state-reading request") def check_security_profile(self) -> None: - # The earlier requests already entered their own namespaces, so the - # probe's check that each container mount appears once also shows - # that they did not mount into the agent's namespace. - result = self.exec(PROBE, "--arg=entry") + # nvx-init runs each request as a plain process in the VM's root, so + # the probe's check that each mount appears once also shows that the + # earlier requests mounted nothing there. + result = self.exec(PROBE, "--arg=init", "--arg=entry") self.save_evidence("probe.log", result.stdout + result.stderr) for marker in ( b"NVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534", @@ -855,6 +878,8 @@ def check_deprovision_while_running(self, runtime: dict[str, Any]) -> None: ) def stop(self, runtime: dict[str, Any], layer_digest: str) -> None: + # nvx-init syncs the file systems and powers the VM off with them + # mounted, so the restart that reads the state back checks the stop. self.invoke("stop", state=self.state) self.require_entries(self.state, STOPPED_ENTRIES, "after stop") if openvmm_running(runtime, self.root.name): @@ -864,10 +889,6 @@ def stop(self, runtime: dict[str, Any], layer_digest: str) -> None: self.require_processes((), "after stop") self.copy_evidence(self.state / OUTCOME, "outcome.json") check_managed_report(managed_report(self.state), self.backend) - # A clean journal shows that stop unmounted the overlay and scratch - # before the VM powered off. - if not scratch_journal_clean(self.scratch): - raise RuntimeError("stop left the scratch file system mounted") if sha256_file(self.artifacts.distro) != layer_digest: raise RuntimeError("the running sandbox modified the EROFS layer") for operation in ("exec", "stop"): diff --git a/scripts/nvx_tools/sandbox_ram_upper_tests.py b/scripts/nvx_tools/sandbox_ram_upper_tests.py index 0128c142d..0352ba015 100644 --- a/scripts/nvx_tools/sandbox_ram_upper_tests.py +++ b/scripts/nvx_tools/sandbox_ram_upper_tests.py @@ -203,7 +203,7 @@ def check_managed(self) -> None: }: raise RuntimeError(f"provision recorded the RAM upper as {recorded!r}") self.start(description="first start") - result = self.exec(PROBE, "--arg=entry") + result = self.exec(PROBE, "--arg=init", "--arg=entry") self.save_evidence("probe.log", result.stdout + result.stderr) for marker in probe_markers(entry=True): if marker not in result.stdout: diff --git a/scripts/test-layout-allowlist.json b/scripts/test-layout-allowlist.json index a3211e68a..f92d59c85 100644 --- a/scripts/test-layout-allowlist.json +++ b/scripts/test-layout-allowlist.json @@ -12,7 +12,6 @@ "nvx-rs/src/guest/sys/spawn.rs": 7, "nvx-rs/src/guest/time/system.rs": 3, "nvx-rs/src/host/cli/sandbox/state.rs": 1, - "nvx-rs/src/host/cli/store/mod.rs": 1, "nvx-rs/src/host/network/address.rs": 2, "nvx-rs/src/host/platform/windows/spawn.rs": 1, "nvx-rs/src/host/sandbox/plan.rs": 5, diff --git a/scripts/test_microvm_tests.py b/scripts/test_microvm_tests.py index 1ab994326..1688fddde 100644 --- a/scripts/test_microvm_tests.py +++ b/scripts/test_microvm_tests.py @@ -569,8 +569,11 @@ def run( returncode = 124 category = "timeout" elif cwd in ("/does-not-exist", "/etc/passwd", "/root"): - returncode = 125 - stderr = b"cannot use working directory\n" + returncode = 1 + stderr = ( + b"error[exec-refused]: managed guest rejected exec " + b"(status=2, category=cwd-failed)\n" + ) elif entrypoint == "/bin/sh": stdout = b"public stdout" stderr = b"public stderr" @@ -1218,8 +1221,8 @@ def _launch(self, state: Path) -> _CliResult: (state / "runtime.json").write_text( json.dumps( { - "format": 4, - "launch": f"{pid:032x}", + "capability": list(capability), + "vmDir": str(state.resolve()), "process": {"pid": pid, "startTime": pid * 7, "boot": "b" * 32}, "endpoint": endpoint, "runtime": { @@ -1284,7 +1287,7 @@ def _exec( returncode = 1 stderr = b"NVX-UBUNTU-SANDBOX-FAIL: the workload shares a namespace\n" else: - if arguments != ["entry"]: + if arguments != ["init", "entry"]: raise AssertionError(f"unexpected probe arguments {arguments}") stdout = ( b"NVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534\n" @@ -1441,6 +1444,40 @@ def terminate(self, pid: int) -> None: class PublicSandboxLifecycleTests(unittest.TestCase): + def test_runtime_reader_checks_the_exact_saved_vm_and_its_owned_files(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + state = root / "state" + state.mkdir() + (state / "config.json").write_text( + json.dumps({"user": "65534:65534", "hypervisor": "whp"}), + encoding="utf-8", + ) + fake = _FakeSandboxCli() + fake._launch(state) + path = state / "runtime.json" + saved = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(sandbox_lifecycle_tests.read_runtime(state), saved) + other = root / "other" + other.mkdir() + old = { + name: value + for name, value in saved.items() + if name not in ("capability", "vmDir") + } | {"format": 4, "launch": "1" * 32} + mismatched = saved["capability"].copy() + mismatched[0] ^= 1 + for record, message in ( + (old, "unexpected fields"), + (saved | {"vmDir": str(other)}, "another VM directory"), + (saved | {"capability": [0] * 31}, "invalid capability"), + (saved | {"capability": mismatched}, "differs from its owned file"), + ): + with self.subTest(message=message): + path.write_text(json.dumps(record), encoding="utf-8") + with self.assertRaisesRegex(RuntimeError, message): + sandbox_lifecycle_tests.read_runtime(state) + EXPECTED_OPERATIONS = [ # Identity admission. "provision", @@ -1627,6 +1664,15 @@ def test_acceptance_drives_the_complete_public_lifecycle(self): for content in evidence.values(): self.assertNotIn(capability, content) self.assertNotIn(capability.hex().encode(), content) + saved_evidence = json.loads(evidence["sandbox-lifecycle-runtime.json"]) + self.assertNotIn("capability", saved_evidence) + self.assertIn( + saved_evidence["capabilitySha256"], + [ + hashlib.sha256(capability).hexdigest() + for capability in fake.capabilities + ], + ) self.assertIn( b"NVX-UBUNTU-SANDBOX-ENTRY-OK", evidence["sandbox-lifecycle-probe.log"] ) @@ -1779,19 +1825,6 @@ def test_an_interrupted_acceptance_stops_its_sandboxes(self): else: self.assertEqual(stderr.getvalue(), "") - def test_scratch_journal_check_reads_the_ext4_recovery_flag(self): - with tempfile.TemporaryDirectory() as temporary: - image = Path(temporary) / "scratch.ext4" - superblock = bytearray(2048) - image.write_bytes(superblock) - self.assertTrue(sandbox_lifecycle_tests.scratch_journal_clean(image)) - superblock[1024 + 0x60] = sandbox_lifecycle_tests.EXT4_INCOMPAT_RECOVER - image.write_bytes(superblock) - self.assertFalse(sandbox_lifecycle_tests.scratch_journal_clean(image)) - image.write_bytes(bytes(1030)) - with self.assertRaisesRegex(RuntimeError, "truncated"): - sandbox_lifecycle_tests.scratch_journal_clean(image) - def test_kill_ends_the_recorded_process(self): child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py index 1c0cd68b0..1348a34dd 100644 --- a/scripts/test_nvx_tools.py +++ b/scripts/test_nvx_tools.py @@ -20316,7 +20316,7 @@ def run( + " esac; }\n" ) return self._run( - ("status_field", "check_namespaces"), + ("status_field", "check_hostname", "check_namespaces"), f"{readlink}check_namespaces\necho checked", { "/proc/1/status": self._fixture("init-status", init_status), @@ -20371,6 +20371,30 @@ def run( self.assertEqual(result.returncode, 1, result.stdout) self.assertIn("expected nvx-lifecycle", result.stderr) + def test_init_hostname_check_reads_the_encoded_command_line(self): + def run(command_line: str, hostname: str) -> subprocess.CompletedProcess[str]: + return self._run( + ("check_hostname",), + "nvx_init=true\ncheck_hostname\necho checked", + { + "/proc/cmdline": self._fixture("cmdline", command_line + "\n"), + "/proc/sys/kernel/hostname": self._fixture( + "hostname", hostname + "\n" + ), + }, + ) + + # nvx-lifecycle, as nvx-init's command line encodes it. + encoded = "nvx_hostname=6e76782d6c6966656379636c65" + result = run( + f"console=hvc0 {encoded} nvx_machine_id=generated", "nvx-lifecycle" + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "checked\n") + result = run(f"console=hvc0 {encoded}", "nvx-sandbox") + self.assertEqual(result.returncode, 1, result.stdout) + self.assertIn("expected 6e76782d6c6966656379636c65", result.stderr) + def _check_mounts( self, mountinfo: str, command_line: str ) -> subprocess.CompletedProcess[str]: @@ -20673,6 +20697,7 @@ def test_main_body_runs_the_profile_then_limits_and_shares(self): "check_identity", "check_capabilities", "check_namespaces", + "check_hostname", "check_mounts", "check_devices", "check_device_creation", @@ -20729,6 +20754,28 @@ def run(*arguments: str) -> subprocess.CompletedProcess[str]: result = run("/workspace") self.assertEqual(result.returncode, 0, result.stderr) self.assertNotIn("check_entry ", result.stdout.splitlines()) + # An execution of nvx-init shares its namespaces and cgroup, so the + # profile checks only its host name of them. + result = run("init", "entry", "/workspace", "rw") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual( + result.stdout.splitlines(), + [ + "check_identity ", + "check_capabilities ", + "check_hostname ", + "check_mounts ", + "check_devices ", + "check_device_creation ", + "check_machine_id ", + "check_ports ", + "check_image ", + "NVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534", + "check_entry ", + "check_share /workspace rw", + "NVX-UBUNTU-SANDBOX-OK uid=65534 gid=65534", + ], + ) result = run("limits", "33554432") self.assertEqual(result.returncode, 1, result.stdout) self.assertIn("limits requires MEMORY_MAX and PIDS_MAX", result.stderr)