tests: capture and restore named snapshots end to end on real VMs (SL5.1c)

.github/workflows/run-nvx-microvm-tests.yml

@@ -239,6 +239,8 @@ jobs:
           --scenario sandbox-ram-upper
           --scenario sandbox-gpt-layer
           --scenario vm-lifecycle
+          --scenario snapshot-core
+          --scenario snapshot-tiers
           --output-dir "build/test-results/public-exec-${{ inputs.backend }}"
 
       - name: Run public managed sandbox acceptance on Windows
@@ -253,6 +255,8 @@ jobs:
           --scenario sandbox-ram-upper
           --scenario sandbox-gpt-layer
           --scenario vm-lifecycle
+          --scenario snapshot-core
+          --scenario snapshot-tiers
           --output-dir "build/test-results/public-exec-${{ inputs.backend }}"
 
       - name: Run Ubuntu guest smoke tests on Linux
@@ -331,7 +335,6 @@ jobs:
           --scenario managed-lifecycle
           --scenario network-snapshot
           --scenario smp
-          --scenario snapshot-core
           --scenario workload-identity
           --output-dir "build/test-results/ubuntu-kvm-extended"
 

doc/ci.md

@@ -234,21 +234,18 @@ a cold boot that declares that profile, and passing boot checks. It then
 captures the guest and restores it with `auto` and no hook: the restore must
 take the snapshot's host profile without another warning, and pass the warp
 probe and its restore checks.
-The `snapshot-core` scenario first sends a snapshot request to a guest that
-OpenVMM launched without a snapshot destination. OpenVMM releases the request,
-so it returns in the source, which must continue exactly once. Before the
-request, the guest's snapshot agent saved and overrode the stall detectors'
-settings, and it must restore them when the request returns
-(`nvx-time cancel-capture`). Afterwards, `nvx-time status` must exit 0 with a
-passing boot line at generation 0 and no restore line, and
-`/sys/module/rcupdate/parameters/rcu_cpu_stall_suppress` must read 0.
 The `snapshot-rollback` scenario fails a capture after its quiesce. The
 guest's RAM has a file of its own, so the snapshot destination's parent holds
 nothing, and it disappears after boot: the capture quiesces and saves the VM,
 then fails to publish, and OpenVMM rolls it back. The request must return in
 the source with status 0, after OpenVMM's capture profile records exactly one
 quiesce and one save but no publication, and the source must pass the checks
-that `snapshot-core` runs after its released request. Once the parent is back,
+of a released request. Before the request, the guest's snapshot agent saved
+and overrode the stall detectors' settings, and it must restore them when the
+request returns (`nvx-time cancel-capture`): `nvx-time status` must then exit 0
+with a passing boot line at generation 0 and no restore line, and
+`/sys/module/rcupdate/parameters/rcu_cpu_stall_suppress` must read 0. Once the
+parent is back,
 a second request of the same source must publish a snapshot and end the
 source with status 0, leaving only the snapshot in the parent, and the
 snapshot's restore must continue past that request exactly once.
@@ -350,8 +347,8 @@ line and adds it to the GitHub job summary, because CI keeps the guest logs only
 for failed jobs: the number of warp probe runs with their worst
 `max_abs_offset_ns` and `max_backward_ns`, the count and `elapsed_us` range of
 the newest check that each `nvx-time status` query reported (boot after a cold
-boot, restore after a restore, and capture after `snapshot-core`'s released
-request), the exhaustive check's summary, and the
+boot, restore after a restore, and capture after `snapshot-rollback`'s
+rolled-back request), the exhaustive check's summary, and the
 `restore-downtime` stall counts. Where the guest reports a check's CPU time
 (`cpu_us`), the line also gives its range and how many checks exceed the
 backend's CPU-time budget for their phase. The budgets come from guest
@@ -367,20 +364,18 @@ The `nvx-microvm-debug-{kvm,mshv,whp}` jobs run `test-microvm --debug-kernel`
 on the CI debug kernel (`build/vmlinux-debug`, built from
 `kernel/config-microvm-debug`), whose soft-lockup and hung-task detectors
 production kernels leave out. It selects the same-host restore scenarios
-`smp`, `smp-snapshot`, `restore-processors`, `restore-downtime`, and
-`snapshot-tiers`. The jobs skip every other test step, because those boot the
+`smp`, `smp-snapshot`, `restore-processors`, and `restore-downtime`. The jobs
+skip every other test step, because those boot the
 production kernel, which the `nvx-microvm-guests-{kvm,mshv,whp}` jobs and
 WHP's runtime adapter job already cover: the public managed execution
 configuration, the Ubuntu and Azure Linux guest tests, the Ubuntu sandbox
 layer and live-share smoke tests, the caller-owned live-share test, and the
 runtime adapter scenarios. Any
 RCU stall, soft lockup, or hung task makes the guest's time ABI watcher power
 off with status 194, which fails the run. Every guest those scenarios restore
-asks `nvx-time status` before it exits
-(`snapshot-tiers` after its tier assertions), which waits for the restore's
-deferred checks and must report a passing restore line, so no restore leaves
-its checks pending. The one exception is `snapshot-tiers`' gate-timeout check,
-whose guest OpenVMM stops at the restore gate on purpose. The harness refuses
+asks `nvx-time status` before it exits, which waits for the restore's deferred
+checks and must report a passing restore line, so no restore leaves its checks
+pending. The harness refuses
 a kernel whose `vmlinux-debug.config` lacks the detectors, because the guest's
 boot check of those detectors passes vacuously. To bound the cost, pull requests
 run the debug kernel on MSHV only, rather than on KVM, whose general runners
@@ -560,7 +555,14 @@ tmpfs in guest memory, with no scratch image, and sandboxes whose root is a
 GPT disk, over either upper, so that the three backends run the CLI's twelve
 cold cells (see the [case matrix](acceptance-case-matrix.md#cold-boots)). The
 public `vm-lifecycle` scenario then creates, spawns, reattaches to, and stops
-a VM through the public API of `nvx`.
+a VM through the public API of `nvx`. The public `snapshot-core` and
+`snapshot-tiers` scenarios capture a named snapshot with
+`nvx-rs snapshot capture`, with nvx-init as the guest, and restore it with
+`snapshot restore` and `run --restore-snapshot` (see
+[Run](run.md#named-snapshots)): restores of each image format over each upper,
+repeated restores with fresh entropy and machine IDs, clocks that follow the
+host's across the downtime, concurrent restores that take turns at the
+snapshot's lock, failed restores, and snapshots that no restore changes.
 Their bounded evidence is part of the backend's failure logs. The same
 entrypoint then verifies a live virtio-fs
 share inside the container: a read-write `/workspace` share with a denied

doc/run.md

@@ -1113,6 +1113,86 @@ reports of each run and stop. With `sandbox-lifecycle` and
 `sandbox-ram-upper`, CI runs it on KVM, MSHV, and WHP: the CLI's twelve cold
 cells, which the [case matrix](acceptance-case-matrix.md#cold-boots) lists.
 
+### Named snapshots
+
+`nvx-rs snapshot capture` captures a VM that nvx-init runs, before any
+workload, with a placeholder image, a dormant scratch, and no NIC, into a named
+snapshot of a store, which `nvx-rs snapshot restore` and
+`nvx-rs run --restore-snapshot` restore under the snapshot's lock, with an
+image and an upper of their own; see
+[`snapshot capture`](usage.md#snapshot-capture) and
+[`snapshot restore`](usage.md#snapshot-restore). The public `snapshot-core` and
+`snapshot-tiers` scenarios of `test-microvm --backend BACKEND` are their
+acceptance, with the Ubuntu layer and the scratch template that
+`nvx-rs download` installs. The workload of each restore is a probe, a
+script that `/bin/sh -c` runs, which reports the guest's clock and uptime,
+fresh random values, its machine ID, host name, and identity, the lower and
+upper directories of its root's overlay, its block devices and network
+interfaces, the distribution of its image, and the marker file of its host
+share at `/data`, if it has one.
+
+The `snapshot-core` scenario captures one snapshot, whose directory must hold
+exactly `key.bin`, `snapshot.json`, and `openvmm/`, each file with one link,
+while no directory or OpenVMM process of the capture's VM remains. After a
+downtime longer than the capture, two `run --restore-snapshot` restores of the
+Ubuntu EROFS layer, in RAM as `65534:65534` and over a fresh copy of the
+scratch template as `1:1`, each with a host name of its own, must run as the
+restore's identity with its host name, the layer as the root's lower directory
+over the upper, the snapshot's two block devices, and no NIC, with a clock
+that reads the host's time and an uptime that kept the time before the capture
+and counts the downtime, and must report different random values, kernel
+UUIDs, machine IDs, and temporary file names. A `snapshot restore` over a
+scratch image of half the dormant scratch's length is refused before any VM
+starts: its error names the lengths of the requested scratch and of the
+dormant one, which attributes the failure to the request, and its sandbox,
+which a failed start keeps provisioned, must hold `config.json` and
+`sandbox.lock` alone, where a launch leaves OpenVMM's log and outcome report
+and the view of the snapshot, until `sandbox deprovision` removes it. A
+`run --restore-snapshot` over a zeroed scratch image of the dormant scratch's
+length, which the guest cannot mount as its upper, ends its VM before its
+workload runs and frees the snapshot's lock, which the restores that follow
+take, and the run reports the guest's exit:
+`OpenVMM exited during startup: guest-exit status 125`.
+
+Then three `snapshot restore` restores start at once, each with a host name
+and a read-only host share of its own, the last over a scratch upper and the
+others in RAM, and a workload that ends only once the scenario creates a
+`release` file in its share, which the guest reads live. While the first holds
+the snapshot's lock, a `run --restore-snapshot` restore that may wait one
+second for the lock fails with `sandbox-busy` before its workload runs, and
+neither of the other two may link its view of the snapshot, which a restore
+does only once it holds the lock. Once the scenario releases the first, one of
+the other two takes the lock, and the last may link its view only once the
+scenario releases that one. The three sandboxes then run at once: each guest
+must read the marker of its own share, and a file that a `sandbox exec` writes
+in one sandbox's `/tmp`, in its upper, must appear in no other. Each must stop with an outcome report of a successful restored VM
+without a network policy and a complete teardown, and be deprovisioned. No
+restore may change the snapshot's files, leave a link to them, or leave an
+OpenVMM process.
+
+The `snapshot-tiers` scenario captures one snapshot of the platform tier, the
+one tier that `snapshot capture` captures, and restores it with
+`snapshot restore` into a managed sandbox of each image format over each
+upper: the Ubuntu EROFS layer, and a GPT disk of its tree that a guest of the
+Alpine control initramfs writes, as `sandbox-gpt-layer` does, each over a
+fresh copy of the scratch template and in RAM. Each guest must have its image
+as its root's lower directory, `/run/nvx/layers/distro` or
+`/run/nvx/layers/p2`, over its upper, and a `sandbox exec` in its sandbox must
+see the same root of the same guest. Each sandbox must stop cleanly and be
+deprovisioned, and the restores change neither image nor the snapshot.
+
+Both scenarios journal the OpenVMM processes of their managed restores, as
+the other scenarios of the `snapshot-isolation`
+[adversarial campaign](adversarial-testing.md) journal theirs. Their evidence,
+with a `snapshot-core-` or `snapshot-tiers-` prefix, holds the bounded command
+observations, the output of each command, and the outcome reports of the
+managed sandboxes, and that of `snapshot-tiers` the console of the guest that
+wrote its GPT disk. CI runs them on KVM, MSHV, and WHP with the public sandbox
+scenarios. A snapshot that `snapshot capture` captures has no NIC, so its
+restores take no network option, and the raw `untiered-network-policy`
+scenario keeps the coverage of a restore's DNS, egress, forwards, and listener
+publication until a capture can keep a NIC.
+
 ### Live host shares
 
 `sandbox run` and `sandbox provision` accept repeatable

doc/usage.md

@@ -1254,7 +1254,9 @@ which runs only when named: it repeats `smp` and also asserts that every CPU
 uses the one-shot counting LAPIC. `managed-exec-config`, `sandbox-lifecycle`,
 `sandbox-proxy-only`, `sandbox-ram-upper`, and `sandbox-gpt-layer`, the public
 sandbox scenarios, drive the public `nvx-rs sandbox` lifecycle commands (see
-[Run](run.md#managed-lifecycle)), and `vm-lifecycle` drives a VM through the
+[Run](run.md#managed-lifecycle)), `snapshot-core` and `snapshot-tiers` capture
+and restore named snapshots with `nvx-rs snapshot` and `nvx-rs run` (see
+[Run](run.md#named-snapshots)), and `vm-lifecycle` drives a VM through the
 public API of `nvx`; they also read the Ubuntu distro layer, its
 manifest, and the sandbox scratch template, which
 [`download`](#download) installs and the guest-artifact build makes; a run
@@ -1263,16 +1265,16 @@ files is missing. Alpine remains the
 default. Ubuntu and Azure Linux cannot act as sandbox control, so they reject
 the Alpine-control-only `interrupt-map`, `sandbox-blocks`, `scratch-snapshot`,
 and `virtio-features` scenarios and
-the sandbox-control-dependent `snapshot-tiers`, `platform-restore-processors`,
-`platform-zero-fill`, `platform-filesystem`, `platform-aggregate-export`,
-`platform-network-policy`, `managed-exec-config`, `sandbox-lifecycle`,
-`sandbox-proxy-only`, `sandbox-ram-upper`, `sandbox-gpt-layer`, and
-`vm-lifecycle` scenarios. Ubuntu also rejects
+the sandbox-control-dependent `snapshot-core`, `snapshot-tiers`,
+`platform-restore-processors`, `platform-zero-fill`, `platform-filesystem`,
+`platform-aggregate-export`, `platform-network-policy`, `managed-exec-config`,
+`sandbox-lifecycle`, `sandbox-proxy-only`, `sandbox-ram-upper`,
+`sandbox-gpt-layer`, and `vm-lifecycle` scenarios. Ubuntu also rejects
 the Alpine-prompt-specific `console-snapshot` scenario. `--debug-kernel` boots
 the CI debug kernel, `build/vmlinux-debug`, whose soft-lockup and hung-task
 detectors the guest's time ABI watcher reports; without `--scenario`, it runs
 only the same-host restore scenarios `smp`, `smp-snapshot`,
-`restore-processors`, `restore-downtime`, and `snapshot-tiers`.
+`restore-processors`, and `restore-downtime`.
 `--shard INDEX/COUNT`, such as `1/2`, runs shard INDEX of the COUNT shards of
 the default Alpine suite on the default kernel, the scenarios of a run without
 `--scenario` but the public sandbox scenarios, which CI runs in its guest and

nvx-rs/src/host/platform/linux/endpoint.rs

@@ -1,10 +1,10 @@
 //! The Unix sockets of OpenVMM's endpoints: its control endpoint and the transport over it, and
 //! the listener of a clone's restore-ready event.
 
-use std::fs::{self, OpenOptions, Permissions};
+use std::fs::{self, File, Permissions};
 use std::io::{self, Read};
 use std::os::fd::{AsFd, AsRawFd};
-use std::os::unix::fs::{OpenOptionsExt, PermissionsExt};
+use std::os::unix::fs::PermissionsExt;
 use std::os::unix::net::{UnixListener, UnixStream};
 use std::path::Path;
 use std::thread;
@@ -227,11 +227,16 @@ fn set_ready_socket_mode(socket_path: &Path) -> io::Result<()> {
     use std::os::unix::fs::FileTypeExt;
 
     use nix::fcntl::OFlag;
+    use nix::sys::stat::Mode;
 
-    let socket = OpenOptions::new()
-        .read(true)
-        .custom_flags((OFlag::O_PATH | OFlag::O_NOFOLLOW).bits())
-        .open(socket_path)?;
+    // `nix` passes the flags on as they are, where `OpenOptionsExt::custom_flags` masks out the
+    // bits of `O_ACCMODE`, which on musl include `O_PATH`: a musl build would open the socket
+    // itself, which the kernel refuses with `ENXIO`.
+    let socket = File::from(nix::fcntl::open(
+        socket_path,
+        OFlag::O_PATH | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC,
+        Mode::empty(),
+    )?);
     if !socket.metadata()?.file_type().is_socket() {
         return Err(io::Error::other(format!(
             "{} is not the restore-ready socket that the launch bound",

scripts/nvx_tools/adversarial_cases/snapshot-isolation.json

@@ -124,11 +124,11 @@
     {
       "id": "snapshot-tier-boundary",
       "scenario": "snapshot-tiers",
-      "description": "Exercise policy-tier dispatch and cross-instance state isolation across restores.",
+      "description": "Restore one platform-tier snapshot into sandboxes of each image format over each upper, each with a root of its own.",
       "coverage": [
         "snapshot-tiers",
-        "cross-instance-isolation",
-        "tier-validation"
+        "image-formats",
+        "cross-instance-isolation"
       ],
       "repetitions": 1,
       "timeout_multiplier": 6