From dbbd87407f7b5393fa7bc1894e7a8695070d816a Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Wed, 11 Nov 2026 15:52:21 -0800 Subject: [PATCH] host: delete the old clone path With the runtime adapter gone (SL5.2), nothing calls the old clone path, which restored a clone from a template store's leased generation: ClonePlan and CloneRequest, which planned it, the launch that returned a Restoring, the restore sequence through CloneControl, Activated and Started, the clone and failure records, with the backward scan for OpenVMM's fatal line, and the store-based reattach_clone. host::template::clone leaves with its submodules, their exports, their 120 unit tests and ClonePlan's doc test, and the public API snapshots lose its 194 items on each platform. The crate-private code that only the clone path used leaves with it, as the dead-code check of each feature set finds it on Linux and Windows: platform::is_refused, link_count on every host, Windows' listen_ready, Linux's unverified ReadyListener::receive, seal::final_volume_path, SnapshotKey::ipv6_gateway, the distro and scratch bindings' version accessors, SandboxPlan::dns_server, LockedStore's reserved_socket and owned_entries, Starting::end_keeping_journal, and LaunchFailure's spawned and unended, which only a clone's launch read; with the re-exports of launch_reporting, reattach_reporting and SOCKET_PATH_LIMIT, and the template store's test helpers that only the clone tests used. The store identity types and the host::template aliases that the template store still uses stay for SL6.2a and SL6.1. The tests of what leaves go with it: final_volume_path's own test retires, and five tests drop their assertions on is_refused, ipv6_gateway and LaunchFailure's fields. The tests of the restore-ready listeners take the paths that a restore takes: on Windows, the test binds its listener at a fresh pipe itself, now that listen_ready is gone there, and on Linux, it receives through the verified path. The docs stop naming the clone path: the rustdoc of the template module, the supervisor, the sandbox store's file names, the bindings' geometries, the restore-ready listeners, which a restore now uses alone, and the template store's clone sockets; the nvx README's module table and its template, supervisor and checked-handoff sections; and the case matrix, which retires ClonePlan's doc test, follows the template store's doc test to its new line, and names SL6.2a as the PR that retires the template store's two doc tests. Part of #1121 (SL6.2b). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- doc/acceptance-case-matrix.md | 8 +- nvx-rs/README.md | 211 +- nvx-rs/public-api/linux.txt | 194 - nvx-rs/public-api/windows.txt | 194 - nvx-rs/src/host/platform/linux/endpoint.rs | 24 +- nvx-rs/src/host/platform/linux/identity.rs | 5 - nvx-rs/src/host/platform/linux/mod.rs | 2 +- nvx-rs/src/host/platform/linux/tests.rs | 4 +- nvx-rs/src/host/platform/mod.rs | 13 +- nvx-rs/src/host/platform/other.rs | 6 +- nvx-rs/src/host/platform/tests.rs | 3 - nvx-rs/src/host/platform/windows/endpoint.rs | 17 +- nvx-rs/src/host/platform/windows/identity.rs | 7 - nvx-rs/src/host/platform/windows/mod.rs | 4 +- nvx-rs/src/host/platform/windows/tests.rs | 10 +- nvx-rs/src/host/sandbox/distro.rs | 9 +- nvx-rs/src/host/sandbox/plan.rs | 12 +- nvx-rs/src/host/sandbox/scratch.rs | 9 +- nvx-rs/src/host/seal.rs | 8 - nvx-rs/src/host/seal/windows.rs | 7 +- nvx-rs/src/host/seal/windows/tests.rs | 19 - nvx-rs/src/host/snapshot/key.rs | 11 - nvx-rs/src/host/snapshot/key/tests.rs | 15 +- nvx-rs/src/host/store/locked.rs | 33 +- nvx-rs/src/host/store/names.rs | 27 +- nvx-rs/src/host/store/regular.rs | 6 +- nvx-rs/src/host/store/tests.rs | 1 - nvx-rs/src/host/supervisor/launch.rs | 75 +- nvx-rs/src/host/supervisor/mod.rs | 59 +- nvx-rs/src/host/supervisor/tests.rs | 3 +- nvx-rs/src/host/supervisor/vmm.rs | 10 +- nvx-rs/src/host/template/clone.rs | 1449 ------- nvx-rs/src/host/template/clone/export.rs | 168 - nvx-rs/src/host/template/clone/failure.rs | 926 ---- .../src/host/template/clone/failure/tests.rs | 408 -- nvx-rs/src/host/template/clone/launch.rs | 794 ---- .../src/host/template/clone/launch/tests.rs | 8 - nvx-rs/src/host/template/clone/reattach.rs | 635 --- nvx-rs/src/host/template/clone/record.rs | 128 - nvx-rs/src/host/template/clone/records.rs | 1136 ----- .../src/host/template/clone/records/tests.rs | 511 --- nvx-rs/src/host/template/clone/restore.rs | 985 ----- nvx-rs/src/host/template/clone/tests.rs | 3037 ------------- .../src/host/template/clone/tests/launch.rs | 3820 ----------------- .../src/host/template/clone/tests/restore.rs | 1661 ------- nvx-rs/src/host/template/mod.rs | 39 - nvx-rs/src/host/template/store.rs | 1 - nvx-rs/src/host/template/store/leases.rs | 4 +- nvx-rs/src/host/template/store/tests.rs | 15 - 49 files changed, 94 insertions(+), 16637 deletions(-) delete mode 100644 nvx-rs/src/host/template/clone.rs delete mode 100644 nvx-rs/src/host/template/clone/export.rs delete mode 100644 nvx-rs/src/host/template/clone/failure.rs delete mode 100644 nvx-rs/src/host/template/clone/failure/tests.rs delete mode 100644 nvx-rs/src/host/template/clone/launch.rs delete mode 100644 nvx-rs/src/host/template/clone/launch/tests.rs delete mode 100644 nvx-rs/src/host/template/clone/reattach.rs delete mode 100644 nvx-rs/src/host/template/clone/record.rs delete mode 100644 nvx-rs/src/host/template/clone/records.rs delete mode 100644 nvx-rs/src/host/template/clone/records/tests.rs delete mode 100644 nvx-rs/src/host/template/clone/restore.rs delete mode 100644 nvx-rs/src/host/template/clone/tests.rs delete mode 100644 nvx-rs/src/host/template/clone/tests/launch.rs delete mode 100644 nvx-rs/src/host/template/clone/tests/restore.rs diff --git a/doc/acceptance-case-matrix.md b/doc/acceptance-case-matrix.md index ac9303d55..d5fa82c3b 100644 --- a/doc/acceptance-case-matrix.md +++ b/doc/acceptance-case-matrix.md @@ -32,7 +32,7 @@ its own in turn. | [The adapter's other tests](#the-adapters-other-tests) | 169 | 0 | 9 | 160 | | [MicroVM scenarios](#microvm-scenarios) | 54 | 51 | 2 | 1 | | [CI](#ci) | 69 | 43 | 3 | 23 | -| [Docs](#docs) | 36 | 35 | 1 | 0 | +| [Docs](#docs) | 36 | 34 | 1 | 1 | ## The adapter's scenarios @@ -500,9 +500,9 @@ this matrix. The README's own code is no doctest. | `nvx-rs/src/host/sandbox/prepared.rs - host::sandbox::prepared::PreparedLaunch (line 92)` | kept | Its launch type leaves with the old launch paths (#1087), whose PR retires it | | `nvx-rs/src/host/store/mod.rs - host::store (line 75)` | kept | The sandbox store leaves (#1121), whose PR retires it | | `nvx-rs/src/host/supervisor/mod.rs - host::supervisor (line 33)` | kept | The old supervisor leaves (#1087), whose PR retires it | -| `nvx-rs/src/host/template/clone.rs - host::template::clone::ClonePlan (line 247)` | kept | The old clone path leaves (#1121), whose PR retires it | -| `nvx-rs/src/host/template/plan.rs - host::template::plan::TemplatePlan (line 137)` | kept | The old clone path leaves (#1121), whose PR retires it | -| `nvx-rs/src/host/template/store.rs - host::template::store::Store (line 474)` | kept | The old clone path leaves (#1121), whose PR retires it | +| `nvx-rs/src/host/template/clone.rs - host::template::clone::ClonePlan (line 247)` | retired | `ClonePlan` and `CloneRequest` leave with the old clone path (SL6.2b, #1121) | +| `nvx-rs/src/host/template/plan.rs - host::template::plan::TemplatePlan (line 137)` | kept | The template store leaves (SL6.2a, #1121), whose PR retires it | +| `nvx-rs/src/host/template/store.rs - host::template::store::Store (line 473)` | kept | The template store leaves (SL6.2a, #1121), whose PR retires it | | `nvx-rs/src/host/transport.rs - host::transport (line 22)` | kept | | | `nvx-rs/src/lib.rs - (line 58)` | kept | | | `nvx-rs/src/protocol/nvxs/mod.rs - protocol::nvxs (line 68)` | kept | | diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 87ddc0642..b66f3979c 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -139,7 +139,7 @@ changes only when the module's purpose does. | [`nvx::host::transport`](#nvxhosttransport) | A verified VM endpoint connection and the blocking host NVXS driver, with the existing timed transport | | [`nvx::host::process`](#nvxhostprocess) | Verified references to OpenVMM processes, through which callers wait for them and end them | | [`nvx::host::supervisor`](#nvxhostsupervisor) | The supervisor of a sandbox's VMM, beneath the CLI, `Instance`, and external orchestrators | -| [`nvx::host::template`](#nvxhosttemplate) | Legacy template capture and clone plans and store, with aliases of the snapshot metadata | +| [`nvx::host::template`](#nvxhosttemplate) | Legacy template capture plans and store, with aliases of the snapshot metadata | | [`nvx::host::seal`](#nvxhostseal) | `FileSeal`: what the host reports about an open file from its metadata alone, without reading it | | [`nvx::host::session`, `platform`, `report`](#nvxhostsession-platform-report) | The authenticated control session, Linux and Windows primitives, and OpenVMM's outcome report | | [`nvx::host::sandbox`](#nvxhostsandbox) | The launch profile of `nvx-rs sandbox`: its plans, live shares, option checks, bindings, and prepared launches | @@ -1237,9 +1237,9 @@ reattach has verified the VM's exit. Windows' process-creation API limits `VmmCommand::working_dir` to 258 UTF-16 code units in its absolute path, including any verbatim prefix and trailing separator. It may name a shorter directory than the store. A raw `spawn` -refuses an overlong working directory with `InvalidInput`; high-level instance, -capture, and clone preflight return `Unsupported` with the limit in the -top-level message. Neither silently changes the requested directory. +refuses an overlong working directory with `InvalidInput`; high-level instance +and capture preflight return `Unsupported` with the limit in the top-level +message. Neither silently changes the requested directory. ### `nvx::host::template` @@ -1344,206 +1344,16 @@ launch that the lease reserves, `Lease::launch`. A lease never expires: `Store::release` ends it once the sandbox records no VM and no launch. -`ClonePlan::new` plans a clone under the lock that acquired its lease, before -the launch that the lease reserves, from metadata alone: the leased generation, -its key, which must speak the current contracts, the boot and control contracts -of the runtime that captured it included, and its snapshot; a network that -attaches a NIC if and only if the template has one, at the guest's address of -the template's NIC, which OpenVMM restores from the snapshot, so that a template -with a NIC serves the clones of every network policy at its guest's address; a -processor target of 1, 2, 4, or 8 from the template's processors online at boot -to its capacity; a memory -target in whole blocks -of `MEMORY_BLOCK_SIZE` from the template's RAM to its capacity; the clone's -image, which the check of its `DistroBinding` opens; its upper, either the -caller's scratch image, which the check of its `ScratchBinding` claims, and -which nothing formats, copies, or reads, or RAM, whose restore declares the -template's scratch absent, so that OpenVMM attaches it with no file and the plan -creates none; -its export, if it has one, an `ExportPlan` that `filesystem::plan` made for the -container root, an aggregate of up to 256 host directories and files, as a cold -launch exports, none of which may overlap the template store, by the paths that -the plan and the store's directory resolve to, let the guest write in a -directory through which the store's path leads, links included, since the host -reaches the store, and OpenVMM its snapshots, by that path, or be, hold, or for -a file lie in the directory of the clone's sandbox, in which the host writes the -clone's records, checks of paths that leave two rules to whoever names the -export, as they leave the directories above the store: the export must not -reach the store through an alias of it that an administrator configures, such as -a bind mount or a mount point of the store, or of a directory above or inside -it, a loopback share, or a hard link of a payload, nor lie inside a directory -that a writable export of another sandbox or clone holds, since OpenVMM resolves -an export's paths again at each launch and restore, so that the directories -above the export's directories and files must be ones that no guest may change, -as those above a clone's block devices must; and a fresh capability. - -It builds the clone's activation record, `ClonePlan::activation`, which the -clone's guest applies in place of the configuration that the template captured, -from the request's `View`, overlay mode, journal policy, and bound of a -GPT disk's first partition, and from the checked inputs: a restore section with -a fresh transaction ID of 16 random bytes alone; -the image, an EROFS layer or a GPT disk, as its binding names it, with the -filesystem UUID or disk GUID that the adapter's plan -expects of it, and for a GPT disk the bound of its first partition, whose -partition table the guest checks with `nvx::gpt` as at a cold launch; a tmpfs -capped at half of the VM's memory, or the scratch image with the journal policy; -the export, an aggregate, and its mappings, the plan's binds, from 1 -to 4,096, or no export and no mappings, which leaves the slot dormant; the -clone's network, the DNS server that OpenVMM serves the guest -at a gateway under its policy, if any, whether the guest publishes its -listeners, as under a policy that allows ingress or host-loopback access both -ways, and the guest's proxy URL; and the processor and memory targets. - -The image must present the logical and physical block sizes of the template's -placeholder and be no longer than it, since a restore presents every clone's -image at the placeholder's length, its capacity, and reads zeros past the end of -a shorter image, while a scratch image must present the `BlockGeometry` of the -template's scratch, its length included, which a restore compares; both must be, -at the paths by which OpenVMM opens them, the regular files that the checks -found, which are not links or other reparse points, and which no one else may -change. Neither the image nor a scratch image may be a file that the sandbox's -store owns, such as its log or its records, by any path, since the store's -transitions replace or remove them, and a link or a directory under one of the -store's names is refused. - -`ClonePlan::launch` then launches the clone as the launch that its lease -reserves, once it has checked that the sandbox's clone binding records the -lease, that the sandbox records no failure of the clone, which fails once under -its lease, and, with `ExportPlan::verify`, that the objects of its export, if it -has one, are still those that its plan found, and returns a `Restoring`: it -writes the clone's activation record in its canonical encoding, -`clone-activation.bin`, from which a caller resends it unchanged; it listens for -the VMM's restore-ready event, at the socket `restore.sock` on Linux, and at a -named pipe of a random name on Windows; and it runs `supervisor::launch` with -OpenVMM's restore of the clone's devices, with `--restore-absent-scratch` for a -clone in RAM, and of its network, whose policy `--restore-network-policy` -installs on the template's NIC in place of the complete deny-all policy, its -export bound to the dormant slot as a cold launch attaches it, -`--mount-aggregate` with a `--mount-child` of each of its directories and files, -their hidden, re-exposed, and writable paths, and its owner, -`--restore-snapshot` with the generation's snapshot in place, which no clone -copies or changes, so that clones of one generation restore it at once, -`--restore-memory` with the memory target, `--restore-processors` with the -processor target, for a template whose processors were not all online at boot, -and `--restore-ready-path`, under the -template's backend, CPU profile, and processors, with a boot console that -discards its output, as cold boots have, and the control console, but no kernel, -initramfs, command line, or guest address, which the snapshot fixes. - On Linux, where OpenVMM requires a restore's control socket in the snapshot's parent, the clone's VMM listens in the generation's payload, at `.sock`, -named after the launch that the lease reserves, whose path must be shorter than -108 bytes with or without the payload's links resolved; on Windows, it listens -at a named pipe, as every launch does. The VMM inherits the claim of a scratch -image, on Windows through the image's guard, once the launch has closed the -claim's handle, which OpenVMM's open of the image would refuse. +named after the launch that the lease reserves; on Windows, it listens at a +named pipe, as every launch does. A payload holds the sockets of its clones only while they lease it: the VMM removes its socket as it exits, a stop or a recovery removes the one that an abrupt exit leaves, and the release makes the socket's absence durable before the lease ends, and removes the record and `restore.sock`. -`Restoring::activate` then runs the clone's restore sequence through the -adapter's framing of its control session, a `CloneControl`, by a -restore-to-Ready deadline that runs from the launch, `RESTORE_TO_READY_TIMEOUT`, -60 seconds by default: it reads the restore-ready event as soon as it comes, -refusing another process's connection and bytes that differ from the event, and -reading nothing past it, since OpenVMM on Windows waits until the host has read -it, and then closes its end; only then connects to the clone's control endpoint, -whose console bounds the host's authentication from the connection on; attaches -with the launch's capability; sends the record, read again from -`clone-activation.bin`, which must still be byte for byte the launch's, in the -adapter's Activate, as the session's first request; and checks that Ready -reports the record's transaction ID. It replaces a session that the adapter -lost, as `CloneControl::lost` tells, with a new one that sends the same record, -in at most `MAX_ACTIVATION_SESSIONS` sessions by the same deadline, and retries -nothing else. - -`Activated::start` then sends the clone's mapping table, if it has an export, -through `CloneControl::map`, in order, as a cold launch's guest receives it, -each path by its child's index, and starts the sandbox once the guest applied -every mapping, both by a Ready-to-Start deadline from the check of Ready, -`READY_TO_START_TIMEOUT`, 30 seconds by default, never sending either again, -and `Started::commit` commits the launch, which writes its runtime record. A -failure, a partial or failed mapping table included, ends the launch as -`Starting::abandon` does; a crash before the commit leaves the launch's -journal, whose recovery ends the VMM, while the lease lasts until the release, -and a crash after it leaves a VM that reattaches, and keeps its lease while it -runs. - -A failure of a clone is a `CloneFailure`, whose `Evidence` attributes it to an -`Outcome`, at a `Stage`, once the clone got as far as its `Progress`: the -request, for a denied authorization, a stale binding, invalid targets, format, -settings, Activate, or mappings, such as an export that no clone takes, or whose -objects changed since its plan, or a mapping table that the guest refused, as -`CloneControl::refused` tells, or an image, a scratch, or a root that the -template cannot serve, and for a supplied file of the wrong identity, length, or -geometry, which the host's staging takes instead once the caller proves it, -`CloneFailure::with_staging_proven`; the host, for an endpoint that cannot be -bound, resource exhaustion, a host that cannot restore the template, another -runtime, or a failure of its own operations; unknown, for a lost status, a -timeout, a reset, a forced kill, or a broken protocol; the template, for a -malformed generation, or a defect of its capture or repair that a diagnosis -attributes to it, the only outcome that quarantines the generation, through its -tombstone, `CloneFailure::quarantine`; and the repair stage, for a status of the -clone's repair alone. - -OpenVMM's outcome report gives the status with which the guest ended the clone's -VM, which `status_evidence` attributes, and which prevails over what the host or -the caller saw, but not over evidence of the template, which shows the -provenance that a status alone leaves to establish: a status of the repair at -any point, and one of the activation before the start was sent. - -A VMM that exits before its restore-ready event, as OpenVMM does once it cannot -restore the snapshot, after it closed the event's endpoint, which the sequence -then gives ten seconds to exit, leaves the line with which OpenVMM reported its -fatal error in its log, which the failure's message keeps, and whose time ABI -code, if it says that the host cannot present the template's CPU profile, TSC, -or LAPIC timer, attributes the failure to the host. - -From the launch on, a failure ends the clone's VMM through its verified -identity, and the sandbox keeps the clone's progress log, `clone-progress.log`, -which records each step from the spawn to the commit, after a first line that -names the launch and the boot of the host, a few lines that it appends without a -sync, and the record of its failure, `clone-failure.json`, JSON in format -`FAILURE_FORMAT`, which it writes before the teardown, with the teardown -pending, so that a crash of the caller during the teardown leaves it for a -reattach to finish, and again once the teardown ended, with the stage, the -evidence and the status, the original outcome, the transaction, the generation, -the progress, and the teardown: no VM, an exit or a termination that the host -verified, or an unresolved one, which keeps the launch's journal or runtime -record, and the lease; a release removes both, and a sandbox that boots cold -writes neither. A failure that cannot be recorded before its teardown keeps the -launch's journal or runtime record, and the lease, so that a reattach still -finds that the clone failed. - -A step or a check of the caller's own that fails once the clone is ready, or -once its sandbox started, fails the clone with the caller's evidence through -`Activated::fail` or `Started::fail`, which end the launch and record the -failure as the sequence's own failures do. - -`reattach_clone` takes a clone over after the caller of its launch restarted: it -reattaches by the identity of its VMM's process, checks a committed clone in a -new session, in which the clone answers the byte-equal Activate of -`clone-activation.bin` with the Ready that it answered once, and ends a clone -that another runtime identity launched, or that fails the check, through its -verified identity; it records the failure of a launch that a crash interrupted, -but not the journal that a commit may leave beside its runtime record, -classifies a VM that exited by its outcome report, which is the clone's VM's -once its launch began, whether or not a crash left a record of its spawn, -records a failure before it ends a VMM or clears the runtime record of one that -exited, and fails, leaving them as they were, if it cannot, finishes a failure -that a crash left pending as its teardown would have recorded it, keeping its -transaction, resolves the teardown of an earlier failure with what the VMM left -as it ended, its outcome report and OpenVMM's fatal error, attributes a status -of the guest's as one that a workload may have given once the clone's progress -log is lost, damaged, or was written during another boot of the host, whose -crash may have lost its last lines, and reads nothing of the template store, so -that a tombstone or a retirement of the generation leaves the clone its lease. - -`CloneRecords::read` reads what the sandbox keeps of a clone, for the operations -that list, inspect, or remove clones. - `Store::tombstone` quarantines a generation, which takes no new lease while its leases keep working, and `Store::retire` deletes it once its leases have drained, crash-safely, removing the records of leases that no clone binding @@ -1577,8 +1387,6 @@ Capture preflight applies the `\captures\<16-hex-slot>\transaction` before creating the slot or scratch. Without a trailing separator this leaves 220 units for the absolute template-store path, or 210 for a state root whose store is `\templates`. -A clone's sandbox directory is also its VMM's working directory, which -`ClonePlan::launch` checks before writing launch records. A capture that fails publishes nothing, and `Store::recover_captures` ends the captures that a crash interrupted, publishing those that were checked, and @@ -2465,13 +2273,6 @@ opens then keep the images, and what it inherited keeps the scratch image's claim. Staging, the checks, and the preparation of a launch read no content of the images. -The clone of a template takes its block devices by path instead, as the -template's capture does, since OpenVMM opens the files of a restore itself: -`ClonePlan::new` checks the clone's image and scratch image through their -bindings, then checks each at its path, and requires the geometry that the -template recorded for its own, which `CheckedDistro::geometry` and -`CheckedScratch::geometry` read through the checked files. - `nvx-rs sandbox provision` records the scratch image's binding under `scratch_binding` in `config.json`, at the image's resolved path, and `start` checks it after the distro binding, and launches OpenVMM through a prepared diff --git a/nvx-rs/public-api/linux.txt b/nvx-rs/public-api/linux.txt index 00e2038b4..aeae95369 100644 --- a/nvx-rs/public-api/linux.txt +++ b/nvx-rs/public-api/linux.txt @@ -3318,15 +3318,6 @@ mod nvx::host::template struct nvx::host::template::Acquired field nvx::host::template::Acquired::lease field nvx::host::template::Acquired::locked -struct nvx::host::template::Activated -method nvx::host::template::Activated::abandon -method nvx::host::template::Activated::activation -method nvx::host::template::Activated::deadline -method nvx::host::template::Activated::fail -method nvx::host::template::Activated::image -method nvx::host::template::Activated::session -method nvx::host::template::Activated::start -method nvx::host::template::Activated::starting struct nvx::host::template::BlockGeometry field nvx::host::template::BlockGeometry::length field nvx::host::template::BlockGeometry::logical_block_size @@ -3359,66 +3350,6 @@ method nvx::host::template::CaptureClass::to_bytes struct nvx::host::template::ClassIdentity method nvx::host::template::ClassIdentity::as_bytes method nvx::host::template::ClassIdentity::from_bytes -trait nvx::host::template::CloneControl -type nvx::host::template::CloneControl::Error -type nvx::host::template::CloneControl::Session -method nvx::host::template::CloneControl::activate -method nvx::host::template::CloneControl::attach -method nvx::host::template::CloneControl::lost -method nvx::host::template::CloneControl::map -method nvx::host::template::CloneControl::refused -method nvx::host::template::CloneControl::start -struct nvx::host::template::CloneFailure -method nvx::host::template::CloneFailure::code -method nvx::host::template::CloneFailure::error -method nvx::host::template::CloneFailure::into_error -method nvx::host::template::CloneFailure::kind -method nvx::host::template::CloneFailure::message -method nvx::host::template::CloneFailure::outcome -method nvx::host::template::CloneFailure::quarantine -method nvx::host::template::CloneFailure::record -method nvx::host::template::CloneFailure::teardown -method nvx::host::template::CloneFailure::with_staging_proven -struct nvx::host::template::ClonePlan -method nvx::host::template::ClonePlan::activation -method nvx::host::template::ClonePlan::capability -method nvx::host::template::ClonePlan::export -method nvx::host::template::ClonePlan::generation -method nvx::host::template::ClonePlan::image -method nvx::host::template::ClonePlan::key -method nvx::host::template::ClonePlan::launch -method nvx::host::template::ClonePlan::lease -method nvx::host::template::ClonePlan::memory -method nvx::host::template::ClonePlan::new -method nvx::host::template::ClonePlan::processors -method nvx::host::template::ClonePlan::scratch_path -method nvx::host::template::ClonePlan::snapshot -method nvx::host::template::ClonePlan::table -method nvx::host::template::ClonePlan::upper -struct nvx::host::template::CloneRecords -field nvx::host::template::CloneRecords::failure -field nvx::host::template::CloneRecords::journal -field nvx::host::template::CloneRecords::lease -field nvx::host::template::CloneRecords::progress -method nvx::host::template::CloneRecords::read -field nvx::host::template::CloneRecords::report -field nvx::host::template::CloneRecords::runtime -struct nvx::host::template::CloneRequest -field nvx::host::template::CloneRequest::adapter -field nvx::host::template::CloneRequest::binding -field nvx::host::template::CloneRequest::export -field nvx::host::template::CloneRequest::first_partition_bound -field nvx::host::template::CloneRequest::journal -field nvx::host::template::CloneRequest::memory -field nvx::host::template::CloneRequest::overlay -field nvx::host::template::CloneRequest::policy_revision -field nvx::host::template::CloneRequest::processors -field nvx::host::template::CloneRequest::scratch -field nvx::host::template::CloneRequest::view -enum nvx::host::template::CloneUpper -variant nvx::host::template::CloneUpper::Scratch -field nvx::host::template::CloneUpper::Scratch::0 -variant nvx::host::template::CloneUpper::Tmpfs enum nvx::host::template::Created variant nvx::host::template::Created::Captured field nvx::host::template::Created::Captured::0 @@ -3442,53 +3373,6 @@ field nvx::host::template::Eligibility::Eligible::0 variant nvx::host::template::Eligibility::Expired field nvx::host::template::Eligibility::Expired::0 method nvx::host::template::Eligibility::is_eligible -enum nvx::host::template::Evidence -variant nvx::host::template::Evidence::AuthorizationDenied -variant nvx::host::template::Evidence::EndpointBind -variant nvx::host::template::Evidence::ForcedKill -variant nvx::host::template::Evidence::ForwardBind -variant nvx::host::template::Evidence::HostFailure -variant nvx::host::template::Evidence::IncompatibleHost -variant nvx::host::template::Evidence::InvalidActivate -variant nvx::host::template::Evidence::InvalidFormat -variant nvx::host::template::Evidence::InvalidMappings -variant nvx::host::template::Evidence::InvalidSettings -variant nvx::host::template::Evidence::InvalidTargets -variant nvx::host::template::Evidence::LostStatus -variant nvx::host::template::Evidence::MalformedGeneration -variant nvx::host::template::Evidence::Protocol -variant nvx::host::template::Evidence::RepairStatus -variant nvx::host::template::Evidence::Reset -variant nvx::host::template::Evidence::ResourceExhaustion -variant nvx::host::template::Evidence::RuntimeMismatch -variant nvx::host::template::Evidence::StaleBinding -variant nvx::host::template::Evidence::SuppliedGeometry -variant nvx::host::template::Evidence::SuppliedIdentity -variant nvx::host::template::Evidence::SuppliedLength -variant nvx::host::template::Evidence::TemplateDefect -variant nvx::host::template::Evidence::Timeout -variant nvx::host::template::Evidence::UnservableImage -variant nvx::host::template::Evidence::UnusableRoot -variant nvx::host::template::Evidence::UnusableScratch -method nvx::host::template::Evidence::as_str -method nvx::host::template::Evidence::is_supplied_file -method nvx::host::template::Evidence::outcome -const nvx::host::template::FAILURE_FORMAT -struct nvx::host::template::FailureRecord -field nvx::host::template::FailureRecord::code -method nvx::host::template::FailureRecord::error -field nvx::host::template::FailureRecord::evidence -field nvx::host::template::FailureRecord::launch -field nvx::host::template::FailureRecord::lease -field nvx::host::template::FailureRecord::message -field nvx::host::template::FailureRecord::outcome -field nvx::host::template::FailureRecord::progress -field nvx::host::template::FailureRecord::stage -field nvx::host::template::FailureRecord::status -field nvx::host::template::FailureRecord::store -field nvx::host::template::FailureRecord::teardown -field nvx::host::template::FailureRecord::template -field nvx::host::template::FailureRecord::transaction const nvx::host::template::GENERATIONS_NAME struct nvx::host::template::Generation method nvx::host::template::Generation::class @@ -3546,59 +3430,10 @@ struct nvx::host::template::LeaseId method nvx::host::template::LeaseId::as_bytes method nvx::host::template::LeaseId::from_bytes const nvx::host::template::MANIFEST_NAME -const nvx::host::template::MAX_ACTIVATION_SESSIONS const nvx::host::template::MAX_DOWNTIME_NS const nvx::host::template::MAX_GENERATIONS const nvx::host::template::MAX_MANIFEST_LEN const nvx::host::template::MEMORY_BLOCK_SIZE -enum nvx::host::template::Outcome -variant nvx::host::template::Outcome::Host -variant nvx::host::template::Outcome::Repair -variant nvx::host::template::Outcome::Request -variant nvx::host::template::Outcome::Staging -variant nvx::host::template::Outcome::Template -variant nvx::host::template::Outcome::Unknown -method nvx::host::template::Outcome::as_str -method nvx::host::template::Outcome::quarantines -enum nvx::host::template::Progress -variant nvx::host::template::Progress::Acknowledged -variant nvx::host::template::Progress::Activating -variant nvx::host::template::Progress::Committed -variant nvx::host::template::Progress::Event -variant nvx::host::template::Progress::Launching -variant nvx::host::template::Progress::Planned -variant nvx::host::template::Progress::Ready -variant nvx::host::template::Progress::Spawned -variant nvx::host::template::Progress::Started -variant nvx::host::template::Progress::Starting -method nvx::host::template::Progress::as_str -method nvx::host::template::Progress::may_have_run -method nvx::host::template::Progress::stage -const nvx::host::template::READY_TO_START_TIMEOUT -const nvx::host::template::RESTORE_READY_EVENT -const nvx::host::template::RESTORE_TO_READY_TIMEOUT -enum nvx::host::template::ReattachedClone -variant nvx::host::template::ReattachedClone::Ended -field nvx::host::template::ReattachedClone::Ended::progress -field nvx::host::template::ReattachedClone::Ended::report -variant nvx::host::template::ReattachedClone::Failed -field nvx::host::template::ReattachedClone::Failed::0 -variant nvx::host::template::ReattachedClone::Indeterminate -field nvx::host::template::ReattachedClone::Indeterminate::0 -variant nvx::host::template::ReattachedClone::Live -field nvx::host::template::ReattachedClone::Live::session -field nvx::host::template::ReattachedClone::Live::vmm -struct nvx::host::template::RestoreDeadlines -field nvx::host::template::RestoreDeadlines::ready -field nvx::host::template::RestoreDeadlines::start -struct nvx::host::template::Restoring -method nvx::host::template::Restoring::abandon -method nvx::host::template::Restoring::activate -method nvx::host::template::Restoring::activation -method nvx::host::template::Restoring::capability -method nvx::host::template::Restoring::image -method nvx::host::template::Restoring::ready_endpoint -method nvx::host::template::Restoring::starting enum nvx::host::template::Retirement variant nvx::host::template::Retirement::Draining field nvx::host::template::Retirement::Draining::0 @@ -3607,24 +3442,6 @@ const nvx::host::template::SCRATCH_NAME const nvx::host::template::SNAPSHOT_NAME const nvx::host::template::STORE_FORMAT const nvx::host::template::STORE_RECORD_NAME -enum nvx::host::template::Stage -variant nvx::host::template::Stage::Activation -variant nvx::host::template::Stage::Commit -variant nvx::host::template::Stage::Launch -variant nvx::host::template::Stage::Plan -variant nvx::host::template::Stage::Reattach -variant nvx::host::template::Stage::Restore -variant nvx::host::template::Stage::Run -variant nvx::host::template::Stage::Start -method nvx::host::template::Stage::as_str -struct nvx::host::template::Started -method nvx::host::template::Started::abandon -method nvx::host::template::Started::activation -method nvx::host::template::Started::commit -method nvx::host::template::Started::fail -method nvx::host::template::Started::image -method nvx::host::template::Started::session -method nvx::host::template::Started::starting struct nvx::host::template::StorageGeneration method nvx::host::template::StorageGeneration::as_bytes method nvx::host::template::StorageGeneration::from_bytes @@ -3653,15 +3470,6 @@ method nvx::host::template::StoreId::as_bytes method nvx::host::template::StoreId::from_bytes const nvx::host::template::TOMBSTONES_NAME const nvx::host::template::TRANSACTION_NAME -enum nvx::host::template::Teardown -variant nvx::host::template::Teardown::Exited -variant nvx::host::template::Teardown::NoVm -variant nvx::host::template::Teardown::Pending -variant nvx::host::template::Teardown::Terminated -variant nvx::host::template::Teardown::Unresolved -field nvx::host::template::Teardown::Unresolved::0 -method nvx::host::template::Teardown::as_str -method nvx::host::template::Teardown::is_resolved struct nvx::host::template::TemplateKey const nvx::host::template::TemplateKey::MAX_LEN const nvx::host::template::TemplateKey::VERSION @@ -3728,8 +3536,6 @@ field nvx::host::template::TimeContract::host_id field nvx::host::template::TimeContract::lineage field nvx::host::template::TimeContract::tsc_frequency_hz field nvx::host::template::TimeContract::tsc_tolerance_ppm -fn nvx::host::template::reattach_clone -fn nvx::host::template::status_evidence mod nvx::host::transport struct nvx::host::transport::HostConnection method nvx::host::transport::HostConnection::attach diff --git a/nvx-rs/public-api/windows.txt b/nvx-rs/public-api/windows.txt index ad44f0575..1f5e64aff 100644 --- a/nvx-rs/public-api/windows.txt +++ b/nvx-rs/public-api/windows.txt @@ -1920,15 +1920,6 @@ mod nvx::host::template struct nvx::host::template::Acquired field nvx::host::template::Acquired::lease field nvx::host::template::Acquired::locked -struct nvx::host::template::Activated -method nvx::host::template::Activated::abandon -method nvx::host::template::Activated::activation -method nvx::host::template::Activated::deadline -method nvx::host::template::Activated::fail -method nvx::host::template::Activated::image -method nvx::host::template::Activated::session -method nvx::host::template::Activated::start -method nvx::host::template::Activated::starting struct nvx::host::template::BlockGeometry field nvx::host::template::BlockGeometry::length field nvx::host::template::BlockGeometry::logical_block_size @@ -1961,66 +1952,6 @@ method nvx::host::template::CaptureClass::to_bytes struct nvx::host::template::ClassIdentity method nvx::host::template::ClassIdentity::as_bytes method nvx::host::template::ClassIdentity::from_bytes -trait nvx::host::template::CloneControl -type nvx::host::template::CloneControl::Error -type nvx::host::template::CloneControl::Session -method nvx::host::template::CloneControl::activate -method nvx::host::template::CloneControl::attach -method nvx::host::template::CloneControl::lost -method nvx::host::template::CloneControl::map -method nvx::host::template::CloneControl::refused -method nvx::host::template::CloneControl::start -struct nvx::host::template::CloneFailure -method nvx::host::template::CloneFailure::code -method nvx::host::template::CloneFailure::error -method nvx::host::template::CloneFailure::into_error -method nvx::host::template::CloneFailure::kind -method nvx::host::template::CloneFailure::message -method nvx::host::template::CloneFailure::outcome -method nvx::host::template::CloneFailure::quarantine -method nvx::host::template::CloneFailure::record -method nvx::host::template::CloneFailure::teardown -method nvx::host::template::CloneFailure::with_staging_proven -struct nvx::host::template::ClonePlan -method nvx::host::template::ClonePlan::activation -method nvx::host::template::ClonePlan::capability -method nvx::host::template::ClonePlan::export -method nvx::host::template::ClonePlan::generation -method nvx::host::template::ClonePlan::image -method nvx::host::template::ClonePlan::key -method nvx::host::template::ClonePlan::launch -method nvx::host::template::ClonePlan::lease -method nvx::host::template::ClonePlan::memory -method nvx::host::template::ClonePlan::new -method nvx::host::template::ClonePlan::processors -method nvx::host::template::ClonePlan::scratch_path -method nvx::host::template::ClonePlan::snapshot -method nvx::host::template::ClonePlan::table -method nvx::host::template::ClonePlan::upper -struct nvx::host::template::CloneRecords -field nvx::host::template::CloneRecords::failure -field nvx::host::template::CloneRecords::journal -field nvx::host::template::CloneRecords::lease -field nvx::host::template::CloneRecords::progress -method nvx::host::template::CloneRecords::read -field nvx::host::template::CloneRecords::report -field nvx::host::template::CloneRecords::runtime -struct nvx::host::template::CloneRequest -field nvx::host::template::CloneRequest::adapter -field nvx::host::template::CloneRequest::binding -field nvx::host::template::CloneRequest::export -field nvx::host::template::CloneRequest::first_partition_bound -field nvx::host::template::CloneRequest::journal -field nvx::host::template::CloneRequest::memory -field nvx::host::template::CloneRequest::overlay -field nvx::host::template::CloneRequest::policy_revision -field nvx::host::template::CloneRequest::processors -field nvx::host::template::CloneRequest::scratch -field nvx::host::template::CloneRequest::view -enum nvx::host::template::CloneUpper -variant nvx::host::template::CloneUpper::Scratch -field nvx::host::template::CloneUpper::Scratch::0 -variant nvx::host::template::CloneUpper::Tmpfs enum nvx::host::template::Created variant nvx::host::template::Created::Captured field nvx::host::template::Created::Captured::0 @@ -2044,53 +1975,6 @@ field nvx::host::template::Eligibility::Eligible::0 variant nvx::host::template::Eligibility::Expired field nvx::host::template::Eligibility::Expired::0 method nvx::host::template::Eligibility::is_eligible -enum nvx::host::template::Evidence -variant nvx::host::template::Evidence::AuthorizationDenied -variant nvx::host::template::Evidence::EndpointBind -variant nvx::host::template::Evidence::ForcedKill -variant nvx::host::template::Evidence::ForwardBind -variant nvx::host::template::Evidence::HostFailure -variant nvx::host::template::Evidence::IncompatibleHost -variant nvx::host::template::Evidence::InvalidActivate -variant nvx::host::template::Evidence::InvalidFormat -variant nvx::host::template::Evidence::InvalidMappings -variant nvx::host::template::Evidence::InvalidSettings -variant nvx::host::template::Evidence::InvalidTargets -variant nvx::host::template::Evidence::LostStatus -variant nvx::host::template::Evidence::MalformedGeneration -variant nvx::host::template::Evidence::Protocol -variant nvx::host::template::Evidence::RepairStatus -variant nvx::host::template::Evidence::Reset -variant nvx::host::template::Evidence::ResourceExhaustion -variant nvx::host::template::Evidence::RuntimeMismatch -variant nvx::host::template::Evidence::StaleBinding -variant nvx::host::template::Evidence::SuppliedGeometry -variant nvx::host::template::Evidence::SuppliedIdentity -variant nvx::host::template::Evidence::SuppliedLength -variant nvx::host::template::Evidence::TemplateDefect -variant nvx::host::template::Evidence::Timeout -variant nvx::host::template::Evidence::UnservableImage -variant nvx::host::template::Evidence::UnusableRoot -variant nvx::host::template::Evidence::UnusableScratch -method nvx::host::template::Evidence::as_str -method nvx::host::template::Evidence::is_supplied_file -method nvx::host::template::Evidence::outcome -const nvx::host::template::FAILURE_FORMAT -struct nvx::host::template::FailureRecord -field nvx::host::template::FailureRecord::code -method nvx::host::template::FailureRecord::error -field nvx::host::template::FailureRecord::evidence -field nvx::host::template::FailureRecord::launch -field nvx::host::template::FailureRecord::lease -field nvx::host::template::FailureRecord::message -field nvx::host::template::FailureRecord::outcome -field nvx::host::template::FailureRecord::progress -field nvx::host::template::FailureRecord::stage -field nvx::host::template::FailureRecord::status -field nvx::host::template::FailureRecord::store -field nvx::host::template::FailureRecord::teardown -field nvx::host::template::FailureRecord::template -field nvx::host::template::FailureRecord::transaction const nvx::host::template::GENERATIONS_NAME struct nvx::host::template::Generation method nvx::host::template::Generation::class @@ -2148,59 +2032,10 @@ struct nvx::host::template::LeaseId method nvx::host::template::LeaseId::as_bytes method nvx::host::template::LeaseId::from_bytes const nvx::host::template::MANIFEST_NAME -const nvx::host::template::MAX_ACTIVATION_SESSIONS const nvx::host::template::MAX_DOWNTIME_NS const nvx::host::template::MAX_GENERATIONS const nvx::host::template::MAX_MANIFEST_LEN const nvx::host::template::MEMORY_BLOCK_SIZE -enum nvx::host::template::Outcome -variant nvx::host::template::Outcome::Host -variant nvx::host::template::Outcome::Repair -variant nvx::host::template::Outcome::Request -variant nvx::host::template::Outcome::Staging -variant nvx::host::template::Outcome::Template -variant nvx::host::template::Outcome::Unknown -method nvx::host::template::Outcome::as_str -method nvx::host::template::Outcome::quarantines -enum nvx::host::template::Progress -variant nvx::host::template::Progress::Acknowledged -variant nvx::host::template::Progress::Activating -variant nvx::host::template::Progress::Committed -variant nvx::host::template::Progress::Event -variant nvx::host::template::Progress::Launching -variant nvx::host::template::Progress::Planned -variant nvx::host::template::Progress::Ready -variant nvx::host::template::Progress::Spawned -variant nvx::host::template::Progress::Started -variant nvx::host::template::Progress::Starting -method nvx::host::template::Progress::as_str -method nvx::host::template::Progress::may_have_run -method nvx::host::template::Progress::stage -const nvx::host::template::READY_TO_START_TIMEOUT -const nvx::host::template::RESTORE_READY_EVENT -const nvx::host::template::RESTORE_TO_READY_TIMEOUT -enum nvx::host::template::ReattachedClone -variant nvx::host::template::ReattachedClone::Ended -field nvx::host::template::ReattachedClone::Ended::progress -field nvx::host::template::ReattachedClone::Ended::report -variant nvx::host::template::ReattachedClone::Failed -field nvx::host::template::ReattachedClone::Failed::0 -variant nvx::host::template::ReattachedClone::Indeterminate -field nvx::host::template::ReattachedClone::Indeterminate::0 -variant nvx::host::template::ReattachedClone::Live -field nvx::host::template::ReattachedClone::Live::session -field nvx::host::template::ReattachedClone::Live::vmm -struct nvx::host::template::RestoreDeadlines -field nvx::host::template::RestoreDeadlines::ready -field nvx::host::template::RestoreDeadlines::start -struct nvx::host::template::Restoring -method nvx::host::template::Restoring::abandon -method nvx::host::template::Restoring::activate -method nvx::host::template::Restoring::activation -method nvx::host::template::Restoring::capability -method nvx::host::template::Restoring::image -method nvx::host::template::Restoring::ready_endpoint -method nvx::host::template::Restoring::starting enum nvx::host::template::Retirement variant nvx::host::template::Retirement::Draining field nvx::host::template::Retirement::Draining::0 @@ -2209,24 +2044,6 @@ const nvx::host::template::SCRATCH_NAME const nvx::host::template::SNAPSHOT_NAME const nvx::host::template::STORE_FORMAT const nvx::host::template::STORE_RECORD_NAME -enum nvx::host::template::Stage -variant nvx::host::template::Stage::Activation -variant nvx::host::template::Stage::Commit -variant nvx::host::template::Stage::Launch -variant nvx::host::template::Stage::Plan -variant nvx::host::template::Stage::Reattach -variant nvx::host::template::Stage::Restore -variant nvx::host::template::Stage::Run -variant nvx::host::template::Stage::Start -method nvx::host::template::Stage::as_str -struct nvx::host::template::Started -method nvx::host::template::Started::abandon -method nvx::host::template::Started::activation -method nvx::host::template::Started::commit -method nvx::host::template::Started::fail -method nvx::host::template::Started::image -method nvx::host::template::Started::session -method nvx::host::template::Started::starting struct nvx::host::template::StorageGeneration method nvx::host::template::StorageGeneration::as_bytes method nvx::host::template::StorageGeneration::from_bytes @@ -2255,15 +2072,6 @@ method nvx::host::template::StoreId::as_bytes method nvx::host::template::StoreId::from_bytes const nvx::host::template::TOMBSTONES_NAME const nvx::host::template::TRANSACTION_NAME -enum nvx::host::template::Teardown -variant nvx::host::template::Teardown::Exited -variant nvx::host::template::Teardown::NoVm -variant nvx::host::template::Teardown::Pending -variant nvx::host::template::Teardown::Terminated -variant nvx::host::template::Teardown::Unresolved -field nvx::host::template::Teardown::Unresolved::0 -method nvx::host::template::Teardown::as_str -method nvx::host::template::Teardown::is_resolved struct nvx::host::template::TemplateKey const nvx::host::template::TemplateKey::MAX_LEN const nvx::host::template::TemplateKey::VERSION @@ -2330,8 +2138,6 @@ field nvx::host::template::TimeContract::host_id field nvx::host::template::TimeContract::lineage field nvx::host::template::TimeContract::tsc_frequency_hz field nvx::host::template::TimeContract::tsc_tolerance_ppm -fn nvx::host::template::reattach_clone -fn nvx::host::template::status_evidence mod nvx::host::transport struct nvx::host::transport::HostConnection method nvx::host::transport::HostConnection::attach diff --git a/nvx-rs/src/host/platform/linux/endpoint.rs b/nvx-rs/src/host/platform/linux/endpoint.rs index 0d8e8b386..699f58def 100644 --- a/nvx-rs/src/host/platform/linux/endpoint.rs +++ b/nvx-rs/src/host/platform/linux/endpoint.rs @@ -1,5 +1,5 @@ //! The Unix sockets of OpenVMM's endpoints: its control endpoint and the transport over it, and -//! the listener of a clone's restore-ready event. +//! the listener of a restore-ready event. use std::fs::{self, File, Permissions}; use std::io::{self, Read}; @@ -24,7 +24,7 @@ pub(crate) fn control_endpoint(socket_path: &Path) -> io::Result { .ok_or_else(|| io::Error::other("control socket path is not valid UTF-8")) } -/// The listener at which the launch of a clone awaits the restore-ready event of its VMM: a Unix +/// The listener at which a restore awaits the restore-ready event of its VMM: a Unix /// socket, which the VMM connects to once, and which queues its connection until the host /// accepts it, then the connection that the host accepted, from which it reads the event. #[derive(Debug)] @@ -36,7 +36,7 @@ pub(crate) struct ReadyListener { } /// Listens at the Unix socket `socket_path`, in place of a socket that an earlier launch left -/// there, as the restore-ready endpoint of a clone's launch, and returns the endpoint, as +/// there, as the restore-ready endpoint of a restore, and returns the endpoint, as /// OpenVMM's `--restore-ready-path` takes it, and its listener. The socket has mode 0600, /// whatever the umask, as [`set_ready_socket_mode`] gives it. pub(crate) fn listen_ready(socket_path: &Path) -> io::Result<(String, ReadyListener)> { @@ -79,19 +79,9 @@ impl ReadyListener { /// arrive: `false` once the VMM closed its end of the connection. /// /// The endpoint's first connection must be the VMM's: one of another process fails with - /// [`io::ErrorKind::PermissionDenied`]. - pub(crate) fn receive( - &mut self, - pid: u32, - received: &mut Vec, - limit: usize, - timeout: Duration, - ) -> io::Result { - self.receive_checked(pid, received, limit, timeout, false) - } - - /// Receives only from the original socket peer while it still runs, never from a cached - /// numeric PID that a later process reused. Requires Linux's `SO_PEERPIDFD`. + /// [`io::ErrorKind::PermissionDenied`]. It receives only from the original socket peer while + /// it still runs, never from a cached numeric PID that a later process reused. Requires + /// Linux's `SO_PEERPIDFD`. pub(crate) fn receive_verified( &mut self, pid: u32, @@ -224,7 +214,7 @@ fn check_original_peer(connection: &UnixStream, accept_unreaped: bool) -> io::Re } /// Gives the restore-ready socket at `socket_path`, which the caller bound, the mode 0600, whatever -/// the umask: its owner, the user of the clone's VMM, may connect to it, which takes the +/// the umask: its owner, the user of the VMM, may connect to it, which takes the /// permission to write it, and no one else may. /// /// The socket is opened only to locate it, following no link at its name, and refused unless it diff --git a/nvx-rs/src/host/platform/linux/identity.rs b/nvx-rs/src/host/platform/linux/identity.rs index 22fabdd8a..e32d76604 100644 --- a/nvx-rs/src/host/platform/linux/identity.rs +++ b/nvx-rs/src/host/platform/linux/identity.rs @@ -44,11 +44,6 @@ pub(crate) fn opened_file_id(file: &File) -> io::Result<(u64, u128)> { Ok((metadata.dev(), u128::from(metadata.ino()))) } -/// Returns the number of names, its hard links, of the file that `file` holds open. -pub(crate) fn link_count(file: &File) -> io::Result { - Ok(file.metadata()?.nlink()) -} - /// An open directory, through which operations name its entries, so that they reach the entries /// of the directory that it opened, whatever its path names meanwhile. pub(crate) struct Directory(File); diff --git a/nvx-rs/src/host/platform/linux/mod.rs b/nvx-rs/src/host/platform/linux/mod.rs index ef8789520..ae1721964 100644 --- a/nvx-rs/src/host/platform/linux/mod.rs +++ b/nvx-rs/src/host/platform/linux/mod.rs @@ -23,7 +23,7 @@ pub(crate) use self::endpoint::{ }; pub use self::identity::file_identity; pub(crate) use self::identity::{ - Directory, file_id, is_deleted, link_count, opened_file_id, opened_file_identity, + Directory, file_id, is_deleted, opened_file_id, opened_file_identity, }; pub(crate) use self::log_claim::{claim_log, log_claim_held, serves_launch}; pub use self::private::create_private_dir; diff --git a/nvx-rs/src/host/platform/linux/tests.rs b/nvx-rs/src/host/platform/linux/tests.rs index c271d6ebd..7a3029400 100644 --- a/nvx-rs/src/host/platform/linux/tests.rs +++ b/nvx-rs/src/host/platform/linux/tests.rs @@ -388,7 +388,7 @@ fn a_restore_ready_client_that_closed_before_its_connection_was_accepted_is_read let mut received = Vec::new(); let deadline = Instant::now() + Duration::from_secs(10); while listener - .receive(pid, &mut received, 64, Duration::from_millis(50)) + .receive_verified(pid, &mut received, 64, Duration::from_millis(50)) .unwrap() { assert!(Instant::now() < deadline, "the endpoint did not close"); @@ -413,7 +413,7 @@ fn a_restore_ready_client_that_closed_before_its_connection_was_accepted_is_read let mut received = Vec::new(); assert!( listener - .receive(pid, &mut received, 64, Duration::from_millis(50)) + .receive_verified(pid, &mut received, 64, Duration::from_millis(50)) .unwrap() ); drop(client); diff --git a/nvx-rs/src/host/platform/mod.rs b/nvx-rs/src/host/platform/mod.rs index 7f61aed12..5a3b0797b 100644 --- a/nvx-rs/src/host/platform/mod.rs +++ b/nvx-rs/src/host/platform/mod.rs @@ -10,7 +10,7 @@ //! //! Its crate-private reads of regular files open a file without following a link, or on Windows //! any other reparse point, refuse one that is not a regular file before reading anything from -//! it, and read it within a bound; `is_refused` tells their refusals from failures of the host's. +//! it, and read it within a bound. use std::fmt; use std::fs::{self, File, OpenOptions}; @@ -43,7 +43,7 @@ use windows as imp; pub(crate) use imp::*; pub use imp::{SUPPORTED, create_private_dir, file_identity, process_start_time}; -/// Why this module refused a file for what the file is, as [`is_refused`] tells. +/// Why this module refused a file for what the file is. #[derive(Debug)] struct Refused(String); @@ -55,15 +55,6 @@ impl fmt::Display for Refused { impl std::error::Error for Refused {} -/// Returns whether `error` is one with which this module refused a file for what the file is, -/// rather than an operation of the host's that failed: a link, or a file that is not a regular -/// file, which [`open_regular`] refuses, as [`refused_open`] and [`regular`] do for other opens -/// that follow no link, and a file that holds more than its bound, which [`read_at_most`] -/// refuses. -pub(crate) fn is_refused(error: &io::Error) -> bool { - error.get_ref().is_some_and(|inner| inner.is::()) -} - /// Returns the error of a file that is a link, or not a regular file. fn not_regular() -> io::Error { io::Error::other(Refused("it is a link, or not a regular file".to_owned())) diff --git a/nvx-rs/src/host/platform/other.rs b/nvx-rs/src/host/platform/other.rs index c5096aa45..c08450731 100644 --- a/nvx-rs/src/host/platform/other.rs +++ b/nvx-rs/src/host/platform/other.rs @@ -118,10 +118,6 @@ pub(crate) fn opened_file_id(_file: &File) -> io::Result<(u64, u128)> { Err(unsupported()) } -pub(crate) fn link_count(_file: &File) -> io::Result { - Err(unsupported()) -} - /// An open directory, which this host never holds: it keeps no sandbox's store. pub(crate) enum Directory {} @@ -254,7 +250,7 @@ pub(crate) fn control_endpoint(_socket_path: &Path) -> io::Result { Err(unsupported()) } -/// The listener of the restore-ready endpoint of a clone's launch, which this host never makes. +/// The listener of the restore-ready endpoint of a restore, which this host never makes. #[derive(Debug)] pub(crate) struct ReadyListener; diff --git a/nvx-rs/src/host/platform/tests.rs b/nvx-rs/src/host/platform/tests.rs index 4681fdac6..8cff9329f 100644 --- a/nvx-rs/src/host/platform/tests.rs +++ b/nvx-rs/src/host/platform/tests.rs @@ -28,11 +28,9 @@ fn a_bounded_read_refuses_a_regular_file_that_holds_more_than_its_bound() { error.to_string(), "it holds more than the 3 bytes that a record may hold" ); - assert!(is_refused(&error), "{error}"); // A file that is missing is a failure of the host's, not a refusal of what a file is. let error = open_regular(&directory.path().join("missing")).unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::NotFound, "{error}"); - assert!(!is_refused(&error), "{error}"); } #[cfg(unix)] @@ -50,7 +48,6 @@ fn an_open_of_a_regular_file_refuses_a_link_and_a_directory() { "it is a link, or not a regular file", "{name}" ); - assert!(is_refused(&error), "{name}: {error}"); } } diff --git a/nvx-rs/src/host/platform/windows/endpoint.rs b/nvx-rs/src/host/platform/windows/endpoint.rs index 21c295afe..9b71cd78e 100644 --- a/nvx-rs/src/host/platform/windows/endpoint.rs +++ b/nvx-rs/src/host/platform/windows/endpoint.rs @@ -1,5 +1,5 @@ //! The named pipes of OpenVMM's endpoints: its control endpoint and the transport over it, and -//! the listener of a clone's restore-ready event. +//! the listener of a restore-ready event. use std::io; use std::os::windows::io::OwnedHandle; @@ -44,7 +44,7 @@ pub(crate) fn control_endpoint(_socket_path: &Path) -> io::Result { /// The bytes that the inbound buffer of a restore-ready pipe holds, more than OpenVMM's event. const READY_BUFFER_LEN: u32 = 4096; -/// The listener at which the launch of a clone awaits the restore-ready event of its VMM: the +/// The listener at which a restore awaits the restore-ready event of its VMM: the /// one instance of a named pipe, inbound and overlapped, which only the current user may open, /// from this host alone, and which takes the connection of the VMM before the host accepts it. #[derive(Debug)] @@ -57,19 +57,6 @@ pub(crate) struct ReadyListener { connected: bool, } -/// Listens at a fresh named pipe, whose name it draws at random, as the restore-ready endpoint of -/// a clone's launch, and returns the endpoint, as OpenVMM's `--restore-ready-path` takes it, and -/// its listener. The first instance of the pipe is the listener's, so no other process holds the -/// name, and the pipe's protected access control list allows the current user alone. -pub(crate) fn listen_ready(_socket_path: &Path) -> io::Result<(String, ReadyListener)> { - let mut bytes = [0u8; 16]; - getrandom::fill(&mut bytes).map_err(io::Error::other)?; - let suffix: String = bytes.iter().map(|byte| format!("{byte:02x}")).collect(); - let endpoint = format!("//./pipe/nvx-restore-ready-{suffix}"); - let listener = ReadyListener::bind(&endpoint)?; - Ok((endpoint, listener)) -} - impl ReadyListener { /// Binds the first instance of the fresh endpoint that the VM directory selected. pub(crate) fn bind(endpoint: &str) -> io::Result { diff --git a/nvx-rs/src/host/platform/windows/identity.rs b/nvx-rs/src/host/platform/windows/identity.rs index 053ad0128..1ee230a2d 100644 --- a/nvx-rs/src/host/platform/windows/identity.rs +++ b/nvx-rs/src/host/platform/windows/identity.rs @@ -351,13 +351,6 @@ pub(crate) fn opened_file_id(file: &fs::File) -> io::Result<(u64, u128)> { Ok((seal.device(), seal.file_id())) } -/// Returns the number of names, its hard links, of the file that `file` holds open. -pub(crate) fn link_count(file: &fs::File) -> io::Result { - Ok(u64::from( - handle_information(file.as_raw_handle() as HANDLE)?.nNumberOfLinks, - )) -} - /// Returns the volume serial number and 64-bit file index of the file that `handle` holds open. fn handle_identity(handle: HANDLE) -> io::Result<(u64, u64)> { let information = handle_information(handle)?; diff --git a/nvx-rs/src/host/platform/windows/mod.rs b/nvx-rs/src/host/platform/windows/mod.rs index a39767529..52c5ccaed 100644 --- a/nvx-rs/src/host/platform/windows/mod.rs +++ b/nvx-rs/src/host/platform/windows/mod.rs @@ -23,7 +23,7 @@ use windows_sys::Win32::System::Threading::OpenProcess; use crate::hypervisor::Hypervisor; pub(crate) use self::endpoint::{ - ReadyListener, connect_endpoint, control_endpoint, endpoint_server_pid, listen_ready, + ReadyListener, connect_endpoint, control_endpoint, endpoint_server_pid, }; #[cfg(feature = "cli")] pub(crate) use self::endpoint::{ @@ -31,7 +31,7 @@ pub(crate) use self::endpoint::{ }; pub use self::identity::file_identity; pub(crate) use self::identity::{ - Directory, file_id, is_deleted, link_count, opened_file_id, opened_file_identity, + Directory, file_id, is_deleted, opened_file_id, opened_file_identity, }; #[cfg(feature = "cli")] pub(crate) use self::interrupt::{interrupted, watch_interrupts}; diff --git a/nvx-rs/src/host/platform/windows/tests.rs b/nvx-rs/src/host/platform/windows/tests.rs index f9d6c0779..5eefd2213 100644 --- a/nvx-rs/src/host/platform/windows/tests.rs +++ b/nvx-rs/src/host/platform/windows/tests.rs @@ -129,18 +129,24 @@ fn a_restore_ready_client_that_closed_before_its_connection_was_accepted_is_read } received }; + // A fresh endpoint and its listener, as a restore binds them. + let listen = || { + let endpoint = control_endpoint(Path::new("unused")).unwrap(); + let listener = ReadyListener::bind(&endpoint).unwrap(); + (endpoint, listener) + }; // A client that sent nothing, as a VMM that fails as it starts, and one that sent its // event, each closed before the listener accepted their connection: what they sent // remains to read, and then the end of the pipe. for sent in [&b""[..], b"the event\n"] { - let (endpoint, mut listener) = listen_ready(Path::new("unused")).unwrap(); + let (endpoint, mut listener) = listen(); let mut client = fs::OpenOptions::new().write(true).open(&endpoint).unwrap(); client.write_all(sent).unwrap(); drop(client); assert_eq!(receive_all(&mut listener), sent); } // A client that closes once the listener accepted its connection. - let (endpoint, mut listener) = listen_ready(Path::new("unused")).unwrap(); + let (endpoint, mut listener) = listen(); let client = fs::OpenOptions::new().write(true).open(&endpoint).unwrap(); let mut received = Vec::new(); assert!( diff --git a/nvx-rs/src/host/sandbox/distro.rs b/nvx-rs/src/host/sandbox/distro.rs index 25b3329c7..751afe3bc 100644 --- a/nvx-rs/src/host/sandbox/distro.rs +++ b/nvx-rs/src/host/sandbox/distro.rs @@ -226,8 +226,7 @@ impl CheckedDistro { /// It reads the file's metadata alone. A restore of a template presents its `distro` at the /// geometry that the template recorded, the capacity of every clone's image, over an image /// that may be shorter, so the image of a clone must be no longer than the template's - /// `distro`, with the same logical and physical block sizes, as - /// [`ClonePlan`](crate::host::template::ClonePlan) checks before the clone's launch. + /// `distro`, with the same logical and physical block sizes. /// /// Fails with [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for a length or a /// physical block size that OpenVMM refuses, and with [`ErrorKind::Io`](crate::ErrorKind::Io) @@ -340,12 +339,6 @@ impl DistroBinding { self.policy } - /// Returns the binding's version, which a [check](Self::check) requires to be - /// [`BINDING_VERSION`]. - pub(crate) fn version(&self) -> u32 { - self.version - } - /// Returns the binding of `layer`, with the seals of its image and manifest and `policy`. fn new( layer: &SandboxLayer, diff --git a/nvx-rs/src/host/sandbox/plan.rs b/nvx-rs/src/host/sandbox/plan.rs index 6af71e1cd..0c42af9b4 100644 --- a/nvx-rs/src/host/sandbox/plan.rs +++ b/nvx-rs/src/host/sandbox/plan.rs @@ -106,7 +106,7 @@ mod form; use std::ffi::OsString; use std::fmt; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; +use std::net::Ipv4Addr; use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; @@ -341,16 +341,6 @@ impl SandboxNetwork { network::template_arguments(&self.guest_network) }) } - - /// Returns the DNS server that OpenVMM names to the guest under the policy, given the IPv6 - /// gateway of the guest's NIC if the NIC has an IPv6 identity, as [`network::dns_server`] - /// chooses it: the gateway, of either family, whose DNS service the policy lets the guest - /// reach, if any. - /// - /// Fails as [`openvmm_arguments`](Self::openvmm_arguments) does. - pub(crate) fn dns_server(&self, ipv6_gateway: Option) -> Result> { - network::dns_server(&self.policy, self.gateway, ipv6_gateway) - } } /// Returns the gateway of `guest_network`, a static guest address that OpenVMM accepts, as diff --git a/nvx-rs/src/host/sandbox/scratch.rs b/nvx-rs/src/host/sandbox/scratch.rs index 8647314f3..87d985625 100644 --- a/nvx-rs/src/host/sandbox/scratch.rs +++ b/nvx-rs/src/host/sandbox/scratch.rs @@ -170,8 +170,7 @@ impl CheckedScratch { /// read the image's metadata. /// /// It reads the file's metadata alone, never the image's content. The scratch image of a - /// clone must present the geometry of its template's scratch, its length included, as - /// [`ClonePlan`](crate::host::template::ClonePlan) checks before the clone's launch. + /// clone must present the geometry of its template's scratch, its length included. /// /// Fails with [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for a length or a /// physical block size that OpenVMM refuses, and with [`ErrorKind::Io`](crate::ErrorKind::Io) @@ -260,12 +259,6 @@ impl ScratchBinding { self.length } - /// Returns the binding's version, which a [check](Self::check) requires to be - /// [`SCRATCH_BINDING_VERSION`]. - pub(crate) fn version(&self) -> u32 { - self.version - } - /// Returns the binding of the image that `seal` seals. fn of(seal: &FileSeal) -> Self { Self { diff --git a/nvx-rs/src/host/seal.rs b/nvx-rs/src/host/seal.rs index c98d53186..f05f60bb9 100644 --- a/nvx-rs/src/host/seal.rs +++ b/nvx-rs/src/host/seal.rs @@ -237,14 +237,6 @@ pub(crate) fn final_path(file: &File) -> io::Result { imp::final_path(file) } -/// Returns the final path of the open `file` as [`final_path`] does, but with its volume named by -/// its GUID, `\\?\Volume{GUID}\`, which names a volume that has no drive letter or mount point, -/// and so no path by a drive, too. -#[cfg(windows)] -pub(crate) fn final_volume_path(file: &File) -> io::Result { - imp::final_volume_path(file) -} - #[cfg(target_os = "linux")] /// The Linux seal, from `statx` or `fstat` of the open file. mod imp { diff --git a/nvx-rs/src/host/seal/windows.rs b/nvx-rs/src/host/seal/windows.rs index 133320d60..dd9d25d62 100644 --- a/nvx-rs/src/host/seal/windows.rs +++ b/nvx-rs/src/host/seal/windows.rs @@ -14,7 +14,7 @@ use windows_sys::Win32::Storage::FileSystem::{ BY_HANDLE_FILE_INFORMATION, FILE_ATTRIBUTE_DIRECTORY, FILE_FLAG_BACKUP_SEMANTICS, FILE_ID_INFO, FILE_SHARE_READ, FILE_TYPE_DISK, FILE_TYPE_UNKNOWN, FileIdInfo, GETFINALPATHNAMEBYHANDLE_FLAGS, GetFileInformationByHandle, GetFileInformationByHandleEx, GetFileType, - GetFinalPathNameByHandleW, VOLUME_NAME_DOS, VOLUME_NAME_GUID, + GetFinalPathNameByHandleW, VOLUME_NAME_DOS, }; use super::{FileId, FileKind, FileSeal, SEAL_VERSION, SealScheme, SealTime}; @@ -114,11 +114,6 @@ pub(super) fn final_path(file: &File) -> io::Result { final_path_by(file, VOLUME_NAME_DOS) } -/// See [`super::final_volume_path`]. -pub(super) fn final_volume_path(file: &File) -> io::Result { - final_path_by(file, VOLUME_NAME_GUID) -} - /// Returns the final path of the open `file`, which names its volume as `volume` asks. fn final_path_by(file: &File, volume: GETFINALPATHNAMEBYHANDLE_FLAGS) -> io::Result { let handle = file.as_raw_handle() as HANDLE; diff --git a/nvx-rs/src/host/seal/windows/tests.rs b/nvx-rs/src/host/seal/windows/tests.rs index b2b93d8fd..02045e690 100644 --- a/nvx-rs/src/host/seal/windows/tests.rs +++ b/nvx-rs/src/host/seal/windows/tests.rs @@ -61,22 +61,3 @@ fn final_paths_hold_no_nul_whatever_their_length() { let file = File::open(deep.join("long")).unwrap(); assert!(final_path(&file).unwrap().as_os_str().len() > 260); } - -#[test] -fn final_volume_paths_name_the_file_by_its_volume() { - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("file"); - std::fs::write(&path, b"").unwrap(); - let found = final_volume_path(&File::open(&path).unwrap()).unwrap(); - assert!( - found.to_string_lossy().starts_with(r"\\?\Volume{"), - "{}", - found.display() - ); - // The path names the file, which its final path by its drive names too. - let reopened = File::open(&found).unwrap(); - assert_eq!( - final_path(&reopened).unwrap(), - std::fs::canonicalize(&path).unwrap() - ); -} diff --git a/nvx-rs/src/host/snapshot/key.rs b/nvx-rs/src/host/snapshot/key.rs index 8adda8fc8..b52ead232 100644 --- a/nvx-rs/src/host/snapshot/key.rs +++ b/nvx-rs/src/host/snapshot/key.rs @@ -822,17 +822,6 @@ impl SnapshotKey { self.machine.nic.as_ref().map(guest_network_of) } - /// Returns the gateway of the IPv6 identity of the template's NIC, or `None` for a template - /// without a NIC or whose NIC has no IPv6 identity. - pub(crate) fn ipv6_gateway(&self) -> Option { - self.machine - .nic - .as_ref()? - .ipv6 - .as_ref() - .map(|ipv6| ipv6.gateway) - } - /// Returns the tag of the filesystem slot, on which a clone's export is bound. pub fn filesystem_tag(&self) -> &str { &self.filesystem_tag diff --git a/nvx-rs/src/host/snapshot/key/tests.rs b/nvx-rs/src/host/snapshot/key/tests.rs index 99c639dcf..74fb1d7dd 100644 --- a/nvx-rs/src/host/snapshot/key/tests.rs +++ b/nvx-rs/src/host/snapshot/key/tests.rs @@ -434,22 +434,9 @@ fn a_key_holds_what_its_capture_made() { ); assert!(made.has_network()); assert_eq!(made.guest_network().as_deref(), Some("10.0.0.2/24")); - assert_eq!( - made.ipv6_gateway(), - Some(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0xa00, 1)) - ); let template = key(&support::class(), &ManifestSpec::template()); assert!(!template.has_network()); - assert_eq!( - (template.guest_network(), template.ipv6_gateway()), - (None, None) - ); - let mut spec = ManifestSpec::with_network(); - let network = spec.network(); - network.ipv6.clear(); - network.ipv6_prefix_length = 0; - network.ipv6_gateway.clear(); - assert_eq!(key(&network_class(), &spec).ipv6_gateway(), None); + assert_eq!(template.guest_network(), None); assert_eq!(made.filesystem_tag(), "microvm"); assert_eq!(made.invariants(), &Invariants::new()); } diff --git a/nvx-rs/src/host/store/locked.rs b/nvx-rs/src/host/store/locked.rs index 67e6d269a..f1d9326e4 100644 --- a/nvx-rs/src/host/store/locked.rs +++ b/nvx-rs/src/host/store/locked.rs @@ -9,12 +9,11 @@ use std::thread; use std::time::Instant; use super::caller::CallerFiles; -use super::caller::{is_report_staging, is_store_temporary}; use super::files::{io_error, remove_if_present}; use super::identity::RuntimeIdentity; use super::launch::{Launch, Recovery, end_process}; use super::lease::{Lease, clone_socket_name, is_clone_socket, open_payload}; -use super::names::{CLONE_FILE_NAMES, FILE_NAMES, LOCK_NAME}; +use super::names::{CLONE_FILE_NAMES, LOCK_NAME}; use super::record::{LaunchId, LaunchJournal, LogClaim, RuntimeRecord}; use super::{CLAIM_RELEASE_WAIT, LOCK_RETRY_INTERVAL, SandboxStore, record, store_error, vmm}; use crate::error::{Error, ErrorCode, Result}; @@ -102,36 +101,6 @@ impl LockedStore { .map(|reservation| &reservation.launch) } - /// Returns the path of the Linux socket at which the VMM of the launch that - /// [`reserved`](Self::reserved) returns listens. - pub(crate) fn reserved_socket(&self) -> Option<&Path> { - self.reserved - .as_ref() - .map(|reservation| reservation.socket.as_path()) - } - - /// Returns the paths at which the store reaches the entries of the sandbox's directory that - /// it owns, which its transitions replace and remove: its files, [`FILE_NAMES`], at its own - /// paths, whether or not they exist, which the file system resolves as it resolves every - /// name, without regard to case on Windows, say; and the temporary files of its records and - /// the files in which a VMM staged an outcome report, under the names by which the directory - /// lists them now, as a [removal] finds them. - /// - /// [removal]: crate::host::store#removal - pub(crate) fn owned_entries(&self) -> io::Result> { - let mut owned: Vec = FILE_NAMES.iter().map(|name| self.path(name)).collect(); - for entry in std::fs::read_dir(self.dir())? { - let entry = entry?; - let name = entry.file_name(); - if let Some(name) = name.to_str() - && (is_store_temporary(name) || is_report_staging(name)) - { - owned.push(entry.path()); - } - } - Ok(owned) - } - /// Opens the sandbox's directory through its path, as one handle through which a release of /// the lease of its clone names the directory's entries, and checks through that handle that /// the directory holds this lock's file, as a [removal] checks it: no change of the path diff --git a/nvx-rs/src/host/store/names.rs b/nvx-rs/src/host/store/names.rs index b7ffe5387..571ff7f57 100644 --- a/nvx-rs/src/host/store/names.rs +++ b/nvx-rs/src/host/store/names.rs @@ -45,36 +45,25 @@ pub const CLONE_BINDING_NAME: &str = "clone.json"; /// /// [removal]: crate::host::store#removal pub const CLONE_SCRATCH_NAME: &str = "clone-scratch.raw"; -/// Name of the activation record of a clone, in its canonical encoding, which the clone's launch, -/// [`template::ClonePlan::launch`](crate::host::template::ClonePlan::launch), writes before it -/// spawns the clone's VMM, so that the restore activates the clone with that record and its -/// transaction whatever happens to the launch's caller, and which +/// Name of the activation record of a clone, in its canonical encoding, which /// [`template::Store::release`](crate::host::template::Store::release) removes once the clone's VM /// has exited, as a [removal] of the sandbox does. /// /// [removal]: crate::host::store#removal pub const CLONE_ACTIVATION_NAME: &str = "clone-activation.bin"; -/// Name of the Linux socket at which the launch of a clone, -/// [`template::ClonePlan::launch`](crate::host::template::ClonePlan::launch), listens for the -/// restore-ready event of its VMM, which OpenVMM sends once it has prepared the restore, before -/// the restored VM runs; on Windows, the endpoint is a named pipe, whose name each launch draws at -/// random. [`template::Store::release`](crate::host::template::Store::release) removes it once the -/// clone's VM has exited, as a [removal] of the sandbox does. +/// Name of the Linux socket of the restore-ready event of a clone's VMM, which +/// [`template::Store::release`](crate::host::template::Store::release) removes once the clone's VM +/// has exited, as a [removal] of the sandbox does. /// /// [removal]: crate::host::store#removal pub const RESTORE_READY_NAME: &str = "restore.sock"; -/// Name of the progress log of a clone's launch, which the launch, -/// [`template::ClonePlan::launch`](crate::host::template::ClonePlan::launch), creates, and to which -/// it and the clone's restore sequence append a line for each step that the clone reaches, without -/// syncing it, so that a reattach tells how far the clone got once the caller of its launch -/// crashed, and which [`template::Store::release`](crate::host::template::Store::release) removes, -/// as a [removal] of the sandbox does. No other launch writes it. +/// Name of the progress log of a clone's launch, which +/// [`template::Store::release`](crate::host::template::Store::release) removes, as a [removal] of +/// the sandbox does. /// /// [removal]: crate::host::store#removal pub const CLONE_PROGRESS_NAME: &str = "clone-progress.log"; -/// Name of the record of a clone's failure, which a failure of the clone from its launch on -/// writes before its teardown, and again once its teardown is known, as -/// [`template::CloneFailure`](crate::host::template::CloneFailure) describes, and which +/// Name of the record of a clone's failure, which /// [`template::Store::release`](crate::host::template::Store::release) removes, as a [removal] of /// the sandbox does. /// diff --git a/nvx-rs/src/host/store/regular.rs b/nvx-rs/src/host/store/regular.rs index d0721edfe..94060b5a7 100644 --- a/nvx-rs/src/host/store/regular.rs +++ b/nvx-rs/src/host/store/regular.rs @@ -2,15 +2,15 @@ //! binding: regular files that are not links, which they read within a bound, and the identity of //! a file that a template module holds open. //! -//! The reads themselves are [`platform`]'s: this module re-exports its `open_regular`, -//! `read_at_most` and `is_refused` for its callers until #1121 deletes them. +//! The reads themselves are [`platform`]'s: this module re-exports its `open_regular` and +//! `read_at_most` for its callers until #1121 deletes them. use std::fs::File; use std::io; use std::path::Path; use crate::host::platform; -pub(crate) use crate::host::platform::{is_refused, open_regular, read_at_most}; +pub(crate) use crate::host::platform::{open_regular, read_at_most}; use crate::host::seal::FileSeal; /// Returns whether `error`, with which an open or a removal of the file at `path` failed, means diff --git a/nvx-rs/src/host/store/tests.rs b/nvx-rs/src/host/store/tests.rs index c403f16c0..8a3991248 100644 --- a/nvx-rs/src/host/store/tests.rs +++ b/nvx-rs/src/host/store/tests.rs @@ -440,7 +440,6 @@ fn entries_that_are_links_or_not_regular_files_are_refused_as_the_platform_refus "it is a link, or not a regular file", "{name}" ); - assert!(platform::is_refused(&error), "{name}: {error}"); } } diff --git a/nvx-rs/src/host/supervisor/launch.rs b/nvx-rs/src/host/supervisor/launch.rs index 26a59691f..b2829722b 100644 --- a/nvx-rs/src/host/supervisor/launch.rs +++ b/nvx-rs/src/host/supervisor/launch.rs @@ -52,20 +52,15 @@ pub(crate) struct LaunchFailure { /// The error of the step that failed, which says why the launch could not be ended if it /// could not. pub(crate) error: Error, - /// Whether the launch spawned its VMM, which it then ended through the VMM's handle. - pub(crate) spawned: bool, - /// Why the launch was not ended, whose journal then remains for [`LockedStore::recover`]: - /// it could not be, or its caller could not record its failure first. `None` once it ended. - pub(crate) unended: Option, } /// Launches a VMM as [`launch`] does, and tells how a launch that fails ended, as a /// [`LaunchFailure`]. A launch that began, and fails, calls `failing` with the step's error, and /// whether it spawned the VMM, before it ends the launch, so that the caller may record the -/// failure, as a clone's launch does, before anything of the launch ends. If `failing` fails, the -/// launch ends a VMM that it spawned through the VMM's handle, as always, but keeps its journal, -/// as a crash would leave it, for [`LockedStore::recover`]: the journal then still tells that the -/// launch began, though the caller could not record its failure. +/// failure before anything of the launch ends. If `failing` fails, the launch ends a VMM that it +/// spawned through the VMM's handle, as always, but keeps its journal, as a crash would leave it, +/// for [`LockedStore::recover`]: the journal then still tells that the launch began, though the +/// caller could not record its failure. pub(crate) fn launch_reporting<'s, 'h>( locked: &'s mut LockedStore, reserved: &LaunchId, @@ -78,23 +73,19 @@ pub(crate) fn launch_reporting<'s, 'h>( // A launch that does not begin leaves nothing to end. let mut launch = locked .begin_launch(reserved, runtime, capability) - .map_err(|error| LaunchFailure { - error, - spawned: false, - unended: None, - })?; + .map_err(|error| LaunchFailure { error })?; let command = match command(launch.endpoint()) { Ok(command) => command, Err(error) => { let recorded = failing(&error, false); - return Err(abandon_after(launch, error, false, recorded)); + return Err(abandon_after(launch, error, recorded)); } }; let log = match launch.take_log() { Ok(log) => log, Err(error) => { let recorded = failing(&error, false); - return Err(abandon_after(launch, error, false, recorded)); + return Err(abandon_after(launch, error, recorded)); } }; let child = match spawn(&command, capability, log) { @@ -103,7 +94,7 @@ pub(crate) fn launch_reporting<'s, 'h>( let error = Error::io(format!("cannot launch {}", command.program.display())) .with_source(error); let recorded = failing(&error, false); - return Err(abandon_after(launch, error, false, recorded)); + return Err(abandon_after(launch, error, recorded)); } }; let process = match launch.record_process(&child) { @@ -113,7 +104,7 @@ pub(crate) fn launch_reporting<'s, 'h>( // An unrecorded VMM holds the claim of its launch, so only its handle can end it, // before the launch ends. child.kill_and_wait(); - return Err(abandon_after(launch, error, true, recorded)); + return Err(abandon_after(launch, error, recorded)); } }; Ok(Starting { @@ -126,17 +117,11 @@ pub(crate) fn launch_reporting<'s, 'h>( }) } -/// Ends `launch`, which failed with `error`, after it `spawned` a VMM or not, once its caller -/// `recorded` the failure, and returns how it failed, whose error says why the launch was not -/// ended if it was not: its journal then remains for [`LockedStore::recover`]. A launch whose -/// caller could not record its failure keeps its journal, as a crash would leave it, and ends no -/// more of it than its caller did. -fn abandon_after( - launch: Launch<'_>, - error: Error, - spawned: bool, - recorded: Result<()>, -) -> LaunchFailure { +/// Ends `launch`, which failed with `error`, once its caller `recorded` the failure, and returns +/// how it failed, whose error says why the launch was not ended if it was not: its journal then +/// remains for [`LockedStore::recover`]. A launch whose caller could not record its failure keeps +/// its journal, as a crash would leave it, and ends no more of it than its caller did. +fn abandon_after(launch: Launch<'_>, error: Error, recorded: Result<()>) -> LaunchFailure { let reason = match recorded { Err(unrecorded) => { drop(launch); @@ -144,13 +129,7 @@ fn abandon_after( } Ok(()) => match launch.abandon() { Ok(Recovery::Indeterminate(reason)) => reason, - Ok(_) => { - return LaunchFailure { - error, - spawned, - unended: None, - }; - } + Ok(_) => return LaunchFailure { error }, Err(failure) => failure.to_string(), }, }; @@ -164,8 +143,6 @@ fn abandon_after( Some(source) => error.with_source(source), None => error, }, - spawned, - unended: Some(reason), } } @@ -253,8 +230,7 @@ impl<'s> Starting<'s> { /// Commits the launch as [`commit`](Self::commit) does, and calls `failing` with the error of /// a commit that fails, before it ends the VMM, so that the caller may record the failure - /// first, as a clone's launch does. The journal remains all the same, whether or not the - /// caller could. + /// first. The journal remains all the same, whether or not the caller could. pub(crate) fn commit_reporting(mut self, failing: impl FnOnce(&Error)) -> Result { let StartingInner { launch, @@ -290,25 +266,6 @@ impl<'s> Starting<'s> { .expect("a launch is abandoned once") .abandon() } - - /// Ends the launch's VMM through its handle, and keeps the launch's journal, as a crash of the - /// caller would leave it, for [`LockedStore::recover`], which ends the launch once it finds - /// the VMM exited. Returns whether the VMM exited, as [`Launched::kill_and_wait`] does. - /// - /// It ends a launch whose caller could not record its failure before the launch's end, as a - /// clone's launch records it, so that the journal, which keeps the clone's lease, still tells - /// that the launch began. - pub(crate) fn end_keeping_journal(mut self) -> bool { - let StartingInner { - launch, - child, - process: _, - } = self.inner.take().expect("a launch is ended once"); - let exited = child.kill_and_wait(); - // A launch that is dropped keeps its journal. - drop(launch); - exited - } } impl Drop for Starting<'_> { diff --git a/nvx-rs/src/host/supervisor/mod.rs b/nvx-rs/src/host/supervisor/mod.rs index 5596cce43..bbf7463f8 100644 --- a/nvx-rs/src/host/supervisor/mod.rs +++ b/nvx-rs/src/host/supervisor/mod.rs @@ -108,8 +108,8 @@ //! VM whose VMM had exited by then as one that it stopped, and clears the runtime record of its //! launch. [`Vmm::stop_unless_exited`] stops nothing then, and keeps the record, so that //! [`reattach`] finds that the VM exited without its holder, as a caller that tells how the VM -//! ended, such as a clone's, needs: a holder that checked whether the VMM had exited, then stopped -//! the VM, would otherwise attribute to its stop a VMM that exited in between. +//! ended needs: a holder that checked whether the VMM had exited, then stopped the VM, would +//! otherwise attribute to its stop a VMM that exited in between. //! //! # Reattach //! @@ -135,31 +135,21 @@ //! identity of the launch that the lease reserves, //! [`Lease::launch`](crate::host::template::Lease::launch), before any VMM of the clone //! exists. It returns the sandbox's lock. -//! 2. Under that lock, the caller plans the clone with -//! [`ClonePlan::new`](crate::host::template::ClonePlan::new), which checks the clone's image, -//! upper, and memory target against the generation, and claims the clone's scratch image, if -//! it has one, as -//! [`ScratchBinding::check`](crate::host::sandbox::scratch::ScratchBinding::check) does. It -//! then launches the clone with -//! [`ClonePlan::launch`](crate::host::template::ClonePlan::launch), which checks that the -//! binding records the lease, writes the clone's files in the sandbox, and runs [`launch`] as -//! the reserved launch, which [`LockedStore::begin_launch`] requires of a sandbox that holds a -//! lease, and begins once, under that lock alone, whatever its outcome. The clone's VMM -//! restores the generation's snapshot in place, and on Linux listens beside it, at the control -//! socket that the lease reserves in the generation's payload, named after the launch, which -//! [`Starting::endpoint`] and the runtime record name, as every launch's endpoint, rather than -//! at the sandbox's own socket. -//! 3. Still under that lock, the caller runs the clone's restore sequence: -//! [`Restoring::activate`](crate::host::template::Restoring::activate) waits for the VMM's -//! restore-ready event, activates the clone through the adapter's framing of its control -//! session, and checks its Ready, by a deadline that runs from the launch; -//! [`Activated::start`](crate::host::template::Activated::start) starts its sandbox; and -//! [`Started::commit`](crate::host::template::Started::commit) commits the launch, as -//! [`Starting::commit`] does. A failure ends the launch, as [`Starting::abandon`] does, through -//! the VMM's verified identity, and the sandbox records it until the release, with the evidence -//! that attributes it, as a [`CloneFailure`](crate::host::template::CloneFailure) describes: a -//! launch that cannot be ended keeps its journal, and the lease, and so does one whose failure -//! cannot be recorded before its end, whose VMM is ended through its handle. +//! 2. Under that lock, the caller launches the clone with [`launch`], as the reserved launch, +//! which [`LockedStore::begin_launch`] requires of a sandbox that holds a lease, and which +//! begins once, under that lock alone, whatever its outcome. The supervisor prepares nothing of +//! the restore: the caller binds what the restore needs before the launch, such as its +//! restore-ready endpoint, and its `command` gives OpenVMM the restore's options, such as +//! `--restore-ready-path`. The clone's VMM restores the generation's snapshot in place, and on +//! Linux listens beside it, at the control socket that the lease reserves in the generation's +//! payload, named after the launch, which [`Starting::endpoint`] and the runtime record name, +//! as every launch's endpoint, rather than at the sandbox's own socket. +//! 3. Still under that lock, the caller checks, through its own transport, that the clone's +//! restore completed: that the VMM sent its restore-ready event, and that the clone was +//! activated, reported Ready and started, as the caller's protocol requires. Only then does it +//! commit the launch with [`Starting::commit`], which writes the runtime record. A failure ends +//! the launch with [`Starting::abandon`], through the VMM's verified identity: a launch that +//! cannot be ended keeps its journal, and the lease. //! 4. Once the clone's VM has exited, as a stop, a termination, or a reattach verifies, //! [`Store::release`](crate::host::template::Store::release) ends the lease, under the //! sandbox's lock, taken through the lease's path, @@ -194,21 +184,14 @@ //! - After the acquisition, the binding reserves a launch that never began, and that no later lock //! begins: a release ends the lease at once, and another acquisition, which checks the //! generation's eligibility again, reserves another launch. -//! - After the launch's journal, before or after the spawn, or in the clone's restore sequence, -//! before its commit, [`reattach`] ends the launch, as [`LockedStore::recover`] does, and the -//! lease lasts until it has: a launch that cannot be ended yet is -//! [`Reattached::Indeterminate`], and keeps it. +//! - After the launch's journal, before or after the spawn, before its commit, [`reattach`] ends +//! the launch, as [`LockedStore::recover`] does, and the lease lasts until it has: a launch +//! that cannot be ended yet is [`Reattached::Indeterminate`], and keeps it. //! - After the commit, the runtime record names the VM, which [`reattach`] holds again: the lease //! lasts as long as the VM runs, until a stop or a termination verifies its exit. //! - During a release, the binding goes first, durably, and leaves the record of the lease, which //! a retirement removes. //! -//! The caller of a clone's launch that restarts takes the clone over with -//! [`reattach_clone`](crate::host::template::reattach_clone), which reattaches as [`reattach`] -//! does, records the failure of a launch that it ended, checks the activation of a clone whose -//! launch committed in a new session, and ends a clone that another runtime identity launched, or -//! that fails the check, through its verified identity. -//! //! The evidence of the VM's exit decides each outcome. A record of an earlier boot, or of a process //! ID that another process now has, names a VM that has exited: reattach clears it, and the lease //! may be released. A VM that cannot be examined is indeterminate, and keeps its records, and so @@ -232,9 +215,7 @@ mod reattach; mod tests; mod vmm; -pub(crate) use self::launch::launch_reporting; pub use self::launch::{Starting, launch}; -pub(crate) use self::reattach::reattach_reporting; pub use self::reattach::{Reattached, reattach}; pub use self::vmm::{StopOutcome, Stopped, Vmm}; pub use super::platform::inherit::{Inherited, SpawnLock, lock_spawns, spawn_command}; diff --git a/nvx-rs/src/host/supervisor/tests.rs b/nvx-rs/src/host/supervisor/tests.rs index e0d8376dc..91f5f8637 100644 --- a/nvx-rs/src/host/supervisor/tests.rs +++ b/nvx-rs/src/host/supervisor/tests.rs @@ -8,6 +8,7 @@ use std::time::{Duration, Instant}; use serde_json::json; +use super::launch::launch_reporting; use super::reattach::reattach_with; use super::vmm::{Stoppable, stop_process, terminate_process}; use super::*; @@ -602,9 +603,7 @@ fn a_launch_whose_caller_cannot_record_its_failure_keeps_its_journal_for_recover }, ) .unwrap_err(); - assert!(!failure.spawned); let reason = "the test cannot record the failure"; - assert_eq!(failure.unended.as_deref(), Some(reason)); assert_eq!(failure.error.kind(), ErrorKind::Unsupported); assert_eq!( failure.error.message(), diff --git a/nvx-rs/src/host/supervisor/vmm.rs b/nvx-rs/src/host/supervisor/vmm.rs index 969bffcde..1de352cf7 100644 --- a/nvx-rs/src/host/supervisor/vmm.rs +++ b/nvx-rs/src/host/supervisor/vmm.rs @@ -98,12 +98,10 @@ impl Vmm { /// [`stop`](Self::stop) does. A VMM that had exited by then ended without its holder, however /// recently the holder found it running: the stop runs no step, signals nothing, and clears /// nothing, so that the record tells [`reattach`] that the VM exited unheld, as one that exits - /// while no caller holds it does. A caller that tells how the VM ended then reattaches, as - /// [`reattach_clone`](crate::host::template::reattach_clone) does, which records a clone's - /// failure, with the loss of its status if no report of the guest's own end tells why its VM - /// exited, before it clears the record. [`stop`](Self::stop) ends such a VM as one that it - /// stopped, and clears its record, so a holder that checks whether the VMM has exited, then - /// stops the VM with it, attributes to its stop a VMM that exits in between. + /// while no caller holds it does. A caller that tells how the VM ended then reattaches. + /// [`stop`](Self::stop) ends such a VM as one that it stopped, and clears its record, so a + /// holder that checks whether the VMM has exited, then stops the VM with it, attributes to its + /// stop a VMM that exits in between. /// /// A VMM that runs as the stop begins is stopped as [`stop`](Self::stop) stops it, with the /// same result, and its exit is the stop's from then on. Fails as [`stop`](Self::stop) fails. diff --git a/nvx-rs/src/host/template/clone.rs b/nvx-rs/src/host/template/clone.rs deleted file mode 100644 index c4776cb6a..000000000 --- a/nvx-rs/src/host/template/clone.rs +++ /dev/null @@ -1,1449 +0,0 @@ -//! The plan of a clone, [`ClonePlan`]: the generation of a template that a clone leased, with the -//! clone's own image, upper, processor and memory targets, and export, which the plan checks -//! against the generation before the clone's launch, so that a request that the template cannot -//! serve fails before any VMM starts; and the clone's launch, [`ClonePlan::launch`], OpenVMM's -//! restore of the generation's snapshot with what the plan checked. - -#[cfg(all(test, any(target_os = "linux", windows)))] -mod tests; - -mod export; -mod failure; -mod launch; -mod reattach; -mod record; -mod records; -mod restore; - -pub use self::failure::{ - CloneFailure, Evidence, FailureRecord, Outcome, Progress, Stage, Teardown, status_evidence, -}; -pub use self::launch::Restoring; -pub use self::reattach::{ReattachedClone, reattach_clone}; -pub use self::records::{CloneRecords, FAILURE_FORMAT}; -pub use self::restore::{ - Activated, CloneControl, MAX_ACTIVATION_SESSIONS, READY_TO_START_TIMEOUT, RESTORE_READY_EVENT, - RESTORE_TO_READY_TIMEOUT, RestoreDeadlines, Started, -}; - -use std::collections::BTreeSet; -use std::fmt; -use std::fs::{self, File}; -use std::io; -use std::path::{Component, Path, PathBuf}; -use std::time::Instant; - -use self::export::CloneExport; -use self::failure::{Attribute, Fault, host_evidence}; -use super::key::TemplateKey; -use super::manifest::{BlockGeometry, PROCESSOR_COUNTS}; -use super::plan::MEMORY_BLOCK_SIZE; -use super::store::{Acquired, Generation, Lease, Store}; -use crate::activation::{ActivationRecord, Journal, OverlayMode, View}; -use crate::contract; -use crate::error::{Error, ErrorCode, ErrorKind, Result}; -use crate::host::filesystem::{self, ExportPlan}; -use crate::host::instance::generate_capability; -use crate::host::paths; -use crate::host::platform; -use crate::host::sandbox::distro::{ - self, CheckedDistro, DistroBinding, DistroFormat, PolicyResult, -}; -use crate::host::sandbox::plan::bound_image; -use crate::host::sandbox::scratch::{self, CheckedScratch, ScratchBinding}; -use crate::host::sandbox::{ - LayerRole, SandboxLayer, SandboxPlan, reject_container_path, reject_disk_path, -}; -use crate::host::store::LockedStore; -use crate::host::store::regular::{identity_of, is_refused, open_regular, was_removed}; -use crate::protocol::{CAPABILITY_LEN, HostMap}; - -/// Bytes in a mebibyte, the unit in which messages give the size of memory blocks. -const MIB: u64 = 1024 * 1024; - -/// The most links that [`path_directories`] follows in a path, more than a Linux host, 40, or a -/// Windows host, 63, follows in one. -const MAX_PATH_LINKS: usize = 64; - -/// What the clone of a sandbox restores of the generation of a template that it leased: the -/// adapter's plan of the clone's image and upper, the binding of the image, the binding of the -/// scratch image of an upper on one, the export of the host paths that the clone shares with its -/// root, if any, and the processor and memory targets, which [`ClonePlan::new`] checks against the -/// generation, and the settings of the clone's activation that the template does not fix, which -/// the clone's [activation record](ClonePlan::activation) names: the workload's view of its VM, -/// the overlay's mode, the journal policy of a scratch image, and the bound of a GPT disk's first -/// partition. -#[derive(Debug, Clone)] -pub struct CloneRequest { - /// The adapter's plan, which [`SandboxPlan::adapter`] makes, of the clone's image, its - /// `distro`, an EROFS layer or a GPT disk, over its upper: in RAM, a tmpfs, as - /// [`SandboxUpper::RAM`](crate::host::sandbox::SandboxUpper::RAM) plans it, or on the - /// caller's scratch image. Its network is the clone's own, whose policy the restore enforces - /// on the template's NIC in place of the template's complete deny-all policy, at the guest's - /// address that the NIC has, [`TemplateKey::guest_network`], or for a template without a NIC - /// none, or one that lets nothing through, which attaches none either. - pub adapter: SandboxPlan, - /// The binding of the image. - pub binding: DistroBinding, - /// The revision of the distro policy that the caller accepts, such as - /// [`DISTRO_POLICY_REVISION`](crate::host::sandbox::distro::DISTRO_POLICY_REVISION), which - /// [`DistroBinding::check`] requires of the binding's verdict. - pub policy_revision: u32, - /// The binding of the scratch image of an upper on one, or `None` for an upper in RAM. - pub scratch: Option, - /// The export of the host paths that the clone shares with its root, which - /// [`filesystem::plan`] planned for the container root, - /// [`ReservedTargets::CONTAINER_ROOT`](crate::host::filesystem::ReservedTargets::CONTAINER_ROOT), - /// or `None` for a clone that shares none, whose restore leaves the template's filesystem slot - /// dormant. The restore binds the export to the slot as a cold launch attaches it, an - /// aggregate of the plan's directories and files, as [`ClonePlan`] describes. - pub export: Option, - /// The processor target: the processors online in the clone's VM once restored, a prefix of - /// the template's processor capacity, of which the restore brings online those beyond the - /// template's processors online at boot. - pub processors: u32, - /// The memory target, in bytes: the RAM of the clone's VM once restored, to which the restore - /// expands the template's RAM in whole blocks. - pub memory: u64, - /// The view that the clone's workload has of its VM: its hostname and machine ID. Each - /// execution names its own identity. - pub view: View, - /// The mode of the clone's overlay, which selects its options. - pub overlay: OverlayMode, - /// Whether the filesystem of the scratch image of an upper on one may have a journal. An - /// upper in RAM has none, so the record of a clone in RAM leaves the policy out. - pub journal: Journal, - /// The most bytes that the first partition of a GPT disk may hold, at least 1, which the - /// guest checks as it reads the disk's partition table. An EROFS layer has no partitions, so - /// the record of a clone of one leaves the bound out. - pub first_partition_bound: u64, -} - -/// The upper of a clone's plan, which decides how the clone's restore attaches the template's -/// `scratch` block device. -#[derive(Debug)] -pub enum CloneUpper { - /// An upper in RAM, a tmpfs, which leaves the `scratch` block device unused: the restore - /// declares the device absent, and OpenVMM attaches it with no file, at the geometry that the - /// template recorded, as [`ClonePlan::launch`] describes, so the plan creates no file for it. - Tmpfs, - /// An upper on the caller's scratch image, which the plan checked and claimed: the claim - /// lasts as long as the checked image, as [`ScratchBinding::check`] describes. - Scratch(CheckedScratch), -} - -/// The plan of a clone: the generation of a template that the clone leased, with the clone's own -/// image, upper, and memory target, checked against the generation, the clone's activation -/// record, and a fresh capability, which [`new`](Self::new) makes under the lock of the sandbox -/// that acquired the lease, before the clone's launch. -/// -/// A restore of a template attaches the template's two block devices, its `distro` and its -/// `scratch`, at the geometries that the template recorded, as OpenVMM presents them: the -/// `distro`, the clone's image, at the placeholder's, the capacity of every clone's image, over an -/// image that may be shorter, whose blocks past its end read as zeros, and the `scratch` at the -/// template's scratch, over the clone's scratch image, of exactly that geometry, for an upper on -/// one, or with no file, for an upper in RAM, whose restore declares it absent. OpenVMM opens the -/// clone's files itself, by their paths, as [Block devices by path](Store#block-devices-by-path) -/// describes. The plan checks each of the clone's inputs as OpenVMM will find it, and holds what -/// the checks opened: -/// -/// - The image, the clone's `distro`, an EROFS layer or a GPT disk, as its binding names it, must -/// pass the check of its binding, [`DistroBinding::check`], and fit the capacity of the -/// template's placeholder, as its key records it, [`TemplateKey::distro`]: no longer than the -/// placeholder, with the same logical and physical block sizes, which -/// [`CheckedDistro::geometry`] reads through the checked file, whatever the placeholder's -/// format. On Linux, a file's physical block size is the I/O block size of its filesystem, so -/// the image must lie on a filesystem of the placeholder's. At its path, it must be a regular -/// file, not a link, nor on Windows any other reparse point, that is the file that the check -/// found, that belongs to the current user or to an administrator, and that no one else may -/// change. The plan holds the file that the check opened, for reading alone. Nothing resizes, -/// pads, or writes the image. As at a cold launch, the guest reads and checks a GPT disk's -/// partition table, with `nvx::gpt`, before it assembles the root, so the plan reads none of -/// it: of a disk shorter than the capacity, `nvx::gpt` requires only that its backup header lie -/// inside the capacity, as it does at the disk's end. -/// - Neither the image nor a scratch image may be a file that the sandbox's store owns, by any -/// path, through a link among its directories, say, or a hard link: the store's transitions -/// replace or remove its files, as a launch replaces its log and the clone's activation record, -/// and a release removes the clone binding. The store creates nothing but regular files under -/// its names, and its control socket, so the plan also refuses a link or a directory under one -/// of them, through which the path of an image could pass. -/// - An upper in RAM, a tmpfs, leaves the `scratch` block device unused, which the restore -/// attaches all the same, since the template's guest negotiated it: the restore declares it -/// absent, and OpenVMM attaches it with no file, as [`launch`](Self::launch) describes. The plan -/// creates nothing for it. -/// - An upper on the caller's scratch image must pass the check of its binding, -/// [`ScratchBinding::check`], which checks the image's identity, kind, and length and claims it -/// for the clone, and present the geometry of the template's scratch, its length included, -/// which [`CheckedScratch::geometry`] reads through the checked file. At its path, it must be a -/// regular file, not a link, nor on Windows any other reparse point, that is the file that the -/// check found, and that no one else may change. Nor may it be a file of the snapshot of any of -/// the store's generations, by any path, through a hard link, say: every clone of a generation -/// restores its snapshot in place, while its guest writes its scratch image. So, once claimed, -/// it must be outside the store's directory, wherever the links on its path lead: the store's -/// generations hold their snapshots there, and its captures write theirs there, which their -/// publications make generations' by renaming directories within it, so a file in the store -/// could be, or become as the plan examines it, a file of a snapshot, while nothing that the -/// host does moves a file into the store's directory. And it must have a single name: another, -/// a hard link, could make it a file of a snapshot, of this store or of another, that its -/// location does not show. These checks open nothing in the store, however many generations it -/// holds. They resolve the aliases that the file system makes, links, junctions, directories -/// held open, and other names, but not a mount or a network share that exposes the store's -/// directories elsewhere: a store's directories must not be exposed that way. The plan holds -/// the checked image, and with it the claim. Nothing reads, formats, or copies the image. -/// - The processor target must be 1, 2, 4, or 8, from the template's processors online at boot to -/// its processor capacity, as its key records them, [`TemplateKey::boot_processors`] and -/// [`TemplateKey::processors`]. A template whose processors are all online at boot records none -/// online at boot, and serves the target of its capacity alone. -/// - The memory target must be whole blocks of [`MEMORY_BLOCK_SIZE`], and of the template's own -/// memory blocks, from the template's RAM to its RAM capacity. -/// - The network of the adapter's plan must attach a NIC, as -/// [`SandboxNetwork::openvmm_arguments`](crate::host::sandbox::SandboxNetwork::openvmm_arguments) -/// lowers it, if and only if the template has one, at the guest's address of the template's NIC, -/// which its key records, [`TemplateKey::guest_network`]. A template captures its NIC under the -/// complete deny-all policy, which the restore replaces with the policy of the clone's network, -/// so a template serves the clones of every network policy at its guest's address: egress -/// rules, a proxy, ingress, host-loopback access, and host-loopback forwards alike. -/// - The export, if the request has one, must be a plan that [`filesystem::plan`] could have made -/// - The export, if the request has one, must be a plan that [`filesystem::plan`] could have made -/// for the container root, as [`ExportPlan::check`] finds it: an aggregate of up to -/// [`MAX_CHILDREN`](crate::host::filesystem::MAX_CHILDREN) host directories and files, each -/// read-only or read-write, with OpenVMM's hidden, re-exposed, and writable paths, and its -/// owner. A template's filesystem slot is dormant, and the restore of a template, a platform -/// snapshot, binds the aggregate to it as a cold launch attaches one, so the sources of the -/// plan's binds name their paths by their child's index, as the mapping table, -/// [`table`](Self::table), sends them, as at a cold launch. None of its directories and files -/// may lie inside the template store's directory, nor any of its directories hold it, by the -/// paths that the plan and the store's directory resolve to, so that no guest reaches the -/// payload of a generation through it, nor may the export let the guest write in a directory -/// through which the store's path leads, links and `..` included, since the host reaches the -/// store, and OpenVMM its snapshots, by that path. Nor may one of its directories be, or hold, -/// the directory of the clone's sandbox, which the lease names by its resolved path, and in -/// which the host writes the clone's records, which the guest could otherwise read or change, -/// or turn elsewhere, into a leased snapshot, say, nor may one of its files lie in that -/// directory itself, where it may be one of those records, though a directory inside the -/// sandbox's, among the caller's files, may be exported, as may a file in it. These checks -/// compare paths, so an alias of the -/// store that an administrator configures is outside them, as the directories above the store -/// are: a bind mount or a mount point of the store, or of a directory above or inside it, a -/// loopback share, or a hard link of a payload. Whoever names an export must not name such an -/// alias, nor a directory that holds one. The launch checks the plan's host objects again, with -/// [`ExportPlan::verify`], before it restores the clone, but OpenVMM resolves the paths of an -/// export again at each launch and restore, so the directories above each of the export's -/// directories and files must be ones that no guest may change, as those above the clone's -/// block devices must, which [Block devices by path](Store#block-devices-by-path) describes: -/// whoever names an export must not name one inside a directory that a writable export of -/// another sandbox or clone holds, whose guest could replace the directory with a link, and -/// turn the export elsewhere, to a leased payload, say. A request without an export leaves the -/// slot dormant. -/// - The activation record, which the clone's guest applies in place of the configuration that -/// the template captured, names the restore, with a fresh transaction ID, the clone's image, -/// upper, export, and settings, its network, and the resources that the clone must -/// have, as [`activation`](Self::activation) describes. Its settings must keep the limits of a -/// record, as [`Configuration::check`] checks them. -/// - The capability is fresh: the plan draws it for the clone's launch, [`launch`](Self::launch). -/// -/// The plan's checks read the metadata of the clone's files, never their content. -/// -/// ```no_run -/// use std::path::Path; -/// use std::time::{Duration, Instant}; -/// -/// use nvx::activation::{Journal, MachineIdMode, OverlayMode, View}; -/// use nvx::host::sandbox::distro::{DISTRO_POLICY_REVISION, DistroBinding}; -/// use nvx::host::sandbox::{SandboxDistro, SandboxPlan, SandboxUpper}; -/// use nvx::host::store::{CallerFiles, RuntimeIdentity}; -/// use nvx::host::template::{ -/// CloneRequest, ClonePlan, Generation, HostTime, MEMORY_BLOCK_SIZE, Store, -/// }; -/// -/// # fn clone( -/// # store: &Store, -/// # generation: &Generation, -/// # image: SandboxDistro, -/// # binding: DistroBinding, -/// # runtime: &RuntimeIdentity, -/// # ) -> nvx::Result<()> { -/// let files = CallerFiles::new("config.json")?; -/// let mut acquired = store.acquire( -/// generation.generation(), -/// "/var/lib/example/sandboxes/clone", -/// &files, -/// &HostTime::now()?, -/// None, -/// )?; -/// let request = CloneRequest { -/// adapter: SandboxPlan::adapter(image, SandboxUpper::RAM), -/// binding, -/// policy_revision: DISTRO_POLICY_REVISION, -/// scratch: None, -/// export: None, -/// processors: 2, -/// memory: 8 * MEMORY_BLOCK_SIZE, -/// view: View { -/// hostname: b"clone".to_vec(), -/// machine_id: MachineIdMode::Image, -/// }, -/// overlay: OverlayMode::UntrustedLowers, -/// journal: Journal::Allowed, -/// first_partition_bound: 64 * 1024, -/// }; -/// let deadline = Instant::now() + Duration::from_secs(5); -/// let plan = ClonePlan::new(store, &mut acquired, &request, deadline)?; -/// // OpenVMM restores the generation's snapshot under the runtime that captured the template. -/// let restoring = plan.launch(&mut acquired, Path::new("openvmm"), runtime)?; -/// // ... the restore, whose VMM sends its restore-ready event to `restoring.ready_endpoint()`, -/// // serves its control console at `restoring.starting().endpoint()`, and whose guest applies -/// // `restoring.activation()` ... -/// # let _ = restoring; -/// # Ok(()) -/// # } -/// ``` -/// -/// [`Configuration::check`]: crate::activation::Configuration::check -pub struct ClonePlan { - /// The lease of the generation. - lease: Lease, - /// The leased generation, as the store's index lists it. - generation: Generation, - /// The generation's compatibility key. - key: TemplateKey, - /// The generation's snapshot, which the clone restores. - snapshot: PathBuf, - /// The clone's image, which the check of its binding opened. - image: CheckedDistro, - /// The clone's upper, with the file of its `scratch` block device. - upper: CloneUpper, - /// The processor target. - processors: u32, - /// The memory target, in bytes. - memory: u64, - /// The clone's activation record. - activation: ActivationRecord, - /// The capability of the clone's launch. - capability: [u8; CAPABILITY_LEN], - /// The clone's devices, which its restore attaches: the adapter's plan of its image and - /// network, over the file of its `scratch` block device. - devices: SandboxPlan, - /// The clone's export, which its restore binds to the template's dormant filesystem slot, if - /// it has one. - export: Option, -} - -/// How a plan reaches the clone's files: as the checks of their bindings reach them, or, in tests, -/// through opens that read no content, or that reach files on filesystems of other block sizes. -#[derive(Clone, Copy)] -struct Access<'a> { - /// How the check of the image's binding reaches the image. - distro: distro::Access<'a>, - /// How the check of the scratch image's binding reaches the image. - scratch: scratch::Access<'a>, -} - -impl Access<'static> { - /// The access of every plan outside tests. - const CHECKED: Self = Self { - distro: distro::Access::GUARDED, - scratch: scratch::Access::WRITABLE, - }; -} - -impl ClonePlan { - /// Plans the clone of the sandbox of `acquired`, which leased a generation of `store`, as - /// `request` asks, waiting until `deadline` for a claim of its scratch image that another - /// launch holds to be released. - /// - /// The plan is made under the lock that acquired the lease, which `acquired` holds, before the - /// launch that the lease reserves, [`Lease::launch`], which the caller then runs under the - /// same lock, with [`launch`](Self::launch), while the sandbox records no VM and no launch. It - /// takes `acquired` exclusively while it plans, as the transitions of a sandbox's store take - /// its lock, since it checks the files of the sandbox's store, and borrows nothing of it once - /// it returns. It creates nothing in the sandbox's directory. It reads the generation's key, - /// [`Store::read_key`], without computing its identity again, and checks the generation's - /// snapshot, [`Store::snapshot_dir`], then checks the clone's files, as the [type](Self) - /// describes. - /// - /// Fails with a [`CloneFailure`] at [`Stage::Plan`], and holds nothing. The failure's record - /// holds the evidence of the check that failed, as below, no transaction, and no VM, - /// [`Teardown::NoVm`], and the plan records it nowhere: the caller holds it, and releases the - /// lease. The evidence of a failure of the host's own operations is - /// [`Evidence::ResourceExhaustion`] if an I/O error among the error's sources says that the - /// host ran out of storage, memory, or descriptors, and [`Evidence::HostFailure`] otherwise. - /// The plan checks in this order: - /// - /// - with [`ErrorKind::Unsupported`] for a request that no clone takes: a plan of the sandbox - /// profile, with workload settings and its shares, [`Evidence::InvalidSettings`]; a plan - /// whose image is not the format and identity that the binding names, as for a binding whose - /// identity is not a UUID in its canonical text, [`Evidence::InvalidFormat`]; an image or a - /// scratch image whose path is relative, which OpenVMM would resolve against the directory in - /// which it runs, holds a comma or a semicolon, which OpenVMM's block device options would - /// split, or whose name ends in `.vhd`, `.vhdx`, `.vmgs`, or `.iso`, which OpenVMM would not - /// attach as a raw image, and an upper on a scratch image without the image's binding, or one - /// in RAM with a binding, [`Evidence::InvalidSettings`]; and an export plan that - /// [`filesystem::plan`] could not have made for the container root, as [`ExportPlan::check`] - /// finds it, such as one that maps paths but exports no directory or file, exports some but - /// maps no path, or exports more directories and files than OpenVMM attaches, - /// [`Evidence::InvalidMappings`]; - /// - with [`ErrorKind::Other`] for a lease of another store, and a lock that is not the one - /// that acquired the lease, before the launch that it reserves began, - /// [`Evidence::HostFailure`]; - /// - with [`ErrorKind::Unsupported`] for an export of which a directory or a file, by the path - /// that its plan resolved, lies inside the template store's directory, [`Store::dir`], by - /// the path that it resolves to, or of which a directory holds it, or is or holds the - /// directory of the clone's sandbox, [`Lease::sandbox`], or of which a file lies in that - /// directory itself, or that lets the guest write in a directory through which the store's - /// path leads, as the host resolves it, links and `..` included, which the guest could then - /// turn elsewhere, or keep the host from following, [`Evidence::InvalidMappings`], and with - /// [`ErrorKind::Io`] if the store's directory, or a directory through which its path leads, - /// cannot be resolved, the host's evidence; - /// - with [`ErrorKind::NotFound`] if the store no longer indexes the leased generation, and as - /// [`Store::index`] fails, the host's evidence; with [`ErrorKind::Io`] if it indexes it with - /// another key than the lease names, [`Evidence::MalformedGeneration`]; and as - /// [`Store::read_key`] and [`Store::snapshot_dir`] fail, once the store's directory of - /// generations passed their check, whose failure is the host's: the host's evidence for an - /// I/O error, [`Evidence::IncompatibleHost`] for a key of another encoding version than this - /// build reads, with [`ErrorCode::FormatUnsupported`], which a build that reads it may - /// serve, and [`Evidence::MalformedGeneration`] for a generation whose payload, key, or - /// snapshot is missing, is not private to the current user, or is malformed, as a key that - /// [`Store::read_key`] refuses for what it is: a link, a file that is not a regular file, or - /// one that holds more than a key may; - /// - with [`ErrorKind::Unsupported`] for a request that the template cannot serve: a template - /// whose key names other contracts than this build speaks, the boot and control contracts - /// of the runtime that captured it included, [`Evidence::IncompatibleHost`]; a plan whose - /// network does not lower to OpenVMM's arguments, as - /// [`SandboxNetwork::openvmm_arguments`](crate::host::sandbox::SandboxNetwork::openvmm_arguments) - /// finds it, a plan with a network for a template without a NIC, or without one for a - /// template with a NIC, or one at another guest's address than the template's NIC, as its - /// key records it, [`TemplateKey::guest_network`], [`Evidence::InvalidSettings`]; and a - /// processor target other than 1, 2, 4, or 8, or - /// outside the template's processors online at boot and its processor capacity, and a - /// memory target that is not whole blocks of [`MEMORY_BLOCK_SIZE`] and of the template's - /// memory blocks, from the template's RAM to its RAM capacity, [`Evidence::InvalidTargets`]; - /// - with [`ErrorKind::Unsupported`] for settings that no activation record holds, as - /// [`Configuration::check`] finds them, such as a GPT disk's first-partition bound of 0, a - /// hostname that is empty, longer than - /// [`MAX_HOSTNAME_BYTES`](crate::activation::MAX_HOSTNAME_BYTES), or holds a NUL, - /// [`Evidence::InvalidActivate`], and with - /// [`ErrorKind::Other`] if the transaction ID of the clone's restore cannot be drawn, the - /// host's evidence; - /// - with [`ErrorKind::Unavailable`] for a link or a directory under a name that the sandbox's - /// store owns, and with [`ErrorKind::Io`] if the store's files cannot be listed or examined, - /// the host's evidence; - /// - as [`DistroBinding::check`] and [`CheckedDistro::geometry`] fail: with - /// [`ErrorKind::Unavailable`] for a binding that holds a verdict of another revision of the - /// distro policy than the request accepts, [`Evidence::StaleBinding`], and with - /// [`ErrorKind::Unsupported`] for one whose verdict rejects the image, - /// [`Evidence::AuthorizationDenied`]; with [`ErrorKind::Unavailable`] for an image that - /// changed, was replaced, or went since its binding was made, or whose manifest did, and for - /// an image that is not, at its path, the regular file that the check found, of the current - /// user or an administrator, and that no one else may change, - /// [`Evidence::SuppliedIdentity`], and with [`ErrorKind::Io`] if the image cannot be opened - /// at its path otherwise, or identified or examined once open, the host's evidence; with - /// [`ErrorKind::Unsupported`] for an image of other block sizes than the template's - /// placeholder, or of a geometry that OpenVMM refuses, [`Evidence::SuppliedGeometry`], or - /// longer than the placeholder, the capacity of the template's clones' images, - /// [`Evidence::SuppliedLength`]; and with [`ErrorKind::Unsupported`] for an image that is a - /// file of the sandbox's store, [`Evidence::InvalidSettings`]; - /// - for an upper on a scratch image, with [`ErrorKind::Io`] if the image's location cannot be - /// compared with the template store's directory, or its names cannot be counted, the host's - /// evidence, and as [`ScratchBinding::check`] and [`CheckedScratch::geometry`] fail: with - /// [`ErrorKind::TimedOut`] and [`ErrorCode::ScratchClaimed`] for an image that another - /// launch still claims, [`Evidence::UnusableScratch`]; with [`ErrorKind::Unavailable`] for an - /// image that changed, was replaced, or went since its binding was made, or that is not, at - /// its path, the regular file that the check found, of the current user or an administrator, - /// and that no one else may change, [`Evidence::SuppliedIdentity`], and with - /// [`ErrorKind::Io`] if the image cannot be opened at its path otherwise, or identified or - /// examined once open, the host's evidence; with [`ErrorKind::Unsupported`] for an image of - /// another length than the template's scratch, [`Evidence::SuppliedLength`], or of other - /// block sizes, [`Evidence::SuppliedGeometry`]; and with [`ErrorKind::Unsupported`] for an - /// image that is a file of the sandbox's store, or that is in the template store's - /// directory, as the files of its snapshots are, or has another name, or whose path is not - /// Unicode, [`Evidence::InvalidSettings`]. - /// - /// [`Configuration::check`]: crate::activation::Configuration::check - pub fn new( - store: &Store, - acquired: &mut Acquired, - request: &CloneRequest, - deadline: Instant, - ) -> Result { - Self::new_with(store, acquired, request, deadline, Access::CHECKED) - } - - /// Plans a clone as [`new`](Self::new) does, reaching its files through `access`. - fn new_with( - store: &Store, - acquired: &mut Acquired, - request: &CloneRequest, - deadline: Instant, - access: Access<'_>, - ) -> Result { - let lease = acquired.lease.clone(); - Self::plan(store, acquired, request, deadline, access).map_err(|fault| { - let record = FailureRecord::new( - &lease, - None, - (Stage::Plan, Progress::Planned), - fault.evidence, - None, - Teardown::NoVm, - &fault.error, - ); - CloneFailure::new(fault.error, record) - }) - } - - /// Plans a clone as [`new`](Self::new) does, reaching its files through `access`, and - /// returns the fault of the check that failed. - fn plan( - store: &Store, - acquired: &mut Acquired, - request: &CloneRequest, - deadline: Instant, - access: Access<'_>, - ) -> Result { - let export = check_request(request)?; - let Acquired { lease, locked } = acquired; - check_lease(store, lease, locked).attribute(Evidence::HostFailure)?; - if let Some(export) = &export { - // The plan resolved the export's paths, so the store's path is resolved as well, so - // that no other spelling of it hides an overlap, and so are the directories through - // which it leads, in which the guest must not write. The lease names the sandbox's - // directory, in which the host writes the clone's records, by its resolved path. - let dir = store.dir(); - let unresolved = |error: io::Error| { - Error::io(format!( - "cannot resolve the template store {}", - dir.display() - )) - .with_source(error) - }; - let resolved = filesystem::canonicalize(dir) - .map_err(unresolved) - .on_host()?; - let through = path_directories(dir).map_err(unresolved).on_host()?; - export - .check_outside(dir, &resolved, &through, lease.sandbox()) - .attribute(Evidence::InvalidMappings)?; - } - let generation = store - .index() - .on_host()? - .get(lease.generation()) - .cloned() - .ok_or_else(|| { - Fault::new( - Evidence::HostFailure, - Error::new( - ErrorKind::NotFound, - format!( - "the template store {} holds no generation {}, which the lease {} \ - names", - store.dir().display(), - lease.generation(), - lease.id() - ), - ), - ) - })?; - if generation.key() != lease.key() { - return Err(Fault::new( - Evidence::MalformedGeneration, - Error::io(format!( - "the generation {} of the template store {} has the key {}, not the key {} \ - that the lease {} names", - generation.generation(), - store.dir().display(), - generation.key(), - lease.key(), - lease.id() - )), - )); - } - // A store whose directory of generations is not private fails every generation: the - // host's, not the generation's. - store.check_generations_dir().on_host()?; - let key = store.read_key(&generation).map_err(generation_fault)?; - let snapshot = store.snapshot_dir(&generation).map_err(generation_fault)?; - check_template(request, &key, &generation)?; - let activation = record::build(request, &key)?; - let owned = StoreFiles::of(locked).on_host()?; - let image = check_image(request, &key, &owned, access)?; - let capability = generate_capability().on_host()?; - let upper = match (request.adapter.upper().scratch_path(), &request.scratch) { - (Some(path), Some(binding)) => CloneUpper::Scratch(check_scratch( - path, - binding, - &key, - (&owned, store), - deadline, - access, - )?), - _ => CloneUpper::Tmpfs, - }; - let devices = devices_of(request); - Ok(Self { - lease: lease.clone(), - generation, - key, - snapshot, - image, - upper, - processors: request.processors, - memory: request.memory, - activation, - capability, - devices, - export, - }) - } - - /// Returns the lease of the generation, which reserves the clone's launch, - /// [`Lease::launch`]. - pub fn lease(&self) -> &Lease { - &self.lease - } - - /// Returns the leased generation, as the store's index lists it. - pub fn generation(&self) -> &Generation { - &self.generation - } - - /// Returns the generation's compatibility key, as the store read it from the generation's - /// payload. - pub fn key(&self) -> &TemplateKey { - &self.key - } - - /// Returns the generation's snapshot, which the clone restores. - pub fn snapshot(&self) -> &Path { - &self.snapshot - } - - /// Returns the clone's image, which the check of its binding opened, for reading alone, and - /// which OpenVMM opens again at its path, [`CheckedDistro::path`], as the clone's `distro`. - pub fn image(&self) -> &CheckedDistro { - &self.image - } - - /// Returns the clone's upper, which decides how its restore attaches the `scratch` block - /// device. - pub fn upper(&self) -> &CloneUpper { - &self.upper - } - - /// Returns the file of the clone's `scratch` block device, which OpenVMM opens at this path: - /// the caller's scratch image, or `None` for an upper in RAM, whose restore declares the - /// device absent. - pub fn scratch_path(&self) -> Option<&Path> { - match &self.upper { - CloneUpper::Tmpfs => None, - CloneUpper::Scratch(checked) => Some(checked.path()), - } - } - - /// Returns the processor target: the processors online in the clone's VM once restored. - pub fn processors(&self) -> u32 { - self.processors - } - - /// Returns the memory target, in bytes. - pub fn memory(&self) -> u64 { - self.memory - } - - /// Returns the export plan of the host paths that the clone shares with its root, which its - /// restore binds to the template's dormant filesystem slot, or `None` for a clone that shares - /// none, whose slot stays dormant. - pub fn export(&self) -> Option<&ExportPlan> { - self.export.as_ref().map(CloneExport::plan) - } - - /// Returns the clone's mapping table, which its restore sequence sends once the clone is - /// ready, before its sandbox starts, as [`Activated::start`] describes: the binds of the - /// export plan, in their order, each from its path in the export, by its child's index, as a - /// cold launch's guest receives them, [`ExportPlan::guest_table`]. A clone without an export - /// has an empty table, and the sequence sends none. - pub fn table(&self) -> Vec { - self.export - .as_ref() - .map_or_else(Vec::new, CloneExport::table) - } - - /// Returns the clone's activation record, which its guest applies in place of the - /// configuration that the template captured, and whose restore transaction ID, as - /// [`ReadyActivation::of`](crate::activation::ReadyActivation::of) returns it, the guest's - /// Ready reports. - /// - /// The record holds the restore section, with the restore's transaction ID, 16 random bytes - /// that the plan drew; the image, in the format that its binding names: an EROFS layer, with - /// the UUID that the guest expects of its filesystem, or a GPT disk, with the GUID that the - /// guest expects of the disk, each if the adapter's plan names one, and the request's bound of - /// the disk's first partition; the upper, a tmpfs capped at half of the VM's memory, or the - /// scratch image with the request's journal policy; the request's overlay mode and view; - /// the export, if the request has one, an aggregate, and the - /// number of its mappings, the binds of the plan, from 1 to - /// [`MAX_MAPPINGS`](crate::activation::MAX_MAPPINGS), or no export and no mappings; the - /// clone's network, if the template has a NIC: the DNS server that OpenVMM serves the guest at a gateway - /// under that policy, if any, whether the guest publishes its listeners, as it does under a - /// policy that allows ingress or host-loopback access both ways, and the URL of the policy's - /// proxy as the guest sees it, if any; and the processor target, which the restore brings - /// online, and the memory target, as the resources that the clone must have. - pub fn activation(&self) -> &ActivationRecord { - &self.activation - } - - /// Returns the capability of the clone's launch, which the plan drew, and which the VMM - /// receives, as [`LockedStore::begin_launch`] stores it. - pub fn capability(&self) -> &[u8; CAPABILITY_LEN] { - &self.capability - } -} - -impl fmt::Debug for ClonePlan { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - // The capability authenticates control sessions, so it is never shown. - formatter - .debug_struct("ClonePlan") - .field("lease", &self.lease) - .field("generation", &self.generation) - .field("snapshot", &self.snapshot) - .field("image", &self.image) - .field("upper", &self.upper) - .field("processors", &self.processors) - .field("memory", &self.memory) - .field("activation", &self.activation) - .field("export", &self.export()) - .finish_non_exhaustive() - } -} - -/// Returns the devices of the clone of `request`: the adapter's plan of its image over its upper, -/// which renders the `scratch` block device of an upper on a scratch image alone, since the -/// restore declares that of an upper in RAM absent, and of its network, whose policy the restore -/// enforces on the template's NIC. -fn devices_of(request: &CloneRequest) -> SandboxPlan { - let adapter = &request.adapter; - SandboxPlan::adapter(adapter.distro().clone(), adapter.upper().clone()) - .with_network(adapter.network().cloned()) -} - -/// Refuses a request that no clone takes, as [`ClonePlan::new`] describes, without reading or -/// opening anything, and returns the request's export, if it has one, which the checks admitted. -fn check_request(request: &CloneRequest) -> Result, Fault> { - let adapter = &request.adapter; - if adapter.settings().is_some() { - return Err(Fault::new( - Evidence::InvalidSettings, - Error::unsupported( - "a clone restores an adapter's plan, not a plan of the sandbox profile, whose \ - workload settings no template holds", - ), - )); - } - let distro = adapter.distro(); - let binding = &request.binding; - if bound_image(binding).attribute(Evidence::InvalidFormat)? != distro.image() { - return Err(Fault::new( - Evidence::InvalidFormat, - Error::unsupported(format!( - "the image {} is not the {} layer with identity {} that its distro binding names", - distro.path().display(), - binding.format(), - binding.identity() - )), - )); - } - check_disk_path(distro.path(), "image").attribute(Evidence::InvalidSettings)?; - match (adapter.upper().scratch_path(), &request.scratch) { - (Some(path), Some(_)) => { - check_disk_path(path, "scratch image").attribute(Evidence::InvalidSettings)?; - } - (None, None) => {} - (Some(path), None) => { - return Err(Fault::new( - Evidence::InvalidSettings, - Error::unsupported(format!( - "the upper of a clone on the scratch image {} needs the image's scratch \ - binding", - path.display() - )), - )); - } - (None, Some(_)) => { - return Err(Fault::new( - Evidence::InvalidSettings, - Error::unsupported("the upper of a clone in RAM takes no scratch binding"), - )); - } - } - request - .export - .as_ref() - .map(CloneExport::new) - .transpose() - .attribute(Evidence::InvalidMappings) -} - -/// Returns the fault of `error`, with which the template store's read of the key or of the -/// snapshot of the leased generation failed, once the store's directory of generations passed its -/// check: the host's, which cannot restore the template, for a key of another encoding version -/// than this build reads, as [`TemplateKey::from_bytes`] refuses it with -/// [`ErrorCode::FormatUnsupported`], since a build that reads it may serve the generation; the -/// generation's, which is malformed, for a payload, a key, or a snapshot that is missing, that is -/// not private to the current user, or that is not one that the store writes, such as a key that -/// the read refuses for what it is, as [`is_refused`] tells: a link, a file that is not a regular -/// file, or one that holds more than a key may; and the host's for any other I/O error among the -/// error's sources. -fn generation_fault(error: Error) -> Fault { - if error.code() == ErrorCode::FormatUnsupported { - return Fault::new(Evidence::IncompatibleHost, error); - } - let mut source = std::error::Error::source(&error); - while let Some(cause) = source { - if let Some(io) = cause.downcast_ref::() - && io.kind() != io::ErrorKind::NotFound - && !is_refused(io) - { - return Fault::new(host_evidence(&error), error); - } - source = cause.source(); - } - Fault::new(Evidence::MalformedGeneration, error) -} - -/// Returns the directories through which `path` leads, as the host resolves it, each resolved as -/// [`filesystem::canonicalize`] resolves the paths of an export plan: for each name of the path, -/// `..` included, from the current directory's own for a relative path, the directory in which -/// the host looks it up, which the path before the name resolves to, and, for a name that is a -/// link, the directories through which the link's target leads from there, through at most -/// [`MAX_PATH_LINKS`] links in all. -/// -/// The host reaches the template store, and OpenVMM the snapshots in it, by the path that the -/// store names, so a guest that could write in one of these directories could replace a link or -/// a directory of the path, and turn the host's later operations on the store elsewhere, or -/// make the directory unsearchable, and keep the host from the store. -fn path_directories(path: &Path) -> io::Result> { - let mut directories = BTreeSet::new(); - let mut pending = vec![(PathBuf::new(), std::path::absolute(path)?)]; - let mut links = 0; - while let Some((start, rest)) = pending.pop() { - let mut prefix = start; - for component in rest.components() { - match component { - Component::Normal(_) => {} - Component::CurDir | Component::ParentDir => { - // Unix looks `.` and `..` up in the directory before them, as it does a name, - // so a guest that could change that directory could keep the host from - // searching it. Windows resolves them from a path's text, as - // `std::path::absolute` did. - directories.insert(filesystem::canonicalize(&prefix)?); - prefix.push(component); - continue; - } - Component::RootDir | Component::Prefix(_) => { - // The root and a prefix name no entry. - prefix.push(component); - continue; - } - } - let parent = filesystem::canonicalize(&prefix)?; - prefix.push(component); - if fs::symlink_metadata(&prefix)?.file_type().is_symlink() { - links += 1; - if links > MAX_PATH_LINKS { - return Err(io::Error::other(format!( - "{} leads through more than {MAX_PATH_LINKS} links", - path.display() - ))); - } - // A link's target that is absolute replaces the directory that it starts from. - pending.push((parent.clone(), fs::read_link(&prefix)?)); - } - directories.insert(parent); - } - } - Ok(directories.into_iter().collect()) -} - -/// Refuses the path of the file of a clone's block device, which messages call `what`, that -/// OpenVMM would not open as the plan checks it: a relative one, which it would resolve against the -/// directory in which it runs, one with a comma or a semicolon, which its block device options -/// would split, and one whose name ends in `.vhd`, `.vhdx`, `.vmgs`, or `.iso`, which it would not -/// attach as a raw image. -fn check_disk_path(path: &Path, what: &str) -> Result<()> { - let shown = path.display().to_string(); - if !path.is_absolute() { - return Err(Error::unsupported(format!( - "the {what} of a clone must be absolute, since OpenVMM would resolve it against the \ - directory in which it runs: {shown}" - ))); - } - reject_disk_path(path, &shown)?; - reject_container_path(path, &shown) -} - -/// Refuses a lease of a store other than `store`, and a lock, `locked`, that is not the one that -/// acquired `lease`, before the launch that it reserves began. -fn check_lease(store: &Store, lease: &Lease, locked: &LockedStore) -> Result<()> { - if lease.store() != store.id() { - return Err(Error::other(format!( - "the lease {} is one of the template store {}, not of this one, {}", - lease.id(), - lease.store(), - store.id() - ))); - } - if locked.dir() != lease.sandbox() || locked.reserved() != Some(lease.launch()) { - return Err(Error::other(format!( - "a clone is planned under the lock that acquired its lease, before the launch that \ - the lease reserves begins, but the lock of the sandbox in {} reserves no launch {} \ - of the lease {}", - locked.dir().display(), - lease.launch(), - lease.id() - ))); - } - Ok(()) -} - -/// Refuses a request that the template of `generation`, whose key is `key`, cannot serve, as -/// [`ClonePlan::new`] describes, without reading or opening anything. -fn check_template( - request: &CloneRequest, - key: &TemplateKey, - generation: &Generation, -) -> Result<(), Fault> { - let name = generation.generation(); - // The key's contracts are those through which its clones are restored; the runtime's are - // those through which this build boots and drives the guest that the template holds. - let runtime = key.runtime(); - if !key.speaks_current_contracts() - || runtime.boot_contract != contract::BOOT_CONTRACT_VERSION - || runtime.control_contract != contract::CONTROL_CONTRACT_REVISION - || runtime.control_session != contract::CONTROL_SESSION_PROTOCOL_VERSION - { - return Err(Fault::new( - Evidence::IncompatibleHost, - Error::unsupported(format!( - "the template's generation {name} speaks other contracts than this build, which \ - restores none of its clones" - )), - )); - } - // A template captures its NIC under the complete deny-all policy, which every restore replaces - // with the policy of its clone's network, so it serves the clones of every network at its - // guest's address, or, without a NIC, those of none, or of one that lets nothing through, - // which attaches no NIC either. - let requested = match request.adapter.network() { - None => None, - Some(network) => { - let attached = !network - .openvmm_arguments() - .attribute(Evidence::InvalidSettings)? - .is_empty(); - attached.then(|| network.guest_network()) - } - }; - let problem = match (requested, key.guest_network()) { - (Some(_), None) => Some(format!( - "the adapter's plan of a clone attaches a network, but the template's generation \ - {name} has no NIC" - )), - (None, Some(_)) => Some(format!( - "the template's generation {name} has a NIC, which its clones restore with the \ - network of the adapter's plan, but the plan has none" - )), - (Some(requested), Some(template)) if requested != template => Some(format!( - "the adapter's plan of a clone attaches a network at {requested}, but the NIC of the \ - template's generation {name} is at {template}: a template serves the clones of its \ - own guest's address alone" - )), - _ => None, - }; - if let Some(problem) = problem { - return Err(Fault::new( - Evidence::InvalidSettings, - Error::unsupported(problem), - )); - } - let processors = request.processors; - let capacity = key.processors(); - // A template that records no processors online at boot has all of them online, and a restore - // brings none online. - let boot = match key.boot_processors() { - 0 => capacity, - boot => boot, - }; - if !PROCESSOR_COUNTS.contains(&processors) || processors < boot || processors > capacity { - return Err(Fault::new( - Evidence::InvalidTargets, - Error::unsupported(format!( - "a clone of the template's generation {name} cannot restore with a processor \ - target of {processors}: a target must be 1, 2, 4, or 8, from the template's \ - processors online at boot, {boot}, to its capacity, {capacity}" - )), - )); - } - let memory = request.memory; - let block = key.memory_block_size(); - if memory < key.memory() - || memory > key.memory_capacity() - || !memory.is_multiple_of(MEMORY_BLOCK_SIZE) - || !memory.is_multiple_of(block) - { - return Err(Fault::new( - Evidence::InvalidTargets, - Error::unsupported(format!( - "a clone of the template's generation {name} cannot restore with {memory} bytes \ - of RAM: its memory target must be whole blocks of {} MiB, and of the template's \ - memory blocks of {block} bytes, from the template's {} bytes of RAM to its \ - capacity of {} bytes", - MEMORY_BLOCK_SIZE / MIB, - key.memory(), - key.memory_capacity() - )), - )); - } - Ok(()) -} - -/// Checks the image of `request` against its binding, through `access`, and then its geometry, -/// against the capacity of the placeholder that `key` records, the image at its path, and that it -/// is none of `owned`, the files of the sandbox's store, as [`ClonePlan`] describes, and returns -/// the file that the binding's check opened. -fn check_image( - request: &CloneRequest, - key: &TemplateKey, - owned: &StoreFiles, - access: Access<'_>, -) -> Result { - let path = request.adapter.distro().path(); - let shown = path.display(); - let text = path.to_str().ok_or_else(|| { - Fault::new( - Evidence::InvalidSettings, - Error::unavailable(format!( - "the image {shown} has a path that is not valid Unicode" - )), - ) - })?; - let binding = &request.binding; - let layer = match binding.format() { - DistroFormat::Erofs => SandboxLayer::new(LayerRole::Distro, text, binding.identity()), - DistroFormat::Gpt => SandboxLayer::gpt(LayerRole::Distro, text, binding.identity()), - }; - let policy = binding.policy(); - let checked = binding - .check_with(&layer, request.policy_revision, access.distro) - .map_err(|error| { - // The check refuses a stale binding, then one whose verdict rejects the image, before - // it opens anything. - let evidence = if binding.version() != distro::BINDING_VERSION - || policy.revision != request.policy_revision - { - Evidence::StaleBinding - } else if policy.result != PolicyResult::Accepted { - Evidence::AuthorizationDenied - } else { - binding_evidence(&error) - }; - Fault::new(evidence, error) - })?; - let geometry = checked.geometry().map_err(supplied_geometry_fault)?; - check_capacity(geometry, key.distro(), &format!("the image {shown}"))?; - let identity = identity_of(checked.file()).map_err(|error| { - Fault::host(Error::io(format!("the image {shown} cannot be identified")).with_source(error)) - })?; - check_at_path(path, identity, &format!("the image {shown}"))?; - owned - .check(identity, "image", path) - .attribute(Evidence::InvalidSettings)?; - Ok(checked) -} - -/// Checks and claims the scratch image at `path` against `binding`, through `access`, waiting -/// until `deadline` for a claim that another launch holds to be released, and then checks its -/// geometry, against the template's scratch that `key` records, the image at its path, that it -/// is none of `owned`, the files of the sandbox's store, that it is outside the directory of -/// `store`, the template store, and that it has no other name, as [`ClonePlan`] describes, and -/// returns the image that the binding's check opened and claimed. -fn check_scratch( - path: &Path, - binding: &ScratchBinding, - key: &TemplateKey, - (owned, store): (&StoreFiles, &Store), - deadline: Instant, - access: Access<'_>, -) -> Result { - let shown = path.display(); - let checked = binding - .check_with(path, deadline, access.scratch) - .map_err(|error| { - let evidence = if binding.version() != scratch::SCRATCH_BINDING_VERSION { - Evidence::StaleBinding - } else if error.code() == ErrorCode::ScratchClaimed { - // Another launch holds the image, which is not fresh for this clone. - Evidence::UnusableScratch - } else { - binding_evidence(&error) - }; - Fault::new(evidence, error) - })?; - let geometry = checked.geometry().map_err(supplied_geometry_fault)?; - check_geometry( - geometry, - key.scratch(), - &format!("the scratch image {shown}"), - "scratch", - )?; - let identity = identity_of(checked.file()).map_err(|error| { - Fault::host( - Error::io(format!("the scratch image {shown} cannot be identified")).with_source(error), - ) - })?; - check_at_path(path, identity, &format!("the scratch image {shown}"))?; - owned - .check(identity, "scratch image", path) - .attribute(Evidence::InvalidSettings)?; - check_outside_store(&checked, store, path)?; - check_single_name(&checked, path)?; - Ok(checked) -} - -/// Refuses the clone's scratch image at `path`, which `checked` holds open and claims, if it is in -/// the directory of `store`, the template store, where it is now, every link on the way to it -/// resolved: the store's generations hold their snapshots there, which every clone of a -/// generation restores in place, and its captures write theirs there, which their publications -/// make generations' by renaming directories within it, so a file in the store could be, or -/// become as the plan examines it, a file of such a snapshot. Nothing that the host does moves a -/// file into the store's directory, so one outside it stays outside it. The location resolves the -/// aliases that the file system makes, links and junctions on the way and directories held open, -/// but not a mount or a network share that exposes the store's directories elsewhere, which -/// [`ClonePlan`] leaves to the system's administration. The request supplied the image, so its -/// refusal is invalid settings, while the failure to locate it is the host's. -fn check_outside_store(checked: &CheckedScratch, store: &Store, path: &Path) -> Result<(), Fault> { - let failed = |error: io::Error| { - Fault::host( - Error::io(format!( - "cannot compare the location of the scratch image {} with the template store in \ - {}", - path.display(), - store.dir().display() - )) - .with_source(error), - ) - }; - let location = location_of(checked.file()).map_err(failed)?; - if is_within(&location, store.dir()).map_err(failed)? { - return Err(Fault::new( - Evidence::InvalidSettings, - Error::unsupported(format!( - "the scratch image {} is {}, in the directory of the template store {}, where its \ - captures write the snapshots that every clone of a generation restores in \ - place: a clone's scratch image must be outside it", - path.display(), - location.display(), - store.dir().display() - )), - )); - } - Ok(()) -} - -/// Returns whether the directory `dir` is one of those above `location`, by their identities, past -/// those that went from their paths meanwhile, as a publication renames a capture's transaction, -/// which `dir`, the store's own directory, outlives. -fn is_within(location: &Path, dir: &Path) -> io::Result { - let dir = platform::file_id(dir)?; - for ancestor in location.ancestors().skip(1) { - match platform::file_id(ancestor) { - Ok(identity) if identity == dir => return Ok(true), - Ok(_) => {} - Err(error) - if matches!( - error.kind(), - io::ErrorKind::NotFound | io::ErrorKind::NotADirectory - ) => {} - Err(error) => return Err(error), - } - } - Ok(false) -} - -/// Returns where the open `file` is now, every link on the way to it resolved: on Linux, the -/// target of its link in `/proc/self/fd`, which follows renames of its directories, and on -/// Windows its final path, which names its volume by a drive letter or a mount point, or else by -/// the volume's GUID. -fn location_of(file: &File) -> io::Result { - #[cfg(target_os = "linux")] - { - use std::os::fd::AsRawFd; - - fs::read_link(Path::new("/proc/self/fd").join(file.as_raw_fd().to_string())) - } - #[cfg(windows)] - { - // A volume without a drive letter or a mount point has no path by a drive. - crate::host::seal::final_path(file) - .or_else(|error| crate::host::seal::final_volume_path(file).map_err(|_| error)) - } - #[cfg(not(any(target_os = "linux", windows)))] - { - let _ = file; - Err(io::Error::from(io::ErrorKind::Unsupported)) - } -} - -/// Refuses the clone's scratch image at `path`, which `checked` holds open, unless it has a single -/// name: another name, a hard link, could make it a file of a snapshot that the clone's guest then -/// writes, of one of the store's generations or captures, or of another store, which the image's -/// location does not show. The request supplied the image, so its refusal is invalid settings, -/// while the failure to count its names is the host's. -fn check_single_name(checked: &CheckedScratch, path: &Path) -> Result<(), Fault> { - let links = platform::link_count(checked.file()).map_err(|error| { - Fault::host( - Error::io(format!( - "cannot count the names of the scratch image {}", - path.display() - )) - .with_source(error), - ) - })?; - if links > 1 { - return Err(Fault::new( - Evidence::InvalidSettings, - Error::unsupported(format!( - "the scratch image {} has {links} names, but a clone's scratch image must have a \ - single name: another, a hard link, could make it a file of a snapshot, which \ - every clone of its generation restores in place", - path.display() - )), - )); - } - Ok(()) -} - -/// Returns the evidence of `error`, with which the check of the binding of a file that the request -/// supplied failed, past the binding's own version and verdict: a file that changed, was replaced, -/// or went since its binding was made, a supplied file of the wrong identity; a path that the -/// check refuses, invalid settings; and any other failure, the host's. -fn binding_evidence(error: &Error) -> Evidence { - match error.kind() { - ErrorKind::Unavailable => Evidence::SuppliedIdentity, - ErrorKind::Unsupported => Evidence::InvalidSettings, - _ => host_evidence(error), - } -} - -/// Returns the fault of `error`, with which the geometry of a file that the request supplied -/// cannot be read: a geometry that OpenVMM refuses, a supplied file of the wrong geometry, and any -/// other failure, the host's. -fn supplied_geometry_fault(error: Error) -> Fault { - match error.kind() { - ErrorKind::Unsupported => Fault::new(Evidence::SuppliedGeometry, error), - _ => Fault::host(error), - } -} - -/// The regular files that the store of a sandbox owns, with their identities, which no image or -/// scratch image of a clone may be. -struct StoreFiles(Vec<(PathBuf, (u64, u128))>); - -impl StoreFiles { - /// Returns the regular files that the store of the sandbox of `locked` owns now, as - /// [`LockedStore::owned_entries`] lists its entries: its files, among them its log and the - /// clone's activation record, which a launch replaces, and the clone binding, which a release - /// removes, and its temporary files. - /// - /// The store creates nothing but regular files under its names, and its control socket, so a - /// link or a directory among its entries is refused: the path of an image could pass through - /// it as through a directory, and a transition that replaces or removes it would leave that - /// path naming nothing. Other kinds of files, such as a socket, are no image. - fn of(locked: &LockedStore) -> Result { - let failed = |error: io::Error| { - Error::io(format!( - "cannot list the files of the sandbox's store in {}", - locked.dir().display() - )) - .with_source(error) - }; - let mut files = Vec::new(); - for entry in locked.owned_entries().map_err(failed)? { - let shown = paths::normalized(&entry); - let absent = |error: &io::Error| { - error.kind() == io::ErrorKind::NotFound || was_removed(&entry, error) - }; - let failed = |error: io::Error| { - Error::io(format!( - "cannot examine {}, a file of the sandbox's store", - shown.display() - )) - .with_source(error) - }; - let metadata = match fs::symlink_metadata(&entry) { - Ok(metadata) => metadata, - Err(error) if absent(&error) => continue, - Err(error) => return Err(failed(error)), - }; - let kind = metadata.file_type(); - if kind.is_dir() || kind.is_symlink() || filesystem::is_reparse_point(&metadata) { - return Err(Error::unavailable(format!( - "{} is a link or a directory, which the sandbox's store never creates there", - shown.display() - ))); - } - if !kind.is_file() { - continue; - } - let file = match open_regular(&entry) { - Ok(file) => file, - Err(error) if absent(&error) => continue, - Err(error) => return Err(failed(error)), - }; - let identity = identity_of(&file).map_err(failed)?; - files.push((shown, identity)); - } - Ok(Self(files)) - } - - /// Refuses the clone's image or scratch image at `path`, which messages call `what`, and - /// whose identity is `identity`, if it is one of these files, by whatever path `path` names - /// it, as through a link among its directories, or a hard link. - fn check(&self, identity: (u64, u128), what: &str, path: &Path) -> Result<()> { - match self.0.iter().find(|(_, owned)| *owned == identity) { - None => Ok(()), - Some((shown, _)) => Err(Error::unsupported(format!( - "the {what} {} is {}, a file of the sandbox's store, which its transitions \ - replace or remove", - path.display(), - shown.display() - ))), - } - } -} - -/// Refuses `found`, the geometry of a clone's image, which messages call `what`, unless it fits -/// `capacity`, the geometry of the template's placeholder, which a restore presents over the -/// image: of other block sizes, [`Evidence::SuppliedGeometry`], or longer, -/// [`Evidence::SuppliedLength`]. A shorter image fills the capacity's first blocks, and the -/// restore reads zeros past its end. -fn check_capacity(found: BlockGeometry, capacity: BlockGeometry, what: &str) -> Result<(), Fault> { - check_blocks(found, capacity, what, "placeholder")?; - if found.length > capacity.length { - return Err(Fault::new( - Evidence::SuppliedLength, - Error::unsupported(format!( - "{what} is {} bytes long, longer than the template's placeholder, whose {} bytes \ - are the capacity of every clone's image", - found.length, capacity.length - )), - )); - } - Ok(()) -} - -/// Refuses `found`, the geometry of the file of a clone's block device that the request supplied, -/// which messages call `what`, unless it is `expected`, the geometry that the template's `device` -/// presented: of another length, [`Evidence::SuppliedLength`], or of other block sizes, -/// [`Evidence::SuppliedGeometry`]. -fn check_geometry( - found: BlockGeometry, - expected: BlockGeometry, - what: &str, - device: &str, -) -> Result<(), Fault> { - if found.length != expected.length { - return Err(Fault::new( - Evidence::SuppliedLength, - Error::unsupported(format!( - "{what} is {} bytes long, not the {} bytes of the template's {device}", - found.length, expected.length - )), - )); - } - check_blocks(found, expected, what, device) -} - -/// Refuses `found`, the geometry of the file of a clone's block device that the request supplied, -/// which messages call `what`, unless it presents the logical and physical block sizes of -/// `expected`, the geometry that the template's `device` presented, [`Evidence::SuppliedGeometry`]. -fn check_blocks( - found: BlockGeometry, - expected: BlockGeometry, - what: &str, - device: &str, -) -> Result<(), Fault> { - if found.logical_block_size != expected.logical_block_size - || found.physical_block_size != expected.physical_block_size - { - return Err(Fault::new( - Evidence::SuppliedGeometry, - Error::unsupported(format!( - "{what} presents logical blocks of {} bytes and physical blocks of {} bytes, but \ - the template's {device} presented {} and {} bytes", - found.logical_block_size, - found.physical_block_size, - expected.logical_block_size, - expected.physical_block_size - )), - )); - } - Ok(()) -} - -/// Checks the file at `path`, which messages call `what`, as OpenVMM will open it, by its path: a -/// regular file, not a link, nor on Windows any other reparse point, whose identity is -/// `expected`, the identity of the file that a check found, and that no one but the current user, -/// or an administrator, may change. -/// -/// Fails with [`ErrorKind::Unavailable`] and [`Evidence::SuppliedIdentity`] for a path that no -/// longer names that file: one whose file went, a component of which is no longer a directory, -/// that the current user may no longer open, or that names a link, a file that is not regular, or -/// another file, and for a file that others may change; and with [`ErrorKind::Io`] and the host's -/// evidence if the file cannot be opened at its path otherwise, as when the host runs out of -/// descriptors, or if the file that it opened cannot be identified or examined. -fn check_at_path(path: &Path, expected: (u64, u128), what: &str) -> Result<(), Fault> { - let refused = |problem: String| { - Fault::new( - Evidence::SuppliedIdentity, - Error::unavailable(format!("{what} {problem}")), - ) - }; - let failed = |action: &str, error: io::Error| { - Fault::host(Error::io(format!("{what} cannot be {action}")).with_source(error)) - }; - let file = open_regular(path).map_err(|error| { - if path_refused(&error) { - refused(format!("cannot be used: {error}")) - } else { - failed("opened", error) - } - })?; - let identity = identity_of(&file).map_err(|error| failed("identified", error))?; - if identity != expected { - return Err(refused( - "changed while it was checked: its path names another file".to_owned(), - )); - } - if let Some(problem) = - platform::foreign_writer_problem(&file).map_err(|error| failed("checked", error))? - { - return Err(refused(problem)); - } - Ok(()) -} - -/// Returns whether `error`, with which [`open_regular`] failed to open a file that a request -/// supplied at its path, says that the path no longer names the file that a check found: the -/// file went, a component of the path is no longer a directory, the current user may no longer -/// open it, or the open refused a link or a file that is not regular, a failure that the system -/// does not report. Any other failure is the host's. -fn path_refused(error: &io::Error) -> bool { - error.raw_os_error().is_none() - || matches!( - error.kind(), - io::ErrorKind::NotFound - | io::ErrorKind::NotADirectory - | io::ErrorKind::PermissionDenied - ) -} diff --git a/nvx-rs/src/host/template/clone/export.rs b/nvx-rs/src/host/template/clone/export.rs deleted file mode 100644 index 808146920..000000000 --- a/nvx-rs/src/host/template/clone/export.rs +++ /dev/null @@ -1,168 +0,0 @@ -//! The export of a clone: the host paths that it shares with its root, as -//! [`filesystem::plan`](crate::host::filesystem::plan) plans them, which -//! [`ClonePlan::new`](super::ClonePlan::new) checks before the clone's launch, and the forms in -//! which the clone's restore attaches the export, on the template's dormant filesystem slot, and -//! in which its restore sequence sends the export's mapping table. -//! -//! A template's capture leaves its filesystem slot dormant, so that each restore binds a clone's -//! own export to it, or none. A template is a platform snapshot, to whose dormant slot OpenVMM lets -//! a restore bind an aggregate, which it checks as it checks a cold boot's, before any processor -//! runs. So a clone exports what a cold launch exports, any export plan that `filesystem::plan` -//! makes for the container root: an aggregate of up to [`MAX_CHILDREN`] host directories and -//! files, each read-only or read-write, with OpenVMM's hidden, re-exposed, and writable paths, and -//! the plan's owner. The restore attaches it as a cold launch attaches it, -//! [`ExportPlan::openvmm_arguments`], and the clone's guest mounts its root, which lists its -//! children by their indices, so the clone's mapping table is a cold launch's, -//! [`ExportPlan::guest_table`], whose sources name each path by its child's index. -//! -//! [`MAX_CHILDREN`]: crate::host::filesystem::MAX_CHILDREN - -use std::path::{Path, PathBuf}; - -use crate::error::{Error, Result}; -use crate::host::filesystem::{ExportPlan, ReservedTargets}; -use crate::protocol::HostMap; - -/// The export of a clone that [`ClonePlan::new`](super::ClonePlan::new) checked: an export plan -/// that [`filesystem::plan`](crate::host::filesystem::plan) could have made for the container -/// root, which its restore binds to the template's dormant filesystem slot. -#[derive(Debug, Clone)] -pub(super) struct CloneExport { - /// The plan. - plan: ExportPlan, -} - -impl CloneExport { - /// Returns the export of `plan`, an export plan of a clone's request, without reading - /// anything. - /// - /// Fails with [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for a plan that - /// [`filesystem::plan`](crate::host::filesystem::plan) could not have made for the container - /// root, [`ReservedTargets::CONTAINER_ROOT`], as [`ExportPlan::check`] finds it, such as one - /// that maps paths but exports no directory or file, exports some but maps no path, or exports - /// more directories and files than OpenVMM attaches. - pub(super) fn new(plan: &ExportPlan) -> Result { - plan.check(ReservedTargets::CONTAINER_ROOT)?; - Ok(Self { plan: plan.clone() }) - } - - /// Returns the export's plan. - pub(super) fn plan(&self) -> &ExportPlan { - &self.plan - } - - /// Refuses the export if one of its directories or files lies inside the directory of a - /// template store, `resolved`, or one of its directories holds it: the guest could then reach - /// the payload of a generation that clones lease, and write it through a writable child, - /// though nothing may change a leased payload or its snapshot. Refuses it as well if it lets - /// the guest write in one of `through`, the directories through which the store's path, - /// `store`, leads: the host reaches the store, and OpenVMM the snapshots in it, by that path, - /// so the guest could replace a link or a directory of the path, and turn the path elsewhere, - /// or make a directory of it unsearchable, and keep the host from following it. - /// - /// Refuses it too if one of its directories is, or holds, the directory of the clone's - /// sandbox, `sandbox`, which the lease names by its path with its links resolved, or if one of - /// its files lies in that directory itself: the host writes the clone's records there, by - /// their paths, so the guest could otherwise read or change them, or replace the directory, or - /// one above it, with a link, and turn the host's writes elsewhere, into a leased snapshot, - /// say. The directories above it are the only ones through which that path leads, and only an - /// export that holds it could let the guest change one. A directory inside the sandbox's, - /// among the caller's files, holds none of the host's records, nor is a file in such a - /// directory one of them. - /// - /// The check reads nothing: it compares the paths of the export's directories and files, and - /// the paths in which the guest may write, as the plan resolved them, with `resolved`, - /// `through`, and `sandbox`, which the caller and the lease resolved as - /// [`filesystem::canonicalize`] resolves the plan's paths, so that no link, and on Windows no - /// other letter case, short name, or verbatim prefix, hides an overlap. Fails with - /// [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for such an export. - /// - /// Since the check compares the paths that the export and the store resolve to, an alias of - /// the store that an administrator configures is outside it, as the directories above the - /// store are: a bind mount or a mount point of the store, or of a directory above or inside - /// it, a loopback share, or a hard link of a payload. Whoever names an export must not name - /// such an alias, nor a directory that holds one. Nor does the check see what changes after - /// it: OpenVMM resolves the paths of an export again at each launch and restore, so the - /// directories above each of the export's directories and files must be ones that no guest may - /// change, as those above the clone's block devices must, which [Block devices by - /// path](crate::host::template::Store#block-devices-by-path) describes: whoever names an - /// export must not name one inside a directory that a writable export of another sandbox or - /// clone holds. - /// - /// [`filesystem::canonicalize`]: crate::host::filesystem::canonicalize - pub(super) fn check_outside( - &self, - store: &Path, - resolved: &Path, - through: &[PathBuf], - sandbox: &Path, - ) -> Result<()> { - for child in &self.plan.children { - if child.root.starts_with(resolved) || resolved.starts_with(&child.root) { - return Err(Error::unsupported(format!( - "the export of a clone cannot share {}, which overlaps the template store {}, \ - whose leased payloads nothing may change", - child.root.display(), - resolved.display() - ))); - } - if sandbox.starts_with(&child.root) { - return Err(Error::unsupported(format!( - "the export of a clone cannot share {}, which holds the directory of the \ - clone's sandbox {}, in which the host writes the clone's records", - child.root.display(), - sandbox.display() - ))); - } - // A file in the sandbox's directory itself may be one of the host's records there. - if child.file && child.root.parent() == Some(sandbox) { - return Err(Error::unsupported(format!( - "the export of a clone cannot share {}, a file in the directory of the \ - clone's sandbox {}, in which the host writes the clone's records", - child.root.display(), - sandbox.display() - ))); - } - // The guest may write anywhere in a writable directory, and a writable file, unless the - // plan limits its writes to some of the directory's paths. - let written: &[PathBuf] = match (child.writable, child.write.is_empty()) { - (false, _) => &[], - (true, true) => std::slice::from_ref(&child.root), - (true, false) => child.write.as_slice(), - }; - let directory = through - .iter() - .find(|directory| written.iter().any(|path| directory.starts_with(path))); - if let Some(directory) = directory { - return Err(Error::unsupported(format!( - "the export of a clone cannot let its guest write in {}, through which the \ - path of the template store {} leads, since the guest could then turn that \ - path elsewhere, or keep the host from following it", - directory.display(), - store.display() - ))); - } - } - Ok(()) - } - - /// Returns the export's mapping table, which the clone's restore sequence sends once the - /// clone is ready, as a cold launch's guest receives it, [`ExportPlan::guest_table`]: the - /// plan's binds, in their order, each from its path in the export, which a source names by - /// its child's index, to its target, with its access. - pub(super) fn table(&self) -> Vec { - self.plan.guest_table() - } - - /// Returns OpenVMM's arguments that bind the export to the template's dormant filesystem slot - /// at a restore, as a cold launch attaches it, [`ExportPlan::openvmm_arguments`]: - /// `--mount-aggregate` with [`GUEST_EXPORT`](crate::host::filesystem::GUEST_EXPORT), a - /// `--mount-child` of each of its directories and files, named by its index, with its mode, - /// `rw` if the guest may write it, and `ro` otherwise, and `file` for a file; then their - /// hidden, re-exposed, and writable paths, with `--mount-deny`, `--mount-allow`, and - /// `--mount-write`; and `--mount-owner caller` for an export of the caller owner. Each path is - /// absolute, as the plan resolved it. - pub(super) fn arguments(&self) -> Vec { - self.plan.openvmm_arguments() - } -} diff --git a/nvx-rs/src/host/template/clone/failure.rs b/nvx-rs/src/host/template/clone/failure.rs deleted file mode 100644 index 0a75833c9..000000000 --- a/nvx-rs/src/host/template/clone/failure.rs +++ /dev/null @@ -1,926 +0,0 @@ -//! The failures of clones, [`CloneFailure`]: the [`Evidence`] of a failure of a clone's plan, -//! launch, restore sequence, or reattach, the [`Outcome`] to which the evidence attributes it, as -//! [the attribution of evidence](Evidence#attribution) describes, the [`Stage`] at which the clone -//! failed, the clone's [`Progress`] by then, and the [`Teardown`] that followed, which the -//! failure's [`FailureRecord`] keeps, and the clone's sandbox records, as [the records of -//! failures](FailureRecord#records) describe. - -use std::fmt; -use std::io; -use std::time::Instant; - -use serde::{Deserialize, Serialize}; - -use crate::activation::{TemplateReference, TransactionId}; -use crate::error::{Error, ErrorCode, ErrorKind, Result}; -use crate::host::report::Report; -use crate::host::store::{LaunchId, Recovery}; -use crate::host::template::{Lease, LeaseId, Store, StoreId}; -use crate::protocol::status; - -/// What the host saw of a failure of a clone, which attributes the failure to an [`Outcome`]. -/// -/// # Attribution -/// -/// The outcome of a failure follows from its evidence alone, as [`outcome`](Self::outcome) gives -/// it: -/// -/// | Evidence | Outcome | Quarantine | -/// | --- | --- | --- | -/// | An authorization that the image's binding denies; a stale image binding; invalid targets, format, settings, Activate, or mappings; an image that the template can't serve; a scratch that isn't ext4 or isn't fresh; a root that does not assemble from them | [`Request`](Outcome::Request) | No | -/// | A supplied file with the wrong identity, length, or geometry | [`Request`](Outcome::Request), or the host's [`Staging`](Outcome::Staging) once the caller proves it | No | -/// | An endpoint or forward that cannot be bound; resource exhaustion; a destination that cannot restore the template, such as an incompatible CPU or TSC; another runtime; another failure of the host's own operations | [`Host`](Outcome::Host) | No | -/// | A lost status; a timeout; a reset; a forced kill; a broken protocol | [`Unknown`](Outcome::Unknown): the host, the protocol, or neither | No | -/// | A malformed or inconsistent generation; a diagnosed capture or repair defect attributable to it | [`Template`](Outcome::Template) | Yes | -/// | A repair status alone, 193, 194, and 195 included | [`Repair`](Outcome::Repair): the repair stage, whose provenance is still to establish | No | -/// -/// Only a failure of the template quarantines the template's generation, through the generation's -/// tombstone, as [`CloneFailure::quarantine`] does: no other evidence shows that the generation, -/// rather than the request, the host, or the session, failed the clone, and a quarantine changes -/// nothing of the generation's payload or snapshot, which its other clones keep restoring. -/// -/// The host sees the guest of a clone through its control session, and through the status with -/// which its VM ends, which OpenVMM's outcome report gives: a clone's restore discards the -/// guest's console, so no diagnostic tells where a status came from. [`status_evidence`] -/// attributes a status alone, as [`nvx::protocol::status`](crate::protocol::status) classes it: -/// -/// - a status of the time ABI or of a clone's repair is evidence of the repair stage, at any point, -/// since the repair's deferred work may end a clone that is ready, and its provenance is still to -/// establish, since a workload may end its VM with any status; -/// - a status of a clone's activation is evidence of its class while no workload can have run, -/// before the start of the clone's sandbox was sent, and none after; -/// - no other status is evidence. -/// -/// A status that is evidence prevails over what the host or the caller saw of the same end: the -/// loss of the session, the passing of a deadline, or the VMM's exit only show that the clone -/// ended, and a step that fails as the guest ends the clone may fail because it ended. It does not -/// prevail over evidence of the template, a check of the generation or a diagnosis that attributes -/// a defect to it, which shows the provenance that the status alone leaves to establish, and the -/// failure's record holds the status beside it. -/// -/// The host sees the failures of OpenVMM itself through the line with which OpenVMM reports the -/// error that ended it, which the launch's log keeps, and which leads with the error's time ABI -/// code, a spelling that OpenVMM keeps stable for its callers. A VMM that exits before its -/// restore-ready event with a code that says that the host cannot present the template's CPU -/// profile, TSC, or LAPIC timer failed on an [incompatible host](Self::IncompatibleHost). Any -/// other code, such as one of the snapshot's time contract, keeps what the host saw, since it does -/// not tell a defect of the generation from one of this host's OpenVMM, and the failure's message -/// keeps the line. One error that has no code tells the host's fault by its text alone: a VMM -/// whose error's chain names a forward of the clone's network whose port of host loopback OpenVMM -/// cannot bind, as one that another socket holds, which OpenVMM binds as it restores the clone's -/// NIC, before its restore-ready event, failed to [bind a forward](Self::ForwardBind). The failure's -/// message keeps that cause of the chain after the line. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -#[non_exhaustive] -pub enum Evidence { - /// The verdict of the distro policy that the binding of the clone's image holds rejects the - /// image: the request may not run it. - AuthorizationDenied, - /// The binding of the clone's image is stale: it holds a verdict of another revision of the - /// distro policy than the caller accepts, or it is of another version than this build checks, - /// or the image's manifest changed after it was made. - StaleBinding, - /// The request's targets are invalid: a memory target that is not whole blocks of the - /// template's, from its RAM to its capacity, or resources that the repaired clone does not - /// have, as the guest's - /// [`ACTIVATION_RESOURCES`](crate::protocol::status::ACTIVATION_RESOURCES) tells. - InvalidTargets, - /// The request's image is not of the format and identity that its binding names. - InvalidFormat, - /// The request's settings are invalid: a plan that no clone takes, a path that OpenVMM would - /// not open as checked, a network that the template's NIC does not match, an image or a - /// scratch image that is a file of the sandbox's store, or settings that the guest cannot - /// apply, as its [`ACTIVATION_WORKLOAD`](crate::protocol::status::ACTIVATION_WORKLOAD) and - /// [`ACTIVATION_NETWORK`](crate::protocol::status::ACTIVATION_NETWORK) tell. - InvalidSettings, - /// The request's Activate is invalid: settings that no activation record holds, or a record - /// that the guest refused, as its - /// [`ACTIVATION_RECORD`](crate::protocol::status::ACTIVATION_RECORD) tells. - InvalidActivate, - /// The request's mappings are invalid: an export that no clone takes, such as one that - /// overlaps the template store; an export whose read-only mappings the host cannot keep - /// read-only, or whose objects are no longer those that its plan found, a missing one - /// included; a mapping table that the guest refused, as the adapter's framing tells; or a - /// host export that the guest cannot mount, as its - /// [`ACTIVATION_EXPORT`](crate::protocol::status::ACTIVATION_EXPORT) tells. - InvalidMappings, - /// The request's image is one that the template cannot serve, as a diagnostic tells. No - /// clone's restore keeps the guest's diagnostics yet, so the guest's status alone, - /// [`UnusableRoot`](Self::UnusableRoot), tells of such an image. - UnservableImage, - /// The request's scratch image is not ext4, or not fresh: another launch holds it, or, as a - /// diagnostic tells, the guest found a filesystem on it. No clone's restore keeps the guest's - /// diagnostics yet, so the guest's status alone, [`UnusableRoot`](Self::UnusableRoot), tells - /// of a filesystem on it. - UnusableScratch, - /// The clone's root does not assemble from the request's image over its upper, as the guest's - /// [`ACTIVATION_ROOT`](crate::protocol::status::ACTIVATION_ROOT) tells: an image that the - /// template cannot serve, or a scratch that is not ext4 or not fresh, which the status alone - /// does not tell apart. - UnusableRoot, - /// A file that the request supplied is not the one that its binding names: it was replaced, - /// changed, or removed after its binding was made, or its path names another file than the - /// check found, or one that others may change. - SuppliedIdentity, - /// A file that the request supplied is of a length that the template's device cannot take: an - /// image longer than the template's placeholder, or a scratch image of another length than the - /// template's scratch. - SuppliedLength, - /// A file that the request supplied presents other block sizes than the template's device, or - /// a geometry that OpenVMM refuses. - SuppliedGeometry, - /// An endpoint of the clone's launch cannot be bound: the endpoint of the VMM's restore-ready - /// event, or a control socket whose path a Unix socket's address, or OpenVMM's option - /// `listen=`, cannot hold. - EndpointBind, - /// A forward of the clone's network cannot be bound: OpenVMM cannot bind its port of host - /// loopback, as when another socket holds it, as it restores the clone's NIC, and exits before - /// its restore-ready event, as [the attribution](Self#attribution) describes. - ForwardBind, - /// The host ran out of a resource, such as its storage, its memory, or its descriptors. - ResourceExhaustion, - /// The host cannot restore the template: this build speaks other contracts than the template, - /// or reads no key of the template's encoding version, or OpenVMM refused to restore the - /// snapshot with a time ABI code that says that the host cannot present the template's CPU - /// profile, TSC, or LAPIC timer, as [the attribution](Self#attribution) describes. - IncompatibleHost, - /// The clone runs, or would launch, under another runtime identity than the template's or the - /// caller's, which the clone's guest might not speak. - RuntimeMismatch, - /// Another failure of the host's own operations: an I/O error of its files, a store whose - /// records are inconsistent, a clock that cannot hold a deadline, or a caller that drives the - /// clone through another lease or lock than the clone's, or launches it again once it failed. - HostFailure, - /// The clone's status was lost: its VMM ended without an outcome report that tells why, as one - /// that is killed does, or the caller of its launch crashed. - LostStatus, - /// A deadline of the clone's restore passed. - Timeout, - /// The adapter lost the clone's session: its reset, the end of its stream, or a failure of its - /// transport. - Reset, - /// The host's recovery terminated the clone's VMM, whose launch's caller had crashed. - ForcedKill, - /// The clone's session broke its protocol: another process at one of the clone's endpoints, - /// bytes that differ from OpenVMM's restore-ready event, a refusal of the adapter's framing, a - /// Ready that does not report the record's activation, or a session that broke the order of - /// the activation, as the guest's - /// [`ACTIVATION_PROTOCOL`](crate::protocol::status::ACTIVATION_PROTOCOL) and - /// [`ACTIVATION_ORDER`](crate::protocol::status::ACTIVATION_ORDER) tell. - Protocol, - /// The leased generation is malformed or inconsistent: the store indexes it with another key - /// than the lease names, its key or snapshot is malformed, or its payload is not a directory - /// of the current user with mode 0700. - MalformedGeneration, - /// A defect of the generation's capture or repair, which a diagnostic attributes to the - /// generation. No clone's restore keeps the guest's diagnostics yet, so a status of the repair - /// alone is [`RepairStatus`](Self::RepairStatus), and no time ABI code of OpenVMM's tells a - /// defect of the generation from one of the host's OpenVMM. - TemplateDefect, - /// A status of the time ABI or of a clone's repair, alone, which - /// [`FailureRecord::status`] holds. - RepairStatus, -} - -impl Evidence { - /// Returns the outcome to which the evidence attributes a failure, as [the - /// attribution](Self#attribution) describes. - pub const fn outcome(self) -> Outcome { - match self { - Self::AuthorizationDenied - | Self::StaleBinding - | Self::InvalidTargets - | Self::InvalidFormat - | Self::InvalidSettings - | Self::InvalidActivate - | Self::InvalidMappings - | Self::UnservableImage - | Self::UnusableScratch - | Self::UnusableRoot - | Self::SuppliedIdentity - | Self::SuppliedLength - | Self::SuppliedGeometry => Outcome::Request, - Self::EndpointBind - | Self::ForwardBind - | Self::ResourceExhaustion - | Self::IncompatibleHost - | Self::RuntimeMismatch - | Self::HostFailure => Outcome::Host, - Self::LostStatus | Self::Timeout | Self::Reset | Self::ForcedKill | Self::Protocol => { - Outcome::Unknown - } - Self::MalformedGeneration | Self::TemplateDefect => Outcome::Template, - Self::RepairStatus => Outcome::Repair, - } - } - - /// Returns whether the evidence is of a file that the request supplied, which the host's - /// staging may have made wrong, as [`CloneFailure::with_staging_proven`] attributes it once - /// the caller proves it. - pub const fn is_supplied_file(self) -> bool { - matches!( - self, - Self::SuppliedIdentity | Self::SuppliedLength | Self::SuppliedGeometry - ) - } - - /// Returns the evidence's name, in lowercase words joined by hyphens, as its record spells it. - pub const fn as_str(self) -> &'static str { - match self { - Self::AuthorizationDenied => "authorization-denied", - Self::StaleBinding => "stale-binding", - Self::InvalidTargets => "invalid-targets", - Self::InvalidFormat => "invalid-format", - Self::InvalidSettings => "invalid-settings", - Self::InvalidActivate => "invalid-activate", - Self::InvalidMappings => "invalid-mappings", - Self::UnservableImage => "unservable-image", - Self::UnusableScratch => "unusable-scratch", - Self::UnusableRoot => "unusable-root", - Self::SuppliedIdentity => "supplied-identity", - Self::SuppliedLength => "supplied-length", - Self::SuppliedGeometry => "supplied-geometry", - Self::EndpointBind => "endpoint-bind", - Self::ForwardBind => "forward-bind", - Self::ResourceExhaustion => "resource-exhaustion", - Self::IncompatibleHost => "incompatible-host", - Self::RuntimeMismatch => "runtime-mismatch", - Self::HostFailure => "host-failure", - Self::LostStatus => "lost-status", - Self::Timeout => "timeout", - Self::Reset => "reset", - Self::ForcedKill => "forced-kill", - Self::Protocol => "protocol", - Self::MalformedGeneration => "malformed-generation", - Self::TemplateDefect => "template-defect", - Self::RepairStatus => "repair-status", - } - } -} - -impl fmt::Display for Evidence { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(self.as_str()) - } -} - -/// What a failure of a clone is attributed to, by its [`Evidence`]. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -#[non_exhaustive] -pub enum Outcome { - /// The request: its image, upper, targets, or settings. - Request, - /// The host's staging of a file that the request supplied, which the caller proved. - Staging, - /// The host. - Host, - /// The host, the protocol, or neither: the evidence shows no provenance. - Unknown, - /// The leased generation of the template, which [`CloneFailure::quarantine`] quarantines. - Template, - /// The repair stage of the clone, whose provenance is still to establish. - Repair, -} - -impl Outcome { - /// Returns whether a failure of this outcome quarantines the leased generation, through its - /// tombstone: only a failure of the [template](Self::Template) does. - pub const fn quarantines(self) -> bool { - matches!(self, Self::Template) - } - - /// Returns the outcome's name, in lowercase, as its record spells it. - pub const fn as_str(self) -> &'static str { - match self { - Self::Request => "request", - Self::Staging => "staging", - Self::Host => "host", - Self::Unknown => "unknown", - Self::Template => "template", - Self::Repair => "repair", - } - } -} - -impl fmt::Display for Outcome { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(self.as_str()) - } -} - -/// The stage of a clone at which it failed. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -#[non_exhaustive] -pub enum Stage { - /// The clone's plan, [`ClonePlan::new`](super::ClonePlan::new), before anything of its launch. - Plan, - /// The clone's launch, [`ClonePlan::launch`](super::ClonePlan::launch), until its VMM runs. - Launch, - /// OpenVMM's restore, from the spawn of the VMM until its restore-ready event. - Restore, - /// The clone's repair and activation, from the restore-ready event until the check of the - /// clone's Ready. - Activation, - /// The start of the clone's sandbox, from the check of its Ready, through the mapping table of - /// a clone with an export, until the start's confirmation. - Start, - /// The commit of the clone's launch. - Commit, - /// The run of the clone's VM, from the commit of its launch until the VM exits. - Run, - /// A reattach's check of the clone, [`reattach_clone`](super::reattach_clone). - Reattach, -} - -impl Stage { - /// Returns the stage's name, in lowercase, as its record spells it. - pub const fn as_str(self) -> &'static str { - match self { - Self::Plan => "plan", - Self::Launch => "launch", - Self::Restore => "restore", - Self::Activation => "activation", - Self::Start => "start", - Self::Commit => "commit", - Self::Run => "run", - Self::Reattach => "reattach", - } - } -} - -impl fmt::Display for Stage { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(self.as_str()) - } -} - -/// How far a clone got, in the order of its steps: the latest step that it reached. -/// -/// The steps from the spawn of the clone's VMM to the start's confirmation are those that its -/// launch and restore sequence record in its progress log, -/// [`CLONE_PROGRESS_NAME`](crate::host::store::CLONE_PROGRESS_NAME), as each is reached; the -/// store's records tell the spawn too, and the commit: the launch's journal records the spawn of -/// its VMM until the launch ends, and the runtime record its commit. -#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(rename_all = "kebab-case")] -#[non_exhaustive] -pub enum Progress { - /// The clone's plan: nothing of its launch exists. - Planned, - /// The launch wrote the clone's files, and may have begun its journal: no VMM was spawned. - Launching, - /// The launch's VMM was spawned, which its journal records, unless the launch could not record - /// it, and then ended it through its handle. - Spawned, - /// The VMM's restore-ready event was read: the restored VM runs. - Event, - /// A session of the adapter's framing attached: the guest's repair acknowledged the restore, - /// before which OpenVMM reads none of the host's input. - Acknowledged, - /// The Activate of the clone's record was sent. - Activating, - /// The clone's Ready was checked: the clone is activated, with the record's transaction. A - /// clone with an export gets its mapping table at this step, before the start. - Ready, - /// The start of the clone's sandbox was sent, once the guest applied the whole mapping table - /// of a clone with an export. - Starting, - /// The start was confirmed: the clone's sandbox started. - Started, - /// The launch committed: the runtime record names the clone's VM. - Committed, -} - -impl Progress { - /// Returns the stage of a clone that fails once it got this far: the stage of the step that - /// follows. - pub const fn stage(self) -> Stage { - match self { - Self::Planned => Stage::Plan, - Self::Launching => Stage::Launch, - Self::Spawned => Stage::Restore, - Self::Event | Self::Acknowledged | Self::Activating => Stage::Activation, - Self::Ready | Self::Starting => Stage::Start, - Self::Started => Stage::Commit, - Self::Committed => Stage::Run, - } - } - - /// Returns the step's name, in lowercase, as its records spell it. - pub const fn as_str(self) -> &'static str { - match self { - Self::Planned => "planned", - Self::Launching => "launching", - Self::Spawned => "spawned", - Self::Event => "event", - Self::Acknowledged => "acknowledged", - Self::Activating => "activating", - Self::Ready => "ready", - Self::Starting => "starting", - Self::Started => "started", - Self::Committed => "committed", - } - } - - /// Returns whether a workload may have run in the clone once it got this far: once the start - /// of its sandbox was sent. - pub const fn may_have_run(self) -> bool { - matches!(self, Self::Starting | Self::Started | Self::Committed) - } -} - -impl fmt::Display for Progress { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(self.as_str()) - } -} - -/// How the teardown that follows a failure of a clone ended. -#[derive(Clone, Debug, PartialEq, Eq, Hash)] -#[non_exhaustive] -pub enum Teardown { - /// The clone failed before its VMM was spawned: no VM runs. - NoVm, - /// No VMM of the clone runs: the VMM that the launch spawned has exited, as its verified - /// handle proved, or a launch's claim on its log, which no process holds, proved for the - /// launch, whether it spawned a VMM or not. The store's records of the VMM are cleared. - Exited, - /// The teardown terminated the clone's VMM through its verified identity, or through its - /// handle, a VMM that the launch could not record, and confirmed its exit, and the store's - /// records of it are cleared. - Terminated, - /// Whether the clone's VMM still runs cannot be determined, for the reason that this gives: - /// its identity or its exit is indeterminate. The launch's journal or the runtime record - /// remains, and with it the clone's lease, which a release refuses to end until a reattach - /// resolves the teardown. A record that the clone's sandbox keeps holds at most 1024 - /// characters of the reason, as it holds at most 8192 of the - /// [message](FailureRecord::message). - Unresolved(String), - /// The teardown had not ended when the failure was recorded: the record that a failure writes - /// before its teardown, so that a crash of the caller during the teardown leaves it, which the - /// record that the failure writes once its teardown ended replaces. A reattach that finds it - /// finishes the failure, as [`reattach_clone`](super::reattach_clone) describes, so no - /// [`CloneFailure`] that a step of a clone returns has it. - Pending, -} - -impl Teardown { - /// Returns whether the teardown is resolved: whether no VM of the clone runs. - pub fn is_resolved(&self) -> bool { - !matches!(self, Self::Unresolved(_) | Self::Pending) - } - - /// Returns the teardown of a launch that [`Starting::abandon`](crate::host::supervisor::Starting::abandon), - /// or a recovery, ended as `ended` tells. - pub(super) fn of(ended: Result) -> Self { - match ended { - Ok(Recovery::Clean | Recovery::Exited) => Self::Exited, - Ok(Recovery::Terminated(_)) => Self::Terminated, - Ok(Recovery::Indeterminate(reason)) => Self::Unresolved(reason), - Err(error) => Self::Unresolved(error.to_string()), - } - } - - /// Returns the teardown's name, in lowercase words joined by hyphens, as its record spells it. - pub const fn as_str(&self) -> &'static str { - match self { - Self::NoVm => "no-vm", - Self::Exited => "exited", - Self::Terminated => "terminated", - Self::Unresolved(_) => "unresolved", - Self::Pending => "pending", - } - } -} - -/// The record of a failure of a clone: what the failure was about, where and how it failed, and -/// how its teardown ended, as [`CloneFailure::record`] returns it. -/// -/// # Records -/// -/// A failure from the clone's launch on is recorded in the clone's sandbox, as -/// [`CLONE_FAILURE_NAME`](crate::host::store::CLONE_FAILURE_NAME): before the teardown that -/// follows it, whose record is [pending](Teardown::Pending) until then, so that a crash of the -/// caller during the teardown leaves the record, which a reattach finishes, and once its teardown -/// is known, so that the record holds the stage, the evidence and the status, the original -/// outcome, the transaction of the clone's restore, the leased generation, the clone's progress, -/// and the teardown's outcome. A failure that cannot be recorded before its teardown removes -/// nothing that tells that the clone launched: the journal of its launch, or the runtime record of -/// its VM, remains, and with it the lease, so that a reattach still finds that the clone failed, -/// though not how, if a crash of the caller comes before the record. A failure of the plan, before -/// the launch, leaves no VM and no record: the caller holds the failure, and releases the lease. A -/// launch keeps its lease, and -/// with it the leased generation, until its VM has exited, so a failure that comes after the -/// clone's Ready, such as one of the repair's deferred work, is classified against that -/// generation all the same, and so is the status with which the guest ended a VM whose exit a -/// reattach confirms only later. -#[derive(Clone, Debug, PartialEq, Eq)] -#[non_exhaustive] -pub struct FailureRecord { - /// The identity of the template store that leased the clone's generation. - pub store: StoreId, - /// The identity of the clone's lease. - pub lease: LeaseId, - /// The leased generation, and the identity of its key. - pub template: TemplateReference, - /// The launch that the lease reserves. - pub launch: LaunchId, - /// The transaction ID of the clone's restore, once its plan drew it. - pub transaction: Option, - /// The stage at which the clone failed. - pub stage: Stage, - /// How far the clone got: its acknowledgment and the progress of its activation included. - pub progress: Progress, - /// The evidence of the failure. - pub evidence: Evidence, - /// The status with which the guest ended the clone's VM, as OpenVMM's outcome report gives - /// it, if the report has one. - pub status: Option, - /// The original outcome of the failure, to which its evidence attributes it, whatever the - /// teardown that followed it, and the status with which the guest ended the VM, once the VMM - /// exited. - pub outcome: Outcome, - /// How the teardown that followed the failure ended. - pub teardown: Teardown, - /// The code of the failure's error. - pub code: ErrorCode, - /// The message of the failure's error, followed by the messages of its sources. A record that - /// the clone's sandbox keeps holds at most 8192 of its characters: of a longer message, as a - /// caller's diagnosis may be, its first and its last ones, with an ellipsis between them. - pub message: String, -} - -impl FailureRecord { - /// Returns the record of a failure of the clone that holds `lease`, with `transaction`, at - /// the stage and the progress `at`, with `evidence`, the guest's `status`, and `teardown`, - /// whose error is `error`. - pub(super) fn new( - lease: &Lease, - transaction: Option, - (stage, progress): (Stage, Progress), - evidence: Evidence, - status: Option, - teardown: Teardown, - error: &Error, - ) -> Self { - Self { - store: lease.store(), - lease: lease.id(), - template: lease.template(), - launch: lease.launch().clone(), - transaction, - stage, - progress, - evidence, - status, - outcome: evidence.outcome(), - teardown, - code: error.code(), - message: describe(error), - } - } - - /// Returns the error that the record describes, with its code and its message. - pub fn error(&self) -> Error { - Error::new(self.code, self.message.clone()) - } -} - -/// A failure of a clone: the error of the step that failed, with the [`FailureRecord`] that -/// attributes it, as the clone's plan, launch, restore sequence, and reattach return it. -/// -/// It displays as its error, whose [`kind`](Self::kind) and [`code`](Self::code) it has, and -/// converts into it, without the record. A failure from the clone's launch on is recorded in the -/// clone's sandbox as well, as [the records of failures](FailureRecord#records) describe. -#[derive(Debug)] -pub struct CloneFailure { - /// The error of the step that failed. - error: Error, - /// The failure's record. - record: Box, -} - -impl CloneFailure { - /// Returns the failure of `error`, which `record` attributes. - pub(super) fn new(error: Error, record: FailureRecord) -> Self { - Self { - error, - record: Box::new(record), - } - } - - /// Returns the error of the step that failed. - pub fn error(&self) -> &Error { - &self.error - } - - /// Returns the error of the step that failed, without the record. - pub fn into_error(self) -> Error { - self.error - } - - /// Returns the error's kind. - pub fn kind(&self) -> ErrorKind { - self.error.kind() - } - - /// Returns the error's code. - pub fn code(&self) -> ErrorCode { - self.error.code() - } - - /// Returns the error's message. - pub fn message(&self) -> &str { - self.error.message() - } - - /// Returns the failure's record. - pub fn record(&self) -> &FailureRecord { - &self.record - } - - /// Returns the outcome to which the failure is attributed. - pub fn outcome(&self) -> Outcome { - self.record.outcome - } - - /// Returns how the teardown that followed the failure ended. - pub fn teardown(&self) -> &Teardown { - &self.record.teardown - } - - /// Returns the failure attributed to the host's staging, once the caller proved that its - /// staging, rather than the request, made a file that the request supplied wrong: a failure - /// whose evidence is a [supplied file](Evidence::is_supplied_file) with the wrong identity, - /// length, or geometry, as the plan finds it, which records nothing. Any other failure is - /// returned unchanged. - pub fn with_staging_proven(mut self) -> Self { - if self.record.evidence.is_supplied_file() { - self.record.outcome = Outcome::Staging; - } - self - } - - /// Quarantines the leased generation through its tombstone, as - /// [`Store::tombstone`] does, waiting for the store's lock until `deadline`, or for as long as - /// it takes without one, if the failure is the [template's](Outcome::Template), and returns - /// whether it did. A failure of any other outcome quarantines nothing. - /// - /// The tombstone changes nothing of the generation's payload or snapshot: the generation takes - /// no new lease, while the clones that lease it keep running, and a retirement deletes it once - /// they have released it. The store's lock comes before a sandbox's in the - /// [lock order](crate::host::supervisor#the-lock-order), so the caller quarantines once it - /// released the lock of the clone's sandbox, such as the one that an - /// [`Acquired`](crate::host::template::Acquired) holds. - /// - /// Fails with [`ErrorKind::Other`] for a store of another identity than the one that leased - /// the generation, and as [`Store::tombstone`] fails. - pub fn quarantine(&self, store: &Store, deadline: Option) -> Result { - if !self.record.outcome.quarantines() { - return Ok(false); - } - if store.id() != self.record.store { - return Err(Error::other(format!( - "the generation {} that a failure quarantines is one of the template store {}, \ - not of this one, {}", - self.record.template.generation, - self.record.store, - store.id() - ))); - } - store.tombstone(self.record.template.generation, deadline)?; - Ok(true) - } -} - -impl fmt::Display for CloneFailure { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(&self.error, formatter) - } -} - -impl std::error::Error for CloneFailure { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - std::error::Error::source(&self.error) - } -} - -impl From for Error { - fn from(failure: CloneFailure) -> Self { - failure.error - } -} - -/// The error of a step of a clone, with the evidence that attributes it, before the stage at which -/// the clone failed, its progress, and the teardown that follows are known. -#[derive(Debug)] -pub(super) struct Fault { - /// The evidence of the failure. - pub(super) evidence: Evidence, - /// The step's error. - pub(super) error: Error, -} - -impl Fault { - /// Returns the fault of `error`, which `evidence` attributes. - pub(super) fn new(evidence: Evidence, error: Error) -> Self { - Self { evidence, error } - } - - /// Returns the fault of `error`, a failure of the host's own operations, as - /// [`host_evidence`] attributes it. - pub(super) fn host(error: Error) -> Self { - Self::new(host_evidence(&error), error) - } -} - -/// Attributes the error of a step of a clone. -pub(super) trait Attribute { - /// Returns the result with its error attributed by `evidence`. - fn attribute(self, evidence: Evidence) -> std::result::Result; - - /// Returns the result with its error attributed as a failure of the host's own operations, as - /// [`host_evidence`] attributes it. - fn on_host(self) -> std::result::Result; -} - -impl Attribute for Result { - fn attribute(self, evidence: Evidence) -> std::result::Result { - self.map_err(|error| Fault::new(evidence, error)) - } - - fn on_host(self) -> std::result::Result { - self.map_err(Fault::host) - } -} - -/// Returns the evidence of `error`, a failure of the host's own operations: resource exhaustion -/// if an I/O error among its sources says that the host ran out of storage, memory, or -/// descriptors, and a host failure otherwise. -pub(super) fn host_evidence(error: &Error) -> Evidence { - let mut source = std::error::Error::source(error); - while let Some(cause) = source { - if let Some(io) = cause.downcast_ref::() - && exhausted(io) - { - return Evidence::ResourceExhaustion; - } - source = cause.source(); - } - Evidence::HostFailure -} - -/// Returns whether `error` says that the host ran out of a resource. -fn exhausted(error: &io::Error) -> bool { - // Too many open files, in the system or in the process. - #[cfg(unix)] - if [nix::errno::Errno::ENFILE, nix::errno::Errno::EMFILE] - .into_iter() - .any(|errno| error.raw_os_error() == Some(errno as i32)) - { - return true; - } - matches!( - error.kind(), - io::ErrorKind::StorageFull - | io::ErrorKind::QuotaExceeded - | io::ErrorKind::FileTooLarge - | io::ErrorKind::OutOfMemory - ) -} - -/// Returns the message of `error`, followed by the messages of its sources, as the record of a -/// failure keeps it. -pub(super) fn describe(error: &Error) -> String { - let mut text = error.to_string(); - let mut source = std::error::Error::source(error); - while let Some(cause) = source { - text.push_str(": "); - text.push_str(&cause.to_string()); - source = cause.source(); - } - text -} - -/// Returns `error` with `addition` after its message, with its code and its source. -pub(super) fn amended(error: Error, addition: &str) -> Error { - let code = error.code(); - let (_, message, source) = error.into_parts(); - let error = Error::new(code, format!("{message}{addition}")); - match source { - Some(source) => error.with_source(source), - None => error, - } -} - -/// Returns the evidence of `status`, with which the guest ended a clone's VM once the clone got -/// as far as `progress`, if the status alone is evidence, as [the attribution of -/// evidence](Evidence#attribution) describes: a status of the time ABI or of a clone's repair is -/// evidence of the repair stage at any point, a status of a clone's activation is evidence of its -/// class before the start of the clone's sandbox was sent, and no other status is evidence. -pub fn status_evidence(status: u8, progress: Progress) -> Option { - if status::TIME_STATUSES.contains(&status) || status::REPAIR_STATUSES.contains(&status) { - return Some(Evidence::RepairStatus); - } - if progress.may_have_run() { - return None; - } - Some(match status { - status::ACTIVATION_DEADLINE => Evidence::Timeout, - status::ACTIVATION_PROTOCOL | status::ACTIVATION_ORDER => Evidence::Protocol, - status::ACTIVATION_RECORD => Evidence::InvalidActivate, - status::ACTIVATION_RESOURCES => Evidence::InvalidTargets, - status::ACTIVATION_ROOT => Evidence::UnusableRoot, - status::ACTIVATION_WORKLOAD | status::ACTIVATION_NETWORK => Evidence::InvalidSettings, - status::ACTIVATION_EXPORT => Evidence::InvalidMappings, - _ => return None, - }) -} - -/// Returns the guest's status in OpenVMM's outcome report `report` of a clone's VM, and the -/// evidence that the status alone is once the clone got as far as `progress`, as -/// [`status_evidence`] gives it: the status of a report whose category is `guest-exit` or -/// `success`, the guest's own end, and none for any other, such as a failure of the VMM. -pub(super) fn reported( - report: Option<&Report>, - progress: Progress, -) -> (Option, Option) { - let Some(report) = report else { - return (None, None); - }; - let status = match (report.category.as_deref(), report.status_code) { - (Some("guest-exit" | "success"), Some(code)) => u8::try_from(code).ok(), - _ => None, - }; - let evidence = status.and_then(|status| status_evidence(status, progress)); - (evidence, status) -} - -/// Returns the evidence of a failure whose step saw `evidence`, once the guest's status, which -/// ended the clone's VM, is `reported`, as [`reported`] gives it, as [the attribution of -/// evidence](Evidence#attribution) describes: the status's evidence prevails, unless `evidence` -/// is of the template, whose provenance the status alone does not show. -pub(super) fn prevailing(evidence: Evidence, reported: Option) -> Evidence { - match reported { - Some(reported) if evidence.outcome() != Outcome::Template => reported, - _ => evidence, - } -} - -/// The start of the line with which OpenVMM reports, on its standard error, which the launch's log -/// keeps, the error that ended it: the line goes on with the error's first time ABI code in -/// brackets, such as `[E_TSC_RATE_TOLERANCE]`, if the error's chain has one. -pub(super) const FATAL_PREFIX: &str = "fatal error: "; - -/// The time ABI codes with which OpenVMM refuses to restore a snapshot on a host that cannot -/// present the template's CPU or time: a CPU that the template's CPU profile does not serve, a -/// hypervisor that cannot present the profile, as OpenVMM groups the failures of a profile, and a -/// TSC or a LAPIC timer that the snapshot's time contract does not accept. -const INCOMPATIBLE_HOST_CODES: [&str; 13] = [ - "E_PROFILE_HOST_UNKNOWN", - "E_CPU_GENERATION", - "E_PROFILE_UNSUPPORTED", - "E_PROFILE_TIME_BITS", - "E_CPU_SURFACE", - "E_CPU_UNLISTED", - "E_IDENTITY_ROUTING", - "E_TSC_SYNC_UNSUPPORTED", - "E_TSC_SCALING_ACTIVE", - "E_TSC_RATE_UNAVAILABLE", - "E_TSC_RATE_TOLERANCE", - "E_LAPIC_RATE_UNAVAILABLE", - "E_LAPIC_RATE_MISMATCH", -]; - -/// The text with which a cause of OpenVMM's fatal error names a forward of a clone's network whose -/// port of host loopback OpenVMM cannot bind as it restores the clone's NIC: the forward's -/// protocol, `Tcp` or `Udp`, its address, and its port follow, then `)`, as the excerpt of -/// OpenVMM's source `tests/vectors/openvmm/port_forward_socket_creation.rs` spells the error. -/// OpenVMM gives the error no code. -pub(super) const FORWARD_BIND_TEXT: &str = "failed to create socket for port forward ("; - -/// Returns the evidence of `line`, the line with which OpenVMM reported the error that ended a -/// clone's VMM before its restore-ready event, followed by the cause of the error's chain that -/// names a forward that OpenVMM cannot bind, if it has one, as the launch reads them from the end -/// of the VMM's log, as [the attribution of evidence](Evidence#attribution) describes: -/// [`Evidence::IncompatibleHost`] for an error whose time ABI code says that the host cannot -/// present the template's CPU or time, [`Evidence::ForwardBind`] for one that names a forward that -/// OpenVMM cannot bind, [`FORWARD_BIND_TEXT`], and none for any other. -pub(super) fn fatal_evidence(line: &str) -> Option { - let reported = line.strip_prefix(FATAL_PREFIX)?; - let code = reported - .strip_prefix('[') - .and_then(|code| code.get(..code.find(']')?)); - if code.is_some_and(|code| INCOMPATIBLE_HOST_CODES.contains(&code)) { - return Some(Evidence::IncompatibleHost); - } - reported - .contains(FORWARD_BIND_TEXT) - .then_some(Evidence::ForwardBind) -} - -#[cfg(test)] -mod tests; diff --git a/nvx-rs/src/host/template/clone/failure/tests.rs b/nvx-rs/src/host/template/clone/failure/tests.rs deleted file mode 100644 index 2f5e90afe..000000000 --- a/nvx-rs/src/host/template/clone/failure/tests.rs +++ /dev/null @@ -1,408 +0,0 @@ -use super::*; - -/// Every evidence, in the order of the module's table. -const ALL: [Evidence; 27] = [ - Evidence::AuthorizationDenied, - Evidence::StaleBinding, - Evidence::InvalidTargets, - Evidence::InvalidFormat, - Evidence::InvalidSettings, - Evidence::InvalidActivate, - Evidence::InvalidMappings, - Evidence::UnservableImage, - Evidence::UnusableScratch, - Evidence::UnusableRoot, - Evidence::SuppliedIdentity, - Evidence::SuppliedLength, - Evidence::SuppliedGeometry, - Evidence::EndpointBind, - Evidence::ForwardBind, - Evidence::ResourceExhaustion, - Evidence::IncompatibleHost, - Evidence::RuntimeMismatch, - Evidence::HostFailure, - Evidence::LostStatus, - Evidence::Timeout, - Evidence::Reset, - Evidence::ForcedKill, - Evidence::Protocol, - Evidence::MalformedGeneration, - Evidence::TemplateDefect, - Evidence::RepairStatus, -]; - -#[test] -fn each_row_of_the_table_attributes_its_evidence() { - let rows: [(&[Evidence], Outcome); 6] = [ - ( - &[ - Evidence::AuthorizationDenied, - Evidence::StaleBinding, - Evidence::InvalidTargets, - Evidence::InvalidFormat, - Evidence::InvalidSettings, - Evidence::InvalidActivate, - Evidence::InvalidMappings, - Evidence::UnservableImage, - Evidence::UnusableScratch, - Evidence::UnusableRoot, - ], - Outcome::Request, - ), - ( - &[ - Evidence::SuppliedIdentity, - Evidence::SuppliedLength, - Evidence::SuppliedGeometry, - ], - Outcome::Request, - ), - ( - &[ - Evidence::EndpointBind, - Evidence::ForwardBind, - Evidence::ResourceExhaustion, - Evidence::IncompatibleHost, - Evidence::RuntimeMismatch, - Evidence::HostFailure, - ], - Outcome::Host, - ), - ( - &[ - Evidence::LostStatus, - Evidence::Timeout, - Evidence::Reset, - Evidence::ForcedKill, - Evidence::Protocol, - ], - Outcome::Unknown, - ), - ( - &[Evidence::MalformedGeneration, Evidence::TemplateDefect], - Outcome::Template, - ), - (&[Evidence::RepairStatus], Outcome::Repair), - ]; - let mut seen = Vec::new(); - for (evidence, outcome) in rows { - for evidence in evidence { - assert_eq!(evidence.outcome(), outcome, "{evidence}"); - assert_eq!( - evidence.outcome().quarantines(), - outcome == Outcome::Template, - "{evidence}" - ); - assert_eq!( - evidence.is_supplied_file(), - evidence.as_str().starts_with("supplied-"), - "{evidence}" - ); - seen.push(*evidence); - } - } - // The table attributes every evidence once. - assert_eq!(seen, ALL); - // Only the template's outcome quarantines. - for outcome in [ - Outcome::Request, - Outcome::Staging, - Outcome::Host, - Outcome::Unknown, - Outcome::Repair, - ] { - assert!(!outcome.quarantines(), "{outcome}"); - } - assert!(Outcome::Template.quarantines()); -} - -#[test] -fn names_are_those_of_the_records() { - for evidence in ALL { - let json = serde_json::to_string(&evidence).unwrap(); - assert_eq!(json, format!("\"{evidence}\"")); - assert_eq!(serde_json::from_str::(&json).unwrap(), evidence); - } - let progress = [ - Progress::Planned, - Progress::Launching, - Progress::Spawned, - Progress::Event, - Progress::Acknowledged, - Progress::Activating, - Progress::Ready, - Progress::Starting, - Progress::Started, - Progress::Committed, - ]; - // The steps are in order, and spelled as their records spell them. - assert!(progress.windows(2).all(|pair| pair[0] < pair[1])); - for step in progress { - assert_eq!(serde_json::to_string(&step).unwrap(), format!("\"{step}\"")); - } - let stages: Vec = progress.iter().map(|step| step.stage()).collect(); - assert_eq!( - stages, - [ - Stage::Plan, - Stage::Launch, - Stage::Restore, - Stage::Activation, - Stage::Activation, - Stage::Activation, - Stage::Start, - Stage::Start, - Stage::Commit, - Stage::Run, - ] - ); - for stage in [ - Stage::Plan, - Stage::Launch, - Stage::Restore, - Stage::Activation, - Stage::Start, - Stage::Commit, - Stage::Run, - Stage::Reattach, - ] { - assert_eq!( - serde_json::to_string(&stage).unwrap(), - format!("\"{stage}\"") - ); - } - for outcome in [ - Outcome::Request, - Outcome::Staging, - Outcome::Host, - Outcome::Unknown, - Outcome::Template, - Outcome::Repair, - ] { - assert_eq!( - serde_json::to_string(&outcome).unwrap(), - format!("\"{outcome}\"") - ); - } -} - -#[test] -fn a_repair_status_alone_is_evidence_of_the_repair_stage_at_any_point() { - for status in status::TIME_STATUSES.chain(status::REPAIR_STATUSES) { - for progress in [Progress::Spawned, Progress::Ready, Progress::Committed] { - assert_eq!( - status_evidence(status, progress), - Some(Evidence::RepairStatus), - "{status} {progress}" - ); - } - } -} - -#[test] -fn an_activation_status_is_evidence_of_its_class_until_a_workload_may_have_run() { - let classes = [ - (status::ACTIVATION_DEADLINE, Evidence::Timeout), - (status::ACTIVATION_PROTOCOL, Evidence::Protocol), - (status::ACTIVATION_RECORD, Evidence::InvalidActivate), - (status::ACTIVATION_RESOURCES, Evidence::InvalidTargets), - (status::ACTIVATION_ROOT, Evidence::UnusableRoot), - (status::ACTIVATION_WORKLOAD, Evidence::InvalidSettings), - (status::ACTIVATION_EXPORT, Evidence::InvalidMappings), - (status::ACTIVATION_NETWORK, Evidence::InvalidSettings), - (status::ACTIVATION_ORDER, Evidence::Protocol), - ]; - assert_eq!( - classes.map(|(status, _)| status).to_vec(), - status::ACTIVATION_STATUSES.collect::>() - ); - for (status, evidence) in classes { - for progress in [Progress::Spawned, Progress::Activating, Progress::Ready] { - assert_eq!( - status_evidence(status, progress), - Some(evidence), - "{status} {progress}" - ); - } - // Once the start was sent, a workload may end the VM with any status. - for progress in [Progress::Starting, Progress::Started, Progress::Committed] { - assert_eq!( - status_evidence(status, progress), - None, - "{status} {progress}" - ); - } - } - // No other status is evidence: a success, the generic failure, or a workload's. - for status in [0, 1, 2, 3, 39, 46, 49, 59, 137, 192, 196, 255] { - assert_eq!( - status_evidence(status, Progress::Activating), - None, - "{status}" - ); - } -} - -#[test] -fn a_report_gives_the_guest_s_status_and_its_evidence() { - let report = |category: &str, status_code: i64| Report { - category: Some(category.to_owned()), - status_code: Some(status_code), - teardown_complete: Some(true), - }; - assert_eq!(reported(None, Progress::Event), (None, None)); - assert_eq!( - reported(Some(&report("guest-exit", 195)), Progress::Event), - (Some(Evidence::RepairStatus), Some(195)) - ); - assert_eq!( - reported(Some(&report("guest-exit", 54)), Progress::Activating), - (Some(Evidence::UnusableRoot), Some(54)) - ); - assert_eq!( - reported(Some(&report("guest-exit", 54)), Progress::Committed), - (None, Some(54)) - ); - assert_eq!( - reported(Some(&report("success", 0)), Progress::Committed), - (None, Some(0)) - ); - // A failure of the VMM, or of its teardown, has no status of the guest's. - assert_eq!( - reported(Some(&report("vmm-failure", 1)), Progress::Spawned), - (None, None) - ); - assert_eq!( - reported(Some(&report("teardown-failure", 1)), Progress::Spawned), - (None, None) - ); - assert_eq!( - reported(Some(&report("guest-exit", 256)), Progress::Spawned), - (None, None) - ); -} - -#[test] -fn a_status_prevails_over_what_was_seen_of_the_same_end_but_not_over_the_template_s_evidence() { - let template = [Evidence::MalformedGeneration, Evidence::TemplateDefect]; - for evidence in ALL { - // Without a status that is evidence, what the step saw stands. - assert_eq!(prevailing(evidence, None), evidence, "{evidence}"); - for reported in [ - Evidence::RepairStatus, - Evidence::UnusableRoot, - Evidence::Timeout, - ] { - let prevailed = prevailing(evidence, Some(reported)); - // A check of the generation, or a diagnosis that attributes a defect to it, shows - // the provenance that the status alone leaves to establish. - let expected = if template.contains(&evidence) { - evidence - } else { - reported - }; - assert_eq!(prevailed, expected, "{evidence} {reported}"); - } - } -} - -#[test] -fn openvmm_s_fatal_error_is_evidence_of_an_incompatible_host_only_for_its_codes() { - for code in INCOMPATIBLE_HOST_CODES { - // OpenVMM leads its fatal error with the first time ABI code of the error's chain. - let line = format!("fatal error: [{code}] failed to launch vm worker"); - assert_eq!( - fatal_evidence(&line), - Some(Evidence::IncompatibleHost), - "{line}" - ); - } - // A code of the snapshot's time contract, or of this OpenVMM, does not tell the - // generation's defect from the host's, and an error without a code tells nothing. - for line in [ - "fatal error: [E_MANIFEST_TIME] failed to launch vm worker", - "fatal error: [E_PROFILE_DIGEST] failed to launch vm worker", - "fatal error: [E_SNAPSHOT_VERSION] failed to restore", - "fatal error: failed to open the snapshot", - "fatal error: [E_TSC_RATE_TOLERANCE failed to launch vm worker", - // A line that is not the fatal error, such as a warning, names no code of it. - "NVX-CPU-PROFILE-FALLBACK: [E_PROFILE_UNSUPPORTED] the host profile serves", - "warning: fatal error: [E_CPU_GENERATION] quoted", - ] { - assert_eq!(fatal_evidence(line), None, "{line}"); - } -} - -#[test] -fn openvmm_s_fatal_error_is_evidence_of_a_forward_bind_when_it_names_a_forward_it_cannot_bind() { - // OpenVMM spells the error as its source, which the excerpt copies, writes it. - let excerpt = - include_str!("../../../../../tests/vectors/openvmm/port_forward_socket_creation.rs"); - let attribute = format!("#[error(\"{FORWARD_BIND_TEXT}{{details}})\")]"); - assert!( - excerpt.lines().any(|line| line.trim() == attribute), - "{attribute} in {excerpt}" - ); - // The cause of the chain follows the line that leads the error, or is the line's own. - for line in [ - "fatal error: failed to launch vm worker: failed to create socket for port forward \ - (Tcp 127.0.0.1:18080)", - "fatal error: failed to create socket for port forward (Udp 127.0.0.1:5353)", - "fatal error: [E_MANIFEST_TIME] failed to launch vm worker: failed to create socket \ - for port forward (Tcp 127.0.0.1:18080)", - ] { - assert_eq!(fatal_evidence(line), Some(Evidence::ForwardBind), "{line}"); - } - // A code of an incompatible host prevails, and a line that is not the fatal error tells - // nothing. - let incompatible = "fatal error: [E_CPU_GENERATION] failed to launch vm worker: failed to \ - create socket for port forward (Tcp 127.0.0.1:18080)"; - assert_eq!( - fatal_evidence(incompatible), - Some(Evidence::IncompatibleHost) - ); - let quoted = "warning: failed to create socket for port forward (Tcp 127.0.0.1:18080)"; - assert_eq!(fatal_evidence(quoted), None); -} - -#[test] -fn host_failures_that_ran_out_of_a_resource_are_resource_exhaustion() { - let failed = |kind: io::ErrorKind| { - host_evidence(&Error::io("cannot write").with_source(io::Error::from(kind))) - }; - assert_eq!( - failed(io::ErrorKind::StorageFull), - Evidence::ResourceExhaustion - ); - assert_eq!( - failed(io::ErrorKind::OutOfMemory), - Evidence::ResourceExhaustion - ); - assert_eq!( - failed(io::ErrorKind::PermissionDenied), - Evidence::HostFailure - ); - assert_eq!( - host_evidence(&Error::io("no source")), - Evidence::HostFailure - ); - // A cause deeper in the chain counts too. - let nested = Error::io("outer") - .with_source(Error::io("inner").with_source(io::Error::from(io::ErrorKind::StorageFull))); - assert_eq!(host_evidence(&nested), Evidence::ResourceExhaustion); -} - -#[test] -fn teardowns_follow_the_end_of_the_launch() { - assert_eq!(Teardown::of(Ok(Recovery::Exited)), Teardown::Exited); - assert_eq!(Teardown::of(Ok(Recovery::Clean)), Teardown::Exited); - assert_eq!( - Teardown::of(Ok(Recovery::Indeterminate("held".to_owned()))), - Teardown::Unresolved("held".to_owned()) - ); - let failed = Teardown::of(Err(Error::io("cannot read"))); - assert!(!failed.is_resolved()); - assert!(!Teardown::Pending.is_resolved()); - assert!(Teardown::NoVm.is_resolved()); - assert!(Teardown::Terminated.is_resolved()); -} diff --git a/nvx-rs/src/host/template/clone/launch.rs b/nvx-rs/src/host/template/clone/launch.rs deleted file mode 100644 index f6b47104d..000000000 --- a/nvx-rs/src/host/template/clone/launch.rs +++ /dev/null @@ -1,794 +0,0 @@ -//! The launch of a clone, [`ClonePlan::launch`]: OpenVMM's restore of the snapshot of the leased -//! generation, as the launch that the clone's lease reserves, with the clone's devices, its -//! activation record, and the endpoint at which the launch awaits the VMM's restore-ready event. - -use std::ffi::{OsStr, OsString}; -use std::fmt; -use std::fs; -use std::io; -use std::path::{Path, PathBuf}; -use std::time::Instant; - -use super::failure::{ - Attribute, CloneFailure, Evidence, FailureRecord, Fault, Progress, Stage, Teardown, - host_evidence, -}; -use super::records::{End, ProgressLog, Recorder, Tracker, abandon}; -use super::{ClonePlan, CloneUpper, MIB}; -use crate::activation::ActivationRecord; -use crate::error::{Error, ErrorKind, Result}; -use crate::host::artifacts; -use crate::host::filesystem; -use crate::host::platform::{self, ReadyListener}; -use crate::host::sandbox::SandboxPlan; -use crate::host::sandbox::distro::CheckedDistro; -use crate::host::store::files::write_private; -use crate::host::store::{LockedStore, Recovery, RuntimeIdentity}; -use crate::host::supervisor::{self, Inherited, Starting, VmmCommand}; -use crate::host::template::key::TemplateKey; -use crate::host::template::store::{Acquired, Lease, SOCKET_PATH_LIMIT}; -use crate::protocol::{CAPABILITY_LEN, HostMap}; - -/// A point of a clone's launch at which a test interrupts it, before the step that follows the -/// point. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(super) enum Point { - /// The sandbox holds the clone's activation record, the launch listens for the restore-ready - /// event, and the launch has not begun. - Prepared, - /// The launch's journal names it, and its VMM is not spawned yet. - Begun, -} - -/// The seams of a clone's launch, through which tests observe and interrupt it. -#[derive(Clone, Copy)] -pub(super) struct Hooks<'a> { - /// Returns the command that starts the VMM, given the one that the launch made. - pub(super) command: &'a dyn for<'h> Fn(VmmCommand<'h>) -> VmmCommand<'h>, - /// Fails the launch at a point, as a test asks. - pub(super) interrupt: &'a dyn Fn(Point) -> Result<()>, - /// Ends the launch of a clone that failed after its VMM was spawned. - pub(super) end: End, -} - -impl Hooks<'static> { - /// The seams of every launch outside tests. - pub(super) const LAUNCH: Self = Self { - command: &unchanged, - interrupt: &uninterrupted, - end: abandon, - }; -} - -/// Returns `command`, as launches outside tests start their VMMs. -fn unchanged(command: VmmCommand<'_>) -> VmmCommand<'_> { - command -} - -/// Lets a launch pass every point, as launches outside tests do. -fn uninterrupted(_: Point) -> Result<()> { - Ok(()) -} - -impl ClonePlan { - /// Launches the clone that the plan describes, under the lock that acquired its lease, which - /// `acquired` holds, as the launch that the lease reserves: the OpenVMM executable `openvmm`, - /// which a relative path names from the current directory, restores the leased generation's - /// snapshot with the clone's devices, under the runtime identity `runtime`, which must be the - /// template's, [`TemplateKey::runtime`], and receives the plan's capability. Returns the - /// launch, whose VMM runs, as a [`Restoring`], whose restore sequence, - /// [`Restoring::activate`], activates the clone, under a deadline that starts as this - /// launch does. - /// - /// The launch first checks, without writing anything, that `acquired` holds the plan's lease, - /// under the lock that acquired it, before the launch that it reserves began, that the - /// sandbox's clone binding records the lease: the acquisition made the binding durable, and no - /// VMM of the clone starts without it, and that the sandbox records no failure of the clone: - /// a lease's clone fails once, and the record of its failure lasts until the release, even - /// for a launch that failed before it began, which leaves the lock's reservation. It then: - /// - /// 1. Creates the clone's progress log, - /// [`CLONE_PROGRESS_NAME`](crate::host::store::CLONE_PROGRESS_NAME) in the sandbox's - /// directory, in place of one that an earlier launch left, which it removes through a - /// handle of the directory that holds the sandbox's lock, once it removed the same way the - /// outcome report that an earlier VM of the sandbox left, so that a report beside the - /// clone's records is the clone's VM's. The launch and the restore sequence append to the - /// log each step that the clone reaches, without syncing it, as - /// [`CloneRecords`](super::CloneRecords) describes. - /// 2. Writes the clone's activation record, in its canonical encoding, as the file - /// [`CLONE_ACTIVATION_NAME`](crate::host::store::CLONE_ACTIVATION_NAME) in the sandbox's - /// directory, in place of one that an earlier launch left, before the VMM starts: the - /// record, with the transaction ID of its restore section, is the one that the clone - /// activates with, which the caller, or a caller that takes the sandbox over after a - /// crash, reads there again and resends unchanged. - /// 3. Listens at the endpoint of the VMM's restore-ready event, which OpenVMM connects to once - /// it has prepared the restore, before the restored VM runs, and which must exist before - /// the VMM starts: on Linux, the socket - /// [`RESTORE_READY_NAME`](crate::host::store::RESTORE_READY_NAME) in the sandbox's - /// directory, in place of one that an earlier launch left; on Windows, a named pipe whose - /// name the launch draws at random. - /// 4. Runs [`supervisor::launch`] as the launch that the lease reserves, which begins it with - /// the store's launch journal, starts the VMM, and records its identity in the journal. - /// 5. Appends the spawn to the progress log before it returns, since a teardown of the launch - /// removes the journal, which records the spawn too. A launch that cannot append it fails - /// at the launch stage, and ends the VMM. - /// - /// The VMM restores the generation's snapshot in place, in the generation's payload, which it - /// only reads, and which the launch neither copies nor changes, so any number of clones - /// restore one generation at once. OpenVMM requires the control socket of a restore's VMM on - /// Linux in the snapshot's parent, so the VMM listens beside the snapshot, at the socket that - /// the lease reserved, `.sock` in the payload, after the launch's identity, rather - /// than at the sandbox's own control socket, as - /// [`Store::acquire`](crate::host::template::Store::acquire) describes; on Windows, at a named - /// pipe, as every launch does. [`Starting::endpoint`] returns it. - /// - /// The VMM's arguments are those of the clone's devices: the microVM machine, the clone's - /// image as the read-only `distro` block device, which OpenVMM opens at its path, as [Block - /// devices by path](crate::host::template::Store#block-devices-by-path) describes, and presents - /// at the capacity that the template's placeholder fixed, reading zeros past the end of a - /// shorter image; for an upper on a scratch image, the image as the `scratch` block device, - /// which OpenVMM opens alike, and for an upper in RAM, `--restore-absent-scratch`, which - /// declares the template's scratch absent, so that OpenVMM attaches it with no file, at the - /// geometry that the template recorded, as the template's guest negotiated it: its reads return - /// zeros and its writes fail, and the clone's guest never uses it; and the clone's network, if - /// the template has a NIC: `--restore-network-policy`, with which the policy of the clone's - /// network replaces the template's complete deny-all policy on the NIC, and the options of that - /// policy, without `--net` and the guest's address, which OpenVMM takes from the snapshot. The - /// clone's export, if it has one, follows, bound to the template's dormant filesystem slot as a - /// cold launch attaches it, as its [plan](ClonePlan::export) holds it, - /// [`ExportPlan::openvmm_arguments`](crate::host::filesystem::ExportPlan::openvmm_arguments): - /// `--mount-aggregate` with [`GUEST_EXPORT`](crate::host::filesystem::GUEST_EXPORT), a - /// `--mount-child` of each of its directories and files, with its mode, `rw` if a mapping - /// writes it, and `ro` otherwise, and `file` for a file, then their hidden, re-exposed, and - /// writable paths, and the caller owner; without an export, the slot stays dormant. Then come - /// `--single-process`, the template's backend, - /// CPU profile, and processor capacity, `--restore-snapshot` with the directory of the - /// generation's snapshot, [`ClonePlan::snapshot`], for a template that records its processors - /// online at boot, [`TemplateKey::boot_processors`], `--restore-processors` with the processor - /// target, which brings online those of the target beyond them, `--restore-memory` with the - /// memory target, - /// `--restore-ready-path` with the endpoint of the restore-ready event, a boot console that - /// discards its output, as cold boots have, the control console at the launch's endpoint, - /// which authenticates with the capability that the VMM reads from its standard input, and the - /// outcome report, [`SandboxStore::outcome_path`](crate::host::store::SandboxStore::outcome_path). - /// They name no kernel, initramfs, kernel command line, or RAM, which the snapshot fixes. The - /// VMM runs in the sandbox's directory. - /// - /// The claim of a scratch image passes to the VMM, which holds it until it exits. On Linux, - /// the VMM inherits the claim's descriptor, which no argument names. On Windows, the claim's - /// handle, open for writing, would make OpenVMM's open of the image fail, since that open - /// shares no writing: the launch closes it before the VMM starts, and the VMM inherits the - /// image's guard instead, which keeps every other claim out, and the image's path naming the - /// image, while the VMM runs. The [`Restoring`] holds the plan's image, open for reading. - /// - /// A crash of the caller at any step leaves the clone's files in the sandbox, which a release - /// of the lease, [`Store::release`](crate::host::template::Store::release), removes, and a - /// launch that the journal names, which a reattach ends, as - /// [Crashes](crate::host::supervisor#crashes) describes, removing the control socket that the - /// VMM, if it started, left beside the snapshot. - /// - /// Like every launch of the sandbox's store, the launch names the sandbox's files by their - /// paths, under its lock: the progress log that it creates, the activation record that it - /// writes, the restore-ready socket at which it listens on Linux, and the journal, capability, - /// and log with which it begins; and the VMM, which runs in the sandbox's directory, opens the - /// files that its arguments name there. So the directories above the sandbox's must be ones - /// that only the current user, or an administrator, may change, as the - /// [store](crate::host::store) requires of them, and as [Block devices by - /// path](crate::host::template::Store#block-devices-by-path) requires of those above the - /// clone's block devices. The launch removes the outcome report of an earlier VM of the - /// sandbox, and the progress log of an earlier launch, through a handle of the directory that - /// holds the sandbox's lock, though, as a release removes the clone's files, and creates its - /// own log at its path, where no file may exist: no change of the path makes the launch remove - /// another sandbox's report or log, and another sandbox's log fails the launch instead, before - /// it writes anything else. - /// - /// Fails with a [`CloneFailure`] at [`Stage::Launch`], whose evidence is the host's for a - /// failure of its own operations, as [`ClonePlan::new`] describes it, checking in this order: - /// - /// - before anything is written, and so recording nothing, as a plan does: with - /// [`ErrorKind::Other`](crate::ErrorKind::Other) if `acquired` holds another lease than the - /// plan's, under another lock than the one that acquired it, or after the launch that it - /// reserves began, if the sandbox's clone binding does not record the plan's lease, and if - /// the sandbox records a failure of the clone, [`Evidence::HostFailure`], and as - /// [`Lease::of`] fails, or the record of a failure cannot be examined, the host's evidence; - /// with - /// [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for a runtime identity other - /// than the template's, [`Evidence::RuntimeMismatch`], or on Windows for a sandbox directory - /// beyond the VMM working-directory limit, [`Evidence::HostFailure`]; on Linux, with - /// [`ErrorKind::Other`](crate::ErrorKind::Other) if the control socket that the lease - /// reserves is not beside the plan's snapshot, as for a plan and a lease that name the - /// template store by different directories, [`Evidence::HostFailure`], with - /// [`ErrorKind::Io`](crate::ErrorKind::Io) if the generation's payload cannot be resolved - /// or opened, the host's evidence, or is not a directory of the current user with mode - /// 0700, as OpenVMM requires of the directory of a control socket, - /// [`Evidence::MalformedGeneration`], and with - /// [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) if the path of the control - /// socket, with or without the payload's links resolved, or of the restore-ready socket, is - /// not shorter than 108 bytes, as a Unix socket's address requires, or is not valid UTF-8, - /// as OpenVMM and the host take it, or if the path of the control socket holds a comma or an - /// equals sign, which OpenVMM's option `listen=` cannot take, - /// [`Evidence::EndpointBind`]; as - /// [`ExportPlan::verify`](crate::host::filesystem::ExportPlan::verify) fails for the clone's - /// export, if it has one: with [`ErrorKind::Other`](crate::ErrorKind::Other) for a - /// directory or a file of the export that is no longer the kind of object that the export's - /// plan found, a mapped or hidden object that the guest could move that is no longer the - /// object that the plan found, or an object that its path no longer reaches, and with - /// [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for a read-only mapping that - /// the host cannot keep read-only, or an owner that the host no longer supports, - /// [`Evidence::InvalidMappings`], and with [`ErrorKind::Io`](crate::ErrorKind::Io) if an - /// object cannot be inspected otherwise, the host's evidence; with - /// [`ErrorKind::Unavailable`](crate::ErrorKind::Unavailable) for an `openvmm` whose path - /// cannot be resolved, the host's evidence; with - /// [`ErrorKind::Other`](crate::ErrorKind::Other) if the sandbox's path no longer names the - /// directory that holds its lock, and if the progress log cannot be created, as when the - /// path names a directory that holds one already, the host's evidence; - /// - once the progress log exists, recording the failure in the sandbox, as the - /// [failures](CloneFailure) of a clone's launch are: with - /// [`ErrorKind::Io`](crate::ErrorKind::Io) if the activation record cannot be written, the - /// host's evidence, or the endpoint of the restore-ready event cannot be opened, - /// [`Evidence::EndpointBind`] unless the host ran out of a resource; and as - /// [`supervisor::launch`] fails, which ends the launch, the host's evidence, with the - /// teardown [`Teardown::NoVm`] before the spawn of the VMM, [`Teardown::Terminated`] once - /// the launch ended the VMM that it spawned through its handle, and - /// [`Teardown::Unresolved`] for a launch that could not be ended, whose journal remains. - /// - /// A launch that fails leaves the clone's files that it wrote, and the record of its failure, - /// for the release. - pub fn launch<'s>( - self, - acquired: &'s mut Acquired, - openvmm: &Path, - runtime: &RuntimeIdentity, - ) -> Result, CloneFailure> { - self.launch_with(acquired, openvmm, runtime, Hooks::LAUNCH) - } - - /// Launches the clone as [`launch`](Self::launch) does, through `hooks`. - pub(super) fn launch_with<'s>( - self, - acquired: &'s mut Acquired, - openvmm: &Path, - runtime: &RuntimeIdentity, - hooks: Hooks<'_>, - ) -> Result, CloneFailure> { - // The restore-to-Ready deadline of the clone's restore sequence starts here. - let launched = Instant::now(); - let Acquired { lease, locked } = acquired; - let planned = self.lease.clone(); - let transaction = self.activation.restore().map(|restore| restore.transaction); - // Until the launch writes a file, the sandbox may not be the clone's, so its failure is - // recorded nowhere. - let unrecorded = |fault: Fault| { - let record = FailureRecord::new( - &planned, - transaction, - (Stage::Launch, Progress::Planned), - fault.evidence, - None, - Teardown::NoVm, - &fault.error, - ); - CloneFailure::new(fault.error, record) - }; - self.check_launch(lease, locked, runtime) - .map_err(unrecorded)?; - let openvmm = artifacts::absolute(openvmm).on_host().map_err(unrecorded)?; - let Self { - lease: _, - generation: _, - key, - snapshot, - image, - upper, - processors, - memory, - activation, - capability, - devices, - export, - } = self; - let mut devices = device_arguments(&devices) - .attribute(Evidence::InvalidSettings) - .map_err(unrecorded)?; - let table = match &export { - Some(export) => { - devices.extend(export.arguments()); - export.table() - } - None => Vec::new(), - }; - let log = locked - .open_dir() - .and_then(|directory| ProgressLog::create(&directory, locked.dir(), lease.launch())) - .on_host() - .map_err(unrecorded)?; - let recorder = Recorder::new(lease.clone(), transaction, locked); - let mut tracker = Tracker::new(recorder, log, hooks.end); - let unspawned = - |tracker: &Tracker, fault: Fault| tracker.record(Stage::Launch, fault, Teardown::NoVm); - let record = locked.clone_activation_path(); - write_private(&record, &activation.encode()) - .on_host() - .map_err(|fault| unspawned(&tracker, fault))?; - let (ready, listener) = - platform::listen_ready(&locked.restore_ready_path()).map_err(|error| { - let fault = Fault::host( - Error::io(format!( - "cannot listen for the restore-ready event of the clone in {}", - locked.dir().display() - )) - .with_source(error), - ); - let fault = match fault.evidence { - Evidence::HostFailure => Fault::new(Evidence::EndpointBind, fault.error), - _ => fault, - }; - unspawned(&tracker, fault) - })?; - (hooks.interrupt)(Point::Prepared) - .on_host() - .map_err(|fault| unspawned(&tracker, fault))?; - // On Windows, this closes the claim's handle, and keeps the image's guard. - let claim = match upper { - CloneUpper::Scratch(checked) => Some(checked.into_file()), - CloneUpper::Tmpfs => None, - }; - // No spawn passes a descriptor of a standard stream, on which the claim is if the caller - // had closed that stream: it moves above them, with its open file description. - #[cfg(target_os = "linux")] - let claim = match claim { - Some(mut file) => { - crate::host::sys::move_above_standard_streams(&mut file).map_err(|error| { - unspawned( - &tracker, - Fault::host( - Error::io( - "cannot pass the claim of the clone's scratch image to its VMM", - ) - .with_source(error), - ), - ) - })?; - Some(file) - } - None => None, - }; - let restore = Restore { - devices, - key: &key, - snapshot, - processors, - memory, - ready: &ready, - report: locked.outcome_path(), - }; - let dir = locked.dir().to_owned(); - let inherit: Vec> = claim.iter().map(Inherited::new).collect(); - let starting = supervisor::launch_reporting( - locked, - lease.launch(), - runtime, - &capability, - |endpoint| { - (hooks.interrupt)(Point::Begun)?; - let mut command = VmmCommand::new(&openvmm, restore.arguments(endpoint), &dir); - command.inherit = inherit; - Ok((hooks.command)(command)) - }, - |error, spawned| { - // A VMM that the launch spawned, and could not record, got the clone that far, - // which only the record of its failure tells once the launch has ended it. - if spawned { - tracker.advance(Progress::Spawned); - } - // A failure that cannot be recorded keeps the launch's journal. - tracker.pending(Stage::Launch, host_evidence(error), error) - }, - ); - // The VMM holds the claim, if it started. - drop(claim); - let starting = match starting { - Ok(starting) => starting, - Err(failure) => { - let teardown = match (failure.unended, failure.spawned) { - (Some(reason), _) => Teardown::Unresolved(reason), - (None, true) => Teardown::Terminated, - (None, false) => Teardown::NoVm, - }; - return Err(tracker.record(Stage::Launch, Fault::host(failure.error), teardown)); - } - }; - // The progress log records the spawn before the launch returns: the journal records it too, - // but a teardown of the launch removes the journal, and a reattach then takes how far the - // clone got, and so the stage of its failure, from the log. - if let Err(fault) = tracker.reach(Progress::Spawned) { - return Err(tracker.fail(starting, Stage::Launch, fault)); - } - Ok(Restoring { - starting, - launched, - ready, - listener, - image, - activation, - record, - capability, - tracker, - table, - }) - } - - /// Refuses a launch of the plan through `lease` and `locked` that is not the one that the - /// plan's lease reserves, or whose binding the sandbox does not record, one under `runtime` - /// that the template does not take, and one of an export whose host objects changed since its - /// plan found them, as - /// [`launch`](Self::launch) describes, without writing anything. - fn check_launch( - &self, - lease: &Lease, - locked: &LockedStore, - runtime: &RuntimeIdentity, - ) -> Result<(), Fault> { - let planned = &self.lease; - if lease != planned - || locked.dir() != planned.sandbox() - || locked.reserved() != Some(planned.launch()) - { - return Err(Fault::new( - Evidence::HostFailure, - Error::other(format!( - "a clone is launched under the lock that acquired the lease of its plan, {}, \ - before the launch that the lease reserves begins, but the lock of the \ - sandbox in {} holds the lease {}, and reserves no launch {} of it", - planned.id(), - locked.dir().display(), - lease.id(), - planned.launch() - )), - )); - } - if Lease::of(locked).on_host()?.as_ref() != Some(planned) { - return Err(Fault::new( - Evidence::HostFailure, - Error::other(format!( - "the sandbox in {} records no clone binding of the lease {}, which its \ - acquisition makes durable before any VMM of the clone starts", - locked.dir().display(), - planned.id() - )), - )); - } - // A launch that failed before it began leaves the lock's reservation, but the record of - // its failure, which no launch removes, ends the clone's launches until the release. - let failure = locked.clone_failure_path(); - match fs::symlink_metadata(&failure) { - Ok(_) => { - return Err(Fault::new( - Evidence::HostFailure, - Error::other(format!( - "the clone of the lease {} failed, as {} records: a lease's clone fails \ - once, and the release of the lease ends it", - planned.id(), - failure.display() - )), - )); - } - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => { - return Err(Fault::host( - Error::io(format!( - "cannot examine the record of a failure of the clone {}", - failure.display() - )) - .with_source(error), - )); - } - } - if runtime != self.key.runtime() { - return Err(Fault::new( - Evidence::RuntimeMismatch, - Error::unsupported(format!( - "the template's generation {} restores under the runtime that captured it, \ - {:?}, not under {runtime:?}", - self.generation.generation(), - self.key.runtime() - )), - )); - } - platform::check_working_dir(locked.dir()).map_err(Fault::host)?; - if cfg!(target_os = "linux") - && let Some(socket) = locked.reserved_socket() - { - self.check_sockets(socket, &locked.restore_ready_path())?; - } - // The objects of the export must still be those that its plan found, before each restore, - // as before each cold start: a refusal is of the request's mappings, which no longer name - // what they named, as for an object that moved or is missing, and any other failure is the - // host's own. - if let Some(export) = &self.export { - export.plan().verify().map_err(|error| match error.kind() { - ErrorKind::Other | ErrorKind::Unsupported => { - Fault::new(Evidence::InvalidMappings, error) - } - _ => Fault::host(error), - })?; - } - Ok(()) - } - - /// Refuses, on Linux, a control socket of the clone's VMM at `socket`, beside the plan's - /// snapshot, and a socket of the restore-ready event at `ready`, that OpenVMM or the host - /// could not use, as [`launch`](Self::launch) describes, without writing anything. - /// - /// OpenVMM requires the control socket of a restore in the snapshot's parent, a directory of - /// the current user with mode 0700, which the payload of a generation is, and listens at the - /// socket's path with the parent's links resolved, while the host connects at the path that it - /// recorded: each must be shorter than 108 bytes, as a Unix socket's address requires, and - /// valid UTF-8, as OpenVMM and the host take them, as must the path of the restore-ready - /// socket. OpenVMM takes the control socket's path in its option `listen=`, among - /// options that commas separate, and whose values follow an equals sign, so the path must hold - /// neither. - fn check_sockets(&self, socket: &Path, ready: &Path) -> Result<(), Fault> { - let (Some(payload), Some(name)) = (socket.parent(), socket.file_name()) else { - return Err(Fault::new( - Evidence::HostFailure, - Error::other(format!( - "the control socket {} of the clone's VMM names no directory", - socket.display() - )), - )); - }; - let resolve = |path: &Path| { - filesystem::canonicalize(path).map_err(|error| { - Fault::host( - Error::io(format!( - "cannot resolve the payload of the generation {}", - path.display() - )) - .with_source(error), - ) - }) - }; - let canonical = resolve(payload)?; - let beside = match self.snapshot.parent() { - Some(parent) if parent == payload => true, - Some(parent) => resolve(parent)? == canonical, - None => false, - }; - if !beside { - return Err(Fault::new( - Evidence::HostFailure, - Error::other(format!( - "the control socket {} of the clone's VMM, which its lease reserves, is not \ - beside the snapshot {} of its plan, where OpenVMM requires it", - socket.display(), - self.snapshot.display() - )), - )); - } - let problem = platform::check_store_dir(payload).map_err(|error| { - Fault::host( - Error::io(format!( - "cannot open the payload of the generation {}", - payload.display() - )) - .with_source(error), - ) - })?; - if let Some(problem) = problem { - return Err(Fault::new( - Evidence::MalformedGeneration, - Error::io(format!( - "the payload of the generation {} {problem}", - payload.display() - )), - )); - } - let resolved = canonical.join(name); - for path in [socket, &resolved, ready] { - if path.as_os_str().len() >= SOCKET_PATH_LIMIT { - return Err(Fault::new( - Evidence::EndpointBind, - Error::unsupported(format!( - "the path of the socket {} of a clone's launch must be shorter than \ - {SOCKET_PATH_LIMIT} bytes, as a Unix socket's address requires", - path.display() - )), - )); - } - } - for path in [socket, &resolved, ready] { - if path.to_str().is_none() { - return Err(Fault::new( - Evidence::EndpointBind, - Error::unsupported(format!( - "the path of the socket {} of a clone's launch must be valid UTF-8, as \ - OpenVMM takes it", - path.display() - )), - )); - } - } - if socket.to_string_lossy().contains([',', '=']) { - return Err(Fault::new( - Evidence::EndpointBind, - Error::unsupported(format!( - "the path of the control socket {} of a clone's VMM must hold no ',' or '=', \ - which end the path in OpenVMM's option `listen=`", - socket.display() - )), - )); - } - Ok(()) - } -} - -/// Returns OpenVMM's arguments of the devices of a clone's restore, `devices`: those that -/// [`SandboxPlan::openvmm_arguments`] renders, with `--restore-network-policy` in place of `--net` -/// and the guest's address, which OpenVMM takes from the snapshot, and refuses beside it, so that -/// the policy of the clone's network replaces the template's complete deny-all policy, then -/// `--restore-absent-scratch` for an upper in RAM, whose plan renders no `scratch` block device. -fn device_arguments(devices: &SandboxPlan) -> Result> { - let mut arguments = devices.openvmm_arguments()?; - if let Some(at) = arguments.iter().position(|argument| argument == "--net") { - arguments.splice( - at..(at + 2).min(arguments.len()), - ["--restore-network-policy".to_owned()], - ); - } - if devices.upper().scratch_path().is_none() { - arguments.push("--restore-absent-scratch".to_owned()); - } - Ok(arguments) -} - -/// What OpenVMM's arguments of a clone's restore name besides its control endpoint. -struct Restore<'a> { - /// The arguments of the clone's devices, as [`device_arguments`] renders them. - devices: Vec, - /// The generation's key. - key: &'a TemplateKey, - /// The directory of the snapshot that the VMM restores. - snapshot: PathBuf, - /// The processor target. - processors: u32, - /// The memory target, in bytes. - memory: u64, - /// The endpoint of the restore-ready event. - ready: &'a str, - /// The path of the VMM's outcome report. - report: PathBuf, -} - -impl Restore<'_> { - /// Returns OpenVMM's arguments of the restore, whose control console listens on `endpoint`, - /// as [`ClonePlan::launch`] describes them. - fn arguments(&self, endpoint: &str) -> Vec { - let key = self.key; - let profile = key.cpu_profile(); - let capacity = key.processors().to_string(); - let processors = self.processors.to_string(); - let memory = format!("{}M", self.memory / MIB); - let listen = format!("listen={endpoint}"); - let mut options = vec![ - OsStr::new("--single-process"), - OsStr::new("--hypervisor"), - OsStr::new(key.hypervisor().as_str()), - OsStr::new("--cpu-profile"), - OsStr::new(&profile), - OsStr::new("--processors"), - OsStr::new(&capacity), - OsStr::new("--restore-snapshot"), - self.snapshot.as_os_str(), - ]; - // A template that records no processors online at boot has all of them online, which - // OpenVMM restores without the option, and refuses with it. - if key.boot_processors() != 0 { - options.extend([OsStr::new("--restore-processors"), OsStr::new(&processors)]); - } - options.extend([ - OsStr::new("--restore-memory"), - OsStr::new(&memory), - OsStr::new("--restore-ready-path"), - OsStr::new(self.ready), - OsStr::new("--virtio-console"), - OsStr::new("none"), - OsStr::new("--microvm-control-console"), - OsStr::new(&listen), - OsStr::new("--microvm-control-auth-stdin"), - OsStr::new("--microvm-report"), - self.report.as_os_str(), - ]); - let mut arguments: Vec = self.devices.iter().map(OsString::from).collect(); - arguments.extend(options.into_iter().map(OsStr::to_owned)); - arguments - } -} - -/// The launch of a clone whose VMM restores the leased generation's snapshot, as -/// [`ClonePlan::launch`] returns it: the [`Starting`] launch of the VMM, whose journal records -/// the VMM's identity, the endpoint at which the launch awaits the VMM's restore-ready event, the -/// clone's activation record and capability, and its image, open for reading. Its restore -/// sequence, [`activate`](Self::activate), then [`Activated::start`](super::Activated::start), -/// takes the clone to its sandbox's start. -/// -/// It borrows the sandbox's locked store exclusively, as [`Starting`] does. Dropping it abandons -/// the launch, as [`abandon`](Self::abandon) does, ignoring failures, and stops listening for the -/// restore-ready event; a crash of the caller leaves the launch's journal instead, for -/// [`LockedStore::recover`]. -pub struct Restoring<'s> { - /// The launch, whose journal records the VMM. - pub(super) starting: Starting<'s>, - /// When the launch began, at which the restore-to-Ready deadline starts. - pub(super) launched: Instant, - /// The endpoint of the restore-ready event, as OpenVMM's `--restore-ready-path` takes it. - pub(super) ready: String, - /// The listener of the restore-ready event. - pub(super) listener: ReadyListener, - /// The clone's image, open for reading, which the plan checked. - pub(super) image: CheckedDistro, - /// The clone's activation record. - pub(super) activation: ActivationRecord, - /// The path of the activation record in the sandbox, which the launch wrote. - pub(super) record: PathBuf, - /// The capability of the launch. - pub(super) capability: [u8; CAPABILITY_LEN], - /// How far the clone got, and how its failures are recorded. - pub(super) tracker: Tracker, - /// The clone's mapping table, which its restore sequence sends once the clone is ready: empty - /// for a clone without an export. - pub(super) table: Vec, -} - -impl<'s> Restoring<'s> { - /// Returns the launch, whose VMM runs, and whose journal records its identity. - pub fn starting(&self) -> &Starting<'s> { - &self.starting - } - - /// Returns the endpoint at which the launch awaits the restore-ready event of the VMM, as - /// OpenVMM's `--restore-ready-path` takes it: the path of a Unix socket on Linux, and the name - /// of a named pipe, `//./pipe/NAME`, on Windows. - pub fn ready_endpoint(&self) -> &str { - &self.ready - } - - /// Returns the clone's activation record, as the sandbox holds it. - pub fn activation(&self) -> &ActivationRecord { - &self.activation - } - - /// Returns the capability of the launch, with which the control console of the VMM - /// authenticates the caller. - pub fn capability(&self) -> &[u8; CAPABILITY_LEN] { - &self.capability - } - - /// Returns the clone's image, open for reading, which the plan checked. - pub fn image(&self) -> &CheckedDistro { - &self.image - } - - /// Ends the launch, as [`Starting::abandon`] does, which terminates its VMM and ends the - /// launch once nothing of it runs, and stops listening for the restore-ready event. - pub fn abandon(self) -> Result { - self.starting.abandon() - } -} - -impl fmt::Debug for Restoring<'_> { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - // The capability authenticates control sessions, so it is never shown. - formatter - .debug_struct("Restoring") - .field("starting", &self.starting) - .field("ready", &self.ready) - .field("listener", &self.listener) - .field("image", &self.image) - .field("activation", &self.activation) - .finish_non_exhaustive() - } -} - -#[cfg(test)] -pub(crate) mod tests; diff --git a/nvx-rs/src/host/template/clone/launch/tests.rs b/nvx-rs/src/host/template/clone/launch/tests.rs deleted file mode 100644 index 7ebb38054..000000000 --- a/nvx-rs/src/host/template/clone/launch/tests.rs +++ /dev/null @@ -1,8 +0,0 @@ -use super::*; - -impl<'s> Restoring<'s> { - /// Returns the launch, which a test commits as a clone's restore does once its VMM runs. - pub(crate) fn into_starting(self) -> Starting<'s> { - self.starting - } -} diff --git a/nvx-rs/src/host/template/clone/reattach.rs b/nvx-rs/src/host/template/clone/reattach.rs deleted file mode 100644 index 61aa70aeb..000000000 --- a/nvx-rs/src/host/template/clone/reattach.rs +++ /dev/null @@ -1,635 +0,0 @@ -//! The reattach of a clone, [`reattach_clone`], once the caller of its launch restarted: it finds -//! the clone's VM by the identity of its VMM's process, takes it over if this host may manage it, -//! and checks the clone's activation in a new session, or tells how the clone ended, from the -//! records of its sandbox. - -use std::path::Path; -use std::time::{Duration, Instant}; - -use super::failure::{ - Attribute, CloneFailure, Evidence, Fault, Progress, Stage, Teardown, amended, describe, -}; -use super::records::{Recorder, progress_of, read_failure_of, read_progress}; -use super::restore::{ActivationSession, CloneControl}; -use crate::activation::{ActivationRecord, MAX_RECORD_BYTES, ReadyActivation}; -use crate::error::{Error, ErrorCode, Result}; -use crate::host::launch::timeout_problem; -use crate::host::process::BootId; -use crate::host::report::Report; -use crate::host::store::files::sync_dir; -use crate::host::store::regular::{open_regular, read_at_most}; -use crate::host::store::{IdentityPart, LockedStore, Recovery, RuntimeIdentity, RuntimeRecord}; -use crate::host::supervisor::{self, Reattached, StopOutcome, Vmm}; -use crate::host::template::Lease; - -/// What [`reattach_clone`] found of a clone. -#[derive(Debug)] -#[non_exhaustive] -pub enum ReattachedClone { - /// The clone's VMM runs under the caller's runtime identity, and the clone answered the - /// Activate of its record, byte for byte, in a new session of the adapter's framing, with the - /// Ready of the record's activation: the caller holds the VM, and the session, in which it - /// drives the clone. - Live { - /// The clone's VM. - vmm: Box, - /// The session in which the clone answered. - session: S, - }, - /// No VM of the clone runs, and the sandbox records no failure of it: its VM exited once its - /// launch committed, with an outcome report of the guest's own end whose status is no evidence - /// of a failure, or its holder stopped it, or its launch ended, or never began, without a - /// failure, as its progress tells. The caller releases the clone's lease. - Ended { - /// How far the clone got. - progress: Progress, - /// OpenVMM's outcome report of the clone's VM, if its VMM wrote one. - report: Option, - }, - /// The clone failed, as the record of its failure in its sandbox tells, which the reattach - /// found, or wrote: the launch of a caller that crashed, which the reattach ended, or which a - /// recovery of the sandbox ended before, whose VM ended with a status of the guest's that is - /// evidence; a VM that exited once its launch committed, with a status of the guest's that is - /// evidence, or without a report of the guest's own end; a check of the clone that failed; a - /// VM that runs under another runtime identity; or a record of a failure of the clone that - /// cannot be read, which the record that the reattach wrote replaces. The failure's teardown - /// tells whether a VM of the clone may still run: until it is resolved, the sandbox keeps the - /// store's records of the VM, and with them the clone's lease, for a later reattach. - Failed(CloneFailure), - /// Whether the clone's VMM, whose launch committed, runs cannot be determined, for the reason - /// that this gives, and the sandbox records no failure of the clone: the runtime record - /// remains, and with it the clone's lease, for a later reattach, which may succeed. - Indeterminate(String), -} - -/// Reattaches to the clone of the sandbox of `locked`, under the runtime identity `runtime`, -/// which the caller would launch the clone's template with now, through the adapter's framing, -/// `control`, which must answer in a new session by `deadline`, and ends a VMM of the clone whose -/// launch committed, and that it may not keep, within `bound`. -/// -/// It reads the records of the clone that the sandbox keeps, as [`CloneRecords`] describes, then -/// reattaches to the sandbox's VM, as [`supervisor::reattach`] does, which ends a launch that a -/// crash of its caller interrupted, through the verified identity of its VMM, as -/// [`LockedStore::recover`] ends it, within the wait that the store's recovery allows the VMM's -/// termination, which `bound` does not shorten, removes a journal that the runtime record of its -/// launch supersedes, as the launch's commit may leave it, which is no failure, and finds a VMM -/// whose launch committed by the identity of its process, which says which VM it is: -/// -/// - A VMM that runs under `runtime`, whose launch is the one that the clone's lease reserves, is -/// the clone's, which the reattach checks in a new session, as the restore sequence of the -/// clone's launch does, [`Restoring::activate`](super::Restoring::activate), sending the -/// sandbox's activation record, byte for byte, which must still be the canonical encoding of the -/// record of a restore: the clone answers with the Ready that it answered once, and the reattach -/// checks the transaction ID that it reports, which [`ReattachedClone::Live`] returns with the -/// session. The clone's launch committed once the clone applied its mapping table, if it has one, -/// and started its sandbox, so the reattach sends neither again, and no launch that it ends gets -/// either. A check that fails, by the evidence that the restore sequence gives it, ends the VMM -/// through its verified identity, as [`Vmm::terminate`] does within `bound`, and the clone fails -/// at [`Stage::Reattach`]. -/// - A VMM that runs under another runtime identity, which might not speak the caller's -/// protocol, is ended the same way, without a word to its guest, and the clone fails at -/// [`Stage::Reattach`] with [`Evidence::RuntimeMismatch`], and -/// [`ErrorCode::RuntimeMismatch`]. -/// - A launch that the reattach ended fails at the stage of its progress, with -/// [`ErrorCode::Exited`], and with [`Evidence::ForcedKill`] for a VMM that the reattach -/// terminated, and [`Evidence::LostStatus`] otherwise, which the error that OpenVMM reported, for -/// a VMM that exited before its restore-ready event, attributes as the restore sequence does. One -/// that it cannot end yet fails the same way with [`Evidence::LostStatus`], whose teardown is -/// unresolved: its journal remains, and the lease with it, for a later reattach. -/// - A launch that a recovery of the sandbox ended before the reattach, as [`supervisor::reattach`] -/// ends one, and of which the store holds no record, fails at the stage of its progress, with -/// [`ErrorCode::Exited`], if OpenVMM's outcome report gives a status of the guest's that is -/// evidence, and is [`ReattachedClone::Ended`] otherwise. Once the launch began, which removes -/// the report of an earlier VM of the sandbox before it creates the clone's progress log, a -/// report that the sandbox holds is the clone's VM's, even if no record kept the spawn of its -/// VMM, as a crash of the launch's caller before the journal or the progress log recorded it -/// leaves it. -/// - A VMM whose launch committed, and that exited, fails at [`Stage::Run`] with -/// [`ErrorCode::Exited`] if OpenVMM's outcome report gives a status of the guest's that is -/// evidence, as a status of the clone's repair is at any point, and with -/// [`Evidence::LostStatus`] if no report of the guest's own end tells why it exited, and -/// [`ReattachedClone::Ended`] otherwise. One that cannot be examined is -/// [`ReattachedClone::Indeterminate`]. A VM that its holder's stop ended, which cleared its -/// runtime record, lost no status, but one whose VMM had exited before the stop began is one -/// that exited, as above, since [`Vmm::stop_unless_exited`] keeps its record. -/// -/// The evidence of a status of the guest's prevails over any but the template's, as in the restore -/// sequence, and as [the attribution of evidence](Evidence#attribution) describes. A failure is -/// recorded in the sandbox, as the restore sequence records it: before the reattach ends a VMM of -/// the clone, recovers an interrupted launch, or clears the runtime record of a VM that exited, -/// with its teardown [pending](Teardown::Pending), and again once the teardown ended. A reattach -/// that cannot record it then fails, before it ends or forgets anything of the clone's VM, as does -/// one that cannot make the record of an earlier failure durable. A clone -/// whose sandbox records a failure already failed for good: the reattach ends a VMM of the clone -/// that still runs, through its verified identity, then finishes a failure whose teardown a crash -/// of its caller left pending, as the failure would have recorded it once its teardown ended, or -/// records the teardown of one whose teardown was unresolved in place of the record's, which -/// keeps the stage, the evidence, and the original outcome, unless what the clone's VMM left as -/// it ended since tells more, as the guest's status, which the record does not hold, and the -/// error that OpenVMM reported, for a VMM that exited before its restore-ready event, may, and -/// returns the failure, with the record's lease, transaction, stage, progress, and code. A record -/// of a failure that cannot be read tells as much, though not how the clone failed, and so does -/// a record of another clone's failure, whose template store, lease, leased generation, its key, -/// or launch is not the binding's, and which tells nothing of this clone: if the store holds -/// records of the clone's launch, which the reattach may remove, the reattach first replaces it -/// with the record of its own failure at [`Stage::Reattach`], with the error of the record's read -/// and the host's evidence, its teardown pending, which keeps how far the clone got for a -/// reattach after a crash of the caller. It then ends a VMM of the clone that still runs the same -/// way, and records the failure with the teardown that it found or made, as any failure that it -/// records, over whose evidence a status of the guest's prevails. The failure that it returns is -/// then this clone's, so a [quarantine](CloneFailure::quarantine) of it never reaches a -/// generation that the clone does not lease. -/// -/// The reattach reads no record of the template store: the lease holds the clone's generation, -/// and the sandbox's activation record the clone's transaction, so a tombstone, or a retirement, -/// of the generation leaves the clone's reattach, as its run, as it is. It signals no process that -/// it has not verified. The progress log only informs it: one that cannot be read is none. A -/// clone that launched, whose progress log, which its launch created before it began, and appends -/// to without a sync, cannot be read, is missing, or was written during another boot of the host, -/// whose crash may have lost its last lines, may have run its workload: a status of the guest's -/// is then attributed as once the start of the clone's sandbox was sent. -/// -/// Fails with [`ErrorKind::Other`](crate::ErrorKind::Other) if the sandbox holds no clone -/// binding, whose VM reattaches as any sandbox's does, with -/// [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) if `bound` is zero or longer than -/// [`MAX_TIMEOUT`](crate::host::launch::MAX_TIMEOUT), and for an invalid `runtime`, before it -/// writes anything, and as [`Lease::of`] and [`supervisor::reattach`] fail for records that are -/// inconsistent, which leave the clone's VM as it was, and the record of the failure of an -/// interrupted launch, or the one that replaced a record that could not be read, if the reattach -/// wrote it, pending. Fails with [`ErrorKind::Io`](crate::ErrorKind::Io) if it cannot record the -/// clone's failure, or make the record of an earlier one durable, before it would end, or forget, -/// anything of the clone's VM, which it then leaves as it found it, running or not, with the -/// journal of its launch or its runtime record, and the lease, for a later reattach. -/// -/// [`CloneRecords`]: super::CloneRecords -pub fn reattach_clone( - locked: &mut LockedStore, - runtime: &RuntimeIdentity, - control: &mut C, - deadline: Instant, - bound: Duration, -) -> Result> { - reattach_clone_with( - locked, - runtime, - control, - deadline, - bound, - |locked, runtime, exited| supervisor::reattach_reporting(locked, runtime, exited), - ) -} - -/// Reattaches to a clone as [`reattach_clone`] does, reattaching to its VM with `reattach`, which -/// is [`supervisor::reattach_reporting`] outside tests, and which calls its last argument with -/// the runtime record of a VMM that exited before it clears the record, and fails, keeping the -/// record, if that fails. -pub(super) fn reattach_clone_with( - locked: &mut LockedStore, - runtime: &RuntimeIdentity, - control: &mut C, - deadline: Instant, - bound: Duration, - reattach: impl FnOnce( - &mut LockedStore, - &RuntimeIdentity, - &dyn Fn(&RuntimeRecord) -> Result<()>, - ) -> Result, -) -> Result> { - reattach_clone_syncing( - locked, runtime, control, deadline, bound, &sync_dir, reattach, - ) -} - -/// Reattaches to a clone as [`reattach_clone_with`] does, syncing the sandbox's directory, to make -/// the record of an earlier failure durable, with `sync`, which is [`sync_dir`] outside tests. -pub(super) fn reattach_clone_syncing( - locked: &mut LockedStore, - runtime: &RuntimeIdentity, - control: &mut C, - deadline: Instant, - bound: Duration, - sync: &dyn Fn(&Path) -> Result<()>, - reattach: impl FnOnce( - &mut LockedStore, - &RuntimeIdentity, - &dyn Fn(&RuntimeRecord) -> Result<()>, - ) -> Result, -) -> Result> { - let started = Instant::now(); - if let Some(problem) = timeout_problem(bound, true) { - return Err(Error::unsupported(format!( - "the bound of a clone's termination {problem}" - ))); - } - // The reattach fails for an invalid runtime identity before it writes anything. - runtime.check()?; - let Some(lease) = Lease::of(locked)? else { - return Err(Error::other(format!( - "the sandbox in {} holds no clone binding, and so no clone to reattach to", - locked.dir().display() - ))); - }; - let launch = lease.launch(); - let journal = locked - .journal()? - .filter(|journal| journal.launch() == launch); - let runtime_record = locked.runtime()?; - let committed = runtime_record - .as_ref() - .filter(|record| record.launch() == launch); - // The record of an earlier failure of the clone, or the fault of a record that cannot be read, - // or of one of another clone's failure, which still tells that the clone launched, and failed - // for good, though not how. - let earlier = read_failure_of(&locked.clone_failure_path(), &lease) - .map_err(unreadable_failure) - .transpose(); - let logged = read_progress(&locked.clone_progress_path(), launch).unwrap_or(None); - // The launch of a clone creates its progress log before it begins, and before it records a - // failure, and appends to it without a sync, so the log of a clone that launched tells how - // far the clone got for sure only if it can be read, and was written during the current boot - // of the host, whose crash would have ended the clone's VM: a crash of the host may have lost - // the log's last lines, and a workload may then have run in the clone. A log that can be read - // tells that the clone launched as well as the store's records do, which a stop of its VM - // clears, and a release alone removes it. - let launched = - journal.is_some() || committed.is_some() || earlier.is_some() || logged.is_some(); - let certain = logged.is_some_and(|logged| logged.is_certain(BootId::current().ok())); - let uncertain = launched && !certain; - let logged = logged.map(|logged| logged.progress); - let progress = progress_of(&lease, journal.as_ref(), committed, logged); - let activation = read_activation(&locked.clone_activation_path()); - // A failure that the reattach records has the transaction that the clone's activation record - // names, while the record of an earlier failure keeps its own. - let transaction = activation - .as_ref() - .ok() - .and_then(|(_, record)| record.restore().map(|restore| restore.transaction)); - let recorder = Recorder::new(lease.clone(), transaction, locked).uncertain(uncertain); - let fresh = earlier.is_none(); - // A record of an earlier failure may be in place without being durable, as a write whose sync - // of the sandbox's directory failed leaves it: the reattach makes it durable before it ends, - // or forgets, anything that tells that the clone launched, as it records a failure of its own. - if let Some(Ok(_)) = &earlier - && (journal.is_some() || committed.is_some()) - { - sync(locked.dir()).map_err(|error| { - unrecorded(Error::io(format!( - "the record of the clone's failure could not be made durable before its \ - teardown: {}", - describe(&error) - ))) - })?; - } - // A record that cannot be read, or that is another clone's, is replaced by the reattach's own - // failure, with its teardown pending, before the reattach ends or forgets anything of the - // clone's launch: the record keeps how far the clone got, which the launch's journal and - // runtime record hold, and the reattach removes, for a reattach after a crash of this caller, - // which finishes it. Without them, the reattach removes nothing that the record would keep, and - // a later reattach finds the clone as this one does. - if let Some(Err(fault)) = &earlier - && (journal.is_some() || committed.is_some()) - { - recorder - .pending( - Stage::Reattach, - progress, - (fault.evidence, None), - &fault.error, - ) - .map_err(unrecorded)?; - } - // A launch that the journal names, which no runtime record supersedes, and whose failure the - // sandbox does not record, was interrupted by a crash of its caller: its failure is recorded - // before the recovery ends it, so that a crash of this caller during the recovery leaves the - // record. A journal that the runtime record of its launch supersedes, which the launch's - // commit may leave, is no failure: the recovery removes it. - if fresh && journal.is_some() && runtime_record.is_none() { - recorder - .pending( - progress.stage(), - progress, - (Evidence::LostStatus, None), - &interrupted(progress), - ) - .map_err(unrecorded)?; - } - // So is the failure of a VM whose launch committed, and which exited, before the reattach - // clears its runtime record, as the outcome report that the recorder reads then tells, which - // the rest of the reattach takes as it found it. - let exiting = |_: &RuntimeRecord| { - if fresh && let Some((fault, status)) = exit_fault(&recorder, progress, true) { - let evidence = (fault.evidence, status); - return recorder - .pending(progress.stage(), progress, evidence, &fault.error) - .map_err(unrecorded); - } - Ok(()) - }; - let reattached = reattach(locked, runtime, &exiting)?; - let failed = |fault: Fault, stage: Stage, teardown: Teardown| { - let fault = Fault::new(fault.evidence, unended(fault.error, &teardown)); - Ok(ReattachedClone::Failed( - recorder.record(stage, progress, fault, teardown), - )) - }; - if let Some(earlier) = earlier { - let teardown = match reattached { - Reattached::Live(vmm) | Reattached::Mismatched { vmm, .. } => { - stopped(vmm.terminate(locked, bound), bound) - } - Reattached::Exited { - recovery: recovery @ (Recovery::Exited | Recovery::Terminated(_)), - .. - } => Teardown::of(Ok(recovery)), - Reattached::Exited { - record: Some(_), .. - } => Teardown::Exited, - // Nothing of the clone remains: a teardown that was not resolved is now. - Reattached::Exited { record: None, .. } => match &earlier { - Ok(earlier) => match earlier.teardown { - Teardown::Unresolved(_) | Teardown::Pending => Teardown::Exited, - ref teardown => teardown.clone(), - }, - // A VM of the clone ran once its launch spawned it, and so did one whose report the - // sandbox holds once the launch began, as in `exit_fault`, though no record kept - // its spawn. - Err(_) - if progress >= Progress::Spawned - || (progress > Progress::Planned && recorder.observed().is_some()) => - { - Teardown::Exited - } - Err(_) => Teardown::NoVm, - }, - Reattached::Indeterminate(reason) => Teardown::Unresolved(reason), - }; - let earlier = match earlier { - Ok(earlier) => earlier, - // The record cannot tell how the clone failed: the failure is the reattach's own. - Err(fault) => return failed(fault, Stage::Reattach, teardown), - }; - let failure = if earlier.teardown == Teardown::Pending { - // The crash of the caller during the teardown left the failure pending, which is - // recorded as the failure would have recorded it once its teardown ended. - let error = unended(earlier.error(), &teardown); - recorder.finish(earlier, error, teardown) - } else { - recorder.resolve(earlier, teardown) - }; - return Ok(ReattachedClone::Failed(failure)); - } - match reattached { - Reattached::Live(vmm) => { - let checked = activation.and_then(|activation| { - check_activation(locked, &lease, &vmm, control, activation, deadline, started) - }); - match checked { - Ok(session) => Ok(ReattachedClone::Live { - vmm: Box::new(vmm), - session, - }), - Err(fault) => { - let evidence = (fault.evidence, None); - recorder - .pending(Stage::Reattach, progress, evidence, &fault.error) - .map_err(unrecorded)?; - let teardown = stopped(vmm.terminate(locked, bound), bound); - failed(fault, Stage::Reattach, teardown) - } - } - } - Reattached::Mismatched { vmm, differences } => { - let fault = Fault::new(Evidence::RuntimeMismatch, mismatched(&differences)); - let evidence = (fault.evidence, None); - recorder - .pending(Stage::Reattach, progress, evidence, &fault.error) - .map_err(unrecorded)?; - let teardown = stopped(vmm.terminate(locked, bound), bound); - failed(fault, Stage::Reattach, teardown) - } - Reattached::Exited { - recovery: recovery @ (Recovery::Exited | Recovery::Terminated(_)), - .. - } => { - let evidence = match recovery { - Recovery::Terminated(_) => Evidence::ForcedKill, - _ => Evidence::LostStatus, - }; - let fault = Fault::new(evidence, interrupted(progress)); - failed(fault, progress.stage(), Teardown::of(Ok(recovery))) - } - Reattached::Exited { record, .. } => { - match exit_fault(&recorder, progress, record.is_some()) { - Some((fault, _)) => failed(fault, progress.stage(), Teardown::Exited), - // A report is of the clone's VM only once its launch began, as in `exit_fault`. - None => Ok(ReattachedClone::Ended { - progress, - report: (progress > Progress::Planned) - .then(|| recorder.observed().cloned()) - .flatten(), - }), - } - } - Reattached::Indeterminate(reason) => { - if locked.journal()?.is_none() { - return Ok(ReattachedClone::Indeterminate(reason)); - } - // The launch that a crash of its caller interrupted cannot be ended yet. - let fault = Fault::new(Evidence::LostStatus, interrupted(progress)); - failed(fault, progress.stage(), Teardown::Unresolved(reason)) - } - } -} - -/// Returns the fault of the clone whose failures `recorder` records, which got as far as -/// `progress`, and whose VM exited, as OpenVMM's outcome report, which -/// [`Recorder::observed`] reads, tells, with the guest's status, or `None` if the VM ended -/// without a failure: a status of the guest's that is evidence, as [`Recorder::reported`] gives -/// it, or, for a VM that exited `unheld`, whose runtime record the reattach found, as -/// [`Vmm::stop_unless_exited`] keeps that of a VMM that had exited before the stop began, the loss -/// of the clone's status, if no report of the guest's own end tells why. -fn exit_fault( - recorder: &Recorder, - progress: Progress, - unheld: bool, -) -> Option<(Fault, Option)> { - // A report is of the clone's VM only once its launch began, which removes the report of an - // earlier VM of the sandbox before it creates the clone's progress log, as - // `CloneRecords::read` takes it: a VMM that the launch spawned wrote it, though no record may - // keep the spawn, if the launch's caller crashed before the journal or the progress log - // recorded it, and a recovery of the sandbox then removed the journal. - if progress <= Progress::Planned { - return None; - } - let (evidence, status) = recorder.reported(recorder.observed(), progress); - let evidence = match evidence { - Some(evidence) => evidence, - // Only a VM that exited unheld, whose runtime record the reattach found, ends without - // telling why: its holder's stop ends one otherwise, and clears the record, but keeps that - // of one that had exited before the stop began, as `Vmm::stop_unless_exited` does. - None if unheld && status.is_none() => Evidence::LostStatus, - None => return None, - }; - Some((Fault::new(evidence, exited(progress, status)), status)) -} - -/// Reads the clone's activation record at `path`, which must be the canonical encoding of the -/// record of a restore, as the clone's launch wrote it, and returns its bytes and the record, or -/// the fault, the host's, of one that is not. -/// -/// It compares nothing of the record with the clone's lease: the reattach matches the clone's -/// records with the lease as [`CloneRecords`](super::CloneRecords) does, through -/// [`read_failure_of`], which refuses the record of the failure of another template's clone. -fn read_activation(path: &Path) -> std::result::Result<(Vec, ActivationRecord), Fault> { - let bytes = open_regular(path) - .and_then(|file| read_at_most(file, MAX_RECORD_BYTES as u64, "an activation record")) - .map_err(|error| { - Fault::host( - Error::io(format!( - "cannot read the clone's activation record {}", - path.display() - )) - .with_source(error), - ) - })?; - let malformed = |problem: &str| { - Error::io(format!( - "the clone's activation record {} {problem}", - path.display() - )) - }; - let record = ActivationRecord::decode(&bytes) - .map_err(|error| malformed("is malformed").with_source(error)) - .attribute(Evidence::HostFailure)?; - if record.encode() != bytes { - return Err(Fault::new( - Evidence::HostFailure, - malformed("is not in its canonical encoding, in which the clone's launch wrote it"), - )); - } - if record.restore().is_none() { - return Err(Fault::new( - Evidence::HostFailure, - malformed("is not the record of a restore"), - )); - } - Ok((bytes, record)) -} - -/// Checks the clone of `lease`, whose VM `vmm` runs in the sandbox of `locked`, in a new session -/// of `control`, with the sandbox's capability and its activation record, `activation`, by -/// `deadline`, as the reattach that `started` describes, and returns the session in which the -/// clone answered. -fn check_activation( - locked: &LockedStore, - lease: &Lease, - vmm: &Vmm, - control: &mut C, - (encoded, record): (Vec, ActivationRecord), - deadline: Instant, - started: Instant, -) -> std::result::Result { - let launch = vmm.record().launch(); - if launch != lease.launch() { - return Err(Fault::new( - Evidence::HostFailure, - Error::io(format!( - "the VM of the sandbox in {} is of the launch {launch}, not of the launch {} that \ - the clone's lease reserves", - locked.dir().display(), - lease.launch() - )), - )); - } - let capability = locked.read_capability().on_host()?; - let path = locked.clone_activation_path(); - let session = ActivationSession { - capability: &capability, - record: &path, - expected: ReadyActivation::of(&record), - encoded, - deadline, - timeout: deadline.saturating_duration_since(started), - since: "its reattach", - }; - session.activate(control, vmm.endpoint(), vmm.process().reference(), None) -} - -/// Returns the teardown of a termination of a clone's VMM within `bound` that ended as `outcome` -/// tells, as [`Vmm::terminate`] returns it. -fn stopped(outcome: Result, bound: Duration) -> Teardown { - match outcome { - Ok(StopOutcome::Forced) => Teardown::Terminated, - Ok(StopOutcome::Graceful) => Teardown::Exited, - Ok(StopOutcome::TimedOut) => Teardown::Unresolved(format!( - "the clone's VMM did not exit within {bound:?} of its termination" - )), - Err(error) => Teardown::Unresolved(describe(&error)), - } -} - -/// Returns `error`, which says so if `teardown` is unresolved. -fn unended(error: Error, teardown: &Teardown) -> Error { - match teardown { - Teardown::Unresolved(reason) => amended( - error, - &format!("; the clone's VMM could not be ended, and remains for a reattach: {reason}"), - ), - _ => error, - } -} - -/// Returns the error of a reattach that could not record the clone's failure, as `error` tells, -/// before it would end, or forget, anything of the clone's VM, which it leaves as it found it. -fn unrecorded(error: Error) -> Error { - amended( - error, - "; the reattach left the clone's VM and its records as it found them, for a later reattach", - ) -} - -/// Returns the error of the launch of a clone that got as far as `progress` before a crash of its -/// caller interrupted it. -fn interrupted(progress: Progress) -> Error { - Error::exited(format!( - "the launch of the clone was interrupted at its step {progress}, as a crash of its \ - caller interrupts it" - )) -} - -/// Returns the error of a clone's VM that exited once the clone got as far as `progress`, with -/// the guest's `status`, if OpenVMM's outcome report gives one. -fn exited(progress: Progress, status: Option) -> Error { - let ending = match status { - Some(status) => format!("with the guest's status {status}"), - None => "without an outcome report of the guest's own end".to_owned(), - }; - Error::exited(format!( - "the clone's VM exited at its step {progress}, {ending}" - )) -} - -/// Returns the error of a clone's VM that runs under another runtime identity, which differs in -/// `differences`. -fn mismatched(differences: &[IdentityPart]) -> Error { - let parts: Vec<&str> = differences.iter().map(|part| part.name()).collect(); - Error::new( - ErrorCode::RuntimeMismatch, - format!( - "the clone's VM runs under another runtime identity, which differs in {}, so it is \ - not driven, and is ended through its verified identity", - parts.join(", ") - ), - ) -} - -/// Returns the fault of a clone whose sandbox holds a record of a failure that cannot be read, or -/// that is another clone's, as `error` tells: the record still shows that the clone failed, so -/// the reattach keeps none of it, with the host's evidence. -fn unreadable_failure(error: Error) -> Fault { - Fault::host(amended( - error, - "; a clone whose sandbox records a failure failed for good, so the reattach keeps no VM \ - of it", - )) -} diff --git a/nvx-rs/src/host/template/clone/record.rs b/nvx-rs/src/host/template/clone/record.rs deleted file mode 100644 index b768b5eae..000000000 --- a/nvx-rs/src/host/template/clone/record.rs +++ /dev/null @@ -1,128 +0,0 @@ -//! The activation record of a clone, which its [plan](super::ClonePlan) builds before the clone's -//! launch from the request and the checked generation: the record of the settings that the -//! clone's guest applies in place of the configuration that its template captured, as -//! [`nvx::activation`](crate::activation) describes it. - -use super::CloneRequest; -use super::failure::{Attribute, Evidence, Fault}; -use crate::activation::{ - ActivationRecord, Configuration, ExpectedResources, Export, ExportKind, Image, RestoreSection, - TransactionId, Upper, -}; -use crate::error::{Error, Result}; -use crate::host::config::UpperConfig; -use crate::host::sandbox::distro::DistroFormat; -use crate::host::template::key::TemplateKey; - -/// Returns the activation record of the clone that `request` asks for, of the template whose key -/// is `key`, with a fresh transaction ID. -/// -/// The record holds: -/// -/// - the restore section: only a transaction ID of 16 random bytes; -/// - the image, in the format that its binding names: an EROFS layer with its filesystem UUID, -/// or a GPT disk with its GUID and the request's bound of its first partition; -/// - the upper: a tmpfs capped at half of the VM's memory, or the scratch with the request's -/// journal policy; -/// - the request's overlay mode and view; -/// - the export, if the request has one: an aggregate, as every export plan is, and the number -/// of its mappings, the plan's binds, from 1 to -/// [`MAX_MAPPINGS`](crate::activation::MAX_MAPPINGS); or no export, and no mappings; -/// - the clone's network, if the template has a NIC, whose policy the restore enforces there in -/// place of the template's complete deny-all policy: the DNS server that OpenVMM serves the -/// guest at a gateway under the -/// policy, as [`SandboxNetwork::dns_server`](crate::host::sandbox::SandboxNetwork::dns_server) -/// finds it with the IPv6 gateway of the template's -/// NIC, [`TemplateKey::ipv6_gateway`], if it serves one, since the template's guest configured -/// no resolver at boot under the complete deny-all policy; published listeners under a policy -/// that publishes them, [`NetworkPolicy::publishes_listeners`], which allows ingress or -/// host-loopback access both ways, and unpublished ones otherwise; and the URL of the policy's -/// proxy as the guest sees it, if it has one; -/// - the expected resources: the request's processor target, which the restore brings online, and -/// its memory target. -/// -/// Fails with [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for a configuration that -/// no record holds, as [`Configuration::check`] finds it, such as a GPT disk's first-partition -/// bound of 0, a hostname that is empty or longer than -/// [`MAX_HOSTNAME_BYTES`](crate::activation::MAX_HOSTNAME_BYTES), [`Evidence::InvalidActivate`], -/// and for a network whose policy does not lower -/// to OpenVMM's arguments, [`Evidence::InvalidSettings`]; and with -/// [`ErrorKind::Other`](crate::ErrorKind::Other) if the transaction ID cannot be drawn, the -/// host's evidence. -/// -/// [`NetworkPolicy::publishes_listeners`]: crate::host::network::NetworkPolicy::publishes_listeners -pub(super) fn build( - request: &CloneRequest, - key: &TemplateKey, -) -> std::result::Result { - let expected = request.adapter.distro().image().identity; - let image = match request.binding.format() { - DistroFormat::Erofs => Image::Erofs { uuid: expected }, - DistroFormat::Gpt => Image::Gpt { - disk_guid: expected, - first_partition_bound: request.first_partition_bound, - }, - }; - let upper = match &request.scratch { - None => Upper::Tmpfs { - max_memory_percent: UpperConfig::DEFAULT_MAX_MEMORY_PERCENT, - }, - Some(_) => Upper::Scratch { - journal: request.journal, - }, - }; - // The plan's check found that the network attaches a NIC if and only if the template has one. - let network = match request.adapter.network().filter(|_| key.has_network()) { - None => None, - Some(network) => Some( - network.policy().activation_network( - network.gateway(), - network - .dns_server(key.ipv6_gateway()) - .attribute(Evidence::InvalidSettings)?, - ), - ), - }; - let (export, mappings) = match &request.export { - None => (None, 0), - Some(plan) => ( - Some(Export { - kind: ExportKind::Aggregate, - }), - plan.binds.len(), - ), - }; - let configuration = Configuration { - image, - upper, - overlay: request.overlay, - export, - mappings, - network, - view: request.view.clone(), - resources: ExpectedResources { - cpus: request.processors, - memory: request.memory, - }, - }; - let restore = RestoreSection { - transaction: transaction().on_host()?, - }; - ActivationRecord::new(Some(restore), configuration).map_err(|error| { - Fault::new( - Evidence::InvalidActivate, - Error::unsupported(format!( - "no clone activates with the settings of the request: {error}" - )), - ) - }) -} - -/// Returns a fresh transaction ID: 16 random bytes. -fn transaction() -> Result { - let mut bytes = [0; TransactionId::LEN]; - getrandom::fill(&mut bytes).map_err(|error| { - Error::other("cannot draw the transaction ID of a clone's restore").with_source(error) - })?; - Ok(TransactionId::from_bytes(bytes)) -} diff --git a/nvx-rs/src/host/template/clone/records.rs b/nvx-rs/src/host/template/clone/records.rs deleted file mode 100644 index dc5fbc696..000000000 --- a/nvx-rs/src/host/template/clone/records.rs +++ /dev/null @@ -1,1136 +0,0 @@ -//! The records that a clone keeps in its sandbox about its launch, beside its activation record: -//! its progress log, [`CLONE_PROGRESS_NAME`](crate::host::store::CLONE_PROGRESS_NAME), and the -//! record of its failure, [`CLONE_FAILURE_NAME`](crate::host::store::CLONE_FAILURE_NAME), which -//! [`CloneRecords::read`] reads with the clone's other records. -//! -//! # The progress log -//! -//! The launch of a clone creates its progress log, in place of one that an earlier launch left, -//! with a first line that names its format, the launch, and the boot of the host during which the -//! launch writes it: `nvx-clone-progress 1 `. The launch and its restore sequence -//! then append a line for each step of the clone's [`Progress`] as the clone reaches it, from the -//! spawn of its VMM to the commit of its launch, in order, and sync none of them: the log tells a -//! reattach how far the clone got once the caller of its launch crashed, which a crash of the -//! caller alone loses no write of. A crash of the host, which ends the clone's VM, may lose the -//! log's last lines, so how far the clone got is uncertain, once it launched, by a log that was -//! written during another boot, as by one that is damaged or missing: a workload may have run in -//! the clone. The launch appends the spawn before it returns, although the launch's journal -//! records it too, since a teardown of the launch removes the journal. A step that the next step -//! follows at once, the acknowledgment, which the Activate follows, is appended with that next -//! step, in its one write, so that the log takes a write for the spawn and for each step from the -//! restore-ready event on. A reader ignores a last line that a crash cut short. A sandbox -//! that boots cold writes no log. -//! -//! # The failure record -//! -//! The record of a clone's failure is JSON in format [`FAILURE_FORMAT`], which the failure writes -//! before its teardown, with the teardown [pending](Teardown::Pending), and again once its -//! teardown is known, replacing a record atomically, as the sandbox's store replaces its records, -//! and durably: each write syncs the sandbox's directory once the record is in place, so that the -//! record outlasts a crash of the host, as the clone's binding does. Like the store's records, the -//! record, and the directory that a write syncs, are named by their paths: the directories above -//! the sandbox's must be ones that only the current user, or an administrator, may change, as the -//! [store](crate::host::store) requires of them. The record keeps at most -//! 8192 characters of the failure's message, and 1024 of the reason of an unresolved teardown: of -//! a longer text, as the diagnosis of a caller that fails the clone may be, its first and its last -//! characters, with an ellipsis between them, so that a read, which takes at most 64 KiB of a -//! record, takes every record that a failure writes. A record of another format fails rather -//! than being read. The failure of a VMM that exited before its restore-ready event reads the -//! launch's log back from its end for the line with which OpenVMM reported the error that ended -//! it, which the record's message keeps, and whose time ABI code may attribute the failure, as -//! [the attribution of evidence](Evidence#attribution) describes. -//! -//! A release of the clone's lease removes both, as it removes the clone's other files, so that -//! the records of a clone last as long as its lease, which a failure whose teardown is unresolved -//! keeps. - -use std::cell::OnceCell; -use std::fs::File; -use std::io::{self, Read, Seek, SeekFrom, Write}; -use std::num::NonZeroU64; -use std::path::{Path, PathBuf}; - -use serde::{Deserialize, Serialize}; - -use super::failure::{ - self, CloneFailure, Evidence, FailureRecord, Fault, Outcome, Progress, Stage, Teardown, - amended, describe, -}; -use crate::activation::{Identity, StoreGeneration, TemplateReference, TransactionId}; -use crate::error::{Error, ErrorCode, Result}; -use crate::host::platform::{self, parse_hex_id}; -use crate::host::process::BootId; -use crate::host::report::{self, Report}; -use crate::host::store::files::{create_new, sync_dir, write_private}; -use crate::host::store::regular::{open_regular, read_at_most, was_removed}; -use crate::host::store::{ - CLONE_PROGRESS_NAME, LaunchId, LaunchJournal, OUTCOME_NAME, Recovery, RuntimeRecord, - SandboxStore, -}; -use crate::host::supervisor::Starting; -use crate::host::template::{KeyIdentity, Lease}; - -/// The format of the record of a clone's failure. -pub const FAILURE_FORMAT: u32 = 1; - -/// The first word of the first line of a clone's progress log. -const PROGRESS_MAGIC: &str = "nvx-clone-progress"; - -/// The format of a clone's progress log, after [`PROGRESS_MAGIC`] in its first line. -const PROGRESS_FORMAT: u32 = 1; - -/// The most bytes of a clone's progress log that a read takes: its first line, and a line for each -/// of its steps, take a few hundred. -const MAX_PROGRESS_BYTES: u64 = 4096; - -/// The most bytes of the record of a clone's failure that a read takes: more than any record that -/// a failure writes takes, whose texts [`bounded`] bounds, as [`MAX_MESSAGE_CHARS`] and -/// [`MAX_REASON_CHARS`] tell. -const MAX_FAILURE_BYTES: u64 = 64 * 1024; - -/// The most characters of the message of a failure's error that the record of the failure keeps, -/// as [`bounded`] keeps them: the diagnosis of a caller that fails a clone, as -/// [`Activated::fail`](super::Activated::fail) and [`Started::fail`](super::Started::fail) take -/// it, may be longer. -const MAX_MESSAGE_CHARS: usize = 8 * 1024; - -/// The most characters of the reason of an unresolved teardown that the record of a failure keeps, -/// as [`bounded`] keeps them. -const MAX_REASON_CHARS: usize = 1024; - -// JSON takes at most six bytes for a character, as for `\u0001`, and the other fields of a record -// take less than a kilobyte, so that a read takes every record that a failure writes. -const _: () = - assert!(6 * (MAX_MESSAGE_CHARS + MAX_REASON_CHARS) + 1024 <= MAX_FAILURE_BYTES as usize); - -/// What messages call the progress log of a clone. -const PROGRESS_WHAT: &str = "clone progress log"; - -/// What messages call the record of a clone's failure. -const FAILURE_WHAT: &str = "clone failure record"; - -/// The bytes of a VMM's log that a failure reads at a time as it looks back from the log's end for -/// the line of OpenVMM's fatal error, which OpenVMM writes as it exits, followed by the error's -/// chain alone, of any length: the chain of a failed CPU check names every difference that the -/// check found. -const FATAL_BLOCK_BYTES: u64 = 64 * 1024; - -/// The most characters of the line of OpenVMM's fatal error that the record of a failure keeps. -const MAX_FATAL_CHARS: usize = 1024; - -/// The progress log of a clone's launch, open for writing, with the latest step that the clone -/// reached, and the latest that the log records, as the [module](self#the-progress-log) -/// describes. -#[derive(Debug)] -pub(super) struct ProgressLog { - /// The log, open for writing at its end. - file: File, - /// The log's path. - path: Box, - /// The latest step that the clone reached. - progress: Progress, - /// The latest step that the log records. - logged: Progress, -} - -impl ProgressLog { - /// Creates the progress log of the launch `launch` in the sandbox's directory `dir`, in place - /// of one that an earlier launch left, with its first line, which names the current boot of - /// the host, and returns it, at [`Progress::Launching`]. It first removes the outcome report - /// that an earlier VM of the sandbox left, which no acquisition, clearing of a VM's runtime - /// record, or release removes, so that a report beside the clone's records is the clone's VM's, - /// as [`CloneRecords::read`] takes it. - /// - /// `directory` holds the sandbox's directory open, as the sandbox's lock opened and checked it - /// through `dir`. The earlier report and log go through it, as a release removes the clone's - /// files, so that the removals reach the directory that holds the lock, whatever `dir` names - /// meanwhile. The log is then created at its path, where no file may exist, so that a path that - /// names another sandbox's directory by then fails the creation if that sandbox holds a log, - /// rather than replace it. - pub(super) fn create( - directory: &platform::Directory, - dir: &Path, - launch: &LaunchId, - ) -> Result { - let path = dir.join(CLONE_PROGRESS_NAME); - let failed = |error: io::Error| { - Error::io(format!( - "cannot create the {PROGRESS_WHAT} {}", - path.display() - )) - .with_source(error) - }; - let boot = BootId::current().map_err(failed)?; - for name in [OUTCOME_NAME, CLONE_PROGRESS_NAME] { - directory.remove_file(name).map_err(|error| { - Error::io(format!("cannot remove {}", dir.join(name).display())).with_source(error) - })?; - } - let mut file = create_new(&path).map_err(failed)?; - file.write_all(format!("{PROGRESS_MAGIC} {PROGRESS_FORMAT} {launch} {boot}\n").as_bytes()) - .map_err(failed)?; - Ok(Self { - file, - path: path.into(), - progress: Progress::Launching, - logged: Progress::Launching, - }) - } - - /// Returns the latest step that the clone reached. - pub(super) fn progress(&self) -> Progress { - self.progress - } - - /// Notes that the clone reached `step`, unless it reached it, or a later step, already, - /// without writing it: the log records it with the next step that it writes. It is for a - /// step that another record holds, as the record of a launch's failure holds the spawn of a - /// VMM that the launch could not record, or that the next step follows at once. - pub(super) fn advance(&mut self, step: Progress) { - self.progress = self.progress.max(step); - } - - /// Records that the clone reached `step`, unless it reached it, or a later step, already: - /// appends the line of each step that the clone reached since the latest that the log - /// records, through `step`, in order, with one write, which nothing syncs. - pub(super) fn reach(&mut self, step: Progress) -> Result<()> { - self.advance(step); - if self.progress <= self.logged { - return Ok(()); - } - let lines: String = LOGGED - .iter() - .filter(|&&logged| self.logged < logged && logged <= self.progress) - .map(|logged| format!("{logged}\n")) - .collect(); - self.file.write_all(lines.as_bytes()).map_err(|error| { - Error::io(format!( - "cannot record that the clone reached its step {} in the {PROGRESS_WHAT} {}", - self.progress, - self.path.display() - )) - .with_source(error) - })?; - self.logged = self.progress; - Ok(()) - } -} - -/// The steps that a progress log records after its first line, in order. -const LOGGED: [Progress; 8] = [ - Progress::Spawned, - Progress::Event, - Progress::Acknowledged, - Progress::Activating, - Progress::Ready, - Progress::Starting, - Progress::Started, - Progress::Committed, -]; - -/// How the launch of a clone that failed is ended: [`Starting::abandon`], outside tests, which -/// replace it to keep a launch that cannot be ended. -pub(super) type End = for<'a> fn(Starting<'a>) -> Result; - -/// Ends `starting` as [`Starting::abandon`] does: how the launch of a clone that failed is ended -/// outside tests. -pub(super) fn abandon(starting: Starting<'_>) -> Result { - starting.abandon() -} - -/// What a clone's failures are recorded with: the clone's lease and the transaction of its -/// restore, the paths, in its sandbox, of the record of its failure, of OpenVMM's outcome report -/// of its VM, and of its VMM's log, whether a workload may have run in the clone whatever its -/// records say, as for a clone whose progress log was damaged, or written during another boot, -/// and the report, once it was read. -#[derive(Debug)] -pub(super) struct Recorder { - /// The clone's lease. - lease: Lease, - /// The transaction ID of the clone's restore. - transaction: Option, - /// The path of the record of the clone's failure. - failure: PathBuf, - /// The path of OpenVMM's outcome report of the clone's VM. - report: PathBuf, - /// The path of the log of the clone's VMM. - log: PathBuf, - /// Whether how far the clone got is uncertain beyond the progress that its records give, so - /// that a workload may have run in it. - uncertain: bool, - /// OpenVMM's outcome report of the clone's VM, as [`observed`](Self::observed) first read it. - observed: OnceCell>, -} - -impl Recorder { - /// Returns the recorder of the failures of the clone of `lease`, with the transaction of its - /// restore, `transaction`, in the sandbox of `sandbox`. - pub(super) fn new( - lease: Lease, - transaction: Option, - sandbox: &SandboxStore, - ) -> Self { - Self { - lease, - transaction, - failure: sandbox.clone_failure_path(), - report: sandbox.outcome_path(), - log: sandbox.log_path(), - uncertain: false, - observed: OnceCell::new(), - } - } - - /// Returns the recorder, for a clone whose progress is uncertain beyond what its records - /// give, if `uncertain`: a status of the guest's is then attributed as once a workload may - /// have run, as [`reported`](Self::reported) does. - pub(super) fn uncertain(self, uncertain: bool) -> Self { - Self { uncertain, ..self } - } - - /// Returns OpenVMM's outcome report of the clone's VM, or `None` if it is missing or - /// malformed, as the recorder first read it, the first time that it needed it: once the VM - /// has ended, whose report is final then, so that a read that fails later, as a read that - /// another process's open of the report refuses may, takes nothing back of what the first - /// one showed. A recorder serves one step, a launch that fails or a reattach, which reads the - /// report only once the clone's VM ended, or its teardown did. - pub(super) fn observed(&self) -> Option<&Report> { - self.observed - .get_or_init(|| report::read(&self.report).ok().flatten()) - .as_ref() - } - - /// Returns the guest's status in OpenVMM's outcome report `report` of the clone's VM, and the - /// evidence that the status alone is once the clone got as far as `progress`, as - /// [`failure::reported`] gives them, or as once a workload may have run, if the clone's - /// progress is [uncertain](Self::uncertain). - pub(super) fn reported( - &self, - report: Option<&Report>, - progress: Progress, - ) -> (Option, Option) { - let progress = if self.uncertain { - progress.max(Progress::Starting) - } else { - progress - }; - failure::reported(report, progress) - } - - /// Records the failure of the clone at `stage`, once it got as far as `progress`, with - /// `fault`, whose teardown ended as `teardown` tells, as [the records of - /// failures](FailureRecord#records) describe, attributed as [`attribute`](Self::attribute) - /// does, and returns it. A failure that cannot be recorded says so. - pub(super) fn record( - &self, - stage: Stage, - progress: Progress, - fault: Fault, - teardown: Teardown, - ) -> CloneFailure { - let (evidence, status, error) = self.attribute(progress, fault, &teardown); - let record = FailureRecord::new( - &self.lease, - self.transaction, - (stage, progress), - evidence, - status, - teardown, - &error, - ); - self.write(error, record) - } - - /// Finishes `earlier`, the record of a failure whose teardown a crash of the caller left - /// [pending](Teardown::Pending), once its teardown ended as `teardown` tells, with the - /// failure's `error`, as the failure would have recorded it then: attributed from its - /// evidence, as [`attribute`](Self::attribute) does, keeping its lease, generation, launch, - /// transaction, stage, and progress, and the guest's status that it holds, if the outcome - /// report gives none now. Writes the record in place of the earlier one, and returns the - /// failure. - pub(super) fn finish( - &self, - earlier: FailureRecord, - error: Error, - teardown: Teardown, - ) -> CloneFailure { - let fault = Fault::new(earlier.evidence, error); - let (evidence, status, error) = self.attribute(earlier.progress, fault, &teardown); - let record = FailureRecord { - evidence, - status: status.or(earlier.status), - outcome: evidence.outcome(), - teardown, - code: error.code(), - message: describe(&error), - ..earlier - }; - self.write(error, record) - } - - /// Returns the evidence, the guest's status, and the error of the failure of the clone with - /// `fault`, once it got as far as `progress`, whose teardown ended as `teardown` tells: the - /// loss of a VMM that exited before its restore-ready event takes the line of OpenVMM's fatal - /// error that the VMM's log ends with, as [`diagnosed`](Self::diagnosed) does, and the - /// evidence of a status of the guest's, which OpenVMM's outcome report gives once the VMM has - /// exited, as [`observed`](Self::observed) reads it, and as [`reported`](Self::reported) - /// gives it, prevails over the fault's, unless the fault's is the template's, as - /// [`failure::prevailing`] decides. - fn attribute( - &self, - progress: Progress, - fault: Fault, - teardown: &Teardown, - ) -> (Evidence, Option, Error) { - let fault = self.diagnosed(progress, fault); - let report = match teardown { - Teardown::NoVm => None, - _ => self.observed(), - }; - let (reported, status) = self.reported(report, progress); - let evidence = failure::prevailing(fault.evidence, reported); - (evidence, status, fault.error) - } - - /// Returns `fault`, of the clone that got as far as `progress`, which is the loss of a VMM - /// that exited before its restore-ready event, with the line with which OpenVMM reported the - /// error that ended the VMM after its message, if the VMM's log holds one, as - /// [`read_fatal`] reads it, and the message does not hold it yet, as the message of a failure - /// that a reattach resolves may, and with the line's evidence, if it is evidence, as [the - /// attribution of evidence](Evidence#attribution) describes. Returns any other fault as it is. - fn diagnosed(&self, progress: Progress, fault: Fault) -> Fault { - if fault.evidence != Evidence::LostStatus || progress >= Progress::Event { - return fault; - } - let Some(line) = read_fatal(&self.log) else { - return fault; - }; - let evidence = failure::fatal_evidence(&line).unwrap_or(fault.evidence); - let reported = format!("; OpenVMM reported: {line}"); - let error = if describe(&fault.error).contains(&reported) { - fault.error - } else { - amended(fault.error, &reported) - }; - Fault::new(evidence, error) - } - - /// Records the failure of the clone at `stage`, once it got as far as `progress`, with - /// `evidence`, the guest's `status`, if the failure knows it already, and `error`, before the - /// teardown that follows it, whose record is [`Teardown::Pending`], so that a crash of the - /// caller during the teardown leaves the record, which a reattach finishes, as - /// [`reattach_clone`](super::reattach_clone) describes. The record that the failure writes - /// once its teardown ended replaces it. - /// - /// Fails if the record cannot be written, durably, as [`write_failure`] writes it: the caller - /// then removes nothing that tells that the clone launched, the journal of its launch or the - /// runtime record of its VM, which keep its lease, so that a reattach still finds that the - /// clone failed, though not how, if a crash of the caller comes before the failure is - /// recorded. - pub(super) fn pending( - &self, - stage: Stage, - progress: Progress, - (evidence, status): (Evidence, Option), - error: &Error, - ) -> Result<()> { - let record = FailureRecord::new( - &self.lease, - self.transaction, - (stage, progress), - evidence, - status, - Teardown::Pending, - error, - ); - write_failure(&self.failure, &record).map_err(|failed| { - Error::io(format!( - "the failure could not be recorded before its teardown: {}", - describe(&failed) - )) - }) - } - - /// Resolves `earlier`, the record of a failure that its teardown left unresolved, with - /// `teardown`, which ended the clone's VMM, keeping its stage, evidence, outcome, code, and - /// message, unless what the VMM left as it ended tells more: the line of OpenVMM's fatal - /// error, for the loss of a VMM that exited before its restore-ready event, as - /// [`diagnosed`](Self::diagnosed) adds it, and the guest's status in OpenVMM's outcome - /// report, which the record does not hold, whose evidence, as [`reported`](Self::reported) - /// gives it, prevails as [`failure::prevailing`] decides, as in [`record`](Self::record). - /// Writes the record in place of the earlier one, if they differ, and returns the failure. - pub(super) fn resolve(&self, mut earlier: FailureRecord, teardown: Teardown) -> CloneFailure { - let error = earlier.error(); - if earlier.teardown == teardown { - return CloneFailure::new(error, earlier); - } - let fault = self.diagnosed(earlier.progress, Fault::new(earlier.evidence, error)); - let mut evidence = fault.evidence; - if earlier.status.is_none() && teardown != Teardown::NoVm { - let (reported, status) = self.reported(self.observed(), earlier.progress); - evidence = failure::prevailing(evidence, reported); - earlier.status = status; - } - if evidence != earlier.evidence { - earlier.evidence = evidence; - earlier.outcome = evidence.outcome(); - } - earlier.message = describe(&fault.error); - earlier.teardown = teardown; - self.write(fault.error, earlier) - } - - /// Writes `record`, the record of the failure of `error`, in place of the sandbox's record, and - /// returns the failure, with the record as the sandbox keeps it, as [`kept`] returns it, whose - /// error says so if the record could not be written. - pub(super) fn write(&self, error: Error, record: FailureRecord) -> CloneFailure { - let record = kept(record); - let error = match write_failure(&self.failure, &record) { - Ok(()) => error, - Err(failed) => amended( - error, - &format!("; the failure could not be recorded: {}", describe(&failed)), - ), - }; - CloneFailure::new(error, record) - } -} - -/// What the launch of a clone keeps to record how far the clone got and how it failed: the -/// [`Recorder`] of its failures, its progress log, and how a failure ends the launch. -#[derive(Debug)] -pub(super) struct Tracker { - /// The recorder of the clone's failures. - recorder: Recorder, - /// The clone's progress log. - log: ProgressLog, - /// How a failure ends the launch. - end: End, -} - -impl Tracker { - /// Returns the tracker of the launch of a clone whose failures `recorder` records, with its - /// progress `log`, and the end of a launch that fails, `end`. - pub(super) fn new(recorder: Recorder, log: ProgressLog, end: End) -> Self { - Self { recorder, log, end } - } - - /// Notes that the clone reached `step`, as [`ProgressLog::advance`] does, without writing it. - pub(super) fn advance(&mut self, step: Progress) { - self.log.advance(step); - } - - /// Records that the clone reached `step` in its progress log, as [`ProgressLog::reach`] does, - /// and returns the fault of a failure to record it, the host's. - pub(super) fn reach(&mut self, step: Progress) -> std::result::Result<(), Fault> { - self.log.reach(step).map_err(Fault::host) - } - - /// Records that the clone reached `step` in its progress log if it can: a step that another - /// record holds, such as the commit, which the runtime record holds while the VM runs. - pub(super) fn note(&mut self, step: Progress) { - let _ = self.log.reach(step); - } - - /// Records the failure of the clone at `stage` with `evidence` and `error` before the teardown - /// that follows it, once the clone got as far as its progress, as [`Recorder::pending`] does, - /// and fails as it fails. - pub(super) fn pending(&self, stage: Stage, evidence: Evidence, error: &Error) -> Result<()> { - self.recorder - .pending(stage, self.log.progress(), (evidence, None), error) - } - - /// Ends `starting`, the launch of the clone, which failed at `stage` with `fault`, as `end` - /// does, and records the failure before the teardown, as [`pending`](Self::pending) does, and - /// with the teardown's outcome, as [`record`](Self::record) does. The error of a launch that - /// could not be ended says so, and that its journal remains for recovery. - /// - /// A failure that cannot be recorded before the teardown ends the clone's VMM through its - /// handle, and keeps the launch's journal, as [`Starting::end_keeping_journal`] does, and with - /// it the lease, until a reattach ends the launch: the teardown is unresolved, so that a crash - /// of the caller before the failure is recorded leaves a launch that a reattach records as - /// failed, rather than a clone that ended. - pub(super) fn fail(&self, starting: Starting<'_>, stage: Stage, fault: Fault) -> CloneFailure { - let teardown = match self.pending(stage, fault.evidence, &fault.error) { - Ok(()) => Teardown::of((self.end)(starting)), - Err(unrecorded) => { - let ended = if starting.end_keeping_journal() { - "the clone's VMM ended through its handle" - } else { - "the clone's VMM did not exit once it was ended through its handle" - }; - Teardown::Unresolved(format!("{ended}, but {unrecorded}")) - } - }; - let fault = match &teardown { - Teardown::Unresolved(reason) => Fault::new( - fault.evidence, - amended( - fault.error, - &format!("; the launch could not be ended, and remains for recovery: {reason}"), - ), - ), - _ => fault, - }; - self.record(stage, fault, teardown) - } - - /// Records the failure of the clone at `stage` with `fault`, whose teardown ended as - /// `teardown` tells, once the clone got as far as its progress, as [`Recorder::record`] - /// does, and returns it. - pub(super) fn record(&self, stage: Stage, fault: Fault, teardown: Teardown) -> CloneFailure { - self.recorder - .record(stage, self.log.progress(), fault, teardown) - } -} - -/// What a clone's progress log records. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub(super) struct Logged { - /// The latest step that the log records. - pub(super) progress: Progress, - /// The boot of the host during which the launch wrote the log, as its first line names it, - /// unless a crash cut that line short. - pub(super) boot: Option, -} - -impl Logged { - /// Returns whether the log tells for sure how far the clone got: whether the launch wrote it - /// during the boot `current`, the current boot of the host, if it is known, so that no crash - /// of the host lost a line of it since, as [the module](self#the-progress-log) describes. - pub(super) fn is_certain(&self, current: Option) -> bool { - self.boot.is_some() && self.boot == current - } -} - -/// Reads the progress log at `path` of the launch `launch`, and returns what it records, or -/// `None` if it does not exist. -/// -/// Fails with [`ErrorKind::Io`](crate::ErrorKind::Io) for a log that cannot be read, is not a -/// private regular file, names another launch or format, or no boot, or records other lines than -/// its steps, in order, besides a last line that a crash cut short. -pub(super) fn read_progress(path: &Path, launch: &LaunchId) -> Result> { - let Some(bytes) = read_private(path, PROGRESS_WHAT, MAX_PROGRESS_BYTES)? else { - return Ok(None); - }; - let malformed = - |problem: String| Error::io(format!("the {PROGRESS_WHAT} {} {problem}", path.display())); - let text = String::from_utf8(bytes).map_err(|_| malformed("is not UTF-8".to_owned()))?; - // A line without its newline is one that a crash cut short. - let mut lines = text - .split_inclusive('\n') - .filter_map(|line| line.strip_suffix('\n')); - let header = format!("{PROGRESS_MAGIC} {PROGRESS_FORMAT} {launch} "); - let boot = match lines.next() { - Some(line) => match line - .strip_prefix(&header) - .and_then(|boot| boot.parse::().ok()) - { - Some(boot) => boot, - None => { - return Err(malformed(format!( - "begins with {line:?}, not with the first line of the launch {launch}, \ - {header:?} followed by the identity of a boot" - ))); - } - }, - // A log that a crash cut short before its first line ended records the launch alone. - None => { - return Ok(Some(Logged { - progress: Progress::Launching, - boot: None, - })); - } - }; - let mut progress = Progress::Launching; - for line in lines { - match LOGGED.iter().find(|step| step.as_str() == line) { - Some(&step) if step > progress => progress = step, - _ => { - return Err(malformed(format!( - "records {line:?} after the step {progress}, which is not a later step" - ))); - } - } - } - Ok(Some(Logged { - progress, - boot: Some(boot), - })) -} - -/// Returns how far the clone of `lease` got, by the latest step that its progress log records, -/// `logged`, and by the store's records of its launch: its `journal`, which records the spawn of -/// its VMM, and its `runtime` record, which records the commit of its launch. -pub(super) fn progress_of( - lease: &Lease, - journal: Option<&LaunchJournal>, - runtime: Option<&RuntimeRecord>, - logged: Option, -) -> Progress { - let launch = lease.launch(); - let recorded = if runtime.is_some_and(|record| record.launch() == launch) { - Some(Progress::Committed) - } else { - journal - .filter(|journal| journal.launch() == launch) - .map(|journal| match journal.process() { - Some(_) => Progress::Spawned, - None => Progress::Launching, - }) - }; - [logged, recorded] - .into_iter() - .flatten() - .max() - .unwrap_or(Progress::Planned) -} - -/// The record of a clone's failure, as JSON. -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase", deny_unknown_fields)] -struct Recorded { - /// [`FAILURE_FORMAT`]. - format: u32, - /// The identity of the template store that leased the clone's generation. - store: String, - /// The identity of the clone's lease. - lease: String, - /// The leased generation. - generation: u64, - /// The identity of the leased generation's key. - key: String, - /// The launch that the lease reserves. - launch: String, - /// The transaction ID of the clone's restore, in hexadecimal, once the plan drew it. - #[serde(default, skip_serializing_if = "Option::is_none")] - transaction: Option, - /// The stage at which the clone failed. - stage: Stage, - /// How far the clone got. - progress: Progress, - /// The evidence of the failure. - evidence: Evidence, - /// The guest's status, if OpenVMM's outcome report has one. - #[serde(default, skip_serializing_if = "Option::is_none")] - status: Option, - /// The original outcome of the failure. - outcome: Outcome, - /// How the teardown ended, as [`Teardown::as_str`] spells it. - teardown: String, - /// Why the teardown is unresolved, if it is. - #[serde(default, skip_serializing_if = "Option::is_none")] - unresolved: Option, - /// The code of the failure's error, as [`ErrorCode::as_str`] spells it. - code: String, - /// The message of the failure's error. - message: String, -} - -impl Recorded { - /// Returns the JSON form of `record`, as the sandbox keeps it, with texts that [`kept`] - /// bounds. - fn new(record: &FailureRecord) -> Self { - Self { - format: FAILURE_FORMAT, - store: record.store.to_string(), - lease: record.lease.to_string(), - generation: record.template.generation.get().get(), - key: KeyIdentity::from(record.template.key).to_string(), - launch: record.launch.to_string(), - transaction: record - .transaction - .map(|transaction| transaction.to_string()), - stage: record.stage, - progress: record.progress, - evidence: record.evidence, - status: record.status, - outcome: record.outcome, - teardown: record.teardown.as_str().to_owned(), - unresolved: match &record.teardown { - Teardown::Unresolved(reason) => Some(bounded(reason, MAX_REASON_CHARS)), - _ => None, - }, - code: record.code.as_str().to_owned(), - message: bounded(&record.message, MAX_MESSAGE_CHARS), - } - } - - /// Checks the record, and returns it, or says why it is malformed. - fn into_record(self) -> std::result::Result { - let field = - |what: &str, error: Error| format!("its {what} is invalid: {}", error.message()); - let teardown = match (self.teardown.as_str(), self.unresolved) { - ("no-vm", None) => Teardown::NoVm, - ("exited", None) => Teardown::Exited, - ("terminated", None) => Teardown::Terminated, - ("pending", None) => Teardown::Pending, - ("unresolved", Some(reason)) => Teardown::Unresolved(reason), - (teardown, _) => { - return Err(format!( - "its teardown, {teardown:?}, is not one that a failure records, or names a \ - reason only when it is unresolved" - )); - } - }; - let transaction = match self.transaction { - None => None, - Some(digits) => Some(TransactionId::from_bytes( - parse_hex_id(&digits).ok_or_else(|| { - format!("its transaction, {digits:?}, is not 32 hexadecimal digits") - })?, - )), - }; - let key: KeyIdentity = self.key.parse().map_err(|error| field("key", error))?; - // A later build may record a code that this one does not know, which is a failure all - // the same. - let code = ErrorCode::ALL - .iter() - .copied() - .find(|code| code.as_str() == self.code) - .unwrap_or(ErrorCode::Other); - Ok(FailureRecord { - store: self.store.parse().map_err(|error| field("store", error))?, - lease: self.lease.parse().map_err(|error| field("lease", error))?, - template: TemplateReference { - generation: NonZeroU64::new(self.generation) - .map(StoreGeneration::new) - .ok_or("it names store generation 0, which no generation has")?, - key: Identity::from(key), - }, - launch: LaunchId::try_from(self.launch) - .map_err(|problem| format!("its launch is invalid: {problem}"))?, - transaction, - stage: self.stage, - progress: self.progress, - evidence: self.evidence, - status: self.status, - outcome: self.outcome, - teardown, - code, - message: self.message, - }) - } -} - -/// Replaces the record of a clone's failure at `path` with `record`, as the sandbox keeps it, as -/// [`kept`] returns it, atomically, and durably: once the record is in place, it syncs the -/// record's directory, as [`sync_dir`] does, and fails if the sync does, though the record is in -/// place then. -pub(super) fn write_failure(path: &Path, record: &FailureRecord) -> Result<()> { - write_failure_with(path, record, &sync_dir) -} - -/// Replaces the record of a clone's failure as [`write_failure`] does, syncing its directory with -/// `sync`, which tests observe, and fail. -fn write_failure_with( - path: &Path, - record: &FailureRecord, - sync: &dyn Fn(&Path) -> Result<()>, -) -> Result<()> { - let mut text = serde_json::to_vec_pretty(&Recorded::new(record)).map_err(|error| { - Error::other(format!("cannot encode the {FAILURE_WHAT}")).with_source(error) - })?; - text.push(b'\n'); - write_private(path, &text)?; - match path.parent() { - Some(dir) if !dir.as_os_str().is_empty() => sync(dir), - _ => sync(Path::new(".")), - } -} - -/// Returns `record` as the sandbox keeps it: with at most [`MAX_MESSAGE_CHARS`] characters of its -/// message, and [`MAX_REASON_CHARS`] of the reason of its teardown, if it is unresolved, as -/// [`bounded`] keeps them. -fn kept(mut record: FailureRecord) -> FailureRecord { - record.message = bounded(&record.message, MAX_MESSAGE_CHARS); - if let Teardown::Unresolved(reason) = &mut record.teardown { - *reason = bounded(reason, MAX_REASON_CHARS); - } - record -} - -/// Returns `text`, or, if it has more than `max` characters, `max` of them: its first ones and its -/// last ones, about as many of each, with an ellipsis between them in place of those that it -/// leaves out, so that the message of an error keeps what it begins with, and the messages of its -/// sources, which end it. -fn bounded(text: &str, max: usize) -> String { - let count = text.chars().count(); - if count <= max { - return text.to_owned(); - } - let first = (max - 1) / 2; - let last = max - 1 - first; - text.chars() - .take(first) - .chain(std::iter::once('…')) - .chain(text.chars().skip(count - last)) - .collect() -} - -/// Reads the record of a clone's failure at `path`, or returns `None` if it does not exist. -/// -/// Fails with [`ErrorCode::FormatUnsupported`] for a record of another format than -/// [`FAILURE_FORMAT`], and with [`ErrorKind::Io`](crate::ErrorKind::Io) for one that cannot be -/// read, is not a private regular file, or is malformed. -pub(super) fn read_failure(path: &Path) -> Result> { - let Some(bytes) = read_private(path, FAILURE_WHAT, MAX_FAILURE_BYTES)? else { - return Ok(None); - }; - let malformed = || format!("the {FAILURE_WHAT} {} is malformed", path.display()); - let value: serde_json::Value = serde_json::from_slice(&bytes) - .map_err(|error| Error::io(malformed()).with_source(error))?; - match value.get("format") { - Some(format) if format.as_u64() == Some(FAILURE_FORMAT.into()) => {} - Some(format) => { - return Err(Error::new( - ErrorCode::FormatUnsupported, - format!( - "the {FAILURE_WHAT} {} has the unsupported format {format}; this build reads \ - format {FAILURE_FORMAT} only", - path.display() - ), - )); - } - None => return Err(Error::io(format!("{}: it has no format", malformed()))), - } - let recorded: Recorded = - serde_json::from_value(value).map_err(|error| Error::io(malformed()).with_source(error))?; - recorded - .into_record() - .map(Some) - .map_err(|problem| Error::io(format!("{}: {problem}", malformed()))) -} - -/// Reads the record of the failure of the clone of `lease` at `path`, as [`read_failure`] reads -/// it, or returns `None` if it does not exist. -/// -/// Fails as [`read_failure`] does, and, as for a record that is malformed, with -/// [`ErrorKind::Io`](crate::ErrorKind::Io) for a record of another clone's failure, whose template -/// store, lease, leased generation, its key, or launch is not the lease's: it tells nothing of -/// this clone, and a [quarantine](CloneFailure::quarantine) of its failure would tombstone the -/// record's generation, which may not be the one that this clone leases. -pub(super) fn read_failure_of(path: &Path, lease: &Lease) -> Result> { - let Some(record) = read_failure(path)? else { - return Ok(None); - }; - let leased = lease.template(); - let differences: Vec<&str> = [ - (record.store != lease.store(), "the template store"), - (record.lease != lease.id(), "the lease"), - ( - record.template.generation != leased.generation, - "the leased generation", - ), - ( - record.template.key != leased.key, - "the key of the leased generation", - ), - (record.launch != *lease.launch(), "the launch"), - ] - .into_iter() - .filter_map(|(differs, what)| differs.then_some(what)) - .collect(); - let Some((last, others)) = differences.split_last() else { - return Ok(Some(record)); - }; - let differences = if others.is_empty() { - (*last).to_owned() - } else { - format!("{} and {last}", others.join(", ")) - }; - Err(Error::io(format!( - "the {FAILURE_WHAT} {} belongs to another clone than the one of the lease {}, which the \ - sandbox's clone binding records: they differ in {differences}", - path.display(), - lease.id(), - ))) -} - -/// Reads the private regular file at `path`, the clone's `description`, of at most `limit` bytes, -/// or returns `None` if it does not exist, or was removed as this opened it, as [`was_removed`] -/// tells. -fn read_private(path: &Path, description: &str, limit: u64) -> Result>> { - let failed = |error: io::Error| { - Error::io(format!("cannot read the {description} {}", path.display())).with_source(error) - }; - let file = match open_regular(path) { - Ok(file) => file, - Err(error) if error.kind() == io::ErrorKind::NotFound || was_removed(path, &error) => { - return Ok(None); - } - Err(error) => return Err(failed(error)), - }; - if let Some(problem) = platform::private_file_problem(&file).map_err(failed)? { - return Err(Error::io(format!( - "the {description} {} {problem}", - path.display() - ))); - } - read_at_most(file, limit, &format!("the {description}")) - .map(Some) - .map_err(failed) -} - -/// Returns the last line with which OpenVMM reported the error that ended it in the VMM's log at -/// `path`, cut to [`MAX_FATAL_CHARS`], or none if the log holds none, or cannot be read: the line -/// that leads OpenVMM's fatal error, then the error's time ABI code, if it has one, as [the -/// attribution of evidence](Evidence#attribution) describes. The line is looked for back from the -/// log's end, as [`last_line_start`] reads it, so that the error's chain, which follows the line, -/// may be of any length. If the line does not name a forward that OpenVMM cannot bind, -/// [`failure::FORWARD_BIND_TEXT`], but a cause of the error's chain that follows it does, the -/// cause follows the line, after `: `, cut to [`MAX_FATAL_CHARS`] too, without the indentation and -/// the number with which the chain lists it. -fn read_fatal(path: &Path) -> Option { - let mut log = open_regular(path).ok()?; - let start = last_line_start(&mut log, failure::FATAL_PREFIX.as_bytes()).ok()??; - log.seek(SeekFrom::Start(start)).ok()?; - let mut rest = Vec::new(); - log.read_to_end(&mut rest).ok()?; - let text = String::from_utf8_lossy(&rest); - let mut lines = text.lines(); - let mut fatal: String = lines.next()?.chars().take(MAX_FATAL_CHARS).collect(); - if !fatal.contains(failure::FORWARD_BIND_TEXT) { - let forward = lines - .map(cause) - .find(|cause| cause.starts_with(failure::FORWARD_BIND_TEXT)); - if let Some(forward) = forward { - fatal.push_str(": "); - fatal.extend(forward.chars().take(MAX_FATAL_CHARS)); - } - } - Some(fatal) -} - -/// Returns the offset in `log` of its last line that starts with `prefix`, or none if no line -/// does, reading the log back from its end, [`FATAL_BLOCK_BYTES`] at a time: a line starts at -/// the start of the log, and after each newline. -fn last_line_start(log: &mut File, prefix: &[u8]) -> io::Result> { - let length = log.seek(SeekFrom::End(0))?; - let mut block = Vec::new(); - let mut end = length; - while end > 0 { - let start = end.saturating_sub(FATAL_BLOCK_BYTES); - // The block's read goes on past its end by the prefix's length, which a line that starts - // at its end would begin with. - let stop = length.min(end + prefix.len() as u64); - block.resize((stop - start) as usize, 0); - log.seek(SeekFrom::Start(start))?; - log.read_exact(&mut block)?; - let starts = block[..(end - start) as usize] - .iter() - .enumerate() - .rev() - .filter(|&(_, &byte)| byte == b'\n') - .map(|(newline, _)| newline + 1) - .chain((start == 0).then_some(0)); - for line in starts { - if block[line..].starts_with(prefix) { - return Ok(Some(start + line as u64)); - } - } - end = start; - } - Ok(None) -} - -/// Returns the cause of an error's chain that `line` lists, as OpenVMM writes the chain after the -/// line that leads its fatal error, ` N: CAUSE` for each of several causes, or ` CAUSE` for -/// one alone: without its indentation, and its number, if it has one. -fn cause(line: &str) -> &str { - let line = line.trim_start(); - match line.split_once(": ") { - Some((number, cause)) - if !number.is_empty() && number.bytes().all(|b| b.is_ascii_digit()) => - { - cause - } - _ => line, - } -} - -/// What the sandbox of a clone keeps of it, as [`CloneRecords::read`] reads it: the lease that its -/// clone binding records, the store's records of its launch and of its VM, its progress, the -/// record of its failure, and OpenVMM's outcome report of its VM. -/// -/// The records tell the state of a clone, as an operation that lists, inspects, or removes -/// clones reads it: -/// -/// - a [`journal`](Self::journal) without a [`runtime`](Self::runtime) record: a launch that runs, -/// under the lock of the sandbox, or one that was interrupted, which -/// [`reattach_clone`](super::reattach_clone) ends; -/// - a runtime record: a VM that may run, which a reattach holds again, or a stop ends; -/// - a [`failure`](Self::failure): a clone that failed, whose teardown is unresolved while the -/// journal or the runtime record remains, and [pending](Teardown::Pending) if a crash of the -/// caller interrupted its teardown, which a reattach finishes; -/// - neither a journal nor a runtime record: no VM of the clone runs, and a release ends the -/// lease, which removes the clone's records, before a removal of the sandbox. -#[derive(Debug, Clone, PartialEq, Eq)] -#[non_exhaustive] -pub struct CloneRecords { - /// The lease that the sandbox's clone binding records. - pub lease: Lease, - /// The journal of the clone's launch, while it runs, or once it was interrupted. - pub journal: Option, - /// The runtime record of the clone's VM, from the commit of its launch until a stop, a - /// termination, or a reattach verifies that the VM exited. - pub runtime: Option, - /// How far the clone got, as its progress log and the store's records of its launch tell. - pub progress: Progress, - /// The record of the clone's failure, if it failed from its launch on. - pub failure: Option, - /// OpenVMM's outcome report of the clone's VM, once its VMM wrote one, and none before the - /// clone got past its plan, as [`read`](Self::read) tells. - pub report: Option, -} - -impl CloneRecords { - /// Reads the records of the clone of the sandbox of `sandbox`, or returns `None` if the - /// sandbox holds no clone binding. - /// - /// It takes no lock, as an inspection of the sandbox does not: each record is read whole, as - /// the latest transition that replaced it left it, and the records may come from transitions - /// that ran between the reads. A caller that holds the sandbox's lock reads them at one point - /// of its lifecycle. Fails as the reads of the clone binding, [`Lease::of`], of the store's - /// records, of the progress log, and of the failure record fail, a failure record of another - /// clone than the binding's included; an outcome report that is missing or malformed is none. - /// - /// It reads the outcome report only once the clone got past its plan, as the progress log of - /// its launch, the launch's journal, the runtime record of its VM, or the record of a failure - /// past the plan shows. Before the launch writes the first of those, its progress log, it - /// removes the report that an earlier VM of the sandbox left, which no acquisition, clearing - /// of a VM's runtime record, or release removes, and which is not the clone's. - pub fn read(sandbox: &SandboxStore) -> Result> { - let Some(lease) = Lease::of(sandbox)? else { - return Ok(None); - }; - let journal = sandbox.journal()?; - let runtime = sandbox.runtime()?; - let logged = read_progress(&sandbox.clone_progress_path(), lease.launch())? - .map(|logged| logged.progress); - let failure = read_failure_of(&sandbox.clone_failure_path(), &lease)?; - // A failure records how far the clone got, which the store's records of its launch no - // longer tell once the failure ended the launch. - let progress = progress_of(&lease, journal.as_ref(), runtime.as_ref(), logged).max( - failure - .as_ref() - .map_or(Progress::Planned, |failure| failure.progress), - ); - // The launch removed the report of an earlier VM of the sandbox before it wrote the first - // of the clone's records, which a progress past the plan shows. - let report = (progress > Progress::Planned) - .then(|| report::read(&sandbox.outcome_path()).ok().flatten()) - .flatten(); - Ok(Some(Self { - lease, - journal, - runtime, - progress, - failure, - report, - })) - } -} - -#[cfg(test)] -mod tests; diff --git a/nvx-rs/src/host/template/clone/records/tests.rs b/nvx-rs/src/host/template/clone/records/tests.rs deleted file mode 100644 index f500858b4..000000000 --- a/nvx-rs/src/host/template/clone/records/tests.rs +++ /dev/null @@ -1,511 +0,0 @@ -use std::fs; - -use super::*; -use crate::error::ErrorKind; -use crate::host::store::CLONE_FAILURE_NAME; -use crate::host::template::{LeaseId, StoreId}; - -/// Returns a directory private to the current user, as a sandbox's is, whose files are -/// private too, in a temporary directory, which removes it as it drops. -fn private_dir() -> (tempfile::TempDir, PathBuf) { - let directory = tempfile::tempdir().unwrap(); - let private = directory.path().join("sandbox"); - assert_eq!(platform::create_owner_only_dir(&private).unwrap(), None); - (directory, private) -} - -/// Returns the record of a failure of a clone whose teardown ended as `teardown`. -fn failure(teardown: Teardown) -> FailureRecord { - FailureRecord { - store: StoreId::from_bytes([1; 16]), - lease: LeaseId::from_bytes([2; 16]), - template: TemplateReference { - generation: StoreGeneration::new(NonZeroU64::new(7).unwrap()), - key: Identity::from_bytes([3; 32]), - }, - launch: LaunchId::try_from("4".repeat(32)).unwrap(), - transaction: Some(TransactionId::from_bytes([5; 16])), - stage: Stage::Activation, - progress: Progress::Activating, - evidence: Evidence::UnusableRoot, - status: Some(54), - outcome: Outcome::Request, - teardown, - code: ErrorCode::Exited, - message: "the clone's VMM exited".to_owned(), - } -} - -#[test] -fn a_progress_log_records_each_step_once_with_the_steps_that_it_deferred() { - let (_directory, private) = private_dir(); - let path = private.join(CLONE_PROGRESS_NAME); - let launch = LaunchId::generate().unwrap(); - let boot = BootId::current().unwrap(); - let logged = |progress| { - Some(Logged { - progress, - boot: Some(boot), - }) - }; - // The log of an earlier launch goes, and so does the report of an earlier VM. - write_private(&path, b"left").unwrap(); - let report = private.join(OUTCOME_NAME); - write_private(&report, b"left").unwrap(); - let directory = platform::Directory::open(&private).unwrap(); - let mut log = ProgressLog::create(&directory, &private, &launch).unwrap(); - assert!(!report.exists()); - assert_eq!( - read_progress(&path, &launch).unwrap(), - logged(Progress::Launching) - ); - // A step that another record holds waits for the next step that the log writes. - log.advance(Progress::Spawned); - assert_eq!(log.progress(), Progress::Spawned); - assert_eq!( - read_progress(&path, &launch).unwrap(), - logged(Progress::Launching) - ); - log.reach(Progress::Event).unwrap(); - log.advance(Progress::Acknowledged); - log.reach(Progress::Activating).unwrap(); - // A step that the clone reached already is not recorded again. - log.reach(Progress::Event).unwrap(); - log.advance(Progress::Spawned); - assert_eq!(log.progress(), Progress::Activating); - assert_eq!( - fs::read_to_string(&path).unwrap(), - format!("{PROGRESS_MAGIC} 1 {launch} {boot}\nspawned\nevent\nacknowledged\nactivating\n") - ); - assert_eq!( - read_progress(&path, &launch).unwrap(), - logged(Progress::Activating) - ); -} - -#[test] -fn a_progress_log_replaces_an_earlier_one_only_in_the_directory_that_it_holds_open() { - let (temporary, private) = private_dir(); - let launch = LaunchId::generate().unwrap(); - let directory = platform::Directory::open(&private).unwrap(); - write_private(&private.join(CLONE_PROGRESS_NAME), b"left").unwrap(); - write_private(&private.join(OUTCOME_NAME), b"left").unwrap(); - // The sandbox's path names another sandbox's directory by the time that the log is - // created: the earlier log and report of the directory that the handle holds go, and the - // other sandbox's stay, whose log fails the creation. - let moved = temporary.path().join("moved"); - fs::rename(&private, &moved).unwrap(); - let other = temporary.path().join("other"); - assert_eq!(platform::create_owner_only_dir(&other).unwrap(), None); - for name in [CLONE_PROGRESS_NAME, OUTCOME_NAME] { - write_private(&other.join(name), b"the other's").unwrap(); - } - fs::rename(&other, &private).unwrap(); - let error = ProgressLog::create(&directory, &private, &launch).unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Io, "{error}"); - for name in [CLONE_PROGRESS_NAME, OUTCOME_NAME] { - assert!(!moved.join(name).exists(), "{name}"); - assert_eq!( - fs::read(private.join(name)).unwrap(), - b"the other's", - "{name}" - ); - } -} - -#[test] -fn a_progress_log_that_a_crash_cut_short_is_read_and_one_of_another_launch_is_refused() { - let (_directory, private) = private_dir(); - let path = private.join(CLONE_PROGRESS_NAME); - let launch = LaunchId::generate().unwrap(); - assert_eq!(read_progress(&path, &launch).unwrap(), None); - let (boot, other) = (BootId::current().unwrap(), BootId::from_bytes([0xee; 16])); - let header = format!("{PROGRESS_MAGIC} 1 {launch} {boot}\n"); - // A last line that a crash cut short is not a step, nor is the first line, without which - // the log names no boot. A log of another boot is read, with its boot, which a crash of - // the host may have cut short. - for (text, progress, written) in [ - (format!("{header}event\nackno"), Progress::Event, Some(boot)), - (header[..10].to_owned(), Progress::Launching, None), - ( - header[..header.len() - 1].to_owned(), - Progress::Launching, - None, - ), - (String::new(), Progress::Launching, None), - ( - format!("{header}committed\n"), - Progress::Committed, - Some(boot), - ), - ( - format!("{PROGRESS_MAGIC} 1 {launch} {other}\nevent\n"), - Progress::Event, - Some(other), - ), - ] { - write_private(&path, text.as_bytes()).unwrap(); - let read = read_progress(&path, &launch).unwrap().unwrap(); - assert_eq!((read.progress, read.boot), (progress, written), "{text:?}"); - // Only a log of the current boot tells for sure how far the clone got. - assert_eq!( - read.is_certain(Some(boot)), - written == Some(boot), - "{text:?}" - ); - assert!(!read.is_certain(None), "{text:?}"); - } - // Another launch's log, another format, a first line without a boot, and steps out of - // their order, or that no log records, are refused. - let other = LaunchId::generate().unwrap(); - for text in [ - format!("{PROGRESS_MAGIC} 1 {other} {boot}\nevent\n"), - format!("{PROGRESS_MAGIC} 2 {launch} {boot}\n"), - format!("{PROGRESS_MAGIC} 1 {launch}\n"), - format!("{PROGRESS_MAGIC} 1 {launch} the-boot\n"), - format!("{header}ready\nevent\n"), - format!("{header}event\nevent\n"), - format!("{header}planned\n"), - format!("{header}launching\n"), - format!("{header}event\nbogus\n"), - ] { - write_private(&path, text.as_bytes()).unwrap(); - let error = read_progress(&path, &launch).unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Io, "{text:?}: {error}"); - } -} - -#[test] -fn a_failure_record_is_read_as_it_was_written() { - let (_directory, private) = private_dir(); - let path = private.join(CLONE_FAILURE_NAME); - assert_eq!(read_failure(&path).unwrap(), None); - let unresolved = Teardown::Unresolved("its VMM did not exit".to_owned()); - for teardown in [ - Teardown::NoVm, - Teardown::Exited, - Teardown::Terminated, - unresolved, - ] { - let written = failure(teardown); - write_failure(&path, &written).unwrap(); - assert_eq!(read_failure(&path).unwrap(), Some(written)); - } - // Its JSON spells each field as the record's types do. - let value: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); - assert_eq!( - value, - serde_json::json!({ - "format": 1, - "store": "01".repeat(16), - "lease": "02".repeat(16), - "generation": 7, - "key": "03".repeat(32), - "launch": "4".repeat(32), - "transaction": "05".repeat(16), - "stage": "activation", - "progress": "activating", - "evidence": "unusable-root", - "status": 54, - "outcome": "request", - "teardown": "unresolved", - "unresolved": "its VMM did not exit", - "code": "exited", - "message": "the clone's VMM exited", - }) - ); - // A code that a later build records, which this one does not know, is still a failure. - let mut later = value.clone(); - later["code"] = "a-later-code".into(); - write_private(&path, &serde_json::to_vec(&later).unwrap()).unwrap(); - assert_eq!(read_failure(&path).unwrap().unwrap().code, ErrorCode::Other); -} - -#[test] -fn a_failure_record_is_in_place_when_its_directory_is_synced() { - let (_directory, private) = private_dir(); - let path = private.join(CLONE_FAILURE_NAME); - // Each write syncs the record's directory once the record is in place, so that the sync - // makes the record's rename durable. - let synced = std::cell::RefCell::new(Vec::new()); - let sync = |dir: &Path| -> Result<()> { - synced - .borrow_mut() - .push((dir.to_owned(), read_failure(&path).unwrap())); - sync_dir(dir) - }; - let (pending, exited) = (failure(Teardown::Pending), failure(Teardown::Exited)); - write_failure_with(&path, &pending, &sync).unwrap(); - write_failure_with(&path, &exited, &sync).unwrap(); - assert_eq!( - synced.into_inner(), - [ - (private.clone(), Some(pending)), - (private.clone(), Some(exited)) - ] - ); - // A sync that fails fails the write, whose record is in place, though maybe not durably. - let refused = |_: &Path| -> Result<()> { Err(Error::io("the test's sync failed")) }; - let terminated = failure(Teardown::Terminated); - let error = write_failure_with(&path, &terminated, &refused).unwrap_err(); - assert_eq!(error.message(), "the test's sync failed"); - assert_eq!(read_failure(&path).unwrap(), Some(terminated)); -} - -#[test] -fn a_failure_record_keeps_the_ends_of_texts_too_long_for_a_read_to_take() { - // A text keeps its first and its last characters, which it counts as characters. - assert_eq!(bounded("abcdef", 6), "abcdef"); - assert_eq!(bounded("abcdefg", 6), "ab…efg"); - assert_eq!(bounded("ééééééé", 4), "é…éé"); - let (_directory, private) = private_dir(); - let path = private.join(CLONE_FAILURE_NAME); - // JSON takes six bytes for each of these control characters, the most that a character - // takes, so that each text, whole, would take more than a read takes. - let filler = "\u{1}".repeat(64 * 1024); - let mut written = failure(Teardown::Unresolved(format!("why{filler}so"))); - written.message = format!("start{filler}end"); - write_failure(&path, &written).unwrap(); - assert!(fs::metadata(&path).unwrap().len() <= MAX_FAILURE_BYTES); - let read = read_failure(&path).unwrap().unwrap(); - assert_eq!(read, kept(written)); - assert_eq!(read.message.chars().count(), MAX_MESSAGE_CHARS); - assert!(read.message.starts_with("start\u{1}") && read.message.ends_with("\u{1}end")); - let Teardown::Unresolved(reason) = &read.teardown else { - panic!("{:?}", read.teardown); - }; - assert_eq!(reason.chars().count(), MAX_REASON_CHARS); - assert!(reason.starts_with("why\u{1}") && reason.ends_with("\u{1}so")); -} - -#[test] -fn a_failure_record_of_another_format_or_a_malformed_one_is_refused() { - let (_directory, private) = private_dir(); - let path = private.join(CLONE_FAILURE_NAME); - write_failure(&path, &failure(Teardown::Exited)).unwrap(); - let value: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap(); - let changed = |change: &dyn Fn(&mut serde_json::Value)| { - let mut value = value.clone(); - change(&mut value); - write_private(&path, &serde_json::to_vec(&value).unwrap()).unwrap(); - read_failure(&path).unwrap_err() - }; - let error = changed(&|value| value["format"] = 2.into()); - assert_eq!(error.code(), ErrorCode::FormatUnsupported, "{error}"); - type Change = fn(&mut serde_json::Value); - let malformed: [Change; 6] = [ - |value| value["unresolved"] = "a reason of a resolved teardown".into(), - |value| value["teardown"] = "unresolved".into(), - |value| value["teardown"] = "vanished".into(), - |value| value["generation"] = 0.into(), - |value| value["transaction"] = "05".into(), - |value| value["extra"] = true.into(), - ]; - for change in malformed { - let error = changed(&change); - assert_eq!(error.kind(), ErrorKind::Io, "{error}"); - } - write_private(&path, b"{").unwrap(); - assert_eq!(read_failure(&path).unwrap_err().kind(), ErrorKind::Io); -} - -#[test] -fn a_failure_record_of_another_clone_than_the_one_of_the_lease_is_refused() { - let (_directory, private) = private_dir(); - let path = private.join(CLONE_FAILURE_NAME); - let written = failure(Teardown::Exited); - let lease = Lease::new( - written.lease, - written.store, - written.template.generation, - written.template.key, - private.clone(), - written.launch.clone(), - ); - assert_eq!(read_failure_of(&path, &lease).unwrap(), None); - write_failure(&path, &written).unwrap(); - assert_eq!( - read_failure_of(&path, &lease).unwrap(), - Some(written.clone()) - ); - // A record of the failure of a clone of another store, of another lease, of another - // generation, or of another key, or of another launch, tells nothing of the lease's clone, - // though `read_failure` reads it. - let generation = StoreGeneration::new(NonZeroU64::new(8).unwrap()); - let launch = LaunchId::try_from("6".repeat(32)).unwrap(); - let others = [ - ( - FailureRecord { - store: StoreId::from_bytes([6; 16]), - ..written.clone() - }, - "the template store", - ), - ( - FailureRecord { - lease: LeaseId::from_bytes([6; 16]), - ..written.clone() - }, - "the lease", - ), - ( - FailureRecord { - template: TemplateReference { - generation, - ..written.template - }, - ..written.clone() - }, - "the leased generation", - ), - ( - FailureRecord { - template: TemplateReference { - key: Identity::from_bytes([6; 32]), - ..written.template - }, - ..written.clone() - }, - "the key of the leased generation", - ), - ( - FailureRecord { - launch: launch.clone(), - ..written.clone() - }, - "the launch", - ), - ( - FailureRecord { - lease: LeaseId::from_bytes([6; 16]), - template: TemplateReference { - generation, - ..written.template - }, - launch, - ..written.clone() - }, - "the lease, the leased generation and the launch", - ), - ]; - for (other, differences) in others { - write_failure(&path, &other).unwrap(); - assert_eq!(read_failure(&path).unwrap(), Some(other)); - let error = read_failure_of(&path, &lease).unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Io, "{error}"); - assert!( - error - .to_string() - .ends_with(&format!("they differ in {differences}")), - "{error}" - ); - } - // A record that cannot be read fails as it does for `read_failure`. - write_private(&path, b"{").unwrap(); - assert_eq!( - read_failure_of(&path, &lease).unwrap_err().kind(), - ErrorKind::Io - ); -} - -#[test] -fn the_line_of_openvmm_s_fatal_error_is_read_from_the_end_of_its_log() { - let (_directory, private) = private_dir(); - let path = private.join("openvmm.log"); - assert_eq!(read_fatal(&path), None); - // OpenVMM's warnings may name a time ABI code too; the line that leads its fatal error, - // which the error's chain follows, is the one that tells why it exited. - let fatal = "fatal error: [E_TSC_RATE_TOLERANCE] failed to launch vm worker"; - let log = format!( - "NVX-CPU-PROFILE-FALLBACK: [E_PROFILE_UNSUPPORTED] a warning\r\n{fatal}\r\n\r\n\ - Caused by:\r\n 0: [E_TSC_RATE_TOLERANCE] the destination's TSC rate\r\n" - ); - fs::write(&path, &log).unwrap(); - assert_eq!(read_fatal(&path).as_deref(), Some(fatal)); - // The last line of a long log is read, kept to its first characters. - let long = format!("fatal error: {}", "x".repeat(2 * MAX_FATAL_CHARS)); - let filler = "a line of the VMM's log\n".repeat(2 * FATAL_BLOCK_BYTES as usize / 24); - fs::write( - &path, - format!("fatal error: an earlier one\n{filler}{long}\n"), - ) - .unwrap(); - let read = read_fatal(&path).unwrap(); - assert_eq!(read.chars().count(), MAX_FATAL_CHARS); - assert!(long.starts_with(&read)); - // A log without a fatal error, or one that only names it, holds none, however long. - fs::write(&path, "the VMM exited\nwarning: fatal error: quoted\n").unwrap(); - assert_eq!(read_fatal(&path), None); - fs::write( - &path, - format!("{filler}warning: fatal error: quoted\n{filler}"), - ) - .unwrap(); - assert_eq!(read_fatal(&path), None); -} - -#[test] -fn the_line_of_openvmm_s_fatal_error_is_read_before_a_chain_of_any_length() { - let (_directory, private) = private_dir(); - let path = private.join("openvmm.log"); - let block = FATAL_BLOCK_BYTES as usize; - // A failed CPU check names every difference that it found, in a cause that may take - // several blocks of the log, after the line, which may be the log's first. - let fatal = "fatal error: [E_CPU_SURFACE] failed to launch vm worker"; - let differences = "CPUID 0x1 ECX differs; ".repeat(3 * block / 23); - for before in ["", "a line of the VMM's log\n"] { - let log = format!("{before}{fatal}\n\nCaused by:\n {differences}\n"); - fs::write(&path, &log).unwrap(); - assert_eq!(read_fatal(&path).as_deref(), Some(fatal)); - } - // A cause that names a forward that OpenVMM cannot bind follows the line after it. - let forward = "failed to create socket for port forward (Tcp 127.0.0.1:18080)"; - let log = format!("{fatal}\n\nCaused by:\n 0: {differences}\n 1: {forward}\n"); - fs::write(&path, &log).unwrap(); - assert_eq!(read_fatal(&path), Some(format!("{fatal}: {forward}"))); - // The line is read wherever a block starts around it: after the newline before the line, - // at the line's start, within its prefix, or after it. - let filler = "a line of the VMM's log\n".repeat(2 * block / 24); - let prefix = failure::FATAL_PREFIX.len(); - for from_end in block - 2..=block + prefix + 2 { - let rest = "y".repeat(from_end - fatal.len() - 2); - let log = format!("{filler}{fatal}\n{rest}\n"); - assert_eq!(log.len() - log.find(fatal).unwrap(), from_end); - fs::write(&path, &log).unwrap(); - assert_eq!(read_fatal(&path).as_deref(), Some(fatal), "{from_end}"); - } -} - -#[test] -fn the_cause_that_names_a_forward_that_openvmm_cannot_bind_follows_the_fatal_line() { - let (_directory, private) = private_dir(); - let path = private.join("openvmm.log"); - let fatal = "fatal error: failed to launch vm worker"; - let forward = "failed to create socket for port forward (Tcp 127.0.0.1:18080)"; - let expected = format!("{fatal}: {forward}"); - // OpenVMM lists the causes of its error's chain after the line, numbered when there are - // several, and alone otherwise. - for chain in [ - format!( - " 0: failed to resolve the network endpoint\n 1: {forward}\n 2: Address \ - already in use (os error 98)\n" - ), - format!(" {forward}\n"), - ] { - let log = format!("a line of the VMM's log\n{fatal}\n\nCaused by:\n{chain}"); - fs::write(&path, &log).unwrap(); - assert_eq!( - read_fatal(&path).as_deref(), - Some(expected.as_str()), - "{log}" - ); - } - // A line that names the forward itself keeps its text alone. - let named = format!("fatal error: {forward}"); - fs::write(&path, format!("{named}\n\nCaused by:\n {forward}\n")).unwrap(); - assert_eq!(read_fatal(&path).as_deref(), Some(named.as_str())); - // A forward that a line before the fatal error names is no cause of it. - let log = format!("warning: {forward}\n{fatal}\n\nCaused by:\n 0: another cause\n"); - fs::write(&path, log).unwrap(); - assert_eq!(read_fatal(&path).as_deref(), Some(fatal)); -} diff --git a/nvx-rs/src/host/template/clone/restore.rs b/nvx-rs/src/host/template/clone/restore.rs deleted file mode 100644 index 5b3d07ee3..000000000 --- a/nvx-rs/src/host/template/clone/restore.rs +++ /dev/null @@ -1,985 +0,0 @@ -//! The restore sequence of a clone, from its launch, a [`Restoring`], to its sandbox's start, -//! through the adapter's framing of the clone's control session, a [`CloneControl`]: -//! [`Restoring::activate`] activates the clone, [`Activated::start`] sends its mapping table, if -//! it has an export, then starts its sandbox, and [`Started::commit`] commits its launch, as the -//! documentation of [`Restoring::activate`] describes, while [`Activated::fail`] and -//! [`Started::fail`] fail the clone for a step of the caller's own between them. - -use std::fmt; -use std::io; -use std::path::Path; -use std::thread; -use std::time::{Duration, Instant}; - -use super::failure::{Attribute, CloneFailure, Evidence, Fault, Progress, Stage, host_evidence}; -use super::launch::Restoring; -use super::records::Tracker; -use crate::activation::{ActivationRecord, MAX_RECORD_BYTES, ReadyActivation}; -use crate::error::{Error, Result}; -use crate::host::platform::{ReadyListener, Transport}; -use crate::host::process::ProcessRef; -use crate::host::sandbox::distro::CheckedDistro; -use crate::host::session::{self, SessionError}; -use crate::host::store::Recovery; -use crate::host::store::regular::{open_regular, read_at_most}; -use crate::host::supervisor::{Starting, Vmm}; -use crate::protocol::{CAPABILITY_LEN, HostMap}; - -/// The restore-ready event that OpenVMM writes to a restore's `--restore-ready-path` once the -/// restored VM has started, which [`Restoring::activate`] waits for. -pub const RESTORE_READY_EVENT: &[u8] = b"OPENVMM_RESTORE_READY_V1\n"; - -/// The default restore-to-Ready deadline of a clone's restore sequence: how long after the clone's -/// launch its Ready may come. -/// -/// It covers OpenVMM's restore, the guest's repair, the host's attachment, and the activation of -/// the clone's root, which assembles it as a cold boot does, with a wide margin on the CI runs of -/// the slowest backend, WHP, whose restores ran within a third of a second, and whose 25 VMs of the -/// test adapter, at once, assembled their layered roots, applied their mapping tables, and started -/// within 19 seconds in all. It is the test adapter's deadline of a cold launch. -pub const RESTORE_TO_READY_TIMEOUT: Duration = Duration::from_secs(60); - -/// The default Ready-to-Start deadline of a clone's restore sequence: how long after the check of -/// the clone's Ready its sandbox may start, which takes a few of the guest's answers, to the parts -/// of its mapping table and to the start, as the CI runs of [`RESTORE_TO_READY_TIMEOUT`] show. -/// It is the test adapter's deadline of a session's answers. -pub const READY_TO_START_TIMEOUT: Duration = Duration::from_secs(30); - -/// The most sessions in which [`Restoring::activate`] sends a clone's Activate: its first session, -/// and those that replace a session that the adapter lost. -pub const MAX_ACTIVATION_SESSIONS: usize = 3; - -/// How often the wait for the restore-ready event checks whether the VMM still runs. -const EXIT_CHECK_INTERVAL: Duration = Duration::from_millis(50); - -/// How long the wait for the restore-ready event waits for the VMM to exit once it closed the -/// event's endpoint without a byte of the event, as OpenVMM does as it fails to launch its VM -/// worker, which it does after it connected the endpoint, before it reports the error and exits. -const CLOSED_EXIT_WAIT: Duration = Duration::from_secs(10); - -/// How long the sequence waits before a session that replaces a lost one, so that the VMM's -/// control console notices the end of the lost session, which frees the console's one attachment. -const SESSION_RETRY_INTERVAL: Duration = Duration::from_millis(50); - -/// The deadlines of a clone's restore sequence, each a duration from the step at which it starts. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct RestoreDeadlines { - /// The restore-to-Ready deadline: how long after the clone's launch its Ready may come, which - /// bounds the wait for the restore-ready event, the connections to the clone's control - /// endpoint, the adapter's attachments, and the Activates. - pub ready: Duration, - /// The Ready-to-Start deadline: how long after the check of the clone's Ready its sandbox may - /// start. - pub start: Duration, -} - -impl Default for RestoreDeadlines { - /// Returns the default deadlines: [`RESTORE_TO_READY_TIMEOUT`] and [`READY_TO_START_TIMEOUT`]. - fn default() -> Self { - Self { - ready: RESTORE_TO_READY_TIMEOUT, - start: READY_TO_START_TIMEOUT, - } - } -} - -/// The adapter's framing of a clone's control session, through which the clone's restore -/// sequence, [`Restoring::activate`], activates the clone and starts its sandbox, over the -/// session of [`nvxs`](crate::host::nvxs), say. -pub trait CloneControl { - /// A session of the adapter's framing over one connection to the clone's control endpoint. - type Session; - /// A failure of the adapter's framing or of its session. - type Error: std::error::Error + Send + Sync + 'static; - - /// Attaches a session of the adapter's framing over `transport`, a fresh connection to the - /// clone's control endpoint, with the launch's `capability`, and waits until `deadline` while - /// the session's broker waits for the guest, as - /// [`HostConnection::attach`](crate::host::nvxs::HostConnection::attach) does. Sends no - /// request. - fn attach( - &mut self, - transport: Box, - capability: &[u8; CAPABILITY_LEN], - deadline: Instant, - ) -> Result; - - /// Sends `record`, the clone's activation record in its canonical encoding, in the framing's - /// Activate, as the first request of `session`, and waits until `deadline` for the clone's - /// Ready. Returns the activation that Ready reports, or `None` for a Ready that reports - /// none, once the adapter checked its own identities in Ready, such as its build, and - /// refused a Ready that does not match them. - fn activate( - &mut self, - session: &mut Self::Session, - record: &[u8], - deadline: Instant, - ) -> Result, Self::Error>; - - /// Sends `table`, the clone's mapping table, which its record announces, over `session`, once - /// the clone is ready, as the framing sends a mapping table: in order, in as many requests as - /// it takes, each once the guest applied the one before; and waits until `deadline` for the - /// guest to apply every mapping. The restore sequence calls it only for a clone with an export, - /// whose table holds 1 to [`MAX_MAPPINGS`](crate::activation::MAX_MAPPINGS) mappings, as - /// [`ClonePlan::table`](super::ClonePlan::table) returns them, and never again in a session. - /// - /// Fails once the guest refused or failed a mapping, after which a guest starts no sandbox, as - /// does a guest that applied some of the table, and the framing sends no more of it, and once - /// the session failed: the sequence then ends the launch, and never starts the sandbox. - fn map( - &mut self, - session: &mut Self::Session, - table: &[HostMap], - deadline: Instant, - ) -> Result<(), Self::Error>; - - /// Starts the clone's sandbox over `session`, once the clone is ready, and applied its mapping - /// table if it has one: sends the start, and waits until `deadline` for its confirmation. - fn start(&mut self, session: &mut Self::Session, deadline: Instant) -> Result<(), Self::Error>; - - /// Returns whether `error`, with which [`attach`](Self::attach) or - /// [`activate`](Self::activate) failed, lost the session, as a reset of the session, the end - /// of its stream, or a failure of its transport do, rather than another failure, such as the - /// guest's refusal or the passing of the deadline: a clone that answers no more in a lost - /// session may still answer in another. - fn lost(&self, error: &Self::Error) -> bool; - - /// Returns whether `error`, with which [`map`](Self::map) failed, is the guest's refusal of - /// the clone's mapping table: its answer that it would not, or could not, apply a part of the - /// table, or one of its mappings, to the clone's root, which the request's mappings and image - /// decide, rather than a failure of the session or of the framing. The restore sequence - /// attributes such a failure to the request's mappings, - /// [`Evidence::InvalidMappings`](super::Evidence::InvalidMappings), whenever it came, and any - /// other to what the host saw of it, as [`Activated::start`] describes. - /// - /// No failure is, by default: a framing whose answers do not tell the guest's refusal of a - /// table leaves the provenance of its failures unknown. - fn refused(&self, error: &Self::Error) -> bool { - let _ = error; - false - } -} - -impl<'s> Restoring<'s> { - /// Runs the clone's restore sequence until the clone is ready, through the adapter's framing - /// of the clone's control session, `control`, under the restore-to-Ready deadline, - /// `deadlines.ready` after the launch, [`ClonePlan::launch`](super::ClonePlan::launch), in - /// order: - /// - /// 1. Waits for OpenVMM's restore-ready event, [`RESTORE_READY_EVENT`], at the endpoint at - /// which the launch listens, [`ready_endpoint`](Self::ready_endpoint), and reads it as soon - /// as it comes: OpenVMM connects to the endpoint before the restored VM runs, writes the - /// event once the restored VM has started, after which, on Windows, it waits until the - /// host has read the event, and then closes its end, which carries nothing else. The wait - /// refuses a connection of another process than the VMM, and bytes that differ from the - /// event, reads nothing past the event, and ends once the VMM has exited. - /// 2. Reads the clone's activation record again where the launch wrote it, - /// [`CLONE_ACTIVATION_NAME`](crate::host::store::CLONE_ACTIVATION_NAME) in the sandbox, - /// which must still be, byte for byte, the record of the launch, - /// [`activation`](Self::activation), in its canonical encoding: the only record that the - /// clone's restore sends. - /// 3. Connects to the clone's control endpoint, [`Starting::endpoint`], which the VMM serves, - /// as [`session::connect`] does, only after the event: the VMM's control console bounds the - /// host's authentication from its connection on, which the restore may hold up until its - /// event. - /// 4. Attaches the adapter's session over the connection with the launch's capability, - /// [`capability`](Self::capability), which the VMM authenticates, and sends the record in - /// the adapter's Activate, as the session's first request. - /// 5. Checks the activation that the clone's Ready reports, after the adapter checked its own - /// identities in it: the transaction ID of the record's restore section, as - /// [`ReadyActivation::of`] returns it. - /// - /// The sequence records each step that the clone reaches in its progress log, as - /// [`CloneRecords`](super::CloneRecords) describes: the event; its acknowledgment, which the - /// attachment of a session shows, since OpenVMM reads none of the host's input, its - /// authentication included, until the guest's repair acknowledged a restore that gates the - /// input; the Activate; and the Ready. - /// - /// Returns the [`Activated`] clone, whose Ready-to-Start deadline, `deadlines.start`, starts - /// at the check of its Ready, and whose sandbox [`Activated::start`] starts, once the clone - /// applied its mapping table, if it has an export, after which [`Started::commit`] commits the - /// launch, as the cold boot of a sandbox commits its launch once it checked its VM, with - /// [`Starting::commit`]. - /// - /// # Retries - /// - /// A clone answers an Activate of its record, byte for byte, with the same Ready, without - /// activating again, also in a later session once it is ready, and refuses any other Activate - /// for good, as does the reset of its session before it is ready. So once the adapter lost the - /// session, as [`CloneControl::lost`] tells, while it attached it or awaited the clone's - /// Ready, the sequence waits briefly, reads the record again, connects again, and attaches a - /// new session, in which it sends the record again, byte for byte, and checks the Ready again: - /// a clone that was ready answers it, and a clone that took the reset of the lost session - /// before then has ended, so that the connection fails once its VMM has exited. It does so - /// until the restore-to-Ready deadline, which no retry extends, in at most - /// [`MAX_ACTIVATION_SESSIONS`] sessions in all, and never after another failure: a refusal, a - /// Ready that does not match the record, a record that changed, and the passing of the - /// deadline are final. - /// - /// # Failures - /// - /// A step that fails ends the launch, as [`abandon`](Self::abandon) does, which terminates the - /// VMM through its verified identity, and ends the launch once nothing of it runs, and fails - /// with a [`CloneFailure`], which the sandbox records, at [`Stage::Restore`] until the - /// restore-ready event and at [`Stage::Activation`] after it, with the step's error and the - /// evidence below, unless OpenVMM's outcome report of a VMM that exited gives a status of the - /// guest's that is evidence, as [`status_evidence`](super::status_evidence) tells, which - /// prevails: - /// - /// - [`ErrorKind::TimedOut`](crate::ErrorKind::TimedOut) once the restore-to-Ready deadline - /// passed, [`Evidence::Timeout`]; - /// - [`ErrorKind::Exited`](crate::ErrorKind::Exited) once the VMM exited before its event, or - /// before a connection, and once it closed the event's endpoint without a byte of the event, - /// and then exited, as OpenVMM does as it fails to launch its VM worker, within ten seconds - /// of the close, [`Evidence::LostStatus`], and, for a VMM that exited before its - /// event, the error that OpenVMM reported, as its log ends with it, after the error's - /// message, and [`Evidence::IncompatibleHost`] for an error whose time ABI code says that - /// the host cannot present the template's CPU profile, TSC, or LAPIC timer, as [the - /// attribution of evidence](Evidence#attribution) describes; - /// - [`ErrorKind::Protocol`](crate::ErrorKind::Protocol) for a connection to the restore-ready - /// endpoint of another process than the VMM, bytes there that differ from the event, an - /// endpoint that the VMM closed after a part of the event, or without a byte of it and then - /// ran on for ten seconds, a control endpoint that another process serves, a failure of the - /// adapter's framing, and a Ready that reports another activation than the record's, or - /// none, [`Evidence::Protocol`], and for a session that the adapter lost in the last of the - /// sessions, [`Evidence::Reset`]; - /// - [`ErrorKind::Io`](crate::ErrorKind::Io) if the endpoints fail otherwise, if the sandbox's - /// activation record cannot be read, or is no longer the launch's, and if the progress log - /// cannot be appended to, the host's evidence, as [`ClonePlan::new`](super::ClonePlan::new) - /// describes it; - /// - [`ErrorKind::Unsupported`](crate::ErrorKind::Unsupported) for a deadline beyond the - /// host's clock, [`Evidence::HostFailure`]. - /// - /// The failure's teardown says whether the launch could be ended, and the error says so if it - /// could not: its journal then remains for - /// [`LockedStore::recover`](crate::host::store::LockedStore::recover), and the lease with it, - /// as [Crashes](crate::host::supervisor#crashes) describes. - pub fn activate( - self, - control: &mut C, - deadlines: RestoreDeadlines, - ) -> Result, CloneFailure> { - let Self { - starting, - launched, - ready: _, - mut listener, - image, - activation, - record, - capability, - mut tracker, - table, - } = self; - let Some(deadline) = launched.checked_add(deadlines.ready) else { - let fault = beyond_clock("restore-to-Ready", deadlines.ready); - return Err(tracker.fail(starting, Stage::Restore, fault)); - }; - if let Err(fault) = wait_event(&mut listener, &starting, deadline, deadlines.ready) { - return Err(tracker.fail(starting, Stage::Restore, fault)); - } - // OpenVMM sends one event, and the endpoint takes nothing more. - drop(listener); - if let Err(fault) = tracker.reach(Progress::Event) { - return Err(tracker.fail(starting, Stage::Activation, fault)); - } - let activation_session = ActivationSession { - capability: &capability, - record: &record, - encoded: activation.encode(), - expected: ReadyActivation::of(&activation), - deadline, - timeout: deadlines.ready, - since: "its launch", - }; - let endpoint = starting.endpoint().to_owned(); - let reference = starting.process().reference().clone(); - let activated = - activation_session.activate(control, &endpoint, &reference, Some(&mut tracker)); - let session = match activated { - Ok(session) => session, - Err(fault) => return Err(tracker.fail(starting, Stage::Activation, fault)), - }; - let Some(start) = Instant::now().checked_add(deadlines.start) else { - drop(session); - let fault = beyond_clock("Ready-to-Start", deadlines.start); - return Err(tracker.fail(starting, Stage::Start, fault)); - }; - Ok(Activated { - session, - starting, - deadline: start, - image, - activation, - tracker, - table, - }) - } -} - -/// A clone that its restore sequence activated, as [`Restoring::activate`] returns it: its guest -/// answered the Activate of its record with a Ready that reports the record's activation, and -/// awaits its mapping table, if it has an export, and the start of its sandbox, which -/// [`start`](Self::start) sends, by the Ready-to-Start deadline, [`deadline`](Self::deadline). -/// -/// It holds the adapter's session in which the clone answered, and the launch, uncommitted, -/// through which it borrows the sandbox's locked store exclusively, as [`Starting`] does. -/// Dropping it abandons the launch, as [`abandon`](Self::abandon) does, ignoring failures; a crash -/// of the caller leaves the launch's journal instead, for -/// [`LockedStore::recover`](crate::host::store::LockedStore::recover). -pub struct Activated<'s, S> { - /// The adapter's session, in which the clone answered its Activate. - session: S, - /// The launch, whose journal records the VMM. - starting: Starting<'s>, - /// The Ready-to-Start deadline. - deadline: Instant, - /// The clone's image, open for reading, which the plan checked. - image: CheckedDistro, - /// The clone's activation record. - activation: ActivationRecord, - /// How far the clone got, and how its failures are recorded. - tracker: Tracker, - /// The clone's mapping table, which [`start`](Self::start) sends before the start: empty for - /// a clone without an export. - table: Vec, -} - -impl<'s, S> Activated<'s, S> { - /// Returns the launch, whose VMM runs, and whose journal records its identity. - pub fn starting(&self) -> &Starting<'s> { - &self.starting - } - - /// Returns the adapter's session, in which the clone answered its Activate. - pub fn session(&mut self) -> &mut S { - &mut self.session - } - - /// Returns the Ready-to-Start deadline, by which the clone's sandbox must start. - pub fn deadline(&self) -> Instant { - self.deadline - } - - /// Returns the clone's activation record, with which the clone is ready. - pub fn activation(&self) -> &ActivationRecord { - &self.activation - } - - /// Returns the clone's image, open for reading, which the plan checked. - pub fn image(&self) -> &CheckedDistro { - &self.image - } - - /// Starts the clone's sandbox through the adapter's framing, `control`, in the session in - /// which the clone answered, by the Ready-to-Start deadline, and returns the [`Started`] - /// clone, whose launch the caller commits. The progress log records that the start was sent, - /// then that it was confirmed. - /// - /// A clone with an export first gets its mapping table, [`ClonePlan::table`], which its - /// record announces: [`CloneControl::map`] sends it in order, and returns once the guest - /// applied every mapping. Only then does the progress log record that the start was sent, and - /// [`CloneControl::start`] send it, which a clone without an export gets at once. Neither is - /// sent again: a guest that applied some mappings starts no sandbox, and a guest that started - /// its sandbox refuses another start, so a session that fails once the table or the start was - /// sent leaves the clone's state unknown. - /// - /// A failure of either, a partial or failed mapping table included, ends the launch, as - /// [`abandon`](Self::abandon) does, and fails with a [`CloneFailure`] at [`Stage::Start`], as - /// [`Restoring::activate`] does. A failure of the table leaves the clone's progress at - /// [`Progress::Ready`], since the progress log records the start only once the table is - /// applied, so that its record tells it from a failure of the start, whose progress is - /// [`Progress::Starting`]. The failure's evidence is the guest's refusal of the table, - /// [`Evidence::InvalidMappings`], once the adapter's framing tells it, as - /// [`CloneControl::refused`] does; otherwise it fails with - /// [`ErrorKind::TimedOut`](crate::ErrorKind::TimedOut) once the deadline passed, - /// [`Evidence::Timeout`], and with [`ErrorKind::Protocol`](crate::ErrorKind::Protocol) - /// before, [`Evidence::Reset`] for a session that the adapter lost and - /// [`Evidence::Protocol`] otherwise, unless OpenVMM's outcome report of a VMM that exited - /// gives evidence that prevails; and with the host's evidence if the progress log cannot be - /// appended to. None of these is the template's, so no failure of the table or of the start - /// quarantines the template's generation. - /// - /// [`ClonePlan::table`]: super::ClonePlan::table - pub fn start>( - self, - control: &mut C, - ) -> Result, CloneFailure> { - let Self { - mut session, - starting, - deadline, - image, - activation, - mut tracker, - table, - } = self; - match order(control, &mut session, &mut tracker, &table, deadline) { - Ok(()) => Ok(Started { - session, - starting, - image, - activation, - tracker, - }), - // The one place where the order of a clone that is ready ends its launch: a mapping - // table that the guest did not apply in whole, a start that failed, or a progress log - // that could not record them. - Err(fault) => { - drop(session); - Err(tracker.fail(starting, Stage::Start, fault)) - } - } - } - - /// Ends the launch, as [`Starting::abandon`] does, which terminates its VMM and ends the - /// launch once nothing of it runs, once the adapter's session has ended. - pub fn abandon(self) -> Result { - let Self { - session, starting, .. - } = self; - drop(session); - starting.abandon() - } - - /// Fails the clone, once it is ready, for a step of the caller's own before the start of its - /// sandbox, such as one that prepares the host's side of the clone, with `evidence`, - /// which attributes the failure as [the attribution of evidence](Evidence#attribution) - /// describes, and `error`, and returns the failure: once the adapter's session has ended, it - /// ends the launch as [`abandon`](Self::abandon) does, and records the failure at - /// [`Stage::Start`], before its teardown and once its teardown ended, as the restore sequence - /// records its own failures, so that a status of the guest's that is evidence, which - /// OpenVMM's outcome report gives once the VMM has exited, prevails over `evidence`, unless - /// `evidence` is of the template, such as a defect of the generation that the caller - /// diagnosed, whose record then holds the status beside it. - pub fn fail(self, evidence: Evidence, error: Error) -> CloneFailure { - let Self { - session, - starting, - tracker, - .. - } = self; - drop(session); - tracker.fail(starting, Stage::Start, Fault::new(evidence, error)) - } -} - -impl fmt::Debug for Activated<'_, S> { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("Activated") - .field("starting", &self.starting) - .field("deadline", &self.deadline) - .field("image", &self.image) - .field("activation", &self.activation) - .field("table", &self.table) - .finish_non_exhaustive() - } -} - -/// Runs the order of a clone that is ready, as [`Activated::start`] describes, over `session` of -/// `control`, by `deadline`: sends the mapping table, `table`, if it has a mapping, then records in -/// `tracker` that the start was sent, sends it, and records that it was confirmed. Returns the -/// fault of the step that failed, which ends the order. -fn order( - control: &mut C, - session: &mut C::Session, - tracker: &mut Tracker, - table: &[HostMap], - deadline: Instant, -) -> std::result::Result<(), Fault> { - if !table.is_empty() { - control - .map(session, table, deadline) - .map_err(|source| Step::Map.fault(control, source, deadline))?; - } - tracker.reach(Progress::Starting)?; - control - .start(session, deadline) - .map_err(|source| Step::Start.fault(control, source, deadline))?; - tracker.reach(Progress::Started) -} - -/// A step of the order of a clone that is ready, which [`Activated::start`] takes in turn. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum Step { - /// The mapping table, which [`CloneControl::map`] sends. - Map, - /// The start, which [`CloneControl::start`] sends. - Start, -} - -impl Step { - /// Returns the fault of the step's failure with `source`, which `control` returned, by the - /// Ready-to-Start deadline `deadline`, as [`Activated::start`] attributes it: the guest's - /// refusal of the mapping table is the request's mappings, whenever it came, and any other - /// failure is what the host saw of it, the passing of the deadline, a lost session, or a - /// broken protocol. - fn fault(self, control: &C, source: C::Error, deadline: Instant) -> Fault { - let (evidence, error) = if self == Self::Map && control.refused(&source) { - ( - Evidence::InvalidMappings, - Error::protocol("the clone refused its mapping table"), - ) - } else if Instant::now() >= deadline { - ( - Evidence::Timeout, - Error::timed_out(match self { - Self::Map => { - "the clone did not apply its mapping table by its Ready-to-Start deadline" - } - Self::Start => { - "the clone's sandbox did not start by its Ready-to-Start deadline" - } - }), - ) - } else if control.lost(&source) { - ( - Evidence::Reset, - Error::protocol(match self { - Self::Map => { - "the clone's control session was lost: the clone did not apply its \ - mapping table" - } - Self::Start => { - "the clone's control session was lost: cannot start the clone's sandbox" - } - }), - ) - } else { - ( - Evidence::Protocol, - Error::protocol(match self { - Self::Map => "the clone did not apply its mapping table", - Self::Start => "cannot start the clone's sandbox", - }), - ) - }; - Fault::new(evidence, error.with_source(source)) - } -} - -/// A clone whose sandbox started, as [`Activated::start`] returns it, whose launch the caller -/// [commits](Self::commit), once it checked anything else that it requires of a launch, or -/// [abandons](Self::abandon), or [fails](Self::fail), if a check of its own failed. -/// -/// It holds the adapter's session, and the launch, as [`Activated`] does. Dropping it abandons the -/// launch, as [`abandon`](Self::abandon) does, ignoring failures; a crash of the caller leaves the -/// launch's journal instead, for -/// [`LockedStore::recover`](crate::host::store::LockedStore::recover). -pub struct Started<'s, S> { - /// The adapter's session, in which the clone's sandbox started. - session: S, - /// The launch, whose journal records the VMM. - starting: Starting<'s>, - /// The clone's image, open for reading, which the plan checked. - image: CheckedDistro, - /// The clone's activation record. - activation: ActivationRecord, - /// How far the clone got, and how its failures are recorded. - tracker: Tracker, -} - -impl<'s, S> Started<'s, S> { - /// Returns the launch, whose VMM runs, and whose journal records its identity. - pub fn starting(&self) -> &Starting<'s> { - &self.starting - } - - /// Returns the adapter's session, in which the clone's sandbox started. - pub fn session(&mut self) -> &mut S { - &mut self.session - } - - /// Returns the clone's activation record, with which the clone is ready. - pub fn activation(&self) -> &ActivationRecord { - &self.activation - } - - /// Returns the clone's image, open for reading, which the plan checked. - pub fn image(&self) -> &CheckedDistro { - &self.image - } - - /// Commits the launch, as [`Starting::commit`] does, which writes its runtime record, and - /// returns the VM, with the adapter's session. The clone's image closes, and the progress log - /// records the commit, if it can: the runtime record holds it while the VM runs. - /// - /// Fails as [`Starting::commit`] fails, which ends the VMM through its handle and keeps the - /// launch's journal, with a [`CloneFailure`] at [`Stage::Commit`], whose evidence is the - /// host's, and whose teardown is [`Teardown::Unresolved`](super::Teardown::Unresolved) until - /// a reattach ends the launch that the journal names, and releases the lease's hold. - pub fn commit(self) -> Result<(Vmm, S), CloneFailure> { - let Self { - session, - starting, - mut tracker, - .. - } = self; - let committed = starting.commit_reporting(|error| { - // The launch's journal remains, whether or not the failure can be recorded. - let _ = tracker.pending(Stage::Commit, host_evidence(error), error); - }); - match committed { - Ok(vmm) => { - tracker.note(Progress::Committed); - Ok((vmm, session)) - } - Err(error) => { - drop(session); - let teardown = super::Teardown::Unresolved( - "the clone's VMM ended through its handle, and the launch's journal remains \ - until a reattach ends the launch" - .to_owned(), - ); - Err(tracker.record(Stage::Commit, Fault::host(error), teardown)) - } - } - } - - /// Ends the launch, as [`Activated::abandon`] does. - pub fn abandon(self) -> Result { - let Self { - session, starting, .. - } = self; - drop(session); - starting.abandon() - } - - /// Fails the clone, once its sandbox started, for a check of the caller's own before the - /// commit of its launch, with `evidence` and `error`, as [`Activated::fail`] does, at - /// [`Stage::Commit`]. - pub fn fail(self, evidence: Evidence, error: Error) -> CloneFailure { - let Self { - session, - starting, - tracker, - .. - } = self; - drop(session); - tracker.fail(starting, Stage::Commit, Fault::new(evidence, error)) - } -} - -impl fmt::Debug for Started<'_, S> { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter - .debug_struct("Started") - .field("starting", &self.starting) - .field("image", &self.image) - .field("activation", &self.activation) - .finish_non_exhaustive() - } -} - -/// What every session of a clone's activation takes: the launch's capability, the sandbox's -/// activation record, and the activation that its Ready must report, by the restore-to-Ready -/// deadline, or by the deadline of a reattach. -pub(super) struct ActivationSession<'a> { - /// The capability of the launch. - pub(super) capability: &'a [u8; CAPABILITY_LEN], - /// The path of the activation record in the sandbox. - pub(super) record: &'a Path, - /// The canonical encoding of the launch's activation record. - pub(super) encoded: Vec, - /// The activation that the clone's Ready must report. - pub(super) expected: ReadyActivation, - /// The deadline by which the clone must be ready. - pub(super) deadline: Instant, - /// The deadline's duration from its start. - pub(super) timeout: Duration, - /// What starts the deadline, as messages say: `its launch`, or `its reattach`. - pub(super) since: &'static str, -} - -/// How a session of a clone's activation failed. -enum Attempt { - /// The adapter lost the session, after which another session may find the clone ready. - Lost(Fault), - /// Any other failure, which is final. - Failed(Fault), -} - -impl ActivationSession<'_> { - /// Activates the clone whose VMM, `reference`, serves its control endpoint at `endpoint`, in - /// sessions of `control`, as [`Restoring::activate`] describes: in a new session after a - /// session that the adapter lost, in [`MAX_ACTIVATION_SESSIONS`] at most, and returns the - /// session in which the clone answered with its Ready. A launch records the clone's progress - /// in its `tracker`. - pub(super) fn activate( - &self, - control: &mut C, - endpoint: &str, - reference: &ProcessRef, - mut tracker: Option<&mut Tracker>, - ) -> std::result::Result { - let mut sessions = 0; - loop { - sessions += 1; - match self.run(control, endpoint, reference, tracker.as_deref_mut()) { - Ok(session) => return Ok(session), - Err(Attempt::Lost(_)) if sessions < MAX_ACTIVATION_SESSIONS => { - let left = self.deadline.saturating_duration_since(Instant::now()); - thread::sleep(SESSION_RETRY_INTERVAL.min(left)); - } - Err(Attempt::Lost(fault) | Attempt::Failed(fault)) => return Err(fault), - } - } - } - - /// Activates the clone in a new session of `control`, as [`activate`](Self::activate) - /// describes, records its progress in `tracker`, if it has one, and returns the session once - /// the clone is ready. - fn run( - &self, - control: &mut C, - endpoint: &str, - reference: &ProcessRef, - mut tracker: Option<&mut Tracker>, - ) -> std::result::Result { - let record = self.read_record().map_err(Attempt::Failed)?; - let transport = self.connect(endpoint, reference).map_err(Attempt::Failed)?; - let attached = control.attach(transport, self.capability, self.deadline); - let mut session = - attached.map_err(|error| self.failed(control, error, "attach its control session"))?; - if let Some(tracker) = &mut tracker { - // OpenVMM authenticated the session, which it does only once the guest acknowledged - // a restore that gates the host's input. The Activate follows at once, with which - // the progress log records the acknowledgment. - tracker.advance(Progress::Acknowledged); - tracker - .reach(Progress::Activating) - .map_err(Attempt::Failed)?; - } - let reported = control - .activate(&mut session, &record, self.deadline) - .map_err(|error| self.failed(control, error, "activate it"))?; - self.check(reported).map_err(Attempt::Failed)?; - if let Some(tracker) = &mut tracker { - tracker.reach(Progress::Ready).map_err(Attempt::Failed)?; - } - Ok(session) - } - - /// Connects to the control endpoint `endpoint`, which the VMM `reference` serves, as - /// [`session::connect`] does, by the deadline. - fn connect( - &self, - endpoint: &str, - reference: &ProcessRef, - ) -> std::result::Result, Fault> { - session::connect(endpoint, reference.pid, reference.start_time, self.deadline).map_err( - |error| { - let message = format!("cannot connect to the clone's control endpoint {endpoint}"); - match error { - SessionError::TimedOut => Fault::new( - Evidence::Timeout, - Error::timed_out(format!( - "{message} within {:?} of {}", - self.timeout, self.since - )), - ), - SessionError::ProcessExited => Fault::new( - Evidence::LostStatus, - Error::exited(format!("{message}: its VMM exited")), - ), - SessionError::Io(source) => Fault::host(Error::io(message).with_source(source)), - error => Fault::new( - Evidence::Protocol, - Error::protocol(message).with_source(error), - ), - } - }, - ) - } - - /// Reads the activation record in the sandbox, which must be, byte for byte, the encoding of - /// the launch's record, and returns it. - fn read_record(&self) -> std::result::Result, Fault> { - let path = self.record; - let bytes = open_regular(path) - .and_then(|file| read_at_most(file, MAX_RECORD_BYTES as u64, "an activation record")) - .map_err(|error| { - Fault::host( - Error::io(format!( - "cannot read the clone's activation record {}", - path.display() - )) - .with_source(error), - ) - })?; - if bytes != self.encoded { - return Err(Fault::new( - Evidence::HostFailure, - Error::io(format!( - "{} no longer holds the activation record that the clone's launch wrote, the \ - only record that its restore sends", - path.display() - )), - )); - } - Ok(bytes) - } - - /// Checks `reported`, the activation that the clone's Ready reports, against the record's. - fn check(&self, reported: Option) -> std::result::Result<(), Fault> { - check_ready(&self.expected, reported).attribute(Evidence::Protocol) - } - - /// Returns how a session failed with `error`, which `control` returned as it tried to `what`. - fn failed(&self, control: &C, error: C::Error, what: &str) -> Attempt { - if Instant::now() >= self.deadline { - return Attempt::Failed(Fault::new( - Evidence::Timeout, - Error::timed_out(format!( - "the clone was not ready within {:?} of {}: cannot {what}", - self.timeout, self.since - )) - .with_source(error), - )); - } - if control.lost(&error) { - let error = Error::protocol(format!( - "the clone's control session was lost: cannot {what}" - )) - .with_source(error); - Attempt::Lost(Fault::new(Evidence::Reset, error)) - } else { - let error = Error::protocol(format!("cannot {what}")).with_source(error); - Attempt::Failed(Fault::new(Evidence::Protocol, error)) - } - } -} - -/// Checks `reported`, the activation that a clone's Ready reports, against `expected`, its -/// record's: the transaction ID of the record's restore section. -fn check_ready(expected: &ReadyActivation, reported: Option) -> Result<()> { - let Some(reported) = reported else { - return Err(Error::protocol( - "the clone's Ready reports no activation, as a VM that boots cold reports none", - )); - }; - if reported == *expected { - return Ok(()); - } - let transaction = |activation: &ReadyActivation| { - activation - .transaction - .map_or_else(|| "none".to_owned(), |transaction| transaction.to_string()) - }; - Err(Error::protocol(format!( - "the clone's Ready reports the transaction {}, not its record's transaction {}", - transaction(&reported), - transaction(expected) - ))) -} - -/// Waits until `deadline`, `timeout` after the launch, for the restore-ready event of the VMM of -/// `starting` at `listener`, as [`Restoring::activate`] describes. -fn wait_event( - listener: &mut ReadyListener, - starting: &Starting<'_>, - deadline: Instant, - timeout: Duration, -) -> std::result::Result<(), Fault> { - let process = starting.process(); - let pid = process.reference().pid; - // OpenVMM writes the event once, and then closes its end, so the wait reads the event's bytes - // and nothing past them. - let length = RESTORE_READY_EVENT.len(); - let mut received = Vec::with_capacity(length); - while received.len() < length { - let left = deadline.saturating_duration_since(Instant::now()); - if left.is_zero() { - return Err(Fault::new( - Evidence::Timeout, - Error::timed_out(format!( - "the clone's VMM sent no restore-ready event within {timeout:?} of its launch" - )), - )); - } - let before = received.len(); - let open = listener - .receive(pid, &mut received, length, left.min(EXIT_CHECK_INTERVAL)) - .map_err(|error| { - let message = "cannot receive the restore-ready event of the clone's VMM"; - if error.kind() == io::ErrorKind::PermissionDenied { - Fault::new( - Evidence::Protocol, - Error::protocol(message).with_source(error), - ) - } else { - Fault::host(Error::io(message).with_source(error)) - } - })?; - if !RESTORE_READY_EVENT.starts_with(&received) { - return Err(Fault::new( - Evidence::Protocol, - Error::protocol(format!( - "the clone's VMM sent {:?} at its restore-ready endpoint, which is not \ - OpenVMM's restore-ready event", - String::from_utf8_lossy(&received) - )), - )); - } - if received.len() == length { - break; - } - if !open { - if received.is_empty() { - let wait = deadline.min(Instant::now() + CLOSED_EXIT_WAIT); - let exited = process.wait_bool(wait).map_err(|error| { - Fault::host( - Error::io("cannot tell whether the clone's VMM still runs") - .with_source(error), - ) - })?; - if exited { - return Err(Fault::new( - Evidence::LostStatus, - Error::exited( - "the clone's VMM closed its restore-ready endpoint without the event, \ - and exited, as OpenVMM does once it cannot restore the snapshot", - ), - )); - } - } - return Err(Fault::new( - Evidence::Protocol, - Error::protocol(format!( - "the clone's VMM closed its restore-ready endpoint after {} bytes of the event", - received.len() - )), - )); - } - if received.len() == before { - let exited = process.has_exited().map_err(|error| { - Fault::host( - Error::io("cannot tell whether the clone's VMM still runs").with_source(error), - ) - })?; - if exited { - return Err(Fault::new( - Evidence::LostStatus, - Error::exited( - "the clone's VMM exited before its restore-ready event, as OpenVMM does \ - once it cannot restore the snapshot", - ), - )); - } - } - } - Ok(()) -} - -/// Returns the fault of the deadline `name`, `timeout` after its start, which this host's clock -/// cannot represent. -fn beyond_clock(name: &str, timeout: Duration) -> Fault { - Fault::new( - Evidence::HostFailure, - Error::unsupported(format!( - "the {name} deadline of a clone's restore, {timeout:?} after its start, is beyond \ - this host's clock" - )), - ) -} diff --git a/nvx-rs/src/host/template/clone/tests.rs b/nvx-rs/src/host/template/clone/tests.rs deleted file mode 100644 index a3ac200ac..000000000 --- a/nvx-rs/src/host/template/clone/tests.rs +++ /dev/null @@ -1,3037 +0,0 @@ -//! Tests of the plans of clones: the checks of their images, uppers, exports, and processor and -//! memory targets against the generation that they lease, which read no content, the claims of -//! their scratch images, and the uppers in RAM, whose scratch devices their restores declare -//! absent; in [`launch`], of their launches; and in [`restore`], of their restore sequences. - -mod launch; -mod restore; - -use std::io::Read; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; -use std::time::Duration; - -use super::*; -use crate::ErrorCode; -use crate::activation::{ - Configuration, ExpectedResources, Export, ExportKind, Image, MachineIdMode, Network, - Publication, Upper, -}; -use crate::host::filesystem; -use crate::host::network::{ - Access as NetworkAccess, EgressRule, ForwardProtocol, HostLoopback, LoopbackForward, - NetworkPolicy, NetworkProxy, Protocol, ProxyScheme, -}; -use crate::host::paths; -use crate::host::sandbox::distro::{DISTRO_POLICY_REVISION, PolicyVerdict}; -use crate::host::sandbox::scratch::CLAIM_RELEASE_WAIT; -use crate::host::sandbox::{SandboxDistro, SandboxNetwork, SandboxSettings, SandboxUpper}; -use crate::host::seal::FileSeal; -use crate::host::seal::tests::open_metadata_only; -use crate::host::snapshot::tests::{MIB, ManifestSpec, class, network_class}; -use crate::host::store::{ - CLONE_ACTIVATION_NAME, CLONE_BINDING_NAME, CLONE_SCRATCH_NAME, CallerFiles, LOG_NAME, - SandboxStore, -}; -use crate::host::template::store::{CAPTURES_NAME, SNAPSHOT_NAME, TRANSACTION_NAME}; -use crate::host::template::{CaptureClass, HostTime, KEY_NAME}; -#[cfg(windows)] -use crate::host::tests::junction; -use crate::host::tests::released; -use crate::protocol::root::RootImage; -use crate::uuid::Uuid; - -/// The UUID of the staged images. -const UUID: &str = "11111111-1111-1111-1111-111111111111"; -/// The configuration record of these tests' sandboxes. -const CONFIG: &str = "config.json"; -/// The length of the template's placeholder, and of the staged images: whole huge pages of 2 MiB, -/// as the length of a file on hugetlbfs is. -const IMAGE_LEN: u64 = 4 * MIB; -/// The length of the template's scratch, and of the staged scratch images: a huge page of 2 MiB. -const SCRATCH_LEN: u64 = 2 * MIB; -/// The template's processors, all online at boot, as [`ManifestSpec::template`] records them. -const PROCESSORS: u32 = 2; -/// The template's RAM, in bytes, as [`ManifestSpec::template`] records it. -const MEMORY: u64 = 512 * MIB; -/// The template's RAM capacity, in bytes, as [`ManifestSpec::template`] records it. -const MEMORY_CAPACITY: u64 = 2048 * MIB; -/// The bound of the first partition of the tests' GPT disks, which the record of a clone of an -/// EROFS layer leaves out. -const FIRST_PARTITION_BOUND: u64 = 64 * 1024; -/// Nanoseconds in a second. -const SECOND: u64 = 1_000_000_000; - -/// The access of a host that lets the plan open the clone's files and read their metadata, but -/// fails every read and mapping of their content through what the checks of their bindings -/// open: the image opened for its metadata alone, and the scratch image, on Linux, for writing -/// alone, and on Windows through its guard and the claim's handle, which never ask to read. -const NO_CONTENT: Access<'static> = Access { - distro: distro::Access { - open: &open_metadata_only, - seal: &FileSeal::of, - }, - scratch: WRITE_ONLY, -}; - -/// Opens a scratch image for writing alone, as [`NO_CONTENT`] does. -#[cfg(target_os = "linux")] -const WRITE_ONLY: scratch::Access<'static> = scratch::Access { - open: &open_write_only, -}; -/// Opens a scratch image for writing alone, as [`NO_CONTENT`] does. -#[cfg(windows)] -const WRITE_ONLY: scratch::Access<'static> = scratch::Access::WRITABLE; - -/// Opens `path` for writing alone, without waiting for a reader of a FIFO. -#[cfg(target_os = "linux")] -fn open_write_only(path: &Path) -> io::Result { - use std::os::unix::fs::OpenOptionsExt; - - fs::OpenOptions::new() - .write(true) - .custom_flags(nix::fcntl::OFlag::O_NONBLOCK.bits()) - .open(path) -} - -/// Creates the file `path` of `length` bytes, or truncates it to that length, which no one else -/// may change, whatever the umask, as staging callers keep their images. -fn stage(path: &Path, length: u64) { - let mut options = fs::OpenOptions::new(); - options.write(true).create(true).truncate(true); - #[cfg(unix)] - std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); - options.open(path).unwrap().set_len(length).unwrap(); -} - -/// Returns the physical block size that OpenVMM presents for the files of the directory `dir`: -/// the I/O block size of its filesystem on Linux, and 4096 bytes on Windows. -fn physical_block_size(dir: &Path) -> u32 { - #[cfg(target_os = "linux")] - { - use std::os::unix::fs::MetadataExt; - - let probe = dir.join("probe"); - fs::write(&probe, b"").unwrap(); - let size = fs::metadata(&probe).unwrap().blksize(); - fs::remove_file(&probe).unwrap(); - u32::try_from(size).unwrap() - } - #[cfg(windows)] - { - let _ = dir; - 4096 - } -} - -/// Returns a physical block size other than `physical` that a template may have recorded. -fn other_physical(physical: u32) -> u32 { - if physical > 512 { - physical / 2 - } else { - physical * 2 - } -} - -/// Returns the geometry of a block device of `length` bytes, with logical blocks of 512 bytes and -/// physical blocks of `physical` bytes. -fn geometry(length: u64, physical: u32) -> BlockGeometry { - BlockGeometry { - length, - logical_block_size: 512, - physical_block_size: physical, - } -} - -/// Gives the template of `class`, whose capture `spec` describes, the geometries `distro` and -/// `scratch`. -fn set_geometries( - class: &mut CaptureClass, - spec: &mut ManifestSpec, - distro: BlockGeometry, - scratch: BlockGeometry, -) { - class.distro = distro; - class.scratch = scratch; - for (block, geometry) in spec.contract().blocks.iter_mut().zip([distro, scratch]) { - block.length = geometry.length; - block.logical_block_size = geometry.logical_block_size; - block.physical_block_size = geometry.physical_block_size; - } -} - -/// Returns the files that these tests' callers keep in their sandboxes. -fn files() -> CallerFiles { - CallerFiles::new(CONFIG).unwrap() -} - -/// Gives the template of `class`, whose capture `spec` describes, a capacity of `processors` -/// processors, of which `boot` are online at boot, as a capture records them. -fn set_processors(class: &mut CaptureClass, spec: &mut ManifestSpec, processors: u32, boot: u32) { - class.processors = processors; - class.boot_processors = boot; - spec.vp_count = processors; - let contract = spec.contract(); - contract.topology = (1, 1, processors, 1, (0..processors).collect()); - contract.boot_online_vp_count = boot; -} - -/// Returns `path` as messages show it. -fn shown(path: &Path) -> String { - paths::normalized(path).display().to_string() -} - -/// Returns the distro layer at `path`. -fn layer(path: &Path) -> SandboxLayer { - SandboxLayer::new(LayerRole::Distro, path.to_str().unwrap(), UUID) -} - -/// Returns the distro GPT disk at `path`, whose disk GUID is [`UUID`]. -fn gpt_layer(path: &Path) -> SandboxLayer { - SandboxLayer::gpt(LayerRole::Distro, path.to_str().unwrap(), UUID) -} - -/// Returns the request of a clone of the image at `image`, which `binding` binds, over an upper in -/// RAM, with the template's processors and RAM as its targets, and the settings of [`view`], -/// for untrusted lowers, and with a journal policy, [`Journal::Forbidden`], that its record leaves -/// out, and for a GPT disk, the bound [`FIRST_PARTITION_BOUND`]. -fn tmpfs_request(image: &Path, binding: &DistroBinding) -> CloneRequest { - CloneRequest { - adapter: SandboxPlan::adapter( - SandboxDistro::new(image.to_str().unwrap(), bound_image(binding).unwrap()), - SandboxUpper::RAM, - ), - binding: binding.clone(), - policy_revision: DISTRO_POLICY_REVISION, - scratch: None, - export: None, - processors: PROCESSORS, - memory: MEMORY, - view: view(), - overlay: OverlayMode::UntrustedLowers, - journal: Journal::Forbidden, - first_partition_bound: FIRST_PARTITION_BOUND, - } -} - -/// Returns the view that these tests' clones give their workloads of their VMs. -fn view() -> View { - View { - hostname: b"clone".to_vec(), - machine_id: MachineIdMode::Generated, - } -} - -/// Returns the request of a clone of the image at `image`, which `binding` binds, over an upper on -/// the scratch image at `scratch`, which `scratch_binding` binds, with the template's RAM as its -/// memory target. -fn scratch_request( - image: &Path, - binding: &DistroBinding, - scratch: &Path, - scratch_binding: &ScratchBinding, -) -> CloneRequest { - let request = tmpfs_request(image, binding); - CloneRequest { - adapter: SandboxPlan::adapter( - request.adapter.distro().clone(), - SandboxUpper::scratch(scratch.to_str().unwrap()), - ), - scratch: Some(scratch_binding.clone()), - ..request - } -} - -/// Returns the request `request` with the network `network`. -fn with_network(request: CloneRequest, network: Option) -> CloneRequest { - CloneRequest { - adapter: request.adapter.clone().with_network(network), - ..request - } -} - -/// Returns the network of a guest at `10.0.0.2/24` under a policy whose default is `access`. -fn network(access: NetworkAccess) -> SandboxNetwork { - SandboxNetwork::new("10.0.0.2/24", NetworkPolicy::new(access)).unwrap() -} - -/// Returns the fixture of a template with a NIC, at `10.0.0.2/24`, whose class's network allows -/// egress. -fn network_fixture() -> Fixture { - Fixture::with(|class, spec, physical| { - *class = network_class(); - *spec = ManifestSpec::with_network(); - let (distro, scratch) = ( - geometry(IMAGE_LEN, physical), - geometry(SCRATCH_LEN, physical), - ); - set_geometries(class, spec, distro, scratch); - }) -} - -/// Returns the policies of networks that attach a NIC, a template's of each: egress allowed; -/// egress denied but to some destinations; a proxy alone; ingress allowed; and host-loopback -/// access both ways, with a forward. -fn attaching() -> Vec { - let mut ingress = NetworkPolicy::new(NetworkAccess::Deny); - ingress.ingress = NetworkAccess::Allow; - let mut forwarded = NetworkPolicy::new(NetworkAccess::Allow); - forwarded.host_loopback = HostLoopback::Allow; - forwarded.host_loopback_forwards.push(LoopbackForward { - protocol: ForwardProtocol::Tcp, - host_port: 18080, - guest_port: 8080, - }); - vec![ - NetworkPolicy::new(NetworkAccess::Allow), - NetworkPolicy::new(NetworkAccess::Deny) - .with_allow(EgressRule::to("192.0.2.0/24").on_port(Protocol::Tcp, 443)), - NetworkPolicy::proxy_only(NetworkProxy::new(3128).with_scheme(ProxyScheme::Http)), - ingress, - forwarded, - ] -} - -/// Returns the request `request` with the export `export`. -fn with_export(request: CloneRequest, export: &ExportPlan) -> CloneRequest { - CloneRequest { - export: Some(export.clone()), - ..request - } -} - -/// Plans the export of `mappings`, host paths each at its guest target, read-only or not, for the -/// container root, as an adapter's sandbox plans it, with the paths `denied` hidden, and the paths -/// `allowed` inside them shown again. -fn export_plan( - mappings: &[(&Path, &str, bool)], - denied: &[PathBuf], - allowed: &[PathBuf], -) -> ExportPlan { - let mappings: Vec> = mappings - .iter() - .map(|&(path, target, read_only)| filesystem::Mapping { - name: "mappings", - path, - target: filesystem::Target::Guest(target), - read_only, - }) - .collect(); - let request = filesystem::ExportRequest { - denied: filesystem::PathList { - name: "denied", - paths: denied, - }, - allowed: filesystem::PathList { - name: "allowed", - paths: allowed, - }, - ..filesystem::ExportRequest::new(&mappings, filesystem::ReservedTargets::CONTAINER_ROOT) - }; - filesystem::plan(&request).unwrap().unwrap() -} - -/// Returns another spelling of the existing directory `directory`, which names it, but which -/// differs from the path that resolves it: through a directory in it and its `..` on Unix, where -/// an absolute path keeps its `..` components, and with the verbatim prefix on Windows. -fn alias(directory: &Path) -> PathBuf { - #[cfg(windows)] - { - let resolved = filesystem::canonicalize(directory).unwrap(); - PathBuf::from(format!(r"\\?\{}", resolved.display())) - } - #[cfg(not(windows))] - { - let inner = directory.join("alias"); - fs::create_dir(&inner).unwrap(); - inner.join("..") - } -} - -/// Returns the mapping table of these tests' exports, [`Fixture::data_export`], as a cold launch's -/// guest receives it: the exported directory, child 0, and the directory `work` in it, each by its -/// path in the export, which names the child's index. -fn data_table() -> [HostMap; 2] { - [ - HostMap { - source: "0".to_owned(), - target: "/data".to_owned(), - read_only: true, - }, - HostMap { - source: "0/work".to_owned(), - target: "/work".to_owned(), - read_only: false, - }, - ] -} - -/// Returns the kind and the message of the error of `result`, a clone's plan that failed with -/// `evidence`, as [`refused_at`] checks it. -fn refused( - result: std::result::Result, - evidence: Evidence, -) -> (ErrorKind, String) { - refused_at(result, Stage::Plan, evidence) -} - -/// Returns the kind and the message of the error of `result`, which must have failed at `stage`, -/// the plan or the launch, with `evidence`, before anything of the launch existed: with the -/// transaction of the clone's restore once the plan drew it, and no VM to tear down. -fn refused_at( - result: std::result::Result, - stage: Stage, - evidence: Evidence, -) -> (ErrorKind, String) { - let failure = result.unwrap_err(); - let record = failure.record(); - assert_eq!( - (record.stage, record.evidence, failure.outcome()), - (stage, evidence, evidence.outcome()), - "{failure}" - ); - assert_eq!( - (record.progress, failure.teardown(), record.status), - (Progress::Planned, &Teardown::NoVm, None), - "{failure}" - ); - assert_eq!( - record.transaction.is_some(), - stage != Stage::Plan, - "{failure}" - ); - assert_eq!(record.code, failure.code()); - let error = failure.into_error(); - (error.kind(), error.to_string()) -} - -/// Returns the time of the host that captured `generation`, during the boot of the capture, -/// `seconds` after it. -fn after(generation: &Generation, seconds: u64) -> HostTime { - let time = generation.time(); - HostTime { - host_id: time.host_id, - boot_id: time.boot_id, - clock: time.host_clock, - monotonic_ns: time.capture_monotonic_ns + seconds * SECOND, - utc_ns: time.capture_utc_ns + seconds * SECOND, - } -} - -/// A template store in a temporary directory, with a published generation whose `distro` and -/// scratch have the geometries of the files of the directory, and the directories of the staged -/// images, scratch images, and sandboxes. -struct Fixture { - /// The temporary directory. - directory: tempfile::TempDir, - /// The store. - store: Store, - /// The published generation. - generation: Generation, - /// The physical block size of the files of the directory. - physical: u32, -} - -impl Fixture { - /// Publishes a template whose `distro` is [`IMAGE_LEN`] bytes long and whose scratch is - /// [`SCRATCH_LEN`] bytes long, both with the physical block size of the files of the - /// directory. - fn new() -> Self { - Self::with(|_, _, _| {}) - } - - /// Publishes a template as [`new`](Self::new) does, of the class and manifest that `change` - /// changes, given the physical block size of the files of the directory. - fn with(change: impl FnOnce(&mut CaptureClass, &mut ManifestSpec, u32)) -> Self { - Self::in_store(|directory| directory.join("templates"), change) - } - - /// Publishes a template as [`with`](Self::with) does, in a store at the path that `store` - /// returns, given the temporary directory, in which it creates the store's parent directory. - fn in_store( - store: impl FnOnce(&Path) -> PathBuf, - change: impl FnOnce(&mut CaptureClass, &mut ManifestSpec, u32), - ) -> Self { - let directory = tempfile::tempdir().unwrap(); - let physical = physical_block_size(directory.path()); - let store = Store::create(store(directory.path())).unwrap(); - let (mut class, mut spec) = (class(), ManifestSpec::template()); - set_geometries( - &mut class, - &mut spec, - geometry(IMAGE_LEN, physical), - geometry(SCRATCH_LEN, physical), - ); - change(&mut class, &mut spec, physical); - let generation = store.publish_spec_for_tests(&class, &spec); - // Staging callers keep images in directories that only their own account can write. - for name in ["images", "scratches", "sandboxes"] { - let dir = directory.path().join(name); - assert_eq!(platform::create_owner_only_dir(&dir).unwrap(), None); - } - Self { - directory, - store, - generation, - physical, - } - } - - /// Returns the path of the file `name` of the directory `kind`. - fn path(&self, kind: &str, name: &str) -> PathBuf { - self.directory.path().join(kind).join(name) - } - - /// Stages the image `name` of `length` bytes, and returns its path and its binding. - fn image(&self, name: &str, length: u64) -> (PathBuf, DistroBinding) { - let path = self.path("images", name); - stage(&path, length); - let binding = DistroBinding::import(&layer(&path), PolicyVerdict::ACCEPTED).unwrap(); - (path, binding) - } - - /// Stages the GPT disk `name` of `length` bytes, whose disk GUID is [`UUID`], as - /// [`image`](Self::image) stages an EROFS layer, and returns its path and its binding. - fn gpt_image(&self, name: &str, length: u64) -> (PathBuf, DistroBinding) { - let path = self.path("images", name); - stage(&path, length); - let binding = DistroBinding::import(&gpt_layer(&path), PolicyVerdict::ACCEPTED).unwrap(); - (path, binding) - } - - /// Stages the scratch image `name` of `length` bytes, and returns its path and its binding. - fn scratch(&self, name: &str, length: u64) -> (PathBuf, ScratchBinding) { - let path = self.path("scratches", name); - stage(&path, length); - let binding = ScratchBinding::bind(&path).unwrap(); - (path, binding) - } - - /// Creates the directory `name` among the exported directories, with the directories `inside` - /// it, and returns its path, with its links resolved, as an export plan names it. - fn exported(&self, name: &str, inside: &[&str]) -> PathBuf { - let dir = self.path("exports", name); - fs::create_dir_all(&dir).unwrap(); - for path in inside { - fs::create_dir_all(dir.join(path)).unwrap(); - } - filesystem::canonicalize(&dir).unwrap() - } - - /// Plans the export of the exported directory `name`, which the guest reads at `/data`, and - /// in which it writes the directory `work`, at `/work`, as [`data_table`] maps them: an - /// aggregate of one directory, whose writes the plan narrows to `work`. - fn data_export(&self, name: &str) -> ExportPlan { - let data = self.exported(name, &["work"]); - export_plan( - &[(&data, "/data", true), (&data.join("work"), "/work", false)], - &[], - &[], - ) - } - - /// Plans the export of an aggregate of the directory `name` among the exported directories, as - /// a cold launch exports one, and returns it with its directories and files, in the order of - /// their children: the directory `data`, which the guest reads at `/data`, but for its - /// directory `work`, which it writes at `/work`, and which hides its directory `secret`, but - /// for the directory `shown` in it; the file `notes.txt`, which the guest writes at - /// `/notes.txt`; the file `settings.json`, which it reads at `/settings.json`; and the - /// directory `tools`, which it reads at `/tools`. - fn aggregate_export(&self, name: &str) -> (ExportPlan, [PathBuf; 4]) { - let root = self.exported(name, &["data/work", "data/secret/shown", "tools"]); - let [data, notes, settings, tools] = - ["data", "notes.txt", "settings.json", "tools"].map(|entry| root.join(entry)); - for file in [¬es, &settings] { - fs::write(file, b"file").unwrap(); - } - let secret = data.join("secret"); - let export = export_plan( - &[ - (&data, "/data", true), - (&data.join("work"), "/work", false), - (¬es, "/notes.txt", false), - (&settings, "/settings.json", true), - (&tools, "/tools", true), - ], - std::slice::from_ref(&secret), - &[secret.join("shown")], - ); - (export, [data, notes, settings, tools]) - } - - /// Provisions the sandbox `name`, and returns its directory, with its links resolved, as a - /// lease names it. - fn provision(&self, name: &str) -> PathBuf { - let dir = self.path("sandboxes", name); - let locked = SandboxStore::lock_for_provision(&dir, &files(), None).unwrap(); - fs::write(dir.join(CONFIG), b"{}").unwrap(); - drop(locked); - filesystem::canonicalize(&dir).unwrap() - } - - /// Provisions the sandbox `name`, and leases the generation to it, a second after its - /// capture. - fn acquire(&self, name: &str) -> Acquired { - let sandbox = self.provision(name); - self.store - .acquire( - self.generation.generation(), - sandbox, - &files(), - &after(&self.generation, 1), - None, - ) - .unwrap() - } - - /// Plans the clone of `acquired` as `request` asks, through `access`, waiting - /// [`CLAIM_RELEASE_WAIT`] at most for the claim of its scratch image. - fn plan( - &self, - acquired: &mut Acquired, - request: &CloneRequest, - access: Access<'_>, - ) -> Result { - ClonePlan::new_with( - &self.store, - acquired, - request, - Instant::now() + CLAIM_RELEASE_WAIT, - access, - ) - } -} - -/// Returns the names of the entries of the directory `dir`, in order. -fn entries(dir: &Path) -> Vec { - let mut names: Vec = fs::read_dir(dir) - .unwrap() - .map(|entry| entry.unwrap().file_name().into_string().unwrap()) - .collect(); - names.sort(); - names -} - -/// Asserts that the sandbox of `acquired` holds no record of a clone's progress or failure, and -/// that its lease still reserves its launch, as after a plan that failed. -fn assert_untouched(acquired: &Acquired) { - let locked = &acquired.locked; - for path in [locked.clone_progress_path(), locked.clone_failure_path()] { - assert!(!path.exists(), "{}", path.display()); - } - assert_eq!(locked.reserved(), Some(acquired.lease.launch())); -} - -/// Returns a file of `length` bytes on hugetlbfs, whose I/O block size is a huge page of 2 MiB, -/// or `None`, saying why, if the host offers none. -#[cfg(target_os = "linux")] -fn huge_file(length: u64) -> Option { - use nix::sys::memfd::{MFdFlags, memfd_create}; - - let flags = MFdFlags::MFD_CLOEXEC | MFdFlags::MFD_HUGETLB | MFdFlags::MFD_HUGE_2MB; - let file = match memfd_create("nvx-clone-test", flags) { - Ok(descriptor) => File::from(descriptor), - Err(error) => { - eprintln!("this host offers no file on hugetlbfs: {error}"); - return None; - } - }; - if let Err(error) = file.set_len(length) { - eprintln!("this host cannot size a file on hugetlbfs: {error}"); - return None; - } - Some(file) -} - -/// Links `link` to the file `target` with a symbolic link, and returns whether it could: Windows -/// lets only Developer Mode or a privilege create one. -fn symlink_file(target: &Path, link: &Path) -> bool { - #[cfg(unix)] - let linked = std::os::unix::fs::symlink(target, link); - #[cfg(windows)] - let linked = std::os::windows::fs::symlink_file(target, link); - created_link(linked) -} - -/// Links `link` to the directory `target` with a symbolic link, and returns whether it could, as -/// [`symlink_file`] does. -fn symlink_dir(target: &Path, link: &Path) -> bool { - #[cfg(unix)] - let linked = std::os::unix::fs::symlink(target, link); - #[cfg(windows)] - let linked = std::os::windows::fs::symlink_dir(target, link); - created_link(linked) -} - -/// Returns whether a symbolic link was created, as `linked` reports, and fails unless Windows -/// refused it. -fn created_link(linked: io::Result<()>) -> bool { - match linked { - Ok(()) => true, - Err(error) if cfg!(windows) => { - eprintln!("cannot create a symbolic link: {error}"); - false - } - Err(error) => panic!("cannot create a symbolic link: {error}"), - } -} - -/// Links `link` to the directory `target`: a symbolic link on Unix, and a junction on Windows, -/// which any user may create. -fn link_dir(target: &Path, link: &Path) { - #[cfg(unix)] - std::os::unix::fs::symlink(target, link).unwrap(); - #[cfg(windows)] - junction(link, target); -} - -#[test] -fn a_clone_in_ram_restores_its_lease_with_its_image_and_no_scratch_file() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let sandbox = acquired.locked.dir().to_owned(); - let before = entries(&sandbox); - let request = tmpfs_request(&image, &binding); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let store = &fixture.store; - assert_eq!(plan.lease(), &acquired.lease); - assert_eq!(plan.generation(), &fixture.generation); - assert_eq!(plan.key(), &store.read_key(&fixture.generation).unwrap()); - assert_eq!( - plan.snapshot(), - store.snapshot_dir(&fixture.generation).unwrap().as_path() - ); - assert_eq!(plan.memory(), MEMORY); - // The image is the file that its binding's check opened, which reads nothing. - assert_eq!(plan.image().path().to_str().unwrap(), shown(&image)); - assert_eq!( - plan.image().geometry().unwrap(), - geometry(IMAGE_LEN, fixture.physical) - ); - let mut byte = [0]; - assert!(plan.image().file().read(&mut byte).is_err()); - // The restore declares the `scratch` block device of an upper in RAM absent, so the plan has - // no file for it, and creates nothing in the sandbox's directory. - assert!(matches!(plan.upper(), CloneUpper::Tmpfs)); - assert_eq!(plan.scratch_path(), None); - assert_eq!(entries(&sandbox), before); - // The capability is fresh, and never shown. - let capability = *plan.capability(); - assert_ne!(capability, [0; CAPABILITY_LEN]); - let shown_plan = format!("{plan:?}"); - assert!(shown_plan.starts_with("ClonePlan {"), "{shown_plan}"); - assert!( - !shown_plan.contains(&format!("{capability:?}")), - "{shown_plan}" - ); - assert!(!shown_plan.contains("capability"), "{shown_plan}"); - // The plan borrows nothing of the acquisition, whose lease still reserves its launch. - assert_eq!(acquired.locked.reserved(), Some(acquired.lease.launch())); - drop(plan); - // The access of launches plans alike, with another capability. - let again = ClonePlan::new(store, &mut acquired, &request, Instant::now()).unwrap(); - assert_ne!(*again.capability(), capability); - assert_eq!(again.scratch_path(), None); - assert_eq!(entries(&sandbox), before); -} - -#[test] -fn files_of_the_sandboxs_store_are_refused_as_images() { - let fixture = Fixture::new(); - let mut acquired = fixture.acquire("sandbox"); - let record = acquired.locked.clone_activation_path(); - // The image or the scratch image of a request is a file of the sandbox's store: the request's - // settings are invalid. - let store_file = |result: std::result::Result| { - refused(result, Evidence::InvalidSettings) - }; - let refusal = |what: &str, path: &Path, owned: &Path| { - ( - ErrorKind::Unsupported, - format!( - "the {what} {} is {}, a file of the sandbox's store, which its transitions \ - replace or remove", - shown(path), - shown(owned) - ), - ) - }; - let seal_of = |path: &Path| FileSeal::of(&File::open(path).unwrap()).unwrap(); - // An image that is the clone's activation record, which a launch would replace, is refused - // before anything replaces it... - stage(&record, IMAGE_LEN); - let binding = DistroBinding::import(&layer(&record), PolicyVerdict::ACCEPTED).unwrap(); - let staged = seal_of(&record); - assert_eq!( - store_file(fixture.plan(&mut acquired, &tmpfs_request(&record, &binding), NO_CONTENT)), - refusal("image", &record, &record) - ); - assert_eq!(seal_of(&record), staged); - // So is the scratch device's file that an earlier version left for a clone in RAM, which a - // release removes. - let left = acquired.locked.dir().join(CLONE_SCRATCH_NAME); - stage(&left, IMAGE_LEN); - let left_binding = DistroBinding::import(&layer(&left), PolicyVerdict::ACCEPTED).unwrap(); - assert_eq!( - store_file(fixture.plan( - &mut acquired, - &tmpfs_request(&left, &left_binding), - NO_CONTENT - )), - refusal("image", &left, &left) - ); - fs::remove_file(&left).unwrap(); - // ...by whatever path the request names it: through a hard link... - let hard = fixture.path("images", "hard.erofs"); - fs::hard_link(&record, &hard).unwrap(); - let hard_binding = DistroBinding::import(&layer(&hard), PolicyVerdict::ACCEPTED).unwrap(); - assert_eq!( - store_file(fixture.plan( - &mut acquired, - &tmpfs_request(&hard, &hard_binding), - NO_CONTENT - )), - refusal("image", &hard, &record) - ); - // ...or through a link among its directories. - let alias = fixture.path("images", "alias"); - if symlink_dir(acquired.locked.dir(), &alias) { - let aliased = alias.join(CLONE_ACTIVATION_NAME); - let aliased_binding = - DistroBinding::import(&layer(&aliased), PolicyVerdict::ACCEPTED).unwrap(); - assert_eq!( - store_file(fixture.plan( - &mut acquired, - &tmpfs_request(&aliased, &aliased_binding), - NO_CONTENT - )), - refusal("image", &aliased, &record) - ); - } - // A release of the lease removes that file, so a clone on a scratch image takes it neither - // as its image, which is bound again, since the hard link changed its change time on - // Linux... - let binding = DistroBinding::import(&layer(&record), PolicyVerdict::ACCEPTED).unwrap(); - let staged = seal_of(&record); - let (image, image_binding) = fixture.image("distro.erofs", IMAGE_LEN); - let (scratch, scratch_binding) = fixture.scratch("scratch.ext4", SCRATCH_LEN); - let request = scratch_request(&record, &binding, &scratch, &scratch_binding); - assert_eq!( - store_file(fixture.plan(&mut acquired, &request, NO_CONTENT)), - refusal("image", &record, &record) - ); - assert_eq!(seal_of(&record), staged); - // ...nor as its scratch image, whose claim the refused plan releases. - fs::remove_file(&record).unwrap(); - stage(&record, SCRATCH_LEN); - let record_binding = ScratchBinding::bind(&record).unwrap(); - let request = scratch_request(&image, &image_binding, &record, &record_binding); - assert_eq!( - store_file(fixture.plan(&mut acquired, &request, NO_CONTENT)), - refusal("scratch image", &record, &record) - ); - assert_eq!(fs::metadata(&record).unwrap().len(), SCRATCH_LEN); - assert!(released(|| record_binding - .check(&record, Instant::now()) - .is_ok())); - fs::remove_file(&record).unwrap(); - // The store's other files are refused alike, such as its log, which a launch replaces, and - // the temporary files of its records. - for name in [LOG_NAME, ".runtime.json.1.tmp"] { - let owned = acquired.locked.dir().join(name); - stage(&owned, IMAGE_LEN); - let binding = DistroBinding::import(&layer(&owned), PolicyVerdict::ACCEPTED).unwrap(); - assert_eq!( - store_file(fixture.plan(&mut acquired, &tmpfs_request(&owned, &binding), NO_CONTENT)), - refusal("image", &owned, &owned) - ); - assert_eq!(fs::metadata(&owned).unwrap().len(), IMAGE_LEN); - fs::remove_file(&owned).unwrap(); - } - assert_untouched(&acquired); -} - -#[test] -fn links_and_directories_under_the_names_of_the_sandboxs_store_are_refused() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let record = acquired.locked.clone_activation_path(); - // The sandbox's store holds an entry that it never creates: the host's state is not as the - // store keeps it. - let store_entry = |result: std::result::Result| { - refused(result, Evidence::HostFailure) - }; - let refusal = |owned: &Path| { - ( - ErrorKind::Unavailable, - format!( - "{} is a link or a directory, which the sandbox's store never creates there", - shown(owned) - ), - ) - }; - let request = tmpfs_request(&image, &binding); - // A directory, which a launch could not replace, nor a release remove. - for name in [CLONE_ACTIVATION_NAME, LOG_NAME] { - let owned = acquired.locked.dir().join(name); - fs::create_dir(&owned).unwrap(); - assert_eq!( - store_entry(fixture.plan(&mut acquired, &request, NO_CONTENT)), - refusal(&owned) - ); - assert!(owned.is_dir()); - fs::remove_dir(&owned).unwrap(); - } - // A link to the directory of the images, through which the path of an image can pass, and - // which a launch would replace, leaving that path naming nothing. - if symlink_dir(image.parent().unwrap(), &record) { - let through = record.join("distro.erofs"); - let through_binding = - DistroBinding::import(&layer(&through), PolicyVerdict::ACCEPTED).unwrap(); - for request in [tmpfs_request(&through, &through_binding), request.clone()] { - assert_eq!( - store_entry(fixture.plan(&mut acquired, &request, NO_CONTENT)), - refusal(&record) - ); - } - assert!( - fs::symlink_metadata(&record) - .unwrap() - .file_type() - .is_symlink() - ); - assert!(fs::metadata(&through).unwrap().is_file()); - #[cfg(unix)] - fs::remove_file(&record).unwrap(); - #[cfg(windows)] - fs::remove_dir(&record).unwrap(); - } - assert_untouched(&acquired); - // A control socket that a launch left is no image, and the launch removes it. - #[cfg(unix)] - let _socket = std::os::unix::net::UnixListener::bind( - acquired.locked.dir().join(crate::host::store::SOCKET_NAME), - ) - .unwrap(); - // Once nothing else is there, the clone is planned. - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - assert_eq!(plan.scratch_path(), None); -} - -#[test] -fn a_scratch_image_that_is_a_file_of_a_generation_s_snapshot_is_refused() { - let fixture = Fixture::new(); - let (image, image_binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - // The RAM of the leased generation's snapshot, and that of another generation's, which the - // store's index need not list, of the template's scratch length: the clone's guest would write - // a snapshot that every clone of its generation restores in place. - let snapshot = fixture.store.snapshot_dir(&fixture.generation).unwrap(); - let leased = snapshot.join("memory.bin"); - stage(&leased, SCRATCH_LEN); - let next = fixture.generation.generation().get().get() + 1; - let other = crate::activation::StoreGeneration::new(std::num::NonZeroU64::new(next).unwrap()); - let other = fixture - .store - .payload_dir(other) - .join(snapshot.file_name().unwrap()); - fs::create_dir_all(&other).unwrap(); - let other = other.join("memory.bin"); - stage(&other, SCRATCH_LEN); - // Through a hard link among the scratch images, another name of the file, and by its own - // path, in the store's directory: the request's settings are invalid. - let cases = [ - (&leased, fixture.path("scratches", "leased.raw")), - (&other, fixture.path("scratches", "other.raw")), - (&other, other.clone()), - ]; - for (memory, scratch) in cases { - let linked = scratch != *memory; - if linked { - fs::hard_link(memory, &scratch).unwrap(); - } - let binding = ScratchBinding::bind(&scratch).unwrap(); - let request = scratch_request(&image, &image_binding, &scratch, &binding); - let expected = if linked { - format!( - "the scratch image {} has 2 names, but a clone's scratch image must have a single \ - name: another, a hard link, could make it a file of a snapshot, which every \ - clone of its generation restores in place", - scratch.display() - ) - } else { - format!( - "the scratch image {} is {}, in the directory of the template store {}, where its \ - captures write the snapshots that every clone of a generation restores in place: \ - a clone's scratch image must be outside it", - scratch.display(), - location_of(&File::open(&scratch).unwrap()) - .unwrap() - .display(), - fixture.store.dir().display() - ) - }; - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, NO_CONTENT), - Evidence::InvalidSettings - ), - (ErrorKind::Unsupported, expected) - ); - assert_untouched(&acquired); - } -} - -#[test] -fn a_scratch_image_in_the_template_store_is_refused() { - let fixture = Fixture::new(); - let (image, image_binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - // A file in the store's directory, and one of a snapshot that a capture of the store writes, - // which its publication makes a generation's by renaming the capture's transaction, of the - // template's scratch length: by their own paths, and through links on the way to them, which - // the image's location resolves. - let dir = fixture.store.dir().to_owned(); - let loose = dir.join("scratch.raw"); - let transaction = dir.join(CAPTURES_NAME).join("slot").join(TRANSACTION_NAME); - let memory = transaction.join(SNAPSHOT_NAME).join("memory.bin"); - fs::create_dir_all(memory.parent().unwrap()).unwrap(); - for file in [&loose, &memory] { - stage(file, SCRATCH_LEN); - } - let through = |link: &Path| link.join(SNAPSHOT_NAME).join("memory.bin"); - let mut cases = vec![(&loose, loose.clone()), (&memory, memory.clone())]; - let link = fixture.path("links", "transaction"); - fs::create_dir_all(link.parent().unwrap()).unwrap(); - #[cfg(unix)] - std::os::unix::fs::symlink(&transaction, &link).unwrap(); - #[cfg(windows)] - junction(&link, &transaction); - cases.push((&memory, through(&link))); - // The transaction held open, as a descriptor keeps a directory whatever renames it. - #[cfg(target_os = "linux")] - let held = File::open(&transaction).unwrap(); - #[cfg(target_os = "linux")] - cases.push(( - &memory, - through( - &Path::new("/proc/self/fd").join(std::os::fd::AsRawFd::as_raw_fd(&held).to_string()), - ), - )); - for (file, scratch) in cases { - let binding = ScratchBinding::bind(&scratch).unwrap(); - let request = scratch_request(&image, &image_binding, &scratch, &binding); - let location = location_of(&File::open(file).unwrap()).unwrap(); - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, NO_CONTENT), - Evidence::InvalidSettings - ), - ( - ErrorKind::Unsupported, - format!( - "the scratch image {} is {}, in the directory of the template store {}, \ - where its captures write the snapshots that every clone of a generation \ - restores in place: a clone's scratch image must be outside it", - scratch.display(), - location.display(), - dir.display() - ) - ) - ); - assert_untouched(&acquired); - } -} - -/// On Windows, a file of a volume without a drive letter or a mount point is located by the -/// volume's GUID, above which the store's directory is found all the same. -#[cfg(windows)] -#[test] -fn locations_that_name_their_volume_by_its_guid_are_compared_with_the_store() { - let fixture = Fixture::new(); - let dir = fixture.store.dir().to_owned(); - let inside = dir - .join(CAPTURES_NAME) - .join("slot") - .join(TRANSACTION_NAME) - .join("memory.bin"); - fs::create_dir_all(inside.parent().unwrap()).unwrap(); - stage(&inside, SCRATCH_LEN); - let (outside, _) = fixture.scratch("scratch.ext4", SCRATCH_LEN); - for (file, within) in [(&inside, true), (&outside, false)] { - let location = crate::host::seal::final_volume_path(&File::open(file).unwrap()).unwrap(); - assert!( - location.to_string_lossy().starts_with(r"\\?\Volume{"), - "{}", - location.display() - ); - assert_eq!( - is_within(&location, &dir).unwrap(), - within, - "{}", - location.display() - ); - } -} - -#[test] -fn a_scratch_image_with_another_name_is_refused() { - let fixture = Fixture::new(); - let (image, image_binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - // Another name of the image, outside the template store, as a file of the snapshot of another - // store would be, or one of a snapshot that a capture of this store writes, which neither the - // store's generations nor the image's location show. - let (scratch, _) = fixture.scratch("scratch.ext4", SCRATCH_LEN); - let other = fixture.path("stores", "memory.bin"); - fs::create_dir_all(other.parent().unwrap()).unwrap(); - fs::hard_link(&scratch, &other).unwrap(); - let binding = ScratchBinding::bind(&scratch).unwrap(); - let request = scratch_request(&image, &image_binding, &scratch, &binding); - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, NO_CONTENT), - Evidence::InvalidSettings - ), - ( - ErrorKind::Unsupported, - format!( - "the scratch image {} has 2 names, but a clone's scratch image must have a single \ - name: another, a hard link, could make it a file of a snapshot, which every clone \ - of its generation restores in place", - scratch.display() - ) - ) - ); - assert_untouched(&acquired); - // Once its other name went, the image is the clone's alone. - fs::remove_file(&other).unwrap(); - drop(fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap()); -} - -#[test] -fn names_that_the_file_system_takes_for_those_of_the_stores_files_are_the_stores() { - let fixture = Fixture::new(); - let (image, image_binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let dir = acquired.locked.dir().to_owned(); - // The image or the scratch image of a request is a file of the sandbox's store: the request's - // settings are invalid. - let store_file = |result: std::result::Result| { - refused(result, Evidence::InvalidSettings) - }; - let refusal = |what: &str, path: &Path, owned: &Path| { - ( - ErrorKind::Unsupported, - format!( - "the {what} {} is {}, a file of the sandbox's store, which its transitions \ - replace or remove", - shown(path), - shown(owned) - ), - ) - }; - // Whether the file system takes the paths `a` and `b` for one file. - let same_file = |a: &Path, b: &Path| { - let identity = |path: &Path| { - File::open(path) - .and_then(|file| FileSeal::of(&file)) - .map(|seal| (seal.device(), seal.file_id())) - .ok() - }; - identity(a).is_some() && identity(a) == identity(b) - }; - // An image under the name of one of the store's files in other letter cases is that file - // where the file system ignores case, as Windows' do, and another file where it does not. - for name in [CLONE_ACTIVATION_NAME, LOG_NAME] { - let cased = dir.join(name.to_ascii_uppercase()); - stage(&cased, IMAGE_LEN); - let binding = DistroBinding::import(&layer(&cased), PolicyVerdict::ACCEPTED).unwrap(); - let owned = dir.join(name); - let same = same_file(&cased, &owned); - let result = fixture.plan(&mut acquired, &tmpfs_request(&cased, &binding), NO_CONTENT); - if same { - assert_eq!(store_file(result), refusal("image", &cased, &owned)); - } else { - drop(result.unwrap()); - } - assert_eq!(fs::metadata(&cased).unwrap().len(), IMAGE_LEN); - fs::remove_file(&cased).unwrap(); - } - // So is a scratch image, which a release would remove as the store's file where the file - // system takes it for that file, and keeps where it does not. - let record = acquired.locked.clone_activation_path(); - let cased = dir.join(CLONE_ACTIVATION_NAME.to_ascii_uppercase()); - stage(&cased, SCRATCH_LEN); - let binding = ScratchBinding::bind(&cased).unwrap(); - let request = scratch_request(&image, &image_binding, &cased, &binding); - let same = same_file(&cased, &record); - let result = fixture.plan(&mut acquired, &request, NO_CONTENT); - if same { - assert_eq!( - store_file(result), - refusal("scratch image", &cased, &record) - ); - } else { - drop(result.unwrap()); - assert_eq!( - fixture.store.release(&mut acquired.locked).unwrap(), - Some(acquired.lease.clone()) - ); - } - assert_eq!(fs::metadata(&cased).unwrap().len(), SCRATCH_LEN); -} - -#[test] -fn templates_that_a_runtime_of_other_contracts_captured_are_refused() { - // A store that a build of other boot or control contracts kept holds templates whose guests - // this build cannot drive. - type ClassChange = fn(&mut CaptureClass); - let changes: [(&str, ClassChange); 3] = [ - ("boot contract", |class| class.runtime.boot_contract += 1), - ("control contract", |class| { - class.runtime.control_contract = "other-revision".to_owned(); - }), - ("control session", |class| { - class.runtime.control_session += 1 - }), - ]; - for (name, change) in changes { - let fixture = Fixture::with(|class, _, _| change(class)); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let plan = fixture.plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT); - assert_eq!( - refused(plan, Evidence::IncompatibleHost), - ( - ErrorKind::Unsupported, - format!( - "the template's generation {} speaks other contracts than this build, which \ - restores none of its clones", - fixture.generation.generation() - ) - ), - "{name}" - ); - assert_untouched(&acquired); - } -} - -#[test] -fn images_that_their_binding_does_not_authorize_and_malformed_generations_are_refused() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = tmpfs_request(&image, &binding); - // A binding whose verdict rejects the image authorizes no clone of it. - let verdict = PolicyVerdict { - result: PolicyResult::Rejected, - ..PolicyVerdict::ACCEPTED - }; - let rejected = DistroBinding::import(&layer(&image), verdict).unwrap(); - let plan = fixture.plan(&mut acquired, &tmpfs_request(&image, &rejected), NO_CONTENT); - refused(plan, Evidence::AuthorizationDenied); - // A binding of another revision of the distro policy than the request's is stale. - let stale = CloneRequest { - policy_revision: DISTRO_POLICY_REVISION + 1, - ..request.clone() - }; - refused( - fixture.plan(&mut acquired, &stale, NO_CONTENT), - Evidence::StaleBinding, - ); - // A failure of the request quarantines nothing. - let failure = fixture.plan(&mut acquired, &stale, NO_CONTENT).unwrap_err(); - assert!(!failure.quarantine(&fixture.store, None).unwrap()); - assert_untouched(&acquired); - // A generation whose key is gone is malformed: a failure of the template, whose generation it - // quarantines through its tombstone, once the sandbox's lock is released, as the lock order - // asks. - let generation = fixture.generation.generation(); - let key = fixture - .store - .payload_dir(generation) - .join(crate::host::template::KEY_NAME); - fs::remove_file(key).unwrap(); - let failure = fixture - .plan(&mut acquired, &request, NO_CONTENT) - .unwrap_err(); - assert_eq!( - (failure.record().evidence, failure.outcome()), - (Evidence::MalformedGeneration, Outcome::Template), - "{failure}" - ); - assert_untouched(&acquired); - drop(acquired); - let other = Fixture::new(); - let error = failure.quarantine(&other.store, None).unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Other, "{error}"); - assert!(other.store.tombstones().unwrap().is_empty()); - assert!(failure.quarantine(&fixture.store, None).unwrap()); - assert_eq!(fixture.store.tombstones().unwrap(), [generation]); - // The generation takes no new lease. - let error = fixture - .store - .acquire( - generation, - fixture.provision("next"), - &files(), - &after(&fixture.generation, 1), - None, - ) - .unwrap_err(); - assert_eq!(error.code(), ErrorCode::TemplateTombstoned, "{error}"); -} - -#[test] -fn keys_that_their_read_refuses_for_what_they_are_are_failures_of_the_template() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = tmpfs_request(&image, &binding); - let generation = fixture.generation.generation(); - let key = fixture - .store - .payload_dir(generation) - .join(crate::host::template::KEY_NAME); - let malformed = |acquired: &mut Acquired, what: &str| { - let failure = fixture.plan(acquired, &request, NO_CONTENT).unwrap_err(); - assert_eq!( - (failure.record().evidence, failure.outcome()), - (Evidence::MalformedGeneration, Outcome::Template), - "{what}: {failure}" - ); - assert_untouched(acquired); - failure - }; - // A link in place of the key, a directory there, and a key that holds more than a key may are - // none that the store writes: the read refuses each for what it is, a failure of the template, - // not of an operation of the host's. Windows denies the open of a directory, as a file's access - // control list may deny one, which is the host's, so only Unix tries the directory. - let moved = key.with_extension("moved"); - fs::rename(&key, &moved).unwrap(); - if symlink_file(&moved, &key) { - malformed(&mut acquired, "a link"); - fs::remove_file(&key).unwrap(); - } - if cfg!(unix) { - fs::create_dir(&key).unwrap(); - malformed(&mut acquired, "a directory"); - fs::remove_dir(&key).unwrap(); - } - fs::rename(&moved, &key).unwrap(); - crate::host::store::files::write_private(&key, &vec![0; TemplateKey::MAX_LEN + 1]).unwrap(); - let failure = malformed(&mut acquired, "an oversized key"); - // So the failure quarantines the generation, once the sandbox's lock is released. - drop(acquired); - assert!(failure.quarantine(&fixture.store, None).unwrap()); - assert_eq!(fixture.store.tombstones().unwrap(), [generation]); -} - -#[test] -fn keys_of_another_encoding_version_are_failures_of_the_host_that_quarantine_nothing() { - // A store that a build of another encoding of keys kept, as one that a rollback of the host's - // build leaves, holds generations whose keys this build cannot read: the host cannot restore - // them, but a build that reads their keys can, so their failures quarantine nothing. - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let generation = fixture.generation.generation(); - let key = fixture - .store - .payload_dir(generation) - .join(crate::host::template::KEY_NAME); - let mut bytes = fs::read(&key).unwrap(); - let version = TemplateKey::VERSION + 1; - bytes[..2].copy_from_slice(&version.to_le_bytes()); - crate::host::store::files::write_private(&key, &bytes).unwrap(); - let failure = fixture - .plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT) - .unwrap_err(); - assert_eq!( - (failure.record().evidence, failure.outcome()), - (Evidence::IncompatibleHost, Outcome::Host), - "{failure}" - ); - assert_eq!(failure.code(), ErrorCode::FormatUnsupported, "{failure}"); - let unread = format!( - "the snapshot key has encoding version {version}; this build reads version {} only", - TemplateKey::VERSION - ); - assert!(failure.message().contains(&unread), "{failure}"); - assert_untouched(&acquired); - drop(acquired); - assert!(!failure.quarantine(&fixture.store, None).unwrap()); - assert!(fixture.store.tombstones().unwrap().is_empty()); - // The generation still takes new leases. - let next = fixture - .store - .acquire( - generation, - fixture.provision("next"), - &files(), - &after(&fixture.generation, 1), - None, - ) - .unwrap(); - assert_eq!(next.lease.generation(), generation); -} - -#[test] -fn a_clone_on_a_scratch_image_claims_it_and_reads_no_content() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let (scratch, scratch_binding) = fixture.scratch("scratch.ext4", SCRATCH_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = scratch_request(&image, &binding, &scratch, &scratch_binding); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let CloneUpper::Scratch(checked) = plan.upper() else { - panic!("the clone's upper is not on its scratch image: {plan:?}"); - }; - assert_eq!(checked.path().to_str().unwrap(), shown(&scratch)); - assert_eq!(plan.scratch_path(), Some(checked.path())); - assert_eq!( - checked.geometry().unwrap(), - geometry(SCRATCH_LEN, fixture.physical) - ); - // What the checks hold reads nothing. - let mut byte = [0]; - assert!(checked.file().read(&mut byte).is_err()); - assert!(plan.image().file().read(&mut byte).is_err()); - // The plan holds the claim: another check finds the image claimed. - let error = scratch_binding.check(&scratch, Instant::now()).unwrap_err(); - assert_eq!(error.code(), ErrorCode::ScratchClaimed, "{error}"); - drop(plan); - assert!(released(|| scratch_binding - .check(&scratch, Instant::now()) - .is_ok())); -} - -#[test] -fn a_scratch_image_serves_one_clone_at_a_time() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let (scratch, scratch_binding) = fixture.scratch("scratch.ext4", SCRATCH_LEN); - let request = scratch_request(&image, &binding, &scratch, &scratch_binding); - let mut first = fixture.acquire("first"); - let mut second = fixture.acquire("second"); - let plan = fixture.plan(&mut first, &request, NO_CONTENT).unwrap(); - // The other clone's plan waits until its deadline for the claim, then fails, holding nothing. - let wait = Duration::from_millis(200); - let started = Instant::now(); - let error = ClonePlan::new_with( - &fixture.store, - &mut second, - &request, - started + wait, - NO_CONTENT, - ) - .unwrap_err(); - assert!(started.elapsed() >= wait); - assert_eq!(error.code(), ErrorCode::ScratchClaimed, "{error}"); - assert_eq!(error.kind(), ErrorKind::TimedOut, "{error}"); - // The image is not fresh for this clone: the request's. - assert_eq!(error.record().evidence, Evidence::UnusableScratch); - assert_eq!(error.outcome(), Outcome::Request); - assert_eq!( - error.message(), - format!( - "another sandbox claims the ext4 scratch image {}", - shown(&scratch) - ) - ); - assert_untouched(&second); - // Once the first plan is gone, the image serves the other clone. - drop(plan); - let plan = fixture.plan(&mut second, &request, NO_CONTENT).unwrap(); - assert!(matches!(plan.upper(), CloneUpper::Scratch(_))); -} - -#[test] -fn scratch_images_whose_identity_or_kind_changed_are_refused() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let changed = |path: &Path, field: &str| { - ( - ErrorKind::Unavailable, - format!( - "{} changed after its scratch binding was made: its {field} differs", - shown(path) - ), - ) - }; - // Another file in the image's place, of the same length, has another identity. - let (scratch, scratch_binding) = fixture.scratch("replaced.ext4", SCRATCH_LEN); - let copy = fixture.path("scratches", "copy.ext4"); - fs::copy(&scratch, ©).unwrap(); - fs::rename(©, &scratch).unwrap(); - let request = scratch_request(&image, &binding, &scratch, &scratch_binding); - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, NO_CONTENT), - Evidence::SuppliedIdentity - ), - changed(&scratch, "identity") - ); - // So has a directory in its place. The image stays, under another name, so that the - // directory cannot take its inode number, which is its file ID on Linux. - let (scratch, scratch_binding) = fixture.scratch("directory.ext4", SCRATCH_LEN); - fs::rename(&scratch, fixture.path("scratches", "moved.ext4")).unwrap(); - fs::create_dir(&scratch).unwrap(); - let request = scratch_request(&image, &binding, &scratch, &scratch_binding); - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, NO_CONTENT), - Evidence::SuppliedIdentity - ), - changed(&scratch, "identity") - ); - // A binding of another kind names another file. - let (scratch, scratch_binding) = fixture.scratch("kind.ext4", SCRATCH_LEN); - let mut value = serde_json::to_value(&scratch_binding).unwrap(); - value["kind"] = "directory".into(); - let scratch_binding: ScratchBinding = serde_json::from_value(value).unwrap(); - let request = scratch_request(&image, &binding, &scratch, &scratch_binding); - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, NO_CONTENT), - Evidence::SuppliedIdentity - ), - changed(&scratch, "kind") - ); - assert_untouched(&acquired); -} - -#[test] -fn images_no_longer_than_the_placeholder_are_served_and_longer_ones_refused() { - let fixture = Fixture::new(); - let mut acquired = fixture.acquire("sandbox"); - let physical = fixture.physical; - // A template serves images of either format of any length up to its placeholder's, the - // capacity that the restore presents over each, reading zeros past a shorter one's end. - for length in [512, IMAGE_LEN / 2, IMAGE_LEN - 512, IMAGE_LEN] { - for gpt in [false, true] { - let (image, binding) = if gpt { - fixture.gpt_image("served.gpt", length) - } else { - fixture.image("served.erofs", length) - }; - let plan = fixture - .plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT) - .unwrap(); - assert_eq!(plan.image().geometry().unwrap(), geometry(length, physical)); - assert_eq!(plan.key().distro(), geometry(IMAGE_LEN, physical)); - drop(plan); - fs::remove_file(&image).unwrap(); - } - } - let mut refusal = |length: u64, gpt: bool, evidence: Evidence| { - let (image, binding) = if gpt { - fixture.gpt_image("other.gpt", length) - } else { - fixture.image("other.erofs", length) - }; - let plan = fixture.plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT); - let failed = refused(plan, evidence); - fs::remove_file(&image).unwrap(); - (shown(&image), failed) - }; - // A longer image, of either format, is refused. - for (length, gpt) in [ - (IMAGE_LEN + 512, false), - (2 * IMAGE_LEN, false), - (IMAGE_LEN + 512, true), - ] { - let (image, failed) = refusal(length, gpt, Evidence::SuppliedLength); - assert_eq!( - failed, - ( - ErrorKind::Unsupported, - format!( - "the image {image} is {length} bytes long, longer than the template's \ - placeholder, whose {IMAGE_LEN} bytes are the capacity of every clone's image" - ) - ) - ); - } - // OpenVMM attaches no block device whose length is not a nonzero multiple of 512 bytes, however - // short. - for length in [0, IMAGE_LEN - 1, IMAGE_LEN + 1] { - let (image, failed) = refusal(length, false, Evidence::SuppliedGeometry); - assert_eq!( - failed, - ( - ErrorKind::Unsupported, - format!( - "the distro EROFS layer {image} of {length} bytes cannot be attached: OpenVMM \ - attaches block devices of nonzero multiples of 512 bytes" - ) - ) - ); - } - assert_untouched(&acquired); - - // A template whose placeholder presented physical blocks of another size serves no image of - // the filesystem of this one, whatever its length. - let fixture = Fixture::with(|class, spec, physical| { - let scratch = class.scratch; - let distro = geometry(IMAGE_LEN, other_physical(physical)); - set_geometries(class, spec, distro, scratch); - }); - let mut acquired = fixture.acquire("sandbox"); - let physical = fixture.physical; - for length in [IMAGE_LEN / 2, IMAGE_LEN, 2 * IMAGE_LEN] { - let (image, binding) = fixture.image("distro.erofs", length); - let plan = fixture.plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT); - assert_eq!( - refused(plan, Evidence::SuppliedGeometry), - ( - ErrorKind::Unsupported, - format!( - "the image {} presents logical blocks of 512 bytes and physical blocks of \ - {physical} bytes, but the template's placeholder presented 512 and {} bytes", - shown(&image), - other_physical(physical) - ) - ) - ); - fs::remove_file(&image).unwrap(); - } - assert_untouched(&acquired); -} - -#[test] -fn scratch_images_of_another_length_or_geometry_are_refused() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - for length in [SCRATCH_LEN - 512, SCRATCH_LEN + 512] { - let (scratch, scratch_binding) = fixture.scratch("other.ext4", length); - let request = scratch_request(&image, &binding, &scratch, &scratch_binding); - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, NO_CONTENT), - Evidence::SuppliedLength - ), - ( - ErrorKind::Unsupported, - format!( - "the scratch image {} is {length} bytes long, not the {SCRATCH_LEN} bytes of \ - the template's scratch", - shown(&scratch) - ) - ) - ); - // The refused plan holds no claim. - drop( - scratch_binding - .check(&scratch, Instant::now() + CLAIM_RELEASE_WAIT) - .unwrap(), - ); - fs::remove_file(&scratch).unwrap(); - } - let (scratch, scratch_binding) = fixture.scratch("unaligned.ext4", SCRATCH_LEN + 1); - let request = scratch_request(&image, &binding, &scratch, &scratch_binding); - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, NO_CONTENT), - Evidence::SuppliedGeometry - ), - ( - ErrorKind::Unsupported, - format!( - "the ext4 scratch image {} of {} bytes cannot be attached: OpenVMM attaches block \ - devices of nonzero multiples of 512 bytes", - shown(&scratch), - SCRATCH_LEN + 1 - ) - ) - ); - assert_untouched(&acquired); -} - -#[test] -fn a_template_serves_clones_in_ram_whatever_geometry_its_scratch_presented() { - // A template whose scratch presented physical blocks of another size than the filesystem of - // the sandboxes and of the scratch images: the restore of a clone in RAM attaches the scratch - // with no file, at the template's geometry, so the template serves clones in RAM there, while - // a scratch image must present that geometry, which no file of that filesystem does. - let fixture = Fixture::with(|class, spec, physical| { - let distro = class.distro; - let scratch = geometry(SCRATCH_LEN, other_physical(physical)); - set_geometries(class, spec, distro, scratch); - }); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let (physical, other) = (fixture.physical, other_physical(fixture.physical)); - let plan = fixture - .plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT) - .unwrap(); - assert_eq!(plan.scratch_path(), None); - assert_eq!(plan.key().scratch(), geometry(SCRATCH_LEN, other)); - drop(plan); - let (scratch, scratch_binding) = fixture.scratch("scratch.ext4", SCRATCH_LEN); - let request = scratch_request(&image, &binding, &scratch, &scratch_binding); - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, NO_CONTENT), - Evidence::SuppliedGeometry - ), - ( - ErrorKind::Unsupported, - format!( - "the scratch image {} presents logical blocks of 512 bytes and physical blocks of \ - {physical} bytes, but the template's scratch presented 512 and {other} bytes", - shown(&scratch) - ) - ) - ); - assert_untouched(&acquired); -} - -/// Files on hugetlbfs, whose I/O block size is a huge page, present physical blocks of 2 MiB, -/// unlike the template's: an image, as long as the placeholder or shorter, and a scratch image, -/// which each reach such a file. -#[cfg(target_os = "linux")] -#[test] -fn files_on_a_filesystem_of_another_block_size_are_refused() { - use crate::host::sandbox::distro::import_with; - use crate::host::sandbox::scratch::bind_with; - - const HUGE_PAGE: u64 = 2 * MIB; - - let fixture = Fixture::new(); - if u64::from(fixture.physical) == HUGE_PAGE { - eprintln!("the files of the test's directory present physical blocks of a huge page"); - return; - } - let Some(huge_images) = [IMAGE_LEN, HUGE_PAGE] - .into_iter() - .map(huge_file) - .collect::>>() - else { - return; - }; - let mut acquired = fixture.acquire("sandbox"); - let presented = |what: String, device: &str| { - ( - ErrorKind::Unsupported, - format!( - "{what} presents logical blocks of 512 bytes and physical blocks of {HUGE_PAGE} \ - bytes, but the template's {device} presented 512 and {} bytes", - fixture.physical - ), - ) - }; - // The image, as long as the placeholder or shorter: its binding and the check of the binding - // reach a file on hugetlbfs at its path. - let image = fixture.path("images", "huge.erofs"); - for huge_image in &huge_images { - let open = |path: &Path| { - if path == image { - huge_image.try_clone() - } else { - Err(io::Error::from(io::ErrorKind::NotFound)) - } - }; - let huge_distro = distro::Access { - open: &open, - seal: &FileSeal::of, - }; - let binding = import_with(&layer(&image), PolicyVerdict::ACCEPTED, huge_distro).unwrap(); - let access = Access { - distro: huge_distro, - ..NO_CONTENT - }; - let plan = fixture.plan(&mut acquired, &tmpfs_request(&image, &binding), access); - assert_eq!( - refused(plan, Evidence::SuppliedGeometry), - presented(format!("the image {}", shown(&image)), "placeholder") - ); - } - // The scratch image: its binding and its check reach a file on hugetlbfs at its path. - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let huge_scratch = huge_file(SCRATCH_LEN).unwrap(); - let scratch = fixture.path("scratches", "huge.ext4"); - let open = |_: &Path| huge_scratch.try_clone(); - let scratch_access = scratch::Access { open: &open }; - let scratch_binding = bind_with(&paths::normalized(&scratch), scratch_access).unwrap(); - let access = Access { - scratch: scratch_access, - ..NO_CONTENT - }; - let request = scratch_request(&image, &binding, &scratch, &scratch_binding); - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, access), - Evidence::SuppliedGeometry - ), - presented(format!("the scratch image {}", shown(&scratch)), "scratch") - ); - assert_untouched(&acquired); -} - -#[test] -fn memory_targets_are_whole_blocks_from_the_templates_ram_to_its_capacity() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let block = MEMORY_BLOCK_SIZE; - let generation = fixture.generation.generation(); - for memory in [ - 0, - MEMORY - block, - MEMORY + 1, - MEMORY + block / 2, - MEMORY_CAPACITY - 4096, - MEMORY_CAPACITY + block, - u64::MAX, - ] { - let request = CloneRequest { - memory, - ..tmpfs_request(&image, &binding) - }; - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, NO_CONTENT), - Evidence::InvalidTargets - ), - ( - ErrorKind::Unsupported, - format!( - "a clone of the template's generation {generation} cannot restore with \ - {memory} bytes of RAM: its memory target must be whole blocks of 128 MiB, \ - and of the template's memory blocks of {block} bytes, from the template's \ - {MEMORY} bytes of RAM to its capacity of {MEMORY_CAPACITY} bytes" - ) - ) - ); - } - assert_untouched(&acquired); - for memory in [MEMORY, MEMORY + block, MEMORY_CAPACITY] { - let request = CloneRequest { - memory, - ..tmpfs_request(&image, &binding) - }; - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - assert_eq!(plan.memory(), memory); - } -} - -#[test] -fn processor_targets_are_counts_from_the_templates_processors_online_at_boot_to_its_capacity() { - let refusal = |generation: &str, processors: u32, boot: u32, capacity: u32| { - ( - ErrorKind::Unsupported, - format!( - "a clone of the template's generation {generation} cannot restore with a \ - processor target of {processors}: a target must be 1, 2, 4, or 8, from the \ - template's processors online at boot, {boot}, to its capacity, {capacity}" - ), - ) - }; - // A template that boots with 2 of its 8 processors online serves each supported count from 2 - // to 8. - let fixture = Fixture::with(|class, spec, _| set_processors(class, spec, 8, 2)); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let generation = fixture.generation.generation().to_string(); - for processors in [0, 1, 3, 6, 16, u32::MAX] { - let request = CloneRequest { - processors, - ..tmpfs_request(&image, &binding) - }; - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, NO_CONTENT), - Evidence::InvalidTargets - ), - refusal(&generation, processors, 2, 8) - ); - } - assert_untouched(&acquired); - for processors in [2, 4, 8] { - let request = CloneRequest { - processors, - ..tmpfs_request(&image, &binding) - }; - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - assert_eq!(plan.processors(), processors); - assert_eq!(plan.activation().configuration().resources.cpus, processors); - } - // A template whose processors are all online at boot records none online at boot, and serves - // its capacity alone. - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let generation = fixture.generation.generation().to_string(); - for processors in [1, 4] { - let request = CloneRequest { - processors, - ..tmpfs_request(&image, &binding) - }; - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, NO_CONTENT), - Evidence::InvalidTargets - ), - refusal(&generation, processors, PROCESSORS, PROCESSORS) - ); - } - let plan = fixture - .plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT) - .unwrap(); - assert_eq!(plan.processors(), PROCESSORS); -} - -#[test] -fn requests_that_no_clone_takes_are_refused_before_anything_is_opened() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let (scratch, scratch_binding) = fixture.scratch("scratch.ext4", SCRATCH_LEN); - let mut acquired = fixture.acquire("sandbox"); - let mut refusal = |request: &CloneRequest, evidence: Evidence| { - let (kind, message) = refused(fixture.plan(&mut acquired, request, NO_CONTENT), evidence); - assert_eq!(kind, ErrorKind::Unsupported, "{message}"); - message - }; - let tmpfs = tmpfs_request(&image, &binding); - let distro = tmpfs.adapter.distro().clone(); - // A plan of the sandbox profile. - let profile = CloneRequest { - adapter: SandboxPlan::new( - distro.clone(), - SandboxUpper::RAM, - SandboxSettings::default(), - ) - .unwrap(), - ..tmpfs.clone() - }; - let message = refusal(&profile, Evidence::InvalidSettings); - assert!( - message.starts_with("a clone restores an adapter's plan, not a plan of the sandbox"), - "{message}" - ); - // An image that is not the one that the binding names. - let other = Uuid::parse("22222222-2222-2222-2222-222222222222").unwrap(); - let mistaken = CloneRequest { - adapter: SandboxPlan::adapter( - SandboxDistro::new(image.to_str().unwrap(), RootImage::erofs(Some(other))), - SandboxUpper::RAM, - ), - ..tmpfs.clone() - }; - assert_eq!( - refusal(&mistaken, Evidence::InvalidFormat), - format!( - "the image {} is not the erofs layer with identity {UUID} that its distro binding \ - names", - shown(&image) - ) - ); - // An image of another format than the one that the binding names, either way. - let identity = Some(Uuid::parse(UUID).unwrap()); - let as_gpt = CloneRequest { - adapter: SandboxPlan::adapter( - SandboxDistro::new(image.to_str().unwrap(), RootImage::gpt(identity)), - SandboxUpper::RAM, - ), - ..tmpfs.clone() - }; - assert_eq!( - refusal(&as_gpt, Evidence::InvalidFormat), - format!( - "the image {} is not the erofs layer with identity {UUID} that its distro binding \ - names", - shown(&image) - ) - ); - let (gpt, gpt_binding) = fixture.gpt_image("distro.gpt", IMAGE_LEN); - let as_erofs = CloneRequest { - adapter: SandboxPlan::adapter( - SandboxDistro::new(gpt.to_str().unwrap(), RootImage::erofs(identity)), - SandboxUpper::RAM, - ), - binding: gpt_binding, - ..tmpfs.clone() - }; - assert_eq!( - refusal(&as_erofs, Evidence::InvalidFormat), - format!( - "the image {} is not the gpt layer with identity {UUID} that its distro binding names", - shown(&gpt) - ) - ); - // Paths that OpenVMM would not open as the plan checks them. - let at = |path: &str| CloneRequest { - adapter: SandboxPlan::adapter(SandboxDistro::new(path, distro.image()), SandboxUpper::RAM), - ..tmpfs.clone() - }; - let message = refusal(&at("distro.erofs"), Evidence::InvalidSettings); - assert!( - message.starts_with("the image of a clone must be absolute"), - "{message}" - ); - let comma = at(&format!("{},x", image.to_str().unwrap())); - let message = refusal(&comma, Evidence::InvalidSettings); - assert!( - message.starts_with("disk paths containing commas, semicolons, or unbalanced brackets"), - "{message}" - ); - let vhdx = at(image.with_extension("vhdx").to_str().unwrap()); - let message = refusal(&vhdx, Evidence::InvalidSettings); - assert!( - message.starts_with("disk paths ending in .vhdx"), - "{message}" - ); - let on_scratch = |path: &str| CloneRequest { - adapter: SandboxPlan::adapter(distro.clone(), SandboxUpper::scratch(path)), - scratch: Some(scratch_binding.clone()), - ..tmpfs.clone() - }; - let message = refusal(&on_scratch("scratch.ext4"), Evidence::InvalidSettings); - assert!( - message.starts_with("the scratch image of a clone must be absolute"), - "{message}" - ); - let iso = on_scratch(scratch.with_extension("iso").to_str().unwrap()); - let message = refusal(&iso, Evidence::InvalidSettings); - assert!( - message.starts_with("disk paths ending in .iso"), - "{message}" - ); - // An upper on a scratch image needs its binding, and one in RAM takes none. - let unbound = CloneRequest { - scratch: None, - ..scratch_request(&image, &binding, &scratch, &scratch_binding) - }; - assert_eq!( - refusal(&unbound, Evidence::InvalidSettings), - format!( - "the upper of a clone on the scratch image {} needs the image's scratch binding", - shown(&scratch) - ) - ); - let bound = CloneRequest { - scratch: Some(scratch_binding.clone()), - ..tmpfs.clone() - }; - assert_eq!( - refusal(&bound, Evidence::InvalidSettings), - "the upper of a clone in RAM takes no scratch binding" - ); - // None of them opened anything: the scratch image is free, and the lease still reserves the - // clone's launch, which a plan of a request that a clone takes then serves. - drop(scratch_binding.check(&scratch, Instant::now()).unwrap()); - assert_untouched(&acquired); - drop(fixture.plan(&mut acquired, &tmpfs, NO_CONTENT).unwrap()); -} - -#[test] -fn templates_serve_the_networks_of_every_policy_at_their_nics_address_alone() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = tmpfs_request(&image, &binding); - let generation = fixture.generation.generation(); - let attached = with_network(request.clone(), Some(network(NetworkAccess::Allow))); - assert_eq!( - refused( - fixture.plan(&mut acquired, &attached, NO_CONTENT), - Evidence::InvalidSettings - ), - ( - ErrorKind::Unsupported, - format!( - "the adapter's plan of a clone attaches a network, but the template's generation \ - {generation} has no NIC" - ) - ) - ); - assert_untouched(&acquired); - // A network that lets nothing through attaches no NIC. - let closed = with_network(request, Some(network(NetworkAccess::Deny))); - drop(fixture.plan(&mut acquired, &closed, NO_CONTENT).unwrap()); - - // A template with a NIC serves no clone without a network. - let fixture = network_fixture(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = tmpfs_request(&image, &binding); - let generation = fixture.generation.generation(); - let closed = with_network(request.clone(), Some(network(NetworkAccess::Deny))); - for request in [&request, &closed] { - assert_eq!( - refused( - fixture.plan(&mut acquired, request, NO_CONTENT), - Evidence::InvalidSettings - ), - ( - ErrorKind::Unsupported, - format!( - "the template's generation {generation} has a NIC, which its clones restore \ - with the network of the adapter's plan, but the plan has none" - ) - ) - ); - assert_untouched(&acquired); - } - // Nor the clones of a network at another address. - for (other, policy) in [ - ("10.0.0.3/24", NetworkPolicy::new(NetworkAccess::Allow)), - ("10.0.0.2/16", NetworkPolicy::new(NetworkAccess::Allow)), - ] { - let other = SandboxNetwork::new(other, policy).unwrap(); - let address = other.guest_network().to_owned(); - let other = with_network(request.clone(), Some(other)); - assert_eq!( - refused( - fixture.plan(&mut acquired, &other, NO_CONTENT), - Evidence::InvalidSettings - ), - ( - ErrorKind::Unsupported, - format!( - "the adapter's plan of a clone attaches a network at {address}, but the NIC \ - of the template's generation {generation} is at 10.0.0.2/24: a template \ - serves the clones of its own guest's address alone" - ) - ) - ); - assert_untouched(&acquired); - } - // It serves the clones of every policy at its guest's address, whatever the policy of its - // class's network, since each clone's restore replaces the complete deny-all policy of the - // template's NIC with its own. - for policy in attaching() { - let attached = SandboxNetwork::new("10.0.0.2/24", policy).unwrap(); - let attached = with_network(request.clone(), Some(attached)); - drop(fixture.plan(&mut acquired, &attached, NO_CONTENT).unwrap()); - } -} - -#[test] -fn clones_are_planned_under_the_lock_that_acquired_their_lease_before_their_launch() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let request = tmpfs_request(&image, &binding); - // A lease of another store. - let other = Fixture::new(); - let mut elsewhere = other.acquire("sandbox"); - let plan = fixture.plan(&mut elsewhere, &request, NO_CONTENT); - let (kind, message) = refused(plan, Evidence::HostFailure); - assert_eq!(kind, ErrorKind::Other, "{message}"); - assert_eq!( - message, - format!( - "the lease {} is one of the template store {}, not of this one, {}", - elsewhere.lease.id(), - other.store.id(), - fixture.store.id() - ) - ); - assert_untouched(&elsewhere); - // Another lock of the sandbox, which reserves no launch, as one taken once the lock that - // acquired the lease is gone. - let Acquired { lease, locked } = fixture.acquire("sandbox"); - drop(locked); - let locked = SandboxStore::lock_existing(lease.sandbox(), &files(), None).unwrap(); - let mut relocked = Acquired { lease, locked }; - let plan = fixture.plan(&mut relocked, &request, NO_CONTENT); - let (kind, message) = refused(plan, Evidence::HostFailure); - assert_eq!(kind, ErrorKind::Other, "{message}"); - assert!( - message.starts_with("a clone is planned under the lock that acquired its lease"), - "{message}" - ); - // The lock that acquired a lease, once the launch that the lease reserves has begun. - let mut acquired = fixture.acquire("begun"); - let key = fixture.store.read_key(&fixture.generation).unwrap(); - drop( - acquired - .locked - .begin_launch(acquired.lease.launch(), key.runtime(), &[7; CAPABILITY_LEN]) - .unwrap(), - ); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT); - let (kind, message) = refused(plan, Evidence::HostFailure); - assert_eq!(kind, ErrorKind::Other, "{message}"); -} - -#[test] -fn files_that_their_paths_do_not_name_as_checked_are_refused() { - let fixture = Fixture::new(); - let mut acquired = fixture.acquire("sandbox"); - // An image whose path is a link, which OpenVMM's open does not follow. - let (target, _) = fixture.image("target.erofs", IMAGE_LEN); - let link = fixture.path("images", "link.erofs"); - if symlink_file(&target, &link) { - let binding = DistroBinding::import(&layer(&link), PolicyVerdict::ACCEPTED).unwrap(); - let plan = fixture.plan(&mut acquired, &tmpfs_request(&link, &binding), NO_CONTENT); - assert_eq!( - refused(plan, Evidence::SuppliedIdentity), - ( - ErrorKind::Unavailable, - format!( - "the image {} cannot be used: it is a link, or not a regular file", - shown(&link) - ) - ) - ); - // A scratch image whose path is a link. - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let (target, _) = fixture.scratch("target.ext4", SCRATCH_LEN); - let link = fixture.path("scratches", "link.ext4"); - assert!(symlink_file(&target, &link)); - let scratch_binding = ScratchBinding::bind(&link).unwrap(); - let request = scratch_request(&image, &binding, &link, &scratch_binding); - assert_eq!( - refused( - fixture.plan(&mut acquired, &request, NO_CONTENT), - Evidence::SuppliedIdentity - ), - ( - ErrorKind::Unavailable, - format!( - "the scratch image {} cannot be used: it is a link, or not a regular file", - shown(&link) - ) - ) - ); - // The refused plan holds no claim of the image. - drop( - scratch_binding - .check(&link, Instant::now() + CLAIM_RELEASE_WAIT) - .unwrap(), - ); - } - // An image that others may change. - let path = fixture.path("images", "shared.erofs"); - File::create(&path).unwrap().set_len(IMAGE_LEN).unwrap(); - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - - fs::set_permissions(&path, fs::Permissions::from_mode(0o666)).unwrap(); - } - #[cfg(windows)] - platform::tests::set_access(&path, "D:P(A;;FA;;;OW)(A;;FA;;;SY)(A;;FA;;;WD)").unwrap(); - let binding = DistroBinding::import(&layer(&path), PolicyVerdict::ACCEPTED).unwrap(); - let problem = if cfg!(windows) { - "allows other users to change it" - } else { - "is writable by other users (mode 666)" - }; - let plan = fixture.plan(&mut acquired, &tmpfs_request(&path, &binding), NO_CONTENT); - assert_eq!( - refused(plan, Evidence::SuppliedIdentity), - ( - ErrorKind::Unavailable, - format!("the image {} {problem}", shown(&path)) - ) - ); - assert_untouched(&acquired); -} - -#[test] -fn the_activation_record_of_a_clone_in_ram_names_its_restore_image_and_settings() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = tmpfs_request(&image, &binding); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let record = plan.activation(); - let restore = record.restore().unwrap(); - assert_eq!( - record.configuration(), - &Configuration { - image: Image::Erofs { - uuid: Some(Uuid::parse(UUID).unwrap()), - }, - // A clone in RAM has no journal policy, whatever the request's. - upper: Upper::Tmpfs { - max_memory_percent: 50 - }, - overlay: OverlayMode::UntrustedLowers, - export: None, - mappings: 0, - network: None, - view: view(), - resources: ExpectedResources { - cpus: PROCESSORS, - memory: MEMORY, - }, - } - ); - // The plan shows the record, which holds nothing secret. - let shown_plan = format!("{plan:?}"); - assert!( - shown_plan.contains(&format!("activation: {record:?}")), - "{shown_plan}" - ); - // Each plan draws its own transaction. - let transaction = restore.transaction; - drop(plan); - let again = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let restored = again.activation().restore().unwrap(); - assert_ne!(restored.transaction, transaction); - // A larger memory target is the resources that the clone must have. - drop(again); - let larger = CloneRequest { - memory: 2 * MEMORY, - ..request - }; - let plan = fixture.plan(&mut acquired, &larger, NO_CONTENT).unwrap(); - assert_eq!( - plan.activation().configuration().resources.memory, - 2 * MEMORY - ); -} - -#[test] -fn the_activation_record_of_a_clone_on_a_scratch_image_names_only_its_journal() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let (scratch, scratch_binding) = fixture.scratch("scratch.ext4", SCRATCH_LEN); - let mut acquired = fixture.acquire("sandbox"); - for journal in [Journal::Forbidden, Journal::Allowed] { - let request = CloneRequest { - overlay: OverlayMode::TrustedLowers, - journal, - ..scratch_request(&image, &binding, &scratch, &scratch_binding) - }; - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let configuration = plan.activation().configuration(); - assert_eq!(configuration.upper, Upper::Scratch { journal }); - assert_eq!(configuration.overlay, OverlayMode::TrustedLowers); - assert_eq!( - configuration.image.expected(), - Some(Uuid::parse(UUID).unwrap()) - ); - } - // Another binding does not change the guest-applied upper. - let (other, other_binding) = fixture.scratch("other.ext4", SCRATCH_LEN); - let request = scratch_request(&image, &binding, &other, &other_binding); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - assert_eq!( - plan.activation().configuration().upper, - Upper::Scratch { - journal: Journal::Forbidden, - } - ); -} - -#[test] -fn the_activation_record_of_a_gpt_clone_names_its_disk_and_the_bound_of_its_first_partition() { - let fixture = Fixture::new(); - let (image, binding) = fixture.gpt_image("distro.gpt", IMAGE_LEN); - let (scratch, scratch_binding) = fixture.scratch("scratch.ext4", SCRATCH_LEN); - let mut acquired = fixture.acquire("sandbox"); - let gpt = |first_partition_bound| Image::Gpt { - disk_guid: Some(Uuid::parse(UUID).unwrap()), - first_partition_bound, - }; - // A GPT disk of the placeholder's geometry serves over RAM and over a scratch image, as an - // EROFS layer does, and its plan reads none of its content: the guest reads and checks the - // disk's partition table as it assembles the root. - for request in [ - tmpfs_request(&image, &binding), - scratch_request(&image, &binding, &scratch, &scratch_binding), - ] { - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - assert_eq!(plan.image().path().to_str().unwrap(), shown(&image)); - assert_eq!( - plan.image().geometry().unwrap(), - geometry(IMAGE_LEN, fixture.physical) - ); - let mut byte = [0]; - assert!(plan.image().file().read(&mut byte).is_err()); - let configuration = plan.activation().configuration(); - assert_eq!(configuration.image, gpt(FIRST_PARTITION_BOUND)); - assert_eq!(configuration.view, view()); - } - // The record keeps the request's bound, whatever it is from 1 on. - for bound in [1, u64::MAX] { - let request = CloneRequest { - first_partition_bound: bound, - ..tmpfs_request(&image, &binding) - }; - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - assert_eq!(plan.activation().configuration().image, gpt(bound)); - } -} - -#[test] -fn the_activation_record_of_a_clone_of_a_template_with_a_nic_names_its_own_network() { - let fixture = network_fixture(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let gateway = IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)); - // The template's NIC has the IPv6 gateway fd00::a00:1. - let ipv6_gateway = IpAddr::V6(Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0xa00, 1)); - let mut ipv6_dns = NetworkPolicy::new(NetworkAccess::Deny); - for protocol in [Protocol::Tcp, Protocol::Udp] { - ipv6_dns = ipv6_dns.with_allow(EgressRule::to("fd00::a00:1/128").on_port(protocol, 53)); - } - let mut outbound = NetworkPolicy::new(NetworkAccess::Allow); - outbound.host_loopback = HostLoopback::Outbound; - let mut policies = vec![ipv6_dns, outbound]; - policies.extend(attaching()); - let (unpublished, published) = (Publication::Unpublished, Publication::Published); - let expected = [ - (Some(ipv6_gateway), unpublished, None), - (Some(gateway), unpublished, None), - (Some(gateway), unpublished, None), - (None, unpublished, None), - (None, unpublished, Some("http://10.0.0.1:3128")), - (None, published, None), - (Some(gateway), published, None), - ]; - assert_eq!(policies.len(), expected.len()); - // Each clone's record names its own network, whatever the policy of the template's class: - // the DNS server that OpenVMM serves the guest at a gateway - // under its policy, whether the guest publishes its listeners, and the proxy's URL as the - // guest sees it, at its gateway. - for (policy, (dns, publication, proxy)) in policies.into_iter().zip(expected) { - let network = SandboxNetwork::new("10.0.0.2/24", policy).unwrap(); - let request = with_network(tmpfs_request(&image, &binding), Some(network.clone())); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let recorded = plan.activation().configuration().network.clone(); - assert_eq!( - recorded, - Some(Network { - dns, - publication, - proxy: proxy.map(str::to_owned), - }), - "{network:?}" - ); - } -} - -#[test] -fn settings_that_no_activation_record_holds_are_refused_before_anything_is_created() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = tmpfs_request(&image, &binding); - let with_view = |view: View| CloneRequest { - view, - ..request.clone() - }; - for (view, problem) in [ - ( - View { - hostname: Vec::new(), - ..view() - }, - "the hostname is not 1 to 64 bytes without a NUL", - ), - ( - View { - hostname: vec![b'h'; 65], - ..view() - }, - "the hostname is not 1 to 64 bytes without a NUL", - ), - ( - View { - hostname: b"clone\0".to_vec(), - ..view() - }, - "the hostname is not 1 to 64 bytes without a NUL", - ), - ] { - let plan = fixture.plan(&mut acquired, &with_view(view), NO_CONTENT); - assert_eq!( - refused(plan, Evidence::InvalidActivate), - ( - ErrorKind::Unsupported, - format!("no clone activates with the settings of the request: {problem}") - ) - ); - assert_untouched(&acquired); - } - // A GPT disk's first-partition bound of 0, which an EROFS layer's record leaves out. - let (path, gpt_binding) = fixture.gpt_image("distro.gpt", IMAGE_LEN); - let unbounded = CloneRequest { - first_partition_bound: 0, - ..tmpfs_request(&path, &gpt_binding) - }; - let plan = fixture.plan(&mut acquired, &unbounded, NO_CONTENT); - assert_eq!( - refused(plan, Evidence::InvalidActivate), - ( - ErrorKind::Unsupported, - "no clone activates with the settings of the request: the bound of a GPT disk's first \ - partition is 0" - .to_owned() - ) - ); - assert_untouched(&acquired); - let erofs = CloneRequest { - first_partition_bound: 0, - ..request.clone() - }; - drop(fixture.plan(&mut acquired, &erofs, NO_CONTENT).unwrap()); - drop(fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap()); -} - -#[test] -fn only_a_path_that_no_longer_names_its_supplied_file_is_the_request_s() { - // The open's refusal of a link or of a file that is not regular, which the system does not - // report, a file that went, and a path that the current user may no longer open, are the - // path's. - assert!(path_refused(&io::Error::other( - "it is a link, or not a regular file" - ))); - #[cfg(unix)] - let (refusals, failures) = { - use nix::errno::Errno; - - ( - [Errno::ENOENT, Errno::ENOTDIR, Errno::EACCES], - [Errno::EIO, Errno::EMFILE, Errno::ENOMEM], - ) - }; - // ERROR_FILE_NOT_FOUND, ERROR_PATH_NOT_FOUND, and ERROR_ACCESS_DENIED; ERROR_IO_DEVICE, - // ERROR_NOT_ENOUGH_MEMORY, and ERROR_NO_SYSTEM_RESOURCES. - #[cfg(windows)] - let (refusals, failures) = ([2_u32, 3, 5], [1117_u32, 8, 1450]); - for errno in refusals { - let error = io::Error::from_raw_os_error(errno as i32); - assert!(path_refused(&error), "{error}"); - } - for errno in failures { - let error = io::Error::from_raw_os_error(errno as i32); - assert!(!path_refused(&error), "{error}"); - } - // A file that went from its path is the request's. - let directory = tempfile::tempdir().unwrap(); - let path = directory.path().join("image"); - let fault = check_at_path(&path, (1, 2), "the image").unwrap_err(); - assert_eq!(fault.evidence, Evidence::SuppliedIdentity); - assert_eq!( - fault.error.kind(), - ErrorKind::Unavailable, - "{}", - fault.error - ); - assert!( - fault - .error - .message() - .starts_with("the image cannot be used: "), - "{}", - fault.error - ); -} - -#[test] -fn a_clone_with_an_export_records_its_plan_and_maps_from_its_children() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - // An aggregate of directories and files, read-only and read-write, with hidden and re-exposed - // paths, as a cold launch exports one. - let (export, _) = fixture.aggregate_export("aggregate"); - assert_eq!(export.children.len(), 4); - let request = with_export(tmpfs_request(&image, &binding), &export); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - assert_eq!(plan.export(), Some(&export)); - // The record names the export's shape and announces its mappings, not its plan identity. - let configuration = plan.activation().configuration(); - assert_eq!( - configuration.export, - Some(Export { - kind: ExportKind::Aggregate, - }) - ); - assert_eq!(configuration.mappings, export.binds.len()); - // The table is a cold launch's: its sources name each path by its child's index, as the - // aggregate's root lists the children. - assert_eq!(plan.table(), export.guest_table()); - let sources: Vec = plan - .table() - .into_iter() - .map(|mapping| mapping.source) - .collect(); - assert_eq!(sources, ["0", "1", "2", "3", "0/work"]); - // Another aggregate still records only its shape and mapping count. - drop(plan); - let other = fixture.data_export("other"); - let request = with_export(tmpfs_request(&image, &binding), &other); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let recorded = plan.activation().configuration().export.unwrap(); - assert_eq!(recorded.kind, ExportKind::Aggregate); - assert_eq!( - plan.activation().configuration().mappings, - other.binds.len() - ); - assert_eq!(plan.table(), data_table()); - // A clone without mappings has no export, and its record none, so its template's slot stays - // dormant. - drop(plan); - let request = tmpfs_request(&image, &binding); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - assert_eq!(plan.export(), None); - assert!(plan.table().is_empty()); - let configuration = plan.activation().configuration(); - assert_eq!((&configuration.export, configuration.mappings), (&None, 0)); -} - -#[test] -fn exports_that_no_cold_launch_attaches_are_refused_before_anything_is_opened() { - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = tmpfs_request(&image, &binding); - let mut refusal = |export: &ExportPlan| { - let request = with_export(request.clone(), export); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT); - let (kind, message) = refused(plan, Evidence::InvalidMappings); - assert_eq!(kind, ErrorKind::Unsupported, "{message}"); - message - }; - // Mappings without an export, an export without mappings, and an export of more directories - // and files than OpenVMM attaches. - let export = fixture.data_export("data"); - let unexported = ExportPlan { - children: Vec::new(), - ..export.clone() - }; - assert_eq!( - refusal(&unexported), - "the recorded export plan is invalid: it exports 0 directories and files, but OpenVMM \ - exports from 1 to 256" - ); - let unmapped = ExportPlan { - binds: Vec::new(), - ..export.clone() - }; - assert_eq!( - refusal(&unmapped), - "the recorded export plan is invalid: it maps 0 paths, but a sandbox maps from 1 to 4096" - ); - let crowded = ExportPlan { - children: vec![export.children[0].clone(); filesystem::MAX_CHILDREN + 1], - ..export.clone() - }; - assert_eq!( - refusal(&crowded), - "the recorded export plan is invalid: it exports 257 directories and files, but OpenVMM \ - exports from 1 to 256" - ); - // A plan that maps a target that the container root keeps for itself. - let mut reserved = export.clone(); - reserved.binds[1].target = "/proc/work".to_owned(); - let message = refusal(&reserved); - assert!( - message.starts_with("the recorded export plan is invalid"), - "{message}" - ); - // None of them opened anything, and an aggregate of directories and files serves a clone, as - // the export of one directory does. - assert_untouched(&acquired); - let (aggregate, _) = fixture.aggregate_export("aggregate"); - for export in [aggregate, export] { - let request = with_export(request.clone(), &export); - drop(fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap()); - } -} - -#[test] -fn exports_that_overlap_the_template_store_are_refused_before_anything_is_opened() { - // The plan compares the paths that the export's plan resolved with the store's, resolved as - // well: this store's path names its directory in another spelling than its resolved one. - let fixture = Fixture::in_store(|directory| alias(directory).join("templates"), |_, _, _| {}); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = tmpfs_request(&image, &binding); - let store = filesystem::canonicalize(fixture.store.dir()).unwrap(); - assert_ne!(store, fixture.store.dir()); - let payload = fixture.store.payload_dir(fixture.generation.generation()); - // The store's directory, the payload of a generation inside it, and a directory that holds - // the store, which a guest could reach the payload through, and write it if writable. - for (dir, read_only) in [ - (store.clone(), true), - (payload.clone(), false), - (fixture.directory.path().to_owned(), true), - ] { - let export = export_plan(&[(&dir, "/data", read_only)], &[], &[]); - let request = with_export(request.clone(), &export); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT); - assert_eq!( - refused(plan, Evidence::InvalidMappings), - ( - ErrorKind::Unsupported, - format!( - "the export of a clone cannot share {}, which overlaps the template store {}, \ - whose leased payloads nothing may change", - export.children[0].root.display(), - store.display() - ) - ) - ); - assert_untouched(&acquired); - } - // Every directory and file of an aggregate is checked: a file of the store's, beside a - // directory that is no part of it. - let beside = fixture.exported("beside", &[]); - let key = filesystem::canonicalize(&payload.join(KEY_NAME)).unwrap(); - let export = export_plan( - &[(&beside, "/beside", false), (&key, "/key.bin", true)], - &[], - &[], - ); - assert_eq!(export.children.len(), 2); - let request = with_export(request, &export); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT); - assert_eq!( - refused(plan, Evidence::InvalidMappings), - ( - ErrorKind::Unsupported, - format!( - "the export of a clone cannot share {}, which overlaps the template store {}, \ - whose leased payloads nothing may change", - key.display(), - store.display() - ) - ) - ); - assert_untouched(&acquired); - // A directory beside the store is no part of it. - let export = fixture.data_export("data"); - let request = with_export(request, &export); - drop(fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap()); -} - -#[test] -fn exports_that_let_the_guest_turn_the_template_store_s_path_are_refused() { - // The store's path leads through two links: `share/link`, to `hop/next`, which links to - // `private`, which holds the store. Neither directory of a link holds any part of the store, - // but a guest that could write in either could replace its link, and turn the path, by which - // the host reaches the store, and OpenVMM the snapshots in it, elsewhere. - let fixture = Fixture::in_store( - |directory| { - for name in ["share", "hop", "private"] { - fs::create_dir(directory.join(name)).unwrap(); - } - fs::create_dir(directory.join("share").join("work")).unwrap(); - let next = directory.join("hop").join("next"); - link_dir(&directory.join("private"), &next); - let link = directory.join("share").join("link"); - link_dir(&next, &link); - link.join("templates") - }, - |_, _, _| {}, - ); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = tmpfs_request(&image, &binding); - let resolved = |name: &str| filesystem::canonicalize(&fixture.directory.path().join(name)); - let (share, hop) = (resolved("share").unwrap(), resolved("hop").unwrap()); - // The store's path resolves outside both directories, so no export of either overlaps it. - let store = filesystem::canonicalize(fixture.store.dir()).unwrap(); - assert_eq!(store, resolved("private").unwrap().join("templates")); - // A writable export of either directory. - for dir in [&share, &hop] { - let export = export_plan(&[(dir, "/data", false)], &[], &[]); - let request = with_export(request.clone(), &export); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT); - assert_eq!( - refused(plan, Evidence::InvalidMappings), - ( - ErrorKind::Unsupported, - format!( - "the export of a clone cannot let its guest write in {}, through which the \ - path of the template store {} leads, since the guest could then turn that \ - path elsewhere, or keep the host from following it", - dir.display(), - fixture.store.dir().display() - ) - ) - ); - assert_untouched(&acquired); - } - // Every directory of an aggregate is checked: a writable one beside a read-only directory - // through which the store's path does not lead. - let beside = fixture.exported("beside", &[]); - let export = export_plan( - &[(&beside, "/beside", true), (&hop, "/hop", false)], - &[], - &[], - ); - assert_eq!(export.children.len(), 2); - let plan = fixture.plan( - &mut acquired, - &with_export(request.clone(), &export), - NO_CONTENT, - ); - assert_eq!( - refused(plan, Evidence::InvalidMappings), - ( - ErrorKind::Unsupported, - format!( - "the export of a clone cannot let its guest write in {}, through which the path \ - of the template store {} leads, since the guest could then turn that path \ - elsewhere, or keep the host from following it", - hop.display(), - fixture.store.dir().display() - ) - ) - ); - assert_untouched(&acquired); - // A read-only export of the directory of a link, one in which the guest writes only a - // directory beside the link, and one of a file beside the link, whose content alone the - // guest writes, leave the link as it is. - let work = share.join("work"); - let notes = share.join("notes.txt"); - fs::write(¬es, b"notes").unwrap(); - for mappings in [ - &[(share.as_path(), "/data", true)][..], - &[ - (share.as_path(), "/data", true), - (work.as_path(), "/work", false), - ], - &[(notes.as_path(), "/notes.txt", false)], - ] { - let export = export_plan(mappings, &[], &[]); - let request = with_export(request.clone(), &export); - drop(fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap()); - } -} - -#[test] -fn exports_that_let_the_guest_block_the_template_store_s_path_are_refused() { - // The store's path leads through `traverse` and its `..`, which Unix looks up in `traverse`: - // a guest that could write there could make it unsearchable, and keep the host from the - // store. Windows resolves `..` from the path's text, and never looks in `traverse`. - let fixture = Fixture::in_store( - |directory| { - for name in ["traverse", "private"] { - fs::create_dir(directory.join(name)).unwrap(); - } - fs::create_dir(directory.join("traverse").join("work")).unwrap(); - directory - .join("traverse") - .join("..") - .join("private") - .join("templates") - }, - |_, _, _| {}, - ); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = tmpfs_request(&image, &binding); - let traverse = filesystem::canonicalize(&fixture.directory.path().join("traverse")).unwrap(); - // A writable export of the directory. - let export = export_plan(&[(&traverse, "/data", false)], &[], &[]); - let writable = with_export(request.clone(), &export); - let plan = fixture.plan(&mut acquired, &writable, NO_CONTENT); - if cfg!(windows) { - drop(plan.unwrap()); - } else { - assert_eq!( - refused(plan, Evidence::InvalidMappings), - ( - ErrorKind::Unsupported, - format!( - "the export of a clone cannot let its guest write in {}, through which the \ - path of the template store {} leads, since the guest could then turn that \ - path elsewhere, or keep the host from following it", - traverse.display(), - fixture.store.dir().display() - ) - ) - ); - assert_untouched(&acquired); - } - // A read-only export of the directory, and one in which the guest writes only a directory - // in it, leave the directory as it is. - let work = traverse.join("work"); - for mappings in [ - &[(traverse.as_path(), "/data", true)][..], - &[ - (traverse.as_path(), "/data", true), - (work.as_path(), "/work", false), - ], - ] { - let export = export_plan(mappings, &[], &[]); - let request = with_export(request.clone(), &export); - drop(fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap()); - } -} - -#[test] -fn exports_that_hold_the_sandbox_s_directory_are_refused() { - // The host writes the clone's records in the sandbox's directory, by their paths, so a guest - // that could reach the directory could change them, or replace it with a link, and turn the - // host's writes elsewhere, into a leased snapshot, say. - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = tmpfs_request(&image, &binding); - let sandbox = acquired.lease.sandbox().to_owned(); - // The sandbox's directory, writable, and the directory that holds it, read-only. - for (dir, read_only) in [ - (sandbox.clone(), false), - (sandbox.parent().unwrap().to_owned(), true), - ] { - let export = export_plan(&[(&dir, "/data", read_only)], &[], &[]); - let request = with_export(request.clone(), &export); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT); - assert_eq!( - refused(plan, Evidence::InvalidMappings), - ( - ErrorKind::Unsupported, - format!( - "the export of a clone cannot share {}, which holds the directory of the \ - clone's sandbox {}, in which the host writes the clone's records", - dir.display(), - sandbox.display() - ) - ) - ); - assert_untouched(&acquired); - } - // A directory inside the sandbox's, among the caller's files, holds none of the host's - // records. - let shared = sandbox.join("shared"); - fs::create_dir(&shared).unwrap(); - let export = export_plan(&[(&shared, "/data", false)], &[], &[]); - let request = with_export(request, &export); - drop(fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap()); -} - -#[test] -fn files_in_the_sandbox_s_directory_are_refused() { - // A file in the sandbox's directory itself may be one of the host's records, such as the - // clone's binding, which names its lease, and which a guest could otherwise read or change. - let fixture = Fixture::new(); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = tmpfs_request(&image, &binding); - let sandbox = acquired.lease.sandbox().to_owned(); - let record = sandbox.join(CLONE_BINDING_NAME); - assert!(record.is_file(), "{}", record.display()); - // The record, read-only and writable, beside a directory that is no part of the sandbox's: an - // aggregate's every file is checked. - let beside = fixture.exported("beside", &[]); - for read_only in [true, false] { - let export = export_plan( - &[ - (&beside, "/beside", true), - (&record, "/clone.json", read_only), - ], - &[], - &[], - ); - assert_eq!(export.children.len(), 2); - let plan = fixture.plan( - &mut acquired, - &with_export(request.clone(), &export), - NO_CONTENT, - ); - assert_eq!( - refused(plan, Evidence::InvalidMappings), - ( - ErrorKind::Unsupported, - format!( - "the export of a clone cannot share {}, a file in the directory of the \ - clone's sandbox {}, in which the host writes the clone's records", - record.display(), - sandbox.display() - ) - ) - ); - assert_untouched(&acquired); - } - // A file in a directory inside the sandbox's, among the caller's files, is none of the host's - // records. - let shared = sandbox.join("shared"); - fs::create_dir(&shared).unwrap(); - let notes = shared.join("notes.txt"); - fs::write(¬es, b"notes").unwrap(); - let export = export_plan(&[(¬es, "/notes.txt", false)], &[], &[]); - assert!(export.children[0].file); - let request = with_export(request, &export); - drop(fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap()); -} - -#[test] -fn the_export_of_a_clone_is_attached_and_mapped_as_a_cold_launch_attaches_and_maps_it() { - let fixture = Fixture::new(); - let (mut export, [data, notes, settings, tools]) = fixture.aggregate_export("aggregate"); - let (work, secret) = (data.join("work"), data.join("secret")); - let shown = secret.join("shown"); - let child = |index: usize, path: &Path, mode: &str| { - format!("{index},{},{mode}", path.to_str().unwrap()) - }; - let text = |path: &Path| path.to_str().unwrap().to_owned(); - let attached = CloneExport::new(&export).unwrap(); - assert_eq!(attached.plan(), &export); - // The restore attaches the aggregate with a cold launch's arguments: each directory and file - // a child, by its index, with its mode, `rw` if a mapping writes it, a file marked as one, - // then the paths that OpenVMM hides, shows again, and lets the guest write. The guest reads - // `data` but for `work`, and `secret` hides all of it but `shown`. - assert_eq!(attached.arguments(), export.openvmm_arguments()); - assert_eq!( - attached.arguments(), - [ - "--mount-aggregate".to_owned(), - filesystem::GUEST_EXPORT.to_owned(), - "--mount-child".to_owned(), - child(0, &data, "rw"), - "--mount-child".to_owned(), - child(1, ¬es, "rw,file"), - "--mount-child".to_owned(), - child(2, &settings, "ro,file"), - "--mount-child".to_owned(), - child(3, &tools, "ro"), - "--mount-deny".to_owned(), - text(&secret), - "--mount-allow".to_owned(), - text(&shown), - "--mount-write".to_owned(), - text(&work), - ] - ); - // Its table is a cold launch's, each mapping in the plan's order, by its child's index. - let mapping = |source: &str, target: &str, read_only| HostMap { - source: source.to_owned(), - target: target.to_owned(), - read_only, - }; - assert_eq!(attached.table(), export.guest_table()); - assert_eq!( - attached.table(), - [ - mapping("0", "/data", true), - mapping("1", "/notes.txt", false), - mapping("2", "/settings.json", true), - mapping("3", "/tools", true), - mapping("0/work", "/work", false), - ] - ); - // An export whose operations run as the guest's caller says so last. - export.owner = filesystem::Owner::Caller; - let arguments = CloneExport::new(&export).unwrap().arguments(); - assert_eq!( - arguments[arguments.len() - 2..], - ["--mount-owner", "caller"] - ); - // An aggregate that the guest only reads is attached read-only. - let read_only = export_plan(&[(&data, "/data", true)], &[], &[]); - assert_eq!( - CloneExport::new(&read_only).unwrap().arguments(), - [ - "--mount-aggregate".to_owned(), - filesystem::GUEST_EXPORT.to_owned(), - "--mount-child".to_owned(), - child(0, &data, "ro"), - ] - ); -} diff --git a/nvx-rs/src/host/template/clone/tests/launch.rs b/nvx-rs/src/host/template/clone/tests/launch.rs deleted file mode 100644 index ccd088aa8..000000000 --- a/nvx-rs/src/host/template/clone/tests/launch.rs +++ /dev/null @@ -1,3820 +0,0 @@ -//! Tests of the launches of clones: OpenVMM's restore of the leased snapshot in place with the -//! clone's devices, activation record, and restore-ready endpoint, for which a -//! [`restore_vmm_role`] stands in, listening beside the snapshot on Linux as OpenVMM does; the -//! checks that keep a VMM from starting before the clone's binding is durable, or with a control -//! socket that OpenVMM or the host could not use; the removal of that socket; and the crashes of -//! a launch's caller, which a [`launch_role`] runs, in the launch and in the restore sequence that -//! follows it, around the clone's Ready and the start of its sandbox. - -use std::cell::RefCell; -use std::env; -use std::ffi::OsString; -use std::io::{Read, Write}; -use std::process::{Command, Stdio}; -use std::thread; -use std::time::{Duration, Instant}; - -use super::*; -use crate::activation::{ReadyActivation, StoreGeneration, TransactionId}; -use crate::host::artifacts; -use crate::host::platform::Transport; -use crate::host::process::{BootId, Opened, Process, ProcessRef}; -use crate::host::snapshot::tests::artifact; -use crate::host::store::files::{sync_dir, write_private}; -use crate::host::store::{ - CLONE_ACTIVATION_NAME, CLONE_BINDING_NAME, CLONE_FAILURE_NAME, CLONE_PROGRESS_NAME, - JOURNAL_NAME, LaunchState, LockedStore, OUTCOME_NAME, RESTORE_READY_NAME, Recovery, - RuntimeIdentity, RuntimeRecord, -}; -use crate::host::supervisor::{self, Reattached, StopOutcome, Vmm, VmmCommand}; -use crate::host::template::Retirement; -use crate::host::template::clone::launch::{Hooks, Point}; -use crate::host::template::clone::reattach::{reattach_clone_syncing, reattach_clone_with}; -use crate::host::template::clone::records::{read_failure, read_progress, write_failure}; -use crate::host::template::store::tests::SNAPSHOT_ARTIFACTS; - -/// The environment variable that names the template store of a [`launch_role`]. -const ROLE_STORE: &str = "NVX_CLONE_LAUNCH_TEST_STORE"; -/// The environment variable that names the store generation that a [`launch_role`] leases. -const ROLE_GENERATION: &str = "NVX_CLONE_LAUNCH_TEST_GENERATION"; -/// The environment variable that names the directory of the sandbox of a [`launch_role`]. -const ROLE_SANDBOX: &str = "NVX_CLONE_LAUNCH_TEST_SANDBOX"; -/// The environment variable that names the image of the clone of a [`launch_role`]. -const ROLE_IMAGE: &str = "NVX_CLONE_LAUNCH_TEST_IMAGE"; -/// The environment variable that names where a [`launch_role`] crashes: at the point `prepared` -/// or `begun` of its launch, once its launch returned, `launched`, at the steps `activating`, -/// `activated`, `starting`, `started`, and `committed` of its restore sequence, and once its -/// commit left the launch's journal beside the runtime record, `superseded`. -const ROLE_CRASH: &str = "NVX_CLONE_LAUNCH_TEST_CRASH"; -/// The environment variable that names the working directory of the VMM of a [`launch_role`]. -const ROLE_VMM: &str = "NVX_CLONE_LAUNCH_TEST_VMM"; -/// The start of the line in which [`launch_role`] prints the reference of the VMM that it -/// spawned, as JSON. -const REFERENCE: &str = "nvx-clone-launch-test-vmm "; -/// The start of the line that [`launch_role`] prints as it crashes. -const CRASHED: &str = "nvx-clone-launch-test-crashed "; -/// File whose presence in its working directory marks a [`restore_vmm_role`]. Without it, the -/// role returns at once, as it does when `cargo test -- --ignored` runs it. -const VMM_MARKER: &str = "vmm-role"; -/// File whose appearance in its working directory ends a [`restore_vmm_role`]. -const EXIT_MARKER: &str = "exit"; -/// File in which [`fake_vmm`] writes the arguments that a launch gave its VMM, as JSON, for the -/// [`restore_vmm_role`] that runs instead. -const ARGUMENTS: &str = "arguments.json"; -/// File in which a [`restore_vmm_role`] writes, once it has tried to send its restore-ready -/// event, the capability that it read, in hexadecimal, and whether it sent the event. -const STARTED: &str = "started"; -/// File in the working directory of a [`restore_vmm_role`] whose words change how it sends its -/// restore-ready event, as [`VmmDir::plan`] describes them. Without it, the VMM sends the event at -/// once, and closes its end of the endpoint. -const EVENT_PLAN: &str = "event-plan"; -/// File in which a [`restore_vmm_role`] whose plan asks it to flush its event as OpenVMM does -/// notes that the flush returned: on Windows, once the host read the event. -const FLUSHED: &str = "flushed"; -/// The OpenVMM executable that the tests' launches name, which [`fake_vmm`] replaces. -pub(super) const OPENVMM: &str = "openvmm"; -/// Longest wait for a VMM that is expected to start or exit, for a sandbox's lock, or for a -/// reattach to become determinate. -pub(super) const TIMEOUT: Duration = Duration::from_secs(60); -/// Longest life of a [`restore_vmm_role`] that nothing ends, so that a failed test leaves no -/// process. -const VMM_LIFETIME: Duration = Duration::from_secs(180); -/// How long a [`restore_vmm_role`] whose plan closes the event's endpoint waits once it connected -/// the endpoint, and once it closed it, before it reports its error. -const CLOSE_PAUSE: Duration = Duration::from_millis(300); - -/// Returns `bytes` in lowercase hexadecimal. -pub(super) fn hex(bytes: &[u8]) -> String { - bytes.iter().map(|byte| format!("{byte:02x}")).collect() -} - -/// Returns the arguments that run only the role `name` of this test executable, on libtest's main -/// thread, so that libtest prints the role's name before the role's own output. -fn role(name: &str) -> Vec { - [ - "--exact", - &format!("host::template::clone::tests::launch::{name}"), - "--ignored", - "--nocapture", - "--test-threads=1", - ] - .map(OsString::from) - .to_vec() -} - -/// Gives the snapshot of the generation of `fixture` its RAM and saved state beside its manifest, -/// as OpenVMM publishes them, and returns the snapshot's directory, and the content of each of -/// its files, by name. -pub(super) fn complete_snapshot(fixture: &Fixture) -> (PathBuf, Vec<(&'static str, Vec)>) { - let snapshot = fixture.store.snapshot_dir(&fixture.generation).unwrap(); - let mut contents = Vec::new(); - for name in SNAPSHOT_ARTIFACTS { - let path = snapshot.join(name); - if !path.exists() { - fs::write(&path, format!("the {name} of the template")).unwrap(); - } - contents.push((name, fs::read(&path).unwrap())); - } - (snapshot, contents) -} - -/// Returns the seals of the files of the snapshot in `snapshot`, by name, which change with their -/// content, their length, their modification time, and on Linux their change time, which a -/// restore checks. -fn seals(snapshot: &Path) -> Vec<(&'static str, FileSeal)> { - SNAPSHOT_ARTIFACTS - .map(|name| { - let file = File::open(snapshot.join(name)).unwrap(); - (name, FileSeal::of(&file).unwrap()) - }) - .to_vec() -} - -/// Returns the runtime identity that captured the template of `fixture`, under which its clones -/// launch. -pub(super) fn runtime_of(fixture: &Fixture) -> RuntimeIdentity { - fixture - .store - .read_key(&fixture.generation) - .unwrap() - .runtime() - .clone() -} - -/// Returns whether a socket, or another file, is at `path`. -pub(super) fn exists(path: &Path) -> bool { - fs::symlink_metadata(path).is_ok() -} - -/// Asserts that the payload of the generation of `fixture` holds no control socket of a clone's -/// VMM, and that the files of the generation's snapshot in `snapshot` have the contents -/// `contents` and the seals `sealed` still. -fn assert_untouched( - fixture: &Fixture, - snapshot: &Path, - contents: &[(&str, Vec)], - sealed: &[(&str, FileSeal)], -) { - let payload = fixture.store.payload_dir(fixture.generation.generation()); - for entry in fs::read_dir(payload).unwrap() { - let name = entry.unwrap().file_name(); - assert!(!name.to_string_lossy().ends_with(".sock"), "{name:?}"); - } - for (name, content) in contents { - assert_eq!(&fs::read(snapshot.join(name)).unwrap(), content, "{name}"); - } - assert_eq!(seals(snapshot), sealed); -} - -/// Asserts that no launch of the clone of `acquired` began or left a file: no journal, activation -/// record, restore-ready socket, or control socket of its VMM, and that the lease still reserves -/// its launch. -fn assert_unlaunched(fixture: &Fixture, acquired: &Acquired) { - let locked = &acquired.locked; - assert!(locked.journal().unwrap().is_none()); - for path in [ - locked.clone_activation_path(), - locked.restore_ready_path(), - fixture.store.clone_socket_path(&acquired.lease), - ] { - assert!(!exists(&path), "{}", path.display()); - } - assert_eq!(locked.reserved(), Some(acquired.lease.launch())); -} - -/// Asserts that the sandbox in `sandbox` holds none of the files that the launch of a clone -/// leaves for its release. -pub(super) fn assert_released(sandbox: &Path) { - for name in [ - CLONE_ACTIVATION_NAME, - CLONE_PROGRESS_NAME, - CLONE_FAILURE_NAME, - RESTORE_READY_NAME, - ] { - assert!(!exists(&sandbox.join(name)), "{name}"); - } -} - -/// Checks that `failure`, of the clone of `lease` in the sandbox of `locked`, failed at `stage`, -/// once the clone got as far as `progress`, with `evidence`, which attributes it, and the teardown -/// `teardown`, and that the sandbox records it, with the clone's lease and the transaction of its -/// restore. -pub(super) fn assert_recorded( - failure: &CloneFailure, - lease: &Lease, - locked: &LockedStore, - (stage, progress): (Stage, Progress), - evidence: Evidence, - teardown: Teardown, -) { - let record = failure.record(); - assert_eq!( - ( - record.stage, - record.progress, - record.evidence, - &record.teardown - ), - (stage, progress, evidence, &teardown), - "{failure}" - ); - assert_eq!(failure.outcome(), evidence.outcome(), "{failure}"); - assert_eq!( - (record.store, record.lease, record.template, &record.launch), - (lease.store(), lease.id(), lease.template(), lease.launch()) - ); - let activation = fs::read(locked.clone_activation_path()).unwrap(); - let activation = ActivationRecord::decode(&activation).unwrap(); - let transaction = activation.restore().unwrap().transaction; - assert_eq!(record.transaction, Some(transaction)); - assert_eq!(record.code, failure.code()); - assert!(record.message.starts_with(failure.message()), "{failure}"); - let records = CloneRecords::read(locked).unwrap().unwrap(); - assert_eq!(records.failure.as_ref(), Some(record)); - assert_eq!(&records.lease, lease); -} - -/// Returns what a reattach found, as test messages say it. -pub(super) fn describe(reattached: &ReattachedClone) -> String { - match reattached { - ReattachedClone::Live { vmm, .. } => format!("the live VM {:?}", vmm.record()), - ReattachedClone::Ended { progress, report } => { - format!("a clone that ended at its step {progress}, with the report {report:?}") - } - ReattachedClone::Failed(failure) => format!("the failure {:?}", failure.record()), - ReattachedClone::Indeterminate(reason) => format!("an indeterminate VM: {reason}"), - } -} - -/// Returns the failure that a reattach found, and fails on anything else. -pub(super) fn failed(reattached: ReattachedClone) -> CloneFailure { - match reattached { - ReattachedClone::Failed(failure) => failure, - other => panic!("the reattach found {}", describe(&other)), - } -} - -/// The seam of the command of a launch that must start no VMM. -fn never_spawned(_: VmmCommand<'_>) -> VmmCommand<'_> { - panic!("a launch that is refused made the command of a VMM") -} - -/// The command that a launch made for its VMM, before [`fake_vmm`] replaced it, and what the -/// sandbox held as the VMM was about to start. -#[derive(Debug)] -pub(super) struct Made { - /// The VMM's executable. - program: PathBuf, - /// The VMM's arguments. - arguments: Vec, - /// The VMM's working directory. - working_dir: PathBuf, - /// How many descriptors or handles the VMM inherits. - inherited: usize, - /// Whether the sandbox held its clone binding. - bound: bool, - /// Whether the sandbox held the launch's journal. - begun: bool, -} - -/// Returns the seam of the command of a launch whose VMM is a [`restore_vmm_role`] in `vmm`: it -/// keeps the command that the launch made in `made`, writes its arguments in [`ARGUMENTS`] in -/// `vmm`, and runs the role there instead, with the descriptors or handles that the command -/// passes. -pub(super) fn fake_vmm<'a>( - vmm: &'a Path, - made: &'a RefCell>, -) -> impl for<'h> Fn(VmmCommand<'h>) -> VmmCommand<'h> + 'a { - move |command| { - let arguments: Vec = command - .arguments - .iter() - .map(|argument| argument.to_str().unwrap().to_owned()) - .collect(); - fs::write(vmm.join(ARGUMENTS), serde_json::to_vec(&arguments).unwrap()).unwrap(); - made.replace(Some(Made { - program: command.program.clone(), - arguments, - working_dir: command.working_dir.clone(), - inherited: command.inherit.len(), - bound: command.working_dir.join(CLONE_BINDING_NAME).is_file(), - begun: command.working_dir.join(JOURNAL_NAME).is_file(), - })); - VmmCommand { - program: env::current_exe().unwrap(), - arguments: role("restore_vmm_role"), - working_dir: vmm.to_owned(), - breakaway_from_job: command.breakaway_from_job, - inherit: command.inherit, - } - } -} - -/// The working directory of a [`restore_vmm_role`], which it marks. Dropping it ends the VMM, and -/// waits for it to exit, so that a failed test leaves no process behind. -pub(super) struct VmmDir { - /// The reference of the VMM, once the test knows it. - pub(super) reference: Option, - /// The directory, which is removed after the VMM exits. - dir: tempfile::TempDir, -} - -impl VmmDir { - /// Creates the directory, and marks it. - pub(super) fn new() -> Self { - let dir = tempfile::tempdir().unwrap(); - fs::write(dir.path().join(VMM_MARKER), b"").unwrap(); - Self { - reference: None, - dir, - } - } - - /// Returns the path of the directory. - pub(super) fn path(&self) -> &Path { - self.dir.path() - } - - /// Waits until the VMM has tried to send its restore-ready event, and returns what it wrote - /// in [`STARTED`]. - pub(super) fn started(&self) -> String { - let path = self.path().join(STARTED); - let deadline = Instant::now() + TIMEOUT; - loop { - match fs::read_to_string(&path) { - Ok(started) => return started, - Err(error) if error.kind() == io::ErrorKind::NotFound => { - assert!(Instant::now() < deadline, "the VMM did not start"); - thread::sleep(Duration::from_millis(10)); - } - Err(error) => panic!("cannot read {}: {error}", path.display()), - } - } - } - - /// Asks the VMM to exit. - pub(super) fn end(&self) { - fs::write(self.path().join(EXIT_MARKER), b"").unwrap(); - } - - /// Asks the VMM, which runs, to exit, and waits until it has. It opens the VMM's process - /// before it asks, since a process that has exited no longer opens. - pub(super) fn end_and_wait(&self) { - let reference = self.reference.as_ref().expect("the test knows the VMM"); - let Opened::Running(process) = Process::open(reference).unwrap() else { - panic!("the VMM runs until the test ends it"); - }; - self.end(); - assert!(process.wait(Instant::now() + TIMEOUT).unwrap().is_some()); - } - - /// Waits until the VMM, which exits on its own, as its [plan](Self::plan) asks, has exited. - pub(super) fn wait_exit(&self) { - let reference = self.reference.as_ref().expect("the test knows the VMM"); - if let Opened::Running(process) = Process::open(reference).unwrap() { - assert!(process.wait(Instant::now() + TIMEOUT).unwrap().is_some()); - } - } - - /// Changes how the VMM sends its restore-ready event, before it starts, by the words of - /// `plan`: - /// - /// - `delay:MS` sends it after MS milliseconds; - /// - `none` sends nothing, and runs on; - /// - `exit` sends nothing, and exits at once; - /// - `close` connects to the event's endpoint, and closes it without a byte of the event, then - /// exits, [`CLOSE_PAUSE`] after each, as OpenVMM does once the launch of its VM worker - /// fails; - /// - `fatal:CODE`, with `exit` or `close`, writes the line with which OpenVMM reports a fatal - /// error of the time ABI code `CODE`, and the error's chain, on its standard error, after a - /// warning that names another code, before it exits; - /// - `forward:PORT`, with `exit` or `close`, writes the line with which OpenVMM reports a fatal - /// error without a code, and the error's chain, whose second cause names the forward of host - /// loopback's TCP port `PORT` that OpenVMM cannot bind, on its standard error, after a - /// warning that names that forward too, before it exits; - /// - `other` sends a version of the event that OpenVMM does not send; - /// - `partial` sends the first bytes of the event, then closes its end of the endpoint; - /// - `flush` flushes the event as OpenVMM does, which waits on Windows until the host read - /// it, before the VMM writes [`STARTED`], then notes it in [`FLUSHED`]; - /// - `report:CATEGORY:STATUS` writes OpenVMM's outcome report of a VM whose outcome is of the - /// category `CATEGORY`, with the status `STATUS`, as the VMM exits. - pub(super) fn plan(&self, plan: &str) { - fs::write(self.path().join(EVENT_PLAN), plan).unwrap(); - } - - /// Returns whether the VMM's flush of its event returned. - pub(super) fn flushed(&self) -> bool { - self.path().join(FLUSHED).exists() - } -} - -impl Drop for VmmDir { - fn drop(&mut self) { - let _ = fs::write(self.path().join(EXIT_MARKER), b""); - if let Some(reference) = &self.reference - && let Ok(Opened::Running(process)) = Process::open(reference) - && !process - .wait(Instant::now() + TIMEOUT) - .is_ok_and(|exited| exited.is_some()) - { - let _ = process.kill(); - let _ = process.wait(Instant::now() + TIMEOUT); - } - } -} - -/// The VMM of a clone, in a process of its own, in a directory that [`VMM_MARKER`] marks: reads -/// its capability as OpenVMM does, finds the endpoints of its control console and of the -/// restore-ready event, and the snapshot that it restores, among the arguments that its launch -/// made, which [`ARGUMENTS`] holds; holds the snapshot as OpenVMM does, which on Linux listens at -/// the control endpoint beside the snapshot, and on Windows opens the snapshot's directory and -/// files sharing nothing but reading, and serves its control endpoint's named pipe; sends the -/// restore-ready event as OpenVMM does, or as its [plan](VmmDir::plan) says; writes [`STARTED`]; -/// then runs until a file named [`EXIT_MARKER`] appears, or for [`VMM_LIFETIME`] at most, writes -/// the outcome report that its plan asks for, and removes its control socket as it exits, as -/// OpenVMM does at a normal exit. It never answers on its control endpoint. -/// -/// Where OpenVMM would refuse its arguments, this VMM writes why in [`STARTED`] after its -/// capability and whether it sent its event. Where OpenVMM would abort a restore whose event it -/// cannot send, as when the launch's caller crashed, this VMM runs on, so that the test ends it as -/// it ends any VMM. -#[test] -#[ignore = "a clone's VMM that the launch tests run in a process of its own"] -fn restore_vmm_role() { - let directory = env::current_dir().unwrap(); - if !directory.join(VMM_MARKER).exists() { - return; - } - let mut capability = [0; CAPABILITY_LEN]; - std::io::stdin().read_exact(&mut capability).unwrap(); - let plan = fs::read_to_string(directory.join(EVENT_PLAN)).unwrap_or_default(); - let words: Vec<&str> = plan.split_whitespace().collect(); - let arguments: Vec = - serde_json::from_slice(&fs::read(directory.join(ARGUMENTS)).unwrap()).unwrap(); - if words.contains(&"exit") || words.contains(&"close") { - if words.contains(&"close") { - // As OpenVMM does once the launch of its VM worker fails: it connected the event's - // endpoint before it launched the worker, closes the endpoint as the launch fails, - // tears down, and only then reports the error as it exits. - let ready = argument(&arguments, "--restore-ready-path").unwrap(); - let sink = send_ready(Path::new(&ready), b""); - thread::sleep(CLOSE_PAUSE); - drop(sink); - thread::sleep(CLOSE_PAUSE); - } - if let Some(code) = words.iter().find_map(|word| word.strip_prefix("fatal:")) { - // OpenVMM's warnings may name a time ABI code too; its fatal error leads with the - // first code of the error's chain, which follows it. - writeln!( - std::io::stderr(), - "NVX-CPU-PROFILE-FALLBACK: [E_PROFILE_UNSUPPORTED] the test's warning\n\ - fatal error: [{code}] failed to launch vm worker\n\nCaused by:\n \ - 0: [{code}] the test's restore" - ) - .unwrap(); - } - if let Some(port) = words.iter().find_map(|word| word.strip_prefix("forward:")) { - // OpenVMM gives the error no code, and numbers the causes of its chain. - writeln!( - std::io::stderr(), - "warning: the test's forward (Tcp 127.0.0.1:{port}) is not yet bound\n\ - fatal error: failed to launch vm worker\n\nCaused by:\n \ - 0: failed to resolve the test's network endpoint\n \ - 1: failed to create socket for port forward (Tcp 127.0.0.1:{port})\n \ - 2: the test's port is held" - ) - .unwrap(); - } - write_report(&words, &arguments); - return; - } - let (held, problem) = match hold_snapshot(&arguments) { - Ok(held) => (Some(held), String::new()), - Err(problem) => (None, format!(" {problem}")), - }; - let delay = words - .iter() - .find_map(|word| word.strip_prefix("delay:")) - .map_or(0, |milliseconds| milliseconds.parse().unwrap()); - thread::sleep(Duration::from_millis(delay)); - let event = if words.contains(&"other") { - b"OPENVMM_RESTORE_READY_V2\n" - } else if words.contains(&"partial") { - &RESTORE_READY_EVENT[..10] - } else { - RESTORE_READY_EVENT - }; - let ready = argument(&arguments, "--restore-ready-path").unwrap(); - let sent = if words.contains(&"none") { - false - } else { - match send_ready(Path::new(&ready), event) { - Ok(sink) if words.contains(&"flush") => { - // OpenVMM flushes its event, and then closes its end of the endpoint. - #[cfg(windows)] - sink.sync_all().unwrap(); - drop(sink); - fs::write(directory.join(FLUSHED), b"").unwrap(); - true - } - Ok(_) => true, - Err(_) => false, - } - }; - let staged = directory.join(format!(".{STARTED}")); - fs::write(&staged, format!("{} {sent}{problem}", hex(&capability))).unwrap(); - fs::rename(&staged, directory.join(STARTED)).unwrap(); - let started = Instant::now(); - while !directory.join(EXIT_MARKER).exists() && started.elapsed() < VMM_LIFETIME { - thread::sleep(Duration::from_millis(10)); - } - write_report(&words, &arguments); - if let Some(held) = held { - held.release(); - } -} - -/// Writes OpenVMM's outcome report that the word `report:CATEGORY:STATUS` among `words` of a -/// [`restore_vmm_role`]'s plan asks for, if any: a VM whose outcome is of the category -/// `CATEGORY`, with the status `STATUS`, at the `--microvm-report` path among `arguments`, as -/// OpenVMM writes it before it exits. -fn write_report(words: &[&str], arguments: &[String]) { - let Some(report) = words.iter().find_map(|word| word.strip_prefix("report:")) else { - return; - }; - let (category, status) = report.split_once(':').unwrap(); - let status: i64 = status.parse().unwrap(); - let path = argument(arguments, "--microvm-report").unwrap(); - let report = serde_json::json!({ - "schema_version": 1, - "outcome": {"operation": "managed", "category": category, "status_code": status}, - "teardown": {"workers": true}, - }); - fs::write(path, serde_json::to_vec(&report).unwrap()).unwrap(); -} - -/// Returns the value of the option `name` among `arguments`, if they name it. -fn argument(arguments: &[String], name: &str) -> Option { - arguments - .iter() - .position(|argument| argument == name) - .and_then(|at| arguments.get(at + 1).cloned()) -} - -/// What a [`restore_vmm_role`] holds of its restore while it runs. -struct Held { - /// The listener at the control endpoint, on Linux, and the path at which it listens. - #[cfg(unix)] - listener: (std::os::unix::net::UnixListener, PathBuf), - /// The snapshot's directory and files, open, on Windows. - #[cfg(windows)] - _files: Vec, - /// The instances of the control endpoint's named pipe, on Windows. - #[cfg(windows)] - _control: Vec, -} - -impl Held { - /// Lets go of the restore, as OpenVMM does at a normal exit, which removes its control socket. - fn release(self) { - #[cfg(unix)] - { - let (listener, path) = self.listener; - drop(listener); - fs::remove_file(path).unwrap(); - } - } -} - -/// Holds the snapshot that `arguments` restore as OpenVMM does, or returns why OpenVMM would -/// refuse them. On Linux, OpenVMM requires the parent of its control endpoint to be a directory -/// of its user with mode 0700, not a link, whose path with its links resolved is that of the -/// snapshot's parent, and listens at the endpoint's path with its parent's links resolved. -#[cfg(unix)] -fn hold_snapshot(arguments: &[String]) -> std::result::Result { - use std::os::unix::fs::{MetadataExt, PermissionsExt}; - - let endpoint = argument(arguments, "--microvm-control-console") - .and_then(|console| console.strip_prefix("listen=").map(PathBuf::from)) - .ok_or("no control endpoint")?; - let snapshot = PathBuf::from(argument(arguments, "--restore-snapshot").ok_or("no snapshot")?); - let (Some(parent), Some(name), Some(snapshot_parent)) = - (endpoint.parent(), endpoint.file_name(), snapshot.parent()) - else { - return Err("the endpoint or the snapshot names no directory".to_owned()); - }; - let metadata = fs::symlink_metadata(parent).map_err(|error| error.to_string())?; - if !metadata.is_dir() - || metadata.uid() != crate::host::sys::effective_ids().0 - || metadata.permissions().mode() & 0o7777 != 0o700 - { - return Err(format!( - "the endpoint's directory {} is not one that OpenVMM takes", - parent.display() - )); - } - let canonical = fs::canonicalize(parent).map_err(|error| error.to_string())?; - if fs::canonicalize(snapshot_parent).map_err(|error| error.to_string())? != canonical { - return Err(format!( - "the endpoint {} is not beside the snapshot {}", - endpoint.display(), - snapshot.display() - )); - } - let path = canonical.join(name); - let listener = std::os::unix::net::UnixListener::bind(&path) - .map_err(|error| format!("cannot listen at {}: {error}", path.display()))?; - Ok(Held { - listener: (listener, path), - }) -} - -/// Holds the snapshot that `arguments` restore as OpenVMM does, or returns why OpenVMM would -/// refuse them: on Windows, OpenVMM opens the snapshot's directory and files for reading, sharing -/// nothing but reading, so that a restore of another clone, which shares reading, reads them too, -/// while no one changes them. It also serves the named pipe of its control endpoint, as -/// [`control_pipe::serve`] does. -#[cfg(windows)] -fn hold_snapshot(arguments: &[String]) -> std::result::Result { - use std::os::windows::fs::OpenOptionsExt; - - /// `FILE_SHARE_READ`. - const SHARE_READ: u32 = 0x1; - /// `FILE_FLAG_BACKUP_SEMANTICS`, which opens a directory. - const BACKUP_SEMANTICS: u32 = 0x0200_0000; - let snapshot = PathBuf::from(argument(arguments, "--restore-snapshot").ok_or("no snapshot")?); - let mut files = Vec::new(); - let paths = SNAPSHOT_ARTIFACTS.map(|name| (snapshot.join(name), 0)); - for (path, flags) in std::iter::once((snapshot.clone(), BACKUP_SEMANTICS)).chain(paths) { - let file = fs::OpenOptions::new() - .read(true) - .share_mode(SHARE_READ) - .custom_flags(flags) - .open(&path) - .map_err(|error| format!("cannot open {}: {error}", path.display()))?; - files.push(file); - } - let endpoint = argument(arguments, "--microvm-control-console") - .and_then(|console| console.strip_prefix("listen=").map(str::to_owned)) - .ok_or("no control endpoint")?; - let control = control_pipe::serve(&endpoint) - .map_err(|error| format!("cannot serve the control endpoint {endpoint}: {error}"))?; - Ok(Held { - _files: files, - _control: control, - }) -} - -/// The named pipe of the control endpoint that a [`restore_vmm_role`] serves on Windows, as -/// OpenVMM does, through the Win32 calls that the standard library lacks. -#[cfg(windows)] -#[allow(unsafe_code)] -mod control_pipe { - use std::io; - use std::os::windows::io::{FromRawHandle, OwnedHandle}; - - use windows_sys::Win32::Foundation::INVALID_HANDLE_VALUE; - use windows_sys::Win32::Storage::FileSystem::PIPE_ACCESS_DUPLEX; - use windows_sys::Win32::System::Pipes::{ - CreateNamedPipeW, PIPE_READMODE_BYTE, PIPE_TYPE_BYTE, PIPE_WAIT, - }; - - /// How many hosts may connect to the endpoint at once, one to each instance of its pipe: as - /// many as a test's restore sequence connects, at most. - const INSTANCES: u32 = 4; - - /// Creates the instances of the named pipe `endpoint`, `//./pipe/NAME`, each of which a host - /// connects to before the role, which never answers, would accept it. - pub(super) fn serve(endpoint: &str) -> io::Result> { - let name: Vec = endpoint - .replace('/', "\\") - .encode_utf16() - .chain(Some(0)) - .collect(); - (0..INSTANCES) - .map(|_| { - // SAFETY: the name is NUL-terminated, and outlives the call. - let handle = unsafe { - CreateNamedPipeW( - name.as_ptr(), - PIPE_ACCESS_DUPLEX, - PIPE_TYPE_BYTE | PIPE_READMODE_BYTE | PIPE_WAIT, - INSTANCES, - 4096, - 4096, - 0, - std::ptr::null(), - ) - }; - if handle == INVALID_HANDLE_VALUE { - return Err(io::Error::last_os_error()); - } - // SAFETY: CreateNamedPipeW returned a fresh handle that nothing else owns. - Ok(unsafe { OwnedHandle::from_raw_handle(handle) }) - }) - .collect() - } -} - -/// Sends `event` to the restore-ready endpoint at `path`, as OpenVMM sends its event: through a -/// Unix socket, or a named pipe on Windows. Returns the VMM's end of the endpoint. -fn send_ready(path: &Path, event: &[u8]) -> io::Result { - #[cfg(unix)] - let mut sink = File::from(std::os::fd::OwnedFd::from( - std::os::unix::net::UnixStream::connect(path)?, - )); - #[cfg(windows)] - let mut sink = fs::OpenOptions::new().write(true).open(path)?; - sink.write_all(event)?; - Ok(sink) -} - -#[test] -fn a_clone_restores_its_leased_snapshot_with_its_devices_and_activation_record() { - let fixture = Fixture::new(); - let (snapshot, contents) = complete_snapshot(&fixture); - let sealed = seals(&snapshot); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = tmpfs_request(&image, &binding); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let (key, activation) = (plan.key().clone(), plan.activation().clone()); - let capability = *plan.capability(); - let sandbox = acquired.locked.dir().to_owned(); - let lease = acquired.lease.clone(); - let mut vmm = VmmDir::new(); - let dir = vmm.path().to_owned(); - let made = RefCell::new(None); - let command = fake_vmm(&dir, &made); - let hooks = Hooks { - command: &command, - ..Hooks::LAUNCH - }; - let restoring = plan - .launch_with(&mut acquired, Path::new(OPENVMM), key.runtime(), hooks) - .unwrap(); - let reference = restoring.starting().process().reference().clone(); - vmm.reference = Some(reference.clone()); - // The VMM read the plan's capability, took its arguments, and sent its event to the - // restore-ready endpoint, which listened before the VMM started. - assert_eq!(vmm.started(), format!("{} true", hex(&capability))); - // The VMM started once the sandbox held the clone binding and the launch's journal, in the - // sandbox's directory, with nothing of the caller's besides its standard streams. - let made = made.take().unwrap(); - assert!(made.bound && made.begun, "{made:?}"); - assert_eq!( - made.program, - artifacts::absolute(Path::new(OPENVMM)).unwrap() - ); - assert_eq!(made.working_dir, sandbox); - assert_eq!(made.inherited, 0); - // It restores the generation's snapshot in place, with the clone's devices: its image, and its - // `scratch` block device, which an upper in RAM declares absent, so that OpenVMM attaches it - // with no file. On Linux, its control console listens beside the snapshot, at the socket that - // the lease reserved, named after the launch. - let endpoint = restoring.starting().endpoint().to_owned(); - let ready = restoring.ready_endpoint().to_owned(); - let socket = fixture.store.clone_socket_path(&lease); - if cfg!(target_os = "linux") { - assert_eq!( - socket, - snapshot - .parent() - .unwrap() - .join(format!("{}.sock", lease.launch())) - ); - assert_eq!(endpoint, socket.to_str().unwrap()); - assert!(exists(&socket)); - assert_eq!(ready, sandbox.join(RESTORE_READY_NAME).to_str().unwrap()); - } else { - assert!( - endpoint.starts_with("//./pipe/openvmm-microvm-"), - "{endpoint}" - ); - assert!(!exists(&socket)); - assert!(ready.starts_with("//./pipe/nvx-restore-ready-"), "{ready}"); - } - assert_eq!(restoring.starting().journal().endpoint(), endpoint); - let distro = format!("distro:file:{},ro", shown(&image)); - let (profile, processors) = (key.cpu_profile(), key.processors().to_string()); - let listen = format!("listen={endpoint}"); - let report = sandbox.join(OUTCOME_NAME); - let expected = [ - "--machine", - "microvm", - "--microvm-sandbox-block", - &distro, - "--restore-absent-scratch", - "--single-process", - "--hypervisor", - key.hypervisor().as_str(), - "--cpu-profile", - &profile, - "--processors", - &processors, - "--restore-snapshot", - snapshot.to_str().unwrap(), - "--restore-memory", - "512M", - "--restore-ready-path", - &ready, - "--virtio-console", - "none", - "--microvm-control-console", - &listen, - "--microvm-control-auth-stdin", - "--microvm-report", - report.to_str().unwrap(), - ]; - assert_eq!(made.arguments, expected); - // The sandbox holds the clone's activation record, in its canonical encoding, which the - // launch keeps. - let recorded = fs::read(sandbox.join(CLONE_ACTIVATION_NAME)).unwrap(); - assert_eq!(recorded, activation.encode()); - assert_eq!(ActivationRecord::decode(&recorded).unwrap(), activation); - assert_eq!(restoring.activation(), &activation); - assert_eq!(restoring.capability(), &capability); - assert_eq!(restoring.image().path().to_str().unwrap(), shown(&image)); - // The launch never shows its capability. - let shown_launch = format!("{restoring:?}"); - assert!(shown_launch.starts_with("Restoring {"), "{shown_launch}"); - assert!(!shown_launch.contains("capability"), "{shown_launch}"); - assert!( - !shown_launch.contains(&format!("{capability:?}")), - "{shown_launch}" - ); - // Abandoning the launch ends its VMM, which leaves its control socket, and the end of the - // launch removes it. The release then removes the clone's files, and nothing of the - // generation, whose snapshot the restore left as it was. - assert_eq!( - restoring.abandon().unwrap(), - Recovery::Terminated(reference) - ); - assert!(!exists(&socket)); - assert_eq!( - fixture.store.release(&mut acquired.locked).unwrap(), - Some(acquired.lease.clone()) - ); - assert_released(&sandbox); - assert_untouched(&fixture, &snapshot, &contents, &sealed); -} - -#[test] -fn clones_of_one_generation_restore_its_snapshot_in_place_at_once() { - let fixture = Fixture::new(); - let (snapshot, contents) = complete_snapshot(&fixture); - let sealed = seals(&snapshot); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let request = tmpfs_request(&image, &binding); - let mut clones = [fixture.acquire("first"), fixture.acquire("second")]; - let leases = clones.each_ref().map(|acquired| acquired.lease.clone()); - let mut vmms = [VmmDir::new(), VmmDir::new()]; - // Both clones plan and launch at once, each under its own sandbox's lock. - let launched: Vec<(Restoring<'_>, Made)> = thread::scope(|scope| { - let launches: Vec<_> = clones - .iter_mut() - .zip(&vmms) - .map(|(acquired, vmm)| { - let (fixture, request, dir) = (&fixture, &request, vmm.path().to_owned()); - scope.spawn(move || { - let plan = fixture.plan(acquired, request, NO_CONTENT).unwrap(); - let runtime = plan.key().runtime().clone(); - let made = RefCell::new(None); - let command = fake_vmm(&dir, &made); - let hooks = Hooks { - command: &command, - ..Hooks::LAUNCH - }; - let restoring = plan - .launch_with(acquired, Path::new(OPENVMM), &runtime, hooks) - .unwrap(); - (restoring, made.take().unwrap()) - }) - }) - .collect(); - launches - .into_iter() - .map(|launch| launch.join().unwrap()) - .collect() - }); - for ((restoring, _), vmm) in launched.iter().zip(&mut vmms) { - vmm.reference = Some(restoring.starting().process().reference().clone()); - } - // Both VMMs run, restoring the one snapshot in place, each listening at its own endpoint. - for ((restoring, made), vmm) in launched.iter().zip(&vmms) { - let capability = hex(restoring.capability()); - assert_eq!(vmm.started(), format!("{capability} true")); - let restored = argument(&made.arguments, "--restore-snapshot").unwrap(); - assert_eq!(restored, snapshot.to_str().unwrap()); - } - let endpoints: Vec<&str> = launched - .iter() - .map(|(restoring, _)| restoring.starting().endpoint()) - .collect(); - assert_ne!(endpoints[0], endpoints[1]); - if cfg!(target_os = "linux") { - for (endpoint, lease) in endpoints.iter().zip(&leases) { - let socket = fixture.store.clone_socket_path(lease); - assert_eq!(*endpoint, socket.to_str().unwrap()); - assert!(exists(&socket)); - } - } - // Each VMM exits as OpenVMM does, removing its socket, and each lease is released. - for ((restoring, _), vmm) in launched.into_iter().zip(&vmms) { - vmm.end_and_wait(); - assert_eq!(restoring.abandon().unwrap(), Recovery::Exited); - } - for (acquired, lease) in clones.iter_mut().zip(leases) { - assert_eq!( - fixture.store.release(&mut acquired.locked).unwrap(), - Some(lease) - ); - } - assert_untouched(&fixture, &snapshot, &contents, &sealed); -} - -/// Launches the clone of `acquired`, whose plan `fixture` makes, with a [`restore_vmm_role`] in -/// `vmm`, and commits the launch once the VMM runs, as a clone's restore does. -fn committed( - fixture: &Fixture, - acquired: &mut Acquired, - image: &(PathBuf, DistroBinding), - vmm: &mut VmmDir, -) -> supervisor::Vmm { - let plan = fixture - .plan(acquired, &tmpfs_request(&image.0, &image.1), NO_CONTENT) - .unwrap(); - let runtime = plan.key().runtime().clone(); - let dir = vmm.path().to_owned(); - let made = RefCell::new(None); - let command = fake_vmm(&dir, &made); - let hooks = Hooks { - command: &command, - ..Hooks::LAUNCH - }; - let restoring = plan - .launch_with(acquired, Path::new(OPENVMM), &runtime, hooks) - .unwrap(); - vmm.reference = Some(restoring.starting().process().reference().clone()); - let capability = hex(restoring.capability()); - assert_eq!(vmm.started(), format!("{capability} true")); - restoring.into_starting().commit().unwrap() -} - -#[test] -fn the_control_socket_of_a_clone_goes_once_its_vm_has_ended_and_for_good_with_its_lease() { - let fixture = Fixture::new(); - let (snapshot, contents) = complete_snapshot(&fixture); - let sealed = seals(&snapshot); - let image = fixture.image("distro.erofs", IMAGE_LEN); - // A clone's VM whose VMM a termination ends, which leaves its control socket: the teardown - // removes it with the runtime record. - let mut acquired = fixture.acquire("terminated"); - let mut vmm = VmmDir::new(); - let running = committed(&fixture, &mut acquired, &image, &mut vmm); - let socket = fixture.store.clone_socket_path(&acquired.lease); - assert_eq!( - running.endpoint(), - acquired.locked.runtime().unwrap().unwrap().endpoint() - ); - assert_eq!(exists(&socket), cfg!(target_os = "linux")); - assert_eq!( - running.terminate(&mut acquired.locked, TIMEOUT).unwrap(), - StopOutcome::Forced - ); - assert!(acquired.locked.runtime().unwrap().is_none()); - assert!(!exists(&socket)); - // A socket that comes back, as an unsynced removal can after a crash of the host, goes with - // the release, which makes its absence durable before the lease ends. - if cfg!(target_os = "linux") { - fs::write(&socket, b"").unwrap(); - } - assert_eq!( - fixture.store.release(&mut acquired.locked).unwrap(), - Some(acquired.lease.clone()) - ); - assert!(!exists(&socket)); - // A clone whose VMM exits on its own removes its socket itself, and a stop that finds it - // gone ends the VM. - let mut acquired = fixture.acquire("stopped"); - let mut vmm = VmmDir::new(); - let running = committed(&fixture, &mut acquired, &image, &mut vmm); - let socket = fixture.store.clone_socket_path(&acquired.lease); - let deadline = Instant::now() + TIMEOUT; - let stopped = running - .stop(&mut acquired.locked, deadline, None, |_, _| { - vmm.end(); - Ok::<_, ()>(()) - }) - .unwrap(); - assert_eq!(stopped.outcome, StopOutcome::Graceful); - assert!(acquired.locked.runtime().unwrap().is_none()); - assert!(!exists(&socket)); - assert_eq!( - fixture.store.release(&mut acquired.locked).unwrap(), - Some(acquired.lease.clone()) - ); - assert_untouched(&fixture, &snapshot, &contents, &sealed); -} - -#[test] -fn a_clone_of_a_template_with_a_nic_restores_its_own_policy_in_place_of_the_templates() { - let fixture = network_fixture(); - complete_snapshot(&fixture); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - // The class's network allows egress; the clone's denies it but to one destination, and - // publishes the guest's listeners on host loopback, with a forward. - let mut policy = NetworkPolicy::new(NetworkAccess::Deny) - .with_allow(EgressRule::to("192.0.2.0/24").on_port(Protocol::Tcp, 443)); - policy.host_loopback = HostLoopback::Allow; - policy.host_loopback_forwards.push(LoopbackForward { - protocol: ForwardProtocol::Tcp, - host_port: 18080, - guest_port: 8080, - }); - let attached = SandboxNetwork::new("10.0.0.2/24", policy).unwrap(); - let request = with_network(tmpfs_request(&image, &binding), Some(attached.clone())); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let runtime = plan.key().runtime().clone(); - let mut vmm = VmmDir::new(); - let dir = vmm.path().to_owned(); - let made = RefCell::new(None); - let command = fake_vmm(&dir, &made); - let hooks = Hooks { - command: &command, - ..Hooks::LAUNCH - }; - let restoring = plan - .launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks) - .unwrap(); - vmm.reference = Some(restoring.starting().process().reference().clone()); - // After the image, the devices end with `--restore-network-policy` and the options of the - // clone's own policy, without `--net` and the guest's address, which OpenVMM takes from the - // snapshot, and refuses beside it, then with `--restore-absent-scratch`, which declares the - // scratch of the clone's upper in RAM absent. - let mut expected = attached.openvmm_arguments().unwrap(); - assert_eq!(expected[..2], ["--net", "10.0.0.2/24"]); - expected.splice(..2, ["--restore-network-policy".to_owned()]); - expected.push("--restore-absent-scratch".to_owned()); - let arguments = made.take().unwrap().arguments; - let devices = arguments - .iter() - .position(|argument| argument == "--single-process") - .unwrap(); - assert_eq!(arguments[4..devices], expected); - assert!(!arguments.iter().any(|argument| argument == "--net")); -} - -#[test] -fn a_clone_of_a_template_with_processors_offline_at_boot_restores_its_processor_target() { - let fixture = Fixture::with(|class, spec, _| set_processors(class, spec, 4, 1)); - let (snapshot, _) = complete_snapshot(&fixture); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = CloneRequest { - processors: 2, - ..tmpfs_request(&image, &binding) - }; - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let runtime = plan.key().runtime().clone(); - let mut vmm = VmmDir::new(); - let dir = vmm.path().to_owned(); - let made = RefCell::new(None); - let command = fake_vmm(&dir, &made); - let hooks = Hooks { - command: &command, - ..Hooks::LAUNCH - }; - let restoring = plan - .launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks) - .unwrap(); - vmm.reference = Some(restoring.starting().process().reference().clone()); - // The VM keeps the template's processor capacity, of which the restore brings the target's - // prefix online. The clone in RAM declares the template's scratch absent among its devices, - // as any clone in RAM does. - let arguments = made.take().unwrap().arguments; - let at = arguments - .iter() - .position(|argument| argument == "--processors") - .unwrap(); - assert!( - arguments[..at] - .iter() - .any(|argument| argument == "--restore-absent-scratch"), - "{arguments:?}" - ); - assert_eq!( - arguments[at..at + 8], - [ - "--processors", - "4", - "--restore-snapshot", - snapshot.to_str().unwrap(), - "--restore-processors", - "2", - "--restore-memory", - "512M", - ] - ); -} - -#[test] -fn a_clone_on_a_scratch_image_passes_its_claim_to_its_vmm() { - let fixture = Fixture::new(); - complete_snapshot(&fixture); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let (scratch, scratch_binding) = fixture.scratch("scratch.ext4", SCRATCH_LEN); - let mut acquired = fixture.acquire("sandbox"); - let request = scratch_request(&image, &binding, &scratch, &scratch_binding); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let runtime = plan.key().runtime().clone(); - let mut vmm = VmmDir::new(); - let dir = vmm.path().to_owned(); - let made = RefCell::new(None); - let command = fake_vmm(&dir, &made); - let hooks = Hooks { - command: &command, - ..Hooks::LAUNCH - }; - let restoring = plan - .launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks) - .unwrap(); - let reference = restoring.starting().process().reference().clone(); - vmm.reference = Some(reference.clone()); - vmm.started(); - // The VMM opens the image at its path, as the `scratch` block device, and inherits the one - // descriptor or handle that holds the claim, which no argument names. The scratch is not - // absent. - let made = made.take().unwrap(); - assert_eq!(made.inherited, 1); - let device = format!("scratch:file:{}", shown(&scratch)); - assert!(made.arguments.contains(&device), "{:?}", made.arguments); - assert!( - !made - .arguments - .iter() - .any(|argument| argument == "--restore-absent-scratch"), - "{:?}", - made.arguments - ); - // On Windows, no handle that may write the image is open any more, so OpenVMM's open, which - // shares no writing, succeeds. - #[cfg(windows)] - { - use std::os::windows::fs::OpenOptionsExt; - - /// `FILE_SHARE_READ | FILE_SHARE_DELETE`. - const SHARE_READ_DELETE: u32 = 0x1 | 0x4; - /// `FILE_FLAG_OPEN_REPARSE_POINT`. - const OPEN_REPARSE_POINT: u32 = 0x0020_0000; - let opened = fs::OpenOptions::new() - .read(true) - .write(true) - .share_mode(SHARE_READ_DELETE) - .custom_flags(OPEN_REPARSE_POINT) - .open(&scratch); - assert!(opened.is_ok(), "{opened:?}"); - } - // The VMM holds the claim: another check finds the image claimed. - let error = scratch_binding.check(&scratch, Instant::now()).unwrap_err(); - assert_eq!(error.code(), ErrorCode::ScratchClaimed, "{error}"); - // The claim is the VMM's alone: once the VMM has exited, the image serves another check, - // while the launch lasts. - vmm.end_and_wait(); - assert!(released(|| scratch_binding - .check(&scratch, Instant::now()) - .is_ok())); - assert_eq!(restoring.abandon().unwrap(), Recovery::Exited); -} - -#[test] -fn a_clone_with_an_export_binds_it_to_its_template_s_dormant_filesystem_slot() { - let fixture = Fixture::new(); - complete_snapshot(&fixture); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let (export, _) = fixture.aggregate_export("aggregate"); - let request = with_export(tmpfs_request(&image, &binding), &export); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let runtime = plan.key().runtime().clone(); - let mut vmm = VmmDir::new(); - let dir = vmm.path().to_owned(); - let made = RefCell::new(None); - let command = fake_vmm(&dir, &made); - let hooks = Hooks { - command: &command, - ..Hooks::LAUNCH - }; - let restoring = plan - .launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks) - .unwrap(); - let reference = restoring.starting().process().reference().clone(); - vmm.reference = Some(reference.clone()); - vmm.started(); - // The clone in RAM declares its scratch absent, and the export follows the clone's block - // device, as a cold launch attaches it: the aggregate, on the dormant slot, with its - // directories and files, their modes and policies. - let arguments = made.take().unwrap().arguments; - let devices = arguments - .iter() - .position(|argument| argument == "--single-process") - .unwrap(); - assert_eq!(arguments[4], "--restore-absent-scratch"); - assert_eq!(arguments[5..devices], export.openvmm_arguments()); - assert_eq!( - arguments[5..7], - ["--mount-aggregate", filesystem::GUEST_EXPORT] - ); - // The restore sequence sends the export's table once the clone is ready, as a cold launch's - // guest receives it. - assert_eq!(restoring.table, export.guest_table()); - assert_eq!( - restoring.abandon().unwrap(), - Recovery::Terminated(reference) - ); -} - -#[test] -fn a_clone_whose_exported_objects_changed_since_its_plan_starts_nothing() { - let fixture = Fixture::new(); - complete_snapshot(&fixture); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let request = tmpfs_request(&image, &binding); - let runtime = runtime_of(&fixture); - let hooks = Hooks { - command: &never_spawned, - ..Hooks::LAUNCH - }; - let changed = |path: &Path| { - ( - ErrorKind::Other, - format!( - "{} no longer names the host object it named at provision; deprovision the \ - sandbox and provision it again", - path.display() - ), - ) - }; - // A directory that the guest writes, in which it maps a directory that a workload could - // move, which another directory has replaced since the plan. - let mut moved = fixture.acquire("moved"); - let data = fixture.exported("moved", &["inner"]); - let inner = data.join("inner"); - let export = export_plan( - &[(&data, "/data", false), (&inner, "/inner", false)], - &[], - &[], - ); - assert!(export.binds[1].identity.is_some()); - let plan = fixture - .plan( - &mut moved, - &with_export(request.clone(), &export), - NO_CONTENT, - ) - .unwrap(); - fs::rename(&inner, data.join("renamed")).unwrap(); - fs::create_dir(&inner).unwrap(); - assert_eq!( - refused_at( - plan.launch_with(&mut moved, Path::new(OPENVMM), &runtime, hooks), - Stage::Launch, - Evidence::InvalidMappings, - ), - changed(&inner) - ); - assert_unlaunched(&fixture, &moved); - // The exported directory, which is missing since the plan. - let mut missing = fixture.acquire("missing"); - let export = fixture.data_export("missing"); - let data = export.children[0].root.clone(); - let plan = fixture - .plan( - &mut missing, - &with_export(request.clone(), &export), - NO_CONTENT, - ) - .unwrap(); - fs::remove_dir_all(&data).unwrap(); - assert_eq!( - refused_at( - plan.launch_with(&mut missing, Path::new(OPENVMM), &runtime, hooks), - Stage::Launch, - Evidence::InvalidMappings, - ), - changed(&data) - ); - assert_unlaunched(&fixture, &missing); - // The exported directory, which a file has replaced since the plan. - let mut replaced = fixture.acquire("replaced"); - let export = fixture.data_export("replaced"); - let data = export.children[0].root.clone(); - let plan = fixture - .plan( - &mut replaced, - &with_export(request.clone(), &export), - NO_CONTENT, - ) - .unwrap(); - fs::remove_dir_all(&data).unwrap(); - fs::write(&data, b"file").unwrap(); - assert_eq!( - refused_at( - plan.launch_with(&mut replaced, Path::new(OPENVMM), &runtime, hooks), - Stage::Launch, - Evidence::InvalidMappings, - ), - changed(&data) - ); - assert_unlaunched(&fixture, &replaced); - // A file of an aggregate, which a directory has replaced since the plan, and which would - // grant the whole directory. - let mut swapped = fixture.acquire("swapped"); - let (export, [_, notes, ..]) = fixture.aggregate_export("swapped"); - let plan = fixture - .plan(&mut swapped, &with_export(request, &export), NO_CONTENT) - .unwrap(); - fs::remove_file(¬es).unwrap(); - fs::create_dir(¬es).unwrap(); - assert_eq!( - refused_at( - plan.launch_with(&mut swapped, Path::new(OPENVMM), &runtime, hooks), - Stage::Launch, - Evidence::InvalidMappings, - ), - changed(¬es) - ); - assert_unlaunched(&fixture, &swapped); -} - -#[test] -fn a_clone_whose_export_cannot_be_inspected_starts_nothing_and_fails_with_the_host_s_evidence() { - let fixture = Fixture::new(); - complete_snapshot(&fixture); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let request = tmpfs_request(&image, &binding); - let runtime = runtime_of(&fixture); - let hooks = Hooks { - command: &never_spawned, - ..Hooks::LAUNCH - }; - // The plan pins an object at a name that the host's file system cannot hold: the failure to - // inspect it shows nothing of the object, so it is the host's own, not the request's. - let mut acquired = fixture.acquire("uninspected"); - let data = fixture.exported("uninspected", &["inner"]); - let inner = data.join("inner"); - let mut export = export_plan( - &[(&data, "/data", false), (&inner, "/inner", false)], - &[], - &[], - ); - assert!(export.binds[1].identity.is_some()); - export.binds[1].source = "n".repeat(300); - let unnamed = export.children[0].root.join(&export.binds[1].source); - let plan = fixture - .plan(&mut acquired, &with_export(request, &export), NO_CONTENT) - .unwrap(); - assert_eq!( - refused_at( - plan.launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks), - Stage::Launch, - Evidence::HostFailure, - ), - ( - ErrorKind::Io, - format!("cannot inspect {}", unnamed.display()) - ) - ); - assert_unlaunched(&fixture, &acquired); -} - -#[test] -fn a_clone_launches_only_as_the_launch_that_its_durable_binding_reserves() { - let fixture = Fixture::new(); - complete_snapshot(&fixture); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let request = tmpfs_request(&image, &binding); - let runtime = runtime_of(&fixture); - let openvmm = Path::new(OPENVMM); - let hooks = Hooks { - command: &never_spawned, - ..Hooks::LAUNCH - }; - // A sandbox that no longer records the clone binding starts no VMM of the clone, and keeps - // nothing of its launch. - let mut unbound = fixture.acquire("unbound"); - let plan = fixture.plan(&mut unbound, &request, NO_CONTENT).unwrap(); - fs::remove_file(unbound.locked.binding_path()).unwrap(); - assert_eq!( - refused_at( - plan.launch_with(&mut unbound, openvmm, &runtime, hooks), - Stage::Launch, - Evidence::HostFailure - ), - ( - ErrorKind::Other, - format!( - "the sandbox in {} records no clone binding of the lease {}, which its \ - acquisition makes durable before any VMM of the clone starts", - unbound.locked.dir().display(), - unbound.lease.id() - ) - ) - ); - assert_unlaunched(&fixture, &unbound); - // Nor does an acquisition of another lease. - let mut first = fixture.acquire("first"); - let mut second = fixture.acquire("second"); - let plan = fixture.plan(&mut first, &request, NO_CONTENT).unwrap(); - assert_eq!( - refused_at( - plan.launch_with(&mut second, openvmm, &runtime, hooks), - Stage::Launch, - Evidence::HostFailure - ), - ( - ErrorKind::Other, - format!( - "a clone is launched under the lock that acquired the lease of its plan, {}, \ - before the launch that the lease reserves begins, but the lock of the sandbox in \ - {} holds the lease {}, and reserves no launch {} of it", - first.lease.id(), - second.locked.dir().display(), - second.lease.id(), - first.lease.launch() - ) - ) - ); - assert_unlaunched(&fixture, &first); - assert_unlaunched(&fixture, &second); - // Nor a runtime other than the one that captured the template. - let other = RuntimeIdentity::new(artifact('d'), artifact('e'), artifact('f')); - assert_ne!(other, runtime); - let plan = fixture.plan(&mut first, &request, NO_CONTENT).unwrap(); - let launched = plan.launch_with(&mut first, openvmm, &other, hooks); - let (kind, message) = refused_at(launched, Stage::Launch, Evidence::RuntimeMismatch); - assert_eq!(kind, ErrorKind::Unsupported, "{message}"); - assert!( - message.starts_with(&format!( - "the template's generation {} restores under the runtime that captured it", - fixture.generation.generation() - )), - "{message}" - ); - assert_unlaunched(&fixture, &first); - // On Linux, a sandbox whose restore-ready socket's path would not fit a Unix socket's - // address is refused. - #[cfg(target_os = "linux")] - { - let mut deep = fixture.acquire(&"d".repeat(100)); - let plan = fixture.plan(&mut deep, &request, NO_CONTENT).unwrap(); - assert_eq!( - refused_at( - plan.launch_with(&mut deep, openvmm, &runtime, hooks), - Stage::Launch, - Evidence::EndpointBind - ), - ( - ErrorKind::Unsupported, - too_long(&deep.locked.restore_ready_path()) - ) - ); - assert_unlaunched(&fixture, &deep); - } - // The acquisition launches once its plan is its own. - let plan = fixture.plan(&mut second, &request, NO_CONTENT).unwrap(); - let mut vmm = VmmDir::new(); - let dir = vmm.path().to_owned(); - let made = RefCell::new(None); - let command = fake_vmm(&dir, &made); - let hooks = Hooks { - command: &command, - ..Hooks::LAUNCH - }; - let restoring = plan - .launch_with(&mut second, openvmm, &runtime, hooks) - .unwrap(); - vmm.reference = Some(restoring.starting().process().reference().clone()); -} - -#[cfg(windows)] -#[test] -fn working_directory_limits_are_checked_before_clone_launches() { - use std::os::windows::ffi::OsStrExt; - - let fixture = Fixture::new(); - complete_snapshot(&fixture); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let request = tmpfs_request(&image, &binding); - let mut parent = fixture.path("sandboxes", ""); - while parent.as_os_str().encode_wide().count() < 300 { - parent.push("long-state-root-component"); - assert_eq!(platform::create_owner_only_dir(&parent).unwrap(), None); - } - let sandbox = parent.join("sandbox"); - let mut acquired = fixture.acquire(sandbox.to_str().unwrap()); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let hooks = Hooks { - command: &never_spawned, - ..Hooks::LAUNCH - }; - let (kind, message) = refused_at( - plan.launch_with( - &mut acquired, - Path::new(OPENVMM), - &runtime_of(&fixture), - hooks, - ), - Stage::Launch, - Evidence::HostFailure, - ); - assert_eq!(kind, ErrorKind::Unsupported); - assert!(message.contains("working directory")); - assert!(message.contains("258")); - assert_unlaunched(&fixture, &acquired); -} - -/// Returns OpenVMM's outcome report of a VM whose guest ended it with `status`. -fn guest_exit(status: u8) -> Vec { - serde_json::to_vec(&serde_json::json!({ - "schema_version": 1, - "outcome": {"operation": "managed", "category": "guest-exit", "status_code": status}, - "teardown": {"workers": true}, - })) - .unwrap() -} - -#[test] -fn a_clone_whose_launch_failed_before_it_began_launches_no_more_until_its_release() { - let fixture = Fixture::new(); - complete_snapshot(&fixture); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let request = tmpfs_request(&image, &binding); - let runtime = runtime_of(&fixture); - let openvmm = Path::new(OPENVMM); - // The launch fails once it wrote the clone's files and recorded its progress, before it - // began, which leaves the lock's reservation. - let interrupt = |point: Point| match point { - Point::Prepared => Err(Error::io("the test fails the launch")), - Point::Begun => Ok(()), - }; - let hooks = Hooks { - command: &never_spawned, - interrupt: &interrupt, - ..Hooks::LAUNCH - }; - let mut acquired = fixture.acquire("failed"); - // The report that an earlier VM of the sandbox left, which the acquisition kept, is not the - // clone's: an inspection takes none, and the launch removes it before it records anything. - let report = acquired.locked.outcome_path(); - fs::write(&report, guest_exit(195)).unwrap(); - assert!(crate::host::report::read(&report).unwrap().is_some()); - let records = CloneRecords::read(&acquired.locked).unwrap().unwrap(); - assert_eq!( - (records.progress, records.report), - (Progress::Planned, None) - ); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - let failure = plan - .launch_with(&mut acquired, openvmm, &runtime, hooks) - .unwrap_err(); - assert_recorded( - &failure, - &acquired.lease, - &acquired.locked, - (Stage::Launch, Progress::Launching), - Evidence::HostFailure, - Teardown::NoVm, - ); - assert!(!exists(&report)); - assert_eq!(acquired.locked.reserved(), Some(acquired.lease.launch())); - // Another launch under the lease and the lock that still reserves it is refused before it - // writes anything, so the record of the failure stays as it was. - let path = acquired.locked.clone_failure_path(); - let recorded = fs::read(&path).unwrap(); - let plan = fixture.plan(&mut acquired, &request, NO_CONTENT).unwrap(); - assert_eq!( - refused_at( - plan.launch_with(&mut acquired, openvmm, &runtime, hooks), - Stage::Launch, - Evidence::HostFailure - ), - ( - ErrorKind::Other, - format!( - "the clone of the lease {} failed, as {} records: a lease's clone fails once, \ - and the release of the lease ends it", - acquired.lease.id(), - path.display() - ) - ) - ); - assert_eq!(fs::read(&path).unwrap(), recorded); - // The release ends the lease with the record. - assert_eq!( - fixture.store.release(&mut acquired.locked).unwrap(), - Some(acquired.lease.clone()) - ); - assert!(!path.exists()); -} - -#[test] -fn a_failure_that_a_reattach_records_before_any_launch_shows_no_report_of_an_earlier_vm() { - // A reattach before any launch of the clone that cannot read the record of a failure, as a - // transient error of the host may keep it from reading one, records a failure of its own, at - // the plan, with no VM: the report that an earlier VM of the sandbox left, which no launch - // removed, stays none of the clone's. - let fixture = Fixture::new(); - let runtime = runtime_of(&fixture); - let mut acquired = fixture.acquire("sandbox"); - let locked = &mut acquired.locked; - fs::write(locked.outcome_path(), guest_exit(195)).unwrap(); - write_private(&locked.clone_failure_path(), b"not a record").unwrap(); - let mut control = ReattachControl::answering(Some); - let deadline = Instant::now() + TIMEOUT; - let reattached = reattach_clone(locked, &runtime, &mut control, deadline, TIMEOUT).unwrap(); - let failure = failed(reattached); - assert_eq!( - (failure.record().progress, &failure.record().teardown), - (Progress::Planned, &Teardown::NoVm) - ); - let records = CloneRecords::read(locked).unwrap().unwrap(); - assert_eq!(records.failure.as_ref(), Some(failure.record())); - assert_eq!( - (records.progress, records.report), - (Progress::Planned, None) - ); -} - -/// Returns the message of the refusal of a clone's launch whose socket at `path` would not fit a -/// Unix socket's address. -#[cfg(target_os = "linux")] -fn too_long(path: &Path) -> String { - format!( - "the path of the socket {} of a clone's launch must be shorter than 108 bytes, as a Unix \ - socket's address requires", - path.display() - ) -} - -#[cfg(target_os = "linux")] -#[test] -fn a_clone_whose_control_socket_openvmm_or_the_host_cannot_use_starts_nothing() { - use std::os::unix::fs::PermissionsExt; - - /// Returns a directory of 60 characters in `directory`, private to the current user. - fn deep(directory: &Path) -> PathBuf { - let deep = directory.join("d".repeat(60)); - assert_eq!(platform::create_owner_only_dir(&deep).unwrap(), None); - deep - } - - let hooks = Hooks { - command: &never_spawned, - ..Hooks::LAUNCH - }; - let launch_refusal = |fixture: &Fixture, sandbox: &str, evidence: Evidence| { - complete_snapshot(fixture); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire(sandbox); - let plan = fixture - .plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT) - .unwrap(); - let runtime = runtime_of(fixture); - let launched = plan.launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks); - let refusal = refused_at(launched, Stage::Launch, evidence); - assert_unlaunched(fixture, &acquired); - (refusal, fixture.store.clone_socket_path(&acquired.lease)) - }; - // A store whose path leaves no room for the socket of a clone's VMM in a payload. - let fixture = Fixture::in_store(|directory| deep(directory).join("templates"), |_, _, _| {}); - let ((kind, message), socket) = launch_refusal(&fixture, "sandbox", Evidence::EndpointBind); - assert!(socket.as_os_str().len() >= 108); - assert_eq!((kind, message), (ErrorKind::Unsupported, too_long(&socket))); - // A store whose path is short, but long once its links are resolved, at which OpenVMM listens. - let fixture = Fixture::in_store( - |directory| { - let link = directory.join("l"); - std::os::unix::fs::symlink(deep(directory), &link).unwrap(); - link.join("templates") - }, - |_, _, _| {}, - ); - let ((kind, message), socket) = launch_refusal(&fixture, "sandbox", Evidence::EndpointBind); - assert!(socket.as_os_str().len() < 108, "{}", socket.display()); - let resolved = filesystem::canonicalize(socket.parent().unwrap()) - .unwrap() - .join(socket.file_name().unwrap()); - assert_eq!( - (kind, message), - (ErrorKind::Unsupported, too_long(&resolved)) - ); - // A store whose path holds a comma or an equals sign, which OpenVMM's option `listen=` - // takes for the end of the path. - for name in ["a,b", "a=b"] { - let fixture = Fixture::in_store( - |directory| { - let parent = directory.join(name); - assert_eq!(platform::create_owner_only_dir(&parent).unwrap(), None); - parent.join("templates") - }, - |_, _, _| {}, - ); - let bind = Evidence::EndpointBind; - let ((kind, message), socket) = launch_refusal(&fixture, "sandbox", bind); - assert_eq!( - (kind, message), - ( - ErrorKind::Unsupported, - format!( - "the path of the control socket {} of a clone's VMM must hold no ',' or '=', \ - which end the path in OpenVMM's option `listen=`", - socket.display() - ) - ) - ); - } - // A store whose path is not valid UTF-8, as OpenVMM takes the paths of its sockets. - let fixture = Fixture::in_store( - |directory| { - use std::os::unix::ffi::OsStrExt; - - let parent = directory.join(std::ffi::OsStr::from_bytes(b"a\xffb")); - assert_eq!(platform::create_owner_only_dir(&parent).unwrap(), None); - parent.join("templates") - }, - |_, _, _| {}, - ); - let ((kind, message), socket) = launch_refusal(&fixture, "sandbox", Evidence::EndpointBind); - assert!(socket.to_str().is_none(), "{}", socket.display()); - assert_eq!( - (kind, message), - ( - ErrorKind::Unsupported, - format!( - "the path of the socket {} of a clone's launch must be valid UTF-8, as OpenVMM \ - takes it", - socket.display() - ) - ) - ); - // A store whose path is valid UTF-8, but not once its links are resolved, as OpenVMM takes - // the path at which it listens. - let fixture = Fixture::in_store( - |directory| { - use std::os::unix::ffi::OsStrExt; - - let target = directory.join(std::ffi::OsStr::from_bytes(b"a\xffb")); - assert_eq!(platform::create_owner_only_dir(&target).unwrap(), None); - let link = directory.join("u"); - std::os::unix::fs::symlink(&target, &link).unwrap(); - link.join("templates") - }, - |_, _, _| {}, - ); - let ((kind, message), socket) = launch_refusal(&fixture, "sandbox", Evidence::EndpointBind); - assert!(socket.to_str().is_some(), "{}", socket.display()); - let resolved = filesystem::canonicalize(socket.parent().unwrap()) - .unwrap() - .join(socket.file_name().unwrap()); - assert!(resolved.to_str().is_none(), "{}", resolved.display()); - assert_eq!( - (kind, message), - ( - ErrorKind::Unsupported, - format!( - "the path of the socket {} of a clone's launch must be valid UTF-8, as OpenVMM \ - takes it", - resolved.display() - ) - ) - ); - // A payload private to the current user, as a plan requires, but whose mode is not 0700, as - // OpenVMM requires of the directory of its control socket. An acquisition refuses such a - // payload too, so its mode changes here once the lease is granted. - let fixture = Fixture::new(); - complete_snapshot(&fixture); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("sandbox"); - let plan = fixture - .plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT) - .unwrap(); - let payload = fixture.store.payload_dir(fixture.generation.generation()); - fs::set_permissions(&payload, fs::Permissions::from_mode(0o500)).unwrap(); - let launched = plan.launch_with( - &mut acquired, - Path::new(OPENVMM), - &runtime_of(&fixture), - hooks, - ); - let refusal = refused_at(launched, Stage::Launch, Evidence::MalformedGeneration); - fs::set_permissions(&payload, fs::Permissions::from_mode(0o700)).unwrap(); - assert_unlaunched(&fixture, &acquired); - assert_eq!( - refusal, - ( - ErrorKind::Io, - format!( - "the payload of the generation {} has mode 500, not the mode 700 that the \ - directory of a control socket must have", - payload.display() - ) - ) - ); -} - -/// A clone's launch in a process of its own, which [`run_launch_role`] runs: it leases the store -/// generation [`ROLE_GENERATION`] of the store in [`ROLE_STORE`] to the sandbox in -/// [`ROLE_SANDBOX`], plans its clone of the image in [`ROLE_IMAGE`] in RAM, and launches it with a -/// [`restore_vmm_role`] in [`ROLE_VMM`] as its VMM; and it exits without unwinding where -/// [`ROLE_CRASH`] says, as a crash would, once it has printed the reference of the VMM that it -/// spawned, if any. -#[test] -#[ignore = "a clone's launch that the launch tests run in a process of its own"] -fn launch_role() { - let (Some(dir), Ok(generation), Some(sandbox), Some(image), Ok(step), Some(vmm)) = ( - env::var_os(ROLE_STORE), - env::var(ROLE_GENERATION), - env::var_os(ROLE_SANDBOX).map(PathBuf::from), - env::var_os(ROLE_IMAGE).map(PathBuf::from), - env::var(ROLE_CRASH), - env::var_os(ROLE_VMM).map(PathBuf::from), - ) else { - return; - }; - let store = Store::open(dir).unwrap(); - let generation = StoreGeneration::new(generation.parse().unwrap()); - let indexed = store.index().unwrap().get(generation).unwrap().clone(); - let journal = sandbox.join(JOURNAL_NAME); - let kept = sandbox.join(format!("{JOURNAL_NAME}.kept")); - let mut acquired = store - .acquire(generation, sandbox, &files(), &after(&indexed, 1), None) - .unwrap(); - let binding = DistroBinding::import(&layer(&image), PolicyVerdict::ACCEPTED).unwrap(); - let plan = ClonePlan::new_with( - &store, - &mut acquired, - &tmpfs_request(&image, &binding), - Instant::now() + CLAIM_RELEASE_WAIT, - NO_CONTENT, - ) - .unwrap(); - let runtime = plan.key().runtime().clone(); - let made = RefCell::new(None); - let command = fake_vmm(&vmm, &made); - let interrupt = |point: Point| -> Result<()> { - if format!("{point:?}").to_lowercase() == step { - crash(&step); - } - Ok(()) - }; - let hooks = Hooks { - command: &command, - interrupt: &interrupt, - ..Hooks::LAUNCH - }; - let restoring = plan - .launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks) - .unwrap(); - println!( - "\n{REFERENCE}{}", - serde_json::to_string(restoring.starting().process().reference()).unwrap() - ); - if step == "launched" { - crash(&step); - } - // The restore sequence, through a framing that crashes within the Activate or the start. - let mut control = RoleControl { step: &step }; - let activated = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap(); - if step == "activated" { - crash(&step); - } - let started = activated.start(&mut control).unwrap(); - if step == "started" { - crash(&step); - } - if step == "superseded" { - fs::hard_link(&journal, &kept).unwrap(); - } - let committed = started.commit().unwrap(); - if step == "superseded" { - // The commit's removal of the journal failed, as a commit tolerates: the journal, the - // same file, remains beside the runtime record that supersedes it. - fs::rename(&kept, &journal).unwrap(); - crash(&step); - } - if step == "committed" { - crash(&step); - } - drop(committed); - panic!("the clone's launch did not crash at {step}"); -} - -/// The adapter's framing of a [`launch_role`], whose clone answers as the record asks, and which -/// crashes as it awaits the clone's Ready, at the step `activating`, or the start of its sandbox, -/// at the step `starting`. -struct RoleControl<'a> { - /// The step at which the role crashes. - step: &'a str, -} - -impl CloneControl for RoleControl<'_> { - type Session = Box; - type Error = io::Error; - - fn attach( - &mut self, - transport: Box, - _capability: &[u8; CAPABILITY_LEN], - _deadline: Instant, - ) -> io::Result> { - Ok(transport) - } - - fn activate( - &mut self, - _session: &mut Box, - record: &[u8], - _deadline: Instant, - ) -> io::Result> { - if self.step == "activating" { - crash(self.step); - } - let record = ActivationRecord::decode(record).unwrap(); - Ok(Some(ReadyActivation::of(&record))) - } - - fn map( - &mut self, - _session: &mut Box, - _table: &[HostMap], - _deadline: Instant, - ) -> io::Result<()> { - // The role's clone has no export, so its sequence sends no mapping table. - unreachable!("a clone without an export got a mapping table") - } - - fn start(&mut self, _session: &mut Box, _deadline: Instant) -> io::Result<()> { - if self.step == "starting" { - crash(self.step); - } - Ok(()) - } - - fn lost(&self, _error: &io::Error) -> bool { - false - } -} - -/// Ends this process at once, once it has printed that it crashes at `step`, without unwinding -/// or running destructors, as a crash would. -fn crash(step: &str) -> ! { - println!("\n{CRASHED}{step}"); - std::io::stdout().flush().unwrap(); - std::process::exit(0) -} - -/// Runs [`launch_role`] for the generation of `fixture`, the sandbox in `sandbox`, the image -/// `image`, and a VMM in `vmm`, until it crashes at `step`, and returns the reference of the VMM -/// that it spawned, if it spawned one. -fn run_launch_role( - step: &str, - fixture: &Fixture, - sandbox: &Path, - image: &Path, - vmm: &Path, -) -> Option { - let mut command = Command::new(env::current_exe().unwrap()); - command - .args(role("launch_role")) - .env(ROLE_STORE, fixture.store.dir()) - .env(ROLE_GENERATION, fixture.generation.generation().to_string()) - .env(ROLE_SANDBOX, sandbox) - .env(ROLE_IMAGE, image) - .env(ROLE_CRASH, step) - .env(ROLE_VMM, vmm) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let output = supervisor::spawn_command(&mut command) - .unwrap() - .wait_with_output() - .unwrap(); - let stdout = String::from_utf8_lossy(&output.stdout); - let crashed = format!("{CRASHED}{step}"); - assert!( - output.status.success() && stdout.lines().any(|line| line == crashed), - "{}: {stdout}{}", - output.status, - String::from_utf8_lossy(&output.stderr) - ); - stdout - .lines() - .find_map(|line| line.strip_prefix(REFERENCE)) - .map(|reference| serde_json::from_str(reference).unwrap()) -} - -/// A clone whose launch's caller, a [`launch_role`], crashed, in a fresh fixture: the VMM's -/// working directory, the sandbox, and the lease that the sandbox holds. -struct Crashed { - /// The working directory of the clone's VMM, which ends the VMM as it drops, before the rest. - vmm: VmmDir, - /// The template store, its generation, and the clone's image. - fixture: Fixture, - /// The sandbox's directory. - sandbox: PathBuf, - /// The lease that the sandbox holds. - lease: Lease, - /// The runtime identity under which the clone launched. - runtime: RuntimeIdentity, -} - -impl Crashed { - /// Leases a fresh fixture's generation to a new sandbox, whose clone a [`launch_role`] - /// launches until it crashes at `step`. - fn at(step: &str) -> Self { - Self::at_with(step, "") - } - - /// Leases a fresh fixture's generation to a new sandbox, whose clone a [`launch_role`] - /// launches until it crashes at `step`, with a VMM whose [plan](VmmDir::plan) holds `words`. - fn at_with(step: &str, words: &str) -> Self { - let fixture = Fixture::new(); - complete_snapshot(&fixture); - let (image, _) = fixture.image("distro.erofs", IMAGE_LEN); - let sandbox = fixture.provision("sandbox"); - let mut vmm = VmmDir::new(); - let mut plan = words.to_owned(); - if !matches!(step, "prepared" | "begun" | "launched") { - // The restore sequence reads the event, whose flush waits for that on Windows. - plan.push_str(" flush"); - } - vmm.plan(&plan); - vmm.reference = run_launch_role(step, &fixture, &sandbox, &image, vmm.path()); - let lease = Lease::of(&SandboxStore::open(&sandbox).unwrap()) - .unwrap() - .expect("the crash kept the clone's lease"); - let runtime = runtime_of(&fixture); - Self { - vmm, - fixture, - sandbox, - lease, - runtime, - } - } - - /// Takes the lock of the clone's sandbox, as the restarted caller does. - fn lock(&self) -> LockedStore { - SandboxStore::lock_existing(&self.sandbox, &files(), Some(Instant::now() + TIMEOUT)) - .unwrap() - } - - /// Checks that the sandbox holds the clone's activation record, with its restore - /// transaction, and, on Linux, the restore-ready socket. - fn kept_its_files(&self, locked: &LockedStore) { - let recorded = fs::read(locked.clone_activation_path()).unwrap(); - let record = ActivationRecord::decode(&recorded).unwrap(); - assert!(record.restore().is_some()); - if cfg!(target_os = "linux") { - assert!(exists(&locked.restore_ready_path())); - } - } - - /// Returns the path of the control socket of the clone's VMM, beside the snapshot that it - /// restores. - fn socket(&self) -> PathBuf { - self.fixture.store.clone_socket_path(&self.lease) - } - - /// Reattaches to the VM of the sandbox of `locked` until the result is determinate, as a - /// caller retries an indeterminate reattach, and returns it. - fn reattach(&self, locked: &mut LockedStore) -> Reattached { - let deadline = Instant::now() + TIMEOUT; - loop { - match supervisor::reattach(locked, &self.runtime).unwrap() { - Reattached::Indeterminate(reason) if Instant::now() < deadline => { - eprintln!("reattach is indeterminate: {reason}"); - thread::sleep(Duration::from_millis(100)); - } - reattached => return reattached, - } - } - } - - /// Takes the clone of the sandbox of `locked` over, under `runtime`, through `control`, with - /// [`reattach_clone`], until the result is determinate, as a caller retries an indeterminate - /// reattach, and returns it. - fn take_over( - &self, - locked: &mut LockedStore, - runtime: &RuntimeIdentity, - control: &mut C, - ) -> ReattachedClone { - let deadline = Instant::now() + TIMEOUT; - loop { - let session = Instant::now() + TIMEOUT; - match reattach_clone(locked, runtime, control, session, TIMEOUT).unwrap() { - ReattachedClone::Indeterminate(reason) if Instant::now() < deadline => { - eprintln!("reattach is indeterminate: {reason}"); - thread::sleep(Duration::from_millis(100)); - } - reattached => return reattached, - } - } - } - - /// Checks that the store refuses to release the lease of the sandbox of `locked`, whose - /// launch has not ended, and that the sandbox keeps it. - fn refuses_release(&self, locked: &mut LockedStore) { - let error = self.fixture.store.release(locked).unwrap_err(); - assert_eq!(error.code(), ErrorCode::SandboxRunning, "{error}"); - assert_eq!(Lease::of(locked).unwrap(), Some(self.lease.clone())); - } - - /// Checks that the store releases the lease of the sandbox of `locked`, which removes the - /// clone's files, and nothing of the generation, whose payload holds no socket of the clone. - fn releases(&self, locked: &mut LockedStore) { - assert_eq!( - self.fixture.store.release(locked).unwrap(), - Some(self.lease.clone()) - ); - assert_released(&self.sandbox); - assert!(CloneRecords::read(locked).unwrap().is_none()); - assert!(!exists(&self.socket())); - let snapshot = self.fixture.store.snapshot_dir(&self.fixture.generation); - for name in SNAPSHOT_ARTIFACTS { - assert!(snapshot.as_ref().unwrap().join(name).is_file(), "{name}"); - } - } -} - -#[test] -fn a_launch_whose_caller_crashes_before_it_begins_leaves_files_that_the_release_removes() { - let crashed = Crashed::at("prepared"); - assert_eq!(crashed.vmm.reference, None); - let mut locked = crashed.lock(); - // The launch never began: the sandbox records no launch, and keeps the clone's files. - assert!(locked.journal().unwrap().is_none()); - crashed.kept_its_files(&locked); - match crashed.reattach(&mut locked) { - Reattached::Exited { - record: None, - recovery: Recovery::Clean, - } => {} - other => panic!("unexpected reattach: {other:?}"), - } - crashed.releases(&mut locked); -} - -#[test] -fn a_launch_whose_caller_crashes_once_it_began_ends_as_reattach_recovers_it() { - let crashed = Crashed::at("begun"); - assert_eq!(crashed.vmm.reference, None); - let mut locked = crashed.lock(); - // The journal names the launch that the lease reserved, which spawned nothing, and keeps the - // lease until a reattach ends it. - let journal = locked.journal().unwrap().unwrap(); - assert_eq!(journal.launch(), crashed.lease.launch()); - assert_eq!(journal.state(), LaunchState::Prepared); - crashed.kept_its_files(&locked); - crashed.refuses_release(&mut locked); - match crashed.reattach(&mut locked) { - Reattached::Exited { - record: None, - recovery: Recovery::Exited, - } => {} - other => panic!("unexpected reattach: {other:?}"), - } - crashed.releases(&mut locked); -} - -#[test] -fn a_launch_that_began_shows_the_report_of_a_vmm_that_its_journal_does_not_record() { - // The caller crashed once the launch began, before the journal recorded the VMM that it - // spawned, which left the report: the clone's, since the launch removed any earlier VM's - // report before it recorded anything, though the clone's progress shows no spawn. An - // inspection takes the report once a reattach of the clone ended the launch, recording its - // failure, and once a recovery of the sandbox's launch did, recording none. - let shown = |records: &CloneRecords| { - let report = records - .report - .as_ref() - .expect("the report of the clone's VM"); - (report.category.clone(), report.status_code) - }; - let ended = (Some("guest-exit".to_owned()), Some(3)); - for of_the_clone in [true, false] { - let crashed = Crashed::at("begun"); - let mut locked = crashed.lock(); - fs::write(locked.outcome_path(), guest_exit(3)).unwrap(); - let records = CloneRecords::read(&locked).unwrap().unwrap(); - assert_eq!( - (records.progress, records.journal.is_some()), - (Progress::Launching, true) - ); - assert_eq!(shown(&records), ended); - if of_the_clone { - let mut control = ReattachControl::answering(Some); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.record().teardown, Teardown::Exited); - } else { - match crashed.reattach(&mut locked) { - Reattached::Exited { - record: None, - recovery: Recovery::Exited, - } => {} - other => panic!("unexpected reattach: {other:?}"), - } - } - let records = CloneRecords::read(&locked).unwrap().unwrap(); - assert_eq!( - ( - records.progress, - records.journal.is_none(), - records.failure.is_some() - ), - (Progress::Launching, true, of_the_clone) - ); - assert_eq!(shown(&records), ended, "{of_the_clone}"); - crashed.releases(&mut locked); - } -} - -#[test] -fn a_reattach_takes_the_report_of_a_vmm_whose_launch_a_recovery_of_the_sandbox_ended() { - // The caller crashed once the launch began, before the journal recorded the VMM that it - // spawned, which left the report, and a recovery of the sandbox's launch then removed the - // journal: nothing records the spawn, but the report is the clone's VM's, as an inspection - // takes it, so a reattach of the clone takes it too. - let recovered = |status: u8| { - let crashed = Crashed::at("begun"); - let mut locked = crashed.lock(); - fs::write(locked.outcome_path(), guest_exit(status)).unwrap(); - match crashed.reattach(&mut locked) { - Reattached::Exited { - record: None, - recovery: Recovery::Exited, - } => {} - other => panic!("unexpected reattach: {other:?}"), - } - assert!(locked.journal().unwrap().is_none()); - (crashed, locked) - }; - let mut control = ReattachControl::answering(Some); - // The status of the guest's repair is evidence of the clone's failure at any point. - let (crashed, mut locked) = recovered(195); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.kind(), ErrorKind::Exited, "{failure}"); - assert_recorded( - &failure, - &crashed.lease, - &locked, - (Stage::Launch, Progress::Launching), - Evidence::RepairStatus, - Teardown::Exited, - ); - assert_eq!(failure.record().status, Some(195)); - crashed.releases(&mut locked); - // Another status is no evidence of a failure: the clone ended, with the report of its VM. - let (crashed, mut locked) = recovered(3); - match crashed.take_over(&mut locked, &crashed.runtime, &mut control) { - ReattachedClone::Ended { - progress: Progress::Launching, - report: Some(report), - } => { - let outcome = (report.category.as_deref(), report.status_code); - assert_eq!(outcome, (Some("guest-exit"), Some(3))); - } - other => panic!("the reattach found {}", describe(&other)), - } - let records = CloneRecords::read(&locked).unwrap().unwrap(); - assert!(records.failure.is_none()); - crashed.releases(&mut locked); - assert!(control.capabilities.is_empty()); -} - -#[test] -fn a_launch_whose_caller_crashes_once_its_vmm_started_ends_the_vmm_as_reattach_recovers_it() { - let crashed = Crashed::at("launched"); - let reference = crashed.vmm.reference.clone().unwrap(); - let Opened::Running(process) = Process::open(&reference).unwrap() else { - panic!("the crashed caller's VMM runs"); - }; - // The VMM listens beside the snapshot on Linux, as it has once it tried to send its event. - crashed.vmm.started(); - assert_eq!(exists(&crashed.socket()), cfg!(target_os = "linux")); - let mut locked = crashed.lock(); - // The journal records the VMM, which never committed: reattach ends it through its verified - // reference, and the lease lasts until then. The VMM, which the termination ends, leaves its - // control socket, which the end of the launch removes. - let journal = locked.journal().unwrap().unwrap(); - assert_eq!(journal.launch(), crashed.lease.launch()); - assert_eq!(journal.state(), LaunchState::Spawned); - assert_eq!(journal.process(), Some(&reference)); - if cfg!(target_os = "linux") { - assert_eq!(journal.endpoint(), crashed.socket().to_str().unwrap()); - } - crashed.kept_its_files(&locked); - crashed.refuses_release(&mut locked); - match crashed.reattach(&mut locked) { - Reattached::Exited { - record: None, - recovery: Recovery::Terminated(terminated), - } => assert_eq!(terminated, reference), - other => panic!("unexpected reattach: {other:?}"), - } - assert!(process.wait(Instant::now() + TIMEOUT).unwrap().is_some()); - assert!(!exists(&crashed.socket())); - crashed.releases(&mut locked); -} - -#[test] -fn a_restore_whose_caller_crashes_before_its_commit_ends_its_vmm_as_reattach_recovers_it() { - // The caller crashes as it awaits the clone's Ready, once the clone is ready, as it awaits the - // start of the clone's sandbox, and once the sandbox started: each time, the launch has not - // committed, and its journal names the VMM. - for step in ["activating", "activated", "starting", "started"] { - let crashed = Crashed::at(step); - let reference = crashed.vmm.reference.clone().unwrap(); - let Opened::Running(process) = Process::open(&reference).unwrap() else { - panic!("the crashed caller's VMM runs"); - }; - // The caller read the VMM's restore-ready event, whose flush waited for that on Windows. - assert!(crashed.vmm.started().ends_with(" true"), "{step}"); - assert!(crashed.vmm.flushed(), "{step}"); - let mut locked = crashed.lock(); - let journal = locked.journal().unwrap().unwrap(); - assert_eq!(journal.launch(), crashed.lease.launch(), "{step}"); - assert_eq!(journal.state(), LaunchState::Spawned, "{step}"); - assert_eq!(journal.process(), Some(&reference), "{step}"); - assert!(locked.runtime().unwrap().is_none(), "{step}"); - crashed.kept_its_files(&locked); - // The lease lasts until reattach ended the VMM, through its verified reference. - crashed.refuses_release(&mut locked); - match crashed.reattach(&mut locked) { - Reattached::Exited { - record: None, - recovery: Recovery::Terminated(terminated), - } => assert_eq!(terminated, reference, "{step}"), - other => panic!("unexpected reattach at {step}: {other:?}"), - } - assert!( - process.wait(Instant::now() + TIMEOUT).unwrap().is_some(), - "{step}" - ); - assert!(!exists(&crashed.socket()), "{step}"); - crashed.releases(&mut locked); - } -} - -#[test] -fn a_restore_whose_caller_crashes_once_it_committed_keeps_its_lease_while_its_vm_runs() { - let crashed = Crashed::at("committed"); - let reference = crashed.vmm.reference.clone().unwrap(); - let mut locked = crashed.lock(); - // The runtime record names the VM, which reattach holds again, and the lease lasts as long as - // the VM runs. - assert!(locked.journal().unwrap().is_none()); - let record = locked.runtime().unwrap().unwrap(); - assert_eq!(record.process(), &reference); - crashed.kept_its_files(&locked); - let vmm = match crashed.reattach(&mut locked) { - Reattached::Live(vmm) => vmm, - other => panic!("unexpected reattach: {other:?}"), - }; - assert_eq!(vmm.process().reference(), &reference); - crashed.refuses_release(&mut locked); - // Once a termination verified the VM's exit, the release ends the lease. - assert_eq!( - vmm.terminate(&mut locked, TIMEOUT).unwrap(), - StopOutcome::Forced - ); - assert!(!exists(&crashed.socket())); - crashed.releases(&mut locked); -} - -/// The Ready of a clone that a [`ReattachControl`] answers, from the activation of the record that -/// the clone's Activate carries. -type Answered = fn(ReadyActivation) -> Option; - -/// The adapter's framing of the sessions of a reattach: it attaches a session to each connection -/// to the clone's control endpoint, keeps the capability of each, and the record of each Activate, -/// which it answers as its `answer` says. -struct ReattachControl { - /// The capabilities of the sessions, in order. - capabilities: Vec<[u8; CAPABILITY_LEN]>, - /// The records of the Activates, in order. - records: Vec>, - /// How the clone answers an Activate. - answer: Answered, -} - -impl ReattachControl { - /// Returns a framing whose clone answers each Activate as `answer` says. - fn answering(answer: Answered) -> Self { - Self { - capabilities: Vec::new(), - records: Vec::new(), - answer, - } - } -} - -impl CloneControl for ReattachControl { - type Session = Box; - type Error = io::Error; - - fn attach( - &mut self, - transport: Box, - capability: &[u8; CAPABILITY_LEN], - _deadline: Instant, - ) -> io::Result> { - self.capabilities.push(*capability); - Ok(transport) - } - - fn activate( - &mut self, - _session: &mut Box, - record: &[u8], - _deadline: Instant, - ) -> io::Result> { - self.records.push(record.to_vec()); - let record = ActivationRecord::decode(record).unwrap(); - Ok((self.answer)(ReadyActivation::of(&record))) - } - - fn map( - &mut self, - _session: &mut Box, - _table: &[HostMap], - _deadline: Instant, - ) -> io::Result<()> { - panic!("a reattach sends no mapping table") - } - - fn start(&mut self, _session: &mut Box, _deadline: Instant) -> io::Result<()> { - panic!("a reattach starts no sandbox") - } - - fn lost(&self, _error: &io::Error) -> bool { - false - } -} - -#[test] -fn a_clone_whose_caller_crashed_once_it_committed_is_taken_over_in_a_new_session() { - let crashed = Crashed::at("committed"); - let reference = crashed.vmm.reference.clone().unwrap(); - let started = crashed.vmm.started(); - let (capability, _) = started.split_once(' ').unwrap(); - // A tombstone quarantines the clone's generation, which a retirement deletes once its leases - // have drained: neither takes the clone's lease from it. - let store = &crashed.fixture.store; - let generation = crashed.fixture.generation.generation(); - store.tombstone(generation, None).unwrap(); - assert_eq!( - store.retire(generation, None).unwrap(), - Retirement::Draining(vec![crashed.lease.clone()]) - ); - let mut locked = crashed.lock(); - let recorded = fs::read(locked.clone_activation_path()).unwrap(); - let mut control = ReattachControl::answering(Some); - let (vmm, session) = match crashed.take_over(&mut locked, &crashed.runtime, &mut control) { - ReattachedClone::Live { vmm, session } => (vmm, session), - other => panic!("the reattach found {}", describe(&other)), - }; - // The VM is the one that the launch committed, whose clone answered the Activate of the - // sandbox's record, byte for byte, in a session that the launch's capability authenticated, - // and runs on, with its lease. - assert_eq!(vmm.process().reference(), &reference); - assert_eq!(control.records, [recorded]); - let attached: Vec = control.capabilities.iter().map(|key| hex(key)).collect(); - assert_eq!(attached, [capability]); - drop(session); - let records = CloneRecords::read(&locked).unwrap().unwrap(); - assert_eq!( - (records.progress, records.failure), - (Progress::Committed, None) - ); - crashed.refuses_release(&mut locked); - assert_eq!( - vmm.terminate(&mut locked, TIMEOUT).unwrap(), - StopOutcome::Forced - ); - crashed.releases(&mut locked); - // Its lease released, the retirement deletes the generation. - assert_eq!(store.retire(generation, None).unwrap(), Retirement::Retired); -} - -#[test] -fn a_journal_that_a_clone_s_commit_left_is_no_failure_of_the_clone() { - // The commit of the clone's launch could not remove the launch's journal, which remains - // beside the runtime record of the same launch, as its caller crashed. - let crashed = Crashed::at("superseded"); - let mut locked = crashed.lock(); - let journal = locked - .journal() - .unwrap() - .expect("the commit left the journal"); - let runtime = locked - .runtime() - .unwrap() - .expect("the commit wrote its record"); - assert_eq!(journal.launch(), runtime.launch()); - // Each reattach takes the clone over, and records no failure: the first removes the journal. - let mut control = ReattachControl::answering(Some); - let mut live = None; - for attempt in 0..2 { - let vmm = match crashed.take_over(&mut locked, &crashed.runtime, &mut control) { - ReattachedClone::Live { vmm, .. } => vmm, - other => panic!("the reattach {attempt} found {}", describe(&other)), - }; - assert_eq!( - Some(vmm.process().reference()), - crashed.vmm.reference.as_ref() - ); - let records = CloneRecords::read(&locked).unwrap().unwrap(); - assert_eq!( - (records.progress, records.failure, records.journal.is_some()), - (Progress::Committed, None, false), - "{attempt}" - ); - live = Some(vmm); - } - assert_eq!(control.records.len(), 2); - crashed.refuses_release(&mut locked); - let vmm = live.unwrap(); - assert_eq!( - vmm.terminate(&mut locked, TIMEOUT).unwrap(), - StopOutcome::Forced - ); - crashed.releases(&mut locked); -} - -#[test] -fn a_reattach_ends_a_clone_that_it_may_not_keep_through_its_verified_identity() { - let reattach = (Stage::Reattach, Progress::Committed); - // A clone that runs under another runtime identity, which might not speak this one's - // protocol: the reattach sends its guest nothing. - let crashed = Crashed::at("committed"); - let mut locked = crashed.lock(); - let other = RuntimeIdentity::new(artifact('d'), artifact('e'), artifact('f')); - let mut control = ReattachControl::answering(Some); - let failure = failed(crashed.take_over(&mut locked, &other, &mut control)); - assert_eq!(failure.code(), ErrorCode::RuntimeMismatch, "{failure}"); - let (lease, terminated) = (&crashed.lease, Teardown::Terminated); - let mismatch = Evidence::RuntimeMismatch; - assert_recorded(&failure, lease, &locked, reattach, mismatch, terminated); - assert_eq!(failure.outcome(), Outcome::Host); - assert!(control.capabilities.is_empty()); - assert!(locked.runtime().unwrap().is_none()); - crashed.releases(&mut locked); - // A clone whose Ready reports another activation than its record's. - let crashed = Crashed::at("committed"); - let mut locked = crashed.lock(); - let mut control = ReattachControl::answering(|_| Some(ReadyActivation { transaction: None })); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.kind(), ErrorKind::Protocol, "{failure}"); - let (lease, terminated) = (&crashed.lease, Teardown::Terminated); - assert_recorded( - &failure, - lease, - &locked, - reattach, - Evidence::Protocol, - terminated, - ); - assert_eq!(control.records.len(), 1); - assert!(locked.runtime().unwrap().is_none()); - crashed.releases(&mut locked); -} - -#[test] -fn a_reattach_fails_a_clone_whose_record_of_a_failure_cannot_be_read() { - let reattach = (Stage::Reattach, Progress::Committed); - let host = Evidence::HostFailure; - // A record that is malformed, and one of a format that this build does not read: the clone - // failed, so the reattach sends its guest nothing, and ends its VM through its verified - // identity, though its Ready would match its activation record. - let unreadable: [(&[u8], ErrorCode); 2] = [ - (b"not a record", ErrorCode::Io), - (br#"{"format": 99}"#, ErrorCode::FormatUnsupported), - ]; - for (bytes, code) in unreadable { - let crashed = Crashed::at("committed"); - let mut locked = crashed.lock(); - write_private(&locked.clone_failure_path(), bytes).unwrap(); - let mut control = ReattachControl::answering(Some); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.code(), code, "{failure}"); - let (lease, terminated) = (&crashed.lease, Teardown::Terminated); - assert_recorded(&failure, lease, &locked, reattach, host, terminated); - assert!(control.capabilities.is_empty()); - assert!(locked.runtime().unwrap().is_none()); - crashed.releases(&mut locked); - } - // A clone whose VM exited with a status that is no evidence of a failure failed all the same. - let crashed = Crashed::at_with("committed", "report:guest-exit:3"); - crashed.vmm.end_and_wait(); - let mut locked = crashed.lock(); - write_private(&locked.clone_failure_path(), b"not a record").unwrap(); - let mut control = ReattachControl::answering(Some); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - let (lease, exited) = (&crashed.lease, Teardown::Exited); - assert_recorded(&failure, lease, &locked, reattach, host, exited); - assert_eq!(failure.record().status, Some(3)); - assert!(control.capabilities.is_empty()); - crashed.releases(&mut locked); - // A clone whose VM cannot be examined failed all the same, with its teardown unresolved, which - // a later reattach resolves. - let crashed = Crashed::at("committed"); - let mut locked = crashed.lock(); - write_private(&locked.clone_failure_path(), b"not a record").unwrap(); - let mut control = ReattachControl::answering(Some); - let reason = "the test cannot examine the VM"; - let reattached = reattach_clone_with( - &mut locked, - &crashed.runtime, - &mut control, - Instant::now() + TIMEOUT, - TIMEOUT, - |_, _, _| Ok(Reattached::Indeterminate(reason.to_owned())), - ) - .unwrap(); - let failure = failed(reattached); - let (lease, unresolved) = (&crashed.lease, Teardown::Unresolved(reason.to_owned())); - assert_recorded(&failure, lease, &locked, reattach, host, unresolved); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - let terminated = Teardown::Terminated; - assert_recorded(&failure, lease, &locked, reattach, host, terminated); - assert!(control.capabilities.is_empty()); - crashed.releases(&mut locked); -} - -#[test] -fn a_reattach_fails_a_clone_whose_sandbox_records_the_failure_of_another_clone() { - // A record of the failure of another lease's clone, of another launch, which leased another - // generation of the same store, whose evidence is the template's: it tells nothing of this - // clone, so the reattach takes it as a record that it cannot read, and the failure that it - // records in its place, and returns, is this clone's, which quarantines nothing. - let crashed = Crashed::at("committed"); - let mut locked = crashed.lock(); - let lease = &crashed.lease; - let generation = lease.generation().get().checked_add(1).unwrap(); - let other = Lease::new( - crate::host::template::LeaseId::from_bytes([0xa5; 16]), - lease.store(), - StoreGeneration::new(generation), - lease.template().key, - lease.sandbox().to_owned(), - crate::host::store::LaunchId::generate().unwrap(), - ); - let defect = Error::io("the test's generation is malformed"); - let at = ((Stage::Restore, Progress::Spawned), None); - let template = (Evidence::MalformedGeneration, Teardown::Exited); - record_earlier(&other, &locked, at, template, &defect); - let mut control = ReattachControl::answering(Some); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.code(), ErrorCode::Io, "{failure}"); - assert!( - failure.message().contains("belongs to another clone"), - "{failure}" - ); - let reattach = (Stage::Reattach, Progress::Committed); - let (host, terminated) = (Evidence::HostFailure, Teardown::Terminated); - assert_recorded(&failure, lease, &locked, reattach, host, terminated); - assert!(control.capabilities.is_empty()); - assert!(locked.runtime().unwrap().is_none()); - crashed.releases(&mut locked); - drop(locked); - let store = &crashed.fixture.store; - assert!(!failure.quarantine(store, None).unwrap()); - assert!(store.tombstones().unwrap().is_empty()); -} - -#[test] -fn a_reattach_fails_a_clone_whose_record_of_a_failure_is_of_another_template() { - // A clone whose sandbox records no failure is healthy: the reattach takes it over, and needs - // no record of a failure to match its lease. - let crashed = Crashed::at("committed"); - let mut locked = crashed.lock(); - let mut control = ReattachControl::answering(Some); - let vmm = match crashed.take_over(&mut locked, &crashed.runtime, &mut control) { - ReattachedClone::Live { vmm, .. } => vmm, - other => panic!("the reattach found {}", describe(&other)), - }; - let records = CloneRecords::read(&locked).unwrap().unwrap(); - assert_eq!(records.failure, None); - assert_eq!( - vmm.terminate(&mut locked, TIMEOUT).unwrap(), - StopOutcome::Forced - ); - crashed.releases(&mut locked); - // A record of a failure of the clone's own store, lease, and launch, but of another template: - // of another generation, or of another key of the leased generation. It tells nothing of this - // clone, whose activation record the reattach no longer compares with the lease, so the - // reattach takes it as a record that it cannot read, as it takes another lease's. - let leased = crashed.lease.template(); - let mut key = *leased.key.as_bytes(); - key[0] ^= 0xff; - let cases = [ - ( - leased.generation.get().checked_add(1).unwrap(), - leased.key, - "the leased generation", - ), - ( - leased.generation.get(), - crate::activation::Identity::from_bytes(key), - "the key of the leased generation", - ), - ]; - for (generation, key, difference) in cases { - let crashed = Crashed::at("committed"); - let mut locked = crashed.lock(); - let lease = &crashed.lease; - let other = Lease::new( - lease.id(), - lease.store(), - StoreGeneration::new(generation), - key, - lease.sandbox().to_owned(), - lease.launch().clone(), - ); - let defect = Error::io("the test's generation is malformed"); - let at = ((Stage::Restore, Progress::Spawned), None); - let template = (Evidence::MalformedGeneration, Teardown::Exited); - record_earlier(&other, &locked, at, template, &defect); - let mut control = ReattachControl::answering(Some); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.code(), ErrorCode::Io, "{difference}: {failure}"); - let differs = format!("they differ in {difference};"); - assert!(failure.message().contains(&differs), "{failure}"); - let reattach = (Stage::Reattach, Progress::Committed); - let (host, terminated) = (Evidence::HostFailure, Teardown::Terminated); - assert_recorded(&failure, lease, &locked, reattach, host, terminated); - assert!(control.capabilities.is_empty(), "{difference}"); - assert!(locked.runtime().unwrap().is_none(), "{difference}"); - crashed.releases(&mut locked); - drop(locked); - let store = &crashed.fixture.store; - assert!(!failure.quarantine(store, None).unwrap()); - assert!(store.tombstones().unwrap().is_empty()); - } -} - -#[test] -fn a_record_of_a_failure_that_cannot_be_read_is_a_failure_whatever_the_reattach_finds() { - let reattach = (Stage::Reattach, Progress::Committed); - // A VM that exited once its launch committed, with a status of the guest's that is evidence, - // or without a report of the guest's own end: the clone failed before, so the failure is the - // reattach's, with the error of the record's read, over whose evidence the status's prevails. - let cases = [ - ("report:guest-exit:195", Evidence::RepairStatus, Some(195)), - ("", Evidence::HostFailure, None), - ]; - for (words, evidence, status) in cases { - let crashed = Crashed::at_with("committed", words); - crashed.vmm.end_and_wait(); - let mut locked = crashed.lock(); - write_private(&locked.clone_failure_path(), b"not a record").unwrap(); - let mut control = ReattachControl::answering(Some); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.code(), ErrorCode::Io, "{words}: {failure}"); - let (lease, exited) = (&crashed.lease, Teardown::Exited); - assert_recorded(&failure, lease, &locked, reattach, evidence, exited); - assert_eq!(failure.record().status, status, "{words}"); - assert!(locked.runtime().unwrap().is_none(), "{words}"); - assert!(control.capabilities.is_empty(), "{words}"); - crashed.releases(&mut locked); - } - // A VM that runs under another runtime identity is ended, and the failure is the same. - let crashed = Crashed::at("committed"); - let mut locked = crashed.lock(); - write_private(&locked.clone_failure_path(), b"not a record").unwrap(); - let other = RuntimeIdentity::new(artifact('d'), artifact('e'), artifact('f')); - let mut control = ReattachControl::answering(Some); - let failure = failed(crashed.take_over(&mut locked, &other, &mut control)); - assert_eq!(failure.code(), ErrorCode::Io, "{failure}"); - let (lease, terminated) = (&crashed.lease, Teardown::Terminated); - let host = Evidence::HostFailure; - assert_recorded(&failure, lease, &locked, reattach, host, terminated); - assert!(control.capabilities.is_empty()); - crashed.releases(&mut locked); - // A launch that a crash of its caller interrupted: the reattach replaces the record with its - // own failure, pending, before its recovery ends the launch, which a crash of the restarted - // caller leaves, for the next reattach to finish. - let spawned = (Stage::Reattach, Progress::Spawned); - let assert_pending = |locked: &LockedStore| { - let pending = read_failure(&locked.clone_failure_path()) - .unwrap() - .expect("the reattach replaced the record that it could not read"); - assert_eq!( - ( - pending.stage, - pending.progress, - pending.evidence, - pending.code - ), - (spawned.0, spawned.1, host, ErrorCode::Io) - ); - assert_eq!(pending.teardown, Teardown::Pending); - }; - let crashed = Crashed::at("launched"); - let mut locked = crashed.lock(); - write_private(&locked.clone_failure_path(), b"not a record").unwrap(); - let mut control = ReattachControl::answering(Some); - let crash = reattach_clone_with( - &mut locked, - &crashed.runtime, - &mut control, - Instant::now() + TIMEOUT, - TIMEOUT, - |_, _, _| Err(Error::other("the test's caller crashes")), - ); - assert!(crash.is_err()); - assert_pending(&locked); - assert!(locked.journal().unwrap().is_some()); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.code(), ErrorCode::Io, "{failure}"); - let (lease, terminated) = (&crashed.lease, Teardown::Terminated); - assert_recorded(&failure, lease, &locked, spawned, host, terminated); - assert!(control.capabilities.is_empty()); - crashed.releases(&mut locked); - // Before the restore-ready event, only the launch's journal, which the recovery removes, holds - // the spawn: once it went, the pending record still tells how far the clone got, so the next - // reattach takes the status with which the guest ended the VM. - let crashed = Crashed::at_with("launched", "report:guest-exit:195"); - crashed.vmm.end_and_wait(); - let mut locked = crashed.lock(); - write_private(&locked.clone_failure_path(), b"not a record").unwrap(); - let mut control = ReattachControl::answering(Some); - let crash = reattach_clone_with( - &mut locked, - &crashed.runtime, - &mut control, - Instant::now() + TIMEOUT, - TIMEOUT, - |locked, runtime, exited| { - supervisor::reattach_reporting(locked, runtime, exited)?; - Err(Error::other("the test's caller crashes")) - }, - ); - assert!(crash.is_err()); - assert_pending(&locked); - assert!(locked.journal().unwrap().is_none()); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.code(), ErrorCode::Io, "{failure}"); - let (lease, exited) = (&crashed.lease, Teardown::Exited); - let repair = Evidence::RepairStatus; - assert_recorded(&failure, lease, &locked, spawned, repair, exited); - assert_eq!(failure.record().status, Some(195)); - assert!(control.capabilities.is_empty()); - crashed.releases(&mut locked); - // A clone whose launch did not begin: the store holds no record of the launch that the - // reattach would remove, so the record stays as it is, and a reattach after a crash of the - // caller finds the clone as this one does, with no VM. - let crashed = Crashed::at("prepared"); - let mut locked = crashed.lock(); - write_private(&locked.clone_failure_path(), b"not a record").unwrap(); - let mut control = ReattachControl::answering(Some); - let crash = reattach_clone_with( - &mut locked, - &crashed.runtime, - &mut control, - Instant::now() + TIMEOUT, - TIMEOUT, - |_, _, _| Err(Error::other("the test's caller crashes")), - ); - assert!(crash.is_err()); - let kept = fs::read(locked.clone_failure_path()).unwrap(); - assert_eq!(kept, b"not a record"); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.code(), ErrorCode::Io, "{failure}"); - let launching = (Stage::Reattach, Progress::Launching); - let (lease, no_vm) = (&crashed.lease, Teardown::NoVm); - assert_recorded(&failure, lease, &locked, launching, host, no_vm); - assert_eq!(failure.record().status, None); - crashed.releases(&mut locked); - // A clone whose launch a recovery of the sandbox ended, once its caller crashed before the - // journal recorded the VMM that it spawned, which left the report: nothing records the spawn, - // but the report is the clone's VM's, since the launch removed any earlier VM's report before - // it recorded anything, and the status of the guest's repair prevails. - let crashed = Crashed::at("begun"); - let mut locked = crashed.lock(); - fs::write(locked.outcome_path(), guest_exit(195)).unwrap(); - match crashed.reattach(&mut locked) { - Reattached::Exited { - record: None, - recovery: Recovery::Exited, - } => {} - other => panic!("unexpected reattach: {other:?}"), - } - write_private(&locked.clone_failure_path(), b"not a record").unwrap(); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.code(), ErrorCode::Io, "{failure}"); - let (lease, exited) = (&crashed.lease, Teardown::Exited); - let repair = Evidence::RepairStatus; - assert_recorded(&failure, lease, &locked, launching, repair, exited); - assert_eq!(failure.record().status, Some(195)); - crashed.releases(&mut locked); -} - -#[test] -fn a_reattach_ends_the_launch_of_a_caller_that_crashed_and_records_its_failure() { - let cases = [ - ( - "begun", - (Stage::Launch, Progress::Launching), - Evidence::LostStatus, - Teardown::Exited, - ), - ( - "launched", - (Stage::Restore, Progress::Spawned), - Evidence::ForcedKill, - Teardown::Terminated, - ), - ( - "activated", - (Stage::Start, Progress::Ready), - Evidence::ForcedKill, - Teardown::Terminated, - ), - ( - "started", - (Stage::Commit, Progress::Started), - Evidence::ForcedKill, - Teardown::Terminated, - ), - ]; - for (step, at, evidence, teardown) in cases { - let crashed = Crashed::at(step); - let mut locked = crashed.lock(); - let mut control = ReattachControl::answering(Some); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.kind(), ErrorKind::Exited, "{step}: {failure}"); - assert_recorded(&failure, &crashed.lease, &locked, at, evidence, teardown); - assert!(control.capabilities.is_empty(), "{step}"); - // Once the launch has ended, a reattach finds its failure as it was recorded. - let again = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(again.record(), failure.record(), "{step}"); - assert_eq!(again.message(), failure.record().message, "{step}"); - crashed.releases(&mut locked); - } - // A caller that crashed before its launch began left no VM, and no failure. - let crashed = Crashed::at("prepared"); - let mut locked = crashed.lock(); - let mut control = ReattachControl::answering(Some); - match crashed.take_over(&mut locked, &crashed.runtime, &mut control) { - ReattachedClone::Ended { - progress: Progress::Launching, - report: None, - } => {} - other => panic!("the reattach found {}", describe(&other)), - } - crashed.releases(&mut locked); -} - -#[test] -fn a_reattach_records_the_failure_of_an_interrupted_launch_before_it_ends_the_launch() { - let crashed = Crashed::at("launched"); - let mut locked = crashed.lock(); - let mut control = ReattachControl::answering(Some); - // The restarted caller crashes in turn as its reattach would end the interrupted launch. - let crash = reattach_clone_with( - &mut locked, - &crashed.runtime, - &mut control, - Instant::now() + TIMEOUT, - TIMEOUT, - |_, _, _| Err(Error::other("the test's caller crashes")), - ); - assert!(crash.is_err()); - // The sandbox records the failure, whose teardown is pending, beside the launch's journal, - // which keeps the lease. - let records = CloneRecords::read(&locked).unwrap().unwrap(); - let pending = records - .failure - .expect("the reattach recorded the failure before the recovery"); - assert_eq!( - (pending.stage, pending.progress, pending.evidence), - (Stage::Restore, Progress::Spawned, Evidence::LostStatus) - ); - assert_eq!(pending.teardown, Teardown::Pending); - assert!(records.journal.is_some()); - crashed.refuses_release(&mut locked); - // The next reattach ends the launch, and resolves the teardown of the recorded failure. - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.kind(), ErrorKind::Exited, "{failure}"); - assert_recorded( - &failure, - &crashed.lease, - &locked, - (Stage::Restore, Progress::Spawned), - Evidence::LostStatus, - Teardown::Terminated, - ); - assert!(control.capabilities.is_empty()); - crashed.releases(&mut locked); -} - -/// A launch that its caller abandons before the clone's activation leaves a progress log that -/// records the spawn of the clone's VMM, which the launch's journal, which the teardown removes, -/// records no more, so that a reattach reads the outcome report of the clone's VM, and records the -/// failure that the guest's status gives. -#[test] -fn a_launch_abandoned_before_its_activation_leaves_the_spawn_for_a_reattach() { - let fixture = Fixture::new(); - complete_snapshot(&fixture); - let (image, binding) = fixture.image("distro.erofs", IMAGE_LEN); - let mut acquired = fixture.acquire("abandoned"); - let progress = acquired.locked.clone_progress_path(); - let launch = acquired.lease.launch().clone(); - let plan = fixture - .plan(&mut acquired, &tmpfs_request(&image, &binding), NO_CONTENT) - .unwrap(); - let runtime = plan.key().runtime().clone(); - let mut vmm = VmmDir::new(); - // The guest's repair ends the clone's VM with its status, which the VMM reports as it exits. - vmm.plan("report:guest-exit:195"); - let dir = vmm.path().to_owned(); - let made = RefCell::new(None); - let command = fake_vmm(&dir, &made); - let hooks = Hooks { - command: &command, - ..Hooks::LAUNCH - }; - let restoring = plan - .launch_with(&mut acquired, Path::new(OPENVMM), &runtime, hooks) - .unwrap(); - vmm.reference = Some(restoring.starting().process().reference().clone()); - // The log records the spawn once the launch returns. - let logged = read_progress(&progress, &launch).unwrap().unwrap(); - assert_eq!( - (logged.progress, logged.boot), - (Progress::Spawned, Some(BootId::current().unwrap())) - ); - let capability = hex(restoring.capability()); - assert_eq!(vmm.started(), format!("{capability} true")); - vmm.end_and_wait(); - assert_eq!(restoring.abandon().unwrap(), Recovery::Exited); - assert!(acquired.locked.journal().unwrap().is_none()); - let mut control = ReattachControl::answering(Some); - let reattached = reattach_clone( - &mut acquired.locked, - &runtime, - &mut control, - Instant::now() + TIMEOUT, - TIMEOUT, - ); - let failure = failed(reattached.unwrap()); - assert_recorded( - &failure, - &acquired.lease, - &acquired.locked, - (Stage::Restore, Progress::Spawned), - Evidence::RepairStatus, - Teardown::Exited, - ); - assert_eq!(failure.record().status, Some(195)); - assert!(control.capabilities.is_empty()); - assert_eq!( - fixture.store.release(&mut acquired.locked).unwrap(), - Some(acquired.lease.clone()) - ); - assert_released(acquired.lease.sandbox()); -} - -#[test] -fn a_clone_whose_vm_exited_once_it_committed_is_classified_by_its_report() { - let run = (Stage::Run, Progress::Committed); - // A repair's status is evidence of the repair stage, after the clone's Ready too, and a VM - // that ended without a report of the guest's own end lost the clone's status. - let cases = [ - ("report:guest-exit:195", Evidence::RepairStatus, Some(195)), - ("", Evidence::LostStatus, None), - ]; - for (words, evidence, status) in cases { - let crashed = Crashed::at_with("committed", words); - crashed.vmm.end_and_wait(); - let mut locked = crashed.lock(); - let mut control = ReattachControl::answering(Some); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.kind(), ErrorKind::Exited, "{words}: {failure}"); - let exited = Teardown::Exited; - assert_recorded(&failure, &crashed.lease, &locked, run, evidence, exited); - assert_eq!(failure.record().status, status, "{words}"); - assert!(locked.runtime().unwrap().is_none(), "{words}"); - crashed.releases(&mut locked); - } - // A VM that its guest ended with a status that is no evidence of a failure ended. - let crashed = Crashed::at_with("committed", "report:guest-exit:3"); - crashed.vmm.end_and_wait(); - let mut locked = crashed.lock(); - let mut control = ReattachControl::answering(Some); - match crashed.take_over(&mut locked, &crashed.runtime, &mut control) { - ReattachedClone::Ended { - progress: Progress::Committed, - report: Some(report), - } => { - let outcome = (report.category.as_deref(), report.status_code); - assert_eq!(outcome, (Some("guest-exit"), Some(3))); - } - other => panic!("the reattach found {}", describe(&other)), - } - // An inspection of the clone takes the report of its VM as the reattach did, and no failure. - let records = CloneRecords::read(&locked).unwrap().unwrap(); - assert!(records.failure.is_none()); - let report = records.report.expect("the report of the clone's VM"); - let outcome = (report.category.as_deref(), report.status_code); - assert_eq!(outcome, (Some("guest-exit"), Some(3))); - crashed.releases(&mut locked); -} - -#[test] -fn a_reattach_records_the_failure_of_a_vm_that_exited_before_it_forgets_the_vm() { - let run = (Stage::Run, Progress::Committed); - let crashed = Crashed::at_with("committed", "report:guest-exit:195"); - crashed.vmm.end_and_wait(); - let mut locked = crashed.lock(); - let mut control = ReattachControl::answering(Some); - // The restarted caller crashes once its reattach cleared the runtime record of the VM that - // exited, before it recorded the failure once more. - let crash = reattach_clone_with( - &mut locked, - &crashed.runtime, - &mut control, - Instant::now() + TIMEOUT, - TIMEOUT, - |locked, runtime, exited| { - supervisor::reattach_reporting(locked, runtime, exited)?; - Err(Error::other("the test's caller crashes")) - }, - ); - assert!(crash.is_err()); - // The sandbox recorded the failure before the runtime record went, with its teardown pending. - let pending = read_failure(&locked.clone_failure_path()) - .unwrap() - .expect("the reattach recorded the failure before it cleared the runtime record"); - assert_eq!( - (pending.stage, pending.progress, pending.evidence), - (run.0, run.1, Evidence::RepairStatus) - ); - assert_eq!(pending.teardown, Teardown::Pending); - assert_eq!(pending.status, Some(195)); - assert!(locked.runtime().unwrap().is_none()); - // The next reattach finishes it, with the status that it holds, even once no read finds the - // report, as when another process holds it. - fs::write(locked.outcome_path(), b"not a report").unwrap(); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - let (lease, exited) = (&crashed.lease, Teardown::Exited); - assert_recorded( - &failure, - lease, - &locked, - run, - Evidence::RepairStatus, - exited, - ); - assert_eq!(failure.record().status, Some(195)); - assert!(control.capabilities.is_empty()); - crashed.releases(&mut locked); -} - -#[test] -fn a_reattach_takes_the_outcome_report_of_a_vm_that_exited_as_it_first_read_it() { - let run = (Stage::Run, Progress::Committed); - let crashed = Crashed::at_with("committed", "report:guest-exit:195"); - crashed.vmm.end_and_wait(); - let mut locked = crashed.lock(); - let outcome = locked.outcome_path(); - let mut control = ReattachControl::answering(Some); - // Once the reattach read the report of the VM that exited, before it cleared the VM's runtime - // record, no read finds the report, as when another process holds it: the reattach takes the - // report as it first read it. - let reattached = reattach_clone_with( - &mut locked, - &crashed.runtime, - &mut control, - Instant::now() + TIMEOUT, - TIMEOUT, - |locked, runtime, exited| { - supervisor::reattach_reporting(locked, runtime, |record| { - exited(record)?; - fs::write(&outcome, b"not a report").unwrap(); - Ok(()) - }) - }, - ) - .unwrap(); - let failure = failed(reattached); - let (lease, exited) = (&crashed.lease, Teardown::Exited); - assert_recorded( - &failure, - lease, - &locked, - run, - Evidence::RepairStatus, - exited, - ); - assert_eq!(failure.record().status, Some(195)); - crashed.releases(&mut locked); -} - -/// Records the failure of the clone of `lease`, in the sandbox of `locked`, at `at`, with -/// `evidence` and `error`, whose teardown is `teardown`, with the transaction `transaction`, as -/// a failure of the clone's restore sequence records it before its teardown, which a crash of its -/// caller interrupted, if the teardown is pending, or once a teardown that could not end the -/// clone's VMM, if it is unresolved. -fn record_earlier( - lease: &Lease, - locked: &LockedStore, - (at, transaction): ((Stage, Progress), Option), - (evidence, teardown): (Evidence, Teardown), - error: &Error, -) { - let record = FailureRecord::new(lease, transaction, at, evidence, None, teardown, error); - write_failure(&locked.clone_failure_path(), &record).unwrap(); -} - -/// Returns the transaction of the restore that the activation record of the clone in the sandbox -/// of `locked` names. -fn transaction_of(locked: &LockedStore) -> Option { - let activation = fs::read(locked.clone_activation_path()).unwrap(); - let activation = ActivationRecord::decode(&activation).unwrap(); - activation.restore().map(|restore| restore.transaction) -} - -#[test] -fn a_reattach_finishes_an_earlier_failure_with_what_the_clone_s_vmm_left_as_it_ended() { - let restore = (Stage::Restore, Progress::Spawned); - let mut control = ReattachControl::answering(Some); - let unresolved = Teardown::Unresolved("the test could not end the clone's VMM".to_owned()); - for earlier in [Teardown::Pending, unresolved] { - // A failure of the clone's restore sequence, whose teardown a crash of its caller - // interrupted, or could not end the clone's VMM, which the guest then ended with the - // status of a failed repair, which prevails. - let crashed = Crashed::at_with("launched", "report:guest-exit:195"); - let mut locked = crashed.lock(); - let reset = Error::protocol("the test's clone reset its session"); - let at = (restore, transaction_of(&locked)); - let teardown = earlier.clone(); - record_earlier( - &crashed.lease, - &locked, - at, - (Evidence::Reset, teardown), - &reset, - ); - crashed.vmm.end_and_wait(); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!( - failure.kind(), - ErrorKind::Protocol, - "{earlier:?}: {failure}" - ); - let (lease, exited) = (&crashed.lease, Teardown::Exited); - assert_recorded( - &failure, - lease, - &locked, - restore, - Evidence::RepairStatus, - exited, - ); - assert_eq!(failure.record().status, Some(195), "{earlier:?}"); - assert_eq!(failure.message(), "the test's clone reset its session"); - crashed.releases(&mut locked); - // One of the template, such as a defect of the generation that the caller diagnosed, - // which the status does not override: the record holds the status beside it. - let crashed = Crashed::at_with("launched", "report:guest-exit:195"); - let mut locked = crashed.lock(); - let defect = Error::other("the test diagnosed a defect of the generation's repair"); - let at = (restore, transaction_of(&locked)); - let teardown = earlier.clone(); - record_earlier( - &crashed.lease, - &locked, - at, - (Evidence::TemplateDefect, teardown), - &defect, - ); - crashed.vmm.end_and_wait(); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - let (lease, template) = (&crashed.lease, Evidence::TemplateDefect); - assert_recorded( - &failure, - lease, - &locked, - restore, - template, - Teardown::Exited, - ); - assert_eq!( - (failure.outcome(), failure.record().status), - (Outcome::Template, Some(195)), - "{earlier:?}" - ); - crashed.releases(&mut locked); - // One whose VMM exited before its restore-ready event, with OpenVMM's error, which tells - // that this host cannot present the template's CPU, once its record was written. - let crashed = Crashed::at_with("launched", "close fatal:E_CPU_GENERATION"); - let mut locked = crashed.lock(); - let lost = Error::exited("the test's VMM exited before its restore-ready event"); - let at = (restore, transaction_of(&locked)); - let teardown = earlier.clone(); - record_earlier( - &crashed.lease, - &locked, - at, - (Evidence::LostStatus, teardown), - &lost, - ); - crashed.vmm.wait_exit(); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.kind(), ErrorKind::Exited, "{earlier:?}: {failure}"); - let (lease, incompatible) = (&crashed.lease, Evidence::IncompatibleHost); - assert_recorded( - &failure, - lease, - &locked, - restore, - incompatible, - Teardown::Exited, - ); - let message = failure.message(); - assert!( - message.starts_with("the test's VMM exited before its restore-ready event; ") - && message.ends_with("failed to launch vm worker") - && message.matches("[E_CPU_GENERATION]").count() == 1, - "{earlier:?}: {failure}" - ); - // A later reattach finds the failure as it was recorded. - let again = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(again.record(), failure.record(), "{earlier:?}"); - crashed.releases(&mut locked); - } - assert!(control.capabilities.is_empty()); - // The record keeps the transaction of the restore that failed, whatever the sandbox's - // activation record names since. - let crashed = Crashed::at_with("launched", "report:guest-exit:195"); - let mut locked = crashed.lock(); - let reset = Error::protocol("the test's clone reset its session"); - let transaction = Some(TransactionId::from_bytes([0x5a; 16])); - assert_ne!(transaction, transaction_of(&locked)); - let pending = (Evidence::Reset, Teardown::Pending); - record_earlier( - &crashed.lease, - &locked, - (restore, transaction), - pending, - &reset, - ); - crashed.vmm.end_and_wait(); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - let record = failure.record(); - assert_eq!( - (record.transaction, record.evidence, &record.teardown), - (transaction, Evidence::RepairStatus, &Teardown::Exited) - ); - let recorded = read_failure(&locked.clone_failure_path()).unwrap(); - assert_eq!(recorded.as_ref(), Some(record)); - crashed.releases(&mut locked); -} - -#[test] -fn a_clone_whose_progress_log_may_have_lost_its_steps_may_have_run_its_workload() { - // A launch whose caller crashed once the clone was ready, or once it sent the start of the - // clone's sandbox, whose VM the guest ended with a status that the clone's root gives before - // that start, and a workload may give after it. - let (current, rebooted) = (BootId::current().unwrap(), BootId::from_bytes([0xee; 16])); - assert_ne!(current, rebooted); - let ready = (Stage::Start, Progress::Ready); - let spawned = (Stage::Restore, Progress::Spawned); - let cases = [ - // Its intact log tells that the clone was only ready: its root's status is evidence. - ("intact", "activated", ready, Evidence::UnusableRoot), - // A log that was lost, or damaged, tells nothing, and one that the launch wrote during - // another boot of the host may have lost the steps of the start with the host's crash: - // the status is no evidence of the root's failure, and the clone lost its status. - ("lost", "started", spawned, Evidence::LostStatus), - ("damaged", "started", spawned, Evidence::LostStatus), - ("rebooted", "started", ready, Evidence::LostStatus), - ]; - for (damage, step, at, evidence) in cases { - let crashed = Crashed::at_with(step, "report:guest-exit:54"); - let mut locked = crashed.lock(); - let log = locked.clone_progress_path(); - let launch = crashed.lease.launch(); - match damage { - "lost" => fs::remove_file(log).unwrap(), - "damaged" => fs::write(log, b"not a progress log\n").unwrap(), - "rebooted" => fs::write( - log, - format!( - "nvx-clone-progress 1 {launch} {rebooted}\nspawned\nevent\nacknowledged\n\ - activating\nready\n" - ), - ) - .unwrap(), - _ => {} - } - crashed.vmm.end_and_wait(); - let mut control = ReattachControl::answering(Some); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - let record = failure.record(); - assert_eq!( - ( - (record.stage, record.progress), - record.evidence, - record.status, - &record.teardown - ), - (at, evidence, Some(54), &Teardown::Exited), - "{damage}: {failure}" - ); - let recorded = read_failure(&locked.clone_failure_path()).unwrap(); - assert_eq!(recorded.as_ref(), Some(record), "{damage}"); - crashed.releases(&mut locked); - } -} - -#[test] -fn a_clone_that_its_holder_stopped_ended_whatever_a_reboot_left_of_its_progress_log() { - let other = BootId::from_bytes([0xee; 16]); - assert_ne!(other, BootId::current().unwrap()); - for rebooted in [false, true] { - let crashed = Crashed::at_with("committed", "report:guest-exit:54"); - let mut locked = crashed.lock(); - let mut control = ReattachControl::answering(Some); - let vmm = match crashed.take_over(&mut locked, &crashed.runtime, &mut control) { - ReattachedClone::Live { vmm, .. } => vmm, - other => panic!("the reattach found {}", describe(&other)), - }; - // Its holder stops the clone, whose workload ends with a status that the clone's root - // gives before the start of its sandbox, which clears the store's records of its VM. - let stopped = vmm - .stop(&mut locked, Instant::now() + TIMEOUT, None, |_, _| { - crashed.vmm.end(); - Ok::<(), io::Error>(()) - }) - .unwrap(); - assert_eq!(stopped.outcome, StopOutcome::Graceful, "{rebooted}"); - assert!(locked.runtime().unwrap().is_none(), "{rebooted}"); - if rebooted { - // A crash of the host, before the release, lost the last lines of the progress log, - // which the launch wrote during the boot that the crash ended. - let launch = crashed.lease.launch(); - fs::write( - locked.clone_progress_path(), - format!( - "nvx-clone-progress 1 {launch} {other}\nspawned\nevent\nacknowledged\n\ - activating\nready\n" - ), - ) - .unwrap(); - } - // The clone ended, and its workload's status is no failure of its root. - let progress = if rebooted { - Progress::Ready - } else { - Progress::Committed - }; - match crashed.take_over(&mut locked, &crashed.runtime, &mut control) { - ReattachedClone::Ended { - progress: ended, - report: Some(report), - } if ended == progress => { - let outcome = (report.category.as_deref(), report.status_code); - assert_eq!(outcome, (Some("guest-exit"), Some(54)), "{rebooted}"); - } - other => panic!("{rebooted}: the reattach found {}", describe(&other)), - } - let recorded = read_failure(&locked.clone_failure_path()).unwrap(); - assert!(recorded.is_none(), "{rebooted}: {recorded:?}"); - crashed.releases(&mut locked); - } -} - -#[test] -fn a_clone_whose_vmm_exited_before_its_holder_s_stop_began_lost_its_status() { - let never = |_: &Vmm, _: Instant| -> io::Result<()> { - panic!("a stop ran the graceful step of a VMM that had exited") - }; - // Its holder found the clone's VMM running, which then exits, without a report of the - // guest's own end, before the holder's stop begins. - let crashed = Crashed::at("committed"); - let mut locked = crashed.lock(); - let mut control = ReattachControl::answering(Some); - let vmm = match crashed.take_over(&mut locked, &crashed.runtime, &mut control) { - ReattachedClone::Live { vmm, .. } => vmm, - other => panic!("the reattach found {}", describe(&other)), - }; - crashed.vmm.end_and_wait(); - let stopped = vmm - .stop_unless_exited(&mut locked, Instant::now() + TIMEOUT, None, never) - .unwrap(); - assert!(stopped.is_none(), "{stopped:?}"); - // The stop kept the runtime record, and with it the lease, so the reattach finds that the VM - // exited unheld: the clone lost its status, which quarantines nothing. - assert_eq!(locked.runtime().unwrap().as_ref(), Some(vmm.record())); - crashed.refuses_release(&mut locked); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - assert_eq!(failure.kind(), ErrorKind::Exited, "{failure}"); - let (run, lost) = ((Stage::Run, Progress::Committed), Evidence::LostStatus); - assert_recorded( - &failure, - &crashed.lease, - &locked, - run, - lost, - Teardown::Exited, - ); - assert_eq!(failure.record().status, None); - let deadline = Some(Instant::now() + TIMEOUT); - assert!( - !failure - .quarantine(&crashed.fixture.store, deadline) - .unwrap() - ); - assert!(locked.runtime().unwrap().is_none()); - crashed.releases(&mut locked); - - // A stop that ends such a VM as one that it stopped clears the record, and the clone ended. - let crashed = Crashed::at("committed"); - let mut locked = crashed.lock(); - let vmm = match crashed.take_over(&mut locked, &crashed.runtime, &mut control) { - ReattachedClone::Live { vmm, .. } => vmm, - other => panic!("the reattach found {}", describe(&other)), - }; - crashed.vmm.end_and_wait(); - let stopped = vmm - .stop(&mut locked, Instant::now() + TIMEOUT, None, never) - .unwrap(); - assert_eq!(stopped.outcome, StopOutcome::Graceful); - assert!(locked.runtime().unwrap().is_none()); - match crashed.take_over(&mut locked, &crashed.runtime, &mut control) { - ReattachedClone::Ended { - progress: Progress::Committed, - report: None, - } => {} - other => panic!("the reattach found {}", describe(&other)), - } - let recorded = read_failure(&locked.clone_failure_path()).unwrap(); - assert!(recorded.is_none(), "{recorded:?}"); - crashed.releases(&mut locked); -} - -#[test] -fn an_interrupted_launch_that_a_reattach_cannot_end_yet_keeps_its_lease_until_one_can() { - let crashed = Crashed::at("activated"); - let mut locked = crashed.lock(); - let mut control = ReattachControl::answering(Some); - // The reattach cannot tell whether the launch's VMM still runs. - let reason = "the test cannot end the launch yet"; - let reattached = reattach_clone_with( - &mut locked, - &crashed.runtime, - &mut control, - Instant::now() + TIMEOUT, - TIMEOUT, - |_, _, _| Ok(Reattached::Indeterminate(reason.to_owned())), - ) - .unwrap(); - let failure = failed(reattached); - let ready = (Stage::Start, Progress::Ready); - let unresolved = Teardown::Unresolved(reason.to_owned()); - let lost = Evidence::LostStatus; - assert_recorded(&failure, &crashed.lease, &locked, ready, lost, unresolved); - assert!(failure.message().ends_with(reason), "{failure}"); - // The journal remains, and with it the lease, and the clone's records. - assert!(locked.journal().unwrap().is_some()); - crashed.refuses_release(&mut locked); - // A reattach that can end the launch resolves the teardown, and keeps the failure as it was. - let resolved = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - let terminated = Teardown::Terminated; - assert_recorded(&resolved, &crashed.lease, &locked, ready, lost, terminated); - assert_eq!(resolved.message(), failure.record().message); - assert!(control.capabilities.is_empty()); - crashed.releases(&mut locked); -} - -/// Returns the error of `reattached`, which must have failed. -fn reattach_error(reattached: crate::Result>) -> Error { - match reattached { - Err(error) => error, - Ok(other) => panic!("the reattach found {}", describe(&other)), - } -} - -/// Takes the clone of the sandbox of `locked` over, under `runtime`, through `control`, with -/// [`reattach_clone`], while a directory takes the place of the record of the clone's failure, as -/// on a storage that refuses to write it, once the reattach read the sandbox's records, and -/// returns the error of the reattach. -fn unrecorded_take_over( - locked: &mut LockedStore, - runtime: &RuntimeIdentity, - control: &mut ReattachControl, -) -> Error { - let path = locked.clone_failure_path(); - let reattached = reattach_clone_with( - locked, - runtime, - control, - Instant::now() + TIMEOUT, - TIMEOUT, - |locked, runtime, exited| { - fs::create_dir(&path).unwrap(); - supervisor::reattach_reporting(locked, runtime, exited) - }, - ); - fs::remove_dir(&path).unwrap(); - reattach_error(reattached) -} - -#[test] -fn a_reattach_that_cannot_record_a_failure_ends_and_forgets_nothing_of_the_clone() { - let unrecorded = |error: &Error| { - let left = "; the reattach left the clone's VM and its records as it found them, for a \ - later reattach"; - let message = error.message(); - assert_eq!(error.kind(), ErrorKind::Io, "{error}"); - let start = "the failure could not be recorded before its teardown: "; - assert!( - message.starts_with(start) && message.ends_with(left), - "{error}" - ); - }; - let mut control = ReattachControl::answering(Some); - // A record of a failure that cannot be read, which the reattach would replace with its own - // before it ends the interrupted launch that the journal names. - let crashed = Crashed::at("activated"); - let mut locked = crashed.lock(); - let path = locked.clone_failure_path(); - fs::create_dir(&path).unwrap(); - let deadline = Instant::now() + TIMEOUT; - let runtime = &crashed.runtime; - let reattached = reattach_clone(&mut locked, runtime, &mut control, deadline, TIMEOUT); - unrecorded(&reattach_error(reattached)); - let journal = locked - .journal() - .unwrap() - .expect("the reattach kept the journal"); - assert!(matches!( - Process::open(journal.process().unwrap()).unwrap(), - Opened::Running(_) - )); - crashed.refuses_release(&mut locked); - // Once the record can be written, the reattach ends the launch, and records its failure. - fs::remove_dir(&path).unwrap(); - let failure = failed(crashed.take_over(&mut locked, runtime, &mut control)); - let (ready, killed) = ((Stage::Start, Progress::Ready), Evidence::ForcedKill); - let terminated = Teardown::Terminated; - assert_recorded(&failure, &crashed.lease, &locked, ready, killed, terminated); - crashed.releases(&mut locked); - // A VM whose launch committed, and that exited without a report of the guest's own end: the - // reattach would record that the clone lost its status before it clears the VM's record. - let crashed = Crashed::at("committed"); - crashed.vmm.end_and_wait(); - let mut locked = crashed.lock(); - let record = locked.runtime().unwrap(); - assert!(record.is_some()); - unrecorded(&unrecorded_take_over( - &mut locked, - &crashed.runtime, - &mut control, - )); - assert_eq!(locked.runtime().unwrap(), record); - crashed.refuses_release(&mut locked); - let failure = failed(crashed.take_over(&mut locked, &crashed.runtime, &mut control)); - let (run, lost) = ((Stage::Run, Progress::Committed), Evidence::LostStatus); - assert_recorded( - &failure, - &crashed.lease, - &locked, - run, - lost, - Teardown::Exited, - ); - crashed.releases(&mut locked); - // A VM that runs under another runtime identity, and one whose Ready reports another - // activation than its record's: the reattach would end either once it recorded why. - let other = RuntimeIdentity::new(artifact('d'), artifact('e'), artifact('f')); - let mut refusing = ReattachControl::answering(|_| Some(ReadyActivation { transaction: None })); - let reattach = (Stage::Reattach, Progress::Committed); - let cases = [ - (Some(&other), Evidence::RuntimeMismatch), - (None, Evidence::Protocol), - ]; - for (runtime, evidence) in cases { - let crashed = Crashed::at("committed"); - let mut locked = crashed.lock(); - let (runtime, control) = match runtime { - Some(other) => (other, &mut control), - None => (&crashed.runtime, &mut refusing), - }; - unrecorded(&unrecorded_take_over(&mut locked, runtime, control)); - let record = locked - .runtime() - .unwrap() - .expect("the reattach kept the record"); - assert!( - matches!(Process::open(record.process()).unwrap(), Opened::Running(_)), - "{evidence:?}" - ); - crashed.refuses_release(&mut locked); - let failure = failed(crashed.take_over(&mut locked, runtime, control)); - let terminated = Teardown::Terminated; - assert_recorded( - &failure, - &crashed.lease, - &locked, - reattach, - evidence, - terminated, - ); - crashed.releases(&mut locked); - } -} - -#[test] -fn a_reattach_makes_the_record_of_an_earlier_failure_durable_before_it_ends_the_launch() { - // A failure of the clone's launch, whose record is in place, but whose teardown a crash of - // its caller interrupted, maybe once the sync of the record's directory failed. - let crashed = Crashed::at("activated"); - let mut locked = crashed.lock(); - let ready = (Stage::Start, Progress::Ready); - let error = Error::other("the test's launch failed"); - let at = (ready, transaction_of(&locked)); - let earlier = (Evidence::Protocol, Teardown::Pending); - record_earlier(&crashed.lease, &locked, at, earlier, &error); - let mut control = ReattachControl::answering(Some); - let reattach = |locked: &mut LockedStore, - runtime: &RuntimeIdentity, - exited: &dyn Fn(&RuntimeRecord) -> Result<()>| { - supervisor::reattach_reporting(locked, runtime, exited) - }; - // A reattach that cannot make the record durable ends nothing. - let refused = |_: &Path| -> Result<()> { Err(Error::io("the test's sync failed")) }; - let deadline = Instant::now() + TIMEOUT; - let runtime = &crashed.runtime; - let reattached = reattach_clone_syncing( - &mut locked, - runtime, - &mut control, - deadline, - TIMEOUT, - &refused, - reattach, - ); - let error = reattach_error(reattached); - assert_eq!(error.kind(), ErrorKind::Io, "{error}"); - let refusal = "could not be made durable before its teardown: the test's sync failed"; - assert!(error.message().contains(refusal), "{error}"); - let journal = locked - .journal() - .unwrap() - .expect("the reattach kept the journal"); - assert!(matches!( - Process::open(journal.process().unwrap()).unwrap(), - Opened::Running(_) - )); - crashed.refuses_release(&mut locked); - // One that can syncs the sandbox's directory before it ends the launch. - let synced = RefCell::new(Vec::new()); - let sync = |dir: &Path| -> Result<()> { - synced.borrow_mut().push(dir.to_owned()); - sync_dir(dir) - }; - let dir = locked.dir().to_owned(); - let reattached = reattach_clone_syncing( - &mut locked, - runtime, - &mut control, - Instant::now() + TIMEOUT, - TIMEOUT, - &sync, - |locked, runtime, exited| { - assert_eq!(*synced.borrow(), std::slice::from_ref(&dir)); - reattach(locked, runtime, exited) - }, - ); - let failure = failed(reattached.unwrap()); - let (protocol, terminated) = (Evidence::Protocol, Teardown::Terminated); - assert_recorded( - &failure, - &crashed.lease, - &locked, - ready, - protocol, - terminated, - ); - crashed.releases(&mut locked); -} diff --git a/nvx-rs/src/host/template/clone/tests/restore.rs b/nvx-rs/src/host/template/clone/tests/restore.rs deleted file mode 100644 index 54d8f45eb..000000000 --- a/nvx-rs/src/host/template/clone/tests/restore.rs +++ /dev/null @@ -1,1661 +0,0 @@ -//! Tests of the restore sequence of clones, [`Restoring::activate`], [`Activated::start`], and -//! [`Started::commit`], with a [`restore_vmm_role`](super::launch) as the clone's VMM, which -//! sends its restore-ready event as its plan says, and a [`FakeControl`] in place of the adapter's -//! framing: the wait for the event, and the connection only once it came; the Activate of the -//! sandbox's record, byte for byte, as the first request of each session; the check of the -//! activation that Ready reports; the mapping table of a clone with an export, and the end of the -//! launch at a partial or failed table; the start; the sessions that replace those that the -//! adapter lost, after a reset of the session, say, and the clone that ended once it took that -//! reset before it was ready; and the deadlines. The crashes of the sequence's caller are among -//! the launch's tests. - -use std::cell::RefCell; -use std::collections::VecDeque; -use std::thread; -use std::time::{Duration, Instant}; - -use super::launch::{ - OPENVMM, TIMEOUT, VmmDir, assert_recorded, complete_snapshot, failed, fake_vmm, runtime_of, -}; -use super::*; -use crate::activation::{ReadyActivation, RestoreSection, TransactionId}; -use crate::host::platform::Transport; -use crate::host::process::{BootId, Opened, Process}; -use crate::host::store::{CLONE_ACTIVATION_NAME, Recovery}; -use crate::host::supervisor::{Starting, StopOutcome}; -use crate::host::template::clone::launch::Hooks; -use crate::host::template::clone::records::{End, read_failure}; -use crate::protocol::{ - APP_HEADER_LEN, OUTER_MAX_PAYLOAD, decode_maps_payload, encode_maps_payloads, -}; - -/// How long a VMM whose plan delays its event waits before it sends it. -const DELAY: Duration = Duration::from_millis(300); - -/// A short deadline, which a test expects to pass. -const SHORT: Duration = Duration::from_secs(2); - -/// The restore-to-Ready deadline of a clone that never answers its Activate, which leaves its VMM -/// time to start on a busy host. -const SILENT: Duration = Duration::from_secs(10); - -/// A call that the restore sequence made to a [`FakeControl`]. -#[derive(Clone, Debug, PartialEq, Eq)] -enum Call { - /// An attachment of a session, numbered from 1, with the capability, when the call came, by - /// the deadline. - Attach { - session: usize, - capability: [u8; CAPABILITY_LEN], - at: Instant, - deadline: Instant, - }, - /// An Activate of the record in a session, by the deadline. - Activate { - session: usize, - record: Vec, - deadline: Instant, - }, - /// A mapping table in a session, by the deadline, as the guest decodes the `MAPS` payloads in - /// which the framing encodes it. - Map { - session: usize, - table: Vec, - deadline: Instant, - }, - /// A start in a session, by the deadline. - Start { session: usize, deadline: Instant }, -} - -/// How a [`FakeControl`] answers an Activate or a start. -enum Answer { - /// With the clone's Ready, which reports the activation of the record that the Activate - /// carries, or with the start's confirmation. - Done, - /// With a Ready that reports this activation, or none. - Report(Option), - /// With this failure. - Fail(Failure), - /// With no answer until the deadline, then a failure that keeps the session. - Silent, -} - -/// A failure of a [`FakeControl`]: one that lost its session, the guest's refusal of a mapping -/// table, or another. -#[derive(Debug)] -struct Failure { - /// Whether the failure lost the session. - lost: bool, - /// Whether the failure is the guest's refusal of a mapping table, which the framing tells. - table: bool, -} - -impl Failure { - /// The loss of the session, as its reset or the end of its stream are. - const LOST: Self = Self { - lost: true, - table: false, - }; - /// Another failure, which keeps the session, as the guest's refusal does. - const REFUSED: Self = Self { - lost: false, - table: false, - }; - /// The guest's refusal of a mapping table, which keeps the session, and which the framing - /// tells, as [`CloneControl::refused`] does. - const TABLE: Self = Self { - lost: false, - table: true, - }; -} - -impl fmt::Display for Failure { - fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { - formatter.write_str(if self.lost { - "the fake framing lost its session" - } else if self.table { - "the fake guest refused its mapping table" - } else { - "the fake guest refused" - }) - } -} - -impl std::error::Error for Failure {} - -/// A session of a [`FakeControl`], over a connection to the clone's control endpoint. -struct FakeSession { - /// The session's number, from 1. - number: usize, - /// The connection, which the session holds while it lasts, as a real one does. - _transport: Box, -} - -/// The adapter's framing of the tests: it attaches a session to each connection that the sequence -/// makes, numbered from 1, answers each Activate as the next of its `activates` says, each mapping -/// table, which it encodes in `MAPS` payloads that its guest decodes, as the next of its `maps` -/// says, and each start as the next of its `starts` says, as [`Answer::Done`] does once they run -/// out, and records each call. Its `on_activate` runs at each Activate, with the session's number, -/// before the answer. -#[derive(Default)] -struct FakeControl<'a> { - /// The calls, in order. - calls: Vec, - /// The sessions that it attached. - sessions: usize, - /// How it answers the next Activates. - activates: VecDeque, - /// How its guest answers the next mapping tables: how many of a table's mappings it applies, - /// in order, at most all, and its answer. - maps: VecDeque<(usize, Answer)>, - /// The mappings that its guest applied, in order. - applied: Vec, - /// How it answers the next starts. - starts: VecDeque, - /// What runs at each Activate. - on_activate: Option>, -} - -impl<'a> FakeControl<'a> { - /// Returns a framing that answers the Activates as `activates` say. - fn activating(activates: impl IntoIterator) -> Self { - Self { - activates: activates.into_iter().collect(), - ..Self::default() - } - } - - /// Returns the records of the Activates, in order. - fn records(&self) -> Vec<&[u8]> { - self.calls - .iter() - .filter_map(|call| match call { - Call::Activate { record, .. } => Some(record.as_slice()), - _ => None, - }) - .collect() - } - - /// Returns the deadlines of the attachments and Activates, in order. - fn ready_deadlines(&self) -> Vec { - self.calls - .iter() - .filter_map(|call| match call { - Call::Attach { deadline, .. } | Call::Activate { deadline, .. } => Some(*deadline), - Call::Map { .. } | Call::Start { .. } => None, - }) - .collect() - } - - /// Checks that each session of the framing attached, then sent an Activate first, in its - /// order, and that the mapping tables and the starts came last, in the last session. - fn assert_activates_first(&self) { - let mut session = 0; - let mut pending = false; - for call in &self.calls { - match call { - Call::Attach { - session: number, .. - } => { - assert!(!pending, "{:?}", self.calls); - session += 1; - assert_eq!(*number, session, "{:?}", self.calls); - pending = true; - } - Call::Activate { - session: number, .. - } => { - assert!(pending && *number == session, "{:?}", self.calls); - pending = false; - } - Call::Map { - session: number, .. - } - | Call::Start { - session: number, .. - } => { - assert!(!pending && *number == session, "{:?}", self.calls); - } - } - } - } -} - -/// Answers `answer` by `deadline`, with the Ready `ready` for [`Answer::Done`]. -fn answered(answer: Answer, ready: T, deadline: Instant) -> Result -where - T: From>, -{ - match answer { - Answer::Done => Ok(ready), - Answer::Report(reported) => Ok(T::from(reported)), - Answer::Fail(failure) => Err(failure), - Answer::Silent => { - thread::sleep(deadline.saturating_duration_since(Instant::now())); - Err(Failure::REFUSED) - } - } -} - -impl CloneControl for FakeControl<'_> { - type Session = FakeSession; - type Error = Failure; - - fn attach( - &mut self, - transport: Box, - capability: &[u8; CAPABILITY_LEN], - deadline: Instant, - ) -> Result { - self.sessions += 1; - self.calls.push(Call::Attach { - session: self.sessions, - capability: *capability, - at: Instant::now(), - deadline, - }); - Ok(FakeSession { - number: self.sessions, - _transport: transport, - }) - } - - fn activate( - &mut self, - session: &mut FakeSession, - record: &[u8], - deadline: Instant, - ) -> Result, Failure> { - self.calls.push(Call::Activate { - session: session.number, - record: record.to_vec(), - deadline, - }); - if let Some(on_activate) = &mut self.on_activate { - on_activate(session.number); - } - let ready = ReadyActivation::of(&ActivationRecord::decode(record).unwrap()); - let answer = self.activates.pop_front().unwrap_or(Answer::Done); - answered(answer, Some(ready), deadline) - } - - fn map( - &mut self, - session: &mut FakeSession, - table: &[HostMap], - deadline: Instant, - ) -> Result<(), Failure> { - // The framing sends the table in `MAPS` payloads, which the guest decodes. - let parts = encode_maps_payloads(table, OUTER_MAX_PAYLOAD - APP_HEADER_LEN) - .expect("the framing encodes the clone's table"); - let decoded: Vec = parts - .iter() - .flat_map(|part| decode_maps_payload(part).unwrap().maps) - .collect(); - let (applied, answer) = self - .maps - .pop_front() - .unwrap_or((decoded.len(), Answer::Done)); - self.applied - .extend_from_slice(&decoded[..applied.min(decoded.len())]); - self.calls.push(Call::Map { - session: session.number, - table: decoded, - deadline, - }); - answered(answer, Confirmed, deadline).map(|_| ()) - } - - fn start(&mut self, session: &mut FakeSession, deadline: Instant) -> Result<(), Failure> { - self.calls.push(Call::Start { - session: session.number, - deadline, - }); - let answer = self.starts.pop_front().unwrap_or(Answer::Done); - answered(answer, Confirmed, deadline).map(|_| ()) - } - - fn lost(&self, error: &Failure) -> bool { - error.lost - } - - fn refused(&self, error: &Failure) -> bool { - error.table - } -} - -/// The confirmation of a start, which [`answered`] returns. -struct Confirmed; - -impl From> for Confirmed { - fn from(_: Option) -> Self { - Self - } -} - -/// The instants around the launch of a clone, at which its restore-to-Ready deadline started. -struct Launched { - /// Before the launch. - before: Instant, - /// Once the launch returned. - after: Instant, -} - -impl Launched { - /// Checks that `deadline` is `timeout` after the launch. - fn assert_deadline(&self, deadline: Instant, timeout: Duration) { - assert!( - self.before + timeout <= deadline && deadline <= self.after + timeout, - "{deadline:?} is not {timeout:?} after the launch" - ); - } -} - -/// Launches the clone of `acquired`, which leases the generation of `fixture`, of `image` in RAM, -/// with a [`restore_vmm_role`](super::launch) in `vmm` as its VMM. Returns the launch, and when it -/// happened. -fn launch<'a>( - fixture: &Fixture, - acquired: &'a mut Acquired, - image: &(PathBuf, DistroBinding), - vmm: &mut VmmDir, -) -> (Restoring<'a>, Launched) { - launch_ending(fixture, acquired, image, vmm, Hooks::LAUNCH.end) -} - -/// Launches the clone as [`launch`] does, whose failures end the launch with `end`. -fn launch_ending<'a>( - fixture: &Fixture, - acquired: &'a mut Acquired, - image: &(PathBuf, DistroBinding), - vmm: &mut VmmDir, - end: End, -) -> (Restoring<'a>, Launched) { - let request = tmpfs_request(&image.0, &image.1); - launch_request(fixture, acquired, &request, vmm, end) -} - -/// Launches the clone of `acquired` as [`launch`] does, as `request` asks, whose failures end the -/// launch with `end`. -fn launch_request<'a>( - fixture: &Fixture, - acquired: &'a mut Acquired, - request: &CloneRequest, - vmm: &mut VmmDir, - end: End, -) -> (Restoring<'a>, Launched) { - let plan = fixture.plan(acquired, request, NO_CONTENT).unwrap(); - let runtime = plan.key().runtime().clone(); - let dir = vmm.path().to_owned(); - let made = RefCell::new(None); - let command = fake_vmm(&dir, &made); - let hooks = Hooks { - command: &command, - end, - ..Hooks::LAUNCH - }; - let before = Instant::now(); - let restoring = plan - .launch_with(acquired, Path::new(OPENVMM), &runtime, hooks) - .unwrap(); - let after = Instant::now(); - vmm.reference = Some(restoring.starting().process().reference().clone()); - (restoring, Launched { before, after }) -} - -/// A fresh fixture whose generation's snapshot is complete, and an image of its clones. -fn fixture() -> (Fixture, (PathBuf, DistroBinding)) { - let fixture = Fixture::new(); - complete_snapshot(&fixture); - let image = fixture.image("distro.erofs", IMAGE_LEN); - (fixture, image) -} - -/// Returns a VMM's working directory, whose VMM sends its event as `plan` says. -fn vmm_with(plan: &str) -> VmmDir { - let vmm = VmmDir::new(); - vmm.plan(plan); - vmm -} - -/// Checks that the VMM of `vmm` has exited. -fn assert_vmm_ended(vmm: &VmmDir) { - let reference = vmm.reference.as_ref().unwrap(); - if let Opened::Running(process) = Process::open(reference).unwrap() { - assert!(process.wait(Instant::now() + TIMEOUT).unwrap().is_some()); - } -} - -/// Checks that the restore sequence of the clone of `acquired` ended its launch, after its VMM in -/// `vmm`, and left the lease, which a release then ends. -fn assert_launch_ended(fixture: &Fixture, acquired: &mut Acquired, vmm: &VmmDir) { - assert_vmm_ended(vmm); - let locked = &mut acquired.locked; - assert!(locked.journal().unwrap().is_none()); - assert!(locked.runtime().unwrap().is_none()); - assert_eq!(Lease::of(locked).unwrap(), Some(acquired.lease.clone())); - assert_eq!( - fixture.store.release(locked).unwrap(), - Some(acquired.lease.clone()) - ); -} - -/// Checks that `failure`, of the clone of `acquired`, failed at `stage`, once the clone got as far -/// as `progress`, with `evidence`, and the teardown `teardown`, as [`assert_recorded`] does. -fn assert_failed( - failure: &CloneFailure, - acquired: &Acquired, - at: (Stage, Progress), - evidence: Evidence, - teardown: Teardown, -) { - assert_recorded( - failure, - &acquired.lease, - &acquired.locked, - at, - evidence, - teardown, - ); -} - -#[test] -fn a_clone_is_activated_once_its_restore_is_ready_then_started_and_committed() { - let (fixture, image) = fixture(); - let mut acquired = fixture.acquire("sandbox"); - let record = acquired.locked.clone_activation_path(); - // The VMM sends its event late, and flushes it as OpenVMM does, which waits on Windows until - // the host read it. - let mut vmm = vmm_with(&format!("delay:{} flush", DELAY.as_millis())); - let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); - let activation = restoring.activation().clone(); - let capability = *restoring.capability(); - let endpoint = restoring.starting().endpoint().to_owned(); - let mut control = FakeControl::default(); - let deadlines = RestoreDeadlines::default(); - assert_eq!(deadlines.ready, RESTORE_TO_READY_TIMEOUT); - assert_eq!(deadlines.start, READY_TO_START_TIMEOUT); - let activating = Instant::now(); - let mut activated = restoring.activate(&mut control, deadlines).unwrap(); - let ready = Instant::now(); - // The host read the event, and only then connected, and attached one session with the - // launch's capability, whose first request was the Activate of the sandbox's record, byte for - // byte the launch's, by the restore-to-Ready deadline, which started with the launch. - assert!(vmm.started().ends_with(" true")); - assert!(vmm.flushed()); - control.assert_activates_first(); - let [ - Call::Attach { - session: 1, - capability: attached, - at, - deadline, - }, - Call::Activate { - session: 1, - record: sent, - deadline: activated_by, - }, - ] = control.calls.as_slice() - else { - panic!("unexpected calls: {:?}", control.calls); - }; - assert_eq!(attached, &capability); - assert!( - *at >= launched.before + DELAY, - "the host connected before the event" - ); - launched.assert_deadline(*deadline, RESTORE_TO_READY_TIMEOUT); - assert_eq!(activated_by, deadline); - assert_eq!(sent, &activation.encode()); - assert_eq!(&fs::read(&record).unwrap(), sent); - // The Ready-to-Start deadline started at the check of the clone's Ready. - let start = activated.deadline(); - assert!( - activating + READY_TO_START_TIMEOUT <= start && start <= ready + READY_TO_START_TIMEOUT - ); - assert_eq!(activated.activation(), &activation); - assert_eq!(activated.starting().endpoint(), endpoint); - assert_eq!(activated.session().number, 1); - let shown = format!("{activated:?}"); - assert!(shown.starts_with("Activated {"), "{shown}"); - assert!(!shown.contains("capability"), "{shown}"); - // The sandbox starts in the same session, by that deadline, and the launch commits. - let started = activated.start(&mut control).unwrap(); - assert_eq!( - control.calls.last(), - Some(&Call::Start { - session: 1, - deadline: start - }) - ); - let (running, session) = started.commit().unwrap(); - assert_eq!(session.number, 1); - // The progress log records each step, the spawn and the acknowledgment with the steps that - // follow them at once, after a first line that names the launch and the current boot of the - // host, and the sandbox's records show the committed clone. - let log = fs::read_to_string(acquired.locked.clone_progress_path()).unwrap(); - assert_eq!( - log, - format!( - "nvx-clone-progress 1 {} {}\nspawned\nevent\nacknowledged\nactivating\nready\n\ - starting\nstarted\ncommitted\n", - acquired.lease.launch(), - BootId::current().unwrap() - ) - ); - let records = CloneRecords::read(&acquired.locked).unwrap().unwrap(); - assert_eq!( - (records.progress, records.failure, records.journal), - (Progress::Committed, None, None) - ); - assert_eq!(records.lease, acquired.lease); - assert_eq!(records.runtime, acquired.locked.runtime().unwrap()); - // The runtime record names the VM, whose lease lasts as long as it runs. - let locked = &mut acquired.locked; - let recorded = locked.runtime().unwrap().unwrap(); - assert_eq!(recorded.endpoint(), endpoint); - assert_eq!(running.endpoint(), endpoint); - assert!(locked.journal().unwrap().is_none()); - let error = fixture.store.release(locked).unwrap_err(); - assert_eq!(error.code(), ErrorCode::SandboxRunning, "{error}"); - assert_eq!(Lease::of(locked).unwrap(), Some(acquired.lease.clone())); - assert_eq!( - running.terminate(locked, TIMEOUT).unwrap(), - StopOutcome::Forced - ); - assert_eq!( - fixture.store.release(locked).unwrap(), - Some(acquired.lease.clone()) - ); -} - -/// The Ready that a clone reports, given another transaction than the record's. -type Reported = fn(TransactionId) -> Option; - -#[test] -fn a_ready_that_does_not_report_the_record_s_activation_ends_the_launch() { - let (fixture, image) = fixture(); - let other_transaction = TransactionId::from_bytes([0x6b; TransactionId::LEN]); - let cases: [(&str, Reported); 3] = [ - // Another transaction. - ("transaction", |transaction| { - Some(ReadyActivation { - transaction: Some(transaction), - }) - }), - // A cold boot's activation, without a transaction. - ("cold", |_| Some(ReadyActivation { transaction: None })), - // No activation. - ("none", |_| None), - ]; - for (name, reported) in cases { - let mut acquired = fixture.acquire(name); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let reported = reported(other_transaction); - let mut control = FakeControl::activating([Answer::Report(reported)]); - let error = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Protocol, "{name}: {error}"); - assert!( - error.message().starts_with("the clone's Ready reports"), - "{error}" - ); - // One session, which the host ended, and no start. - assert_eq!(control.calls.len(), 2, "{name}: {:?}", control.calls); - let ready = (Stage::Activation, Progress::Activating); - let terminated = Teardown::Terminated; - assert_failed(&error, &acquired, ready, Evidence::Protocol, terminated); - assert_launch_ended(&fixture, &mut acquired, &vmm); - } -} - -#[test] -fn a_session_that_the_adapter_lost_is_replaced_by_one_that_resends_the_same_record_first() { - let (fixture, image) = fixture(); - let mut acquired = fixture.acquire("sandbox"); - let record = acquired.locked.clone_activation_path(); - let mut vmm = vmm_with("flush"); - let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); - // The adapter loses its first session as it awaits the clone's Ready, as a reset of the - // session does: the clone, which was ready, answers the Activate of a new session. - let mut control = FakeControl::activating([Answer::Fail(Failure::LOST), Answer::Done]); - let activated = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap(); - control.assert_activates_first(); - assert_eq!(control.sessions, 2); - // Both Activates carry the sandbox's record, byte for byte, by the same deadline, which the - // retry did not extend. - let recorded = fs::read(&record).unwrap(); - assert_eq!(control.records(), [&recorded[..], &recorded[..]]); - let deadlines = control.ready_deadlines(); - assert!(deadlines.iter().all(|deadline| *deadline == deadlines[0])); - launched.assert_deadline(deadlines[0], RESTORE_TO_READY_TIMEOUT); - // The sandbox starts in the new session. - let started = activated.start(&mut control).unwrap(); - assert!(matches!( - control.calls.last(), - Some(Call::Start { session: 2, .. }) - )); - assert_eq!( - started.abandon().unwrap(), - Recovery::Terminated(vmm.reference.clone().unwrap()) - ); - assert_launch_ended(&fixture, &mut acquired, &vmm); -} - -#[test] -fn a_retry_never_sends_another_record_nor_takes_another_activation() { - let (fixture, image) = fixture(); - - // The sandbox's record changed once the first session was lost: the retry sends nothing, and - // connects no more. - let mut acquired = fixture.acquire("changed"); - let record = acquired.locked.clone_activation_path(); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let original = restoring.activation().clone(); - let restore = original.restore().unwrap(); - let other = ActivationRecord::new( - Some(RestoreSection { - transaction: TransactionId::from_bytes([0x77; TransactionId::LEN]), - }), - original.configuration().clone(), - ) - .unwrap(); - let path = record.clone(); - let mut control = FakeControl::activating([Answer::Fail(Failure::LOST)]); - control.on_activate = Some(Box::new(move |_| { - fs::write(&path, other.encode()).unwrap(); - })); - let error = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Io, "{error}"); - assert!(error.message().contains("no longer holds"), "{error}"); - assert_eq!(control.sessions, 1); - assert_eq!(control.records(), [&original.encode()[..]]); - // The host's own record changed: the record of the failure names the launch's transaction. - let record = error.record(); - assert_eq!( - (record.evidence, record.outcome, record.stage), - (Evidence::HostFailure, Outcome::Host, Stage::Activation) - ); - assert_eq!(record.transaction, Some(restore.transaction)); - assert_launch_ended(&fixture, &mut acquired, &vmm); - - // The Ready of the retry reports another transaction. - let mut acquired = fixture.acquire("diverged"); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let diverged = ReadyActivation { - transaction: Some(TransactionId::from_bytes([0x78; TransactionId::LEN])), - }; - let mut control = - FakeControl::activating([Answer::Fail(Failure::LOST), Answer::Report(Some(diverged))]); - let error = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Protocol, "{error}"); - assert_eq!(control.sessions, 2); - let ready = (Stage::Activation, Progress::Activating); - assert_failed( - &error, - &acquired, - ready, - Evidence::Protocol, - Teardown::Terminated, - ); - assert_launch_ended(&fixture, &mut acquired, &vmm); -} - -#[test] -fn only_lost_sessions_are_replaced_and_only_so_many_times() { - let (fixture, image) = fixture(); - - // A failure that did not lose the session is final. - let mut acquired = fixture.acquire("refused"); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::activating([Answer::Fail(Failure::REFUSED)]); - let error = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Protocol, "{error}"); - assert_eq!(error.message(), "cannot activate it"); - assert_eq!(control.sessions, 1); - let ready = (Stage::Activation, Progress::Activating); - assert_failed( - &error, - &acquired, - ready, - Evidence::Protocol, - Teardown::Terminated, - ); - assert_launch_ended(&fixture, &mut acquired, &vmm); - - // A session lost again and again ends the sequence once the last of its sessions is lost. - let mut acquired = fixture.acquire("lost"); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let lost = (0..MAX_ACTIVATION_SESSIONS).map(|_| Answer::Fail(Failure::LOST)); - let mut control = FakeControl::activating(lost); - let error = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Protocol, "{error}"); - assert!(error.message().contains("was lost"), "{error}"); - control.assert_activates_first(); - assert_eq!(control.sessions, MAX_ACTIVATION_SESSIONS); - assert_failed( - &error, - &acquired, - ready, - Evidence::Reset, - Teardown::Terminated, - ); - assert_launch_ended(&fixture, &mut acquired, &vmm); -} - -#[test] -fn a_clone_that_took_the_reset_of_a_lost_session_before_its_ready_ends_the_retries() { - let (fixture, image) = fixture(); - let mut acquired = fixture.acquire("sandbox"); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - // The reset of the lost session ends a clone that was not ready yet, and with it its VMM: the - // retry cannot connect any more. - let ending = &vmm; - let mut control = FakeControl::activating([Answer::Fail(Failure::LOST)]); - control.on_activate = Some(Box::new(move |_| ending.end_and_wait())); - let error = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Exited, "{error}"); - assert_eq!(control.sessions, 1); - drop(control); - // The clone's status was lost with its VMM, which ended without a report. - let ready = (Stage::Activation, Progress::Activating); - assert_failed( - &error, - &acquired, - ready, - Evidence::LostStatus, - Teardown::Exited, - ); - assert_launch_ended(&fixture, &mut acquired, &vmm); -} - -#[test] -fn the_restore_to_ready_deadline_runs_from_the_launch_through_every_step() { - let (fixture, image) = fixture(); - let deadlines = RestoreDeadlines { - ready: SHORT, - ..RestoreDeadlines::default() - }; - - // A VMM that sends no event: the wait ends at the deadline, before any connection. - let mut acquired = fixture.acquire("no-event"); - let mut vmm = vmm_with("none"); - let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); - let error = restoring.activate(&mut control, deadlines).unwrap_err(); - assert_eq!(error.kind(), ErrorKind::TimedOut, "{error}"); - assert!(launched.before.elapsed() >= SHORT); - assert!(control.calls.is_empty()); - let restore = (Stage::Restore, Progress::Spawned); - assert_failed( - &error, - &acquired, - restore, - Evidence::Timeout, - Teardown::Terminated, - ); - assert_launch_ended(&fixture, &mut acquired, &vmm); - - // A clone that never answers its Activate, after a late event: the attachment and the - // Activate have the deadline that started with the launch. - let mut acquired = fixture.acquire("no-ready"); - let mut vmm = vmm_with(&format!("delay:{} flush", DELAY.as_millis())); - let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::activating([Answer::Silent]); - let deadlines = RestoreDeadlines { - ready: SILENT, - ..RestoreDeadlines::default() - }; - let error = restoring.activate(&mut control, deadlines).unwrap_err(); - assert_eq!(error.kind(), ErrorKind::TimedOut, "{error}"); - assert_eq!(control.sessions, 1); - for deadline in control.ready_deadlines() { - launched.assert_deadline(deadline, deadlines.ready); - } - assert!(launched.before.elapsed() >= deadlines.ready); - let ready = (Stage::Activation, Progress::Activating); - assert_failed( - &error, - &acquired, - ready, - Evidence::Timeout, - Teardown::Terminated, - ); - assert_launch_ended(&fixture, &mut acquired, &vmm); -} - -#[test] -fn the_ready_to_start_deadline_runs_from_the_ready_and_a_start_is_never_resent() { - let (fixture, image) = fixture(); - let deadlines = RestoreDeadlines { - start: SHORT, - ..RestoreDeadlines::default() - }; - - // A start that the guest never confirms ends at the deadline that started at the Ready. - let mut acquired = fixture.acquire("silent"); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); - control.starts.push_back(Answer::Silent); - let ready = Instant::now(); - let activated = restoring.activate(&mut control, deadlines).unwrap(); - let start = activated.deadline(); - assert!(start >= ready + SHORT); - let error = activated.start(&mut control).unwrap_err(); - assert_eq!(error.kind(), ErrorKind::TimedOut, "{error}"); - assert!(Instant::now() >= start); - assert_eq!( - control.calls.last(), - Some(&Call::Start { - session: 1, - deadline: start - }) - ); - let starting = (Stage::Start, Progress::Starting); - assert_failed( - &error, - &acquired, - starting, - Evidence::Timeout, - Teardown::Terminated, - ); - assert_launch_ended(&fixture, &mut acquired, &vmm); - - // A start that fails, even in a lost session, is not sent again. - let mut acquired = fixture.acquire("lost"); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); - control.starts.push_back(Answer::Fail(Failure::LOST)); - let activated = restoring.activate(&mut control, deadlines).unwrap(); - let error = activated.start(&mut control).unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Protocol, "{error}"); - let starts = control - .calls - .iter() - .filter(|call| matches!(call, Call::Start { .. })) - .count(); - assert_eq!((control.sessions, starts), (1, 1)); - assert_failed( - &error, - &acquired, - starting, - Evidence::Reset, - Teardown::Terminated, - ); - assert_launch_ended(&fixture, &mut acquired, &vmm); -} - -#[test] -fn a_vmm_that_exits_or_sends_anything_but_its_event_ends_the_launch() { - let (fixture, image) = fixture(); - // A VMM that exits before its event, as OpenVMM does once it cannot restore the snapshot, ends - // the wait at once, and loses the clone's status, unless OpenVMM's fatal error tells that the - // host cannot present the template's CPU or time, or bind a forward of the clone's network; - // another version of the event, and part of it, are refused, and the VMM that sent them is - // terminated. - let cases = [ - ( - "exit", - "exit", - ErrorKind::Exited, - Evidence::LostStatus, - Teardown::Exited, - ), - ( - "incompatible", - "exit fatal:E_TSC_RATE_TOLERANCE", - ErrorKind::Exited, - Evidence::IncompatibleHost, - Teardown::Exited, - ), - ( - "contract", - "exit fatal:E_MANIFEST_TIME", - ErrorKind::Exited, - Evidence::LostStatus, - Teardown::Exited, - ), - // OpenVMM connects the event's endpoint before it launches its VM worker, whose launch - // fails on a host that cannot present the template's CPU or time: it closes the endpoint - // without the event, and reports its error only as it exits, which the wait awaits. - ( - "closed", - "close fatal:E_TSC_RATE_TOLERANCE", - ErrorKind::Exited, - Evidence::IncompatibleHost, - Teardown::Exited, - ), - // The launch of its VM worker fails too where OpenVMM cannot bind a forward of the - // clone's network as it restores the clone's NIC, because another socket holds its port. - ( - "forward", - "close forward:18080", - ErrorKind::Exited, - Evidence::ForwardBind, - Teardown::Exited, - ), - ( - "other", - "other flush", - ErrorKind::Protocol, - Evidence::Protocol, - Teardown::Terminated, - ), - ( - "partial", - "partial flush", - ErrorKind::Protocol, - Evidence::Protocol, - Teardown::Terminated, - ), - ]; - let restore = (Stage::Restore, Progress::Spawned); - for (name, plan, kind, evidence, teardown) in cases { - let mut acquired = fixture.acquire(name); - let mut vmm = vmm_with(plan); - let (restoring, launched) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); - let error = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap_err(); - assert_eq!(error.kind(), kind, "{plan}: {error}"); - assert!(launched.before.elapsed() < TIMEOUT, "{plan}"); - assert!(control.calls.is_empty(), "{plan}"); - assert_failed(&error, &acquired, restore, evidence, teardown); - // The record keeps the line of OpenVMM's fatal error, and not the warning before it, with - // the cause of the error's chain that names the forward that OpenVMM cannot bind. - let line = plan.split_whitespace().find_map(|word| { - if let Some(code) = word.strip_prefix("fatal:") { - Some(format!("[{code}] failed to launch vm worker")) - } else { - word.strip_prefix("forward:").map(|port| { - format!( - "failed to launch vm worker: failed to create socket for port forward \ - (Tcp 127.0.0.1:{port})" - ) - }) - } - }); - if let Some(line) = line { - let line = format!("OpenVMM reported: fatal error: {line}"); - assert!(error.message().ends_with(&line), "{error}"); - assert!(error.record().message.ends_with(&line), "{error}"); - assert_eq!(error.outcome(), evidence.outcome(), "{error}"); - } - assert_launch_ended(&fixture, &mut acquired, &vmm); - } - - // Another process than the VMM connects to the restore-ready endpoint. - let mut acquired = fixture.acquire("intruder"); - let mut vmm = vmm_with("none"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let ready = PathBuf::from(restoring.ready_endpoint()); - #[cfg(unix)] - let _intruder = std::os::unix::net::UnixStream::connect(&ready).unwrap(); - #[cfg(windows)] - let _intruder = fs::OpenOptions::new().write(true).open(&ready).unwrap(); - let mut control = FakeControl::default(); - let error = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap_err(); - assert_eq!(error.kind(), ErrorKind::Protocol, "{error}"); - let source = std::error::Error::source(&error).unwrap().to_string(); - assert!(source.contains("not the VMM"), "{source}"); - assert!(control.calls.is_empty()); - assert_failed( - &error, - &acquired, - restore, - Evidence::Protocol, - Teardown::Terminated, - ); - assert_launch_ended(&fixture, &mut acquired, &vmm); - assert!(!acquired.locked.dir().join(CLONE_ACTIVATION_NAME).exists()); -} - -#[test] -fn a_status_with_which_the_guest_ended_the_clone_s_vm_attributes_its_failure() { - let (fixture, image) = fixture(); - let ready = (Stage::Activation, Progress::Activating); - let starting = (Stage::Start, Progress::Starting); - // Each VMM ends as its clone awaits its Ready, or once the start of its sandbox was sent, with - // OpenVMM's outcome report, which the failure's record reads once the VMM has exited. - let cases = [ - // The guest's activation could not assemble the clone's root, which only the request's - // image and upper make, before any workload ran. - ( - "root", - "guest-exit:54", - ready, - Evidence::UnusableRoot, - Some(54), - ), - // A repair's status is evidence of the repair stage at any point. - ( - "repair", - "guest-exit:195", - ready, - Evidence::RepairStatus, - Some(195), - ), - ( - "late-repair", - "guest-exit:195", - starting, - Evidence::RepairStatus, - Some(195), - ), - // Once the start was sent, a workload may have ended its VM with any status, so the loss - // of the session is all that the host knows. - ( - "late-root", - "guest-exit:54", - starting, - Evidence::Reset, - Some(54), - ), - // A failure of the VMM tells nothing of the guest. - ("vmm", "vmm-failure:1", ready, Evidence::LostStatus, None), - ]; - for (name, report, at, evidence, status) in cases { - let mut acquired = fixture.acquire(name); - let mut vmm = vmm_with(&format!("flush report:{report}")); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let error = if at == starting { - let mut control = FakeControl::default(); - control.starts.push_back(Answer::Fail(Failure::LOST)); - let activated = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap(); - vmm.end_and_wait(); - activated.start(&mut control).unwrap_err() - } else { - let ending = &vmm; - let mut control = FakeControl::activating([Answer::Fail(Failure::LOST)]); - control.on_activate = Some(Box::new(move |_| ending.end_and_wait())); - restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap_err() - }; - assert_failed(&error, &acquired, at, evidence, Teardown::Exited); - assert_eq!(error.record().status, status, "{name}"); - assert_launch_ended(&fixture, &mut acquired, &vmm); - } -} - -thread_local! { - /// The path of the record of a failure that [`ended_once_recorded`] reads, and the record that - /// it read there. - static RECORDED: RefCell)>> = const { RefCell::new(None) }; -} - -/// Ends a launch as [`Starting::abandon`] does, once it read the record of the failure at the path -/// in [`RECORDED`], as the teardown begins, which a crash of the caller during the teardown would -/// leave. -fn ended_once_recorded(starting: Starting<'_>) -> Result { - RECORDED.with_borrow_mut(|recorded| { - let (path, record) = recorded.as_mut().expect("the test names the record's path"); - *record = read_failure(path).unwrap(); - }); - starting.abandon() -} - -#[test] -fn a_failure_is_recorded_before_its_teardown_and_again_once_the_teardown_ended() { - let (fixture, image) = fixture(); - let mut acquired = fixture.acquire("sandbox"); - RECORDED.set(Some((acquired.locked.clone_failure_path(), None))); - let mut vmm = vmm_with("other flush"); - let (restoring, _) = launch_ending( - &fixture, - &mut acquired, - &image, - &mut vmm, - ended_once_recorded, - ); - let mut control = FakeControl::default(); - let error = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap_err(); - // As the teardown began, the sandbox recorded the failure, with its teardown pending. - let (_, pending) = RECORDED.take().unwrap(); - let pending = pending.expect("the failure was recorded before its teardown"); - let restore = (Stage::Restore, Progress::Spawned); - assert_eq!( - (pending.stage, pending.progress, pending.evidence), - (restore.0, restore.1, Evidence::Protocol) - ); - assert_eq!(pending.teardown, Teardown::Pending); - assert_eq!(pending.message, error.record().message); - // Once the teardown ended, the record holds its outcome. - assert_failed( - &error, - &acquired, - restore, - Evidence::Protocol, - Teardown::Terminated, - ); - assert_launch_ended(&fixture, &mut acquired, &vmm); -} - -/// Why [`kept`] keeps a launch. -const KEPT: &str = "the test keeps the launch"; - -/// Ends no launch of a clone that failed: keeps its journal and its VMM, as a launch whose VMM's -/// identity or exit is indeterminate keeps them. -fn kept(starting: Starting<'_>) -> Result { - // Nothing of the launch ends: its journal and its VMM outlive it, as a crash leaves them. - std::mem::forget(starting); - Ok(Recovery::Indeterminate(KEPT.to_owned())) -} - -#[test] -fn a_launch_that_cannot_be_ended_keeps_its_records_and_its_lease_until_a_reattach_ends_it() { - let (fixture, image) = fixture(); - let mut acquired = fixture.acquire("sandbox"); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch_ending(&fixture, &mut acquired, &image, &mut vmm, kept); - let mut control = FakeControl::activating([Answer::Fail(Failure::REFUSED)]); - let error = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap_err(); - let ready = (Stage::Activation, Progress::Activating); - let unresolved = Teardown::Unresolved(KEPT.to_owned()); - assert_failed(&error, &acquired, ready, Evidence::Protocol, unresolved); - let unended = format!("; the launch could not be ended, and remains for recovery: {KEPT}"); - assert!(error.message().ends_with(&unended), "{error}"); - // The journal remains, and with it the lease, which no release ends, and the clone's records. - let reference = vmm.reference.clone().unwrap(); - let locked = &mut acquired.locked; - let refusal = fixture.store.release(locked).unwrap_err(); - assert_eq!(refusal.code(), ErrorCode::SandboxRunning, "{refusal}"); - let records = CloneRecords::read(locked).unwrap().unwrap(); - assert_eq!(records.journal.unwrap().process(), Some(&reference)); - assert!(matches!( - Process::open(&reference).unwrap(), - Opened::Running(_) - )); - // A reattach ends the launch through its VMM's verified identity: the record keeps the - // failure's stage, evidence, and original outcome, with the teardown that resolved it. - let mut control = FakeControl::default(); - let reattached = reattach_clone( - locked, - &runtime_of(&fixture), - &mut control, - Instant::now() + TIMEOUT, - TIMEOUT, - ) - .unwrap(); - let resolved = failed(reattached); - assert_failed( - &resolved, - &acquired, - ready, - Evidence::Protocol, - Teardown::Terminated, - ); - assert_eq!( - (resolved.code(), resolved.message()), - (error.code(), error.record().message.as_str()) - ); - assert!(control.calls.is_empty()); - assert_launch_ended(&fixture, &mut acquired, &vmm); -} - -#[test] -fn a_caller_fails_a_ready_or_started_clone_with_its_own_evidence_as_the_sequence_fails_one() { - let (fixture, image) = fixture(); - let ready = (Stage::Start, Progress::Ready); - // A step of the caller's own, such as one that maps a directory of the host into the clone, - // fails once the clone is ready, which the request's mappings attribute to the request. - let mut acquired = fixture.acquire("ready"); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); - let activated = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap(); - let mapping = "the test's mapping names no directory of the host"; - let failure = activated.fail(Evidence::InvalidMappings, Error::unsupported(mapping)); - assert_eq!(failure.kind(), ErrorKind::Unsupported, "{failure}"); - assert_eq!(failure.message(), mapping); - let mappings = Evidence::InvalidMappings; - assert_failed(&failure, &acquired, ready, mappings, Teardown::Terminated); - assert_eq!(failure.outcome(), Outcome::Request); - assert_launch_ended(&fixture, &mut acquired, &vmm); - // A check of the caller's own fails once the clone's sandbox started, before the commit. - let mut acquired = fixture.acquire("started"); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); - let activated = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap(); - let started = activated.start(&mut control).unwrap(); - let failure = started.fail(Evidence::HostFailure, Error::io("the test's check failed")); - let commit = (Stage::Commit, Progress::Started); - let host = Evidence::HostFailure; - assert_failed(&failure, &acquired, commit, host, Teardown::Terminated); - assert_eq!(failure.outcome(), Outcome::Host); - assert_launch_ended(&fixture, &mut acquired, &vmm); - // The guest ended the clone's VM with the status of a failed repair as the caller's step - // failed: the status prevails, as over the sequence's own failures. - let mut acquired = fixture.acquire("repair"); - let mut vmm = vmm_with("flush report:guest-exit:195"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); - let activated = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap(); - vmm.end_and_wait(); - let failure = activated.fail(Evidence::InvalidMappings, Error::unsupported(mapping)); - let repair = Evidence::RepairStatus; - assert_failed(&failure, &acquired, ready, repair, Teardown::Exited); - assert_eq!(failure.record().status, Some(195)); - assert_launch_ended(&fixture, &mut acquired, &vmm); - // Evidence of the template, such as a defect of the generation that the caller diagnosed, - // shows the provenance that the status alone leaves to establish: it prevails, and the record - // holds the status beside it. The failure quarantines the generation once the sandbox's lock - // is released, as the lock order asks. - let mut acquired = fixture.acquire("defect"); - let mut vmm = vmm_with("flush report:guest-exit:195"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); - let activated = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap(); - vmm.end_and_wait(); - let defect = "the test diagnosed a defect of the generation's repair"; - let failure = activated.fail(Evidence::TemplateDefect, Error::other(defect)); - let template = Evidence::TemplateDefect; - assert_failed(&failure, &acquired, ready, template, Teardown::Exited); - assert_eq!( - (failure.outcome(), failure.record().status), - (Outcome::Template, Some(195)) - ); - assert_launch_ended(&fixture, &mut acquired, &vmm); - drop(acquired); - assert!(failure.quarantine(&fixture.store, None).unwrap()); - assert_eq!( - fixture.store.tombstones().unwrap(), - [failure.record().template.generation] - ); -} - -#[test] -fn a_failure_that_cannot_be_recorded_before_its_teardown_keeps_its_launch_for_a_reattach() { - let (fixture, image) = fixture(); - let mut acquired = fixture.acquire("unrecorded"); - let path = acquired.locked.clone_failure_path(); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); - let activated = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap(); - // The caller diagnoses a defect of the template once the clone is ready, but the record of - // its failure cannot be written, as on a full disk: a directory takes its place. - fs::create_dir(&path).unwrap(); - let defect = "the test diagnosed a defect of the generation's repair"; - let failure = activated.fail(Evidence::TemplateDefect, Error::other(defect)); - let ready = (Stage::Start, Progress::Ready); - let record = failure.record(); - assert_eq!( - ((record.stage, record.progress), record.evidence), - (ready, Evidence::TemplateDefect) - ); - // The caller holds the failure, the template's, whose teardown ended the clone's VMM through - // its handle, and kept the launch's journal, and with it the lease. - assert_eq!(failure.outcome(), Outcome::Template); - let Teardown::Unresolved(reason) = &record.teardown else { - panic!("the teardown of {failure} was resolved"); - }; - let unrecorded = "the failure could not be recorded before its teardown: "; - let ended = format!("the clone's VMM ended through its handle, but {unrecorded}"); - assert!(reason.starts_with(&ended), "{reason}"); - assert!(failure.message().starts_with(defect), "{failure}"); - let lost = "; the failure could not be recorded: "; - assert!(failure.message().contains(lost), "{failure}"); - assert_vmm_ended(&vmm); - let journal = acquired.locked.journal().unwrap(); - let journal = journal.expect("the launch kept its journal"); - assert_eq!(journal.process(), vmm.reference.as_ref()); - let refusal = fixture.store.release(&mut acquired.locked).unwrap_err(); - assert_eq!(refusal.code(), ErrorCode::SandboxRunning, "{refusal}"); - // Once the record can be written, a reattach, as after a crash of the caller, finds that the - // clone failed, though not how, rather than a clone that ended. - fs::remove_dir(&path).unwrap(); - let mut control = FakeControl::default(); - let reattached = reattach_clone( - &mut acquired.locked, - &runtime_of(&fixture), - &mut control, - Instant::now() + TIMEOUT, - TIMEOUT, - ) - .unwrap(); - let resolved = failed(reattached); - let exited = Teardown::Exited; - assert_failed(&resolved, &acquired, ready, Evidence::LostStatus, exited); - assert!(control.calls.is_empty()); - assert_launch_ended(&fixture, &mut acquired, &vmm); -} - -#[test] -fn a_diagnosis_too_long_for_a_record_keeps_its_ends_and_its_evidence_through_a_reattach() { - let (fixture, image) = fixture(); - let mut acquired = fixture.acquire("diagnosis"); - let mut vmm = vmm_with("flush report:guest-exit:195"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - let mut control = FakeControl::default(); - let activated = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap(); - vmm.end_and_wait(); - // The caller's diagnosis of a defect of the generation is longer than a record keeps: JSON - // takes six bytes for each of its control characters, the most that a character takes. - let (first, last) = ( - "the test's diagnosis begins", - "and the test's diagnosis ends", - ); - let defect = format!("{first}{}{last}", "\u{1}".repeat(64 * 1024)); - let failure = activated.fail(Evidence::TemplateDefect, Error::other(defect.clone())); - // The failure's error keeps the diagnosis whole, and its record, which the sandbox keeps, - // its ends. - assert_eq!(failure.message(), defect); - let record = failure.record(); - let ready = (Stage::Start, Progress::Ready); - assert_eq!( - ( - (record.stage, record.progress), - record.evidence, - &record.teardown - ), - (ready, Evidence::TemplateDefect, &Teardown::Exited) - ); - assert!(record.message.starts_with(first) && record.message.ends_with(last)); - assert!(record.message.chars().count() < defect.chars().count()); - let records = CloneRecords::read(&acquired.locked).unwrap().unwrap(); - assert_eq!(records.failure.as_ref(), Some(record)); - // A reattach finds the failure as it was recorded, with the template's evidence, which - // quarantines the generation once the sandbox's lock is released. - let mut control = FakeControl::default(); - let reattached = reattach_clone( - &mut acquired.locked, - &runtime_of(&fixture), - &mut control, - Instant::now() + TIMEOUT, - TIMEOUT, - ) - .unwrap(); - let again = failed(reattached); - assert_eq!(again.record(), record); - assert_launch_ended(&fixture, &mut acquired, &vmm); - drop(acquired); - assert!(again.quarantine(&fixture.store, None).unwrap()); - assert_eq!( - fixture.store.tombstones().unwrap(), - [record.template.generation] - ); -} - -#[test] -fn a_clone_with_an_export_gets_its_mapping_table_once_ready_and_its_start_once_it_applied_it() { - let (fixture, image) = fixture(); - let export = fixture.data_export("data"); - let request = with_export(tmpfs_request(&image.0, &image.1), &export); - let mut acquired = fixture.acquire("exported"); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch_request( - &fixture, - &mut acquired, - &request, - &mut vmm, - Hooks::LAUNCH.end, - ); - let mut control = FakeControl::default(); - let activated = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap(); - let deadline = activated.deadline(); - let started = activated.start(&mut control).unwrap(); - // In the session in which the clone answered, the table follows its Ready, in order, and the - // start follows once the guest applied every mapping, both by the Ready-to-Start deadline. - // The table names each path by its child's index, as a cold launch's does, through the - // `MAPS` payloads that carry it. - control.assert_activates_first(); - assert_eq!(data_table().to_vec(), export.guest_table()); - assert_eq!( - control.calls[2..], - [ - Call::Map { - session: 1, - table: data_table().to_vec(), - deadline, - }, - Call::Start { - session: 1, - deadline, - }, - ] - ); - assert_eq!(control.applied, data_table()); - let (running, _) = started.commit().unwrap(); - let locked = &mut acquired.locked; - assert_eq!( - running.terminate(locked, TIMEOUT).unwrap(), - StopOutcome::Forced - ); - assert_eq!( - fixture.store.release(locked).unwrap(), - Some(acquired.lease.clone()) - ); - - // A clone without an export gets its start at once. - let mut acquired = fixture.acquire("plain"); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch(&fixture, &mut acquired, &image, &mut vmm); - assert!(restoring.table.is_empty()); - let mut control = FakeControl::default(); - let activated = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap(); - let deadline = activated.deadline(); - let started = activated.start(&mut control).unwrap(); - assert_eq!( - control.calls[2..], - [Call::Start { - session: 1, - deadline, - }] - ); - assert!(control.applied.is_empty()); - drop(started.abandon().unwrap()); - assert_launch_ended(&fixture, &mut acquired, &vmm); -} - -#[test] -fn a_partial_or_failed_mapping_table_ends_the_launch_and_its_sandbox_never_starts() { - let (fixture, image) = fixture(); - let export = fixture.data_export("data"); - let request = with_export(tmpfs_request(&image.0, &image.1), &export); - let deadlines = RestoreDeadlines { - start: SHORT, - ..RestoreDeadlines::default() - }; - let refused = "the clone refused its mapping table"; - let failed = "the clone did not apply its mapping table"; - let lost = "the clone's control session was lost: the clone did not apply its mapping table"; - let late = "the clone did not apply its mapping table by its Ready-to-Start deadline"; - // A guest that refuses the table once it applied some of it, or none of it, as the request's - // mappings decide; a failure of the framing that tells no refusal; a session lost once the - // table was sent; and a guest that never answers. - let cases = [ - ( - "partial", - 1, - Answer::Fail(Failure::TABLE), - (ErrorKind::Protocol, refused), - Evidence::InvalidMappings, - ), - ( - "refused", - 0, - Answer::Fail(Failure::TABLE), - (ErrorKind::Protocol, refused), - Evidence::InvalidMappings, - ), - ( - "failed", - 1, - Answer::Fail(Failure::REFUSED), - (ErrorKind::Protocol, failed), - Evidence::Protocol, - ), - ( - "lost", - 1, - Answer::Fail(Failure::LOST), - (ErrorKind::Protocol, lost), - Evidence::Reset, - ), - ( - "silent", - 0, - Answer::Silent, - (ErrorKind::TimedOut, late), - Evidence::Timeout, - ), - ]; - // The progress log records the start only once the guest applied the whole table, so each - // failure is recorded at the start's stage, with the clone's progress still at its Ready. - let ready = (Stage::Start, Progress::Ready); - for (name, applied, answer, (kind, message), evidence) in cases { - let mut acquired = fixture.acquire(name); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch_request( - &fixture, - &mut acquired, - &request, - &mut vmm, - Hooks::LAUNCH.end, - ); - let mut control = FakeControl::default(); - let source = match &answer { - Answer::Fail(failure) => failure.to_string(), - _ => Failure::REFUSED.to_string(), - }; - control.maps.push_back((applied, answer)); - let activated = restoring.activate(&mut control, deadlines).unwrap(); - let deadline = activated.deadline(); - let error = activated.start(&mut control).unwrap_err(); - assert_eq!(error.kind(), kind, "{name}: {error}"); - assert_eq!(error.message(), message, "{name}"); - let failed = std::error::Error::source(&error).unwrap().to_string(); - assert_eq!(failed, source, "{name}"); - if kind == ErrorKind::TimedOut { - assert!(Instant::now() >= deadline, "{name}"); - } - // The table was sent once, in the session in which the clone answered, and no start, nor - // another session, followed it. - assert_eq!(control.sessions, 1, "{name}"); - assert_eq!( - control.calls[2..], - [Call::Map { - session: 1, - table: data_table().to_vec(), - deadline, - }], - "{name}" - ); - assert_eq!(control.applied, data_table()[..applied], "{name}"); - // The launch ended, through the VMM's verified identity, and the sandbox records the - // failure, which is never the template's, so that none quarantines its generation. - assert_failed(&error, &acquired, ready, evidence, Teardown::Terminated); - assert!(!error.outcome().quarantines(), "{name}"); - assert_launch_ended(&fixture, &mut acquired, &vmm); - } - - // The guest's refusal of the table is the request's whenever it came, even past the deadline. - let mut acquired = fixture.acquire("late"); - let mut vmm = vmm_with("flush"); - let (restoring, _) = launch_request( - &fixture, - &mut acquired, - &request, - &mut vmm, - Hooks::LAUNCH.end, - ); - let mut control = FakeControl::default(); - control.maps.push_back((0, Answer::Fail(Failure::TABLE))); - let passed = RestoreDeadlines { - start: Duration::ZERO, - ..RestoreDeadlines::default() - }; - let activated = restoring.activate(&mut control, passed).unwrap(); - let error = activated.start(&mut control).unwrap_err(); - assert_eq!( - (error.kind(), error.message()), - (ErrorKind::Protocol, refused) - ); - assert_failed( - &error, - &acquired, - ready, - Evidence::InvalidMappings, - Teardown::Terminated, - ); - assert_launch_ended(&fixture, &mut acquired, &vmm); - - // The guest ended the clone's VM as its table failed, with the status of an export that does - // not mount: no start was sent, so no workload ran, and the status is evidence, which prevails - // over the loss of the session. - let mut acquired = fixture.acquire("status"); - let mut vmm = vmm_with("flush report:guest-exit:56"); - let (restoring, _) = launch_request( - &fixture, - &mut acquired, - &request, - &mut vmm, - Hooks::LAUNCH.end, - ); - let mut control = FakeControl::default(); - control.maps.push_back((0, Answer::Fail(Failure::LOST))); - let activated = restoring - .activate(&mut control, RestoreDeadlines::default()) - .unwrap(); - vmm.end_and_wait(); - let error = activated.start(&mut control).unwrap_err(); - assert_eq!(error.message(), lost); - assert_failed( - &error, - &acquired, - ready, - Evidence::InvalidMappings, - Teardown::Exited, - ); - assert_eq!(error.record().status, Some(56)); - assert_launch_ended(&fixture, &mut acquired, &vmm); -} diff --git a/nvx-rs/src/host/template/mod.rs b/nvx-rs/src/host/template/mod.rs index 4d8d208c5..9f3997a5e 100644 --- a/nvx-rs/src/host/template/mod.rs +++ b/nvx-rs/src/host/template/mod.rs @@ -31,45 +31,12 @@ //! sandbox, until [`Store::release`] ends the lease once the clone's VM has exited. A tombstone //! quarantines a generation, which takes no new lease, and [`Store::retire`] deletes it once its //! leases have drained. -//! - [`ClonePlan::new`] plans the clone of a leased generation, as a [`CloneRequest`] asks, under -//! the lock that acquired the lease, before the clone's launch: it checks the clone's image -//! against its binding, its upper, in RAM or on the caller's scratch image, which it claims, its -//! memory target against the template, and its export, if it has one, an aggregate of host -//! directories and files, as a cold launch exports it, which the template's dormant filesystem -//! slot binds, and refuses a request that the template cannot serve, such as an image longer -//! than the template's placeholder, the capacity of every clone's image, or of other block -//! sizes, or a network that attaches a NIC where the template has none, none where it has one, -//! or one at another guest address than the template's, before any VMM starts. -//! - [`ClonePlan::launch`] launches the clone, as the launch that its lease reserves, once the -//! objects of its export are still those that its plan found: OpenVMM restores the generation's -//! snapshot in place with the clone's devices and export, and the clone's activation record -//! waits in its sandbox, as a [`Restoring`] describes. -//! - [`Restoring::activate`] runs the clone's restore sequence through the adapter's framing of -//! the clone's control session, a [`CloneControl`]: it waits for OpenVMM's restore-ready event, -//! then connects, sends the clone's activation record in the adapter's Activate, and checks the -//! activation that the clone's Ready reports, by a deadline that starts with the launch, and -//! resends the same record in a new session once the adapter lost one. [`Activated::start`] -//! then sends the clone's mapping table, if it has an export, and starts the clone's sandbox -//! once the guest applied every mapping, by a deadline that starts at its Ready, and -//! [`Started::commit`] commits the launch. -//! - The plan, the launch, and the restore sequence fail with a [`CloneFailure`], whose -//! [`Evidence`] attributes it to an [`Outcome`], at a [`Stage`], once the clone got as far as -//! its [`Progress`], with the [`Teardown`] that followed: a failure from the launch on ends the -//! clone's VMM through its verified identity, and is recorded in the clone's sandbox, which -//! keeps its lease while a VM of the clone may run. Only a failure of the template quarantines -//! the leased generation, through its tombstone, [`CloneFailure::quarantine`]. -//! - [`reattach_clone`] takes a clone over once the caller of its launch restarted: it checks the -//! clone's activation in a new session, and ends a clone that another runtime identity -//! launched, or that fails its check, or classifies how it ended; [`CloneRecords::read`] reads -//! what the clone's sandbox keeps of it, for the operations that list, inspect, or remove -//! clones. //! //! # Snapshot metadata //! //! The metadata types remain aliases of [`snapshot`](super::snapshot)'s types for template //! users. That module describes their canonical encoding and conformance vectors. -mod clone; mod plan; mod store; @@ -80,12 +47,6 @@ pub use super::snapshot::{ }; pub(super) use super::snapshot::{eligibility, key, manifest}; -pub use self::clone::{ - Activated, CloneControl, CloneFailure, ClonePlan, CloneRecords, CloneRequest, CloneUpper, - Evidence, FAILURE_FORMAT, FailureRecord, MAX_ACTIVATION_SESSIONS, Outcome, Progress, - READY_TO_START_TIMEOUT, RESTORE_READY_EVENT, RESTORE_TO_READY_TIMEOUT, ReattachedClone, - RestoreDeadlines, Restoring, Stage, Started, Teardown, reattach_clone, status_evidence, -}; pub use self::plan::{MEMORY_BLOCK_SIZE, SCRATCH_NAME, TemplatePlan, TemplateResources}; pub use self::store::{ Acquired, CAPTURE_RECORD_NAME, CAPTURES_NAME, Capture, Created, GENERATIONS_NAME, Generation, diff --git a/nvx-rs/src/host/template/store.rs b/nvx-rs/src/host/template/store.rs index 2a7cd7b91..46e7a1f98 100644 --- a/nvx-rs/src/host/template/store.rs +++ b/nvx-rs/src/host/template/store.rs @@ -28,7 +28,6 @@ use crate::host::store::lease::{ ID_LEN, StoreRecord, hex, read_identity, read_private, read_record, store_error, }; -pub(super) use self::capture::SOCKET_PATH_LIMIT; pub use self::capture::{CAPTURE_RECORD_NAME, CAPTURES_NAME, Capture, Created, TRANSACTION_NAME}; pub use self::leases::{Acquired, LEASES_NAME, Retirement, TOMBSTONES_NAME}; pub use crate::host::store::lease::{ diff --git a/nvx-rs/src/host/template/store/leases.rs b/nvx-rs/src/host/template/store/leases.rs index 347e04318..45e249530 100644 --- a/nvx-rs/src/host/template/store/leases.rs +++ b/nvx-rs/src/host/template/store/leases.rs @@ -344,9 +344,7 @@ impl Store { /// Returns the path of the Linux socket at which the VMM of the clone that holds `lease` /// listens, its control endpoint: `.sock`, after the launch that the lease reserves, /// in the payload of the leased generation, beside the snapshot that the VMM restores in - /// place, since OpenVMM requires a restore's control socket in the snapshot's parent. The - /// clone's launch refuses a path that a Unix socket, or OpenVMM's option `listen=`, - /// cannot take. + /// place, since OpenVMM requires a restore's control socket in the snapshot's parent. /// /// A payload holds the sockets of the clones of its generation only while they lease it: the /// VMM removes its socket as it exits, a stop or a recovery removes what an abrupt exit diff --git a/nvx-rs/src/host/template/store/tests.rs b/nvx-rs/src/host/template/store/tests.rs index f38b612af..45467d334 100644 --- a/nvx-rs/src/host/template/store/tests.rs +++ b/nvx-rs/src/host/template/store/tests.rs @@ -12,9 +12,6 @@ use crate::host::template::MANIFEST_NAME; use crate::host::template::key::CaptureClass; use crate::host::template::manifest::TemplateManifest; -#[cfg(any(target_os = "linux", windows))] -pub(in crate::host::template) use crate::host::snapshot::SNAPSHOT_ARTIFACTS; - impl Store { /// Publishes a generation of a template, whose fixed time contract names a host and boot of /// its own, for the tests of other modules, and returns it. @@ -25,18 +22,6 @@ impl Store { tests::storage(1), ) } - - /// Publishes a capture of `class` whose manifest `spec` encodes, as - /// [`publish_for_tests`](Self::publish_for_tests) does, for the tests of the other modules of - /// templates, and returns it. - #[cfg(any(target_os = "linux", windows))] - pub(in crate::host::template) fn publish_spec_for_tests( - &self, - class: &crate::host::template::CaptureClass, - spec: &crate::host::snapshot::tests::ManifestSpec, - ) -> Generation { - tests::publish_with(self, class, spec, tests::storage(1)) - } } /// A change to a record, as JSON.