diff --git a/.github/workflows/run-nvx-microvm-tests.yml b/.github/workflows/run-nvx-microvm-tests.yml index efdb67468..016699138 100644 --- a/.github/workflows/run-nvx-microvm-tests.yml +++ b/.github/workflows/run-nvx-microvm-tests.yml @@ -341,12 +341,14 @@ jobs: ) scratch="build/ubuntu-smoke-${{ inputs.backend }}.ext4" cp --reflink=auto build/ubuntu-smoke-scratch.ext4 "${scratch}" - # The probe checks the security profile. + # A run executes the probe once, as nvx-init runs each execution, + # and the probe checks the security profile of that execution. nvx-rs/target/release/nvx-rs sandbox \ --hypervisor "${{ inputs.backend }}" \ --layer "distro,build/ubuntu-distro.erofs,${uuid}" \ --scratch "${scratch}" \ --entrypoint /sbin/nvx-sandbox-smoke \ + --arg init \ --workload-user 65534:65534 \ --memory-mib 256 @@ -377,6 +379,7 @@ jobs: --mount "/workspace,${share},rw" \ --mount-deny nvx-denied \ --entrypoint /sbin/nvx-sandbox-smoke \ + --arg init \ --arg /workspace \ --arg rw \ --workload-user 65534:65534 \ @@ -403,6 +406,7 @@ jobs: --mount "/opt/hostedtoolcache,${share},ro" \ --mount-deny nvx-denied \ --entrypoint /sbin/nvx-sandbox-smoke \ + --arg init \ --arg /opt/hostedtoolcache \ --arg ro \ --workload-user 65534:65534 \ @@ -511,18 +515,30 @@ jobs: # group. With --mount-owner caller, every share operation must fail # with EPERM instead of running as the runner. The OpenVMM unit tests # cover the identity of each request, including guest root squash. + # A run maps its shares as a start does: its guest agent reaches the + # share as guest root, which the mode squashes to the share's owner, + # before any workload runs, so that request fails with EPERM, and the + # run fails without running its workload. cp --reflink=auto build/ubuntu-smoke-scratch.ext4 "${scratch}" - nvx-rs/target/release/nvx-rs sandbox \ + errors=$(mktemp) + if nvx-rs/target/release/nvx-rs sandbox \ --hypervisor "${{ inputs.backend }}" \ --layer "distro,build/ubuntu-distro.erofs,${uuid}" \ --scratch "${scratch}" \ --mount "/workspace,${share},rw" \ --mount-owner caller \ --entrypoint /sbin/nvx-sandbox-smoke \ + --arg init \ --arg /workspace \ --arg eperm \ --workload-user 65534:65534 \ - --memory-mib 256 + --memory-mib 256 2>"${errors}"; then + echo "the caller-owned run reached its share as the runner" >&2 + exit 1 + fi + cat "${errors}" >&2 + grep -q 'could not mount a mapped host path (error 1)' "${errors}" + rm -f "${errors}" test "$(ls -A "${share}")" = nvx-host-marker # A managed sandbox persists the ownership mode and applies it on # start. Its guest agent reaches the share as guest root, which the @@ -645,7 +661,7 @@ jobs: --layer "distro,build/ubuntu-distro.erofs,${uuid}" \ --scratch "${scratch}" \ "${share_options[@]}" \ - "${smoke_arguments[@]}" \ + --arg init "${smoke_arguments[@]}" \ --workload-user 65534:65534 \ --memory-mib 256 check_shares @@ -770,7 +786,7 @@ jobs: --layer "distro,build/ubuntu-distro.erofs,${uuid}" \ --scratch "${scratch}" \ "${share_options[@]}" \ - "${smoke_arguments[@]}" \ + --arg init "${smoke_arguments[@]}" \ --workload-user 65534:65534 \ --memory-mib 256 check_share @@ -844,12 +860,14 @@ jobs: ConvertFrom-Json $scratch = "build\ubuntu-smoke-${{ inputs.backend }}.ext4" Copy-Item build\ubuntu-smoke-scratch.ext4 $scratch - # The probe checks the security profile. + # A run executes the probe once, as nvx-init runs each execution, + # and the probe checks the security profile of that execution. nvx-rs\target\release\nvx-rs.exe sandbox ` --hypervisor "${{ inputs.backend }}" ` --layer "distro,build\ubuntu-distro.erofs,$($manifest.uuid)" ` --scratch $scratch ` --entrypoint /sbin/nvx-sandbox-smoke ` + --arg init ` --workload-user 65534:65534 ` --memory-mib 256 @@ -878,6 +896,7 @@ jobs: --mount "/workspace,$share,rw" ` --mount-deny nvx-denied ` --entrypoint /sbin/nvx-sandbox-smoke ` + --arg init ` --arg /workspace ` --arg rw ` --workload-user 65534:65534 ` @@ -921,6 +940,7 @@ jobs: --mount "/opt/hostedtoolcache,$share,ro" ` --mount-deny nvx-denied ` --entrypoint /sbin/nvx-sandbox-smoke ` + --arg init ` --arg /opt/hostedtoolcache ` --arg ro ` --workload-user 65534:65534 ` @@ -1101,7 +1121,7 @@ jobs: if ([IO.File]::ReadAllText((Join-Path $files "results\other.txt")) -ne "other`n") { throw "guest changed a file beside a shared file" } } Initialize-Shares - Invoke-Nvx sandbox --hypervisor "${{ inputs.backend }}" --layer $layer --scratch $scratch @shareOptions @smokeArguments --workload-user 65534:65534 --memory-mib 256 + Invoke-Nvx sandbox --hypervisor "${{ inputs.backend }}" --layer $layer --scratch $scratch @shareOptions --arg init @smokeArguments --workload-user 65534:65534 --memory-mib 256 Test-Shares # A managed sandbox persists every share and reattaches them on start. Initialize-Shares @@ -1202,7 +1222,7 @@ jobs: } } Initialize-Share - Invoke-Nvx sandbox --hypervisor "${{ inputs.backend }}" --layer $layer --scratch $scratch @shareOptions @smokeArguments --workload-user 65534:65534 --memory-mib 256 + Invoke-Nvx sandbox --hypervisor "${{ inputs.backend }}" --layer $layer --scratch $scratch @shareOptions --arg init @smokeArguments --workload-user 65534:65534 --memory-mib 256 Test-Share # A managed sandbox persists the access policy and reapplies it on # start. diff --git a/doc/ci.md b/doc/ci.md index f3ca57a09..653194bbe 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -537,13 +537,13 @@ that read it as a workflow artifact of its own, under its build filenames (see [Guest artifact handoff](#guest-artifact-handoff)). Each backend also boots the Ubuntu initramfs and runs `/sbin/nvx-sandbox-smoke` from the Ubuntu EROFS layer as UID/GID 65534 over a -fresh ext4 scratch copy. The probe verifies the -[sandbox security profile](run.md#experimental-single-workload-sandbox): the -fixed identity, empty capability sets and `no_new_privs`, private mount, PID, -and UTS namespaces, the private device view and read-only sysfs, the -container cgroup, and the overlay root over read-only image layers with its -upper directory on scratch. The live-share runs below repeat the profile -checks. Each +fresh ext4 scratch copy, as the one workload of an `nvx-rs sandbox run`, which +nvx-init executes. The probe verifies the +[sandbox security profile](run.md#experimental-single-workload-sandbox) of +such an execution: the fixed identity, empty capability sets and +`no_new_privs`, the host name, the device view and read-only sysfs, and the +overlay root over read-only image layers with its upper directory on scratch. +The live-share runs below repeat the profile checks. Each backend also runs the public `managed-exec-config`, `sandbox-lifecycle`, `sandbox-proxy-only`, and `sandbox-ram-upper` scenarios of `test-microvm`, which drive `nvx-rs sandbox run`, `provision`, `start`, @@ -587,7 +587,7 @@ read-write file but neither remove nor rename either one. The host requires the guest's writes in the workspace and the read-write file, an unchanged tool cache, read-only file, and denied files, and unchanged files beside the shared files, both for a -one-shot run and for a managed sandbox whose format-6 configuration must +run and for a managed sandbox whose format-6 configuration must persist every share and its kind and whose `stop` must leave a cleanly unmounted scratch filesystem. Each backend then attaches a read-write `/workspace` share whose @@ -599,8 +599,8 @@ writes to succeed only in `nvx-writable` and to fail with `EROFS` everywhere else, a hard link of a read-only file into `nvx-writable` to fail, and `nvx-denied` to list only `nvx-allowed`, whose file stays readable while `nvx-denied/secret` stays hidden. The host requires the guest's writes in -`nvx-writable` and no other change to the share, both for a one-shot run and -for a managed sandbox whose format-5 configuration must persist the denied, +`nvx-writable` and no other change to the share, both for a run and for a +managed sandbox whose format-5 configuration must persist the denied, allowed, and writable paths and whose `stop` must leave a cleanly unmounted scratch filesystem. The default `filesystem-policy` scenario of `test-microvm` covers the same policy as guest root, whose permission checks mode bits never @@ -612,13 +612,14 @@ writable handle open across the capture. On Linux, a world-writable share then runs with `--mount-owner caller`. The runners grant OpenVMM neither `CAP_SETUID` nor `CAP_SETGID`, so it can neither assume the 65534 workload identity nor drop the backend group through which it opens -`/dev/kvm` or `/dev/mshv`. The smoke script's `eperm` mode therefore -requires reading and listing the share to fail with `EPERM` and writes to -fail, and the host requires the share to be unchanged, both for a one-shot -run and for a managed sandbox whose configuration must persist the mode. The `filesystem-owner` scenario of -`test-microvm` likewise requires guest root and a foreign guest identity to -fail with `EPERM` on the runners, and a root-owned export to be rejected -before boot; on Windows, `--mount-owner caller` must be rejected before boot. +`/dev/kvm` or `/dev/mshv`. The guest agent of a run, and of a managed +sandbox whose configuration must persist the mode, therefore fails to map the +share with `EPERM` before any workload runs, which fails the run and the +start, and the host requires the share to be unchanged. The `filesystem-owner` +scenario of `test-microvm` likewise requires guest root and a foreign guest +identity to fail with `EPERM` on the runners, and a root-owned export to be +rejected before boot; on Windows, `--mount-owner caller` must be rejected +before boot. OpenVMM's unit tests, which its own CI runs (see [OpenVMM CI](#openvmm-ci)), check the host identity that each request runs as, including guest root squashed to an export owner other than OpenVMM. Where diff --git a/doc/run.md b/doc/run.md index c4eb78cae..bdb042708 100644 --- a/doc/run.md +++ b/doc/run.md @@ -277,7 +277,7 @@ IPv4 or IPv6 exclusions to those native rules: } ``` -Pass the file with `--network-egress-policy-file PATH` on `run`, one-shot +Pass the file with `--network-egress-policy-file PATH` on `run`, `sandbox run`, or `sandbox provision`. An explicit `--network-egress allow` or `deny` is required. The file option cannot be mixed with `--network-egress-allow` or `--network-egress-deny`. @@ -857,10 +857,10 @@ mount -t virtiofs microvm /mnt/host ## Experimental single-workload sandbox The `sandbox` command of [`nvx-rs`](usage.md#nvx-rs) launches the microVM -with one compressed EROFS distro layer under a writable upper: by default one -preformatted ext4 scratch image, or a tmpfs in guest memory with -`--upper ram`, as [Overlay upper](#overlay-upper) describes. Build the distro -layer with `nvx.py`: +with one distro layer, a compressed EROFS image or a GPT disk, under a writable +upper: by default one preformatted ext4 scratch image, or a tmpfs in guest +memory with `--upper ram`, as [Overlay upper](#overlay-upper) describes. Build +the distro layer with `nvx.py`: ```bash python3 scripts/nvx.py build-distro-layer \ @@ -872,22 +872,40 @@ Read the deterministic UUID from `build/ubuntu-distro.erofs.manifest.json`, create scratch, for example with [`build-scratch-template`](usage.md#build-scratch-template), which formats the sandbox scratch template at the path that `--output` gives, or with -`mkfs.ext4` itself, then launch the Ubuntu workload through the existing -Alpine control initramfs: +`mkfs.ext4` itself, then run an Ubuntu workload in a sandbox that the Alpine +control initramfs boots and nvx-init manages: ```bash python3 scripts/nvx.py build-scratch-template --output /var/lib/nvx/scratch.ext4 nvx-rs/target/release/nvx-rs sandbox \ --layer distro,build/ubuntu-distro.erofs,11111111-1111-1111-1111-111111111111 \ --scratch /var/lib/nvx/scratch.ext4 \ - --entrypoint /bin/sh \ + --entrypoint /bin/sh --arg=-c --arg='cat /etc/os-release' \ --workload-user 65534:65534 \ --memory-mib 256 ``` -CI uses `/sbin/nvx-sandbox-smoke` as the entrypoint to verify the security -profile that this section describes before clean guest exit. Before any other -check, it requires: +`sandbox run`, the default operation, keeps no state. It launches the VM that +a `provision` and a `start` of its options would launch, in a fresh temporary +VM directory, and prints its OpenVMM command as `>> COMMAND`. Once nvx-init is +ready and has mapped the shares, the run executes its workload once over the +control session, as [`sandbox exec`](#managed-lifecycle) does, with the same +`--cwd`, `--environment`, `--environment-file`, +`--inherit-default-environment`, and `--exec-timeout-ms` options. It then asks +nvx-init to stop the VM, terminates OpenVMM if it does not exit within +`--timeout` seconds, warning that it did, and removes the VM directory, and +finally prints the workload's standard output and standard error and exits +with its status. A VM that exits before nvx-init is ready, as when the guest +refuses the workload identity, fails the run as it fails a `start`, with the +category and status of the outcome report that OpenVMM published, such as +`guest-exit status 125`. `--outcome-report` forwards that report, of the VM's +stop, with the permissions that OpenVMM gave it. Ctrl-C ends the VM, and the +run then exits with status 130. `--dry-run` prints the command and launches +nothing. + +CI runs `/sbin/nvx-sandbox-smoke`, with `--arg init`, as below, as the workload +of `sandbox run` and `sandbox exec`, to verify the security profile of the +executions that nvx-init runs. Before any other check, it requires: - the fixed `65534:65534` identity as the real, effective, saved, and file-system IDs, no supplementary groups, and `HOME=/nonexistent`; @@ -916,9 +934,10 @@ check, it requires: A managed workload that mounted into the agent's namespace would stack its runtime mounts on those of earlier requests, so the single-mount check also covers repeated managed execs. With `--arg init` as its first argument, for an -execution in a sandbox that `sandbox start` started, which nvx-init runs as a -plain process in the VM's root, it checks the host name, which nvx-init's kernel -command line encodes in hexadecimal, instead of the namespaces, and no cgroup: +execution of `sandbox run`, or in a sandbox that `sandbox start` started, which +nvx-init runs as a plain process in the VM's root, it checks the host name, +which nvx-init's kernel command line encodes in hexadecimal, instead of the +namespaces, and no cgroup: such an execution shares nvx-init's mount, PID, and UTS namespaces and its cgroup. With `--arg entry`, for a managed execution, it also requires every mount other than `/dev` and its devpts instance to be `nodev`, as the managed @@ -974,30 +993,26 @@ unused fields, no object may repeat a field, and arrays and objects nest at most 32 deep, whatever features of `serde_json` other crates of a build enable. The artifact name, SHA-256 digest, and package metadata still bind it to the layer. -This is the cold-filesystem bootstrap described in -[the sandbox design](design/4.1-guest-filesystem-and-image-model.md#411-layered-sandbox-root), not the final -production agent. It accepts no environment variables or secrets and does not -expose sandbox snapshot capture or restore, the configuration region, or runtime RPC. -Entrypoints, arguments, and guest share targets are individual kernel-command-line -tokens and therefore cannot contain whitespace, control characters, double -quotes, or any UTF-8 byte that Linux treats as whitespace, including `0xa0`. +A run is the cold-filesystem bootstrap described in +[the sandbox design](design/4.1-guest-filesystem-and-image-model.md#411-layered-sandbox-root), which nvx-init +manages. It does not expose sandbox snapshot capture or restore, the +configuration region, or runtime RPC. Guest share targets cannot contain +whitespace, control characters, double quotes, or any UTF-8 byte that Linux +treats as whitespace, including `0xa0`, while the workload's entrypoint and +arguments, which nvx-init receives over the control session, may. Sandbox `--cmdline` uses Linux's quote and separator rules, and refuses reserved names even when quoted or written with hyphens instead of underscores. Validation uses the same trimmed text that is rendered and also checks the guest bootstrap's whitespace-only view, so reserved settings inside quoted values cannot override the generated settings. It also refuses a `--` or unclosed quote that hides the generated root tokens. -The workload enters private mount/PID/UTS namespaces with -a private `/dev`, an agent-owned cgroup, no capabilities, and `no_new_privs`. -It always runs as the fixed non-root `UID:GID` selected at VM creation -(`65534:65534` by default). The guest verifies that exactly one matching user, -its primary group, and its absolute home directory exist in the assembled -root; otherwise the workload is never started. -The outer agent retains the initramfs root; the capability-stripped child -enters only the assembled root with `chroot`, because Linux cannot -`pivot_root` away from an initramfs `rootfs`. A -[managed](#managed-lifecycle) execution instead moves the assembled root over -`/` in its own mount namespace. +nvx-init runs the workload as a plain process in the VM's root, with no +capabilities and `no_new_privs`, as it runs every +[managed](#managed-lifecycle) execution. The workload always runs as the fixed +non-root `UID:GID` selected at VM creation (`65534:65534` by default). The +guest verifies that exactly one matching user, its primary group, and its +absolute home directory exist in the assembled root; otherwise the workload is +never started. ### Overlay upper @@ -1009,9 +1024,10 @@ for `sandbox run` and `sandbox provision`: writes uses host storage, fails with `ENOSPC` when the image is full, and survives a managed sandbox's `stop` and next `start`. - `ram` keeps the upper in guest memory, with no scratch image or block device. - The kernel command line declares `nvx_overlay_upper=ramfs`, and the guest - mounts a tmpfs for the upper, `nosuid` and `nodev`, at the kernel's default - size, half of the guest's RAM, beyond which a write fails with `ENOSPC`. The + The kernel command line declares `nvx_overlay_upper=ramfs` and the cap of + `--ram-upper-percent`, from 1 through 90 percent of the guest's RAM, 50 by + default, and the guest mounts a tmpfs for the upper, `nosuid` and `nodev`, of + that size, beyond which a write fails with `ENOSPC`. The kernel cannot reclaim its pages while the files exist, since the guest has no swap. Nothing that the workload writes outlives its VM: each `start` of a managed sandbox begins with an empty upper. @@ -1038,20 +1054,23 @@ acceptance for the RAM upper. Like the subprocesses with the Ubuntu layer that `nvx-rs download` installs, but it attaches no scratch image. It checks that `provision` refuses `--upper ram` beside `--scratch`, and `start` refuses `--upper`, before -either creates any state. A one-shot `run` of `/sbin/nvx-sandbox-smoke` must -print an OpenVMM command with one sandbox block device, the read-only distro -layer, and `nvx_overlay_upper=ramfs` in place of the scratch token, and the -probe must pass its profile checks over a tmpfs upper. The run's -`--outcome-report`, under the scenario's temporary directory, must report -success and a complete teardown, and also names its OpenVMM to the scenario's -process checks and cleanup, which find VMs by that directory. A managed sandbox +either creates any state. A run of `/sbin/nvx-sandbox-smoke`, which keeps no +state, must print an OpenVMM command with one sandbox block device, the +read-only distro layer, `nvx_overlay_upper=ramfs` in place of the scratch +token, with the default cap, `nvx_ram_upper_percent=50`, and `nvx_sandbox=1` +without an OpenVMM lifecycle option, so that nvx-init manages its VM, and the +probe must pass its checks of an execution of nvx-init over a tmpfs upper. The +run's `--outcome-report`, under the scenario's temporary directory, must report +the success of the VM's stop and a complete teardown, and its `--cmdline` names +that directory, which names its OpenVMM to the scenario's process checks and +cleanup, which find VMs by that directory. A managed sandbox must record `ram` without a scratch image, pass the probe's managed checks, show its guest one virtio block device and the RAM upper's token, and read back a file that an earlier request wrote. After a `stop` and a new `start`, the file is gone. The scenario's evidence, with a `sandbox-ram-upper-` prefix, holds its -bounded command observations, the runtime record, the output of the one-shot -run, of the probe, and of the guest's view, and the outcome reports of the -one-shot run and of both stops. +bounded command observations, the runtime record, the output of the run, of +the probe, and of the guest's view, and the outcome reports of the run and of +both stops. ### Live host shares @@ -1109,17 +1128,9 @@ the guest mounts the shares inside the container root with `nosuid,nodev`, before any workload enters its private mount namespace. How it learns of them depends on the lifecycle. -A one-shot `sandbox run` delivers them at boot. The guest agent creates each -target inside the container root and mounts the share there. With several -shares, or a shared file, it mounts the aggregate at `/run/nvx/shares`, outside -the container root, and binds each child at its target with its mode, in -order, as the `nvx_share=NAME,TARGET,MODE[,file]` tokens that NVX adds to the -kernel command line describe. It binds a shared file at a regular file, which -it creates when the container root has nothing at the target. The number of -shares is bounded only by the sandbox's 1024-byte kernel command-line budget. - -A managed sandbox delivers them once its guest agent is ready. OpenVMM exports -every share as a child of one aggregate, which the guest mounts at +A sandbox, of `run` or of `start`, delivers them once its guest agent is +ready. OpenVMM exports every share as a child of one aggregate, which the guest +mounts at `/run/nvx/hostfs/root`, where only root can enter, and the kernel command line announces only the number of mappings, as `nvx_maps=COUNT`. Once the guest agent answers, `start` sends it the mapping table, in order, in as many @@ -1127,17 +1138,16 @@ requests as it takes. The agent mounts each entry at its target with its mode, and creates a missing target only on the container overlay, never inside another mapping, as a directory or, for a shared file, a regular file. It refuses every workload until it has mounted the whole table. A mapping that -the guest refuses or cannot mount fails `start`, after the guest shuts down as -`stop` has it do, so the sandbox never runs with part of its shares; `start` -terminates OpenVMM only if the guest does not shut down within `--timeout`. -The kernel command line does not bound the number of a managed sandbox's +the guest refuses or cannot mount fails `run` or `start`, after the guest +shuts down as `stop` has it do, so the sandbox never runs with part of its +shares; they terminate OpenVMM only if the guest does not shut down within +`--timeout`. The kernel command line does not bound the number of a sandbox's shares; the 256 children of one aggregate do. -A one-shot workload exit, a managed `stop`, and any failure after a share is -mounted unmount the binds or mappings in reverse order, then the aggregate or -the export, before the overlay is unmounted or the VM powers off, unless the -guest stops answering, or `start` is interrupted, and OpenVMM is terminated -first. Each target +A managed `stop`, and any failure after a share is mounted, unmount the binds +or mappings in reverse order, then the aggregate or the export, before the +overlay is unmounted or the VM powers off, unless the guest stops answering, or +`start` is interrupted, and OpenVMM is terminated first. Each target must be an absolute, canonical path; `/`, `/etc`, `/run`, and `/run/nvx`, and the `/etc/machine-id`, `/etc/resolv.conf`, `/proc`, `/sys`, `/dev`, `/.nvx-agent`, and `/run/nvx/time` trees, are reserved for the container @@ -1146,9 +1156,9 @@ runtime, which replaces `/etc/machine-id` with the workload's machine ID, keeps `/run/nvx`, and binds its time state over `/run/nvx/time`. The guest refuses a target whose path crosses a symbolic link in a container layer, a target that a container layer holds as another kind of file than the -share's, and overlapping targets, and a one-shot guest a repeated child. Any -validation or mount failure aborts a one-shot sandbox with status 125, and -fails a managed `start`, instead of starting the workload without its shares. +share's, and overlapping targets. Any validation or mount failure fails a +`run` or a managed `start`, instead of starting the workload without its +shares. An `rw` share supports the symbolic links that package managers and language toolchains create; see @@ -1173,11 +1183,15 @@ deprovision such a sandbox and provision it again. A share must not reach the sandbox's own files, which the workload could otherwise read or change: `provision` refuses a share whose resolved host path equals, contains, or lies inside the sandbox's state directory, its image, its -scratch image, or the OpenVMM, kernel, or initramfs of the artifact root, before -it records the sandbox, and `start` refuses one again before it launches the VM. +scratch image, or the OpenVMM, kernel, or initramfs of the artifact root, or +the OpenVMM that `NVX_OPENVMM` names, before it records the sandbox, and +`start` refuses one again before it launches the VM. `start` resolves the recorded host paths again, so a share that came to reach one of these files through a link since `provision` is refused too. A -`provision` that finds no artifact root leaves the artifacts to `start`. Only +`provision` that finds no artifact root leaves the artifacts to `start`. A +`run`, which has no state directory, refuses a share that reaches any of the +others, or the temporary VM directory in which it would launch its VM, before +it launches that VM. Only resolved paths are compared, so don't share a directory that holds a link through which the host reaches one of these files, nor a hard link or bind mount of one. @@ -1478,19 +1492,10 @@ it runs, a second sandbox, or a `run`, on the same image fails with the device and inode number, which a file that is created once the image is removed can take again: such a replacement, of the same length, is not refused. A sandbox whose upper is in RAM has no scratch image to bind or claim. -`run` and `start` hand OpenVMM the files that their checks opened, rather than -paths that something could replace meanwhile. On Linux, OpenVMM inherits their -descriptors, and its command line names them `/proc/self/fd/N`, as the command -that `run` prints shows; on Windows, its command line names them by their final -paths, `\\?\` paths that go through no link, and the checked files keep the -images from being replaced, renamed, or deleted, until the guest is ready for -`start`, and until OpenVMM exits for `run`. OpenVMM inherits the checked files -too, so that its copies keep the images in place even if `run` or `start` ends -before OpenVMM has opened them, as when it crashes, until OpenVMM exits; the -sandbox's next transition ends the OpenVMM of such a `start`. -There, a final path with a comma, semicolon, or unbalanced brackets, or ending -in `.vhd`, `.vhdx`, `.vmgs`, or `.iso`, which OpenVMM could not attach as a raw -image, fails the launch. Images are attached raw: stage an image in a container +`run` binds the scratch image itself, and checks the binding and claims the +image as `start` does. `run` and `start` then hand OpenVMM the paths of the +images that they checked, which OpenVMM opens itself: keep the images in place +while the VM uses them. Images are attached raw: stage an image in a container format, such as a VHD, as a raw file first. `provision` records the guest memory of `--memory-mib`, which it takes from 1 @@ -1589,18 +1594,17 @@ array requests an empty environment. `--inherit-default-environment` layers the supplied entries over that default environment instead of replacing it, each entry replacing the default variable of the same name; without supplied entries it has no effect. Inline values are visible in host process arguments -and should not be used for secrets. These options apply only to managed -`sandbox exec`; one-shot execution rejects them. nvx-init, the VM's init, runs -each managed exec as a plain process in a process group of its own, in the VM's -root and in nvx-init's mount, PID, and UTS namespaces and cgroup: the VM is the +and should not be used for secrets. These options apply to `sandbox exec`, +and to `sandbox run`, which executes its one workload as `exec` does. nvx-init, +the VM's init, runs each managed exec as a plain process in a process group of +its own, in the VM's root and in nvx-init's mount, PID, and UTS namespaces and +cgroup: the VM is the boundary of its workloads. Every mount of the root but `/dev` and its devpts instance is `nodev`. The workload sees one machine ID for the life of the VM. On `stop`, nvx-init syncs the file systems and powers the VM off with them mounted, and the next `start` replays the journal of the scratch file system. -The legacy -operation-less `sandbox` -form is `sandbox run`; it remains one-shot and rejects `--state-dir` or any -request to retain VM state. +The operation-less `sandbox` form is `sandbox run`, which keeps no state and +rejects `--state-dir`. Environment files are limited to 1 MiB of UTF-8 JSON. Environments contain at most 256 unique, nonempty names. Each `KEY=VALUE` entry and working-directory @@ -1643,10 +1647,10 @@ control initramfs, Ubuntu EROFS layer, and fresh copies of the same scratch template, so it has the same artifact requirements. It checks that: - `provision` rejects a root `--workload-user` before it creates any state, and - the guest refuses an identity that the Ubuntu image lacks before a one-shot - workload starts, and in a managed `start`, which then fails with + the guest refuses an identity that the Ubuntu image lacks before any + workload starts, in a run and in a managed `start`, which both then fail with `guest-exit status 125` from the outcome report that OpenVMM must publish - before it exits, and leaves only `config.json`, `sandbox.lock`, + before it exits, and the start leaves only `config.json`, `sandbox.lock`, `openvmm.log`, and that `outcome.json`, without a runtime record, capability, control socket, report staging file, or OpenVMM process; - every operation fails on a state directory that was never provisioned, a @@ -1700,20 +1704,22 @@ fixture for recovery only if that cleanup fails. Its evidence, written to the output directory with a `sandbox-lifecycle-` prefix, holds bounded command observations (arguments, statuses, output sizes, and state-directory entries), the runtime record, the OpenVMM logs, the VM-level and exec outcome reports, the -guest probe's transcript, and the console of the refused one-shot run, each -limited to its last 64 KiB. The command observations hold no workload output, +guest probe's transcript, and the output of the refused run, each limited to +its last 64 KiB. The command observations hold no workload output, and no evidence holds the control capability. Decoder, helper, and direct control-session tests remain useful supplemental coverage for protocol boundaries and guest implementation details. They do not replace or establish support through the public `nvx-rs sandbox` interface. -`run --outcome-report PATH` and one-shot `sandbox run --outcome-report PATH` -forward OpenVMM's bounded local JSON report. Managed `sandbox exec` writes only -the operation, bounded result category, numeric status, and an opaque operation -ID to its requested report; stdout, stderr, arguments, environment values, and -credentials remain excluded. `sandbox stop` waits for OpenVMM teardown and -retains the latest VM-level report as `outcome.json` in the state directory. +`run --outcome-report PATH` and `sandbox run --outcome-report PATH` forward +OpenVMM's bounded local JSON report, with the permissions that OpenVMM gave it: +that of the VM's stop, for `sandbox run`, whose exit status is its workload's. +Managed `sandbox exec` writes only the operation, bounded result category, +numeric status, and an opaque operation ID to its requested report; stdout, +stderr, arguments, environment values, and credentials remain excluded. +`sandbox stop` waits for OpenVMM teardown and retains the latest VM-level report +as `outcome.json` in the state directory. On Linux, NVX starts OpenVMM with the caller's umask without the owner's bits, so OpenVMM's reports keep their owner's read and write permissions whatever that umask, unless their directory has a default POSIX ACL, which Linux diff --git a/doc/usage.md b/doc/usage.md index 9be8779a4..01746cffe 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -301,27 +301,27 @@ do, and their names cannot be abbreviated either. | Option | Default | Description | | --- | --- | --- | -| `{run,provision,start,exec,stop,deprovision}` | `run` | Select a one-shot run or a managed lifecycle operation. | -| `--layer ROLE,PATH,IMAGE_UUID` | required for `run` and `provision` | Attach exactly one `distro` image by path. `run` takes EROFS; `provision` takes the format of `--layer-format`. The filesystem UUID or GPT disk GUID must be lowercase hexadecimal in canonical hyphenated form; braces, prefixes, uppercase, and unhyphenated forms are refused. | -| `--layer-format {erofs,gpt}` | `erofs` | Select provision's image format explicitly, without probing the file or inferring its format from its name. Other sandbox commands reject this option. | +| `{run,provision,start,exec,stop,deprovision}` | `run` | Select a run, which keeps no state, or a managed lifecycle operation. | +| `--layer ROLE,PATH,IMAGE_UUID` | required for `run` and `provision` | Attach exactly one `distro` image by path, of the format of `--layer-format`. The filesystem UUID or GPT disk GUID must be lowercase hexadecimal in canonical hyphenated form; braces, prefixes, uppercase, and unhyphenated forms are refused. | +| `--layer-format {erofs,gpt}` | `erofs` | Select the image format of `run` or `provision` explicitly, without probing the file or inferring its format from its name. Other sandbox operations reject this option. | | `--scratch PATH` | required for `run` and `provision` with a scratch upper | Attach a preformatted ext4 scratch image as the overlay's writable upper. `--upper ram` refuses it. | -| `--upper {scratch,ram}` | `scratch` | Select the overlay's writable upper for `run` or `provision`: `scratch`, the image of `--scratch`, or `ram`, a tmpfs in guest memory with no scratch image, which loses the workload's writes when the VM stops. The default RAM cap is half of guest memory; provision can set it with `--ram-upper-percent`. See [Overlay upper](run.md#overlay-upper). | -| `--ram-upper-percent PERCENT` | `50` | Set provision's RAM upper cap to an ASCII decimal integer from 1 through 90 percent of VM memory. Requires `--upper ram`; other sandbox commands reject this option. The typed configuration records the cap, image format and path, and upper kind, which each start reads unchanged. Older configuration formats are refused rather than migrated. | -| `--state-dir PATH` | required for managed operations | Select persistent sandbox state. One-shot `run` rejects this option. | -| `--entrypoint PATH` | `/bin/sh` | Select an absolute workload entrypoint without whitespace, control characters, double quotes, or any UTF-8 byte that Linux treats as whitespace. | -| `--arg VALUE` | none | Append one nonempty entrypoint argument without whitespace, control characters, double quotes, or any UTF-8 byte that Linux treats as whitespace. Repeat to pass multiple arguments. | +| `--upper {scratch,ram}` | `scratch` | Select the overlay's writable upper for `run` or `provision`: `scratch`, the image of `--scratch`, or `ram`, a tmpfs in guest memory with no scratch image, which loses the workload's writes when the VM stops. The default RAM cap is half of guest memory, which `--ram-upper-percent` changes. See [Overlay upper](run.md#overlay-upper). | +| `--ram-upper-percent PERCENT` | `50` | Set the RAM upper's cap of `run` or `provision` to an ASCII decimal integer from 1 through 90 percent of VM memory. Requires `--upper ram`; other sandbox operations reject this option. The typed configuration of `provision` records the cap, image format and path, and upper kind, which each start reads unchanged. Older configuration formats are refused rather than migrated. | +| `--state-dir PATH` | required for managed operations | Select persistent sandbox state. `run`, which keeps no state, rejects this option. | +| `--entrypoint PATH` | `/bin/sh` | Select the absolute entrypoint of the workload of `run` or `exec`, which runs it over the control protocol. `provision`, which runs no workload, refuses one with whitespace, control characters, double quotes, or any UTF-8 byte that Linux treats as whitespace. | +| `--arg VALUE` | none | Append one nonempty argument of the workload of `run` or `exec`. Repeat to pass multiple arguments. `provision` refuses one with whitespace, control characters, double quotes, or any UTF-8 byte that Linux treats as whitespace. | | `--hostname NAME` | `nvx-sandbox` | Set the workload UTS hostname. | | `--workload-user UID:GID` | `65534:65534` | Select the fixed non-root workload identity for `run` or `provision`. Both IDs must be ASCII decimal integers from 1 to 4294967294, without signs, whitespace, or underscores. | | `--memory-mib MIB` | `256` | Set guest memory in MiB, an ASCII decimal integer from 1 to 17592186044415, without signs, whitespace, or underscores. | -| `--timeout SECONDS` | `60` | Set the timeout for each managed control request and its response in `start`, `exec`, and `stop`, and the longest time that `provision`, `start`, `stop`, and `deprovision` wait for another lifecycle transition of the sandbox. It is a decimal number, with an optional fraction and exponent, that must be finite and greater than 0. | +| `--timeout SECONDS` | `60` | Set the timeout for each managed control request and its response in `run`, `start`, `exec`, and `stop`, the longest time that `run` and `stop` wait for the VM to exit once the guest shuts it down, and the longest time that `provision`, `start`, `stop`, and `deprovision` wait for another lifecycle transition of the sandbox. It is a decimal number, with an optional fraction and exponent, that must be finite and greater than 0. | | `--wait SECONDS` | `60` | Set the longest time that a managed `exec` waits for the `exec` that another `nvx-rs` process runs in the sandbox to end, since a sandbox runs one `exec` at a time; it then fails with `sandbox-busy`. `stop` never waits for an `exec`. Like `--timeout`, which still bounds each wait for another lifecycle transition, it is a decimal number that must be finite and greater than 0. | -| `--exec-timeout-ms MILLISECONDS` | `0` | Set the managed `exec` guest workload timeout to an ASCII decimal integer from 0 to 4294967295, without signs, whitespace, or underscores; zero disables the workload deadline. This is separate from the finite host `--timeout` control deadline. | -| `--cwd GUEST_PATH` | `/` | Set an absolute working directory inside the workload root for managed `exec`. It is resolved after the workload identity and root are applied; missing, inaccessible, or non-directory paths fail the workload launch. Default and layered environments point `PWD` at this directory; an exact replacement environment controls `PWD` itself. | -| `--environment KEY=VALUE` | omitted | Set the exact managed `exec` environment. Repeat for multiple entries. Empty values, spaces, additional equals signs, and UTF-8 are preserved. Inline values are visible in the invoking host process arguments; use `--environment-file` for sensitive values. | -| `--environment-file PATH` | omitted | Read the exact managed `exec` environment from a strict UTF-8 JSON array of `KEY=VALUE` strings, limited to 1 MiB of input. Invalid JSON, non-finite numbers, and unpaired surrogate escapes are refused. An empty array requests an empty environment. This option is mutually exclusive with `--environment`; omitting both preserves guest defaults. | -| `--inherit-default-environment` | off | Layer the managed `exec` environment from `--environment` or `--environment-file` over the guest default environment instead of replacing it; an entry replaces the default variable of the same name. Without either option the workload already receives the defaults, so the flag has no effect. | +| `--exec-timeout-ms MILLISECONDS` | `0` | Set the guest's timeout of the workload of `run` or `exec` to an ASCII decimal integer from 0 to 4294967295, without signs, whitespace, or underscores; zero disables the workload deadline. This is separate from the finite host `--timeout` control deadline. | +| `--cwd GUEST_PATH` | `/` | Set an absolute working directory inside the workload root for `run` or `exec`. It is resolved after the workload identity and root are applied; missing, inaccessible, or non-directory paths fail the workload launch. Default and layered environments point `PWD` at this directory; an exact replacement environment controls `PWD` itself. | +| `--environment KEY=VALUE` | omitted | Set the exact environment of the workload of `run` or `exec`. Repeat for multiple entries. Empty values, spaces, additional equals signs, and UTF-8 are preserved. Inline values are visible in the invoking host process arguments; use `--environment-file` for sensitive values. | +| `--environment-file PATH` | omitted | Read the exact environment of the workload of `run` or `exec` from a strict UTF-8 JSON array of `KEY=VALUE` strings, limited to 1 MiB of input. Invalid JSON, non-finite numbers, and unpaired surrogate escapes are refused. An empty array requests an empty environment. This option is mutually exclusive with `--environment`; omitting both preserves guest defaults. | +| `--inherit-default-environment` | off | Layer the environment of the workload of `run` or `exec` from `--environment` or `--environment-file` over the guest default environment instead of replacing it; an entry replaces the default variable of the same name. Without either option the workload already receives the defaults, so the flag has no effect. | | `--hypervisor {auto,whp,kvm,mshv}` | `auto` | Select the host hypervisor. | -| `--mount GUEST_TARGET,HOST_PATH[,ro\|rw]` | none | Live-share a host directory, or a single regular host file without its directory, at the absolute target inside the container rootfs for `run` or `provision`; defaults to `ro`. The target cannot contain whitespace, control characters, double quotes, backslashes, `=`, or any UTF-8 byte that Linux treats as whitespace, such as the `0xa0` in `à`, so that it stays one kernel command-line token in the shares of a one-shot `run`. The mode follows the last comma, so `HOST_PATH` may contain commas when the mode is given. Repeat to attach more shares, each with its own target and mode, for example a read-write workspace and a read-only tool cache, as the children of an aggregate that the guest binds at their targets; a shared file always uses the aggregate, and so does every share of a managed sandbox. Targets and host paths must not overlap, and `provision` and `start` refuse a host path that equals, contains, or lies inside the sandbox's state directory, image, scratch image, OpenVMM, kernel, or initramfs; see [Live host shares](run.md#live-host-shares). A one-shot `run` describes its shares in kernel command-line tokens that count toward the sandbox's 1024-byte budget; a managed sandbox announces only their number and sends their mapping table once its guest agent is ready, for at most 256 shares. An `rw` share accepts guest-created symbolic links, which the host never follows. `/`, `/etc`, `/run`, and `/run/nvx`, and the `/etc/machine-id`, `/etc/resolv.conf`, `/proc`, `/sys`, `/dev`, `/.nvx-agent`, and `/run/nvx/time` trees, are reserved. | +| `--mount GUEST_TARGET,HOST_PATH[,ro\|rw]` | none | Live-share a host directory, or a single regular host file without its directory, at the absolute target inside the container rootfs for `run` or `provision`; defaults to `ro`. The target cannot contain whitespace, control characters, double quotes, backslashes, `=`, or any UTF-8 byte that Linux treats as whitespace, such as the `0xa0` in `à`. The mode follows the last comma, so `HOST_PATH` may contain commas when the mode is given. Repeat to attach more shares, each with its own target and mode, for example a read-write workspace and a read-only tool cache, as the children of an aggregate that the guest binds at their targets. Targets and host paths must not overlap, and `run`, `provision`, and `start` refuse a host path that equals, contains, or lies inside the sandbox's image, scratch image, OpenVMM, whether the artifact root's or the one that `NVX_OPENVMM` names, kernel, or initramfs, the state directory of `provision` and `start`, or the temporary VM directory of `run`; see [Live host shares](run.md#live-host-shares). The kernel command line announces only the number of the shares, whose mapping table the sandbox sends once its guest agent is ready, for at most 256 shares. An `rw` share accepts guest-created symbolic links, which the host never follows. `/`, `/etc`, `/run`, and `/run/nvx`, and the `/etc/machine-id`, `/etc/resolv.conf`, `/proc`, `/sys`, `/dev`, `/.nvx-agent`, and `/run/nvx/time` trees, are reserved. | | `--mount-deny HOST_PATH` | none | Hide one existing file or directory inside a `--mount` host directory; relative paths are resolved inside it. It may name the directory itself when `--mount-allow` paths inside it remain. With several shares, it applies to the `--mount` before it, which must share a directory. Repeat to deny multiple paths. | | `--mount-allow HOST_PATH` | none | Expose one existing file or directory inside a `--mount-deny` path of the same share again, for example a readable `mcp-payloads` directory inside hidden logs. The hidden directories on the way list only the entries that lead to it, and the workload cannot modify them. Relative paths are resolved inside the share; with several shares, it applies to the `--mount` before it. Repeat to allow multiple paths. See [Access policy](run.md#access-policy). | | `--mount-write HOST_PATH` | none | Make one existing file or directory one of the only writable parts of an `rw` share; the workload's writes anywhere else in the share fail with `EROFS`. Relative paths are resolved inside the share; with several shares, it applies to the `--mount` before it, which must be `rw`. Repeat to declare multiple writable paths. `provision` persists the denied, allowed, and writable paths. | @@ -336,15 +336,18 @@ do, and their names cannot be abbreviated either. | `--host-loopback {allow,deny}` | guest to host | Control access between the workload and host loopback services for `run` or `provision`. Without the option, the workload reaches host loopback through its gateway, subject to the egress policy, and no port is published; `deny` closes both directions. Explicit `allow` is bidirectional: the workload reaches host loopback through its gateway, and every TCP and UDP port on which the workload listens is published on host loopback for the host alone, or with `--network-ingress allow` admits the host as well; see [Host-loopback access](run.md#host-loopback-access). | | `--network-proxy IPV4:TCP-PORT` | none | Make the network proxy-only: the workload reaches the HTTP or HTTPS proxy that listens on host `127.0.0.1:TCP-PORT` at `IPV4:TCP-PORT`, where `IPV4` must be the guest gateway, and nothing else. `TCP-PORT` can't be 53, where OpenVMM answers DNS at the gateway. Requires `--network-egress deny` and `--host-loopback deny`, and refuses egress rules, `--network-ingress allow`, and forwards. NVX gives the workload no proxy settings; see [Proxy-only networking](run.md#proxy-only-networking). | | `--host-loopback-forward PROTOCOL:HOST_PORT:GUEST_PORT` | none | Publish one TCP or UDP localhost port to the guest; repeat to add forwards. Requires `--host-loopback allow`. A UDP forward also requires egress allowed by default and no deny rule that covers the guest network, which carries its replies; OpenVMM lets the replies on a forwarded TCP connection through whatever the egress policy. | -| `--outcome-report PATH` | none | Write a bounded local JSON outcome report for one-shot `run` or managed `exec`. | +| `--outcome-report PATH` | none | Write a bounded local JSON outcome report: for `run`, OpenVMM's report of its VM, which the guest stopped after the workload; for `exec`, the workload's outcome. | | `--cmdline TEXT` | empty | Append non-sandbox kernel parameters, which `nvx-rs` reads as Linux does: split at whitespace, which includes the `0xa0` in `à`, outside double quotes, with names that match whether they spell a dash or an underscore. Quoted values, such as `a="b c"`, keep their whitespace. Parameters whose names start with `nvx_` or `virtfs_` are reserved and refused, even inside a quoted value, and so is a `--` or unclosed quote that hides the sandbox's own tokens; see [Experimental single-workload sandbox](run.md#experimental-single-workload-sandbox). The guest powers off if its kernel applies a clock parameter that the time ABI forbids, such as `tsc=unstable`. | -| `--dry-run` | off | For one-shot `run` only, print the generated OpenVMM microVM command without running it. Managed operations reject this option. | +| `--dry-run` | off | For `run` only, print the generated OpenVMM microVM command without launching it. Other operations reject this option. | `run` and `start` install the release of a Git checkout that holds no -artifacts before they launch, as `nvx-rs run` does. A one-shot `run` runs -OpenVMM with the standard streams, and attaches the guest's console, which -carries its workload's output, to them through the boot console, as a cold -boot of `nvx-rs run` does. +artifacts before they launch, as `nvx-rs run` does. A `run` keeps no state: +it launches the VM that a `provision` and a `start` of its options would +launch, in a temporary VM directory that it removes, executes its workload +once, as `exec` does, has the guest shut the VM down, which it terminates if +it does not exit within `--timeout` seconds, and then prints the workload's +standard output and error and exits with its status; see +[Experimental single-workload sandbox](run.md#experimental-single-workload-sandbox). See [Run](run.md) for artifact preparation, the security boundary, and current snapshot/configuration limitations. diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 4bf4ea8c0..6f89a0218 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -1651,8 +1651,7 @@ refusing one whose root tokens the guest's parser refuses. Both deliver the live shares as the managed lifecycle does: the arguments export them at `nvx::host::filesystem::GUEST_EXPORT`, the command line announces the number of their mappings with `nvx_maps=`, and `mapping_table` gives the ordered table -that `start` sends once the guest agent is ready; the CLI's one-shot `run` still -renders them as boot tokens. +that `start` and `sandbox run` send once the guest agent is ready. An adapter's plan renders the caller's tokens before those of its root, and refuses tokens that its root or OpenVMM's shares own. @@ -2369,9 +2368,9 @@ another name, `start` checks the manifest beside the link's target, so that the check accepts there. `start` checks the binding before it launches OpenVMM, and hands OpenVMM the checked layer; see [Checked handoff](#checked-handoff). `run` and `start` -plan the launch with the image format and the identity that the binding -records, which name an EROFS layer, and launch it over a scratch disk, the one -layered root that the guest's shell init assembles. A configuration +launch the image of the format and the identity that the binding records, an +EROFS layer or a GPT disk, over a scratch or a RAM upper, the layered root that +nvx-init assembles. A configuration without a binding, as `nvx.py` wrote, fails with `sandbox configuration has no distro binding`, and a stale binding with what changed, such as `LAYER changed after its distro binding was made: its length differs`; both add `deprovision @@ -2484,9 +2483,8 @@ exited. A scratch image that changed fails as a distro layer does, such as with `IMAGE changed after its scratch binding was made: its length differs; deprovision the sandbox and provision it again`, and one that another sandbox claims with `another sandbox claims the ext4 scratch image IMAGE`. `run` binds -and claims the scratch image itself, and runs OpenVMM through a prepared -launch until it exits; its command, which it prints, names the checked files, -except in a dry run, which launches nothing. A sandbox whose upper is in RAM, +the scratch image itself, checks the binding and claims the image as `start` +does, and holds the claim until its VM exits. A sandbox whose upper is in RAM, `--upper ram`, has no scratch image: `provision` records `null` for the image and its binding, and neither `start` nor `run` binds or claims one. @@ -2740,7 +2738,7 @@ at a time installs into a checkout, holding the lock of its holder exits. A command shares that lock, creating its file if needed and if it may, while it decides, reads the artifacts, and launches OpenVMM on them, `sandbox run` from before it reads its distro layer, which may be a release's, -and `sandbox start` until its VM is ready. It thus never sees part of an +and `sandbox start`, until their VM is ready. It thus never sees part of an installation, no download replaces the files that it reads, and a download that it waited for has it decide again. `sandbox provision` shares the lock where it can take it while it reads its distro layer and records the sandbox, @@ -2833,16 +2831,19 @@ as `--option VALUE` or `--option=VALUE`, repeated options accumulate, and `--mount-deny`, `--mount-allow`, and `--mount-write` apply to the `--mount` before them. -- `run` prints `>> COMMAND`, OpenVMM's command line, quoted as the Microsoft C - runtime splits a command line on Windows and as a POSIX shell splits one - elsewhere, as `nvx.py` quoted it with `subprocess.list2cmdline` and - `shlex.join`. Unless `--dry-run` is given, it then runs OpenVMM with the - standard streams, to which `--virtio-console console` attaches the guest's - boot console, and exits with its status; the command then names the - checked distro layer, and the scratch image of an upper on scratch, as - `/proc/self/fd/N` on Linux, which OpenVMM inherits, and by their final paths - on Windows, as [Checked handoff](#checked-handoff) describes. - `--outcome-report` asks OpenVMM for its outcome report. +- `run` runs one workload in a sandbox of its own, which keeps no state + directory: it launches the VM that a `provision` and a `start` of its + options would launch, in a fresh temporary VM directory, and prints + `>> COMMAND`, OpenVMM's command line, quoted as the Microsoft C runtime + splits a command line on Windows and as a POSIX shell splits one elsewhere, + as `nvx.py` quoted it with `subprocess.list2cmdline` and `shlex.join`. + Unless `--dry-run` is given, it then executes the workload once through the + control session, as `exec` does, with its working directory and environment + options, asks the guest to shut the VM down, waits up to `--timeout` seconds + for OpenVMM to exit before it terminates it, removes the VM directory, and + prints the workload's standard output and then its standard error, and exits + with its status. `--outcome-report` forwards OpenVMM's outcome report of the + VM's stop. - `provision` records the sandbox in its state directory. `start` launches OpenVMM detached with the managed lifecycle, passes it a fresh capability through its standard input, and waits until the guest answers. `exec` runs a @@ -2946,11 +2947,12 @@ switch too, since this build finds no VM in that sandbox. #### Guest features -After the guest answers, `start` asks it for its control features and -requires `GuestFeatures::LAYERED_SANDBOX`, the features that its launches use: -per-execution environments (bit 4), since `exec` sends a working directory and -an environment, and host path mapping tables (bit 6), since `start` then sends -the mapping table of the sandbox's shares, which the guest agent mounts before +After the guest answers, `start`, and `sandbox run`, ask it for its control +features and require `GuestFeatures::LAYERED_SANDBOX`, the features that their +launches use: per-execution environments (bit 4), since `exec` and +`sandbox run` send a working directory and an environment, and host path +mapping tables (bit 6), since they then send the mapping table of the +sandbox's shares, which the guest agent mounts before it runs any workload; a mapping that it refuses fails the start, after the guest shuts down as `stop` has it do. It needs neither cancellation (bit 0), since the CLI never cancels a workload, and Ctrl-C ends the session, nor @@ -2967,8 +2969,8 @@ guest agents of NVX advertise bits 0, 4, and 6 in sandbox mode. #### Other differences from nvx.py -- The cold boots of `run`, and `sandbox run`, attach the guest's console to the - terminal through the boot console, with `--virtio-console console`, where +- The cold boots of `run` attach the guest's console to the terminal through + the boot console, with `--virtio-console console`, where `nvx.py` left it on the byte channel, `hvc0`, which took the terminal's input. The byte channel's output goes to standard error. - Long options cannot be abbreviated, where `argparse` accepted any prefix that @@ -3071,15 +3073,9 @@ guest agents of NVX advertise bits 0, 4, and 6 in sandbox mode. way out. The command spells out OpenVMM's defaults and the rules in their canonical form, with the network options after the plan's other options; see [Sandbox networking](../doc/run.md#sandbox-networking). -- `run` and `start` hand OpenVMM the files that their checks opened: its - command line names them `/proc/self/fd/N` on Linux, and by their final - paths, `\\?\` paths, on Windows, and on every host, a sandbox's scratch image - is claimed while it runs, so a second sandbox, or a `run`, on the same image - fails; see [Checked handoff](#checked-handoff). OpenVMM inherits no other - descriptor or handle of `run` beyond its standard streams, as `subprocess.run` - passes none with `close_fds`: on Windows, OpenVMM's handle list holds copies - of `run`'s standard handles, its checked files, and the scratch image's claim - alone. +- A sandbox's scratch image is claimed while its VM runs, so a second + sandbox, or a `run`, on the same image fails; see + [Checked handoff](#checked-handoff). - On Linux, OpenVMM starts with the caller's umask without the owner's bits, where `nvx.py` passed the umask unchanged, so that a umask that clears the owner's own permissions, such as 0400, no longer keeps the caller, or `start` @@ -3156,8 +3152,9 @@ and the arguments that OpenVMM receives, which the fake OpenVMM records, and into which the `>> ` line of a run splits back, with `CommandLineToArgvW` on Windows and a shell-word splitter on POSIX, as a property test checks for arbitrary kernel command lines too. The tables cover usage errors and help, the -checks that refuse a command line before anything changes, dry runs, one-shot -runs, the lifecycles of sandboxes of every configuration, their executions, the +checks that refuse a command line before anything changes, dry runs, the runs +of `nvx-rs run` and of `nvx-rs sandbox run`, the lifecycles of sandboxes of +every configuration, their executions, the refusals of lifecycle operations, configurations of every format, from the hand-written `config.json` files of `tests/fixtures/state/`, `egress compile`, `artifacts`, and `download`, with the downloads of `run` and `sandbox`, which diff --git a/nvx-rs/src/host/cli/sandbox/lifecycle/mod.rs b/nvx-rs/src/host/cli/sandbox/lifecycle/mod.rs index 4c3a632ca..664647302 100644 --- a/nvx-rs/src/host/cli/sandbox/lifecycle/mod.rs +++ b/nvx-rs/src/host/cli/sandbox/lifecycle/mod.rs @@ -8,6 +8,9 @@ //! Exec and stop reattach with runtime and process verification. Stop requests NVXC shutdown, //! waits for exit, then forces only the verified process. Deprovision removes only this stopped //! sandbox's owned files. No running sandbox from the old runtime store is migrated. +//! +//! A run launches the VM that a provision and a start would launch, in a temporary VM directory, +//! without a state directory, executes one workload over NVXC, and stops it the way stop does. mod directory; mod outcome; @@ -39,7 +42,7 @@ pub(crate) use self::directory::{create_state_directory, prepare_state_directory pub(crate) use self::outcome::{ copy_openvmm_outcome, validate_outcome_destination, write_exec_outcome, }; -pub(crate) use self::start::{exec_workload, start}; +pub(crate) use self::start::{exec_workload, run, start}; pub(crate) use self::stop::{deprovision, stop}; /// The launch configuration, the CLI's own record beside those of the store. diff --git a/nvx-rs/src/host/cli/sandbox/lifecycle/start.rs b/nvx-rs/src/host/cli/sandbox/lifecycle/start.rs index 14bbeda69..daf35c872 100644 --- a/nvx-rs/src/host/cli/sandbox/lifecycle/start.rs +++ b/nvx-rs/src/host/cli/sandbox/lifecycle/start.rs @@ -1,25 +1,38 @@ -//! Starting a managed sandbox and executing workloads in its running VM. +//! Starting a managed sandbox and executing workloads in its running VM, and running one workload +//! in a VM of its own, without a state directory. +use std::fs; +use std::io; use std::path::Path; use std::time::Instant; use serde_json::Value; use super::directory::prepare_state_directory; +use super::outcome::{copy_openvmm_outcome, validate_outcome_destination}; use super::{ - CONFIG_NAME, Found, REQUIRED_FEATURES, connect, find_vm, mismatched, require_config, - running_vm, runtime_for, + CONFIG_NAME, Found, ProvisionConfig, REQUIRED_FEATURES, connect, find_vm, mismatched, + require_config, running_vm, runtime_for, }; use crate::host::cli::artifacts::locate; use crate::host::cli::deadline::{Deadline, RETRY_INTERVAL}; +use crate::host::cli::download::Hold; use crate::host::cli::exec_lock::ExecLock; use crate::host::cli::sandbox::lock::LifecycleLock; -use crate::host::cli::sandbox::state::{Config, read_config}; -use crate::host::cli::sandbox::{Artifacts, error_with_cause, refuse_own_files}; +use crate::host::cli::sandbox::state::{ + Config, RecordedDistro, RecordedScratch, read_config, serialize_launch, +}; +use crate::host::cli::sandbox::{ + Artifacts, Options, error_with_cause, hypervisor, path_string, refuse_own_files, +}; use crate::host::cli::session::{ExecRequest, ExecResult, Session}; -use crate::host::cli::{Exit, Result, output}; +use crate::host::cli::shell::format_command; +use crate::host::cli::store::StateFile; +use crate::host::cli::{Exit, Result, hex, io_error, output, random_bytes}; use crate::host::process::KILL_TIMEOUT; -use crate::host::runtime::Runtime; +use crate::host::report::Report; +use crate::host::runtime::{ArtifactIdentity, Runtime, RuntimeIdentity}; +use crate::host::sandbox::SandboxLaunch; use crate::host::sandbox::distro::DISTRO_POLICY_REVISION; use crate::host::sandbox::scratch::CLAIM_RELEASE_WAIT; use crate::host::vm::{self, SavedVm, VmDir}; @@ -64,7 +77,11 @@ fn start_locked( } let config = Config::new(config)?; let launch = config.launch()?; - refuse_own_files(state_dir, &launch, Some(&locate(artifacts.root)?.dir))?; + refuse_own_files( + Some(("state directory", state_dir)), + &launch, + Some(&locate(artifacts.root)?), + )?; let binding = config.distro_binding(); let scratch_binding = config.scratch_binding()?; let distro = binding @@ -157,21 +174,27 @@ fn start_locked( drop((distro, scratch)); let report = stopped.report.map_err(error_with_cause)?; if stopped.self_exited && !stopping && !matches!(error, Exit::Interrupted) { - let detail = match report { - Some(report) => match (report.category, report.status_code) { - (Some(category), Some(status)) => format!(": {category} status {status}"), - _ => String::new(), - }, - None => " without an outcome report".to_owned(), - }; - return Err(Exit::error( - ErrorCode::Exited, - format!("OpenVMM exited during startup{detail}"), - )); + return Err(exited_during_startup(report)); } Err(error) } +/// Returns the error of a launch whose OpenVMM exited on its own before its guest was ready, with +/// the category and status of `report`, its outcome report, if it holds them. +fn exited_during_startup(report: Option) -> Exit { + let detail = match report { + Some(report) => match (report.category, report.status_code) { + (Some(category), Some(status)) => format!(": {category} status {status}"), + _ => String::new(), + }, + None => " without an outcome report".to_owned(), + }; + Exit::error( + ErrorCode::Exited, + format!("OpenVMM exited during startup{detail}"), + ) +} + /// Returns the error of a start whose distro layer or scratch image fails the check of its /// binding: a binding that no longer holds needs a new provision, which verifies the layer and /// binds the image again, while a scratch image that another sandbox claims, and a failure of the @@ -252,3 +275,262 @@ pub(crate) fn exec_workload( let mut session = connect(&running.saved, artifacts, response_timeout)?; session.exec(request, response_timeout) } + +/// Runs one workload in a VM of its own and stops the VM, as `sandbox run` does, and returns how +/// the workload ended, or nothing for a dry run, which launches nothing. +/// +/// The VM is the one that `start` launches for the configuration that `provision` records for +/// `launch`, whose distro layer `distro` is at the path that `provision` records, with the options +/// of `options`: the run serializes that configuration and reads it back as `start` does, then +/// launches it as `start` does, but in a temporary VM directory rather than a state directory. +/// Before it downloads or launches anything, it refuses what `start` refuses of the configuration +/// and `exec` of `request`, after a missing artifact; it then refuses the shares that `start` +/// refuses, with its VM directory in place of a state directory, and those of the OpenVMM that it +/// runs, and prints the OpenVMM command as `>> COMMAND`. +/// +/// Once nvx-init is ready and has mapped the shares, the run executes `request` over NVXC, asks +/// nvx-init to stop the VM, and waits up to `--timeout` seconds for the VM to exit before it +/// forces it down. A VM that exits before it is ready fails the run as it fails `start`. Once the +/// VM has exited, the run forwards OpenVMM's outcome report to `--outcome-report`, if given, and +/// removes the VM directory, which it keeps only if it cannot confirm that exit. It holds `held`, +/// the lock of the downloads, until the VM is ready or has failed to be. +pub(crate) fn run( + launch: &SandboxLaunch, + distro: &RecordedDistro, + artifacts: &Artifacts<'_>, + options: &Options, + request: &ExecRequest, + held: Hold, +) -> Result> { + let hypervisor = hypervisor(options.hypervisor); + let provision = ProvisionConfig { + hypervisor, + memory_mib: options.memory_mib, + cmdline: &options.cmdline, + ram_upper_percent: options.ram_upper_percent, + }; + let recorded = record(launch, distro, &provision).and_then(|(config, launch)| { + // As `start` does, refuse the user's own nvx_* tokens, and the command line that the + // launch would refuse, such as one that holds double quotes. + launch + .bound_plan(config.distro_binding())? + .kernel_command_line(config.cmdline())?; + let vm_config = config.vm_config(&launch)?; + vm_config.check_command_line().map_err(error_with_cause)?; + Session::exec_payload(request)?; + Ok((config, launch, vm_config)) + }); + // A run that its options refuse downloads nothing, and reports a missing artifact first. + let hold = match recorded { + Ok(_) => artifacts.ensure_under(held, hypervisor)?, + Err(_) => held, + }; + artifacts.require()?; + let (config, launch, vm_config) = recorded?; + let name = hex(&random_bytes::<16>()?); + let path = std::env::temp_dir().join(format!("nvx-sandbox-run-{name}")); + refuse_own_files( + Some(("VM directory", &path)), + &launch, + Some(&locate(artifacts.root)?), + )?; + if !options.dry_run + && let Some(report) = &options.outcome_report + { + validate_outcome_destination(report)?; + } + // A run never records or reattaches its VM, so it names no runtime identity of its artifacts. + let identity = ArtifactIdentity::new(format!("unrecorded-run:{name}"))?; + let runtime = runtime_for( + artifacts, + &config, + RuntimeIdentity::new(identity.clone(), identity.clone(), identity), + )?; + let images = if options.dry_run { + None + } else { + let distro = config + .distro_binding() + .check(launch.distro(), DISTRO_POLICY_REVISION) + .map_err(error_with_cause)?; + let scratch = match (config.scratch_binding()?, launch.scratch()) { + (Some(binding), Some(scratch)) => Some( + binding + .check(scratch, Instant::now() + CLAIM_RELEASE_WAIT) + .map_err(error_with_cause)?, + ), + _ => None, + }; + Some((distro, scratch)) + }; + if path.try_exists().map_err(Exit::from_os_error)? { + return Err(Exit::error( + ErrorCode::Io, + "the temporary VM directory already exists", + )); + } + vm::create_private_dir(&path).map_err(error_with_cause)?; + let dir = VmDir::open(&path).map_err(error_with_cause)?; + let timeout = options.timeout; + let mut cleanup = true; + let result = (|| { + let mut prepared = vm::create(&runtime, &vm_config, None, &dir) + .map_err(error_with_cause)? + .for_foreground(); + let command = std::iter::once(path_string(&runtime.openvmm)) + .chain(prepared.arguments().iter().map(|argument| { + argument.to_str().map(str::to_owned).ok_or_else(|| { + Exit::error(ErrorCode::Unsupported, "an OpenVMM argument is not Unicode") + }) + })) + .collect::>>()?; + output::print_stdout(&format!(">> {}", format_command(&command))) + .map_err(Exit::from_os_error)?; + let Some((_, scratch)) = &images else { + return Ok(None); + }; + if let Some(scratch) = scratch { + #[cfg(not(windows))] + prepared.inherit(scratch.file().try_clone().map_err(Exit::from_os_error)?); + #[cfg(windows)] + prepared.inherit(scratch.claim().try_clone().map_err(Exit::from_os_error)?); + } + let launched = vm::spawn(prepared).map_err(|error| { + cleanup = error.kind() != ErrorKind::StillRunning; + error_with_cause(error) + })?; + cleanup = false; + let saved = SavedVm { + vm_dir: dir.clone(), + endpoint: launched.outputs.endpoint.clone(), + capability: *launched.capability.as_bytes(), + process: launched.process.reference().clone(), + runtime: runtime.identity.clone(), + }; + let mut session = None; + let ready = check_ready(&saved, &runtime, timeout).and_then(|ready| { + let maps = vm_config.mapping_table(); + let session = session.insert(ready); + if maps.is_empty() { + Ok(()) + } else { + session.map_host_paths(&maps, timeout) + } + }); + drop(hold); + let started = ready.is_ok(); + let executed = ready.and_then(|()| { + session + .as_mut() + .expect("a ready VM has a session") + .exec(request, timeout) + }); + // A VM whose control endpoint closed is shutting down already. + let requested = match session.take() { + Some(mut session) + if !matches!(executed, Err(Exit::Interrupted | Exit::EndpointClosed)) => + { + Some(session.stop(timeout)) + } + _ => None, + }; + // So is one whose endpoint closed while it was asked to stop: it gets a stop's grace. + let stopping = matches!(requested, Some(Ok(()) | Err(Exit::EndpointClosed))); + let grace = if stopping || matches!(executed, Err(Exit::EndpointClosed)) { + Deadline::after(timeout) + } else { + Deadline::after(0.0) + }; + let mut executed = executed; + // No record of the VM outlives the run, so it forces the VM down whatever ended the wait. + if let Err(error) = super::stop::wait_for_exit(&launched.process, grace) { + executed = Err(error); + } + let now = Instant::now(); + let stopped = vm::stop( + &launched.process, + &launched.outputs, + |_| false, + now, + now + KILL_TIMEOUT, + ) + .map_err(error_with_cause)?; + cleanup = true; + if let Some(report) = &options.outcome_report + && let Err(error) = copy_openvmm_outcome(&launched.outputs.outcome, report) + { + error.report(); + } + // Ctrl-C during the request ends the run as interrupted, which is no failed shutdown. + if let Some(Err(error)) = &requested + && !matches!(error, Exit::Interrupted) + { + output::print_stderr(&format!( + "warning: managed guest shutdown failed: {error:?}" + )); + } + if !started { + let report = stopped.report.map_err(error_with_cause)?; + if stopped.self_exited && !stopping && !matches!(executed, Err(Exit::Interrupted)) { + return Err(exited_during_startup(report)); + } + } else if executed.is_ok() && stopped.forced { + output::print_stderr("warning: OpenVMM was forcibly terminated after the run"); + } + executed.map(Some) + })(); + if cleanup && let Err(error) = remove_vm_directory(&dir) { + output::print_stderr(&format!( + "warning: cannot remove the temporary VM directory {}: {error:?}", + dir.path().display() + )); + } + result +} + +/// Returns the configuration that `provision` records for `launch`, whose distro layer `distro` is +/// at the path that `provision` records, with the options of `config`, as `start` reads it back, +/// and the launch that `start` makes of it, which checks its shares again. +fn record( + launch: &SandboxLaunch, + distro: &RecordedDistro, + config: &ProvisionConfig<'_>, +) -> Result<(Config, SandboxLaunch)> { + let scratch = RecordedScratch::new(launch)?; + let record = serialize_launch(launch, distro, scratch.as_ref(), config)?; + let config = Config::new( + serde_json::from_str(&record) + .map_err(|error| Exit::error(ErrorCode::Other, error.to_string()))?, + )?; + let launch = config.launch()?; + Ok((config, launch)) +} + +/// Removes the temporary VM directory of a run, with what its VM left in it: OpenVMM's log and +/// outcome report, its control socket on Linux, and the staging file of a report that OpenVMM +/// did not finish writing, which [`StateFile::owns_temporary`] names. +fn remove_vm_directory(dir: &VmDir) -> Result { + let remove = |path: &Path| match fs::remove_file(path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(io_error(ErrorCode::Io, &error, path)), + }; + for file in [StateFile::Log, StateFile::Outcome, StateFile::Socket] { + remove(&dir.path().join(file.name()))?; + } + let failed = |error: io::Error| io_error(ErrorCode::Io, &error, dir.path()); + for entry in fs::read_dir(dir.path()).map_err(failed)? { + let entry = entry.map_err(failed)?; + if entry + .file_name() + .to_str() + .is_some_and(StateFile::owns_temporary) + { + remove(&entry.path())?; + } + } + fs::remove_dir(dir.path()).map_err(failed) +} + +#[cfg(test)] +mod tests; diff --git a/nvx-rs/src/host/cli/sandbox/lifecycle/start/tests.rs b/nvx-rs/src/host/cli/sandbox/lifecycle/start/tests.rs new file mode 100644 index 000000000..d585117a1 --- /dev/null +++ b/nvx-rs/src/host/cli/sandbox/lifecycle/start/tests.rs @@ -0,0 +1,98 @@ +//! Tests of the configuration that a run launches, and of the removal of its VM directory. + +use std::fs; + +use super::*; +use crate::host::paths::resolve; +use crate::host::sandbox::layer::{LayerRole, SandboxLayer}; +use crate::host::sandbox::mount::{plan_mounts, validate_mounts}; +use crate::host::sandbox::options::{sandbox_mounts, validate_sandbox_systemd_policy}; +use crate::host::sandbox::{SandboxSettings, SandboxUpper}; + +#[test] +fn run_checks_again_that_its_shares_do_not_overlap() { + let directory = tempfile::tempdir().unwrap(); + let root = resolve(directory.path()).unwrap(); + let path = |name: &str| root.join(name).to_str().unwrap().to_owned(); + for (name, content) in [ + ("distro.erofs", "distro"), + ("scratch.ext4", "scratch"), + ("settings.json", "{}"), + ("output.log", ""), + ] { + fs::write(path(name), content).unwrap(); + } + let mounts = sandbox_mounts( + &[ + format!("/etc/settings.json,{}", path("settings.json")), + format!("/var/log/output.log,{},rw", path("output.log")), + ], + &[], + None, + ) + .unwrap(); + validate_mounts(&mounts).unwrap(); + let launch = SandboxLaunch::new( + vec![SandboxLayer::new( + LayerRole::Distro, + &path("distro.erofs"), + "11111111-1111-1111-1111-111111111111", + )], + SandboxUpper::scratch(&path("scratch.ext4")), + SandboxSettings::default(), + ) + .unwrap() + .with_export(plan_mounts(&mounts).unwrap()) + .unwrap(); + let binding = validate_sandbox_systemd_policy(&launch).unwrap(); + let distro = RecordedDistro::new(&launch, binding).unwrap(); + let config = ProvisionConfig { + hypervisor: "kvm", + memory_mib: 256, + cmdline: "", + ram_upper_percent: None, + }; + record(&launch, &distro, &config).unwrap(); + + // A hard link that replaced the read-only file with the writable one while the run verified + // its distro layer would export one file under both access modes; the export plan's own + // checks pass, since a file share's root is not pinned. + fs::remove_file(path("settings.json")).unwrap(); + fs::hard_link(path("output.log"), path("settings.json")).unwrap(); + launch.export().unwrap().verify().unwrap(); + let Err(error) = record(&launch, &distro, &config) else { + panic!("the run launched shares that came to overlap"); + }; + assert_eq!( + error.code(), + Some(crate::ErrorCode::Unsupported), + "{error:?}" + ); + let message = error.message().unwrap_or_default(); + assert!( + message.contains("name the same directory or file"), + "{message}" + ); +} + +/// A run removes its VM directory with the staging file of an outcome report that OpenVMM did +/// not finish writing, as one that the run forced down can leave, and keeps a directory that +/// holds a file of another name. +#[test] +fn a_run_removes_its_vm_directory_with_a_report_s_staging_file() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("vm"); + for (staging, removed) in [(".tmpA1b2C3", true), ("other", false)] { + vm::create_private_dir(&path).unwrap(); + for name in [StateFile::Log.name(), StateFile::Outcome.name(), staging] { + fs::write(path.join(name), "").unwrap(); + } + let result = remove_vm_directory(&VmDir::open(&path).unwrap()); + assert_eq!(result.is_ok(), removed, "{staging}: {result:?}"); + assert_eq!(path.exists(), !removed, "{staging}"); + if !removed { + assert!(path.join(staging).exists(), "{staging}"); + fs::remove_dir_all(&path).unwrap(); + } + } +} diff --git a/nvx-rs/src/host/cli/sandbox/mod.rs b/nvx-rs/src/host/cli/sandbox/mod.rs index 768fbc456..a42943898 100644 --- a/nvx-rs/src/host/cli/sandbox/mod.rs +++ b/nvx-rs/src/host/cli/sandbox/mod.rs @@ -2,13 +2,12 @@ //! `sandbox` parser of `scripts/nvx.py`, with its options, defaults, checks, outputs, and exit //! statuses. //! -//! - `run`, the default, boots a one-shot sandbox: it prints the OpenVMM command as `>> COMMAND`, -//! then runs it on the terminal, unless `--dry-run`, and exits with OpenVMM's status. It runs -//! OpenVMM through a [`PreparedLaunch`] of the distro layer that it verified and, for an upper -//! on scratch, of the scratch image, which it claims, so the command names their checked files: -//! on Linux as `/proc/self/fd/N`, and on Windows by their final paths. A dry run, which launches -//! nothing, names their paths. `--upper ram` keeps the upper in RAM, on a tmpfs that the guest -//! mounts, and attaches no scratch image. +//! - `run`, the default, runs one workload in a sandbox of its own, without a state directory: it +//! launches the VM that a `provision` and a `start` of the same options would launch, in a +//! temporary VM directory, prints its OpenVMM command as `>> COMMAND`, executes the workload +//! once over NVXC, as `exec` would, with `exec`'s options, stops the VM, and exits with the +//! workload's status, after its output. A dry run prints the command and launches nothing. See +//! [`lifecycle::run`]. //! - `provision`, `start`, `exec`, `stop`, and `deprovision` manage a persistent sandbox in its //! `--state-dir`; see [`lifecycle`]. //! @@ -23,24 +22,20 @@ mod options; mod state; use std::path::{Path, PathBuf}; -use std::time::Instant; use super::artifacts::{Located, OPENVMM_VARIABLE, locate}; use super::download::{self, Hold}; -use super::shell::format_command; +use super::session::ExecRequest; use super::{Exit, Result, io_error, output}; use crate::ErrorCode; -use crate::host::artifacts::{ArtifactDir, Guest, KERNEL_NAME, require_artifact}; +use crate::host::artifacts::{Guest, KERNEL_NAME, require_artifact}; use crate::host::paths; -use crate::host::sandbox::distro::{DISTRO_POLICY_REVISION, DistroBinding, DistroFormat}; +use crate::host::sandbox::distro::DistroFormat; use crate::host::sandbox::mount::{plan_mounts, validate_mounts}; use crate::host::sandbox::options::{ SandboxOperation, SandboxOptions, sandbox_mounts, validate_sandbox_systemd_policy, }; -use crate::host::sandbox::scratch::{CLAIM_RELEASE_WAIT, ScratchBinding}; -use crate::host::sandbox::{ - PreparedLaunch, SandboxLaunch, SandboxPlan, SandboxSettings, SandboxUpper, -}; +use crate::host::sandbox::{SandboxLaunch, SandboxSettings, SandboxUpper}; use crate::host::store::RuntimeIdentity; use crate::protocol::root::RootUpper; @@ -139,12 +134,14 @@ pub(crate) fn error_with_cause(error: crate::Error) -> Exit { /// Runs the operation of `options`, as `command_sandbox` did. pub(super) fn run(root: Option<&str>, options: Options) -> Result { let operation = options.operation; - if !matches!(operation, SandboxOperation::Provision) - && (options.layer_format.is_some() || options.ram_upper_percent.is_some()) + if !matches!( + operation, + SandboxOperation::Run | SandboxOperation::Provision + ) && (options.layer_format.is_some() || options.ram_upper_percent.is_some()) { return Err(Exit::error( ErrorCode::Unsupported, - "--layer-format and --ram-upper-percent are only supported by provision", + "--layer-format and --ram-upper-percent are only valid for sandbox run or provision", )); } if options.ram_upper_percent.is_some() && options.upper != Some(RootUpper::Ram) { @@ -174,12 +171,12 @@ pub(super) fn run(root: Option<&str>, options: Options) -> Result { }; let environment = checks.check()?; let artifacts = Artifacts { root }; - // A one-shot run holds the lock of the downloads into a checkout from before it reads the - // distro layer, which may be a release's, until it has launched OpenVMM, so that no download - // replaces the layer or the artifacts meanwhile. It reports a failure to take it where it - // decides to download, after its options. A provision holds it while it reads the layer and - // records the sandbox, where it can take it: it runs no artifact, so it needs no artifact root. - // Ctrl-C ends the wait for a download, and the command with it. + // A run holds the lock of the downloads into a checkout from before it reads the distro + // layer, which may be a release's, until its VM is ready, so that no download replaces the + // layer or the artifacts meanwhile. It reports a failure to take it where it decides to + // download, after its options. A provision holds it while it reads the layer and records the + // sandbox, where it can take it: it runs no artifact, so it needs no artifact root. Ctrl-C + // ends the wait for a download, and the command with it. let early = match operation { SandboxOperation::Run | SandboxOperation::Provision => match artifacts.hold() { Err(Exit::Interrupted) => return Err(Exit::Interrupted), @@ -194,13 +191,19 @@ pub(super) fn run(root: Option<&str>, options: Options) -> Result { ) { let rules = options.network.egress_rules()?; let mounts = sandbox_mounts(&options.mounts, &options.policy_paths, options.mount_owner)?; - let settings = SandboxSettings { + let mut settings = SandboxSettings { entrypoint: options.entrypoint.clone(), args: options.args.clone(), hostname: options.hostname.clone(), workload_identity: options.workload_user, ..SandboxSettings::default() }; + // A run's workload is its exec request, which no boot token carries, and which its control + // session checks. + if operation == SandboxOperation::Run { + let workload = SandboxSettings::default(); + (settings.entrypoint, settings.args) = (workload.entrypoint, workload.args); + } // The checks above require --scratch for an upper on scratch, and refuse it for one in // RAM. let upper = match options.upper.unwrap_or(RootUpper::Scratch) { @@ -246,6 +249,16 @@ pub(super) fn run(root: Option<&str>, options: Options) -> Result { ) }) }; + // The workload that `exec` runs, or the one that `run` runs once. + let mut arguments = vec![options.entrypoint.clone()]; + arguments.extend(options.args.iter().cloned()); + let request = ExecRequest { + arguments, + timeout_ms: options.exec_timeout_ms, + cwd: options.cwd.clone(), + environment, + inherit_default_environment: options.inherit_default_environment, + }; match (operation, launch) { (SandboxOperation::Provision, Some((launch, binding))) => { // `provision` records the layer at its resolved path, where `start` checks the @@ -256,9 +269,9 @@ pub(super) fn run(root: Option<&str>, options: Options) -> Result { // checks the shares against the artifacts that it runs. let located = locate(root).ok(); refuse_own_files( - Path::new(state_dir), + Some(("state directory", Path::new(state_dir))), &launch, - located.as_ref().map(|located| &located.dir), + located.as_ref(), )?; lifecycle::provision( state_dir, @@ -281,18 +294,10 @@ pub(super) fn run(root: Option<&str>, options: Options) -> Result { if let Some(report) = &options.outcome_report { lifecycle::validate_outcome_destination(report)?; } - let mut arguments = vec![options.entrypoint.clone()]; - arguments.extend(options.args.iter().cloned()); let result = lifecycle::exec_workload( state_dir, &artifacts, - &super::session::ExecRequest { - arguments, - timeout_ms: options.exec_timeout_ms, - cwd: options.cwd.clone(), - environment, - inherit_default_environment: options.inherit_default_environment, - }, + &request, options.timeout, options.wait, )?; @@ -318,77 +323,21 @@ pub(super) fn run(root: Option<&str>, options: Options) -> Result { if options.state_dir.is_some() { return Err(Exit::error( ErrorCode::Unsupported, - "one-shot sandbox execution rejects persistent --state-dir settings", + "sandbox run keeps no state: it rejects --state-dir", )); } - let plan = run_plan(&launch, &binding)?; - // The shares and the command line depend on the options alone, and a missing artifact - // is reported before them, as before, but a run that they refuse downloads nothing. - let refused = plan.one_shot_openvmm_arguments().is_err() - || plan.one_shot_kernel_command_line(&options.cmdline).is_err(); + // A run launches the layer at the path that a provision would record. + let distro = RecordedDistro::new(&launch, binding)?; let held = early.transpose()?.unwrap_or_else(Hold::none); - let hold = if refused { - held - } else { - artifacts.ensure_under(held, hypervisor(options.hypervisor))? - }; - let [openvmm, kernel, initrd] = artifacts.require()?; - // A one-shot run delivers the shares as boot tokens. A dry run launches nothing, so it - // shows the plan's own paths; a run launches through the checked files, which it holds - // until OpenVMM exits. - let (arguments, command_line, prepared) = if options.dry_run { - ( - plan.one_shot_openvmm_arguments()?, - plan.one_shot_kernel_command_line(&options.cmdline)?, - None, - ) - } else { - let prepared = prepare_run(&launch, &binding, plan, &options.cmdline)?; - ( - prepared.openvmm_arguments().to_vec(), - prepared.kernel_command_line().to_owned(), - Some(prepared), - ) - }; - let mut command = vec![openvmm]; - command.extend(arguments); - // The boot console carries the guest's console, with the workload's output, to the - // terminal. - command.extend( - [ - "--microvm-lifecycle", - "one-shot", - "--virtio-console", - "console", - "--single-process", - "--hypervisor", - hypervisor(options.hypervisor), - ] - .map(str::to_owned), - ); - command.extend([ - "--memory".to_owned(), - format!("{}M", options.memory_mib), - "--kernel".to_owned(), - kernel, - "--initrd".to_owned(), - initrd, - "--cmdline".to_owned(), - command_line, - ]); - if let Some(report) = &options.outcome_report { - command.extend(["--microvm-report".to_owned(), path_text(report)]); - } - output::print_stdout(&format!(">> {}", format_command(&command))) - .map_err(Exit::from_os_error)?; - let Some(prepared) = prepared else { + let Some(result) = + lifecycle::run(&launch, &distro, &artifacts, &options, &request, held)? + else { return Ok(()); }; - Err(Exit::Status(run_openvmm( - &command, - &prepared.inherited(), - hold, - )?)) + output::write_stdout(&result.stdout) + .and_then(|()| output::write_stderr(&result.stderr)) + .map_err(Exit::from_os_error)?; + Err(Exit::Status(result.returncode)) } (SandboxOperation::Run | SandboxOperation::Provision, None) => { unreachable!("run and provision build their launch") @@ -397,34 +346,39 @@ pub(super) fn run(root: Option<&str>, options: Options) -> Result { } /// Refuses a share of the sandbox's own files: one whose host path equals, contains, or lies -/// inside the state directory `state_dir`, the image or scratch image of `launch`, or the OpenVMM, -/// kernel, or Alpine initramfs of `artifacts`, if known. Through it, the guest could read or -/// change the sandbox's records, its storage, or what the host runs. +/// inside `directory`, if the sandbox has one, with its name, its state directory or a run's VM +/// directory; the image or scratch image of `launch`; or, if known, the OpenVMM, kernel, or Alpine +/// initramfs of the artifact root of `artifacts`, or the OpenVMM that `NVX_OPENVMM` names instead, +/// which a run runs. Through it, the guest could read or change the sandbox's records, its +/// storage, or what the host runs. /// /// The paths are compared once [`paths::resolve`] has resolved them, without a Windows verbatim /// prefix. Resolution follows symbolic links and junctions, but not hard links or bind mounts. /// `provision` checks before it records the sandbox, and `start` again before it launches it, -/// since the shares' paths may have come to resolve elsewhere. +/// since the shares' paths may have come to resolve elsewhere; `run`, which has no state +/// directory, checks before it creates its VM directory and launches its VM. fn refuse_own_files( - state_dir: &Path, + directory: Option<(&str, &Path)>, launch: &SandboxLaunch, - artifacts: Option<&ArtifactDir>, + artifacts: Option<&Located>, ) -> Result { let Some(export) = launch.export() else { return Ok(()); }; - let mut owned = vec![ - ("state directory", state_dir.to_owned()), - ("image", launch.distro().path().to_owned()), - ]; + let mut owned: Vec<_> = directory + .map(|(name, path)| (name, path.to_owned())) + .into_iter() + .collect(); + owned.push(("image", launch.distro().path().to_owned())); owned.extend( launch .scratch() .map(|path| ("scratch image", path.to_owned())), ); - if let Some(dir) = artifacts { + if let Some(Located { dir, openvmm }) = artifacts { owned.extend([ ("OpenVMM", dir.openvmm()), + ("OpenVMM", openvmm.clone()), ("kernel", dir.kernel()), ("initramfs", dir.initramfs(Guest::Alpine)), ]); @@ -469,42 +423,5 @@ fn without_verbatim_prefix(path: PathBuf) -> PathBuf { } } -/// Checks the launch of a one-shot run again, and returns its plan with the format and identity of -/// the distro layer that `binding`, which the run's verification of the layer made, records. -/// -/// The verification read all of the layer after the shares were planned, so the launch must still -/// pass [`SandboxLaunch::validated`], as `provision` checks it before recording it and `start` -/// checks a recorded one: the directories and files of its export plan must still not overlap, by -/// path or through a hard link or bind mount, and must pass [`ExportPlan::verify`]. -/// -/// [`ExportPlan::verify`]: crate::host::filesystem::ExportPlan::verify -fn run_plan(launch: &SandboxLaunch, binding: &DistroBinding) -> Result { - launch.validated()?; - launch.bound_plan(binding).map_err(Exit::from) -} - -/// Checks the distro layer of a one-shot run against `binding`, which the run's verification of -/// the layer made, binds and claims the scratch image of an upper on scratch, and prepares the -/// one-shot launch of `plan`, which delivers the shares as boot tokens, with the checked files and -/// the user's kernel command line `cmdline`. -/// -/// The checks read no content, so the layer that the run verified is the one that OpenVMM gets: -/// one that changed meanwhile fails its check. -fn prepare_run( - launch: &SandboxLaunch, - binding: &DistroBinding, - plan: SandboxPlan, - cmdline: &str, -) -> Result { - let distro = binding.check(launch.distro(), DISTRO_POLICY_REVISION)?; - let scratch = match launch.scratch() { - Some(scratch) => Some( - ScratchBinding::bind(scratch)?.check(scratch, Instant::now() + CLAIM_RELEASE_WAIT)?, - ), - None => None, - }; - PreparedLaunch::one_shot(plan, distro, scratch, cmdline).map_err(Exit::from) -} - #[cfg(test)] pub(super) mod tests; diff --git a/nvx-rs/src/host/cli/sandbox/options.rs b/nvx-rs/src/host/cli/sandbox/options.rs index edc637213..dd2b706c6 100644 --- a/nvx-rs/src/host/cli/sandbox/options.rs +++ b/nvx-rs/src/host/cli/sandbox/options.rs @@ -33,16 +33,18 @@ options: -h, --help show this help message and exit --layer ROLE,PATH,IMAGE_UUID --layer-format {erofs,gpt} - provision's image format (default: erofs); IMAGE_UUID - is the filesystem UUID or GPT disk GUID + the image format of run and provision (default: + erofs); IMAGE_UUID is the filesystem UUID or GPT disk + GUID --scratch SCRATCH --upper {scratch,ram} where the overlay keeps the sandbox's writes: scratch (default), on the ext4 image of --scratch, or ram, on a tmpfs in guest memory, without a scratch image --ram-upper-percent PERCENT - provision's RAM upper cap, from 1 to 90 percent of - guest memory (default: 50); requires --upper ram + the RAM upper's cap of run and provision, from 1 to 90 + percent of guest memory (default: 50); requires + --upper ram --state-dir STATE_DIR --entrypoint ENTRYPOINT --arg SANDBOX_ARG @@ -59,17 +61,18 @@ options: --exec-timeout-ms EXEC_TIMEOUT_MS guest workload timeout in milliseconds; zero disables it - --cwd CWD absolute guest working directory for managed exec - (default: /) + --cwd CWD absolute guest working directory of the workload of run + or exec (default: /) --environment KEY=VALUE - set the exact managed exec environment; repeat for - multiple entries + set the exact environment of the workload of run or + exec; repeat for multiple entries --environment-file PATH - read the exact managed exec environment from a JSON - string array + read the exact environment of the workload of run or + exec from a JSON string array --inherit-default-environment - layer the managed exec environment over the guest - default environment instead of replacing it + layer the environment of the workload of run or exec + over the guest default environment instead of + replacing it --hypervisor {auto,whp,kvm,mshv} --mount GUEST_TARGET,HOST_PATH[,ro|rw] live-share a host directory, or a single regular host @@ -119,14 +122,14 @@ options: write a bounded local JSON outcome report for run or exec --cmdline CMDLINE - --dry-run print the one-shot run command without launching it", + --dry-run print the OpenVMM command of run without launching it", }; /// The operations, in the order in which usage messages list them. const OPERATIONS: [&str; 6] = ["run", "provision", "start", "exec", "stop", "deprovision"]; /// The choices of `--upper`, the names of the uppers. const UPPERS: [&str; 2] = [RootUpper::Scratch.as_str(), RootUpper::Ram.as_str()]; -/// The image formats that provision accepts. +/// The image formats that run and provision accept. const LAYER_FORMATS: [&str; 2] = ["erofs", "gpt"]; /// The choices of `--hypervisor`. pub(in crate::host::cli) const HYPERVISORS: [&str; 4] = ["auto", "whp", "kvm", "mshv"]; @@ -195,7 +198,7 @@ pub(crate) struct Options { pub(crate) operation: SandboxOperation, /// `--layer`, parsed as `SandboxLayer.parse` parses it. pub(crate) layers: Vec, - /// `--layer-format`, if given; provision defaults to EROFS. + /// `--layer-format`, if given; run and provision default to EROFS. pub(crate) layer_format: Option, /// `--scratch`, as given. pub(crate) scratch: Option, diff --git a/nvx-rs/src/host/cli/sandbox/tests.rs b/nvx-rs/src/host/cli/sandbox/tests.rs index 76e293fe0..ed0feeca6 100644 --- a/nvx-rs/src/host/cli/sandbox/tests.rs +++ b/nvx-rs/src/host/cli/sandbox/tests.rs @@ -1,13 +1,9 @@ -//! Tests of the checks of a one-shot run. - -use std::fs; +//! Tests of the options of `nvx-rs sandbox`. use proptest::prelude::*; pub(in crate::host::cli) use super::options::{OPTIONS, SYNTAX}; use super::*; -use crate::host::paths::resolve; -use crate::host::sandbox::layer::{LayerRole, SandboxLayer}; use crate::host::sandbox::mount::PolicyKind; use crate::host::sandbox::options::PolicyPathOption; @@ -56,8 +52,8 @@ fn the_upper_is_scratch_or_ram() { } #[test] -fn provision_options_do_not_change_other_commands() { - for operation in ["run", "start", "exec", "stop", "deprovision"] { +fn image_options_are_only_valid_for_run_and_provision() { + for operation in ["start", "exec", "stop", "deprovision"] { for flag in ["--layer-format", "--ram-upper-percent"] { let value = if flag == "--layer-format" { "erofs" @@ -71,23 +67,25 @@ fn provision_options_do_not_change_other_commands() { error .message() .unwrap() - .contains("only supported by provision") + .contains("only valid for sandbox run or provision") ); } } - let (options, _) = parse(&[ - "provision".to_owned(), - "--ram-upper-percent".to_owned(), - "50".to_owned(), - ]) - .unwrap(); - assert!( - run(None, options) - .unwrap_err() - .message() - .unwrap() - .contains("requires --upper ram") - ); + for operation in ["run", "provision"] { + let (options, _) = parse(&[ + operation.to_owned(), + "--ram-upper-percent".to_owned(), + "50".to_owned(), + ]) + .unwrap(); + assert!( + run(None, options) + .unwrap_err() + .message() + .unwrap() + .contains("requires --upper ram") + ); + } for (name, format) in [("erofs", DistroFormat::Erofs), ("gpt", DistroFormat::Gpt)] { let (options, _) = parse(&["--layer-format".to_owned(), name.to_owned()]).unwrap(); assert_eq!(options.layer_format, Some(format)); @@ -127,60 +125,3 @@ proptest! { prop_assert_eq!(options.policy_paths, expected); } } - -#[test] -fn run_checks_again_that_its_shares_do_not_overlap() { - let directory = tempfile::tempdir().unwrap(); - let root = resolve(directory.path()).unwrap(); - let path = |name: &str| root.join(name).to_str().unwrap().to_owned(); - for (name, content) in [ - ("distro.erofs", "distro"), - ("scratch.ext4", "scratch"), - ("settings.json", "{}"), - ("output.log", ""), - ] { - fs::write(path(name), content).unwrap(); - } - let mounts = sandbox_mounts( - &[ - format!("/etc/settings.json,{}", path("settings.json")), - format!("/var/log/output.log,{},rw", path("output.log")), - ], - &[], - None, - ) - .unwrap(); - validate_mounts(&mounts).unwrap(); - let launch = SandboxLaunch::new( - vec![SandboxLayer::new( - LayerRole::Distro, - &path("distro.erofs"), - "11111111-1111-1111-1111-111111111111", - )], - SandboxUpper::scratch(&path("scratch.ext4")), - SandboxSettings::default(), - ) - .unwrap() - .with_export(plan_mounts(&mounts).unwrap()) - .unwrap(); - let binding = validate_sandbox_systemd_policy(&launch).unwrap(); - run_plan(&launch, &binding).unwrap(); - - // A hard link that replaced the read-only file with the writable one while the run verified - // its distro layer would export one file under both access modes; the export plan's own - // checks pass, since a file share's root is not pinned. - fs::remove_file(path("settings.json")).unwrap(); - fs::hard_link(path("output.log"), path("settings.json")).unwrap(); - launch.export().unwrap().verify().unwrap(); - let error = run_plan(&launch, &binding).unwrap_err(); - assert_eq!( - error.code(), - Some(crate::ErrorCode::Unsupported), - "{error:?}" - ); - let message = error.message().unwrap_or_default(); - assert!( - message.contains("name the same directory or file"), - "{message}" - ); -} diff --git a/nvx-rs/src/host/cli/session.rs b/nvx-rs/src/host/cli/session.rs index ef819c211..991f91e14 100644 --- a/nvx-rs/src/host/cli/session.rs +++ b/nvx-rs/src/host/cli/session.rs @@ -610,19 +610,11 @@ impl Session { Ok(()) } - /// Runs a workload and returns how it ended, as `ControlSession.exec` does: the request and - /// every response must arrive within `response_timeout` seconds of each other's start. - pub(crate) fn exec( - &mut self, - request: &ExecRequest, - response_timeout: f64, - ) -> Result { + /// Checks the workload of `request` and returns the payload of its exec request, as + /// [`exec`](Self::exec) sends it, so that a caller can refuse a request before it starts a + /// VM. + pub(crate) fn exec_payload(request: &ExecRequest) -> Result> { let refused = |message: &str| Exit::error(ErrorCode::Unsupported, message); - if !(response_timeout > 0.0 && response_timeout < f64::INFINITY) { - return Err(refused( - "managed exec response timeout must be positive and finite", - )); - } let arguments = &request.arguments; if !(1..=MAX_ARGUMENTS).contains(&arguments.len()) { return Err(refused("managed exec requires 1 through 64 arguments")); @@ -665,8 +657,24 @@ impl Session { environment, }; // The checks above leave only the size of the request for the encoder to refuse. - let payload = encode_exec_payload(&workload) - .map_err(|_| refused("managed exec request exceeds the protocol limit"))?; + encode_exec_payload(&workload) + .map_err(|_| refused("managed exec request exceeds the protocol limit")) + } + + /// Runs a workload and returns how it ended, as `ControlSession.exec` does: the request and + /// every response must arrive within `response_timeout` seconds of each other's start. + pub(crate) fn exec( + &mut self, + request: &ExecRequest, + response_timeout: f64, + ) -> Result { + if !(response_timeout > 0.0 && response_timeout < f64::INFINITY) { + return Err(Exit::error( + ErrorCode::Unsupported, + "managed exec response timeout must be positive and finite", + )); + } + let payload = Self::exec_payload(request)?; let request_id = request_id()?; let deadline = Deadline::after(response_timeout); self.send(APP_EXEC, request_id, &payload, deadline)?; diff --git a/nvx-rs/src/host/sandbox/options.rs b/nvx-rs/src/host/sandbox/options.rs index b26ae643f..1c18be835 100644 --- a/nvx-rs/src/host/sandbox/options.rs +++ b/nvx-rs/src/host/sandbox/options.rs @@ -39,7 +39,8 @@ pub const MAX_EXEC_ENVIRONMENT_BYTES: usize = 4096; /// An operation of `nvx-rs sandbox`. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] pub enum SandboxOperation { - /// `run`: a one-shot sandbox, the default. + /// `run`: a sandbox that runs one managed execution and stops, without a state directory, the + /// default. Run, /// `provision`: record a managed sandbox's launch in its state directory. Provision, @@ -70,6 +71,11 @@ impl SandboxOperation { fn launches(self) -> bool { matches!(self, Self::Run | Self::Provision) } + + /// Returns whether the operation runs a managed execution: `run` or `exec`. + fn executes(self) -> bool { + matches!(self, Self::Run | Self::Exec) + } } impl fmt::Display for SandboxOperation { @@ -157,10 +163,10 @@ impl SandboxOptions { } /// Applies the checks of `nvx-rs sandbox` that precede the launch, in its order, and - /// returns the managed exec environment of an `exec`, if one was given. + /// returns the managed exec environment of a `run` or `exec`, if one was given. /// /// It refuses `--dry-run` except for `run`, both `--environment` and `--environment-file`, - /// the managed exec options except for `exec`, an `exec` environment that + /// the managed exec options except for `run` and `exec`, an environment that /// [`exec_environment_from_file`] or [`validate_exec_environment`] refuses, network policy /// options except for `run` and `provision`, a systemd entrypoint for `run`, `provision`, /// and `exec`, `--outcome-report` except for `run` and `exec`, share options without a @@ -180,18 +186,18 @@ impl SandboxOptions { "--environment and --environment-file are mutually exclusive", )); } - if operation != SandboxOperation::Exec + if !operation.executes() && (self.cwd.is_some() || !self.environment.is_empty() || self.environment_file.is_some() || self.inherit_default_environment) { return Err(refused( - "managed execution options require the sandbox exec operation", + "managed execution options are only valid for sandbox run or exec", )); } let mut environment = None; - if operation == SandboxOperation::Exec { + if operation.executes() { if let Some(file) = &self.environment_file { environment = Some(exec_environment_from_file(file)?); } else if !self.environment.is_empty() { @@ -210,11 +216,9 @@ impl SandboxOptions { ) { validate_sandbox_entrypoint(&self.entrypoint)?; } - if self.outcome_report - && !matches!(operation, SandboxOperation::Run | SandboxOperation::Exec) - { + if self.outcome_report && !operation.executes() { return Err(refused( - "--outcome-report is only valid for one-shot run or managed exec", + "--outcome-report is only valid for sandbox run or exec", )); } for kind in POLICY_KINDS { diff --git a/nvx-rs/src/host/sandbox/options/tests.rs b/nvx-rs/src/host/sandbox/options/tests.rs index 1e7589014..111bc9f4b 100644 --- a/nvx-rs/src/host/sandbox/options/tests.rs +++ b/nvx-rs/src/host/sandbox/options/tests.rs @@ -11,7 +11,7 @@ use super::*; use crate::ErrorKind; use crate::host::paths::{self, resolve}; use crate::host::sandbox::layer::LayerRole; -use crate::host::sandbox::mount::{aggregate_child_name, plan_mounts, validate_mounts}; +use crate::host::sandbox::mount::{plan_mounts, validate_mounts}; use crate::host::sandbox::{SandboxLayer, SandboxSettings, SandboxUpper}; fn message(result: Result) -> String { @@ -606,7 +606,7 @@ fn sandbox_exec_forwards_environment_inheritance() { } #[test] -fn sandbox_exec_rejects_ambiguous_or_one_shot_environment() { +fn sandbox_run_and_exec_alone_take_the_execution_options() { let both = SandboxOptions { environment: vec!["A=1".to_owned()], environment_file: Some("environment.json".to_owned()), @@ -616,13 +616,26 @@ fn sandbox_exec_rejects_ambiguous_or_one_shot_environment() { message(both.check()), "--environment and --environment-file are mutually exclusive" ); - let one_shot = SandboxOptions { + // A run executes its workload as exec does. + let run = SandboxOptions { environment: vec!["A=1".to_owned()], - ..SandboxOptions::new(SandboxOperation::Run) + cwd: Some("/tmp".to_owned()), + inherit_default_environment: true, + ..launching(SandboxOperation::Run) }; - let exec_only = "managed execution options require the sandbox exec operation"; - assert_eq!(message(one_shot.check()), exec_only); - for operation in [SandboxOperation::Run, SandboxOperation::Start] { + assert_eq!(run.check().unwrap(), Some(vec!["A=1".to_owned()])); + let exec_only = "managed execution options are only valid for sandbox run or exec"; + for operation in [ + SandboxOperation::Provision, + SandboxOperation::Start, + SandboxOperation::Stop, + SandboxOperation::Deprovision, + ] { + let environment = SandboxOptions { + environment: vec!["A=1".to_owned()], + ..SandboxOptions::new(operation) + }; + assert_eq!(message(environment.check()), exec_only); let inheriting = SandboxOptions { inherit_default_environment: true, ..SandboxOptions::new(operation) @@ -784,7 +797,7 @@ fn managed_non_launch_operations_reject_network_policy_options() { }; assert_eq!( message(report.check()), - "--outcome-report is only valid for one-shot run or managed exec" + "--outcome-report is only valid for sandbox run or exec" ); } @@ -920,8 +933,9 @@ impl Shares { path } - /// Runs the steps of `nvx-rs sandbox --dry-run` that build the OpenVMM arguments and the - /// kernel command line of these shares. + /// Runs the steps of `nvx-rs sandbox --dry-run` that build the launch of these shares, and + /// returns the OpenVMM arguments and the kernel command line of its plan, which exports the + /// shares for the mapping table that the host sends once the guest is ready. fn run( &self, mounts: &[String], @@ -959,10 +973,9 @@ impl Shares { launch.validated()?; let launch = launch.with_export(plan_mounts(&mounts)?)?; validate_sandbox_systemd_policy(&launch)?; - // The command runs one-shot, which renders the shares as boot tokens. Ok(( - launch.plan().one_shot_openvmm_arguments()?, - launch.plan().one_shot_kernel_command_line("")?, + launch.plan().openvmm_arguments()?, + launch.plan().kernel_command_line("")?, )) } } @@ -989,8 +1002,8 @@ fn sandbox_command_forwards_mount_to_openvmm() { ) .unwrap(); assert_eq!( - values(&arguments, "--mount"), - [format!("/workspace,{share},rw")] + values(&arguments, "--mount-child"), + [format!("0,{share},rw")] ); // The plan resolves each policy path, which must exist. assert_eq!(values(&arguments, "--mount-deny"), [secrets]); @@ -1023,15 +1036,15 @@ fn sandbox_command_shares_a_file() { None, ) .unwrap(); - let name = aggregate_child_name(0, "/etc/app/settings.json"); // The regular file is a child of its own, without its directory, which OpenVMM must find to - // be a regular file. + // be a regular file, and the guest maps it from the table that the command line counts. assert!(!arguments.contains(&"--mount".to_owned())); assert_eq!( values(&arguments, "--mount-child"), - [format!("{name},{settings},ro,file")] + [format!("0,{settings},ro,file")] ); - assert!(command_line.contains(&format!("nvx_share={name},/etc/app/settings.json,ro,file"))); + assert!(command_line.ends_with(" nvx_maps=1"), "{command_line}"); + assert!(!command_line.contains("nvx_share="), "{command_line}"); let error = shares .run( &[format!("/etc/app/settings.json,{settings},rw")], @@ -1056,7 +1069,10 @@ fn sandbox_command_shares_host_paths_with_commas() { None, ) .unwrap(); - assert_eq!(values(&arguments, "--mount"), [format!("/work,{alpha},rw")]); + assert_eq!( + values(&arguments, "--mount-child"), + [format!("0,{alpha},rw")] + ); assert_eq!( values(&arguments, "--mount-deny"), [shares.path("alpha,beta/secret")] @@ -1074,11 +1090,8 @@ fn sandbox_command_shares_host_paths_with_commas() { None, ) .unwrap(); - // The children follow their resolved host paths. - let (report, work) = ( - aggregate_child_name(0, "/reports"), - aggregate_child_name(1, "/work"), - ); + // The children follow their resolved host paths, and are named by their positions. + let (report, work) = ("0", "1"); let pairs: Vec<(String, String)> = arguments .windows(2) .filter(|pair| pair[0] == "--mount-child" || pair[0] == "--mount-deny") @@ -1125,7 +1138,7 @@ fn sandbox_command_forwards_mount_owner() { let (arguments, _) = result.unwrap(); assert_eq!(values(&arguments, "--mount-owner"), ["caller"]); let position = |option: &str| arguments.iter().position(|argument| argument == option); - assert!(position("--mount") < position("--mount-owner")); + assert!(position("--mount-aggregate") < position("--mount-owner")); } } @@ -1252,10 +1265,7 @@ fn sandbox_command_hides_a_share_root_behind_its_allowed_paths() { ) .unwrap(); // OpenVMM hides the workspace's root behind its one allowed path. - let (toolcache, workspace) = ( - aggregate_child_name(0, "/opt/hostedtoolcache"), - aggregate_child_name(1, "/workspace"), - ); + let (toolcache, workspace) = ("0", "1"); assert_eq!( values(&arguments, "--mount-child"), [ @@ -1285,13 +1295,14 @@ fn sandbox_command_attributes_mount_deny_to_the_preceding_mount() { (PolicyKind::Denied, Some(1), "logs"), ]; let (arguments, command_line) = shares.run(&mounts, &policy, None).unwrap(); - // The children follow their resolved host paths, each with its own denied paths. - let (toolcache, workspace) = ( - aggregate_child_name(0, "/opt/hostedtoolcache"), - aggregate_child_name(1, "/workspace"), - ); + // The children follow their resolved host paths, each with its own denied paths, and the + // command line counts their mappings rather than naming their targets. + let (toolcache, workspace) = ("0", "1"); assert!(!arguments.contains(&"--mount".to_owned())); - assert_eq!(values(&arguments, "--mount-aggregate"), ["/run/nvx/shares"]); + assert_eq!( + values(&arguments, "--mount-aggregate"), + [crate::host::filesystem::GUEST_EXPORT] + ); assert_eq!( values(&arguments, "--mount-child"), [ @@ -1307,9 +1318,8 @@ fn sandbox_command_attributes_mount_deny_to_the_preceding_mount() { shares.path("work/secrets"), ] ); - assert!(command_line.contains(&format!( - "nvx_share={toolcache},/opt/hostedtoolcache,ro nvx_share={workspace},/workspace,rw" - ))); + assert!(command_line.ends_with(" nvx_maps=2"), "{command_line}"); + assert!(!command_line.contains("nvx_share="), "{command_line}"); // With one share, a --mount-deny may come first; with several, it must follow the share // whose directory it hides. diff --git a/nvx-rs/src/host/sandbox/plan.rs b/nvx-rs/src/host/sandbox/plan.rs index 6e7ab5cd7..6af71e1cd 100644 --- a/nvx-rs/src/host/sandbox/plan.rs +++ b/nvx-rs/src/host/sandbox/plan.rs @@ -31,9 +31,8 @@ //! at [`GUEST_EXPORT`](crate::host::filesystem::GUEST_EXPORT), where the guest mounts the export, //! the kernel command line announces the number of their mappings with `nvx_maps=`, and the host //! sends the plan's [`mapping_table`](SandboxPlan::mapping_table) in `MAPS` requests once the -//! guest agent is ready, which maps each in order beneath the container's root. The CLI's -//! one-shot lifecycle still renders them as boot tokens, from which the guest's shell binds them, -//! as [`mount::boot_shares`] does. An adapter's plan renders its root and devices alone, and lets +//! guest agent is ready, which maps each in order beneath the container's root. An adapter's plan +//! renders its root and devices alone, and lets //! the adapter's own boot tokens through, as //! [`kernel_command_line`](SandboxPlan::kernel_command_line) says, and //! [`check_adapter_launch`](SandboxPlan::check_adapter_launch) checks, as the adapter plans its @@ -141,10 +140,6 @@ pub(crate) enum Delivery { /// the number of their mappings, and the host sends the mapping table once the guest agent is /// ready. MappingTable, - /// The one-shot lifecycle's, which only the CLI launches: boot tokens, from which the guest's - /// shell binds the shares. - #[cfg(any(test, feature = "cli"))] - BootTokens, } /// The role of a block device of a plan, which fixes its MMIO base and its access. @@ -572,8 +567,7 @@ impl SandboxPlan { } } - /// Returns the shares as `delivery` delivers them, which [`mount::boot_shares`] renders as - /// boot tokens; nothing without shares. + /// Returns the shares as `delivery` delivers them; nothing without shares. fn shares(&self, delivery: Delivery) -> Result { let Some(export) = &self.export else { return Ok(BootShares::default()); @@ -584,8 +578,6 @@ impl SandboxPlan { tokens: vec![export.kernel_token()], fragment: mount::export_command_line_fragment(export), }, - #[cfg(any(test, feature = "cli"))] - Delivery::BootTokens => mount::boot_shares(export)?, }) } @@ -607,16 +599,6 @@ impl SandboxPlan { ) } - /// Returns the OpenVMM arguments of the plan as the one-shot lifecycle launches it: as - /// [`openvmm_arguments`](Self::openvmm_arguments) does, but with the arguments that attach - /// the shares as the guest's shell binds them at boot. - /// - /// Fails as [`mount::boot_shares`] and [`SandboxNetwork::openvmm_arguments`] do. - #[cfg(any(test, feature = "cli"))] - pub(crate) fn one_shot_openvmm_arguments(&self) -> Result> { - self.render_arguments(&|_, path| path.display().to_string(), Delivery::BootTokens) - } - /// Renders the OpenVMM arguments of the plan, with `block` naming the file of each block /// device, given its role and its path in the plan, and with the shares as `delivery` delivers /// them. @@ -731,17 +713,6 @@ impl SandboxPlan { } } - /// Returns the kernel command line of the plan as the one-shot lifecycle boots it: as - /// [`kernel_command_line`](Self::kernel_command_line) does, but with the `nvx_share=` tokens - /// of an aggregate, which [`mount::boot_shares`] renders, instead of `nvx_maps=`. - /// - /// Fails as [`kernel_command_line`](Self::kernel_command_line) and [`mount::boot_shares`] - /// do. - #[cfg(any(test, feature = "cli"))] - pub(crate) fn one_shot_kernel_command_line(&self, user_command_line: &str) -> Result { - self.command_line(user_command_line, Delivery::BootTokens) - } - /// Returns the kernel command line of the plan, with the shares as `delivery` delivers them. pub(crate) fn command_line( &self, @@ -795,13 +766,8 @@ impl SandboxPlan { // OpenVMM appends the shares' bootstrap tokens after this command line, so they consume // the same x86 budget. if command_line.len() + fragment.len() + 1 > SANDBOX_COMMAND_LINE_MAX_SIZE { - // Only boot tokens put the shares' targets on the command line. let advice = match delivery { Delivery::MappingTable => "use shorter arguments and kernel parameters", - #[cfg(any(test, feature = "cli"))] - Delivery::BootTokens => { - "attach fewer --mount shares or use shorter targets and arguments" - } }; return Err(refused(format!( "sandbox kernel command line exceeds its 1024-byte x86 budget; {advice}" diff --git a/nvx-rs/src/host/sandbox/plan/tests.rs b/nvx-rs/src/host/sandbox/plan/tests.rs index 3e1d16a82..e7d729146 100644 --- a/nvx-rs/src/host/sandbox/plan/tests.rs +++ b/nvx-rs/src/host/sandbox/plan/tests.rs @@ -14,7 +14,7 @@ use crate::host::network::{ Protocol, ProxyScheme, }; use crate::host::sandbox::distro::DistroBinding; -use crate::host::sandbox::mount::{boot_shares, export_command_line_fragment}; +use crate::host::sandbox::mount::export_command_line_fragment; use crate::host::sandbox::tests::{deny, exported, host, host_text, native}; use crate::host::sandbox::{LayerRole, MountAccess, MountKind, SandboxLaunch, SandboxLayer}; @@ -112,24 +112,15 @@ type Rendering = ( fn(&ExportPlan) -> String, ); -/// Returns the renderings of the managed lifecycle, which announces a mapping table, and of the -/// one-shot lifecycle, which carries boot tokens, by name. -fn renderings() -> [(&'static str, Rendering); 2] { - [ +/// Returns the renderings of the managed lifecycle, which announces a mapping table, by name. +fn renderings() -> [(&'static str, Rendering); 1] { + [( + "managed", ( - "managed", - ( - SandboxPlan::kernel_command_line, - export_command_line_fragment, - ), + SandboxPlan::kernel_command_line, + export_command_line_fragment, ), - ( - "one-shot", - (SandboxPlan::one_shot_kernel_command_line, |export| { - boot_shares(export).unwrap().fragment - }), - ), - ] + )] } #[test] @@ -378,20 +369,8 @@ fn the_managed_lifecycle_exports_the_shares_and_announces_their_mapping_table() .collect::>(), ["/workspace", "/srv/cache"] ); - // The one-shot lifecycle renders the same plan as boot tokens. - let boot = boot_shares(&export).unwrap(); - assert_eq!( - plan.one_shot_openvmm_arguments().unwrap()[blocks.len()..], - boot.arguments[..] - ); - assert_eq!( - plan.one_shot_kernel_command_line("quiet").unwrap(), - format!("{base} {}", boot.tokens.join(" ")) - ); - // Without shares, both render alike, and nothing is mapped. + // Without shares, nothing is mapped. assert!(bare.mapping_table().is_empty()); - assert_eq!(bare.one_shot_openvmm_arguments().unwrap(), blocks); - assert_eq!(bare.one_shot_kernel_command_line("quiet").unwrap(), base); assert!(!base.contains("nvx_maps=")); } diff --git a/nvx-rs/src/host/sandbox/prepared.rs b/nvx-rs/src/host/sandbox/prepared.rs index f32dfa251..7a3c9f0dc 100644 --- a/nvx-rs/src/host/sandbox/prepared.rs +++ b/nvx-rs/src/host/sandbox/prepared.rs @@ -177,29 +177,7 @@ impl PreparedLaunch { ) } - /// Prepares the one-shot launch of `plan`, as [`new`](Self::new) does, but with the shares as - /// the one-shot lifecycle delivers them, which - /// [`SandboxPlan::one_shot_openvmm_arguments`] and - /// [`SandboxPlan::one_shot_kernel_command_line`] render: as the guest's shell binds them at - /// boot. - #[cfg(feature = "cli")] - pub(crate) fn one_shot( - plan: SandboxPlan, - distro: CheckedDistro, - scratch: Option, - user_command_line: &str, - ) -> Result { - Self::prepare( - plan, - distro, - scratch, - user_command_line, - Delivery::BootTokens, - ) - } - - /// Implements [`new`](Self::new) and [`one_shot`](Self::one_shot), with the shares as - /// `delivery` delivers them. + /// Implements [`new`](Self::new), with the shares as `delivery` delivers them. fn prepare( plan: SandboxPlan, distro: CheckedDistro, @@ -288,8 +266,7 @@ impl PreparedLaunch { } /// Returns the OpenVMM arguments of the plan, as [`SandboxPlan::openvmm_arguments`] renders - /// them, or for the CLI's one-shot launch with the shares as the guest's shell binds them at - /// boot, except that each block device names its checked file: on Linux as `/proc/self/fd/N`, + /// them, except that each block device names its checked file: on Linux as `/proc/self/fd/N`, /// which [`inherited`](Self::inherited) passes the VMM under the number `N`, and on Windows by /// its final path. /// @@ -299,8 +276,7 @@ impl PreparedLaunch { } /// Returns the kernel command line of the plan, with the user's command line given to - /// [`new`](Self::new), as [`SandboxPlan::kernel_command_line`] renders it, or for the CLI's - /// one-shot launch with the shares' boot tokens. + /// [`new`](Self::new), as [`SandboxPlan::kernel_command_line`] renders it. pub fn kernel_command_line(&self) -> &str { &self.kernel_command_line } diff --git a/nvx-rs/src/host/sandbox/tests.rs b/nvx-rs/src/host/sandbox/tests.rs index cc4a999df..0f0e920d0 100644 --- a/nvx-rs/src/host/sandbox/tests.rs +++ b/nvx-rs/src/host/sandbox/tests.rs @@ -207,10 +207,8 @@ fn launch_contract_rejects_duplicates_and_reserved_tokens() { assert_eq!(error.to_string(), "duplicate sandbox layer role(s): distro"); let launch = with_export(None).unwrap(); let plan = launch.plan(); - let renderings: [fn(&SandboxPlan, &str) -> Result; 2] = [ - SandboxPlan::kernel_command_line, - SandboxPlan::one_shot_kernel_command_line, - ]; + let renderings: [fn(&SandboxPlan, &str) -> Result; 1] = + [SandboxPlan::kernel_command_line]; for render in renderings { assert_eq!( message(render(plan, "nvx_sandbox=0")), @@ -225,18 +223,12 @@ fn launch_contract_rejects_duplicates_and_reserved_tokens() { "kernel command line contains an embedded NUL" ); } - // Only the one-shot lifecycle puts the shares' targets on the command line. let long = "x".repeat(SANDBOX_COMMAND_LINE_MAX_SIZE); assert_eq!( message(plan.kernel_command_line(&long)), "sandbox kernel command line exceeds its 1024-byte x86 budget; use shorter arguments and \ kernel parameters" ); - assert_eq!( - message(plan.one_shot_kernel_command_line(&long)), - "sandbox kernel command line exceeds its 1024-byte x86 budget; attach fewer --mount \ - shares or use shorter targets and arguments" - ); assert_eq!( message(parse_workload_identity("0:0")), "--workload-user UID must be between 1 and 4294967294" @@ -399,16 +391,18 @@ fn launch_contract_attaches_mount_and_reserves_bootstrap_tokens() { )]); deny(&mut plan, 0, &["/host/share/secrets"]); let launch = with_export(Some(plan)).unwrap(); - let arguments = launch.plan().one_shot_openvmm_arguments().unwrap(); + let arguments = launch.plan().openvmm_arguments().unwrap(); let start = arguments .iter() - .position(|argument| argument == "--mount") + .position(|argument| argument == "--mount-aggregate") .unwrap(); assert_eq!( arguments[start..], [ - "--mount".to_owned(), - format!("/workspace,{},rw", host_text("/host/share")), + "--mount-aggregate".to_owned(), + "/run/nvx/hostfs/root".to_owned(), + "--mount-child".to_owned(), + format!("0,{},rw", host_text("/host/share")), "--mount-deny".to_owned(), host_text("/host/share/secrets"), ] @@ -416,7 +410,7 @@ fn launch_contract_attaches_mount_and_reserves_bootstrap_tokens() { assert!( !launch .plan() - .one_shot_kernel_command_line("quiet") + .kernel_command_line("quiet") .unwrap() .contains("virtfs_") ); @@ -426,34 +420,20 @@ fn launch_contract_attaches_mount_and_reserves_bootstrap_tokens() { ("virtfs_mode=rw", "virtfs_mode"), ] { assert_eq!( - message(launch.plan().one_shot_kernel_command_line(token)), + message(launch.plan().kernel_command_line(token)), format!("{name} is owned by the sandbox --mount option") ); } - let base = with_export(None) - .unwrap() - .plan() - .one_shot_kernel_command_line("") - .unwrap(); - let fragment = mount::boot_shares(launch.export().unwrap()) - .unwrap() - .fragment; + let base = launch.plan().kernel_command_line("").unwrap(); + let fragment = mount::export_command_line_fragment(launch.export().unwrap()); assert_eq!( fragment, - " virtfs_dir=/workspace virtfs_tag=microvm virtfs_mode=rw" + " virtfs_dir=/run/nvx/hostfs/root virtfs_tag=microvm virtfs_mode=rw virtfs_aggregate=1" ); let fitting = "x".repeat(SANDBOX_COMMAND_LINE_MAX_SIZE - base.len() - fragment.len() - 2); - launch - .plan() - .one_shot_kernel_command_line(&fitting) - .unwrap(); + launch.plan().kernel_command_line(&fitting).unwrap(); assert!( - message( - launch - .plan() - .one_shot_kernel_command_line(&format!("{fitting}x")) - ) - .contains("1024-byte") + message(launch.plan().kernel_command_line(&format!("{fitting}x"))).contains("1024-byte") ); } @@ -486,9 +466,9 @@ fn launch_contract_attaches_several_shares_as_an_aggregate() { let names: Vec = shares .iter() .enumerate() - .map(|(index, &(target, ..))| mount::aggregate_child_name(index, target)) + .map(|(index, _)| index.to_string()) .collect(); - let arguments = launch.plan().one_shot_openvmm_arguments().unwrap(); + let arguments = launch.plan().openvmm_arguments().unwrap(); // Several shares are the children of one aggregate, and OpenVMM attributes each denied // path, which is absolute, to the child that contains it. assert!(!arguments.iter().any(|argument| argument == "--mount")); @@ -500,7 +480,7 @@ fn launch_contract_attaches_several_shares_as_an_aggregate() { arguments[start..], [ "--mount-aggregate".to_owned(), - "/run/nvx/shares".to_owned(), + "/run/nvx/hostfs/root".to_owned(), "--mount-child".to_owned(), format!("{},{},rw", names[0], host_text("/host/work")), "--mount-child".to_owned(), @@ -513,56 +493,45 @@ fn launch_contract_attaches_several_shares_as_an_aggregate() { host_text("/host/tools/credentials"), ] ); - let fragment = mount::boot_shares(&plan).unwrap().fragment; + let fragment = mount::export_command_line_fragment(&plan); assert_eq!( fragment, - " virtfs_dir=/run/nvx/shares virtfs_tag=microvm virtfs_mode=rw virtfs_aggregate=1" + " virtfs_dir=/run/nvx/hostfs/root virtfs_tag=microvm virtfs_mode=rw virtfs_aggregate=1" ); - // The guest binds each child at its target, in order. - let command_line = launch.plan().one_shot_kernel_command_line("").unwrap(); + // The guest receives each bind in the mapping table, not as boot tokens. + let command_line = launch.plan().kernel_command_line("").unwrap(); assert!(!command_line.contains("virtfs_")); - assert!( - command_line.ends_with(&format!( - " nvx_share={},/workspace,rw nvx_share={},/opt/hostedtoolcache,ro \ - nvx_share={},/srv/cache,ro", - names[0], names[1], names[2] - )), - "{command_line}" + assert!(command_line.ends_with(" nvx_maps=3"), "{command_line}"); + assert!(!command_line.contains("nvx_share=")); + assert_eq!( + launch + .plan() + .mapping_table() + .iter() + .map(|map| map.target.as_str()) + .collect::>(), + ["/workspace", "/opt/hostedtoolcache", "/srv/cache"] ); assert_eq!( - mount::boot_shares(&exported(&shares[1..])) - .unwrap() - .fragment, - " virtfs_dir=/run/nvx/shares virtfs_tag=microvm virtfs_mode=ro virtfs_aggregate=1" + mount::export_command_line_fragment(&exported(&shares[1..])), + " virtfs_dir=/run/nvx/hostfs/root virtfs_tag=microvm virtfs_mode=ro virtfs_aggregate=1" ); assert_eq!( - message( - launch - .plan() - .one_shot_kernel_command_line("nvx_share=0,/etc,rw") - ), + message(launch.plan().kernel_command_line("nvx_share=0,/etc,rw")), "nvx_share is owned by the sandbox profile" ); // Every share's tokens count toward the x86 budget. let base = with_export(None) .unwrap() .plan() - .one_shot_kernel_command_line("") + .kernel_command_line("") .unwrap(); let tokens = command_line.len() - base.len(); let fitting = "x".repeat(SANDBOX_COMMAND_LINE_MAX_SIZE - base.len() - tokens - fragment.len() - 2); - launch - .plan() - .one_shot_kernel_command_line(&fitting) - .unwrap(); + launch.plan().kernel_command_line(&fitting).unwrap(); assert!( - message( - launch - .plan() - .one_shot_kernel_command_line(&format!("{fitting}x")) - ) - .contains("1024-byte") + message(launch.plan().kernel_command_line(&format!("{fitting}x"))).contains("1024-byte") ); let caller = ExportPlan { @@ -602,13 +571,6 @@ fn launch_bounds_shares_by_the_kernel_command_line() { .collect(); with_export(Some(exported(&shares))).unwrap() }; - // No fixed number of shares is the limit of the one-shot lifecycle, only the command line, - // which carries each share's target. - launch(10).plan().one_shot_kernel_command_line("").unwrap(); - assert!( - message(launch(20).plan().one_shot_kernel_command_line("")) - .contains("fewer --mount shares") - ); // The managed lifecycle announces only their number, and sends them once the guest is ready. assert!( launch(20) @@ -663,10 +625,9 @@ fn launch_contract_attaches_a_file_as_an_aggregate_child() { MountKind::File, ); let launch = with_export(Some(exported(&[settings]))).unwrap(); - let name = mount::aggregate_child_name(0, "/etc/app/settings.json"); // Even alone, a file is the child of an aggregate, because the guest mounts the device's // root, which is a directory. - let arguments = launch.plan().one_shot_openvmm_arguments().unwrap(); + let arguments = launch.plan().openvmm_arguments().unwrap(); assert!(!arguments.iter().any(|argument| argument == "--mount")); let start = arguments .iter() @@ -676,24 +637,26 @@ fn launch_contract_attaches_a_file_as_an_aggregate_child() { arguments[start..], [ "--mount-aggregate".to_owned(), - "/run/nvx/shares".to_owned(), + "/run/nvx/hostfs/root".to_owned(), "--mount-child".to_owned(), - format!("{name},{},ro,file", host_text("/host/settings.json")), + format!("0,{},ro,file", host_text("/host/settings.json")), ] ); assert_eq!( - mount::boot_shares(launch.export().unwrap()) - .unwrap() - .fragment, - " virtfs_dir=/run/nvx/shares virtfs_tag=microvm virtfs_mode=ro virtfs_aggregate=1" + mount::export_command_line_fragment(launch.export().unwrap()), + " virtfs_dir=/run/nvx/hostfs/root virtfs_tag=microvm virtfs_mode=ro virtfs_aggregate=1" ); assert!( launch .plan() - .one_shot_kernel_command_line("") + .kernel_command_line("") .unwrap() - .ends_with(&format!(" nvx_share={name},/etc/app/settings.json,ro,file")) + .ends_with(" nvx_maps=1") ); + let mappings = launch.plan().mapping_table(); + assert_eq!(mappings[0].target, "/etc/app/settings.json"); + assert_eq!(mappings[0].source, "0"); + assert!(launch.export().unwrap().children[0].file); // Files and directories share one aggregate, and only a file's token is marked. let shares = [ @@ -714,23 +677,37 @@ fn launch_contract_attaches_a_file_as_an_aggregate_child() { let mut plan = exported(&shares); deny(&mut plan, 0, &["/host/work/secrets"]); let mixed = with_export(Some(plan)).unwrap(); - let names: Vec = shares - .iter() - .enumerate() - .map(|(index, &(target, ..))| mount::aggregate_child_name(index, target)) - .collect(); assert!( mixed .plan() - .one_shot_kernel_command_line("") + .kernel_command_line("") + .unwrap() + .ends_with(" nvx_maps=3") + ); + assert_eq!( + mixed + .plan() + .mapping_table() + .iter() + .map(|map| map.target.as_str()) + .collect::>(), + [ + "/workspace", + "/results/output.txt", + "/etc/app/settings.json" + ] + ); + assert_eq!( + mixed + .export() .unwrap() - .ends_with(&format!( - " nvx_share={},/workspace,rw nvx_share={},/etc/app/settings.json,ro,file \ - nvx_share={},/results/output.txt,rw,file", - names[0], names[1], names[2] - )) + .children + .iter() + .map(|child| child.file) + .collect::>(), + [false, true, true] ); - let arguments = mixed.plan().one_shot_openvmm_arguments().unwrap(); + let arguments = mixed.plan().openvmm_arguments().unwrap(); let denied = arguments .iter() .position(|argument| argument == "--mount-deny") diff --git a/nvx-rs/src/host/sandbox/tests/specification.rs b/nvx-rs/src/host/sandbox/tests/specification.rs index 0546d2123..5cd9345a3 100644 --- a/nvx-rs/src/host/sandbox/tests/specification.rs +++ b/nvx-rs/src/host/sandbox/tests/specification.rs @@ -235,10 +235,6 @@ fn launch_arguments_follow_the_openvmm_contract() { format!("{}:{}", identity.0, identity.1), ]; assert_eq!(launch.plan().openvmm_arguments().unwrap(), expected); - assert_eq!( - launch.plan().one_shot_openvmm_arguments().unwrap(), - expected - ); assert_eq!( launch.plan().kernel_command_line("quiet").unwrap(), format!( @@ -255,27 +251,24 @@ fn accepted_launch_shapes_keep_their_arguments_and_kernel_tokens() { let base = "nvx_sandbox=1 \ nvx_layer=distro,0xd0003000,erofs:11111111-1111-1111-1111-111111111111 \ nvx_scratch=0xd0006000,ext4 nvx_entrypoint=/bin/sh nvx_hostname=nvx-sandbox"; - let workspace = "0-c52ddf65534b7b46035084358ab7902b"; - let cache = "1-2a93e82e011c66d5c7dbd25da5f2e6c1"; - let file = "0-3ddd35a70a0d7386a5451cd0d5dc3c39"; let cases = [ - (862, Vec::new(), String::new(), Vec::new()), + (Vec::new(), String::new(), Vec::new()), ( - 806, vec![( "/workspace", "/host", MountAccess::ReadWrite, MountKind::Directory, )], - String::new(), + " nvx_maps=1".to_owned(), vec![ - "--mount".to_owned(), - format!("/workspace,{},rw", host_text("/host")), + "--mount-aggregate".to_owned(), + "/run/nvx/hostfs/root".to_owned(), + "--mount-child".to_owned(), + format!("0,{},rw", host_text("/host")), ], ), ( - 664, vec![ ( "/workspace", @@ -290,34 +283,46 @@ fn accepted_launch_shapes_keep_their_arguments_and_kernel_tokens() { MountKind::Directory, ), ], - format!(" nvx_share={workspace},/workspace,rw nvx_share={cache},/srv/cache,ro"), + " nvx_maps=2".to_owned(), vec![ "--mount-aggregate".to_owned(), - "/run/nvx/shares".to_owned(), + "/run/nvx/hostfs/root".to_owned(), "--mount-child".to_owned(), - format!("{workspace},{},rw", host_text("/host")), + format!("0,{},rw", host_text("/host")), "--mount-child".to_owned(), - format!("{cache},{},ro", host_text("/cache")), + format!("1,{},ro", host_text("/cache")), ], ), ( - 706, vec![( "/etc/app/settings.json", "/settings.json", MountAccess::ReadOnly, MountKind::File, )], - format!(" nvx_share={file},/etc/app/settings.json,ro,file"), + " nvx_maps=1".to_owned(), vec![ "--mount-aggregate".to_owned(), - "/run/nvx/shares".to_owned(), + "/run/nvx/hostfs/root".to_owned(), "--mount-child".to_owned(), - format!("{file},{},ro,file", host_text("/settings.json")), + format!("0,{},ro,file", host_text("/settings.json")), ], ), ]; - for (padding, shares, suffix, tail) in cases { + for (shares, suffix, tail) in cases { + let fragment = if shares.is_empty() { + String::new() + } else { + let mode = if shares.iter().any(|share| share.2 == MountAccess::ReadWrite) { + "rw" + } else { + "ro" + }; + format!( + " virtfs_dir=/run/nvx/hostfs/root virtfs_tag=microvm virtfs_mode={mode} virtfs_aggregate=1" + ) + }; + let padding = 1024 - base.len() - suffix.len() - fragment.len() - 2; let launch = SandboxLaunch::new( vec![SandboxLayer::new( LayerRole::Distro, @@ -343,22 +348,15 @@ fn accepted_launch_shapes_keep_their_arguments_and_kernel_tokens() { .map(str::to_owned) .to_vec(); expected.extend(tail); - assert_eq!( - launch.plan().one_shot_openvmm_arguments().unwrap(), - expected - ); + assert_eq!(launch.plan().openvmm_arguments().unwrap(), expected); assert_eq!( launch .plan() - .one_shot_kernel_command_line(&"x".repeat(padding)) + .kernel_command_line(&"x".repeat(padding)) .unwrap(), format!("{} {base}{suffix}", "x".repeat(padding)), ); - refused( - launch - .plan() - .one_shot_kernel_command_line(&"x".repeat(padding + 1)), - ); + refused(launch.plan().kernel_command_line(&"x".repeat(padding + 1))); } } @@ -373,32 +371,27 @@ fn shares_follow_the_openvmm_and_guest_contracts() { let launch = launch(SandboxSettings::default()) .with_export(Some(export)) .unwrap(); - let arguments = launch.plan().one_shot_openvmm_arguments().unwrap(); - let command_line = launch.plan().one_shot_kernel_command_line("").unwrap(); - if kind == MountKind::Directory { - assert_eq!( - &arguments[8..], - [ - "--mount", - &format!("/workspace,{},{}", host_text("/host/share"), mode) - ], - ); - assert!(!command_line.contains("nvx_share=")); - } else { - let name = "0-c52ddf65534b7b46035084358ab7902b"; - assert_eq!( - &arguments[8..], - [ - "--mount-aggregate", - "/run/nvx/shares", - "--mount-child", - &format!("{name},{},{mode},file", host_text("/host/share")), - ], - ); - assert!( - command_line.ends_with(&format!(" nvx_share={name},/workspace,{mode},file")) - ); - } + let arguments = launch.plan().openvmm_arguments().unwrap(); + let command_line = launch.plan().kernel_command_line("").unwrap(); + let file = if kind == MountKind::File { ",file" } else { "" }; + assert_eq!( + &arguments[8..], + [ + "--mount-aggregate", + "/run/nvx/hostfs/root", + "--mount-child", + &format!("0,{},{mode}{file}", host_text("/host/share")), + ], + ); + assert!(!command_line.contains("nvx_share=")); + let maps = launch.plan().mapping_table(); + assert_eq!(maps[0].target, "/workspace"); + assert_eq!(maps[0].read_only, access == MountAccess::ReadOnly); + assert_eq!(maps[0].source, "0"); + assert_eq!( + launch.export().unwrap().children[0].file, + kind == MountKind::File + ); assert!( launch .plan() @@ -588,7 +581,6 @@ fn caller_parameters_follow_linux_and_guest_rules() { "value=\"unterminated", ] { refused(launch.plan().kernel_command_line(user)); - refused(launch.plan().one_shot_kernel_command_line(user)); refused(adapter.kernel_command_line(user)); } for user in [ @@ -598,7 +590,6 @@ fn caller_parameters_follow_linux_and_guest_rules() { r#"extra="ignored nvx_pids_max=0 ignored""#, ] { refused(launch.plan().kernel_command_line(user)); - refused(launch.plan().one_shot_kernel_command_line(user)); } for user in [ r#"a="b c" d"#, @@ -607,7 +598,6 @@ fn caller_parameters_follow_linux_and_guest_rules() { ] { for command_line in [ launch.plan().kernel_command_line(user).unwrap(), - launch.plan().one_shot_kernel_command_line(user).unwrap(), adapter.kernel_command_line(user).unwrap(), ] { assert!(command_line.starts_with(&format!("{user} "))); @@ -626,7 +616,6 @@ fn the_reproduction_of_microsoft_nvx_723_is_refused() { r#""nvx_entrypoint=/bin/other""#, ] { refused(launch.plan().kernel_command_line(user)); - refused(launch.plan().one_shot_kernel_command_line(user)); } refused(SandboxLaunch::new( vec![SandboxLayer::new(LayerRole::Distro, "distro.erofs", UUID)], @@ -755,12 +744,8 @@ proptest! { (!shares.is_empty()).then(|| exported(&shares)), ).unwrap(); let user = "x".repeat(user_bytes); - for one_shot in [false, true] { - let render = |user: &str| if one_shot { - launch.plan().one_shot_kernel_command_line(user) - } else { - launch.plan().kernel_command_line(user) - }; + { + let render = |user: &str| launch.plan().kernel_command_line(user); let mut tokens = vec![ "nvx_sandbox=1".to_owned(), format!("nvx_layer=distro,0xd0003000,erofs:{UUID}"), @@ -769,25 +754,15 @@ proptest! { "nvx_hostname=nvx-sandbox".to_owned(), format!("nvx_arg={argument}"), ]; - if one_shot && share_count > 1 { - for (index, target) in targets.iter().enumerate() { - let digest = format!("{:x}", Sha256::digest(target.as_bytes())); - tokens.push(format!("nvx_share={index}-{},{target},{}", &digest[..32], access.as_str())); - } - } else if !one_shot && share_count != 0 { + if share_count != 0 { tokens.push(format!("nvx_maps={share_count}")); } let base = tokens.join(" "); prop_assert_eq!(render("").unwrap(), base.clone()); - let mode = access.as_str(); let fragment = if share_count == 0 { String::new() - } else if !one_shot { - " virtfs_dir=/run/nvx/hostfs/root virtfs_tag=microvm virtfs_mode=ro virtfs_aggregate=1".to_owned() - } else if share_count == 1 { - format!(" virtfs_dir=/share-0 virtfs_tag=microvm virtfs_mode={mode}") } else { - format!(" virtfs_dir=/run/nvx/shares virtfs_tag=microvm virtfs_mode={mode} virtfs_aggregate=1") + " virtfs_dir=/run/nvx/hostfs/root virtfs_tag=microvm virtfs_mode=ro virtfs_aggregate=1".to_owned() }; let bytes = base.len() + user_bytes + usize::from(user_bytes != 0) + fragment.len() + 1; match render(&user) { @@ -833,7 +808,6 @@ proptest! { format!("quiet{separator}{token}") }; refused(launch.plan().kernel_command_line(&user)); - refused(launch.plan().one_shot_kernel_command_line(&user)); } } @@ -880,14 +854,10 @@ proptest! { .map(|(target, root)| (target.as_str(), root, MountAccess::ReadWrite, MountKind::Directory)) .collect(); let launch = launch.with_export(Some(exported(&shares))).unwrap(); - for command_line in [ - launch.plan().kernel_command_line("quiet").unwrap(), - launch.plan().one_shot_kernel_command_line("quiet").unwrap(), - ] { - let reading = model::read(command_line.as_bytes()); - prop_assert_eq!(&reading.rest, &None); - let tokens: Vec<&[u8]> = command_line.split(' ').map(str::as_bytes).collect(); - prop_assert_eq!(reading.lines(), tokens); - } + let command_line = launch.plan().kernel_command_line("quiet").unwrap(); + let reading = model::read(command_line.as_bytes()); + prop_assert_eq!(&reading.rest, &None); + let tokens: Vec<&[u8]> = command_line.split(' ').map(str::as_bytes).collect(); + prop_assert_eq!(reading.lines(), tokens); } } diff --git a/nvx-rs/src/testing/fake_openvmm.rs b/nvx-rs/src/testing/fake_openvmm.rs index c593ba2e5..8004c62e6 100644 --- a/nvx-rs/src/testing/fake_openvmm.rs +++ b/nvx-rs/src/testing/fake_openvmm.rs @@ -48,7 +48,9 @@ //! //! Kernel command-line tokens adjust the emulation: `fake_exit_on_start=CODE` fails the launch, //! `fake_boot_delay_ms=MS` delays the control endpoint, `fake_crash_after_ms=MS` makes the VM -//! die, `fake_ignore_stop=1` ignores stop requests, `fake_exit_delay_ms=MS` delays the exit after +//! die, `fake_ignore_stop=1` ignores stop requests, `fake_drop_stop=1` closes the control +//! connection on a stop request without acknowledging it, and then exits as after an +//! acknowledged stop, `fake_exit_delay_ms=MS` delays the exit after //! an acknowledged stop, `fake_ignore_cancel=1` ignores cancellation, //! `fake_console_bytes=N` emits up to 16 MiB of `x` bytes on a run's console, and //! `fake_report_before_console=1` writes a run's report before console output or its boot delay. @@ -410,6 +412,8 @@ struct Options { crash_after: Option, /// Whether stop requests are deliberately left unanswered. ignore_stop: bool, + /// Whether a stop request closes the control connection without its acknowledgement. + drop_stop: bool, /// Artificial delay between the acknowledgement of a stop and the exit. exit_delay: Duration, /// Whether cancellation requests are deliberately ignored. @@ -1887,6 +1891,7 @@ fn parse(program: &str, arguments: &[String]) -> Result { .transpose()?, crash_after: millis("fake_crash_after_ms"), ignore_stop: knob("fake_ignore_stop") == Some("1"), + drop_stop: knob("fake_drop_stop") == Some("1"), exit_delay: millis("fake_exit_delay_ms").unwrap_or_default(), ignore_cancel: knob("fake_ignore_cancel") == Some("1"), legacy_guest: knob("fake_legacy_guest") == Some("1"), @@ -2541,6 +2546,9 @@ enum Flow { Continue, /// Stop the fake VM after a successful stop request. Exit, + /// Stop the fake VM once the current session's connection has closed, as a guest that powers + /// off before its acknowledgement of a stop request leaves. + Drop, } /// Outcome of control traffic observed while a workload runs. @@ -2698,6 +2706,7 @@ impl Session<'_, S> { // The targeted workload already finished. APP_CANCEL => {} APP_STOP if options.ignore_stop => {} + APP_STOP if options.drop_stop => return Ok(Flow::Drop), APP_STOP => { let _ = self.send(APP_STOPPED, request_id, 0, &[]); return Ok(Flow::Exit); @@ -3014,14 +3023,21 @@ fn serve_client( }; match session.run(options, guest) { Ok(Flow::Exit) => { - write_report(options, options.managed_operation(), "success", 0); - thread::sleep(options.exit_delay); + stop(options); Flow::Exit } + Ok(Flow::Drop) => Flow::Drop, _ => Flow::Continue, } } +/// Ends the fake VM's guest as a stop does: writes the outcome report of a successful stop, and +/// waits for the exit delay. +fn stop(options: &Options) { + write_report(options, options.managed_operation(), "success", 0); + thread::sleep(options.exit_delay); +} + /// Writes the synthetic OpenVMM outcome report of `operation` if a report was requested and its /// destination is new. /// @@ -3171,9 +3187,16 @@ fn serve(options: &Options, capability: &[u8; CAPABILITY_LEN]) -> io::Result<()> let mut stream = stream?; epoch += 1; record_session(options); - if let Flow::Exit = serve_client(&mut stream, options, capability, epoch, &mut guest) { - let _ = fs::remove_file(&options.endpoint); - return Ok(()); + match serve_client(&mut stream, options, capability, epoch, &mut guest) { + Flow::Continue => {} + flow => { + if let Flow::Drop = flow { + drop(stream); + stop(options); + } + let _ = fs::remove_file(&options.endpoint); + return Ok(()); + } } } Ok(()) @@ -3269,8 +3292,13 @@ fn serve(options: &Options, capability: &[u8; CAPABILITY_LEN]) -> io::Result<()> let _ = file.sync_all(); // SAFETY: the handle is a connected pipe server instance. unsafe { DisconnectNamedPipe(pipe) }; - if let Flow::Exit = flow { - return Ok(()); + match flow { + Flow::Continue => {} + Flow::Exit => return Ok(()), + Flow::Drop => { + stop(options); + return Ok(()); + } } } } diff --git a/nvx-rs/tests/cli.rs b/nvx-rs/tests/cli.rs index b15835b04..7d0b95fcb 100644 --- a/nvx-rs/tests/cli.rs +++ b/nvx-rs/tests/cli.rs @@ -6,10 +6,12 @@ //! directories and their records, parsed, POSIX file modes, and the arguments that OpenVMM //! receives. Their tables cover usage errors and help ([`usage`]), the checks that refuse a command //! line before anything changes ([`validation`]), dry runs and the splitting of their `>> ` lines -//! ([`dry_run`]), one-shot runs ([`one_shot`]), the lifecycles of managed sandboxes, their -//! executions, and the refusals of their operations ([`lifecycle`]), the configurations of every -//! format ([`formats`]), `egress compile` ([`egress`]), `artifacts` ([`artifacts`]), and `download` -//! and the download that `run` and `sandbox` start themselves, without a network ([`download`]). +//! ([`dry_run`]), the runs of `nvx-rs run` ([`one_shot`]), the runs of `nvx-rs sandbox run`, each +//! a sandbox without a state directory ([`sandbox_run`]), the lifecycles of managed sandboxes, +//! their executions, and the refusals of their operations ([`lifecycle`]), the configurations of +//! every format ([`formats`]), `egress compile` ([`egress`]), `artifacts` ([`artifacts`]), and +//! `download` and the download that `run` and `sandbox` start themselves, without a network +//! ([`download`]). //! They replaced the scenarios that scripts recorded from the Python implementations of //! `nvx.py sandbox` and `nvx.py run`, and that this test replayed until microsoft/nvx#728 retired //! them. @@ -42,6 +44,8 @@ mod lifecycle; mod one_shot; #[path = "cli/openvmm.rs"] mod openvmm; +#[path = "cli/sandbox_run.rs"] +mod sandbox_run; #[path = "cli/usage.rs"] mod usage; #[path = "cli/validation.rs"] @@ -563,10 +567,10 @@ fn symlink_file(target: &Path, link: &Path) -> bool { #[test] fn a_distro_manifest_path_whose_lookup_fails_refuses_provision_and_start() { - // Verification counts a manifest path whose lookup fails as no manifest, as nvx.py did, so a - // run's outcome is unchanged. A binding of the manifest's absence holds only while nothing is - // at its path: start refuses such a path, and provision refuses to record a binding that - // every start would refuse. + // Verification counts a manifest path whose lookup fails as no manifest, as nvx.py did. A + // binding of the manifest's absence holds only while nothing is at its path: start refuses + // such a path, and provision refuses to record a binding that every start would refuse, as a + // run does, which launches what provision would record, even as a dry run. let root = Root::new(); let manifest = root.path.join("distro.erofs.manifest.json"); provisioned(&root, "absent"); @@ -607,8 +611,7 @@ fn a_distro_manifest_path_whose_lookup_fails_refuses_provision_and_start() { path("scratch.ext4"), "--dry-run".into(), ]; - let output = run(&root, &dry_run); - assert!(output.status.success(), "{}", text(&output.stderr)); + failed(&run(&root, &dry_run)); } #[test] @@ -1532,7 +1535,7 @@ fn tcp_forwards_take_any_egress_policy() { &["--network-egress", "deny"][..], " --net 10.0.0.2/24 --network-profile portable --network-egress deny \ --network-ingress deny --host-loopback allow --host-loopback-forward tcp:18080:8080 \ - --microvm-lifecycle ", + --single-process ", ), ( &[ @@ -1543,7 +1546,7 @@ fn tcp_forwards_take_any_egress_policy() { ][..], " --net 10.0.0.2/24 --network-profile portable --network-egress allow \ --network-ingress deny --host-loopback allow --network-egress-deny 10.0.0.0/8 \ - --host-loopback-forward tcp:18080:8080 --microvm-lifecycle ", + --host-loopback-forward tcp:18080:8080 --single-process ", ), ] { let mut arguments: Vec = [ @@ -1630,7 +1633,7 @@ fn network_policy_files_compile_into_the_typed_policy() { assert!( text(&output.stdout).contains( " --net 10.0.0.2/24 --network-profile portable --network-egress deny \ - --network-ingress deny --network-egress-allow 192.0.2.0/24 --microvm-lifecycle " + --network-ingress deny --network-egress-allow 192.0.2.0/24 --single-process " ), "{}", text(&output.stdout) @@ -1855,7 +1858,7 @@ fn a_running_sandbox_claims_its_scratch_image() { state_entries(&root, "second"), ["config.json", "sandbox.lock"] ); - let one_shot: Vec = vec![ + let sandbox_run: Vec = vec![ "sandbox".into(), "run".into(), "--layer".into(), @@ -1866,9 +1869,9 @@ fn a_running_sandbox_claims_its_scratch_image() { "--scratch".into(), scratch.to_str().unwrap().to_owned(), "--entrypoint".into(), - "/bin/true".into(), + "/bin/echo".into(), ]; - let output = run(&root, &one_shot); + let output = run(&root, &sandbox_run); assert_eq!(output.status.code(), Some(1), "{}", text(&output.stderr)); assert_eq!(text(&output.stderr), claimed); @@ -1876,7 +1879,7 @@ fn a_running_sandbox_claims_its_scratch_image() { let output = run(&root, &lifecycle("stop", &root, "first")); assert!(output.status.success(), "{}", text(&output.stderr)); cycle(&root, "second"); - let output = run(&root, &one_shot); + let output = run(&root, &sandbox_run); assert!(output.status.success(), "{}", text(&output.stderr)); let output = run(&root, &lifecycle("deprovision", &root, "first")); assert!(output.status.success(), "{}", text(&output.stderr)); @@ -2209,6 +2212,8 @@ fn a_run_passes_openvmm_no_handle_of_its_caller() { let (_reader, unrelated) = std::io::pipe().unwrap(); let root = Root::new(); let path = |name: &str| root.path.join(name).to_str().unwrap().to_owned(); + let started = root.path.join("started"); + let script = format!("touch {}; sleep 60000", started.display()); let mut command = Command::new(NVX_RS); command .arg("--root") @@ -2224,9 +2229,10 @@ fn a_run_passes_openvmm_no_handle_of_its_caller() { "--scratch", &path("scratch.ext4"), "--entrypoint", - "/bin/sleep", + "/bin/sh", + "--arg=-c", "--arg", - "60", + &script, ]) .current_dir(&root.path) .env_remove("NVX_OPENVMM") @@ -2238,8 +2244,8 @@ fn a_run_passes_openvmm_no_handle_of_its_caller() { set_inheritable(&unrelated, false); let mut cli = cli.unwrap(); - // Once OpenVMM has started, the CLI holds the handle, and OpenVMM does not. - wait_for(&root.path.join("build").join("fake-openvmm-.json")); + // Once the workload runs, the CLI holds the handle, and OpenVMM does not. + wait_for(&started); let value = unrelated.as_raw_handle() as usize; let openvmm = children_named(cli.id(), "openvmm.exe"); let cli_holds = holds(cli.id(), value); @@ -2272,8 +2278,8 @@ fn batch_openvmm(root: &Root) -> (BTreeMap, PathBuf, String) { wrapper.to_str().unwrap().to_owned(), )]); let refusal = format!( - "error[unsupported]: cannot start the batch file {}: Windows runs batch files through \ - cmd.exe, which would read their arguments as shell syntax\n", + "error[io]: cannot spawn the VM: cannot start the batch file {}: Windows runs batch files \ + through cmd.exe, which would read their arguments as shell syntax\n", wrapper.display() ); (environment, marker, refusal) @@ -2299,7 +2305,7 @@ fn a_run_refuses_a_batch_file_as_openvmm() { "--scratch", scratch.to_str().unwrap(), "--entrypoint", - "/bin/true", + "/bin/echo", ] .map(str::to_owned) .to_vec(); @@ -2758,8 +2764,8 @@ mod interrupts { use super::*; - /// Sends SIGINT once OpenVMM has started, to the CLI alone or, as a terminal's Ctrl-C does, - /// to its whole process group, which a run's OpenVMM shares, and returns the CLI's output. + /// Sends SIGINT once `started` exists, to the CLI alone or, as a terminal's Ctrl-C does, to + /// its whole process group, which a run's OpenVMM shares, and returns the CLI's output. fn interrupt( root: &Root, arguments: &[String], @@ -2798,10 +2804,12 @@ mod interrupts { (output, interrupted.elapsed()) } - /// Returns the arguments of a one-shot run of `/bin/sleep 60` in the fake OpenVMM, and the - /// file that the fake writes when it starts. + /// Returns the arguments of a run whose workload, once it starts, writes the file that this + /// returns too, and then sleeps for a minute in the fake OpenVMM. fn sleeping_run(root: &Root) -> (Vec, PathBuf) { let path = |name: &str| root.path.join(name).to_str().unwrap().to_owned(); + let started = root.path.join("started"); + let script = format!("touch {}; sleep 60000", started.display()); let arguments: Vec = [ "sandbox", "run", @@ -2813,17 +2821,14 @@ mod interrupts { "--scratch", &path("scratch.ext4"), "--entrypoint", - "/bin/sleep", + "/bin/sh", + "--arg=-c", "--arg", - "60", + &script, ] .map(str::to_owned) .to_vec(); - // The fake records its command line when it starts. - ( - arguments, - root.path.join("build").join("fake-openvmm-.json"), - ) + (arguments, started) } #[test] @@ -2833,8 +2838,11 @@ mod interrupts { let (output, elapsed) = interrupt(&root, &arguments, &started, false); assert_eq!(output.status.code(), Some(130)); assert_eq!(text(&output.stderr), "Interrupted\n"); - assert!(text(&output.stdout).starts_with(">> ")); assert!(elapsed < Duration::from_secs(10), "{elapsed:?}"); + // The run ended its VM, whose directory it then removed. + let outcome = harness::Outcome::new(arguments, output); + let (command, _) = outcome.command(); + openvmm::Options::parse(&command[1..]).normalized_run_outputs(); } #[test] @@ -2926,8 +2934,9 @@ mod interrupts { #[test] fn interrupting_a_run_ends_it_as_interrupted_even_if_openvmm_exits_first() { - // A terminal's Ctrl-C reaches OpenVMM too, which the fake lets end it at once; Python's - // wait raises KeyboardInterrupt all the same, rather than return OpenVMM's status. + // A terminal's Ctrl-C reaches OpenVMM too, which a run keeps in its process group, and + // which the fake lets end it at once; the run ends as interrupted all the same, rather + // than as one whose VM ended. let root = Root::new(); let (arguments, started) = sleeping_run(&root); let (output, elapsed) = interrupt(&root, &arguments, &started, true); diff --git a/nvx-rs/tests/cli/download.rs b/nvx-rs/tests/cli/download.rs index b60f9c49f..eb734cd18 100644 --- a/nvx-rs/tests/cli/download.rs +++ b/nvx-rs/tests/cli/download.rs @@ -168,6 +168,17 @@ fn commands_download_nothing_for_options_that_they_refuse() { "--cmdline", "nvx_sandbox=0", ], + // A balanced quote passes the plan, but not the managed command line of the launch. + vec![ + "sandbox", + "run", + "--layer", + LAYER, + "--scratch", + SCRATCH, + "--cmdline", + "a=\"b c\"", + ], ] { let root = checkout(&INSTALLED); let before = root.snapshot(); @@ -301,8 +312,11 @@ fn sandbox_run_reads_a_packaged_layer_once_no_download_replaces_it() { spec.as_str(), "--scratch", SCRATCH, - "--cmdline", - "fake_exit_status=7", + "--entrypoint", + "/bin/sh", + "--arg=-c", + "--arg", + "exit 7", ]; std::thread::scope(|scope| { let run = scope.spawn(|| root.nvx_rs(&arguments)); diff --git a/nvx-rs/tests/cli/dry_run.rs b/nvx-rs/tests/cli/dry_run.rs index 104482699..2ab0ab87f 100644 --- a/nvx-rs/tests/cli/dry_run.rs +++ b/nvx-rs/tests/cli/dry_run.rs @@ -6,9 +6,10 @@ use proptest::prelude::*; use proptest::test_runner::{Config, TestRunner}; use crate::harness::{Content, Hosts, LAYER, OPENVMM, Outcome, Root, SCRATCH, in_parallel, native}; +use crate::lifecycle::exported; use crate::openvmm::{ - DISCONNECTED_CONSOLE, NO_CONSOLE, Options, TERMINAL_CONSOLE, WORKLOAD, child, cold_boot, - one_shot, restore, share_token, snapshot, + DISCONNECTED_CONSOLE, NO_CONSOLE, Options, TERMINAL_CONSOLE, child, cold_boot, managed_run, + restore, share_token, snapshot, }; /// An egress policy file with an allowed range, an exclusion, a port, and a denied protocol, which @@ -119,30 +120,22 @@ fn run(options: &[&str]) -> Vec { .collect() } -/// Returns the default workload tokens of a sandbox, then `tokens`. -fn workload(tokens: &[String]) -> Vec { - WORKLOAD - .iter() - .map(|token| (*token).to_owned()) - .chain(tokens.iter().cloned()) - .collect() -} - /// Returns the `--mount-child` value of the aggregate child at `index`, which the guest binds at /// `target`, of the host path `host`, with the access and kind `mode`, such as `ro` or `ro,file`. fn mount_child(index: usize, target: &str, host: &str, mode: &str) -> String { format!("{},{host},{mode}", child(index, target)) } -/// Returns the dry runs of `nvx-rs sandbox run`. +/// Returns the dry runs of `nvx-rs sandbox run`, with the options of their OpenVMM commands in the +/// form that [`Options::persistent_expected`] projects: those of a start of the same sandbox. fn sandbox_dry_runs() -> Vec { let network = || { - one_shot() + managed_run() .with("--net", ["10.0.0.2/24"]) .with("--network-profile", ["portable"]) }; vec![ - dry_run("minimal", sandbox_run(&[]), one_shot()), + dry_run("minimal", sandbox_run(&[]), managed_run()), dry_run( "balanced disk path brackets", [ @@ -154,7 +147,7 @@ fn sandbox_dry_runs() -> Vec { "--scratch", "{root/scratch[][1].ext4}", ], - one_shot().with( + managed_run().with( "--microvm-sandbox-block", [ "distro:file:{root/distro[[1][2]].erofs},ro", @@ -167,9 +160,10 @@ fn sandbox_dry_runs() -> Vec { dry_run( "explicit scratch upper", sandbox_run(&["--upper", "scratch"]), - one_shot(), + managed_run(), ), - // An upper in RAM attaches the distro layer alone, and asks the guest for a tmpfs upper. + // An upper in RAM attaches the distro layer alone, and asks the guest for a tmpfs upper of + // at most half of its RAM by default. dry_run( "RAM upper", [ @@ -181,8 +175,11 @@ fn sandbox_dry_runs() -> Vec { "ram", "--dry-run", ], - one_shot().in_ram(), + managed_run().in_ram(), ), + // The workload's entrypoint and arguments are its execution's, which neither OpenVMM nor + // the kernel command line receives, and the outcome report is OpenVMM's, which the run + // forwards. dry_run( "every layer and option", sandbox_run(&[ @@ -204,68 +201,45 @@ fn sandbox_dry_runs() -> Vec { "--outcome-report", "{root/reports/openvmm-run.json}", ]), - one_shot() + managed_run() .with("--microvm-workload-identity", ["1000:1000"]) .with("--hypervisor", ["mshv"]) .with("--memory", ["512M"]) - .with("--microvm-report", ["{root/reports/openvmm-run.json}"]) - .sandbox_kernel( - "quiet loglevel=3", - &[ - "nvx_entrypoint=/bin/server", - "nvx_hostname=worker", - "nvx_arg=--port", - "nvx_arg=8080", - ], - ), + .sandbox_kernel("quiet loglevel=3", &["nvx_hostname=worker"]), ), dry_run( "explicit hypervisor", sandbox_run(&["--hypervisor", "kvm"]), - one_shot().with("--hypervisor", ["kvm"]), + managed_run().with("--hypervisor", ["kvm"]), ), // `--memory-mib` takes from 1 MiB to the most whose count of bytes fits in 64 bits, as // OpenVMM reads a size of RAM. dry_run( "least memory", sandbox_run(&["--memory-mib", "1"]), - one_shot().with("--memory", ["1M"]), + managed_run().with("--memory", ["1M"]), ), dry_run( "most memory", sandbox_run(&["--memory-mib", "17592186044415"]), - one_shot().with("--memory", ["17592186044415M"]), + managed_run().with("--memory", ["17592186044415M"]), ), - // One directory is OpenVMM's own share, read-only by default. + // Every share, read-only by default, is a child of the aggregate whose mapping table the + // guest receives once it is ready. dry_run( "one share", sandbox_run(&["--mount", "/data,{root/share}"]), - one_shot().with("--mount", ["/data,{root/share},ro"]), + exported(managed_run(), &["0,{root/share},ro"]), ), dry_run( "writable share", sandbox_run(&["--mount", "/data,{root/share},rw"]), - one_shot().with("--mount", ["/data,{root/share},rw"]), + exported(managed_run(), &["0,{root/share},rw"]), ), - // A file is a child of the aggregate, which a boot token binds at its target. dry_run( "file share", sandbox_run(&["--mount", "/etc/single.txt,{root/single.txt},ro"]), - one_shot() - .with("--mount-aggregate", ["/run/nvx/shares"]) - .with( - "--mount-child", - [mount_child( - 0, - "/etc/single.txt", - "{root/single.txt}", - "ro,file", - )], - ) - .sandbox_kernel( - "", - &workload(&[share_token(0, "/etc/single.txt", "ro", true)]), - ), + exported(managed_run(), &["0,{root/single.txt},ro,file"]), ), dry_run( "several shares with policies", @@ -281,25 +255,10 @@ fn sandbox_dry_runs() -> Vec { "--mount", "/other,{root/share2}", ]), - one_shot() - .with("--mount-aggregate", ["/run/nvx/shares"]) - .with( - "--mount-child", - [ - mount_child(0, "/data", "{root/share}", "rw"), - mount_child(1, "/other", "{root/share2}", "ro"), - ], - ) + exported(managed_run(), &["0,{root/share},rw", "1,{root/share2},ro"]) .with("--mount-deny", ["{root/share/hidden}"]) .with("--mount-allow", ["{root/share/hidden/shown}"]) - .with("--mount-write", ["{root/share/writable}"]) - .sandbox_kernel( - "", - &workload(&[ - share_token(0, "/data", "rw", false), - share_token(1, "/other", "ro", false), - ]), - ), + .with("--mount-write", ["{root/share/writable}"]), ), dry_run( "network", @@ -366,17 +325,18 @@ fn sandbox_dry_runs() -> Vec { .with("--network-egress-deny", COMPILED_DENY), ) .with_file("policy.json", POLICY), + // A double quote of the kernel command line would end the options of the control + // endpoint, as it does a start's. dry_run( "quoted arguments", sandbox_run(&["--cmdline", "a=\"b c\" d"]), - one_shot().sandbox_kernel("a=\"b c\" d", &WORKLOAD), - ), + managed_run(), + ) + .refusing("double quotes"), dry_run( "caller-owned share", sandbox_run(&["--mount", "/data,{root/share}", "--mount-owner", "caller"]), - one_shot() - .with("--mount", ["/data,{root/share},ro"]) - .with("--mount-owner", ["caller"]), + exported(managed_run(), &["0,{root/share},ro"]).with("--mount-owner", ["caller"]), ) .on(Hosts::Posix), ] @@ -1324,6 +1284,10 @@ fn dry_runs_print_the_openvmm_command_and_launch_nothing() { } let before = root.snapshot(); let arguments: Vec<&str> = case.arguments.iter().map(String::as_str).collect(); + let sandbox = case + .arguments + .first() + .is_some_and(|command| command == "sandbox"); let cold = case .arguments .first() @@ -1350,6 +1314,8 @@ fn dry_runs_print_the_openvmm_command_and_launch_nothing() { case.options .cold_run_expected(&root, "one-shot") .unwrap_or_else(|error| panic!("{} unexpectedly refused: {error}", case.name)) + } else if sandbox { + case.options.clone().persistent_expected() } else { case.options.expanded(&root) }; @@ -1366,6 +1332,8 @@ fn dry_runs_print_the_openvmm_command_and_launch_nothing() { let actual = Options::parse(arguments); let actual = if cold { actual.normalized_cold_outputs(true) + } else if sandbox { + actual.normalized_run_outputs() } else { actual }; diff --git a/nvx-rs/tests/cli/lifecycle.rs b/nvx-rs/tests/cli/lifecycle.rs index 8c10bf35b..e615256f3 100644 --- a/nvx-rs/tests/cli/lifecycle.rs +++ b/nvx-rs/tests/cli/lifecycle.rs @@ -243,13 +243,13 @@ fn configuration( } /// Returns the mapping table entry that maps the export's child `source` at `target`. -fn mapped(source: &str, target: &str, read_only: bool, directory: bool) -> Value { +pub(crate) fn mapped(source: &str, target: &str, read_only: bool, directory: bool) -> Value { json!({"source": source, "target": target, "read_only": read_only, "directory": directory}) } /// Returns the options of a start of a sandbox whose aggregate exports `children`, as /// `--mount-child` values, which the guest maps from `/run/nvx/hostfs/root`. -fn exported(options: Options, children: &[&str]) -> Options { +pub(crate) fn exported(options: Options, children: &[&str]) -> Options { let mut tokens: Vec = WORKLOAD.iter().map(|token| (*token).to_owned()).collect(); tokens.push(format!("nvx_maps={}", children.len())); options @@ -1559,11 +1559,11 @@ fn refusals_of_lifecycle_operations() { }); } -/// The host paths of shares of a sandbox's own files, which `provision` and `start` refuse, and -/// which of its files each one reaches first: its state directory, image, scratch image, kernel, -/// initramfs, and OpenVMM, a directory that holds one of them, and a directory inside its state -/// directory. -const OWN_FILES: [(&str, &str); 11] = [ +/// The host paths of shares of a sandbox's own files, which `provision` and `start` refuse, as +/// `sandbox run` does all but those of the state directory, which it has none of, and which of its +/// files each one reaches first: its state directory, image, scratch image, kernel, initramfs, and +/// OpenVMM, a directory that holds one of them, and a directory inside its state directory. +pub(crate) const OWN_FILES: [(&str, &str); 11] = [ ("{root/control/state}", "state directory"), ("{root/control}", "state directory"), ("{root/control/state/inside}", "state directory"), @@ -1578,7 +1578,7 @@ const OWN_FILES: [(&str, &str); 11] = [ ]; /// Returns a root whose image and scratch image are in directories of their own. -fn own_files_root() -> Root { +pub(crate) fn own_files_root() -> Root { let root = Root::new(); root.write("images/distro.erofs", "distro\n"); root.write("upper/scratch.ext4", "scratch\n"); @@ -1602,7 +1602,7 @@ fn own_files_provision(host: Option<&str>) -> Vec { /// Returns `host`, with its placeholders replaced, and resolved if it is a link, as the root's /// OpenVMM is on POSIX hosts, since a share's host path must not be one. -fn share_host(root: &Root, host: &str) -> String { +pub(crate) fn share_host(root: &Root, host: &str) -> String { let host = root.expand(host); if Path::new(&host).is_symlink() { canonical(Path::new(&host)).to_str().unwrap().to_owned() diff --git a/nvx-rs/tests/cli/one_shot.rs b/nvx-rs/tests/cli/one_shot.rs index 6d119b42d..d49dc5356 100644 --- a/nvx-rs/tests/cli/one_shot.rs +++ b/nvx-rs/tests/cli/one_shot.rs @@ -1,14 +1,12 @@ -//! The runs that launch OpenVMM on the terminal: one-shot sandboxes of `nvx-rs sandbox run`, and -//! the microVMs of `nvx-rs run`. Each prints the OpenVMM command as `>> COMMAND`, runs it with the -//! standard streams, which carry the workload's output, or OpenVMM's, byte for byte, and exits -//! with OpenVMM's status, which a one-shot sandbox's OpenVMM takes from its workload. +//! The runs that launch OpenVMM on the terminal: the microVMs of `nvx-rs run`. Each prints the +//! OpenVMM command as `>> COMMAND`, runs it with the standard streams, which carry the workload's +//! output, or OpenVMM's, byte for byte, and exits with OpenVMM's status. The runs of +//! `nvx-rs sandbox run`, which execute their workload over NVXC, are [`crate::sandbox_run`]'s. -use crate::harness::{Content, LAYER, OPENVMM, Root, SCRATCH, in_parallel, native}; +use crate::harness::{Content, OPENVMM, Root, in_parallel, native}; use crate::openvmm::{ DISCONNECTED_CONSOLE, NO_CONSOLE, NO_NETWORK, NetworkPolicy, Options, TERMINAL_CONSOLE, - WORKLOAD, assert_checked_blocks, assert_checked_distro, assert_opened_distro, - assert_opened_images, assert_outcome_report, child, cold_boot, one_shot, restore, share_token, - snapshot, + assert_outcome_report, child, cold_boot, restore, share_token, snapshot, }; /// The outcome report that OpenVMM writes for a run that asks for one. @@ -35,8 +33,7 @@ struct Launch { environment: Vec<(&'static str, String)>, /// The arguments after `--root ROOT`, with placeholders. arguments: Vec, - /// The options of the OpenVMM command, with placeholders, and with the paths of a sandbox's - /// images, which the run passes as files that it checked. + /// The options of the OpenVMM command, with placeholders. options: Options, /// The record of the arguments that the fake OpenVMM received, below the root, unless it /// refuses them. @@ -125,40 +122,6 @@ impl Launch { } } -/// Returns `nvx-rs sandbox run` of the distro layer and scratch image of the root that runs -/// `entrypoint` with `options`. -fn sandbox_run(entrypoint: &str, options: &[&str]) -> Vec { - [ - "sandbox", - "run", - "--layer", - LAYER, - "--scratch", - SCRATCH, - "--entrypoint", - entrypoint, - ] - .iter() - .chain(options) - .map(|argument| (*argument).to_owned()) - .collect() -} - -/// Returns the OpenVMM options of a one-shot sandbox that runs `entrypoint` with `arguments`, -/// after the user's kernel command line `user`. -fn workload(user: &str, entrypoint: &str, arguments: &[&str]) -> Options { - let mut tokens = vec![ - format!("nvx_entrypoint={entrypoint}"), - WORKLOAD[1].to_owned(), - ]; - tokens.extend( - arguments - .iter() - .map(|argument| format!("nvx_arg={argument}")), - ); - one_shot().sandbox_kernel(user, &tokens) -} - /// Returns whether the launch is a cold boot of `nvx-rs run` that fails, which `run` follows with /// its CPU-profile guidance on a host whose CPU no built-in profile serves, as /// `run_guides_failed_cold_boots_on_cpus_that_no_built_in_profile_serves` checks. @@ -182,152 +145,6 @@ fn run(options: &[&str]) -> Vec { .collect() } -/// Returns the one-shot runs of `nvx-rs sandbox run`. -fn sandbox_launches() -> Vec { - let echo = || workload("", "/bin/echo", &["hello", "world"]); - let mut launches = vec![ - launch( - "echo", - sandbox_run("/bin/echo", &["--arg", "hello", "--arg", "world"]), - echo(), - 0, - "hello world\n", - ), - // The operation is `run` by default. - launch( - "default operation", - [ - "sandbox", - "--layer", - LAYER, - "--scratch", - SCRATCH, - "--entrypoint", - "/bin/echo", - "--arg", - "hello", - "--arg", - "world", - ], - echo(), - 0, - "hello world\n", - ), - launch( - "failure", - sandbox_run("/bin/false", &[]), - workload("", "/bin/false", &[]), - 1, - "", - ), - // An upper in RAM attaches the distro layer alone, and needs no scratch image. - launch( - "RAM upper", - [ - "sandbox", - "run", - "--layer", - LAYER, - "--upper", - "ram", - "--entrypoint", - "/bin/echo", - "--arg", - "hello", - "--arg", - "world", - ], - echo().in_ram(), - 0, - "hello world\n", - ) - .without("scratch.ext4"), - launch( - "missing program", - sandbox_run("/bin/missing", &[]), - workload("", "/bin/missing", &[]), - 127, - "", - ) - .with_stderr("openvmm: /bin/missing: not found\n"), - launch( - "outcome report", - sandbox_run( - "/bin/echo", - &[ - "--arg", - "hello", - "--arg", - "world", - "--outcome-report", - "{root/reports/openvmm-run.json}", - ], - ), - echo().with("--microvm-report", ["{root/reports/openvmm-run.json}"]), - 0, - "hello world\n", - ) - .recorded_in(Some("build/fake-openvmm-reports.json")) - .reporting(Report { - path: "reports/openvmm-run.json", - category: "success", - status: 0, - network: NO_NETWORK, - }), - launch( - "network outcome report", - sandbox_run( - "/bin/true", - &[ - "--net", - "10.0.0.2/24", - "--network-profile", - "portable", - "--network-egress", - "deny", - "--network-egress-allow", - "10.0.0.0/8:tcp:443", - "--outcome-report", - "{root/reports/openvmm-run.json}", - ], - ), - workload("", "/bin/true", &[]) - .with("--net", ["10.0.0.2/24"]) - .with("--network-profile", ["portable"]) - .with("--network-egress", ["deny"]) - .with("--network-ingress", ["deny"]) - .with("--network-egress-allow", ["10.0.0.0/8:tcp:443"]) - .with("--microvm-report", ["{root/reports/openvmm-run.json}"]), - 0, - "", - ) - .recorded_in(Some("build/fake-openvmm-reports.json")) - .reporting(Report { - path: "reports/openvmm-run.json", - category: "success", - status: 0, - network: ("applied", "rules", 1, 0, "allow"), - }), - ]; - // OpenVMM exits with the status that its workload exits with, which the fake OpenVMM's - // `fake_exit_status=STATUS` sets. - for (name, status, command_line) in [ - ("exit status", 7, "fake_exit_status=7"), - ("exit status 3", 3, "fake_exit_status=3"), - ("timeout status", 124, "fake_exit_status=124"), - ("killed status", 137, "fake_exit_status=137"), - ] { - launches.push(launch( - name, - sandbox_run("/bin/true", &["--cmdline", command_line]), - workload(command_line, "/bin/true", &[]), - status, - "", - )); - } - launches -} - /// Returns the runs of `nvx-rs run`. fn run_launches() -> Vec { let booted = "openvmm: booted the microVM\n"; @@ -679,10 +496,7 @@ fn run_launches() -> Vec { #[test] fn one_shot_runs_exit_with_the_workload_or_openvmm_status() { - let cases: Vec = sandbox_launches() - .into_iter() - .chain(run_launches()) - .collect(); + let cases: Vec = run_launches(); let (vendor, family, model, stepping) = crate::reported_cpu(); let unserved = !crate::served(&vendor, family, model, stepping); in_parallel(&cases, |case| { @@ -767,27 +581,9 @@ fn one_shot_runs_exit_with_the_workload_or_openvmm_status() { if let Some(record) = record { assert_eq!(root.openvmm_arguments(&record), arguments, "{}", case.name); } - let mut expected = expected_cold + let expected = expected_cold .map(|expected| expected.unwrap()) .unwrap_or_else(|| case.options.clone()); - // A sandbox's OpenVMM opens the images through the files that the run checked: the distro - // layer alone for an upper in RAM, and the scratch image too for one on scratch. - let blocks = expected.take("--microvm-sandbox-block"); - if !blocks.is_empty() { - let checked = options.take("--microvm-sandbox-block"); - let token = case - .record - .and_then(|record| record.strip_prefix("build/fake-openvmm-")) - .and_then(|name| name.strip_suffix(".json")) - .unwrap(); - if blocks.len() == 1 { - assert_checked_distro(&root, &checked); - assert_opened_distro(&root, token); - } else { - assert_checked_blocks(&root, &checked); - assert_opened_images(&root, token); - } - } if cold { options = options.normalized_cold_outputs(true); } diff --git a/nvx-rs/tests/cli/openvmm.rs b/nvx-rs/tests/cli/openvmm.rs index 48b0adb9c..ffba28faf 100644 --- a/nvx-rs/tests/cli/openvmm.rs +++ b/nvx-rs/tests/cli/openvmm.rs @@ -578,10 +578,45 @@ impl Options { .with("--microvm-report", [""]); options } + + /// Returns these options of a `sandbox run` with the outputs of its VM directory named as + /// [`managed_run`] names them, once it has checked that the directory is a fresh one of the + /// run, which is gone once the run has ended, and that it holds the control endpoint on POSIX + /// hosts. + pub(crate) fn normalized_run_outputs(&self) -> Self { + let report = PathBuf::from(self.value("--microvm-report")); + assert!(report.is_absolute(), "{}", report.display()); + assert_eq!(report.file_name().unwrap(), "outcome.json"); + let parent = report.parent().unwrap(); + let name = parent.file_name().unwrap().to_str().unwrap(); + let suffix = name + .strip_prefix("nvx-sandbox-run-") + .expect("a run owns a fresh VM directory"); + assert!(is_hex(suffix, 32), "{name}"); + assert!( + !parent.exists(), + "the run left its VM directory: {}", + parent.display() + ); + let control = self + .value("--microvm-control-console") + .strip_prefix("listen=") + .unwrap(); + if cfg!(windows) { + let pipe = control.strip_prefix("//./pipe/openvmm-microvm-").unwrap(); + assert!(is_hex(pipe, 32), "{control}"); + } else { + assert_eq!(Path::new(control), parent.join("control.sock")); + } + self.clone() + .with("--microvm-control-console", ["listen="]) + .with("--microvm-report", [""]) + } } /// Returns the options with which a one-shot run of the minimal sandbox, `sandbox run --layer -/// LAYER --scratch SCRATCH`, starts OpenVMM, with the paths of the images that a dry run shows. +/// LAYER --scratch SCRATCH`, started OpenVMM, from which [`managed`] and [`managed_run`] derive +/// those of its managed launches. pub(crate) fn one_shot() -> Options { Options::default() .with("--machine", ["microvm"]) @@ -617,6 +652,19 @@ pub(crate) fn managed(state: &str, endpoint: &str) -> Options { ) } +/// Returns the options with which a run of the minimal sandbox, `sandbox run --layer LAYER +/// --scratch SCRATCH`, starts OpenVMM, in the form that [`Options::persistent_expected`] projects: +/// those of a start of the same sandbox, with the control endpoint and outcome report of the run's +/// own VM directory, as [`Options::normalized_run_outputs`] names them. +pub(crate) fn managed_run() -> Options { + one_shot() + .with("--microvm-lifecycle", ["managed"]) + .with("--virtio-console", ["none"]) + .with("--microvm-control-console", ["listen="]) + .flag("--microvm-control-auth-stdin") + .with("--microvm-report", [""]) +} + /// Returns the options with which `nvx-rs run` cold-boots its default guest, Alpine Linux, with /// its boot console on the terminal. pub(crate) fn cold_boot() -> Options { @@ -769,66 +817,6 @@ pub(crate) fn assert_stable_blocks(root: &Root, blocks: &[String], scratch: bool assert_eq!(actual, expected); } -/// Asserts that `blocks`, the `--microvm-sandbox-block` values of a launch, attach the distro -/// layer of the root read-only and then its scratch image through files that the launch checked: -/// on Linux, descriptors that OpenVMM inherits, which it names as `/proc/self/fd/N`, and on -/// Windows, the images' final paths. -pub(crate) fn assert_checked_blocks(root: &Root, blocks: &[String]) { - let [distro, scratch] = blocks else { - panic!("not two sandbox block devices: {blocks:?}"); - }; - let distro = checked_distro(distro); - let scratch = scratch - .strip_prefix("scratch:file:") - .unwrap_or_else(|| panic!("not a scratch disk: {scratch}")); - if cfg!(windows) { - assert_eq!( - distro, - format!(r"\\?\{}", root.expand("{root/distro.erofs}")) - ); - assert_eq!( - scratch, - format!(r"\\?\{}", root.expand("{root/scratch.ext4}")) - ); - } else { - assert_ne!(descriptor(distro), descriptor(scratch)); - } -} - -/// Asserts that `blocks`, the `--microvm-sandbox-block` values of a launch whose upper is in RAM, -/// attach the distro layer of the root alone, read-only, through a file that the launch checked, -/// as [`assert_checked_blocks`] says. -pub(crate) fn assert_checked_distro(root: &Root, blocks: &[String]) { - let [distro] = blocks else { - panic!("not one sandbox block device: {blocks:?}"); - }; - let distro = checked_distro(distro); - if cfg!(windows) { - assert_eq!( - distro, - format!(r"\\?\{}", root.expand("{root/distro.erofs}")) - ); - } else { - descriptor(distro); - } -} - -/// Returns the checked file of `block`, a `--microvm-sandbox-block` value that attaches a -/// read-only distro layer. -fn checked_distro(block: &str) -> &str { - block - .strip_prefix("distro:file:") - .and_then(|rest| rest.strip_suffix(",ro")) - .unwrap_or_else(|| panic!("not a read-only distro layer: {block}")) -} - -/// Returns the number of the descriptor that `path`, `/proc/self/fd/N`, names. -fn descriptor(path: &str) -> u32 { - path.strip_prefix("/proc/self/fd/") - .and_then(|number| number.parse::().ok()) - .unwrap_or_else(|| panic!("not an inherited descriptor: {path}")) -} - /// Asserts that the fake OpenVMM of the launch whose records are named for `token` read the /// distro layer and then the scratch image of the root, whole, through its sandbox block devices. pub(crate) fn assert_opened_images(root: &Root, token: &str) { diff --git a/nvx-rs/tests/cli/sandbox_run.rs b/nvx-rs/tests/cli/sandbox_run.rs new file mode 100644 index 000000000..8ed207373 --- /dev/null +++ b/nvx-rs/tests/cli/sandbox_run.rs @@ -0,0 +1,597 @@ +//! The runs of `nvx-rs sandbox run`, each in a sandbox of its own, without a state directory: a +//! run launches the VM that a `provision` and a `start` of its options would launch, in a fresh +//! temporary VM directory, prints its OpenVMM command as `>> COMMAND`, executes its workload once +//! over NVXC with the execution options of `exec`, stops the VM as `stop` does, removes the VM +//! directory, and exits with the workload's status after the workload's output. + +use std::fs; +use std::path::Path; + +use serde_json::json; + +use crate::harness::{ + HYPERVISOR, LAYER, LINE_END, OPENVMM, Outcome, Root, SCRATCH, Snapshot, UUID, in_parallel, +}; +use crate::lifecycle::{OWN_FILES, exported, mapped, own_files_root, share_host}; +use crate::openvmm::{ + NO_NETWORK, Options, assert_opened_distro, assert_opened_images, assert_outcome_report, + assert_stable_blocks, managed_run, +}; + +/// Returns `nvx-rs sandbox run` of the distro layer and scratch image of the root with `options`. +fn sandbox_run(options: &[&str]) -> Vec { + ["sandbox", "run", "--layer", LAYER, "--scratch", SCRATCH] + .iter() + .chain(options) + .map(|argument| (*argument).to_owned()) + .collect() +} + +/// What a run printed of the VM that it launched. +struct Ran { + /// The options of the OpenVMM command that it printed, with the outputs of its VM directory + /// named as [`managed_run`] names them. + options: Options, + /// The name of its VM directory, which names the records of the fake OpenVMM. + token: String, +} + +/// Returns `snapshot`, a snapshot of a root, without the records of the fake OpenVMM, the outcome +/// reports, and the lock file of the root's downloads. +fn kept(snapshot: &Snapshot) -> Snapshot { + snapshot + .iter() + .filter(|(path, _)| { + !path.starts_with("build/fake-openvmm-") + && !path.starts_with("reports/") + && *path != "build/.nvx-download.lock" + }) + .map(|(path, node)| (path.clone(), node.clone())) + .collect() +} + +/// Returns the options of the OpenVMM command that `outcome`, a run in `root`, printed, with the +/// outputs of its VM directory named as [`managed_run`] names them, once it has checked that the +/// run exited with `status` after the workload's standard output `stdout`, that the fake OpenVMM +/// received the command, and that the run left the root as it held `before`, but for the records +/// of the fake OpenVMM and the outcome reports: without a VM directory or a state directory. +fn assert_launched( + root: &Root, + before: &Snapshot, + outcome: &Outcome, + status: i32, + stdout: &str, +) -> Ran { + assert_eq!(outcome.status, Some(status), "{}", outcome.describe()); + let (command, rest) = outcome.command(); + assert_eq!(rest, stdout.as_bytes(), "{}", outcome.describe()); + let (program, arguments) = command.split_first().unwrap(); + assert_eq!(*program, root.below(OPENVMM).to_str().unwrap()); + let options = Options::parse(arguments); + let report = Path::new(options.value("--microvm-report")); + let token = report + .parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + .unwrap() + .to_owned(); + assert_eq!( + root.openvmm_arguments(&format!("build/fake-openvmm-{token}.json")), + arguments, + "{}", + outcome.describe() + ); + assert_eq!( + kept(&root.snapshot()), + kept(before), + "{}", + outcome.describe() + ); + Ran { + options: options.normalized_run_outputs(), + token, + } +} + +/// Asserts that `outcome`, a run in `root`, which held `before`, ran its workload as +/// [`assert_launched`] says, in a VM whose command line holds `nvx_sandbox=1` and the CLI's managed +/// mode, and no OpenVMM lifecycle or workload option, or token of the workload, and that the run +/// attached to the VM once, through which it executed its workload and then stopped the VM. +fn assert_ran(root: &Root, before: &Snapshot, outcome: &Outcome, status: i32, stdout: &str) -> Ran { + let ran = assert_launched(root, before, outcome, status, stdout); + for option in ["--microvm-lifecycle", "--microvm-workload-identity"] { + assert!( + ran.options.values(option).is_empty(), + "{option}: {}", + outcome.describe() + ); + } + assert_eq!(ran.options.value("--virtio-console"), "none"); + assert_eq!(ran.options.value("--guest-shutdown-action"), "exit"); + let line = ran.options.value("--cmdline"); + let tokens: Vec<&str> = line.split_ascii_whitespace().collect(); + assert_eq!( + tokens + .iter() + .filter(|&&token| token == "nvx_sandbox=1") + .count(), + 1, + "{line}" + ); + assert!(tokens.contains(&"nvx_mode=managed"), "{line}"); + // The workload is the execution's, which no token of the kernel command line carries. + assert!( + !tokens + .iter() + .any(|token| { token.starts_with("nvx_entrypoint=") || token.starts_with("nvx_arg=") }), + "{line}" + ); + assert_eq!( + fs::read_to_string(root.below(&format!("build/fake-openvmm-{}-sessions.log", ran.token))) + .unwrap(), + "connection\n", + "{}", + outcome.describe() + ); + ran +} + +#[test] +fn runs_of_every_format_and_upper_execute_once_and_stop_their_vm() { + let mut cases = Vec::new(); + for format in ["erofs", "gpt"] { + for (upper, cap, expected_cap) in [ + ("scratch", None, None), + ("ram", None, Some(50)), + ("ram", Some("37"), Some(37)), + ] { + cases.push((format, upper, cap, expected_cap)); + } + } + in_parallel(&cases, |&(format, upper, cap, expected_cap)| { + let context = format!("{format}/{upper}/{cap:?}"); + let root = Root::new(); + let mut arguments = vec![ + "sandbox", + "run", + "--layer", + LAYER, + "--layer-format", + format, + "--upper", + upper, + ]; + if upper == "scratch" { + arguments.extend(["--scratch", SCRATCH]); + } else { + // A RAM upper needs no scratch image. + root.remove("scratch.ext4"); + } + if let Some(cap) = cap { + arguments.extend(["--ram-upper-percent", cap]); + } + arguments.extend([ + "--entrypoint", + "/bin/echo", + "--arg", + "hello", + "--arg", + "world", + "--outcome-report", + "{root/reports/run.json}", + ]); + let before = root.snapshot(); + let outcome = root.nvx_rs(&arguments); + let mut ran = assert_ran(&root, &before, &outcome, 0, "hello world\n"); + assert!(outcome.stderr.is_empty(), "{}", outcome.describe()); + assert_stable_blocks( + &root, + &ran.options.take("--microvm-sandbox-block"), + upper == "scratch", + ); + if upper == "scratch" { + assert_opened_images(&root, &ran.token); + } else { + assert_opened_distro(&root, &ran.token); + } + let line = ran.options.value("--cmdline"); + let tokens: Vec<&str> = line.split_ascii_whitespace().collect(); + assert!( + tokens.contains(&format!("nvx_layer=distro,0xd0003000,{format}:{UUID}").as_str()), + "{context}: {line}" + ); + if let Some(cap) = expected_cap { + assert!( + tokens.contains(&format!("nvx_ram_upper_percent={cap}").as_str()), + "{context}: {line}" + ); + assert!( + tokens.contains(&"nvx_overlay_upper=ramfs"), + "{context}: {line}" + ); + assert!( + !tokens.iter().any(|token| token.starts_with("nvx_scratch")), + "{context}: {line}" + ); + } else { + assert!( + tokens.contains(&"nvx_scratch=0xd0006000,ext4"), + "{context}: {line}" + ); + assert!( + !tokens + .iter() + .any(|token| token.starts_with("nvx_ram_upper_percent=")), + "{context}: {line}" + ); + } + // The guest stopped the VM, as `stop` asks it to, and the run forwards OpenVMM's report. + assert_outcome_report( + &root.json("reports/run.json"), + HYPERVISOR, + ("run", "success", 0), + NO_NETWORK, + ); + }); +} + +#[test] +fn runs_execute_their_workload_with_the_options_of_exec() { + let path = "/usr/sbin:/usr/bin:/sbin:/bin\n"; + let cases: [(&[&str], i32, &str, &str); 9] = [ + (&["--entrypoint", "/bin/pwd"], 0, "/\n", ""), + ( + &["--entrypoint", "/bin/pwd", "--cwd", "/tmp"], + 0, + "/tmp\n", + "", + ), + ( + &[ + "--entrypoint", + "/usr/bin/env", + "--environment", + "A=1", + "--environment", + "B=two words", + ], + 0, + "A=1\nB=two words\n", + "", + ), + ( + &[ + "--entrypoint", + "/usr/bin/env", + "--environment-file", + "{root/env.json}", + ], + 0, + "X=1\nY=two words\n", + "", + ), + // The guest's default environment, without an environment of the run, beneath it with + // --inherit-default-environment, and replaced by it otherwise. + ( + &["--entrypoint", "/usr/bin/printenv", "--arg", "PATH"], + 0, + path, + "", + ), + ( + &[ + "--entrypoint", + "/usr/bin/printenv", + "--arg", + "PATH", + "--environment", + "A=1", + "--inherit-default-environment", + ], + 0, + path, + "", + ), + ( + &[ + "--entrypoint", + "/usr/bin/printenv", + "--arg", + "PATH", + "--environment", + "A=1", + ], + 1, + "", + "", + ), + // The arguments may hold whitespace, and the workload's status and standard error reach + // the caller. + ( + &[ + "--entrypoint", + "/bin/sh", + "--arg=-c", + "--arg", + "echo two words; echoerr oops; exit 7", + ], + 7, + "two words\n", + "oops\n", + ), + ( + &[ + "--entrypoint", + "/bin/sh", + "--arg=-c", + "--arg", + "sleep 60000", + "--exec-timeout-ms", + "50", + ], + 124, + "", + "", + ), + ]; + in_parallel(&cases, |&(options, status, stdout, stderr)| { + let root = Root::new(); + root.write("env.json", r#"["X=1","Y=two words"]"#); + let before = root.snapshot(); + let outcome = root.nvx_rs(&sandbox_run(options)); + assert_ran(&root, &before, &outcome, status, stdout); + assert_eq!(outcome.stderr(), stderr, "{}", outcome.describe()); + }); +} + +#[test] +fn runs_map_their_shares_before_they_execute_their_workload() { + let root = Root::new(); + let before = root.snapshot(); + let outcome = root.nvx_rs(&sandbox_run(&[ + "--mount", + "/data,{root/share}", + "--entrypoint", + "/bin/pwd", + "--cwd", + "/data", + ])); + let mut ran = assert_ran(&root, &before, &outcome, 0, "/data\n"); + assert!(outcome.stderr.is_empty(), "{}", outcome.describe()); + // The VM is the one that a start of the same sandbox launches. + assert_stable_blocks(&root, &ran.options.take("--microvm-sandbox-block"), true); + let mut expected = exported(managed_run(), &["0,{root/share},ro"]).persistent_expected(); + expected.take("--microvm-sandbox-block"); + expected.assert_matches(&root, &ran.options, "one share"); + assert_eq!( + root.json(&format!("build/fake-openvmm-{}-maps.json", ran.token)), + json!([mapped("0", "/data", true, true)]) + ); +} + +/// A run refuses a share that reaches the image, scratch image, OpenVMM, kernel, or initramfs of +/// its VM before it launches anything, as `start` does. It has no state directory, so a share of +/// a directory that would be a sandbox's state directory is one like any other. +#[test] +fn runs_refuse_shares_of_their_own_files() { + in_parallel(&OWN_FILES, |&(host, name)| { + let root = own_files_root(); + fs::create_dir_all(root.below("control/state/inside")).unwrap(); + let host = share_host(&root, host); + let layer = format!("distro,{{root/images/distro.erofs}},{UUID}"); + let mount = format!("/data,{host}"); + let arguments = [ + "sandbox", + "run", + "--layer", + &layer, + "--scratch", + "{root/upper/scratch.ext4}", + "--mount", + &mount, + "--entrypoint", + "/bin/pwd", + "--cwd", + "/data", + ]; + let before = root.snapshot(); + let outcome = root.nvx_rs(&arguments); + if name == "state directory" { + assert_ran(&root, &before, &outcome, 0, "/data\n"); + } else { + outcome.assert_error("unsupported", &[&format!("overlaps the sandbox's {name} ")]); + assert_eq!(root.snapshot(), before, "{host}"); + } + }); +} + +/// A run runs the OpenVMM that `NVX_OPENVMM` names, if set, rather than the root's, so it refuses a +/// share that reaches that one too, through which the workload could change what the host runs. +#[test] +fn a_run_refuses_a_share_of_the_openvmm_that_the_variable_names() { + let root = Root::new(); + root.write("tools/openvmm{exe}", "openvmm\n"); + let before = root.snapshot(); + root.nvx_rs_with( + &sandbox_run(&["--mount", "/data,{root/tools}"]), + &[("NVX_OPENVMM", "{root/tools/openvmm{exe}}")], + ) + .assert_error("unsupported", &["overlaps the sandbox's OpenVMM "]); + assert_eq!(root.snapshot(), before); +} + +/// A run refuses a share that reaches the VM directory that it would create in the temporary +/// directory, which holds what OpenVMM writes, as `start` refuses one of a state directory. +#[test] +fn a_run_refuses_a_share_of_its_vm_directory() { + let root = Root::new(); + fs::create_dir(root.below("temporary")).unwrap(); + let before = root.snapshot(); + root.nvx_rs_with( + &sandbox_run(&["--mount", "/data,{root/temporary}"]), + &[ + ("TMPDIR", "{root/temporary}"), + ("TMP", "{root/temporary}"), + ("TEMP", "{root/temporary}"), + ], + ) + .assert_error("unsupported", &["overlaps the sandbox's VM directory "]); + assert_eq!(root.snapshot(), before); +} + +#[test] +fn a_run_whose_guest_exits_during_startup_fails_as_a_start_does() { + let root = Root::new(); + let before = root.snapshot(); + let outcome = root.nvx_rs(&sandbox_run(&[ + "--entrypoint", + "/bin/pwd", + "--cmdline", + "fake_exit_during_attach=5", + "--outcome-report", + "{root/reports/run.json}", + ])); + assert_launched(&root, &before, &outcome, 1, ""); + assert_eq!( + outcome.stderr(), + format!("error[exited]: OpenVMM exited during startup: guest-exit status 5{LINE_END}") + ); + assert_outcome_report( + &root.json("reports/run.json"), + HYPERVISOR, + ("run", "guest-exit", 5), + NO_NETWORK, + ); +} + +#[test] +fn a_run_whose_workload_the_guest_refuses_stops_its_vm() { + let root = Root::new(); + let before = root.snapshot(); + let outcome = root.nvx_rs(&sandbox_run(&[ + "--entrypoint", + "/bin/pwd", + "--cwd", + "/missing", + "--outcome-report", + "{root/reports/run.json}", + ])); + assert_ran(&root, &before, &outcome, 1, ""); + assert_eq!( + outcome.stderr(), + format!( + "error[exec-refused]: managed guest rejected exec (status=2, category=cwd-failed)\ + {LINE_END}" + ) + ); + assert_outcome_report( + &root.json("reports/run.json"), + HYPERVISOR, + ("run", "success", 0), + NO_NETWORK, + ); +} + +/// A run whose guest closes its control endpoint as it stops, before its acknowledgement leaves, +/// waits up to `--timeout` for its VM to exit, as a stop does, rather than forcing it down at +/// once: the VM exits on its own with its report, and the run warns only of the failed request. +#[test] +fn a_run_waits_for_a_vm_whose_endpoint_closes_as_it_stops() { + let root = Root::new(); + let before = root.snapshot(); + let outcome = root.nvx_rs(&sandbox_run(&[ + "--entrypoint", + "/bin/echo", + "--arg", + "once", + "--cmdline", + "fake_drop_stop=1 fake_exit_delay_ms=500", + "--outcome-report", + "{root/reports/run.json}", + ])); + assert_ran(&root, &before, &outcome, 0, "once\n"); + assert_eq!( + outcome.stderr(), + format!("warning: managed guest shutdown failed: EndpointClosed{LINE_END}"), + "{}", + outcome.describe() + ); + assert_outcome_report( + &root.json("reports/run.json"), + HYPERVISOR, + ("run", "success", 0), + NO_NETWORK, + ); +} + +/// A run whose VM does not stop when asked to forces it down once `--timeout` has passed, and +/// warns that it did, but exits with its workload's status all the same. +#[test] +fn a_run_forces_down_a_vm_that_does_not_stop() { + let root = Root::new(); + let before = root.snapshot(); + let outcome = root.nvx_rs(&sandbox_run(&[ + "--entrypoint", + "/bin/echo", + "--arg", + "once", + "--timeout", + "1", + "--cmdline", + "fake_ignore_stop=1", + ])); + assert_ran(&root, &before, &outcome, 0, "once\n"); + let stderr = outcome.stderr(); + let lines: Vec<&str> = stderr.lines().collect(); + assert_eq!(lines.len(), 2, "{}", outcome.describe()); + assert!( + lines[0].starts_with("warning: managed guest shutdown failed: "), + "{}", + outcome.describe() + ); + assert_eq!( + lines[1], + "warning: OpenVMM was forcibly terminated after the run", + "{}", + outcome.describe() + ); +} + +/// A run forwards an outcome report that the caller can read whatever its umask, even one that +/// clears the owner's own permissions, and whose group's and others' permissions the caller's +/// umask narrows: OpenVMM starts with the caller's umask without the owner's bits, and the run +/// keeps the permissions of OpenVMM's report. +#[cfg(target_os = "linux")] +#[test] +fn runs_forward_an_outcome_report_that_the_caller_reads_whatever_its_umask() { + use crate::harness::mode; + use crate::openvmm::FAKE_REPORT_MODE; + + for umask in [0o022, 0o400, 0o200, 0o777] { + let root = Root::new(); + let outcome = root.nvx_rs_with_umask( + &sandbox_run(&[ + "--entrypoint", + "/bin/echo", + "--arg", + "once", + "--outcome-report", + "{root/reports/run.json}", + ]), + umask, + ); + assert_eq!(outcome.status, Some(0), "{}", outcome.describe()); + let report = root.below("reports/run.json"); + let (permissions, expected) = (mode(&report), FAKE_REPORT_MODE & !(umask & 0o077)); + assert_eq!( + permissions, expected, + "umask {umask:03o}: the report has mode {permissions:03o}, not {expected:03o}" + ); + assert_outcome_report( + &root.json("reports/run.json"), + HYPERVISOR, + ("run", "success", 0), + NO_NETWORK, + ); + } +} diff --git a/nvx-rs/tests/cli/validation.rs b/nvx-rs/tests/cli/validation.rs index 7cb9f24ff..d287b7e2a 100644 --- a/nvx-rs/tests/cli/validation.rs +++ b/nvx-rs/tests/cli/validation.rs @@ -135,7 +135,7 @@ fn shares(count: usize) -> Vec { fn sandbox_refusals() -> Vec { let network_outside = &["network policy options are only valid for sandbox run or provision"][..]; - let exec_options = &["managed execution options", "sandbox exec"][..]; + let exec_options = &["managed execution options", "sandbox run or exec"][..]; vec![ refusal( "dry run outside run", @@ -158,19 +158,19 @@ fn sandbox_refusals() -> Vec { ) .with_file("env.json", ENVIRONMENT), refusal( - "cwd outside exec", + "cwd outside run or exec", "unsupported", - ["sandbox", "run", "--cwd", "/"], + ["sandbox", "deprovision", "--cwd", "/"], exec_options, ), refusal( - "environment outside exec", + "environment outside run or exec", "unsupported", ["sandbox", "start", "--environment", "A=1"], exec_options, ), refusal( - "environment file outside exec", + "environment file outside run or exec", "unsupported", [ "sandbox", @@ -182,11 +182,24 @@ fn sandbox_refusals() -> Vec { ) .with_file("env.json", ENVIRONMENT), refusal( - "inherited environment outside exec", + "inherited environment outside run or exec", "unsupported", ["sandbox", "stop", "--inherit-default-environment"], exec_options, ), + // A run checks the execution of its workload, as exec does, before it launches anything. + refusal( + "relative working directory of run", + "unsupported", + sandbox_run(&["--cwd", "tmp"]), + &["working directory", "absolute"], + ), + refusal( + "environment entry without name of run", + "unsupported", + sandbox_run(&["--environment", "=x"]), + &["KEY=VALUE"], + ), refusal( "environment file too large", "unsupported", @@ -559,10 +572,12 @@ fn sandbox_refusals() -> Vec { sandbox_run(&["--hostname", "bad host"]), &["hostname", "RFC 1123"], ), + // A provision checks the arguments, though a run executes them over NVXC, where they may + // hold whitespace. refusal( "argument with space", "unsupported", - sandbox_run(&["--arg", "a b"]), + provision("{root/state}", &["--arg", "a b"]), &["arguments", "whitespace"], ), refusal( diff --git a/scripts/nvx_tools/sandbox_lifecycle_tests.py b/scripts/nvx_tools/sandbox_lifecycle_tests.py index 45aa7df7b..5ee5b6db7 100644 --- a/scripts/nvx_tools/sandbox_lifecycle_tests.py +++ b/scripts/nvx_tools/sandbox_lifecycle_tests.py @@ -100,7 +100,7 @@ def __str__(self) -> str: return f"error {self.code!r}{named}" -# A start that OpenVMM ended names the category of its outcome report. +# A start or a run that OpenVMM ended names the category of its outcome report. REFUSED_START = Refusal("exited", (REFUSAL_CATEGORY,)) # The files of a state directory, as nvx-rs sandbox documents them. @@ -628,8 +628,9 @@ def check_identity_admission(self) -> None: if self.state.exists(): raise RuntimeError("a rejected root identity created sandbox state") - # The guest refuses an identity that the image lacks before it starts - # the workload. + # The guest refuses an identity that the image lacks before it runs any + # workload, so a run fails as a managed start does, below, with the + # guest's status from the outcome report that OpenVMM publishes. run_scratch = self.root / "run-scratch.ext4" shutil.copyfile(self.artifacts.scratch_template, run_scratch) result = self.invoke( @@ -639,16 +640,15 @@ def check_identity_admission(self) -> None: UNAVAILABLE_IDENTITY, "--entrypoint", PROBE, + "--arg=init", state=None, - expected=125, + expected=1, + refusal=REFUSED_START, ) console = result.stdout + result.stderr self.save_evidence("identity-run.log", console) - if ( - b"configured workload UID is unavailable" not in console - or b"NVX-UBUNTU-SANDBOX-PROFILE-OK" in console - ): - raise RuntimeError("an unavailable one-shot identity was not refused") + if b"NVX-UBUNTU-SANDBOX-PROFILE-OK" in console: + raise RuntimeError("a run of an unavailable identity ran its workload") # A managed start with that identity fails closed. OpenVMM exits on its # own and publishes the guest's status, which the start reports, and diff --git a/scripts/nvx_tools/sandbox_ram_upper_tests.py b/scripts/nvx_tools/sandbox_ram_upper_tests.py index 5ecf989e4..52fa0de9e 100644 --- a/scripts/nvx_tools/sandbox_ram_upper_tests.py +++ b/scripts/nvx_tools/sandbox_ram_upper_tests.py @@ -4,13 +4,14 @@ real kernel, the Alpine control initramfs, and the Ubuntu EROFS layer, with no scratch image. It checks that the planner refuses `--upper ram` beside `--scratch`, and `--upper` outside `run` and `provision`, before it creates -any state. A one-shot run attaches the distro layer alone, declares the RAM -upper on the kernel command line, passes the smoke probe, which requires the -overlay's upper on a tmpfs, and reports its success. A managed sandbox records -the RAM upper without a scratch image, and passes the probe's managed checks. -Its guest has one virtio block device and declares the RAM upper, and a later -request reads a file that an earlier one wrote. Once the sandbox stops and -starts again, the file is gone. +any state. A run, which keeps no state, attaches the distro layer alone, +declares the RAM upper and its default cap on the kernel command line of a VM +that nvx-init manages, runs the smoke probe once over NVXC, which requires the +overlay's upper on a tmpfs, and forwards the report of the VM's graceful stop. +A managed sandbox records the RAM upper without a scratch image, and passes the +probe's managed checks. Its guest has one virtio block device and declares the +RAM upper, and a later request reads a file that an earlier one wrote. Once the +sandbox stops and starts again, the file is gone. """ from __future__ import annotations @@ -36,8 +37,15 @@ MEMORY_MIB = 256 RAM_UPPER_TOKEN = "nvx_overlay_upper=ramfs" +# The cap of a RAM upper whose run gives none: half of the guest's RAM. +DEFAULT_CAP_TOKEN = "nvx_ram_upper_percent=50" SCRATCH_TOKEN = "nvx_scratch=" -# The outcome that OpenVMM reports for a one-shot run whose workload succeeded. +# The tokens with which a run's VM boots its layered root through nvx-init, +# which owns the VM's lifecycle, so that the run passes OpenVMM no lifecycle. +MANAGED_TOKENS = ("nvx_sandbox=1", "nvx_mode=managed") +LIFECYCLE_OPTION = "--microvm-lifecycle" +# The outcome that OpenVMM reports for a run whose guest stopped its VM, as it +# does for a managed sandbox's stop. RUN_SUCCESS = {"operation": "run", "category": "success", "status_code": 0} STATE_PATH = "/tmp/nvx-ram-upper-state" STATE_CONTENT = b"kept in RAM\n" @@ -68,20 +76,18 @@ def check_ram_tokens(command_line: str, context: str) -> None: def check_run_report(path: Path, backend: str) -> None: - """Checks the outcome report of a one-shot run whose workload succeeded and - whose VM was torn down.""" + """Checks the outcome report that a run forwards, of a VM that its guest + stopped and that was torn down.""" try: report: object = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError) as error: - raise RuntimeError("the one-shot run published no outcome report") from error + raise RuntimeError("the run forwarded no outcome report") from error if not isinstance(report, dict): - raise RuntimeError(f"the one-shot run's outcome report is {report!r}") + raise RuntimeError(f"the run's outcome report is {report!r}") fields = cast(dict[str, object], report) outcome = fields.get("outcome") if outcome != RUN_SUCCESS or fields.get("backend") != backend: - raise RuntimeError( - f"the one-shot run reported {outcome!r} on {fields.get('backend')!r}" - ) + raise RuntimeError(f"the run reported {outcome!r} on {fields.get('backend')!r}") # microvm_tests, which runs this scenario, imports this module. from .microvm_tests import OUTCOME_TEARDOWN_FIELDS @@ -93,7 +99,7 @@ def check_run_report(path: Path, backend: str) -> None: value is True for value in cast(dict[str, object], teardown).values() ) ): - raise RuntimeError(f"the one-shot run's teardown is {teardown!r}") + raise RuntimeError(f"the run's teardown is {teardown!r}") class RamUpperAcceptance(LifecycleAcceptance): @@ -113,7 +119,7 @@ def ram_options(self) -> list[str]: def run(self) -> None: self.check_refusals() - self.check_one_shot() + self.check_run() self.check_managed() self.require_processes((), "after the RAM-upper sandboxes") @@ -141,20 +147,28 @@ def check_refusals(self) -> None: if self.state.exists(): raise RuntimeError("a refused RAM-upper request created sandbox state") - def check_one_shot(self) -> None: - """A one-shot run attaches the distro layer alone, declares the RAM - upper, passes the smoke probe, and reports its success.""" - # The report's path, under the fixture, also names the run's OpenVMM to - # the process checks and to cleanup, as no other argument does: the - # distro layer is the build's, and the run has no scratch image. + def check_run(self) -> None: + """A run, which keeps no state, attaches the distro layer alone, + declares the RAM upper and its default cap in the VM that nvx-init + manages, passes the probe's managed checks once, and forwards the + report of the VM's graceful stop.""" + # The report's path lies under the fixture, and the kernel command line + # names the fixture, which names the run's OpenVMM to the process checks + # and to cleanup, as no other argument does: the distro layer is the + # build's, the run has no scratch image, and its VM directory and the + # report that OpenVMM writes there are the run's own. report = self.root / "run-outcome.json" result = self.invoke( "run", *self.ram_options(), "--outcome-report", str(report), + "--cmdline", + f"sandbox_fixture={self.root.name}", "--entrypoint", PROBE, + "--arg=init", + "--arg=entry", state=None, ) console = result.stdout + result.stderr @@ -168,22 +182,33 @@ def check_one_shot(self) -> None: or "scratch:file:" in command ): raise RuntimeError( - f"the one-shot run did not attach its distro layer alone: {command!r}" + f"the run did not attach its distro layer alone: {command!r}" ) if self.root.name not in command: raise RuntimeError( - f"the one-shot run's OpenVMM command does not name the fixture: " - f"{command!r}" + f"the run's OpenVMM command does not name the fixture: {command!r}" + ) + check_ram_tokens(command, "the run's OpenVMM command") + tokens = [token.strip("'\"") for token in command.split()] + if ( + LIFECYCLE_OPTION in tokens + or DEFAULT_CAP_TOKEN not in tokens + or not all(token in tokens for token in MANAGED_TOKENS) + ): + raise RuntimeError( + f"the run's VM is not one that nvx-init manages with the RAM " + f"upper's default cap: {command!r}" ) - check_ram_tokens(command, "the one-shot run's OpenVMM command") - for marker in probe_markers(entry=False): + for marker in probe_markers(entry=True): if marker not in result.stdout: raise RuntimeError( - f"the one-shot RAM-upper probe did not report {marker.decode()}" + f"the run's RAM-upper probe did not report {marker.decode()}" ) self.copy_evidence(report, "run-outcome.json") check_run_report(report, self.backend) - self.require_processes((), "after the one-shot run") + if self.state.exists(): + raise RuntimeError("the run created sandbox state") + self.require_processes((), "after the run") def check_managed(self) -> None: """A managed sandbox records the RAM upper without a scratch image, and diff --git a/scripts/test-layout-allowlist.json b/scripts/test-layout-allowlist.json index 19e7ed4eb..5c0555d1e 100644 --- a/scripts/test-layout-allowlist.json +++ b/scripts/test-layout-allowlist.json @@ -15,7 +15,6 @@ "nvx-rs/src/host/network/address.rs": 2, "nvx-rs/src/host/platform/windows/spawn.rs": 1, "nvx-rs/src/host/restore_driver.rs": 1, - "nvx-rs/src/host/sandbox/plan.rs": 5, "nvx-rs/src/host/session.rs": 1, "nvx-rs/src/host/vm.rs": 2, "nvx-rs/src/protocol/app.rs": 6 diff --git a/scripts/test_microvm_tests.py b/scripts/test_microvm_tests.py index 21e750912..34640cd12 100644 --- a/scripts/test_microvm_tests.py +++ b/scripts/test_microvm_tests.py @@ -1110,16 +1110,22 @@ def _other_runtime( ) def _run( - self, state: Path | None, values: dict[str, str], _: list[str] + self, state: Path | None, values: dict[str, str], arguments: list[str] ) -> _CliResult: assert state is None assert values["--workload-user"] == sandbox_lifecycle_tests.UNAVAILABLE_IDENTITY + assert arguments == ["init"] if "run-starts-workload" in self.faults: - return 0, b"NVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534\n", b"" + return ( + 0, + b">> openvmm\nNVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534\n", + b"", + ) + # The guest refuses the identity before it is ready, as at a start. return ( - 125, - b">> openvmm\nNVX-SANDBOX-ERROR: configured workload UID is unavailable\n", - b"", + 1, + b">> openvmm\n", + b"error[exited]: OpenVMM exited during startup: guest-exit status 125\n", ) def _provision( @@ -1688,7 +1694,7 @@ def test_acceptance_drives_the_complete_public_lifecycle(self): def test_acceptance_detects_lifecycle_regressions(self): regressions = { - "run-starts-workload": "run returned 0, expected 125", + "run-starts-workload": "run returned 0, expected 1", "failed-start-keeps-runtime": ( "after a refused managed start: the state directory holds " "['config.json', 'openvmm.log', 'outcome.json', 'runtime.json', " @@ -2676,6 +2682,7 @@ class _FakeRamUpperSandboxCli(_FakeSandboxCli): SMOKE_RUN = ( b"NVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534\n" + + b"NVX-UBUNTU-SANDBOX-ENTRY-OK\n" + b"NVX-UBUNTU-SANDBOX-OK uid=65534 gid=65534\n" ) @@ -2688,29 +2695,33 @@ def _command_line(self) -> str: return f"nvx_sandbox=1 nvx_layer=distro,0xd0003000,erofs {upper}" def _run( - self, state: Path | None, values: dict[str, str], _: list[str] + self, state: Path | None, values: dict[str, str], arguments: list[str] ) -> _CliResult: assert state is None assert values["--upper"] == "ram" assert values["--entrypoint"] == sandbox_lifecycle_tests.PROBE - assert _ == [] + assert arguments == ["init", "entry"] blocks = f"--microvm-sandbox-block distro:file:{values['--layer']},ro" if "attaches-scratch" in self.faults: blocks += " --microvm-sandbox-block scratch:file:scratch.ext4" command = f">> openvmm {blocks}" + if "uses-lifecycle" in self.faults: + command += " --microvm-lifecycle managed" + # OpenVMM writes its report in the run's own VM directory, and the run + # forwards it. + command += " --microvm-report /tmp/nvx-sandbox-run-0/outcome.json" + report = Path(values["--outcome-report"]) + outcome = ( + {"operation": "run", "category": "guest-exit", "status_code": 1} + if "run-reports-failure" in self.faults + else sandbox_ram_upper_tests.RUN_SUCCESS + ) + teardown = dict.fromkeys(microvm_tests.OUTCOME_TEARDOWN_FIELDS, True) + if "partial-teardown" in self.faults: + del teardown["virtiofs_released"] + if "incomplete-teardown" in self.faults: + teardown["openvmm_process_terminated"] = False if "drops-report" not in self.faults: - report = Path(values["--outcome-report"]) - command += f" --microvm-report {report}" - outcome = ( - {"operation": "run", "category": "guest-exit", "status_code": 1} - if "run-reports-failure" in self.faults - else sandbox_ram_upper_tests.RUN_SUCCESS - ) - teardown = dict.fromkeys(microvm_tests.OUTCOME_TEARDOWN_FIELDS, True) - if "partial-teardown" in self.faults: - del teardown["virtiofs_released"] - if "incomplete-teardown" in self.faults: - teardown["openvmm_process_terminated"] = False report.write_text( json.dumps( { @@ -2721,7 +2732,12 @@ def _run( ), encoding="utf-8", ) - command += f" --cmdline '{self._command_line()}'\n" + cap = 51 if "other-cap" in self.faults else 50 + user = "" if "drops-fixture" in self.faults else f" {values['--cmdline']}" + command += ( + f" --cmdline '{self._command_line()} nvx_ram_upper_percent={cap}" + f" nvx_sandbox=1 nvx_mode=managed nvx_identity=65534:65534{user}'\n" + ) output = b"" if "probe-fails" in self.faults else self.SMOKE_RUN return 0, command.encode() + output, b"" @@ -2796,7 +2812,7 @@ class PublicSandboxRamUpperTests(unittest.TestCase): # outside run and provision. "provision", "start", - # A one-shot run. + # A run, which keeps no state. "run", # A managed sandbox: the probe, the guest's view, and a written and a # read file. @@ -2918,11 +2934,14 @@ def test_acceptance_detects_ram_upper_regressions(self): "start-takes-upper": "with error 'unsupported'", "attaches-scratch": "did not attach its distro layer alone", "declares-scratch": "does not declare the RAM upper alone", - "probe-fails": "the one-shot RAM-upper probe did not report", - "drops-report": "the one-shot run's OpenVMM command does not name", - "run-reports-failure": "the one-shot run reported", - "partial-teardown": "the one-shot run's teardown is", - "incomplete-teardown": "the one-shot run's teardown is", + "drops-fixture": "the run's OpenVMM command does not name the fixture", + "uses-lifecycle": "is not one that nvx-init manages", + "other-cap": "is not one that nvx-init manages", + "probe-fails": "the run's RAM-upper probe did not report", + "drops-report": "the run forwarded no outcome report", + "run-reports-failure": "the run reported", + "partial-teardown": "the run's teardown is", + "incomplete-teardown": "the run's teardown is", "records-scratch": "provision recorded the RAM upper as", "records-old-format": "provision recorded the RAM upper as", "records-other-cap": "provision recorded the RAM upper as",