SL3.5b: migrate persistent sandbox lifecycle to CLI-owned VM state

.github/workflows/run-nvx-microvm-tests.yml

@@ -452,25 +452,21 @@ jobs:
           nvx-rs/target/release/nvx-rs sandbox exec \
             --state-dir "${state}" \
             --entrypoint /sbin/nvx-sandbox-smoke \
+            --arg init \
             --arg /workspace \
             --arg rw
           nvx-rs/target/release/nvx-rs sandbox stop --state-dir "${state}"
-          # A clean ext4 journal shows that stop unmounted the share, overlay,
-          # and scratch in order instead of powering off with them mounted.
-          python3 - "${state}/outcome.json" "${scratch}" <<'EOF'
+          # nvx-init owns the sandbox VM's lifecycle, so OpenVMM, which starts
+          # it without a lifecycle option, reports a run.
+          python3 - "${state}/outcome.json" <<'EOF'
           import json
-          import struct
           import sys
 
           with open(sys.argv[1], encoding="utf-8") as report:
               outcome = json.load(report)["outcome"]
-          expected = {"operation": "managed", "category": "success", "status_code": 0}
+          expected = {"operation": "run", "category": "success", "status_code": 0}
           if outcome != expected:
               sys.exit(f"managed sandbox outcome is invalid: {outcome}")
-          with open(sys.argv[2], "rb") as image:
-              image.seek(1024 + 0x60)
-              if struct.unpack("<I", image.read(4))[0] & 0x4:
-                  sys.exit("managed stop left the scratch filesystem mounted")
           EOF
           test "$(cat "${share}/nvx-guest-marker")" = guest-to-host
           test -d "${share}/nvx-guest-directory"
@@ -550,13 +546,18 @@ jobs:
           if config["format"] != 8 or config["export"]["owner"] != "caller":
               sys.exit(f"managed sandbox did not persist caller ownership: {config}")
           EOF
-          if nvx-rs/target/release/nvx-rs sandbox start --state-dir "${state}"; then
+          errors=$(mktemp)
+          if nvx-rs/target/release/nvx-rs sandbox start --state-dir "${state}" 2>"${errors}"; then
             echo "the caller-owned managed sandbox reached its share as the runner" >&2
             exit 1
           fi
+          cat "${errors}" >&2
           grep -q 'virtio-fs request cannot run as its caller; failing it with EPERM' \
             "${state}/openvmm.log"
-          grep -q 'NVX-MANAGED-ERROR: stage=host-mappings' "${state}/openvmm.log"
+          # The guest refuses to map the share with EPERM, 1, which start
+          # reports.
+          grep -q 'could not mount a mapped host path (error 1)' "${errors}"
+          rm -f "${errors}"
           test ! -e "${state}/runtime.json"
           test "$(ls -A "${share}")" = nvx-host-marker
           nvx-rs/target/release/nvx-rs sandbox deprovision --state-dir "${state}"
@@ -690,24 +691,17 @@ jobs:
           EOF
           nvx-rs/target/release/nvx-rs sandbox start --state-dir "${state}"
           nvx-rs/target/release/nvx-rs sandbox exec --state-dir "${state}" \
-            "${smoke_arguments[@]}"
+            --arg init "${smoke_arguments[@]}"
           nvx-rs/target/release/nvx-rs sandbox stop --state-dir "${state}"
-          # A clean ext4 journal shows that stop unmounted every share, the
-          # overlay, and scratch in order.
-          python3 - "${state}/outcome.json" "${scratch}" <<'EOF'
+          python3 - "${state}/outcome.json" <<'EOF'
           import json
-          import struct
           import sys
 
           with open(sys.argv[1], encoding="utf-8") as report:
               outcome = json.load(report)["outcome"]
-          expected = {"operation": "managed", "category": "success", "status_code": 0}
+          expected = {"operation": "run", "category": "success", "status_code": 0}
           if outcome != expected:
               sys.exit(f"managed sandbox outcome is invalid: {outcome}")
-          with open(sys.argv[2], "rb") as image:
-              image.seek(1024 + 0x60)
-              if struct.unpack("<I", image.read(4))[0] & 0x4:
-                  sys.exit("managed stop left the scratch filesystem mounted")
           EOF
           check_shares
           nvx-rs/target/release/nvx-rs sandbox deprovision --state-dir "${state}"
@@ -826,24 +820,17 @@ jobs:
           EOF
           nvx-rs/target/release/nvx-rs sandbox start --state-dir "${state}"
           nvx-rs/target/release/nvx-rs sandbox exec --state-dir "${state}" \
-            "${smoke_arguments[@]}"
+            --arg init "${smoke_arguments[@]}"
           nvx-rs/target/release/nvx-rs sandbox stop --state-dir "${state}"
-          # A clean ext4 journal shows that stop unmounted the share, the
-          # overlay, and scratch in order.
-          python3 - "${state}/outcome.json" "${scratch}" <<'EOF'
+          python3 - "${state}/outcome.json" <<'EOF'
           import json
-          import struct
           import sys
 
           with open(sys.argv[1], encoding="utf-8") as report:
               outcome = json.load(report)["outcome"]
-          expected = {"operation": "managed", "category": "success", "status_code": 0}
+          expected = {"operation": "run", "category": "success", "status_code": 0}
           if outcome != expected:
               sys.exit(f"managed sandbox outcome is invalid: {outcome}")
-          with open(sys.argv[2], "rb") as image:
-              image.seek(1024 + 0x60)
-              if struct.unpack("<I", image.read(4))[0] & 0x4:
-                  sys.exit("managed stop left the scratch filesystem mounted")
           EOF
           check_share
           nvx-rs/target/release/nvx-rs sandbox deprovision --state-dir "${state}"
@@ -982,6 +969,7 @@ jobs:
               Invoke-Nvx sandbox exec `
                 --state-dir $state `
                 --entrypoint /sbin/nvx-sandbox-smoke `
+                --arg init `
                 --arg /workspace `
                 --arg rw
               Invoke-Nvx sandbox stop --state-dir $state
@@ -993,23 +981,12 @@ jobs:
               $log = Join-Path $state "openvmm.log"
               if (Test-Path $log) { Copy-Item -Force $log (Join-Path $logs "managed-share-openvmm.log") }
           }
+          # nvx-init owns the sandbox VM's lifecycle, so OpenVMM, which starts
+          # it without a lifecycle option, reports a run.
           $outcome = (Get-Content -Raw (Join-Path $state "outcome.json") | ConvertFrom-Json).outcome
-          if ($outcome.operation -ne "managed" -or $outcome.category -ne "success" -or $outcome.status_code -ne 0) {
+          if ($outcome.operation -ne "run" -or $outcome.category -ne "success" -or $outcome.status_code -ne 0) {
               throw "managed sandbox outcome is invalid: $($outcome | ConvertTo-Json -Compress)"
           }
-          # A clean ext4 journal shows that stop unmounted the share, overlay,
-          # and scratch in order instead of powering off with them mounted.
-          $image = [IO.File]::OpenRead($scratch)
-          try {
-              $null = $image.Seek(1024 + 0x60, [IO.SeekOrigin]::Begin)
-              $incompat = New-Object byte[] 4
-              if ($image.Read($incompat, 0, 4) -ne 4) { throw "scratch superblock is truncated" }
-          } finally {
-              $image.Dispose()
-          }
-          if ([BitConverter]::ToUInt32($incompat, 0) -band 4) {
-              throw "managed stop left the scratch filesystem mounted"
-          }
           $marker = [IO.File]::ReadAllText((Join-Path $share "nvx-guest-marker"))
           if ($marker -ne "guest-to-host`n") { throw "managed guest write did not reach the host" }
           if (-not (Test-Path (Join-Path $share "nvx-guest-directory") -PathType Container)) {
@@ -1145,7 +1122,7 @@ jobs:
           }
           try {
               Invoke-Nvx sandbox start --state-dir $state
-              Invoke-Nvx sandbox exec --state-dir $state @smokeArguments
+              Invoke-Nvx sandbox exec --state-dir $state --arg init @smokeArguments
               Invoke-Nvx sandbox stop --state-dir $state
           } finally {
               if (Test-Path (Join-Path $state "runtime.json")) {
@@ -1156,22 +1133,9 @@ jobs:
               if (Test-Path $log) { Copy-Item -Force $log (Join-Path $logs "shares-openvmm.log") }
           }
           $outcome = (Get-Content -Raw (Join-Path $state "outcome.json") | ConvertFrom-Json).outcome
-          if ($outcome.operation -ne "managed" -or $outcome.category -ne "success" -or $outcome.status_code -ne 0) {
+          if ($outcome.operation -ne "run" -or $outcome.category -ne "success" -or $outcome.status_code -ne 0) {
               throw "managed sandbox outcome is invalid: $($outcome | ConvertTo-Json -Compress)"
           }
-          # A clean ext4 journal shows that stop unmounted every share, the
-          # overlay, and scratch in order.
-          $image = [IO.File]::OpenRead($scratch)
-          try {
-              $null = $image.Seek(1024 + 0x60, [IO.SeekOrigin]::Begin)
-              $incompat = New-Object byte[] 4
-              if ($image.Read($incompat, 0, 4) -ne 4) { throw "scratch superblock is truncated" }
-          } finally {
-              $image.Dispose()
-          }
-          if ([BitConverter]::ToUInt32($incompat, 0) -band 4) {
-              throw "managed stop left the scratch filesystem mounted"
-          }
           Test-Shares
           Invoke-Nvx sandbox deprovision --state-dir $state
           Remove-Item -Recurse -Force $workspace, $toolcache, $files, $scratch, $outside
@@ -1256,7 +1220,7 @@ jobs:
           }
           try {
               Invoke-Nvx sandbox start --state-dir $state
-              Invoke-Nvx sandbox exec --state-dir $state @smokeArguments
+              Invoke-Nvx sandbox exec --state-dir $state --arg init @smokeArguments
               Invoke-Nvx sandbox stop --state-dir $state
           } finally {
               if (Test-Path (Join-Path $state "runtime.json")) {
@@ -1267,22 +1231,9 @@ jobs:
               if (Test-Path $log) { Copy-Item -Force $log (Join-Path $logs "policy-openvmm.log") }
           }
           $outcome = (Get-Content -Raw (Join-Path $state "outcome.json") | ConvertFrom-Json).outcome
-          if ($outcome.operation -ne "managed" -or $outcome.category -ne "success" -or $outcome.status_code -ne 0) {
+          if ($outcome.operation -ne "run" -or $outcome.category -ne "success" -or $outcome.status_code -ne 0) {
               throw "managed sandbox outcome is invalid: $($outcome | ConvertTo-Json -Compress)"
           }
-          # A clean ext4 journal shows that stop unmounted the share, the
-          # overlay, and scratch in order.
-          $image = [IO.File]::OpenRead($scratch)
-          try {
-              $null = $image.Seek(1024 + 0x60, [IO.SeekOrigin]::Begin)
-              $incompat = New-Object byte[] 4
-              if ($image.Read($incompat, 0, 4) -ne 4) { throw "scratch superblock is truncated" }
-          } finally {
-              $image.Dispose()
-          }
-          if ([BitConverter]::ToUInt32($incompat, 0) -band 4) {
-              throw "managed stop left the scratch filesystem mounted"
-          }
           Test-Share
           Invoke-Nvx sandbox deprovision --state-dir $state
           Remove-Item -Recurse -Force $share, $scratch

doc/guests/ubuntu-guest.md

@@ -152,11 +152,13 @@ one-shot workload, the Alpine
 [container entry helper](../../guest/alpine/nvx-container-enter) injects its
 musl loader, `setpriv`, and libraries into `/.nvx-agent`, so glibc Ubuntu
 binaries run while the trusted agent stays musl-based. A managed sandbox's
-agent, a static binary, enters each execution itself and runs the workload
-without them. The workload gets private mount, PID, and UTS namespaces and
-shares the guest's network namespace. It runs as a fixed non-root identity, by
-default `65534:65534`, the Ubuntu `nobody` account whose home the builder
-creates, with no supplementary groups, no capabilities, and `no_new_privs`.
+init, nvx-init, a static binary, runs each execution itself, as a plain
+process in the VM's root, without them. A one-shot workload gets private
+mount, PID, and UTS namespaces, a managed sandbox's executions share
+nvx-init's, and every workload shares the guest's network namespace. It runs
+as a fixed non-root identity, by default `65534:65534`, the Ubuntu `nobody`
+account whose home the builder creates, with no supplementary groups, no
+capabilities, and `no_new_privs`.
 
 The sandbox rejects systemd entrypoints, and it rejects a distro layer whose
 manifest lists the `systemd` package. Systemd needs a separate compatibility

doc/run.md

@@ -915,9 +915,14 @@ check, it requires:
 
 A managed workload that mounted into the agent's namespace would stack its
 runtime mounts on those of earlier requests, so the single-mount check also
-covers repeated managed execs. With `--arg entry`, for a managed execution, it
-also requires every mount other than `/dev` and its devpts instance to be
-`nodev`, as the managed agent's execution entry makes them. With
+covers repeated managed execs. With `--arg init` as its first argument, for an
+execution in a sandbox that `sandbox start` started, which nvx-init runs as a
+plain process in the VM's root, it checks the host name, which nvx-init's kernel
+command line encodes in hexadecimal, instead of the namespaces, and no cgroup:
+such an execution shares nvx-init's mount, PID, and UTS namespaces and its
+cgroup. With `--arg entry`, for a managed execution, it also requires every
+mount other than `/dev` and its devpts instance to be `nodev`, as the managed
+agent's execution entry and nvx-init's root make them. With
 `--arg limits --arg MEMORY_MAX --arg PIDS_MAX`, for a workload whose cgroup has
 those limits, it also requires the memory controller to kill an allocation of
 twice `MEMORY_MAX` with `SIGKILL` while a 1 MiB allocation succeeds, and the
@@ -1567,14 +1572,14 @@ the supplied entries over that default environment instead of replacing it,
 each entry replacing the default variable of the same name; without supplied
 entries it has no effect. Inline values are visible in host process arguments
 and should not be used for secrets. These options apply only to managed
-`sandbox exec`; one-shot execution rejects them. Each managed exec enters
-private mount, PID, and UTS namespaces of its own, as the first process of its
-PID namespace, through the guest agent's execution entry, which also makes
-every mount but `/dev` and its devpts instance `nodev`. The workload sees one
-machine ID for the life of the VM. On `stop`, the guest agent unmounts the
-mappings of the live shares, and the guest then unmounts the host export,
-overlay, layers, and upper in dependency order before the VM powers off, as
-it unmounts the shares when a one-shot workload exits. The legacy
+`sandbox exec`; one-shot execution rejects them. nvx-init, the VM's init, runs
+each managed exec as a plain process in a process group of its own, in the VM's
+root and in nvx-init's mount, PID, and UTS namespaces and cgroup: the VM is the
+boundary of its workloads. Every mount of the root but `/dev` and its devpts
+instance is `nodev`. The workload sees one machine ID for the life of the VM.
+On `stop`, nvx-init syncs the file systems and powers the VM off with them
+mounted, and the next `start` replays the journal of the scratch file system.
+The legacy
 operation-less `sandbox`
 form is `sandbox run`; it remains one-shot and rejects `--state-dir` or any
 request to retain VM state.

guest/common/nvx-sandbox-smoke

@@ -11,6 +11,15 @@ workload_ids="65534 65534 65534 65534"
 initial_namespaces="mnt:4026531832 pid:4026531836 uts:4026531838"
 # The private /dev that the container runtime creates, in byte order.
 device_view="fd full null ptmx pts random shm stderr stdin stdout tty urandom zero"
+# A first argument `init` checks an execution that nvx-init runs as a plain
+# process in the VM's switched root. It has no container entry: it shares
+# nvx-init's mount, PID, and UTS namespaces and its cgroup, and the VM is its
+# boundary.
+nvx_init=false
+if [ "${1:-}" = init ]; then
+    nvx_init=true
+    shift
+fi
 
 fail() {
     echo "NVX-UBUNTU-SANDBOX-FAIL: $*" >&2
@@ -85,6 +94,13 @@ check_namespaces() {
     init_uids=$(status_field /proc/1/status Uid)
     [ "$init_uids" = "$workload_ids" ] ||
         fail "PID 1 of the workload PID namespace runs as $init_uids"
+    check_hostname
+}
+
+# The workload's host name is the one that the kernel command line names, or
+# nvx-sandbox. nvx-init's command line names its bytes in lowercase
+# hexadecimal.
+check_hostname() {
     expected_hostname=nvx-sandbox
     # Read the whole file: a caller's --cmdline keeps its newlines, so tokens
     # may follow a line break.
@@ -99,6 +115,9 @@ check_namespaces() {
     done
     set +f
     read -r workload_hostname </proc/sys/kernel/hostname
+    if [ "$nvx_init" = true ]; then
+        workload_hostname=$(printf '%s' "$workload_hostname" | od -An -v -tx1 | tr -d ' \n')
+    fi
     [ "$workload_hostname" = "$expected_hostname" ] ||
         fail "the workload hostname is $workload_hostname, expected $expected_hostname"
 }
@@ -253,8 +272,8 @@ check_ports() {
         fail "a request for the port permission ended with status $port_status"
 }
 
-# A managed execution enters its root through the agent's entry, which also
-# makes every mount but /dev and its devpts nodev.
+# A managed execution's entry, like nvx-init's switched root, makes every mount
+# but /dev and its devpts nodev.
 check_entry() {
     while read -r _ _ _ _ mount_point mount_options _; do
         case "$mount_point" in
@@ -609,13 +628,17 @@ check_shares_isolated() {
 
 check_identity
 check_capabilities
-check_namespaces
+if [ "$nvx_init" = true ]; then
+    check_hostname
+else
+    check_namespaces
+fi
 check_mounts
 check_devices
 check_device_creation
 check_machine_id
 check_ports
-check_cgroup
+[ "$nvx_init" = true ] || check_cgroup
 check_image
 echo "NVX-UBUNTU-SANDBOX-PROFILE-OK uid=65534 gid=65534"
 # Each share is a TARGET MODE pair; a final TARGET alone is read-only. A MODE

nvx-rs/src/host/cli/sandbox/lifecycle/directory.rs

@@ -13,14 +13,6 @@ pub(super) fn os_error(error: &std::io::Error, path: &Path) -> Exit {
     io_error(ErrorCode::Io, error, path)
 }
 
-/// Removes `path` unless it is missing.
-pub(super) fn unlink_missing_ok(path: &Path) -> Result {
-    match std::fs::remove_file(path) {
-        Err(error) if error.kind() != std::io::ErrorKind::NotFound => Err(os_error(&error, path)),
-        _ => Ok(()),
-    }
-}
-
 /// Returns a builder of directories of `mode`, which Windows ignores.
 fn dir_builder(mode: u32) -> std::fs::DirBuilder {
     #[cfg_attr(not(unix), allow(unused_mut))]