diff --git a/.github/actions/check-nvx-rs/action.yml b/.github/actions/check-nvx-rs/action.yml index e0d20e0d3..02d37edb2 100644 --- a/.github/actions/check-nvx-rs/action.yml +++ b/.github/actions/check-nvx-rs/action.yml @@ -1,7 +1,5 @@ name: Check nvx-rs -description: >- - Format, lint, document, and test the nvx crate, and the runtime adapter that builds on it - as an independent adapter +description: Format, lint, document, and test the nvx crate inputs: toolchain: @@ -164,56 +162,3 @@ runs: directory="${PWD}/../build/nvx-rs-guest-binaries" python3 ../scripts/nvx.py build-guest-binaries --output "${directory}" python3 ../scripts/nvx.py test-guest-binaries --binaries "${directory}" - - # The runtime adapter builds on nvx-rs as an independent adapter does: with a lockfile of its - # own, default features off, and the public API only. Its tests check that each configuration - # builds that way, and each configuration also builds with nvx-rs's minimum Rust version. As - # for nvx-rs, each configuration's documentation builds with warnings denied, with - # RUSTDOCFLAGS set for that command alone so that it doesn't reach the doctests. - - name: Check the runtime adapter's protocol-only configuration - shell: bash - working-directory: tests/runtime-adapter - run: | - set -euo pipefail - cargo "+${{ steps.toolchains.outputs.toolchain }}" fmt --all --check - cargo "+${{ steps.toolchains.outputs.toolchain }}" clippy --locked --all-targets \ - -- -D warnings - cargo "+${{ steps.toolchains.outputs.toolchain }}" build --locked - cargo "+${{ steps.toolchains.outputs.toolchain }}" test --locked - RUSTDOCFLAGS="-D warnings" cargo "+${{ steps.toolchains.outputs.toolchain }}" doc \ - --locked --no-deps - cargo "+${{ steps.toolchains.outputs.msrv }}" check --locked --all-targets - - - name: Check the runtime adapter's host-only configuration - shell: bash - working-directory: tests/runtime-adapter - run: | - set -euo pipefail - cargo "+${{ steps.toolchains.outputs.toolchain }}" clippy --locked --all-targets \ - --features host -- -D warnings - cargo "+${{ steps.toolchains.outputs.toolchain }}" build --locked --features host - cargo "+${{ steps.toolchains.outputs.toolchain }}" test --locked --features host - RUSTDOCFLAGS="-D warnings" cargo "+${{ steps.toolchains.outputs.toolchain }}" doc \ - --locked --no-deps --features host - cargo "+${{ steps.toolchains.outputs.msrv }}" check --locked --all-targets --features host - - # The guest agent runs in Linux guests, as a static executable that a test checks. - - name: Check the runtime adapter's guest-only configuration - if: runner.os == 'Linux' - shell: bash - working-directory: tests/runtime-adapter - run: | - set -euo pipefail - for toolchain in "${{ steps.toolchains.outputs.toolchain }}" "${{ steps.toolchains.outputs.msrv }}"; do - rustup target add --toolchain "${toolchain}" x86_64-unknown-linux-musl - done - cargo "+${{ steps.toolchains.outputs.toolchain }}" clippy --locked --all-targets \ - --features guest --target x86_64-unknown-linux-musl -- -D warnings - cargo "+${{ steps.toolchains.outputs.toolchain }}" build --locked \ - --features guest --target x86_64-unknown-linux-musl - cargo "+${{ steps.toolchains.outputs.toolchain }}" test --locked \ - --features guest --target x86_64-unknown-linux-musl - RUSTDOCFLAGS="-D warnings" cargo "+${{ steps.toolchains.outputs.toolchain }}" doc \ - --locked --no-deps --features guest --target x86_64-unknown-linux-musl - cargo "+${{ steps.toolchains.outputs.msrv }}" check --locked --all-targets \ - --features guest --target x86_64-unknown-linux-musl diff --git a/.github/actions/validate-nvx/action.yml b/.github/actions/validate-nvx/action.yml index acd6ce385..63d406faf 100644 --- a/.github/actions/validate-nvx/action.yml +++ b/.github/actions/validate-nvx/action.yml @@ -18,7 +18,6 @@ runs: python3 scripts/nvx.py test-openvmm-unit --help python3 scripts/nvx.py test-openvmm --help python3 scripts/nvx.py test-microvm --help - python3 scripts/nvx.py test-runtime-adapter --help python3 scripts/nvx.py test-adversarial --help python3 scripts/nvx.py benchmark --help python3 scripts/nvx.py doctor --help @@ -46,8 +45,6 @@ runs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python scripts\nvx.py test-microvm --help if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } - python scripts\nvx.py test-runtime-adapter --help - if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python scripts\nvx.py test-adversarial --help if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } python scripts\nvx.py benchmark --help diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 504440735..44ffa2d31 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -28,9 +28,7 @@ quality and CLI test definitions. Start with the narrowest affected test, then run the applicable commands from those actions. - For changes under `nvx-rs/`, run the commands in - [check-nvx-rs](./actions/check-nvx-rs/action.yml). It also checks the - runtime adapter in `tests/runtime-adapter/`, which builds on `nvx`; run its - steps for changes to either. + [check-nvx-rs](./actions/check-nvx-rs/action.yml). They need no hypervisor. - OpenVMM source changes also require the package-scoped checks prescribed by the nested repository. Do not claim unavailable hardware or cross-platform gates diff --git a/.github/skills/nvx-openvmm-promote/SKILL.md b/.github/skills/nvx-openvmm-promote/SKILL.md index cc2ea0211..229ea3712 100644 --- a/.github/skills/nvx-openvmm-promote/SKILL.md +++ b/.github/skills/nvx-openvmm-promote/SKILL.md @@ -111,9 +111,8 @@ Each test stops at its first failure, which names a copy that the revision chang After copying a vector file or an excerpt, record the revision in [the vectors' README](../../../nvx-rs/tests/vectors/README.md#openvmms-vectors). Rerun the comparison after each fix until it passes, since only the NVX CLI jobs run these ignored -tests. Then run the checks of [check-nvx-rs](../../actions/check-nvx-rs/action.yml), which -also build the runtime adapter that compiles the excerpts; they may need coordinated -changes too. +tests. Then run the checks of [check-nvx-rs](../../actions/check-nvx-rs/action.yml); they +may need coordinated changes too. `nvx-rs run` also reads a copy of the identities of OpenVMM's built-in CPU profiles, from which it guides a failed cold boot on a CPU that none serves. Compare it with the revision's diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a26a46b97..787f60fa8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -695,30 +695,6 @@ jobs: if-no-files-found: error retention-days: 1 - # The Windows guest and sandbox job runs the runtime adapter's scenarios - # with its guest agent, a static musl executable that Linux builds, as - # it builds the images that the job reads. The Linux jobs build their - # own. - - name: Build the runtime adapter's guest agent - shell: bash - working-directory: tests/runtime-adapter - run: | - set -euo pipefail - toolchain=$(sed -n 's/^channel = "\([^"]*\)".*/\1/p' ../../rust-toolchain.toml) - test -n "${toolchain}" - rustup toolchain install "${toolchain}" --profile minimal --no-self-update \ - --target x86_64-unknown-linux-musl - cargo "+${toolchain}" build --locked --release --features guest \ - --target x86_64-unknown-linux-musl --bin runtime-adapter-guest - - - name: Upload the runtime adapter's guest agent - uses: actions/upload-artifact@v7 - with: - name: runtime-adapter-guest - path: tests/runtime-adapter/target/x86_64-unknown-linux-musl/release/runtime-adapter-guest - if-no-files-found: error - retention-days: 1 - # A dev push and a merge group reuse the guest artifact components that the # pull request that they merge built (see build-guest-artifacts). A # same-repository pull request rehearses that selection and check on the @@ -1090,13 +1066,10 @@ jobs: OPENVMM_DEPLOY_KEY: ${{ secrets.OPENVMM_DEPLOY_KEY }} # The steps that follow each backend's default suite, from the public - # managed sandbox acceptance to the runtime adapter scenarios, depend on + # managed sandbox acceptance to the Ubuntu sandbox steps, depend on # none of its results, so a second job runs them beside it, on the same # runner pool. They alone read the Ubuntu and Azure Linux artifacts, so - # only the jobs that run them wait for distro-artifacts. WHP, whose runtime - # adapter scenarios take about twice as long as on KVM and MSHV, runs them - # in a third job beside the other two, so that its guest and sandbox job no - # longer ends last in most runs (doc/ci.md#job-graph-and-runner-capacity). + # only the jobs that run them wait for distro-artifacts. nvx-microvm-guests-kvm: name: NVX microVM tests / Linux / KVM / Guests and sandboxes needs: [artifacts, distro-artifacts, build-openvmm-linux-gnu, build-nvx-rs-linux-musl, openvmm-changes] @@ -1129,7 +1102,6 @@ jobs: openvmm-artifact: openvmm-linux-gnu nvx-rs-artifact: nvx-rs-linux-musl guests-and-sandboxes: true - runtime-adapter: true secrets: OPENVMM_DEPLOY_KEY: ${{ secrets.OPENVMM_DEPLOY_KEY }} @@ -1165,7 +1137,6 @@ jobs: openvmm-artifact: openvmm-linux-gnu nvx-rs-artifact: nvx-rs-linux-musl guests-and-sandboxes: true - runtime-adapter: true secrets: OPENVMM_DEPLOY_KEY: ${{ secrets.OPENVMM_DEPLOY_KEY }} @@ -1204,41 +1175,6 @@ jobs: secrets: OPENVMM_DEPLOY_KEY: ${{ secrets.OPENVMM_DEPLOY_KEY }} - nvx-microvm-adapter-whp: - name: NVX microVM tests / Windows / WHP / Runtime adapter - needs: [artifacts, distro-artifacts, build-openvmm-windows-msvc, build-nvx-rs-windows-msvc, openvmm-changes] - if: >- - ${{ - always() && - !cancelled() && - ( - github.event_name != 'pull_request' || - github.event.pull_request.draft == false - ) && - ( - github.event_name != 'pull_request' || - github.event.pull_request.head.repo.full_name == github.repository - ) && - needs.artifacts.result == 'success' && - needs.distro-artifacts.result == 'success' && - needs.build-openvmm-windows-msvc.result == 'success' && - needs.build-nvx-rs-windows-msvc.result == 'success' && - ( - needs.openvmm-changes.result != 'success' || - needs.openvmm-changes.outputs.run-microvm-tests == 'true' - ) - }} - uses: ./.github/workflows/run-nvx-microvm-tests.yml - with: - display-name: Windows / WHP / Runtime adapter - runner: '["self-hosted", "windows", "whp", "virtual-machine", "general"]' - backend: whp - openvmm-artifact: openvmm-windows-msvc - nvx-rs-artifact: nvx-rs-windows-msvc - runtime-adapter: true - secrets: - OPENVMM_DEPLOY_KEY: ${{ secrets.OPENVMM_DEPLOY_KEY }} - # Pull requests run the debug kernel on MSHV only, rather than on KVM, whose # general runners queue the most, or on WHP, where it would lengthen the # longest waits. Merge groups keep it on KVM alone, and dev pushes and full @@ -1583,7 +1519,6 @@ jobs: - nvx-microvm-guests-kvm - nvx-microvm-guests-mshv - nvx-microvm-guests-whp - - nvx-microvm-adapter-whp - nvx-microvm-debug-kvm - nvx-microvm-debug-mshv - nvx-microvm-debug-whp @@ -1640,10 +1575,6 @@ jobs: needs.nvx-microvm-guests-whp.result == 'success' || needs.nvx-microvm-guests-whp.result == 'skipped' ) && - ( - needs.nvx-microvm-adapter-whp.result == 'success' || - needs.nvx-microvm-adapter-whp.result == 'skipped' - ) && ( needs.nvx-microvm-debug-kvm.result == 'success' || needs.nvx-microvm-debug-kvm.result == 'skipped' @@ -1736,7 +1667,6 @@ jobs: - nvx-microvm-guests-kvm - nvx-microvm-guests-mshv - nvx-microvm-guests-whp - - nvx-microvm-adapter-whp - nvx-microvm-debug-kvm - nvx-microvm-debug-mshv - nvx-microvm-debug-whp @@ -1791,7 +1721,6 @@ jobs: MICROVM_GUESTS_KVM_RESULT: ${{ needs.nvx-microvm-guests-kvm.result }} MICROVM_GUESTS_MSHV_RESULT: ${{ needs.nvx-microvm-guests-mshv.result }} MICROVM_GUESTS_WHP_RESULT: ${{ needs.nvx-microvm-guests-whp.result }} - MICROVM_ADAPTER_WHP_RESULT: ${{ needs.nvx-microvm-adapter-whp.result }} MICROVM_DEBUG_KVM_RESULT: ${{ needs.nvx-microvm-debug-kvm.result }} MICROVM_DEBUG_MSHV_RESULT: ${{ needs.nvx-microvm-debug-mshv.result }} MICROVM_DEBUG_WHP_RESULT: ${{ needs.nvx-microvm-debug-whp.result }} @@ -1832,7 +1761,6 @@ jobs: - nvx-microvm-guests-kvm - nvx-microvm-guests-mshv - nvx-microvm-guests-whp - - nvx-microvm-adapter-whp - nvx-microvm-debug-kvm - nvx-microvm-debug-mshv - nvx-microvm-debug-whp @@ -1887,10 +1815,6 @@ jobs: needs.nvx-microvm-guests-whp.result == 'success' || needs.nvx-microvm-guests-whp.result == 'skipped' ) && - ( - needs.nvx-microvm-adapter-whp.result == 'success' || - needs.nvx-microvm-adapter-whp.result == 'skipped' - ) && ( needs.nvx-microvm-debug-kvm.result == 'success' || needs.nvx-microvm-debug-kvm.result == 'skipped' diff --git a/.github/workflows/copilot-setup-steps.yml b/.github/workflows/copilot-setup-steps.yml index 9379703dd..b2b647686 100644 --- a/.github/workflows/copilot-setup-steps.yml +++ b/.github/workflows/copilot-setup-steps.yml @@ -104,8 +104,8 @@ jobs: run: | set -euo pipefail rust_toolchain=$(sed -n 's/^channel = "\([^"]*\)".*/\1/p' rust-toolchain.toml) - # The minimum supported Rust version of the nvx crate, with which - # check-nvx-rs also checks the runtime adapter. + # The minimum supported Rust version of the nvx crate, which + # check-nvx-rs checks. crate_msrv=$(sed -n 's/^rust-version = "\(.*\)"$/\1/p' nvx-rs/Cargo.toml) nextest=$(sed -n 's/^cargo_nextest\.version=//p' scripts/setup/tool-versions.conf) { diff --git a/.github/workflows/run-nvx-microvm-tests.yml b/.github/workflows/run-nvx-microvm-tests.yml index 0bfbd703b..3b7757e8a 100644 --- a/.github/workflows/run-nvx-microvm-tests.yml +++ b/.github/workflows/run-nvx-microvm-tests.yml @@ -41,15 +41,6 @@ on: required: false type: boolean default: false - runtime-adapter: - description: >- - Run the runtime adapter scenarios instead of the default suite: - after the guest and sandbox steps with guests-and-sandboxes, or - alone, in a third job of the backend beside the other two. Not for - the debug-kernel jobs, which skip them - required: false - type: boolean - default: false shard: description: >- Run only this shard, from 1 to shard-count, of the default suite, @@ -104,9 +95,9 @@ jobs: # qualifies its runner with the kernel and the Alpine initramfs, the # shards of the default suite boot only those, and the debug-kernel jobs # run only Alpine scenarios on the debug kernel, so only the guest and - # sandbox steps take the Ubuntu and Azure Linux artifacts, and the - # runtime adapter scenarios and an unsharded default run the Ubuntu - # distro layer of their layered roots and public sandbox scenarios. + # sandbox steps take the Ubuntu and Azure Linux artifacts, and an + # unsharded default run the Ubuntu distro layer of its public sandbox + # scenarios. - name: Download guest kernel uses: actions/download-artifact@v8 with: @@ -143,17 +134,7 @@ jobs: # The layer, its manifest, and the sandbox scratch template, which the # public sandbox scenarios of an unsharded default run read. - name: Download Ubuntu distro layer for the whole default run - if: ${{ !inputs.guests-and-sandboxes && !inputs.runtime-adapter && !inputs.debug-kernel && !inputs.shard }} - uses: actions/download-artifact@v8 - with: - name: guest-ubuntu-distro - path: build - - # The layer, its manifest, and the sandbox scratch template, from which - # the layered runtime adapter scenarios boot their roots, in a job that - # runs the scenarios without the guest and sandbox steps. - - name: Download Ubuntu distro layer for the runtime adapter alone - if: ${{ inputs.runtime-adapter && !inputs.guests-and-sandboxes && !inputs.debug-kernel }} + if: ${{ !inputs.guests-and-sandboxes && !inputs.debug-kernel && !inputs.shard }} uses: actions/download-artifact@v8 with: name: guest-ubuntu-distro @@ -208,7 +189,7 @@ jobs: # the Linux steps exec the test harness, which can then stop OpenVMM and # remove its scratch directories (doc/ci.md#cancellation). - name: Run NVX microVM tests on Linux - if: runner.os != 'Windows' && !inputs.guests-and-sandboxes && !inputs.runtime-adapter + if: runner.os != 'Windows' && !inputs.guests-and-sandboxes shell: bash run: >- exec python3 scripts/nvx.py test-microvm @@ -218,7 +199,7 @@ jobs: --output-dir "build/test-results/microvm-${{ inputs.backend }}${{ inputs.debug-kernel && '-debug' || '' }}" - name: Run NVX microVM tests on Windows - if: runner.os == 'Windows' && !inputs.guests-and-sandboxes && !inputs.runtime-adapter + if: runner.os == 'Windows' && !inputs.guests-and-sandboxes shell: powershell run: >- python scripts\nvx.py test-microvm @@ -1265,59 +1246,8 @@ jobs: Invoke-Nvx sandbox deprovision --state-dir $state Remove-Item -Recurse -Force $share, $scratch - # The runtime adapter's consolidated suite on this backend: each scenario - # that tests/runtime-adapter/scenarios.txt names, by name, in its - # host-only configuration, whose VM scenarios are its ignored tests and - # read the backend through `nvx::Hypervisor::from_env_or_default`. The - # command fails once a scenario names a test that its target does not - # have, a named target has a test that no scenario names, the library or - # another target has an ignored test, or a named test does not run and - # pass. Its host program launches VMs of the artifacts above, with the - # runtime identity that their build provenance gives, whose initramfs runs - # the adapter's guest agent in place of NVX's agents. Later processes of - # the host program reattach to a VM, as after a restart of the - # orchestrator, and stop it. The layered scenarios boot the Ubuntu EROFS - # image over a copy of the scratch template, from the guest artifacts, as - # the sandbox steps do, and the guest agent's worker assembles the root - # outside PID 1. The guest agent is a static musl executable: Linux - # builds it, and Windows takes the one that distro-artifacts builds on - # Linux, beside the Ubuntu distro layer. - - name: Build the runtime adapter's guest agent on Linux - if: runner.os != 'Windows' && inputs.runtime-adapter && !inputs.debug-kernel - shell: bash - working-directory: tests/runtime-adapter - run: >- - cargo build --locked --release --features guest - --target x86_64-unknown-linux-musl --bin runtime-adapter-guest - - - name: Run runtime adapter scenarios on Linux - if: runner.os != 'Windows' && inputs.runtime-adapter && !inputs.debug-kernel - shell: bash - run: >- - python3 scripts/nvx.py test-runtime-adapter - --backend "${{ inputs.backend }}" - --guest-agent tests/runtime-adapter/target/x86_64-unknown-linux-musl/release/runtime-adapter-guest - - - name: Download the runtime adapter's guest agent on Windows - if: runner.os == 'Windows' && inputs.runtime-adapter && !inputs.debug-kernel - uses: actions/download-artifact@v8 - with: - name: runtime-adapter-guest - path: build/runtime-adapter - - - name: Run runtime adapter scenarios on Windows - if: runner.os == 'Windows' && inputs.runtime-adapter && !inputs.debug-kernel - shell: powershell - run: >- - python scripts\nvx.py test-runtime-adapter - --backend "${{ inputs.backend }}" - --guest-agent build\runtime-adapter\runtime-adapter-guest - - # A job that runs the runtime adapter scenarios alone runs no - # test-microvm step, so it has neither result directories nor duration - # reports to keep. - name: Upload NVX microVM failure logs - if: failure() && (inputs.guests-and-sandboxes || !inputs.runtime-adapter) + if: failure() uses: actions/upload-artifact@v7 with: name: nvx-microvm-tests-${{ inputs.backend }}${{ inputs.debug-kernel && '-debug' || '' }}${{ inputs.guests-and-sandboxes && '-guests' || '' }}${{ inputs.shard && format('-shard-{0}', inputs.shard) || '' }} @@ -1335,7 +1265,7 @@ jobs: # job passes, fails, or is cancelled; each shard of the default suite # keeps its own. - name: Upload NVX microVM scenario durations - if: always() && (inputs.guests-and-sandboxes || !inputs.runtime-adapter) + if: always() uses: actions/upload-artifact@v7 with: name: nvx-microvm-durations-${{ runner.os }}-${{ inputs.backend }}-${{ inputs.debug-kernel && 'debug' || 'default' }}${{ inputs.guests-and-sandboxes && '-guests' || '' }}${{ inputs.shard && format('-shard-{0}', inputs.shard) || '' }}-${{ github.run_id }}-${{ github.run_attempt }} diff --git a/.gitignore b/.gitignore index eb9036015..dda7733e3 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ /.ruff_cache/ /.nvx-hosts.json /nvx-rs/target/ -/tests/runtime-adapter/target/ __pycache__/ *.py[cod] *.log diff --git a/README.md b/README.md index b226b0903..edaf38af0 100644 --- a/README.md +++ b/README.md @@ -98,7 +98,7 @@ nvx-rs\target\release\nvx-rs.exe run dependencies between them. - [Continuous integration](doc/ci.md) - Instructions for running and maintaining NVX CI. - [Case matrix](doc/acceptance-case-matrix.md) - The fate of each test of the runtime adapter, of - CI and of the docs, and of each microVM scenario, before NVX deletes the adapter. + CI and of the docs, and of each microVM scenario, before NVX deleted the adapter. - [Copilot-driven adversarial testing](doc/adversarial-testing.md) - Bounded adaptive stress and containment campaigns. - [Package and source delivery](doc/distribution.md) - Instructions for packaging and distributing diff --git a/doc/acceptance-case-matrix.md b/doc/acceptance-case-matrix.md index 70c4d4f42..2307d756e 100644 --- a/doc/acceptance-case-matrix.md +++ b/doc/acceptance-case-matrix.md @@ -1,7 +1,7 @@ # Case matrix -NVX deletes its runtime adapter, `tests/runtime-adapter`, once the acceptance that replaces it -passes (#1086, #1119). Before then, this matrix gives each test of the adapter, of CI and of the +NVX deleted its runtime adapter, `tests/runtime-adapter`, once the acceptance that replaces it +passed (#1086, #1119). Before then, this matrix gave each test of the adapter, of CI and of the docs, and each microVM scenario, one fate, by name: - **kept**: the test stays as it is, and runs without the adapter. @@ -36,12 +36,12 @@ its own in turn. ## The adapter's scenarios -`scenarios.txt` names 157 scenarios, which CI's runtime adapter step runs on KVM, MSHV +`scenarios.txt` named 157 scenarios, which CI's runtime adapter step ran on KVM, MSHV and WHP: the 154 of the adapter's stage, the two that `dev` added since, `template_capture::the_host_program_refuses_a_repair_delay_that_it_cannot_read` and `vm_clone::openvmm_reads_none_of_the_host_s_input_until_the_repair_acknowledges_the_restore`, and the Linux-only `vm_clone::restore_exclusion::hard_linked_restore_views_obey_the_callers_exclusion`. -The adapter's README says what each proves. +The adapter's README said what each proved. ### `template_capture` (3) @@ -257,8 +257,8 @@ The adapter's README says what each proves. ## The adapter's other tests -The adapter's library and its targets that `scenarios.txt` doesn't name hold these tests, which -check the adapter's own code, and CI runs in its configurations of the adapter. +The adapter's library and its targets that `scenarios.txt` didn't name held these tests, which +checked the adapter's own code, and CI ran in its configurations of the adapter. | File | Tests | Fate | Coverage, or why it retires | | --- | --- | --- | --- | @@ -407,7 +407,7 @@ capture keep a NIC. ## CI These are the steps of CI's workflows and actions that check something, and the Python tests of -CI's wiring that name the adapter. Builds, downloads, uploads and publications check nothing of +CI's wiring that named the adapter. Builds, downloads, uploads and publications check nothing of their own. The tests that a step runs have rows of their own where they concern the adapter. | Where | Case | Fate | Note | diff --git a/doc/ci.md b/doc/ci.md index 70ce313f1..c382c3093 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -20,23 +20,19 @@ of `feature/microvm/dev`, on each pull request into the branch and on each push to it, on OpenVMM's own runners, and the GitHub-hosted `openvmm-ci` job checks that they passed the jobs that cover NVX's use of OpenVMM; see [OpenVMM CI](#openvmm-ci). -Two jobs per backend, and a third on WHP, exercise Linux, SMP, virtio, -sandbox, and snapshot behavior through the public OpenVMM CLI, on the NVX -Linux kernel plus the selected Alpine, Ubuntu, or Azure Linux initramfs. All -run on the backend's `general` runners, with the same OpenVMM and `nvx-rs` -builds. The +Two jobs per backend exercise Linux, SMP, virtio, sandbox, and snapshot +behavior through the public OpenVMM CLI, on the NVX Linux kernel plus the +selected Alpine, Ubuntu, or Azure Linux initramfs. Both run on the backend's +`general` runners, with the same OpenVMM and `nvx-rs` builds. The `nvx-microvm-tests-{kvm,mshv,whp}` jobs run the default suite on the Alpine initramfs, each in shards of about equal duration that run in parallel, one runner each (see [MicroVM test shards](#microvm-test-shards)). Beside them, the `nvx-microvm-guests-{kvm,mshv,whp}` jobs run the guest smoke and sandbox steps, which need no result of that suite: the public managed sandbox acceptance, the Ubuntu guest tests, the one-vCPU Azure Linux -smoke set, the Ubuntu sandbox steps, and, on KVM and MSHV, the runtime adapter -scenarios, which WHP runs in a third job, `nvx-microvm-adapter-whp` (see -[Job graph and runner capacity](#job-graph-and-runner-capacity)). Only +smoke set, and the Ubuntu sandbox steps. Only those steps read the Ubuntu and Azure -Linux artifacts, so only the guest and sandbox jobs and WHP's runtime adapter -job wait for them (see +Linux artifacts, so only the guest and sandbox jobs wait for them (see [Guest artifact handoff](#guest-artifact-handoff)). Every guest runs under the same time ABI checks. Alpine-control-only scenarios remain explicit and are rejected for the Ubuntu initramfs. The debug-kernel jobs skip the Ubuntu and @@ -52,8 +48,7 @@ too, also uploads the `nvx-microvm-durations---[-guests|-shard-]--`, where `` is `default` or `debug`, the guest and sandbox jobs add `-guests`, and the shards `-shard-`, so that each job and attempt keeps its -own. WHP's runtime adapter job runs no `test-microvm` step, so it uploads -neither. +own. Every harness launch, in the tests and the benchmarks, scans the OpenVMM console for the guest's [time ABI](design/3.2-clock-and-time.md) output, and the tests scan every virtio console they read over TCP the same way, to its end. An @@ -366,11 +361,10 @@ on the CI debug kernel (`build/vmlinux-debug`, built from production kernels leave out. It selects the same-host restore scenarios `smp`, `smp-snapshot`, `restore-processors`, and `restore-downtime`. The jobs skip every other test step, because those boot the -production kernel, which the `nvx-microvm-guests-{kvm,mshv,whp}` jobs and -WHP's runtime adapter job already cover: the public managed execution -configuration, the Ubuntu and Azure Linux guest tests, the Ubuntu sandbox -layer and live-share smoke tests, the caller-owned live-share test, and the -runtime adapter scenarios. Any +production kernel, which the `nvx-microvm-guests-{kvm,mshv,whp}` jobs already +cover: the public managed execution configuration, the Ubuntu and Azure Linux +guest tests, the Ubuntu sandbox layer and live-share smoke tests, and the +caller-owned live-share test. Any RCU stall, soft lockup, or hung task makes the guest's time ABI watcher power off with status 194, which fails the run. Every guest those scenarios restore asks `nvx-time status` before it exits, which waits for the restore's deferred @@ -706,8 +700,7 @@ benchmark lane to pass. The microVM correctness jobs gate every use of benchmark results, so the benchmark action runs no correctness scenario of its own (#286). `Required status check` requires each `nvx-microvm-tests-*` and `nvx-microvm-guests-*` -job, and `nvx-microvm-adapter-whp`, that the change schedules to succeed, and -an `nvx-microvm-tests-*` job +job that the change schedules to succeed, and an `nvx-microvm-tests-*` job succeeds only when each of its shards does. `Publish development release` and `Record performance results` run only on `dev` pushes in which every microVM test job succeeded or was skipped. The pull-request and merge-group @@ -765,8 +758,7 @@ Beside each platform job, a package job on a GitHub-hosted runner of the platform's operating system, `package-kvm`, `package-mshv`, or `package-whp`, packages the platform's release once `artifacts`, `distro-artifacts`, the platform's OpenVMM producer, and the `nvx-rs` build of its operating system -end. The microVM guest and sandbox jobs and WHP's runtime adapter job, -`nvx-microvm-adapter-whp`, also wait for `distro-artifacts`, and +end. The microVM guest and sandbox jobs also wait for `distro-artifacts`, and the debug-kernel jobs for `debug-kernel`. `Performance regression gate`, `Publish development release`, `Record performance results`, and `Required status check` wait for the jobs whose results they check, and in a full run of @@ -890,11 +882,8 @@ merge group then fills KVM's four general runners with the two shards, the guest and sandbox job, and the debug-kernel job, and a pull request, which runs the debug kernel on MSHV, needs three of them. MSHV's four hold the two shards, the guest and sandbox job, and, for a pull request, push, or -full run, the debug-kernel job. On WHP, which runs the runtime adapter -scenarios in a job of their own, as the next paragraphs describe, a pull -request or merge group needs four of the six general runners at once, for the -microVM jobs, and a push or full run, which also runs the debug kernel -there, five. When a pool has fewer +full run, the debug-kernel job. On WHP, a run needs at most four of the six +general runners at once, for the microVM jobs. When a pool has fewer free general runners than a backend's microVM jobs, the jobs that start last wait for the first to end, so the backend's microVM tests take up to about twice as long as one of its jobs, still less than the unsharded suite. @@ -944,30 +933,6 @@ would have ended with their Linux leg, which took a median 17.0 minutes on `ubuntu-latest`, at a median of 18.1 minutes. Since CI stopped running OpenVMM's tests, every run is of the first two kinds. -In the successful first-attempt pull-request, merge-group, and `dev`-push runs -from October 31, 2026, when CI stopped running OpenVMM's tests, to November 3, -the 33 that restored OpenVMM from its cache and ran the benchmarks took a -median of 14.0 minutes, or 13.4 without queueing over 23 of them, and the only -one that ran no benchmark took 16.6 (#982). The 40 that built OpenVMM, for a -change of its pin, took 27.6, or 25.6 without queueing. The WHP guest and -sandbox job ended last in 23 of the 33, and the WHP platform job in 9: the job -took a median of 9.9 minutes, against 5.9 on KVM and 5.2 on MSHV, because the -runtime adapter scenarios, its last step, took 5.4 minutes on WHP and 2.7 on -the others. WHP therefore runs them in a job of their own, -`nvx-microvm-adapter-whp`, beside its guest and sandbox job. It waits for the -same jobs, but downloads, of the Ubuntu and Azure Linux artifacts, only the -Ubuntu distro layer, from which the layered scenarios boot. KVM and MSHV, -whose general runners queue more and whose guest and sandbox jobs take about -as long as a shard of the default suite, keep the scenarios in those jobs. A -replay of the window's WHP general jobs through six first-in, first-out -runners reproduces their observed queueing, with 58 against 56 of 490 jobs -waiting 2 minutes or more, at a p90 of 3.7 against 3.5 minutes. With the -split, it estimates that the 33 runs would have taken a median of 11.9 -minutes, or 11.2 without queueing, the runs that built OpenVMM 22.6 without -queueing, and the run without benchmarks 14.7, while 98 of the pool's 600 -jobs would have waited 2 minutes or more, at a p90 of 4.8 minutes. The WHP -platform job then ends last in most runs that run the benchmarks. - ## MicroVM test shards `nvx.py test-microvm --shard INDEX/COUNT` runs one of COUNT shards of the @@ -1001,8 +966,7 @@ shards that failed. For a `dev` run, use that partial rerun only under the job's condition holds, so a backend's shards run together or not at all. The job's result, which `Required status check`, `Publish development release`, and `Record performance results` read, succeeds only when every shard succeeds. -The guest and sandbox jobs, WHP's runtime adapter job, and the debug-kernel -jobs are not sharded: the +The guest and sandbox jobs and the debug-kernel jobs are not sharded: the debug-kernel jobs take three to five minutes, about as long as a shard, and still run on MSHV only for pull requests, on KVM only for merge groups, and on every backend for pushes and full runs. @@ -1090,7 +1054,7 @@ components that its steps read, and waits only for the jobs that upload them: | `guest-kernel` | `vmlinux`, `vmlinux.config`, `vmlinux.provenance.json` | `artifacts` | microVM tests, debug-kernel jobs, platform jobs, package jobs | | `guest-alpine` | `initramfs.cpio.gz`, its `.packages.json`, `initramfs.provenance.json` | `artifacts` | microVM tests, debug-kernel jobs, platform jobs, package jobs | | `guest-ubuntu` | `initramfs-ubuntu.cpio.gz` and its `.packages.json` | `distro-artifacts` | microVM guest and sandbox jobs, package jobs | -| `guest-ubuntu-distro` | `ubuntu-distro.erofs` and its `.manifest.json`, and `ubuntu-smoke-scratch.ext4`, the sandbox scratch template | `distro-artifacts` | microVM guest and sandbox jobs, WHP's runtime adapter job, package jobs | +| `guest-ubuntu-distro` | `ubuntu-distro.erofs` and its `.manifest.json`, and `ubuntu-smoke-scratch.ext4`, the sandbox scratch template | `distro-artifacts` | microVM guest and sandbox jobs, package jobs | | `guest-azurelinux` | `initramfs-azurelinux.cpio.gz` and its `.packages.json` | `distro-artifacts` | microVM guest and sandbox jobs, package jobs | | `guest-debug-kernel` | `vmlinux-debug`, `vmlinux-debug.config`, `vmlinux-debug.provenance.json` | `debug-kernel` | debug-kernel jobs | @@ -1101,11 +1065,7 @@ microVM guest and sandbox jobs boot every guest and run sandboxes from the Ubuntu distro layer over copies of the scratch template, which comes with the layer: the layer's Docker target formats it, its cache entry holds it, and the Ubuntu record gives its digest, so a merge group or push reuses it with the -layer. WHP's runtime adapter job boots the Alpine initramfs, with the -adapter's guest agent in place of NVX's agents, and the layer over copies of -the scratch template, so of the Ubuntu and Azure Linux artifacts it downloads -only the layer. The microVM test workflow's own default, an unsharded default -run, which +layer. The microVM test workflow's own default, an unsharded default run, which no CI job selects, runs the public sandbox scenarios too, so it also downloads the layer. The platform jobs boot the kernel and the Alpine initramfs to qualify their runner and to benchmark, and the package jobs package every guest @@ -1122,12 +1082,6 @@ that uploads none of its components. It also checks that each guest artifact job builds exactly the components that it uploads, and that no component builds in two jobs. -`distro-artifacts` also builds the runtime adapter's guest agent, which is no -guest component: a static musl executable that it uploads as -`runtime-adapter-guest` for WHP's runtime adapter job, which downloads it -into `build/runtime-adapter/`. The Linux guest and sandbox jobs build their -own (see [Rust crates](#rust-crates)). - Each job restores its components from their caches, and builds, verifies, and saves those that miss, as one job did before. The Ubuntu determinism check, the provenance that each build records, and the cache keys are unchanged. A @@ -1742,8 +1696,7 @@ other path selects each suite that takes it as an input: - `run-microvm-tests` enables the NVX microVM tests and their debug-kernel jobs. Their inputs are the OpenVMM pin, the guests and every input of their cache keys, the `nvx` crate, whose executable their scenarios run and whose - guest binaries every guest ships, the runtime adapter in - `tests/runtime-adapter/`, which they test, the workflows, + guest binaries every guest ships, the workflows, actions, and CLI modules that their jobs and the jobs that build their artifacts run, the runner setup scripts, the Rust pin, and `.gitattributes`, which checks out their scripts on Windows. @@ -1804,7 +1757,7 @@ A push, pull request, or merge group that changes neither the OpenVMM pin nor the check, such as one that changes only the microVM tests, the benchmarks, the guests, or `nvx-rs`, therefore skips the OpenVMM CI check. A pull request or merge group that changes only the microVM tests, such as their -scenarios or the runtime adapter, also skips the platform and package jobs. +scenarios, also skips the platform and package jobs. The platform jobs wait for their pool's single benchmark runner. A change only to the performance tooling, such as `performance.py`, the `run-benchmark` action, or `data/`, skips the microVM tests. One that changes the input of no @@ -2362,68 +2315,12 @@ and the prebuilt-binary integration run to and Docker guest build, and Alpine, Ubuntu, and Azure Linux ship its ten executables. -The `nvx-rs` job also checks the -[runtime adapter](../tests/runtime-adapter/README.md) in -`tests/runtime-adapter/`, an independent adapter that builds on `nvx` with a -lockfile of its own, default features off, and the public API only. For each -of its configurations, protocol-only, host-only, and, on Linux, guest-only for -`x86_64-unknown-linux-musl`, the action runs Clippy with warnings denied, the -build, the tests, rustdoc with warnings denied, and a check with the -minimum Rust version of `nvx`, after rustfmt. The tests check that each -configuration builds as an independent adapter's does: it enables only its -side of `nvx`, the guest-only build pulls in neither Tokio nor a -dependency that only the host side declares, and the guest binary is a -static executable. They also check the adapter's own framing -over the control session between the host and the guest: in the host-only -configuration, on Linux and Windows, that the adapter's host program attaches -to the session and exchanges the framing with the guest agent, and in the -guest-only configuration, that its guest agent serves the session, also on a -tty that it puts in raw mode. Each runs against a test broker that sends it -messages that span records, resets, disconnects, and malformed records. The -host-only tests also check the initramfs that runs the guest agent in place of -NVX's agents, the runtime identity that the host program takes from NVX's -identity of its artifacts, and the plan, OpenVMM arguments, and record of a -launch with a layered root. The guest-only tests also run the guest agent's -worker of a layered root against a fake root: its root configuration, the -failure of each stage of the root, which it reports, and its teardown. - The development release job depends on `nvx-rs` and requires its combined Linux/Windows results to be successful; failed, cancelled, or skipped crate checks cannot publish a release. Before it uploads anything, it checks that each platform's package ships `bin/openvmm`, which `nvx-rs download` installs, and `bin/nvx-rs`, or their `.exe` forms on Windows. -The `nvx-microvm-guests-kvm` and `nvx-microvm-guests-mshv` jobs, and on WHP -the `nvx-microvm-adapter-whp` job, build the runtime adapter's -host-only configuration on their backend and run its consolidated suite, the -[stage scenarios](../tests/runtime-adapter/README.md#stage-scenarios), with -`scripts/nvx.py test-runtime-adapter`, which names the backend in -`NVX_HYPERVISOR`. The command runs each scenario that -`tests/runtime-adapter/scenarios.txt` names, by name, its VM scenarios -included, which are the adapter's ignored tests, and fails once a scenario is -renamed, added, or gone, or does not run and pass. Through NVX's supervisor, -the adapter's host program -launches a VM of the job's artifacts, whose initramfs runs the adapter's guest -agent in place of NVX's agents, with the runtime identity that the artifacts' -build provenance gives. Later processes reattach to the VM, as after a restart -of the orchestrator, find it mismatched under another runtime identity without -signaling it, and stop it. The adapter's layered scenarios boot the Ubuntu -EROFS image over a copy of the scratch template, from the guest artifacts, -where the guest agent's worker, outside PID 1, assembles the root through -`nvx::guest::root`. They check the root, the mappings that the worker applies -beneath it, through the root and through `nvx::guest::mappings` with reserved -targets of its own, and those that the mapper refuses, and that a failed -assembly, an invalid root configuration, and a failed teardown each reach the -host program with the stage that failed, and still end the VM cleanly. Its -template and clone scenarios capture templates, and start and run their -clones. The -Linux jobs build the guest agent, a static musl executable, first, and the -WHP job takes the one that `distro-artifacts` builds on Linux, as it takes -the Ubuntu distro layer. The adapter's other tests, including those -of its guest agent, run in the `nvx-rs` job. Changes under -`tests/runtime-adapter/` or `nvx-rs/`, which the adapter builds on, therefore -trigger the NVX microVM test matrix. - ## nvx-rs mutation testing Mutation testing tells whether one selection of the `nvx` crate's tests catches diff --git a/doc/contribute.md b/doc/contribute.md index a7436a07c..45ea0e118 100644 --- a/doc/contribute.md +++ b/doc/contribute.md @@ -90,7 +90,6 @@ the Python, shell, and link checks. | Markdown files and `doc/` paths | Every relative link and image in a tracked Markdown file resolves to a tracked file, and every anchor to a heading slug that GitHub generates, ignoring code. Every `doc/` path that a tracked file mentions resolves too, except the test fixtures that `DOC_PATH_FIXTURES` lists. | [check-quality](../.github/actions/check-quality/action.yml), [check_markdown_links.py](../scripts/check_markdown_links.py) | | Tests | Every test lives in a test file, and production files hold only the test hooks that the allowlist allows; see [Test layout](#test-layout) | [check-quality](../.github/actions/check-quality/action.yml), [check_test_layout.py](../scripts/check_test_layout.py) | | `nvx-rs/` | rustfmt, Clippy and rustdoc with warnings denied, Clippy also for the `serde`, `cli` and `testing` features alone, tests including a bundled-release fixture, the minimum supported Rust version, a macOS check, the public API against its snapshots in `nvx-rs/public-api/`, and, through `nvx.py`, the musl lint, tests, build, digest manifest, and static-ELF check of the guest binaries; see [Rust crates](ci.md#rust-crates) | [check-nvx-rs](../.github/actions/check-nvx-rs/action.yml), [public_api.py](../scripts/public_api.py) | -| `tests/runtime-adapter/` | rustfmt, then Clippy and rustdoc with warnings denied, the build, the tests, and the minimum supported Rust version of `nvx`, in the protocol-only, host-only, and, on Linux, static musl guest-only configurations; see [Rust crates](ci.md#rust-crates) | [check-nvx-rs](../.github/actions/check-nvx-rs/action.yml) | | NVX CLI | Compiles `scripts/`, runs every `scripts/test_*.py` module, and checks command help, on Linux and Windows, and compares the copies of OpenVMM's protocol vectors in `nvx-rs/tests/vectors/openvmm/`, and of the identities of its built-in CPU profiles in `nvx-rs/src/host/cli/cpu_generations.json`, with the submodule's files | [validate-nvx](../.github/actions/validate-nvx/action.yml), [ci.yml](../.github/workflows/ci.yml) | check-quality names each shell script that it checks, and a CLI test fails @@ -148,13 +147,8 @@ of their own beside its source, as [Test layout](#test-layout) describes, and integration tests in its `tests` directory, and also runs `tests/guest_tools.rs` against the exact guest-profile binaries. Tests of the NVX mechanics, such as the conformance vectors of the control protocol, belong in -`nvx`. Tests of what an independent adapter observes through -the public API of `nvx` alone, such as its own framing over the control -session between the host and the guest, belong in the -[runtime adapter](../tests/runtime-adapter/README.md), whose consolidated -suite the microVM jobs run on each backend with -`scripts/nvx.py test-runtime-adapter --backend BACKEND`. [Continuous integration](ci.md) -describes how CI checks OpenVMM's own CI and what the NVX microVM suite covers. +`nvx`. [Continuous integration](ci.md) describes how CI checks OpenVMM's own CI +and what the NVX microVM suite covers. ## Test layout diff --git a/doc/distribution.md b/doc/distribution.md index 253f34c91..eb70029fb 100644 --- a/doc/distribution.md +++ b/doc/distribution.md @@ -162,9 +162,8 @@ change reaches crates.io within a day of its merge. The version that `nvx-rs/Cargo.toml` declares is the lowest that CI publishes. A pull request changes it to start another series, such as -`0.2.0-alpha.1`. Only `nvx`'s version changes in `nvx-rs/Cargo.lock` and -`tests/runtime-adapter/Cargo.lock`. The `bundled` feature of the published -crate downloads the release that +`0.2.0-alpha.1`. Only `nvx`'s version changes in `nvx-rs/Cargo.lock`. The +`bundled` feature of the published crate downloads the release that [`nvx-rs/artifacts.json`](../nvx-rs/artifacts.json) pins and fails unless that release declares the crate's microVM, time, restore-packet, control-session, and control contracts, which CI checks before it publishes. When a contract has diff --git a/doc/project-structure.md b/doc/project-structure.md index ca4ee977f..714e4a3be 100644 --- a/doc/project-structure.md +++ b/doc/project-structure.md @@ -20,7 +20,6 @@ are build products or caches and are not part of the tracked source tree. The | `ubuntu` | Pinned Ubuntu supplemental binary-package lock | | `azurelinux` | Checksum-pinned Azure Linux supplemental RPM lock | | `nvx-rs` | Rust crate `nvx`: the NVX mechanics | -| `tests/runtime-adapter` | Rust crate `runtime-adapter`: an independent adapter that CI builds against `nvx` through its public API | | `openvmm` | Public OpenVMM Git submodule from `microsoft/openvmm`, tracking its `feature/microvm/dev` branch | | `data` | Tracked performance history and generated benchmark data | | `scripts/nvx_tools` | Implementation of the `nvx.py` commands | @@ -178,10 +177,6 @@ nvx/ | |-- publish_nvx_crate.py nvx crate publisher run by CI's full runs of dev | |-- test-layout-allowlist.json The test hooks that production files may hold | `-- test_*.py Python tooling unit tests -|-- tests/ Tests that build on the repository's crates from outside them -| `-- runtime-adapter/ Rust crate `runtime-adapter`, an independent adapter of nvx -| |-- src/ Its framing, its host program, and its guest agent -| `-- tests/ Its independent build, and its guest agent against a test broker |-- SECURITY.md Microsoft security vulnerability reporting policy |-- .dockerignore Docker build-context exclusions |-- .gitattributes Git path attributes @@ -369,23 +364,6 @@ see [Automated environment bootstrap](setup.md#automated-environment-bootstrap). draw the module diagrams that `doc/modules` publishes (see [Module diagrams](ci.md#module-diagrams)). -### `tests/` - -Tests that build on the repository's crates from outside them. -`runtime-adapter/` is an independent adapter of the NVX runtime: a standalone -crate with its own `Cargo.lock` that depends on `nvx` with default features -off and uses its public API only. Its protocol-only, host-only, and guest-only -configurations build in CI, and its tests check that each one builds as an -independent adapter's does. Its host program and its guest agent exchange the -adapter's own framing over the control session between the host and the -guest, and its tests check each of them against a test broker. Its host -program also runs the guest agent in VMs of NVX's artifacts through NVX's -supervisor, and reattaches to them from other processes. In VMs with a -layered root, the guest agent's worker assembles the root through -`nvx::guest::root`, outside PID 1, and the host program checks it with probes, -in a table of scenarios. Its ignored tests check the VMs on each backend in CI. -See its [README](../tests/runtime-adapter/README.md). - ## Root files | Path | Responsibility | @@ -415,7 +393,6 @@ See its [README](../tests/runtime-adapter/README.md). | `dist/` | Staged binary and source release archives | | `openvmm/target/` | Rust build output produced inside the OpenVMM submodule | | `nvx-rs/target/` | Rust build output of the `nvx` crate | -| `tests/runtime-adapter/target/` | Rust build output of the runtime adapter | | `.ruff_cache/` | Ruff's local lint cache | | `__pycache__/` | Python bytecode caches that may appear below Python source directories | diff --git a/doc/usage.md b/doc/usage.md index d0ebf4914..74c92040b 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -507,8 +507,8 @@ the history with `git`. Update the checkout to install a newer release. file against the package's `SHA256SUMS`, and then installs it: it replaces the packaged guest artifacts under `build/`, among them the sandbox scratch template, `build/ubuntu-smoke-scratch.ext4`, that the sandbox scenarios of -[`test-microvm`](#test-microvm) and the runtime adapter copy, removes any known -guest artifact that the release does not declare, such as the Azure Linux +[`test-microvm`](#test-microvm) copy, removes any known guest artifact that +the release does not declare, such as the Azure Linux guest that source-inclusive packages omit, or the template that releases built before it shipped lack, and installs the packaged OpenVMM at `openvmm/target/release/openvmm[.exe]`, executable, with the build provenance @@ -597,7 +597,6 @@ is authorized for the organization when it enforces single sign-on. | `test-microvm` | Run NVX Linux and device correctness tests through OpenVMM. | | `plan-microvm-shards` | Split the default microVM suite into shards of balanced scenario durations for CI. | | `doctor` | Qualify this host for the NVX time ABI. | -| `test-runtime-adapter` | Run the runtime adapter's consolidated suite by name on a real hypervisor. | | `test-adversarial` | Run a brokered Copilot-driven adversarial campaign. | | `build` | Build the guest artifacts and OpenVMM. | | `benchmark` | Run the OpenVMM-native benchmark coordinator. | @@ -1457,38 +1456,6 @@ check and the subset that CI runs. | `--summary PATH` | none | Append a Markdown summary, for example to `$GITHUB_STEP_SUMMARY`. | | `--timeout SECONDS` | `120` | Set the seconds allowed for each probe or guest. `H4` gets its sampling window on top. | -### `test-runtime-adapter` - -```text -python3 scripts/nvx.py test-runtime-adapter - --backend {kvm,mshv,whp} - [--guest-agent PATH] - [--check-only] - [--cargo CARGO] -``` - -Runs the consolidated suite of the [runtime adapter](../tests/runtime-adapter/README.md), -the [stage scenarios](../tests/runtime-adapter/README.md#stage-scenarios) that -`tests/runtime-adapter/scenarios.txt` names, against a real hypervisor. Each -line of that file names one of the adapter's test targets, then one of its -tests, optionally followed by `linux` or `windows` to select that host platform. -Unqualified entries run on both. The command first checks that the selected -lines name exactly the tests of their targets, and that neither the adapter's -library nor another target has an -ignored test, as the adapter's VM scenarios are, so that a scenario that is -renamed, added, or gone fails it rather than drop out of the suite. It then -runs each target's scenarios by -name, in the adapter's host-only configuration, the ignored ones with the -others, with the backend in `NVX_HYPERVISOR`, NVX's artifacts from the -directory that `NVX_ARTIFACTS_DIR` names, from the current directory if it is -relative, or this checkout unless it names one, and the adapter's guest agent -from `--guest-agent` (default -`tests/runtime-adapter/target/x86_64-unknown-linux-musl/release/runtime-adapter-guest`), -and fails unless each of them ran and passed. Execution and counts use the same -platform-selected entries as the inventory check. `--check-only` compiles and -lists the tests but runs no scenarios, and needs neither a hypervisor nor the -guest agent. - ### `test-adversarial` ```text diff --git a/nvx-rs/README.md b/nvx-rs/README.md index 693f7613e..3608f6df5 100644 --- a/nvx-rs/README.md +++ b/nvx-rs/README.md @@ -1959,7 +1959,7 @@ nothing until `capture` returns. After `Continued` or a `Resumed` failure, report on the same session. After `Restored`, take the session's RESET, then call `GuestConnection::acknowledge(window)` on that session before receiving the activation record. The activation module retains its legacy Device-shaped -capture and repair interfaces for the test adapter until its migration. +capture and repair interfaces. ### `nvx::guest::activation` @@ -3208,8 +3208,7 @@ caller owner. Without default features, the crate has only `protocol`, `activation`, `exec`, `contract`, `net`, `gpt`, `uuid`, `Hypervisor`, and `Error`, and depends on nothing but `serde` when `serde` is enabled and `nix` and `getrandom` when -`guest` is; the protocol-only configuration of the -[runtime adapter](../tests/runtime-adapter/README.md) uses it that way. +`guest` is. ## Example @@ -3547,8 +3546,8 @@ compiles the examples with the rest of the crate's targets, and or a restore up to its restore-ready event. The [case matrix](../doc/acceptance-case-matrix.md) gives each test of the -[runtime adapter](../tests/runtime-adapter/README.md), of CI, and of the docs, -and each microVM scenario, a fate: kept, rewritten, or retired. +runtime adapter, of CI, and of the docs, and each microVM scenario, a fate: +kept, rewritten, or retired. The [`check-nvx-rs`](../.github/actions/check-nvx-rs/action.yml) action runs diff --git a/nvx-rs/tests/vectors/README.md b/nvx-rs/tests/vectors/README.md index 9aaefe167..b53ae6063 100644 --- a/nvx-rs/tests/vectors/README.md +++ b/nvx-rs/tests/vectors/README.md @@ -100,10 +100,7 @@ command-line tokens of a capture at OpenVMM's platform tier, and `openvmm/openvmm_defs/src/microvm.rs` that names the control tty's token, which the function reads. The tests of `nvx::host::template` compile them, and check each token of a template's kernel command line with OpenVMM's own function. The -runtime adapter in `tests/runtime-adapter/` compiles them too, as the allowlist -with which the guest agent of a template's VM checks its kernel command line, -and its tests pin the set of tokens that they admit. The third, -`port_forward_socket_creation.rs`, is the variant of the error of +third, `port_forward_socket_creation.rs`, is the variant of the error of `vm/devices/net/net_consomme/src/resolver.rs` with which OpenVMM reports a port forward whose socket on the host it cannot bind, as when a clone's VMM restores its NIC with a forward whose port another socket holds. OpenVMM gives that diff --git a/scripts/check_test_changes.py b/scripts/check_test_changes.py index d6d9d6691..42a2b4121 100644 --- a/scripts/check_test_changes.py +++ b/scripts/check_test_changes.py @@ -110,9 +110,7 @@ r"scripts/nvx_tools/microvm_test_scripts/.+" r"|guest/common/nvx-(?:identity-probe|sandbox-smoke|virtio-restore-probe)" ) -TEST_DATA_FILES = re.compile( - r"scripts/nvx_tools/adversarial_cases/.+|tests/runtime-adapter/scenarios\.txt" -) +TEST_DATA_FILES = re.compile(r"scripts/nvx_tools/adversarial_cases/.+") # Lines that may keep tests from running when a test action or workflow adds # them: a step's condition or its tolerance of failure, a command that ends the # script early or skips tests, and a shell or PowerShell conditional or loop, diff --git a/scripts/nvx.py b/scripts/nvx.py index 5095a008f..7b1945b90 100644 --- a/scripts/nvx.py +++ b/scripts/nvx.py @@ -110,9 +110,6 @@ package_release, verify_source_tree, ) -from nvx_tools.runtime_adapter_tests import ( - configure_parser as configure_runtime_adapter_test_parser, -) from nvx_tools.time_abi import ( benchmark_cpu_profile_refusal, openvmm_cpu_generations, @@ -840,12 +837,6 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: ) configure_doctor_parser(doctor) - runtime_adapter_tests = subparsers.add_parser( - "test-runtime-adapter", - help="run the runtime adapter's consolidated suite by name on a real hypervisor", - ) - configure_runtime_adapter_test_parser(runtime_adapter_tests) - adversarial_tests = subparsers.add_parser( "test-adversarial", help="run a brokered Copilot-driven adversarial campaign", diff --git a/scripts/nvx_tools/ci.py b/scripts/nvx_tools/ci.py index e7fce7f95..08bae4ccc 100644 --- a/scripts/nvx_tools/ci.py +++ b/scripts/nvx_tools/ci.py @@ -51,7 +51,6 @@ "nvx-microvm-guests-kvm": "MICROVM_GUESTS_KVM_RESULT", "nvx-microvm-guests-mshv": "MICROVM_GUESTS_MSHV_RESULT", "nvx-microvm-guests-whp": "MICROVM_GUESTS_WHP_RESULT", - "nvx-microvm-adapter-whp": "MICROVM_ADAPTER_WHP_RESULT", "nvx-microvm-debug-kvm": "MICROVM_DEBUG_KVM_RESULT", "nvx-microvm-debug-mshv": "MICROVM_DEBUG_MSHV_RESULT", "nvx-microvm-debug-whp": "MICROVM_DEBUG_WHP_RESULT", @@ -82,9 +81,7 @@ # from forks too. REQUIRED_CI_OPENVMM_CI_JOB = "openvmm-ci" # Each backend runs the default suite in one job and, beside it, the guest -# smoke and sandbox steps that follow the suite in another, with the runtime -# adapter scenarios on KVM and MSHV. WHP, where those scenarios take about -# twice as long, runs them in a third job. +# smoke and sandbox steps that follow the suite in another. REQUIRED_CI_MICROVM_TEST_JOBS = ( "nvx-microvm-tests-kvm", "nvx-microvm-tests-mshv", @@ -92,7 +89,6 @@ "nvx-microvm-guests-kvm", "nvx-microvm-guests-mshv", "nvx-microvm-guests-whp", - "nvx-microvm-adapter-whp", ) # Pull requests run the CI debug kernel on MSHV, merge groups run it on KVM, # and dev pushes and full runs run it on every backend. @@ -162,13 +158,13 @@ CI_DOCUMENTATION_PATHS = re.compile(r"^(doc/|.*\.md$)", re.IGNORECASE) # Inputs of the NVX microVM tests: the OpenVMM pin, the guests and every input # of their cache keys, the nvx-rs executable that their scenarios run, the -# runtime adapter that they test, the workflows, actions, and CLI modules that -# their jobs and the jobs that build their artifacts run, the runners, and the -# attributes that check out their scripts on Windows. Every module of the CLI -# is an input but those that none of these jobs runs. +# workflows, actions, and CLI modules that their jobs and the jobs that build +# their artifacts run, the runners, and the attributes that check out their +# scripts on Windows. Every module of the CLI is an input but those that none +# of these jobs runs. MICROVM_TEST_INPUT_PATHS = re.compile( r"^(openvmm($|/)|rust-toolchain\.toml$|scripts/setup/|\.gitattributes$|" - r"nvx-rs/|tests/runtime-adapter/|" + r"nvx-rs/|" r"kernel/|guest/|ubuntu/|azurelinux/|" r"docker/|\.dockerignore$|SOURCE-MANIFEST\.json$|" r"\.github/workflows/(ci|build-openvmm-binary|run-nvx-microvm-tests)\.yml$|" @@ -200,7 +196,7 @@ r"scripts/nvx_tools/(?!(adversarial|adversarial_broker|adversarial_executor|" r"crate_release|create_linux_source_archive|managed_exec_tests|merge_queue|" r"microvm_durations|microvm_shards|microvm_tests|nightly_ci|openvmm_ci|" - r"openvmm_process|performance_history|runtime_adapter_tests|" + r"openvmm_process|performance_history|" r"sandbox_lifecycle_tests|sandbox_proxy_tests|sandbox_ram_upper_tests|" r"shard_plans)\.py$|" r"microvm_shards\.json$|adversarial_cases/|" diff --git a/scripts/nvx_tools/runtime_adapter_tests.py b/scripts/nvx_tools/runtime_adapter_tests.py deleted file mode 100644 index 03ed56608..000000000 --- a/scripts/nvx_tools/runtime_adapter_tests.py +++ /dev/null @@ -1,318 +0,0 @@ -"""The runtime adapter's consolidated suite, run by name on a real hypervisor. - -`tests/runtime-adapter/scenarios.txt` names the suite's scenarios, each a test of one of -the adapter's test targets, optionally limited to Linux or Windows. The command checks -that the host's scenarios name exactly the tests of their targets, and that neither the -library nor another target has an ignored test, as the scenarios that boot VMs are, so -that a scenario that is renamed, added, or gone -fails the command rather than drop out of it. It then runs each target's scenarios by -name, the ignored ones with the others, and checks that each of them ran and passed. -""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import shlex -import subprocess -import sys -from collections.abc import Sequence -from dataclasses import dataclass -from pathlib import Path -from typing import Any, cast - -from .build_constants import BuildConstants, OpenVMMBuildConstants -from .ci import OPENVMM_TEST_BACKENDS, validate_openvmm_test_backend -from .common import ScriptError, require_file - -ADAPTER_DIRECTORY = BuildConstants.REPO_ROOT / "tests" / "runtime-adapter" -PACKAGE = "runtime-adapter" -SCENARIOS = ADAPTER_DIRECTORY / "scenarios.txt" -GUEST_AGENT = ( - ADAPTER_DIRECTORY - / "target" - / OpenVMMBuildConstants.MUSL_RUST_TARGET - / "release" - / "runtime-adapter-guest" -) -# libtest's line for a test that ended, and the summary of a test target's run. -_TEST_LINE = re.compile(r"^test (\S+) \.\.\. (\S+)") -_RESULT_LINE = re.compile(r"^test result: \S+\. (\d+) passed; ") - - -@dataclass(frozen=True) -class Target: - """A test target of the adapter, with the tests of it that the scenarios name.""" - - name: str - tests: tuple[str, ...] - - -def configure_parser(parser: argparse.ArgumentParser) -> None: - parser.description = ( - "Run the runtime adapter's consolidated suite on a real hypervisor: each " - "scenario that tests/runtime-adapter/scenarios.txt names, by its name." - ) - parser.add_argument("--backend", choices=OPENVMM_TEST_BACKENDS, required=True) - parser.add_argument( - "--guest-agent", - type=Path, - default=GUEST_AGENT, - help="the adapter's guest agent, a static musl executable (default: its " - "release build in tests/runtime-adapter)", - ) - parser.add_argument( - "--check-only", - action="store_true", - help="check that the scenarios name exactly the tests of their targets, and " - "run none", - ) - parser.add_argument( - "--cargo", - default="cargo", - help="cargo executable used to build and run the tests (default: cargo)", - ) - parser.set_defaults(handler=command_test_runtime_adapter) - - -def read_scenarios(path: Path = SCENARIOS) -> list[Target]: - """Returns the host's scenarios of `path`, grouped by target in the order of their - first selected lines, each with its tests in the order of their lines. - - A line names a target, then one of its tests, optionally followed by `linux` or - `windows`; `#` starts a comment. Raises `ScriptError` for another line, an unknown - platform, and a scenario that two lines name, even on different platforms. - """ - host_platform = "windows" if sys.platform == "win32" else sys.platform - tests: dict[str, list[str]] = {} - seen: set[tuple[str, str]] = set() - for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): - fields = line.split("#", 1)[0].split() - if not fields: - continue - where = f"{path}:{number}" - if len(fields) not in (2, 3): - raise ScriptError(f"{where}: expected a target and a test, not {line!r}") - target, test = fields[:2] - platform = fields[2] if len(fields) == 3 else None - if platform not in (None, "linux", "windows"): - raise ScriptError(f"{where}: unknown scenario platform {platform!r}") - if (target, test) in seen: - raise ScriptError(f"{where}: {target} {test} is named twice") - seen.add((target, test)) - if platform is None or platform == host_platform: - tests.setdefault(target, []).append(test) - if not tests: - raise ScriptError(f"{path} names no scenario") - return [Target(name, tuple(named)) for name, named in tests.items()] - - -def cargo_test_targets(cargo: str, environment: dict[str, str]) -> list[str]: - """Returns the names of the adapter's test targets as Cargo finds them, those of - the directories of its `tests` directory that hold a `main.rs` included.""" - command = [cargo, "metadata", "--locked", "--no-deps", "--format-version", "1"] - found = listing(command, environment) - if found.returncode != 0: - raise ScriptError(f"cannot read the adapter's targets:\n{found.stderr}") - metadata = cast(dict[str, Any], json.loads(found.stdout)) - for package in cast(list[dict[str, Any]], metadata["packages"]): - if package["name"] == PACKAGE: - return sorted( - cast(str, target["name"]) - for target in cast(list[dict[str, Any]], package["targets"]) - if "test" in target["kind"] - ) - raise ScriptError(f"Cargo's metadata of the adapter has no package {PACKAGE}") - - -def cargo_test(cargo: str, target: str | None, *arguments: str) -> list[str]: - """Returns the command that runs the test target `target` of the adapter's - host-only configuration, or the unit tests of its library for `None`, with - libtest's `arguments`.""" - selection = ["--lib"] if target is None else ["--test", target] - return [ - cargo, - "test", - "--locked", - "--features", - "host", - *selection, - "--", - *arguments, - ] - - -def listed_tests(output: str) -> list[str]: - """Returns the tests that libtest's terse list `output` names.""" - suffix = ": test" - return [ - line.removesuffix(suffix) - for line in output.splitlines() - if line.endswith(suffix) - ] - - -def naming_problems(target: Target, tests: Sequence[str]) -> list[str]: - """Returns why the scenarios of `target` do not name exactly `tests`, the tests - that the target has.""" - present = set(tests) - named = set(target.tests) - problems = [ - f"{target.name} has no test {test}" - for test in target.tests - if test not in present - ] - problems.extend( - f"{target.name} has the test {test}, which no scenario names" - for test in tests - if test not in named - ) - return problems - - -def run_problems(target: Target, output: str, status: int) -> list[str]: - """Returns why the run of the scenarios of `target`, which printed `output` and - exited with `status`, does not show that each of them ran and passed.""" - outcomes: dict[str, str] = {} - passed = 0 - for line in output.splitlines(): - if (ended := _TEST_LINE.match(line)) is not None: - outcomes[ended.group(1)] = ended.group(2) - elif (result := _RESULT_LINE.match(line)) is not None: - passed += int(result.group(1)) - problems: list[str] = [] - for test in target.tests: - outcome = outcomes.get(test) - if outcome is None: - problems.append(f"{target.name} {test} did not run") - elif outcome != "ok": - problems.append(f"{target.name} {test}: {outcome}") - if passed != len(target.tests): - problems.append(f"{target.name} passed {passed} tests, not {len(target.tests)}") - if status != 0 and not problems: - problems.append(f"the tests of {target.name} exited with status {status}") - return problems - - -def listing( - command: list[str], environment: dict[str, str] -) -> subprocess.CompletedProcess[str]: - """Runs `command`, which lists tests, in the adapter's directory, and returns what - it printed.""" - return subprocess.run( - command, - cwd=ADAPTER_DIRECTORY, - env=environment, - check=False, - capture_output=True, - text=True, - encoding="utf-8", - errors="replace", - ) - - -def streamed(command: list[str], environment: dict[str, str]) -> tuple[str, int]: - """Runs `command` in the adapter's directory, printing its output as it comes, - and returns that output and the command's status.""" - print(f">> {shlex.join(command)}", flush=True) - lines: list[str] = [] - with subprocess.Popen( - command, - cwd=ADAPTER_DIRECTORY, - env=environment, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - text=True, - encoding="utf-8", - errors="replace", - ) as process: - assert process.stdout is not None - for line in process.stdout: - sys.stdout.write(line) - sys.stdout.flush() - lines.append(line) - status = process.wait() - return "".join(lines), status - - -def suite_problems( - targets: Sequence[Target], cargo: str, environment: dict[str, str] -) -> list[str]: - """Returns why `targets` are not the adapter's suite: each target that is not one - of the adapter's test targets, each scenario that names a test that its target does - not have, each test of a named target that no scenario names, and each ignored test - of the library or of a test target that no scenario names.""" - available = cargo_test_targets(cargo, environment) - problems: list[str] = [] - for target in targets: - if target.name not in available: - problems.append(f"{target.name} is not a test target of the adapter") - continue - listed = listing( - cargo_test(cargo, target.name, "--list", "--format", "terse"), environment - ) - if listed.returncode != 0: - problems.append(f"cannot list the tests of {target.name}:\n{listed.stderr}") - continue - problems.extend(naming_problems(target, listed_tests(listed.stdout))) - named = {target.name for target in targets} - others = [name for name in available if name not in named] - for name in [None, *others]: - command = cargo_test(cargo, name, "--ignored", "--list", "--format", "terse") - listed = listing(command, environment) - what = "the library" if name is None else name - if listed.returncode != 0: - problems.append(f"cannot list the tests of {what}:\n{listed.stderr}") - continue - problems.extend( - f"{what} has the ignored test {test}, which no scenario names" - for test in listed_tests(listed.stdout) - ) - return problems - - -def command_test_runtime_adapter(args: argparse.Namespace) -> int: - backend = cast(str, args.backend) - cargo = cast(str, args.cargo) - check_only = cast(bool, args.check_only) - targets = read_scenarios() - environment = os.environ.copy() - if not check_only: - validate_openvmm_test_backend(backend) - guest = require_file( - cast(Path, args.guest_agent), "runtime adapter guest agent" - ) - environment["NVX_HYPERVISOR"] = backend - # Cargo runs the tests in the adapter's directory, so a relative directory of - # artifacts is the caller's, and NVX takes an empty one for none. - artifacts = environment.get("NVX_ARTIFACTS_DIR") - environment["NVX_ARTIFACTS_DIR"] = os.fspath( - Path(artifacts).resolve() if artifacts else BuildConstants.REPO_ROOT - ) - environment["RUNTIME_ADAPTER_GUEST"] = os.fspath(guest.resolve()) - problems = suite_problems(targets, cargo, environment) - if not problems and not check_only: - for target in targets: - command = cargo_test( - cargo, target.name, "--include-ignored", "--exact", *target.tests - ) - output, status = streamed(command, environment) - problems.extend(run_problems(target, output, status)) - count = sum(len(target.tests) for target in targets) - if problems: - print( - f"the runtime adapter's suite of {count} scenarios failed:", file=sys.stderr - ) - for problem in problems: - print(f" {problem}", file=sys.stderr) - return 1 - if check_only: - print( - f"{count} scenarios name exactly the tests of {len(targets)} targets, and " - "neither the library nor another target has an ignored test" - ) - else: - print(f"the {count} scenarios of {len(targets)} targets passed on {backend}") - return 0 diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py index bdf9b2407..d19ad4d7e 100644 --- a/scripts/test_nvx_tools.py +++ b/scripts/test_nvx_tools.py @@ -62,7 +62,6 @@ performance, performance_history, release, - runtime_adapter_tests, ubuntu, ) from nvx_tools.build_constants import ( # noqa: E402 @@ -6071,10 +6070,6 @@ def load(name: str) -> Any: "setup-cross-os-cache": set[str](), "verify": set[str](), "test-openvmm": kernel_alpine, - # The runtime adapter's VM scenarios boot what nvx finds, and its - # layered scenarios boot the Ubuntu layer over a copy of the - # scratch template (tests/runtime-adapter/tests/common/mod.rs). - "test-runtime-adapter": kernel_alpine | workload, # A sandbox boots the Alpine control initramfs under its layers. "sandbox": kernel_alpine, } @@ -6574,61 +6569,6 @@ def test_nvx_rs_builds_and_checks_every_guest_binary_for_musl(self): build, ) - def test_nvx_rs_checks_the_runtime_adapter_in_each_configuration(self): - # The runtime adapter builds on the nvx crate as an independent adapter - # does, with a lockfile of its own and default features off. Each - # configuration is linted, built, tested, documented with warnings - # denied, and checked with the minimum Rust version of nvx; the - # guest-only one builds the static musl executable that guests run, so - # it needs Linux. - root = BuildConstants.REPO_ROOT - action = ( - root / ".github" / "actions" / "check-nvx-rs" / "action.yml" - ).read_text(encoding="utf-8") - adapter = root / "tests" / "runtime-adapter" - manifest = (adapter / "Cargo.toml").read_text(encoding="utf-8") - self.assertIn( - 'nvx = { path = "../../nvx-rs", default-features = false }\n', manifest - ) - self.assertIn("\ndefault = []\n", manifest) - self.assertTrue((adapter / "Cargo.lock").is_file()) - toolchain = '"+${{ steps.toolchains.outputs.toolchain }}"' - msrv = '"+${{ steps.toolchains.outputs.msrv }}"' - musl = "x86_64-unknown-linux-musl" - for configuration, options in ( - ("protocol-only", ""), - ("host-only", " --features host"), - ("guest-only", f" --features guest --target {musl}"), - ): - with self.subTest(configuration=configuration): - name = f"Check the runtime adapter's {configuration} configuration" - step = _composite_action_step(action, name) - self.assertIn(" working-directory: tests/runtime-adapter\n", step) - self.assertEqual( - " if: runner.os == 'Linux'\n" in step, - configuration == "guest-only", - ) - script = " ".join( - _composite_action_script(action, name).replace("\\\n", " ").split() - ) - self.assertTrue(script.startswith("set -euo pipefail ")) - for command in ( - f"cargo {toolchain} clippy --locked --all-targets{options} " - "-- -D warnings", - f"cargo {toolchain} build --locked{options}", - f"cargo {toolchain} test --locked{options}", - f'RUSTDOCFLAGS="-D warnings" cargo {toolchain} doc --locked ' - f"--no-deps{options}", - f"cargo {msrv} check --locked --all-targets{options}", - ): - self.assertIn(command, script) - if configuration == "protocol-only": - self.assertIn(f"cargo {toolchain} fmt --all --check", script) - if configuration == "guest-only": - self.assertIn( - f'rustup target add --toolchain "${{toolchain}}" {musl}', script - ) - def test_required_ci_result_policy(self): always_successful = { "quality", @@ -7521,7 +7461,7 @@ def select(*outputs: str) -> ci.CiChanges: (("doc/ci.md", "README.md", "DOC/Guide.MD"), select(), None), # Documentation selects no suite, even in a suite's directory. (("scripts/setup/README.md",), select(), None), - (("nvx-rs/README.md", "tests/runtime-adapter/README.md"), select(), None), + (("nvx-rs/README.md",), select(), None), ( ("scripts/setup/README.md", "scripts/test_nvx_tools.py"), select(), @@ -7615,11 +7555,6 @@ def select(*outputs: str) -> ci.CiChanges: (("nvx-rs/tests/cli.rs",), microvm_and_benchmarks, None), # Inputs of the microVM tests alone, such as the planner of their # shards (#691). - ( - ("tests/runtime-adapter/src/lib.rs",), - microvm_only, - microvm_and_benchmarks, - ), ( ("scripts/nvx_tools/microvm_tests.py",), microvm_only, @@ -7803,165 +7738,6 @@ def test_ci_change_classification_builds_artifacts_for_the_jobs_that_use_them(se changes.run_microvm_tests or changes.run_benchmarks, ) - def test_ci_change_classification_runs_microvm_tests_for_the_runtime_adapter(self): - # The microVM jobs run the runtime adapter's suite on every backend, so - # a change to the adapter, its suite, or the command that runs it alone - # runs them. The OpenVMM CI check reads none of them, and neither the - # benchmarks nor the packages use the adapter, so a pull request skips - # the platform and package jobs. - workflow = ( - BuildConstants.REPO_ROOT - / ".github" - / "workflows" - / "run-nvx-microvm-tests.yml" - ).read_text(encoding="utf-8") - for system in ("Linux", "Windows"): - with self.subTest(system=system): - hook = _workflow_step( - workflow, "test", f"Run runtime adapter scenarios on {system}" - ) - self.assertIn("!inputs.debug-kernel", hook) - self.assertIn("nvx.py test-runtime-adapter\n", hook) - self.assertIn('--backend "${{ inputs.backend }}"\n', hook) - adapter = ( - "tests/runtime-adapter/Cargo.toml", - "tests/runtime-adapter/Cargo.lock", - "tests/runtime-adapter/src/lib.rs", - "tests/runtime-adapter/src/bin/runtime-adapter-guest.rs", - "tests/runtime-adapter/tests/independent_build.rs", - "tests/runtime-adapter/scenarios.txt", - ) - for changed_files in ( - *((path,) for path in adapter), - adapter, - ("scripts/nvx_tools/runtime_adapter_tests.py",), - ): - with self.subTest(changed_files=changed_files): - changes = ci.classify_ci_changes("pull_request", changed_files) - self.assertEqual(changes, ci.CiChanges(True, True, False, True, False)) - self.assertEqual( - ci.classify_ci_changes("push", changed_files), - ci.CiChanges(True, True, False, True, True), - ) - expected = ci.required_ci_expected_results( - "pull_request", same_repository=True, changes=changes - ) - for job in ( - *ci.REQUIRED_CI_MICROVM_TEST_JOBS, - *ci.REQUIRED_CI_MICROVM_DEBUG_JOBS, - ): - self.assertEqual(expected[job], "success", job) - for job in ( - ci.REQUIRED_CI_OPENVMM_CI_JOB, - *ci.REQUIRED_CI_PLATFORM_JOBS, - *ci.REQUIRED_CI_PACKAGE_JOBS, - "performance-gate", - ): - self.assertEqual(expected[job], "skipped", job) - summary = ci.describe_ci_changes( - "pull_request", - adapter, - ci.classify_ci_changes("pull_request", adapter), - ) - self.assertIn( - "### NVX microVM tests: run\nInputs changed: " - "`tests/runtime-adapter/Cargo.toml`, `tests/runtime-adapter/Cargo.lock`, " - "`tests/runtime-adapter/src/lib.rs`, and 3 more.\n", - summary, - ) - self.assertIn( - "### Platform benchmarks and packages: skip\nNo input changed.\n", - summary, - ) - - def test_runtime_adapter_scenarios_take_a_guest_agent_that_linux_builds(self): - # The scenarios boot VMs of the artifacts that the microVM jobs that - # run them download into the checkout, with the adapter's guest agent, - # a static musl executable. The Linux jobs build it before the - # scenarios run, and the WHP job takes the one that distro-artifacts - # builds on Linux, as the WHP guest and sandbox job takes the Ubuntu - # and Azure Linux images. The default suites neither build nor - # download it. - github = BuildConstants.REPO_ROOT / ".github" - workflow = (github / "workflows" / "ci.yml").read_text(encoding="utf-8") - microvm_workflow = ( - github / "workflows" / "run-nvx-microvm-tests.yml" - ).read_text(encoding="utf-8") - build = ( - "cargo build --locked --release --features guest " - "--target x86_64-unknown-linux-musl --bin runtime-adapter-guest" - ) - agent = ( - "tests/runtime-adapter/target/x86_64-unknown-linux-musl/release/" - "runtime-adapter-guest" - ) - - def step(job: str, name: str) -> str: - return _workflow_step( - microvm_workflow if job == "test" else workflow, job, name - ) - - linux_build = step("test", "Build the runtime adapter's guest agent on Linux") - self.assertIn( - "if: runner.os != 'Windows' && inputs.runtime-adapter " - "&& !inputs.debug-kernel\n", - linux_build, - ) - self.assertIn("working-directory: tests/runtime-adapter\n", linux_build) - self.assertIn(build, " ".join(linux_build.split())) - download = step("test", "Download the runtime adapter's guest agent on Windows") - self.assertIn( - "if: runner.os == 'Windows' && inputs.runtime-adapter " - "&& !inputs.debug-kernel\n", - download, - ) - self.assertIn("uses: actions/download-artifact@v8\n", download) - self.assertIn("name: runtime-adapter-guest\n", download) - self.assertIn("path: build/runtime-adapter\n", download) - for system, setup, guest_agent in ( - ("Linux", linux_build, agent), - ( - "Windows", - download, - "build\\runtime-adapter\\runtime-adapter-guest", - ), - ): - with self.subTest(system=system): - name = f"Run runtime adapter scenarios on {system}" - hook = step("test", name) - # The command takes NVX's artifacts from the checkout, into which - # the job downloaded them. - self.assertNotIn("NVX_ARTIFACTS_DIR", hook) - self.assertIn(f"--guest-agent {guest_agent}\n", hook) - self.assertLess( - microvm_workflow.index(setup), - microvm_workflow.index(f" - name: {name}\n"), - ) - - shared = " ".join( - step("distro-artifacts", "Build the runtime adapter's guest agent") - .replace("\\\n", " ") - .split() - ) - self.assertIn("working-directory: tests/runtime-adapter", shared) - self.assertIn( - 'rustup toolchain install "${toolchain}" --profile minimal ' - "--no-self-update --target x86_64-unknown-linux-musl", - shared, - ) - self.assertIn( - build.replace("cargo build", 'cargo "+${toolchain}" build'), shared - ) - upload = step("distro-artifacts", "Upload the runtime adapter's guest agent") - self.assertIn("uses: actions/upload-artifact@v7\n", upload) - self.assertIn("name: runtime-adapter-guest\n", upload) - self.assertIn(f"path: {agent}\n", upload) - self.assertIn( - "needs: [artifacts, distro-artifacts, build-openvmm-windows-msvc, " - "build-nvx-rs-windows-msvc, openvmm-changes]", - _workflow_job(workflow, "nvx-microvm-adapter-whp"), - ) - def test_ci_change_classification_covers_each_suite_input(self): github = BuildConstants.REPO_ROOT / ".github" tools = BuildConstants.REPO_ROOT / "scripts" / "nvx_tools" @@ -8140,10 +7916,6 @@ def job_inputs( all_guest_inputs = ( set[str]().union(*guest_inputs.values()) | guest_binary_inputs ) - microvm_workflow = ( - github / "workflows" / "run-nvx-microvm-tests.yml" - ).read_text(encoding="utf-8") - self.assertIn("working-directory: tests/runtime-adapter\n", microvm_workflow) microvm_inputs = ( job_inputs( *ci.REQUIRED_CI_MICROVM_TEST_JOBS, @@ -8156,7 +7928,6 @@ def job_inputs( ) | nvx_rs_inputs | all_guest_inputs - | {"tests/runtime-adapter/example"} ) # The platform jobs check their history's expected host in data/, the # package jobs name their packages with VERSION and ship LICENSE, and @@ -9117,11 +8888,6 @@ def test_ci_shares_openvmm_inputs_and_binary_artifacts(self): "openvmm-windows-msvc", "run-nvx-microvm-tests.yml", ), - "nvx-microvm-adapter-whp": ( - "build-openvmm-windows-msvc", - "openvmm-windows-msvc", - "run-nvx-microvm-tests.yml", - ), "nvx-microvm-debug-kvm": ( "build-openvmm-linux-gnu", "openvmm-linux-gnu", @@ -9229,13 +8995,12 @@ def test_ci_shares_openvmm_inputs_and_binary_artifacts(self): self.assertEqual(build_workflow.count("retention-days: 1"), 2) # The microVM workflow also downloads nvx-rs, which the benchmarks # don't run, the debug kernel, which neither the platform nor package - # jobs read, the Ubuntu distro layer in two more steps, for an - # unsharded default run and for the runtime adapter scenarios alone, - # and on Windows the runtime adapter's guest agent. The platform jobs - # download only what they boot, and the package jobs every image that - # a package ships; their action downloads nvx-rs. + # jobs read, and the Ubuntu distro layer a second time for an + # unsharded default run. The platform jobs download only what they + # boot, and the package jobs every image that a package ships; their + # action downloads nvx-rs. for consumer_workflow, download_count in ( - (microvm_workflow, 12), + (microvm_workflow, 10), (platform_workflow, 4), (package_workflow, 7), ): @@ -10016,14 +9781,9 @@ def test_ci_runs_the_steps_after_the_default_microvm_suite_beside_it(self): # second job beside the suite, on the same runner pool and with the # same OpenVMM and nvx-rs builds. Of the two, only that job reads the # Ubuntu and Azure Linux artifacts, so only it downloads and waits for - # them. #982: the runtime adapter scenarios, which that job runs last, - # take about twice as long on WHP as on KVM and MSHV, and WHP's job - # then ended last in most runs, so WHP runs them in a third job beside - # the other two, which waits for the same artifacts as the second but - # downloads, of the Ubuntu and Azure Linux ones, only the Ubuntu - # distro layer. Together, each backend's jobs run every step of the - # former single job exactly once, and the debug-kernel jobs, which - # never set a selector, run as before. + # them. Together, the two jobs run every step of the former single job + # exactly once, and the debug-kernel jobs, which never set the + # selector, run as before. workflows = BuildConstants.REPO_ROOT / ".github" / "workflows" jobs = yaml.safe_load((workflows / "ci.yml").read_text(encoding="utf-8"))[ "jobs" @@ -10032,19 +9792,14 @@ def test_ci_runs_the_steps_after_the_default_microvm_suite_beside_it(self): (workflows / "run-nvx-microvm-tests.yml").read_text(encoding="utf-8") ) # PyYAML reads the `on` key as true. - declared = microvm[True]["workflow_call"]["inputs"] - for selector in ("guests-and-sandboxes", "runtime-adapter"): - with self.subTest(selector=selector): - self.assertEqual(declared[selector]["type"], "boolean") - self.assertIs(declared[selector]["default"], False) + selector = microvm[True]["workflow_call"]["inputs"]["guests-and-sandboxes"] + self.assertEqual(selector["type"], "boolean") + self.assertIs(selector["default"], False) backends = { "kvm": "Linux / KVM", "mshv": "Linux / MSHV", "whp": "Windows / WHP", } - # The backends that run the runtime adapter scenarios in a job of - # their own. - separate = {"whp"} self.assertEqual( { name @@ -10055,8 +9810,7 @@ def test_ci_runs_the_steps_after_the_default_microvm_suite_beside_it(self): f"nvx-microvm-{kind}-{backend}" for kind in ("tests", "guests", "debug") for backend in backends - } - | {f"nvx-microvm-adapter-{backend}" for backend in separate}, + }, ) for backend, label in backends.items(): suite = jobs[f"nvx-microvm-tests-{backend}"] @@ -10072,7 +9826,7 @@ def test_ci_runs_the_steps_after_the_default_microvm_suite_beside_it(self): self.assertNotIn("distro-artifacts", suite["needs"]) # The guest and sandbox job also waits for the Ubuntu and Azure # Linux artifacts, and otherwise equals the suite's job but for - # its name, the selectors, and the suite's shards + # its name, the selector, and the suite's shards # (test_ci_runs_each_default_microvm_suite_in_its_shards). conjuncts = _expression_conjuncts(suite["if"]) after = conjuncts.index("needs.artifacts.result == 'success'") + 1 @@ -10096,26 +9850,11 @@ def test_ci_runs_the_steps_after_the_default_microvm_suite_beside_it(self): *conjuncts[after:], ], "with": inputs - | {"display-name": name, "guests-and-sandboxes": True} - | ({} if backend in separate else {"runtime-adapter": True}), + | {"display-name": name, "guests-and-sandboxes": True}, }, ) - if backend in separate: - # The runtime adapter job equals the guest and sandbox job - # but for its name and selector. - adapter = f"{label} / Runtime adapter" - self.assertEqual( - jobs[f"nvx-microvm-adapter-{backend}"], - guests - | { - "name": f"NVX microVM tests / {adapter}", - "with": inputs - | {"display-name": adapter, "runtime-adapter": True}, - }, - ) - for selector in ("guests-and-sandboxes", "runtime-adapter"): - self.assertNotIn(selector, suite["with"]) - self.assertNotIn(selector, debug["with"]) + self.assertNotIn("guests-and-sandboxes", suite["with"]) + self.assertNotIn("guests-and-sandboxes", debug["with"]) self.assertIs(debug["with"]["debug-kernel"], True) steps = microvm["jobs"]["test"]["steps"] @@ -10133,29 +9872,21 @@ def test_ci_runs_the_steps_after_the_default_microvm_suite_beside_it(self): if str(step.get("uses", "")).startswith("actions/upload-artifact@") ] - def selected( - backend: str, *, debug: bool, parts: tuple[bool, bool] | None - ) -> list[str]: - # The steps of a job that sets the guests-and-sandboxes and - # runtime-adapter selectors to `parts`. None stands for the single - # job that ran every step before the splits: it ignores the - # selectors' conjuncts. Only the default-suite jobs run a shard; - # the single job, the jobs that set a selector, and the - # debug-kernel jobs pass none. + def selected(backend: str, *, debug: bool, guests: bool | None) -> list[str]: + # A selector of None stands for the single job that ran every step + # before the split: it ignores the guests-and-sandboxes conjuncts. + # Only the default-suite jobs run a shard; the single job, the + # guest and sandbox jobs, and the debug-kernel jobs pass none. windows = backend == "whp" - single = parts is None - guests, adapter = parts or (True, True) - sharded = not single and not guests and not adapter and not debug + sharded = guests is False and not debug values = { "runner.os != 'Windows'": not windows, "runner.os == 'Windows'": windows, "inputs.backend == 'kvm'": backend == "kvm", "inputs.debug-kernel": debug, "!inputs.debug-kernel": not debug, - "inputs.guests-and-sandboxes": guests, - "!inputs.guests-and-sandboxes": single or not guests, - "inputs.runtime-adapter": adapter, - "!inputs.runtime-adapter": single or not adapter, + "inputs.guests-and-sandboxes": guests is not False, + "!inputs.guests-and-sandboxes": guests is not True, "!inputs.shard": not sharded, } chosen: list[str] = [] @@ -10177,45 +9908,30 @@ def downloaded(selection: list[str]) -> set[str]: for backend in backends: system = "Windows" if backend == "whp" else "Linux" - single = selected(backend, debug=False, parts=None) - suite_steps = selected(backend, debug=False, parts=(False, False)) - if backend in separate: - guest_steps = selected(backend, debug=False, parts=(True, False)) - adapter_steps = selected(backend, debug=False, parts=(False, True)) - split = [suite_steps, guest_steps, adapter_steps] - else: - guest_steps = selected(backend, debug=False, parts=(True, True)) - adapter_steps = [] - split = [suite_steps, guest_steps] + single = selected(backend, debug=False, guests=None) + suite_steps = selected(backend, debug=False, guests=False) + guest_steps = selected(backend, debug=False, guests=True) with self.subTest(backend=backend): - # The jobs take the same builds and validate the runner the + # Both jobs take the same builds and validate the runner the # same way, and every other step of the single job runs in # exactly one of them. Each downloads the guest artifacts that # its steps read (test_ci_jobs_download_exactly_the_guest_ # artifacts_that_they_read): the default suite only the kernel - # and the Alpine initramfs, the guest and sandbox steps every - # guest artifact that the single job downloaded, and the - # runtime adapter scenarios alone those two and the Ubuntu - # distro layer. + # and the Alpine initramfs, and the guest and sandbox steps + # every guest artifact that the single job downloaded. for step_name in setup: if step_name in downloads: continue - for job_steps in split: - self.assertEqual(step_name in job_steps, step_name in single) + self.assertEqual(step_name in suite_steps, step_name in single) + self.assertEqual(step_name in guest_steps, step_name in single) self.assertEqual( downloaded(suite_steps), {"guest-kernel", "guest-alpine"} ) self.assertEqual(downloaded(guest_steps), downloaded(single)) - if adapter_steps: - self.assertEqual( - downloaded(adapter_steps), - {"guest-kernel", "guest-alpine", "guest-ubuntu-distro"}, - ) self.assertEqual( sorted( step_name - for job_steps in split - for step_name in job_steps + for step_name in suite_steps + guest_steps if step_name not in setup ), sorted(step_name for step_name in single if step_name not in setup), @@ -10231,36 +9947,21 @@ def downloaded(selection: list[str]) -> set[str]: f"Run Ubuntu sandbox layer smoke test on {system}", ): self.assertIn(step_name, guest_steps) - scenarios = f"Run runtime adapter scenarios on {system}" - self.assertEqual(scenarios in guest_steps, backend not in separate) - if adapter_steps: - self.assertEqual( - [ - step_name - for step_name in adapter_steps - if step_name not in setup - ], - [ - "Download the runtime adapter's guest agent on Windows", - scenarios, - ], - ) self.assertEqual( "Run extended Ubuntu guest tests on Linux/KVM" in guest_steps, backend == "kvm", ) self.assertEqual( - selected(backend, debug=True, parts=(False, False)), - selected(backend, debug=True, parts=None), + selected(backend, debug=True, guests=False), + selected(backend, debug=True, guests=None), ) # A second upload of an artifact name in a run fails, so each shard of # the default suite, the guest and sandbox job, and the debug-kernel - # job of every backend upload under names of their own. A job that - # runs the runtime adapter scenarios alone runs no test-microvm step, - # so it has nothing to upload and uploads nothing. + # job of every backend upload under names of their own. defaults = { - name: definition.get("default") for name, definition in declared.items() + name: definition.get("default") + for name, definition in microvm[True]["workflow_call"]["inputs"].items() } legs = [ {**defaults, **leg} @@ -10268,31 +9969,12 @@ def downloaded(selection: list[str]) -> set[str]: if job.get("uses") == "./.github/workflows/run-nvx-microvm-tests.yml" for leg in _matrix_legs(job) ] - statuses = {"always": lambda: True, "failure": lambda: True} - - def uploads_in(upload: Mapping[str, Any], leg: Mapping[str, Any]) -> bool: - return _expression_truthy( - _evaluate_expression(str(upload["if"]), {"inputs": leg}, statuses) - ) - self.assertTrue(uploads) for upload in uploads: - uploaded = [ - _resolve_inputs(upload["with"]["name"], leg) - for leg in legs - if uploads_in(upload, leg) - ] + uploaded = [_resolve_inputs(upload["with"]["name"], leg) for leg in legs] with self.subTest(upload=upload["name"]): self.assertNotIn("inputs.", " ".join(uploaded)) self.assertEqual(len(set(uploaded)), len(uploaded), uploaded) - self.assertEqual( - [ - leg["display-name"] - for leg in legs - if not uploads_in(upload, leg) - ], - [f"{backends[backend]} / Runtime adapter" for backend in separate], - ) failure_logs = next( step for step in uploads @@ -12838,9 +12520,7 @@ def test_every_microvm_job_uploads_its_scenario_durations(self): # Each microVM job, the guest and sandbox and debug-kernel ones and # each shard of a default suite too, keeps the duration report of # every test-microvm step whether it passes, fails, or is cancelled, - # under a name of its own in each attempt (#694, #695, #696). A job - # that runs the runtime adapter scenarios alone runs no test-microvm - # step, so it keeps no report, and no failure logs either (#982). + # under a name of its own in each attempt (#694, #695, #696). workflows = BuildConstants.REPO_ROOT / ".github" / "workflows" steps = { step["name"]: step @@ -12849,8 +12529,7 @@ def test_every_microvm_job_uploads_its_scenario_durations(self): )["jobs"]["test"]["steps"] } upload = steps["Upload NVX microVM scenario durations"] - runs_test_microvm = "(inputs.guests-and-sandboxes || !inputs.runtime-adapter)" - self.assertEqual(upload["if"], f"always() && {runs_test_microvm}") + self.assertEqual(upload["if"], "always()") self.assertEqual(upload["uses"], "actions/upload-artifact@v7") self.assertEqual( upload["with"]["path"], @@ -12870,10 +12549,7 @@ def test_every_microvm_job_uploads_its_scenario_durations(self): 1, ) # The failure logs keep their upload, only when a step fails. - self.assertEqual( - steps["Upload NVX microVM failure logs"]["if"], - f"failure() && {runs_test_microvm}", - ) + self.assertEqual(steps["Upload NVX microVM failure logs"]["if"], "failure()") kernel = "${{ inputs.debug-kernel && 'debug' || 'default' }}" part = "${{ inputs.guests-and-sandboxes && '-guests' || '' }}" @@ -12907,14 +12583,12 @@ def test_every_microvm_job_uploads_its_scenario_durations(self): for job_name, job in jobs.items(): # Each shard of a default suite is a leg of its job's matrix. for inputs in _matrix_legs(job): - guests = inputs.get("guests-and-sandboxes", False) is True - if inputs.get("runtime-adapter", False) is True and not guests: - continue legs += 1 system = ( "Windows" if "windows" in json.loads(inputs["runner"]) else "Linux" ) debug = inputs.get("debug-kernel", False) is True + guests = inputs.get("guests-and-sandboxes", False) is True index = inputs.get("shard", 0) for attempt in (1, 2): artifact = ( @@ -20910,449 +20584,6 @@ def test_check_lists_every_executable_and_fails_on_dynamic_ones(self): self.assertIn("not static executables", error.getvalue()) -class RuntimeAdapterSuiteTests(unittest.TestCase): - def _scenarios(self, text: str) -> Path: - """Writes `text` as the scenarios of an adapter, and returns their path.""" - temporary = tempfile.TemporaryDirectory() - self.addCleanup(temporary.cleanup) - path = Path(temporary.name) / "scenarios.txt" - path.write_text(text, encoding="utf-8") - return path - - def _run( - self, - path: Path, - listings: Mapping[tuple[str, str], str], - *, - targets: Mapping[str, str] | None = None, - check_only: bool = False, - artifacts: str | None = None, - ) -> tuple[int, list[tuple[list[str], dict[str, str]]], str, str]: - """Runs the handler on the scenarios at `path`, for an adapter whose Cargo - metadata holds the test targets `targets`, by name with their sources, and - whose targets list the tests of `listings`, by target and by the first of - libtest's arguments, and pass each run of named tests, with `artifacts` in - `NVX_ARTIFACTS_DIR`, or without it. Returns the handler's status, the runs, and - what it printed to its standard output and error.""" - if targets is None: - targets = {name: f"tests/{name}.rs" for name in ("vm_a", "unit", "other")} - package = { - "name": "runtime-adapter", - "targets": [ - {"name": "runtime_adapter", "kind": ["lib"], "src_path": "src/lib.rs"}, - {"name": "runtime-adapter-host", "kind": ["bin"], "src_path": "h.rs"}, - *( - {"name": name, "kind": ["test"], "src_path": source} - for name, source in targets.items() - ), - ], - } - metadata = json.dumps({"packages": [package]}) - runs: list[tuple[list[str], dict[str, str]]] = [] - - def fake_listing( - command: list[str], environment: dict[str, str] - ) -> subprocess.CompletedProcess[str]: - if command[1] == "metadata": - return subprocess.CompletedProcess(command, 0, metadata, "") - target = ( - command[command.index("--test") + 1] if "--test" in command else "lib" - ) - first = command[command.index("--") + 1] - return subprocess.CompletedProcess( - command, 0, listings[(target, first)], "" - ) - - def fake_streamed( - command: list[str], environment: dict[str, str] - ) -> tuple[str, int]: - runs.append((command, environment)) - names = command[command.index("--exact") + 1 :] - output = "".join(f"test {name} ... ok\n" for name in names) - return f"{output}\ntest result: ok. {len(names)} passed; 0 failed;\n", 0 - - guest = path.parent / "runtime-adapter-guest" - guest.write_bytes(b"") - args = argparse.Namespace( - backend="kvm", cargo="cargo", check_only=check_only, guest_agent=guest - ) - scenarios = runtime_adapter_tests.read_scenarios(path) - with ( - patch.object( - runtime_adapter_tests, "read_scenarios", return_value=scenarios - ), - patch.object(runtime_adapter_tests, "ADAPTER_DIRECTORY", path.parent), - patch.object(runtime_adapter_tests, "validate_openvmm_test_backend"), - patch.object(runtime_adapter_tests, "listing", fake_listing), - patch.object(runtime_adapter_tests, "streamed", fake_streamed), - patch.dict(os.environ), - contextlib.redirect_stdout(io.StringIO()) as stdout, - contextlib.redirect_stderr(io.StringIO()) as stderr, - ): - os.environ.pop("NVX_ARTIFACTS_DIR", None) - if artifacts is not None: - os.environ["NVX_ARTIFACTS_DIR"] = artifacts - status = runtime_adapter_tests.command_test_runtime_adapter(args) - return status, runs, stdout.getvalue(), stderr.getvalue() - - def test_cli_registers_the_suite(self): - args = nvx.parse_args(["test-runtime-adapter", "--backend", "kvm"]) - - self.assertEqual(args.backend, "kvm") - self.assertEqual(args.cargo, "cargo") - self.assertFalse(args.check_only) - self.assertEqual(args.guest_agent, runtime_adapter_tests.GUEST_AGENT) - self.assertIs(args.handler, runtime_adapter_tests.command_test_runtime_adapter) - - def test_ci_runs_the_suite_by_name_on_each_backend(self): - workflow = ( - BuildConstants.REPO_ROOT - / ".github" - / "workflows" - / "run-nvx-microvm-tests.yml" - ).read_text(encoding="utf-8") - - self.assertEqual(workflow.count("nvx.py test-runtime-adapter"), 2) - self.assertIn("python3 scripts/nvx.py test-runtime-adapter", workflow) - self.assertIn("python scripts\\nvx.py test-runtime-adapter", workflow) - self.assertNotIn("cargo test --locked --features host -- --ignored", workflow) - - def test_scenarios_name_targets_and_tests_in_the_order_of_their_lines(self): - path = self._scenarios("# The suite.\n\nvm_a b # last\nunit x\nvm_a a\n") - - self.assertEqual( - runtime_adapter_tests.read_scenarios(path), - [ - runtime_adapter_tests.Target("vm_a", ("b", "a")), - runtime_adapter_tests.Target("unit", ("x",)), - ], - ) - - def test_scenarios_refuse_other_lines(self): - for text, problem in ( - ("vm_a\n", r":1: expected a target and a test, not 'vm_a'"), - ("unit x\nvm_a a linux extra\n", r":2: expected a target and a test"), - ("unit x\nvm_a a b\n", r":2: unknown scenario platform 'b'"), - ("vm_a a Linux\n", r":1: unknown scenario platform 'Linux'"), - ("vm_a a\nvm_a a\n", r":2: vm_a a is named twice"), - ("vm_a a linux\nvm_a a windows\n", r":2: vm_a a is named twice"), - ("vm_a a linux\nvm_a a\n", r":2: vm_a a is named twice"), - ("vm_a a windows\nvm_a a windows\n", r":2: vm_a a is named twice"), - ("# Nothing yet.\n", r"names no scenario"), - ): - for platform in ("linux", "win32"): - with ( - self.subTest(text=text, platform=platform), - patch.object(sys, "platform", platform), - self.assertRaisesRegex(common.ScriptError, problem), - ): - runtime_adapter_tests.read_scenarios(self._scenarios(text)) - - def test_scenarios_select_the_host_platform_in_line_order(self): - path = self._scenarios( - "vm_a a windows\nunit x\nvm_a b linux # Linux only\nvm_a common\n" - ) - for platform, expected in ( - ( - "linux", - [ - runtime_adapter_tests.Target("unit", ("x",)), - runtime_adapter_tests.Target("vm_a", ("b", "common")), - ], - ), - ( - "win32", - [ - runtime_adapter_tests.Target("vm_a", ("a", "common")), - runtime_adapter_tests.Target("unit", ("x",)), - ], - ), - ): - with ( - self.subTest(platform=platform), - patch.object(sys, "platform", platform), - ): - self.assertEqual(runtime_adapter_tests.read_scenarios(path), expected) - - with ( - patch.object(sys, "platform", "win32"), - self.assertRaisesRegex(common.ScriptError, "names no scenario"), - ): - runtime_adapter_tests.read_scenarios(self._scenarios("vm_a a linux\n")) - - def test_the_adapter_s_readme_lists_its_scenarios(self): - readme = (runtime_adapter_tests.ADAPTER_DIRECTORY / "README.md").read_text( - encoding="utf-8" - ) - section = re.split(r"\n#+ ", readme.split("\n### Stage scenarios\n", 1)[1])[0] - rows = re.findall( - r"^\| `(\w+)` \| `([^`]+)`(?: \((linux|windows)\))? \|", - section, - re.MULTILINE, - ) - for platform, host in (("linux", "linux"), ("win32", "windows")): - with ( - self.subTest(platform=platform), - patch.object(sys, "platform", platform), - ): - targets = runtime_adapter_tests.read_scenarios() - named = [ - (target.name, test) for target in targets for test in target.tests - ] - self.assertLessEqual( - {"vm_clone", "vm_template", "vm_layered", "template_operations"}, - {target.name for target in targets}, - ) - self.assertEqual( - [ - (target, test) - for target, test, on in rows - if not on or on == host - ], - named, - ) - - def test_libtest_s_terse_list_names_the_tests(self): - self.assertEqual( - runtime_adapter_tests.listed_tests("a: test\nb::c: test\nd: bench\n"), - ["a", "b::c"], - ) - - def test_naming_problems_name_missing_and_unnamed_tests(self): - target = runtime_adapter_tests.Target("vm_a", ("a", "gone")) - - self.assertEqual( - runtime_adapter_tests.naming_problems(target, ["gone", "a"]), [] - ) - self.assertEqual( - runtime_adapter_tests.naming_problems(target, ["a", "new"]), - ["vm_a has no test gone", "vm_a has the test new, which no scenario names"], - ) - - def test_a_run_passes_once_each_scenario_ran_and_passed(self): - target = runtime_adapter_tests.Target("vm_a", ("a", "b")) - output = ( - "running 2 tests\n" - "test b ... ok\n" - "test a has been running for over 60 seconds\n" - "test a ... ok\n\n" - "test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; " - "3 filtered out; finished in 9.00s\n" - ) - - self.assertEqual(runtime_adapter_tests.run_problems(target, output, 0), []) - - def test_a_run_fails_once_a_scenario_failed_or_did_not_run(self): - target = runtime_adapter_tests.Target("vm_a", ("a", "b")) - result = "test result: {}. {} passed; {} failed; 0 ignored; 0 measured; " - for output, status, problems in ( - ( - f"test a ... ok\ntest b ... FAILED\n\n{result.format('FAILED', 1, 1)}", - 101, - ["vm_a b: FAILED", "vm_a passed 1 tests, not 2"], - ), - ( - f"test a ... ok\n\n{result.format('ok', 1, 0)}4 filtered out\n", - 0, - ["vm_a b did not run", "vm_a passed 1 tests, not 2"], - ), - ( - f"running 0 tests\n\n{result.format('ok', 0, 0)}5 filtered out\n", - 0, - [ - "vm_a a did not run", - "vm_a b did not run", - "vm_a passed 0 tests, not 2", - ], - ), - ( - f"test a ... ok\ntest b ... ok\n\n{result.format('ok', 2, 0)}\n", - 1, - ["the tests of vm_a exited with status 1"], - ), - ("error: could not compile `runtime-adapter`\n", 101, None), - ): - with self.subTest(output=output): - found = runtime_adapter_tests.run_problems(target, output, status) - if problems is None: - self.assertIn("vm_a passed 0 tests, not 2", found) - else: - self.assertEqual(found, problems) - - def test_the_handler_checks_the_names_then_runs_each_target_by_name(self): - path = self._scenarios("vm_a a\nvm_a b\nunit x\n") - listings = { - ("vm_a", "--list"): "a: test\nb: test\n", - ("unit", "--list"): "x: test\n", - ("lib", "--ignored"): "", - ("other", "--ignored"): "", - } - - status, runs, stdout, stderr = self._run(path, listings) - - self.assertEqual(status, 0, stderr) - self.assertEqual( - [command for command, _ in runs], - [ - runtime_adapter_tests.cargo_test( - "cargo", "vm_a", "--include-ignored", "--exact", "a", "b" - ), - runtime_adapter_tests.cargo_test( - "cargo", "unit", "--include-ignored", "--exact", "x" - ), - ], - ) - environment = runs[0][1] - self.assertEqual(environment["NVX_HYPERVISOR"], "kvm") - self.assertEqual( - environment["NVX_ARTIFACTS_DIR"], os.fspath(BuildConstants.REPO_ROOT) - ) - self.assertEqual( - Path(environment["RUNTIME_ADAPTER_GUEST"]), - (path.parent / "runtime-adapter-guest").resolve(), - ) - self.assertIn("the 3 scenarios of 2 targets passed on kvm", stdout) - - # A check runs nothing, and needs neither a hypervisor nor the guest agent. - status, runs, stdout, _ = self._run(path, listings, check_only=True) - self.assertEqual((status, runs), (0, [])) - self.assertIn("3 scenarios name exactly the tests of 2 targets", stdout) - - def test_the_handler_takes_a_relative_directory_of_artifacts_from_the_caller(self): - path = self._scenarios("unit x\n") - listings = { - ("unit", "--list"): "x: test\n", - ("lib", "--ignored"): "", - ("vm_a", "--ignored"): "", - ("other", "--ignored"): "", - } - for artifacts, expected in ( - # Cargo runs the tests in the adapter's directory, not the caller's. - ("relative", Path("relative").resolve()), - (os.fspath(path.parent), path.parent.resolve()), - # NVX takes an empty directory for none, and the command this checkout. - ("", BuildConstants.REPO_ROOT), - ): - with self.subTest(artifacts=artifacts): - status, runs, _, stderr = self._run(path, listings, artifacts=artifacts) - self.assertEqual(status, 0, stderr) - self.assertEqual(Path(runs[0][1]["NVX_ARTIFACTS_DIR"]), expected) - - def test_the_handler_checks_and_runs_the_same_platform_scenarios(self): - path = self._scenarios( - "vm_a common\nvm_a linux_case linux\nunit windows_case windows\n" - ) - for platform, selected in ( - ("linux", [("vm_a", ("common", "linux_case"))]), - ("win32", [("vm_a", ("common",)), ("unit", ("windows_case",))]), - ): - listings = { - (target, "--list"): "".join(f"{test}: test\n" for test in tests) - for target, tests in selected - } - listings.update( - ((target, "--ignored"), "") for target in ("lib", "unit", "other") - ) - for check_only in (False, True): - with ( - self.subTest(platform=platform, check_only=check_only), - patch.object(sys, "platform", platform), - ): - status, runs, stdout, stderr = self._run( - path, listings, check_only=check_only - ) - self.assertEqual(status, 0, stderr) - expected = [ - runtime_adapter_tests.cargo_test( - "cargo", target, "--include-ignored", "--exact", *tests - ) - for target, tests in selected - ] - self.assertEqual( - [command for command, _ in runs], [] if check_only else expected - ) - if check_only: - self.assertIn( - f"2 scenarios name exactly the tests of {len(selected)} targets", - stdout, - ) - else: - self.assertIn( - f"the 2 scenarios of {len(selected)} targets passed", stdout - ) - - def test_platform_selection_keeps_unlisted_and_ignored_test_errors(self): - path = self._scenarios( - "vm_a common\nvm_a linux_case linux\nother hidden linux\n" - ) - listings = { - ("vm_a", "--list"): "common: test\nlinux_case: test\n", - ("lib", "--ignored"): "", - ("unit", "--ignored"): "", - ("other", "--ignored"): "hidden: test\n", - } - with patch.object(sys, "platform", "win32"): - status, runs, _, stderr = self._run(path, listings) - - self.assertEqual((status, runs), (1, [])) - self.assertIn("vm_a has the test linux_case, which no scenario names", stderr) - self.assertIn( - "other has the ignored test hidden, which no scenario names", stderr - ) - - def test_the_handler_runs_nothing_once_the_names_are_not_the_tests(self): - path = self._scenarios("vm_a a\nvm_a gone\ngone x\n") - listings = { - ("vm_a", "--list"): "a: test\nnew: test\n", - ("lib", "--ignored"): "unit::vm: test\n", - ("other", "--ignored"): "vm: test\n", - ("vm_dir", "--ignored"): "vm: test\n", - } - targets = { - "vm_a": "tests/vm_a.rs", - "other": "tests/other.rs", - # Cargo also builds a directory of the tests directory with a main.rs. - "vm_dir": "tests/vm_dir/main.rs", - } - - status, runs, _, stderr = self._run(path, listings, targets=targets) - - self.assertEqual((status, runs), (1, [])) - for problem in ( - "vm_a has no test gone", - "vm_a has the test new, which no scenario names", - "gone is not a test target of the adapter", - "the library has the ignored test unit::vm, which no scenario names", - "other has the ignored test vm, which no scenario names", - "vm_dir has the ignored test vm, which no scenario names", - ): - self.assertIn(problem, stderr) - - def test_the_handler_refuses_an_adapter_that_cargo_does_not_describe(self): - path = self._scenarios("vm_a a\n") - for status, output, problem in ( - (101, "", "cannot read the adapter's targets"), - (0, json.dumps({"packages": []}), "has no package runtime-adapter"), - ): - - def fake_listing( - command: list[str], - environment: dict[str, str], - status: int = status, - output: str = output, - ) -> subprocess.CompletedProcess[str]: - return subprocess.CompletedProcess(command, status, output, "error") - - with ( - self.subTest(problem=problem), - patch.object(runtime_adapter_tests, "listing", fake_listing), - self.assertRaisesRegex(common.ScriptError, problem), - ): - runtime_adapter_tests.suite_problems( - runtime_adapter_tests.read_scenarios(path), "cargo", {} - ) - - class _TwoStreamInteraction: """An OpenVMM that prints the boot marker, waits for the guest's script, then writes each of ``steps``, a chunk or the end of its stream, to its diff --git a/scripts/test_test_changes.py b/scripts/test_test_changes.py index 007dc6660..d7fcd2ec0 100644 --- a/scripts/test_test_changes.py +++ b/scripts/test_test_changes.py @@ -1105,7 +1105,7 @@ def current(seams: Mapping[str, int]) -> str: class ClassifyTests(unittest.TestCase): def test_paths_and_their_categories(self) -> None: - crates = ("nvx-rs", "tests/runtime-adapter") + crates = ("nvx-rs",) for path, expected in ( ("scripts/test-layout-allowlist.json", (changes.TEST_POLICY, "allowlist")), ("scripts/check_test_layout.py", (changes.TEST_POLICY, "file")), @@ -1122,12 +1122,10 @@ def test_paths_and_their_categories(self) -> None: ("scripts/nvx_tools/microvm_tests.py", (changes.END_TO_END, "python")), ("scripts/test_nvx_tools.py", (changes.TEST_CODE, "python")), ("scripts/powershell_testing.py", (changes.TEST_CODE, "python")), - ("tests/runtime-adapter/scenarios.txt", (changes.TEST_DATA, "file")), ("nvx-rs/tests/fixtures/store/runtime.json", (changes.TEST_DATA, "file")), ("nvx-rs/tests/vectors/openvmm/a.rs", (changes.TEST_DATA, "file")), ("nvx-rs/tests/cli.rs", (changes.TEST_CODE, "rust")), ("nvx-rs/src/a/tests.rs", (changes.TEST_CODE, "rust")), - ("tests/runtime-adapter/tests/runtime.rs", (changes.TEST_CODE, "rust")), ("nvx-rs/src/a.rs", (changes.TEST_CODE, "rust-production")), ("nvx-rs/tests/vectors/README.md", None), ("nvx-rs/README.md", None), diff --git a/tests/runtime-adapter/Cargo.lock b/tests/runtime-adapter/Cargo.lock deleted file mode 100644 index 57f1e1771..000000000 --- a/tests/runtime-adapter/Cargo.lock +++ /dev/null @@ -1,275 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "bitflags" -version = "2.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" - -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - -[[package]] -name = "cfg-if" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" - -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "libc", - "r-efi", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "libc" -version = "0.2.189" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - -[[package]] -name = "nix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" -dependencies = [ - "bitflags", - "cfg-if", - "cfg_aliases", - "libc", - "memoffset", -] - -[[package]] -name = "nvx" -version = "0.1.0-alpha.3" -dependencies = [ - "getrandom", - "nix", - "serde", - "serde_json", - "sha2", - "windows-sys", -] - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "runtime-adapter" -version = "0.1.0" -dependencies = [ - "nix", - "nvx", - "serde_json", -] - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "syn" -version = "3.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - -[[package]] -name = "unicode-ident" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tests/runtime-adapter/Cargo.toml b/tests/runtime-adapter/Cargo.toml deleted file mode 100644 index 0aebeee74..000000000 --- a/tests/runtime-adapter/Cargo.toml +++ /dev/null @@ -1,68 +0,0 @@ -[package] -name = "runtime-adapter" -version = "0.1.0" -edition = "2024" -# The minimum supported Rust version of `nvx`, which a test keeps equal. -rust-version = "1.89" -description = "An independent adapter of the NVX runtime, built on the nvx crate through its public API only" -license = "MIT" -repository = "https://github.com/microsoft/nvx" -readme = "README.md" -publish = false - -[features] -# No default features: each configuration selects one side of `nvx`, as an independent adapter -# does. Without features, the adapter builds against the protocol-only `nvx`. -default = [] -# The host program: the host API of `nvx` and the `runtime-adapter-host` binary. -host = ["nvx/host", "dep:serde_json"] -# The guest agent: the guest API of `nvx` and the `runtime-adapter-guest` binary, which builds as a -# static `x86_64-unknown-linux-musl` executable. -guest = ["nvx/guest", "dep:nix"] - -[dependencies] -nvx = { path = "../../nvx-rs", default-features = false } -# The host program records each launch in the sandbox's store as JSON, with the persisted form of -# its plan. The host side of `nvx` depends on the same release. -serde_json = { version = "1.0.120", optional = true } - -# The guest agent's PID 1 collects the status of every child and ends the unrelated one, its -# worker polls its nonblocking session and executions, its probes read how much space a file -# system has, its mapping probes mount a procfs in the root, and -# the helper of its `entries` probe enters a mount namespace of its own and leaks a descriptor to -# its executions, which open raw sockets: system calls that `nvx` leaves to the caller. The guest -# side of `nvx` depends on the same release, with these features among its own. -[target.'cfg(target_os = "linux")'.dependencies] -nix = { version = "0.31.3", default-features = false, features = [ - "fs", - "mount", - "poll", - "process", - "sched", - "signal", - "socket", -], optional = true } - -# The guest agent's tests open pseudo-terminals, and trace a child. The guest side of `nvx` -# depends on the same release, with `term` among its own features. -[target.'cfg(target_os = "linux")'.dev-dependencies] -nix = { version = "0.31.3", default-features = false, features = ["ptrace", "term"] } - -# The binaries have no unit tests; tests/independent_build.rs runs them. -[[bin]] -name = "runtime-adapter-host" -path = "src/bin/runtime-adapter-host.rs" -required-features = ["host"] -test = false - -[[bin]] -name = "runtime-adapter-guest" -path = "src/bin/runtime-adapter-guest.rs" -required-features = ["guest"] -test = false - -[lints.rust] -missing_docs = "warn" -# The adapter reaches `nvx` through its public API alone. -unsafe_code = "forbid" -unreachable_pub = "warn" diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md deleted file mode 100644 index e5bc05e64..000000000 --- a/tests/runtime-adapter/README.md +++ /dev/null @@ -1,3311 +0,0 @@ -# Runtime adapter - -The runtime adapter is an independent adapter of the NVX runtime, with a host -program and a guest agent. It lives in this repository so that NVX's own CI -builds and runs it, and it depends on [`nvx`](../../nvx-rs/README.md) the way -an adapter outside the repository does: - -- as a path dependency with default features off; -- with a `Cargo.lock` of its own, outside any other workspace; -- through the public API alone, with unsafe code forbidden; -- with the minimum supported Rust version of `nvx`. - -The adapter brings its own application framing to the control session between -the host and the guest, in place of NVX's `NVXC` frames. Its host program sends -requests in that framing, and its guest agent serves them. The host program -also runs the guest agent in VMs of NVX's artifacts, in place of NVX's agents, -through NVX's supervisor, and finds a VM again after it restarts, as an -orchestrator does. - -A VM may have a layered root: an image under an upper, which the host program -stages and plans with NVX's typed plan for an adapter's own guest, whose staged -bindings a launch checks before it hands OpenVMM the checked files, and which -the guest agent's worker assembles through `nvx::guest::root`, in a process of -its own rather than PID 1, as the host program's root configuration asks. The -plan also holds the VM's [network](#networks), if any. The host program may -share host directories and files with the root, which it exports with NVX's -`filesystem::plan`, and which the worker maps beneath the root from the host's -mapping table, before the host starts the sandbox. The host program then checks -the root from inside the VM with the worker's probes, and runs executions in -it, several at once over one session, in a -[table of scenarios](#layered-scenarios), to which each new check adds a row. - -## Configurations - -Each configuration enables one side of `nvx`: - -| Configuration | Cargo options | `nvx` features | Modules | Binary | -| --- | --- | --- | --- | --- | -| Protocol-only | none | none | `frame`, `root`, `probe`, `exec`, `template` | none | -| Host-only | `--features host` | `host` | `frame`, `root`, `probe`, `exec`, `template`, `host`, `runtime`, `initramfs`, `staged`, `sharing`, `layered`, `capture`, `generations`, `hook` | `runtime-adapter-host` | -| Guest-only | `--features guest --target x86_64-unknown-linux-musl` | `guest` | `frame`, `root`, `probe`, `exec`, `template`, `guest`, `worker` | `runtime-adapter-guest`, a static executable | - -## Framing - -The `frame` module is the adapter's application framing, which its host program -and guest agent exchange over the control session in place of NVX's `NVXC` -frames. A message is a 5-byte header, then a body. The header holds the body's -length, a little-endian `u32`, then the message's kind: - -| Kind | Message | Body | -| --- | --- | --- | -| 1 | `Ping`: asks the guest to answer with `Pong` | none | -| 2 | `Pong`: the guest's answer to `Ping` | none | -| 3 | `Echo`: bytes that the guest sends back in an `Echo` of its own | at most 256 KiB, what four of the session's largest records carry | -| 4 | `Stop`: asks the guest to stop serving | none | -| 5 | `Stopped`: the guest's confirmation of `Stop`, its last message | none | -| 6 | `RootConfig`: the host's [root configuration](#root-configurations) of a layered root, which the guest answers with `Ready` or `RootFailed` | the configuration, at most 4 KiB | -| 7 | `Ready`: the guest assembled the root, or activated a clone | the guest agent's [build](#ready-mapping-tables-and-start), and a clone's activation, at most 4 KiB | -| 8 | `RootFailed`: a stage of the root failed: the guest's answer to `RootConfig`, or its report of a failed teardown before it confirms a `Stop` | the stage, then the problem, at most 4 KiB | -| 9 | `Probe`: asks the guest to run a [probe](#probes) on its root | the probe, at most 4 KiB | -| 10 | `Report`: the guest's answer to `Probe`: the facts that the probe found | at most 256 KiB | -| 11 | `ProbeFailed`: the guest's answer to a `Probe` that it does not know, or that failed | the problem, at most 4 KiB | -| 12 to 21 | The messages of an [execution](#executions) | at most 256 KiB for a start, input, or output, and 64 bytes for the others | -| 22 | `Maps`: a part of the host's [mapping table](#ready-mapping-tables-and-start), which the guest applies beneath the root, in order, and answers with `Mapped` or `MapFailed` | a `MAPS` payload of `nvx::protocol`, at most 256 KiB | -| 23 | `Mapped`: the guest applied the part's mappings | none | -| 24 | `MapFailed`: the guest refused the part, or failed to apply one of its mappings | how many mappings it applied, then the problem, at most 4 KiB | -| 25 | `Start`: asks the guest to start the sandbox's workloads once it applied the mapping table, which it confirms with `Started` | none | -| 26 | `Started`: the guest's confirmation of `Start` | none | -| 27 | `Activate`: the host's activation of a clone, which a restore of a [template's](#template-mode) snapshot started, and which the guest answers with `Ready` | a clone's activation record, as `nvx::activation::ActivationRecord::encode` writes it, at most 1 KiB | - -The session keeps no message boundaries, so a message may span several of its -records, and one record may end a message and start the next. `Decoder` -reassembles the messages. It checks each header before it takes any byte of -the body, and holds at most one partial message. A malformed message ends the -stream, because the framing cannot find the next message after it. A clone's -`Activate` whose header announces more than 1 KiB fails the clone's -activation instead, as a record that long does. - -The framing carries each body as bytes, within its kind's limit. The bodies of -kinds 6 to 22 and 24 have encodings of their own, which the `root`, `probe`, -and `exec` modules write and read, and that of kind 27 is `nvx::activation`'s. -A body that does not decode leaves the stream intact, at the start of the next -message. The guest answers a root configuration that does not decode with -`RootFailed`, as a configuration that it refuses, and a probe with -`ProbeFailed`, while a message about an execution that does not decode ends the -worker's session, as one that breaks the execution's rules does, and an -activation record that does not decode fails the clone's activation. The host -program's client ends on an answer that does not decode, since it cannot tell -what the guest did. - -### Root configurations - -The boot tokens of `nvx::protocol::root` carry what NVX's boot contract -defines: the image's format, the identity that the guest expects of it, the -upper, and the root's settings. The adapter's plan names none of the settings, -and the worker reads only a RAM upper's cap from them. A root configuration, -`root::RootConfig`, carries the rest that the root needs, which only the -adapter defines, once the host attaches. It is -UTF-8 text: one `NAME=VALUE` line per field, each ended by a newline, in any -order, and each at most once. Numbers are decimal, without a sign or leading -zeros: - -| Name | Value | Required | -| --- | --- | --- | -| `version` | 1 | yes | -| `runtime` | The runtime directory, below which the guest assembles the root | yes | -| `agent-cgroup` | The agent's cgroup, which holds the guest's own processes | yes | -| `workload-cgroup` | The workload cgroup, the agent's sibling | yes | -| `device-timeout-ms` | How long the guest waits for each block device, at most 600,000 | yes | -| `agent-memory-low` | The agent's `memory.low`, in bytes | no | -| `workload-memory-max` | The workload cgroup's `memory.max`, in bytes | no | -| `workload-pids-max` | The workload cgroup's `pids.max` | no | -| `distro-sectors` | The size of the image's block device, in 512-byte sectors | no | -| `scratch-sectors` | The size of the scratch block device, in 512-byte sectors | no | -| `first-partition-bound` | The most bytes that the first partition of a GPT disk may hold | no | -| `overlay-mode` | The overlay's mode: `trusted-lowers` or `untrusted-lowers` | no | -| `scratch-journal` | Whether an ext4 scratch may have a journal: `allowed` or `forbidden` | no | -| `mapping-count` | The number of mappings of the host's [mapping table](#ready-mapping-tables-and-start), at most 4,096 | no | -| `workload-uid` | The user ID of the sandbox's workload identity, as which every execution runs | with `workload-gid` | -| `workload-gid` | The group ID of the sandbox's workload identity | with `workload-uid` | -| `listeners` | Whether the guest's listeners are published, with which every execution enters the root: `published` or `unpublished` | no | - -The paths are absolute and canonical guest paths of at most 4,095 bytes, -without a NUL or a newline: not `/`, and without an empty, `.`, or `..` -component, or a trailing `/`. The runtime directory holds no `,`, `:`, `\`, or -`=`, which the overlay's options reserve, and the two cgroups are siblings, -whose parent is not `/`. The optional numbers are positive. A workload -identity has both IDs, each below 4,294,967,295, which the kernel reads as -"unchanged", and no supplementary groups or capabilities, as a layered -sandbox's identity does; the worker checks its account once it assembled the -root, and every execution then runs as it, naming no identity of its own. -Without an overlay mode or a journal policy, the guest takes the defaults of -`nvx::guest::root`: an overlay that trusts its lowers, and a scratch that may -have a journal. Without a bound of the first partition, the guest bounds it at -`root::DEFAULT_FIRST_PARTITION_BOUND`, 64 KiB. Without a mapping count, the -host's mapping table has no mapping. Without `listeners`, the guest's listeners -are published, as `nvx::guest::entry`'s default has them, which refuses the -most to the executions. The host program and the guest agent ship together, so -the guest refuses every other configuration, one with an unknown field -included, rather than ignore what it does not know. -`RootConfig::default()` holds the values that the host program sends unless its -caller names others: the runtime directory `/run/runtime-adapter`, the cgroups -`/sys/fs/cgroup/agent` and `/sys/fs/cgroup/workload`, a wait of 30 seconds, an -agent's `memory.low` of 16 MiB, as NVX's own guest reserves, and a workload's -`memory.max` of 128 MiB and `pids.max` of 128, with neither a bound of the -first partition, an overlay mode, a journal policy, a workload identity, nor a -publication of listeners, and no mapping. A layered launch names the -publication of its network; see [Networks](#networks). - -A stage of the root that fails is a `root::RootFailure`: the stage's code, one -byte, then the problem, as UTF-8 text. The guest's worker, and with it the VM, -then ends with the stage's status: - -| Code | Stage | What failed | Status | -| --- | --- | --- | --- | -| 1 | `configuration` | Receiving and checking the root configuration | 11 | -| 2 | `layout` | Reading the boot layout of the root and the host export from the kernel command line, and checking that the worker assembles them, with an export if the configuration announces mappings | 12 | -| 3 | `bootstrap` | Bootstrapping the guest for the root: its cgroups and their resource values | 13 | -| 4 | `assembly` | Assembling the root, such as from an image of another identity than the boot tokens expect, and mounting the host export that the kernel command line names | 14 | -| 5 | `teardown` | Tearing the root down, such as while a process holds it open | 15 | -| 6 | `identity` | Checking the account of the sandbox's workload identity in the assembled root, in existing-account mode, such as one that the image lacks | 16 | - -### Ready, mapping tables, and Start - -`Ready`, the guest's answer to a root configuration once it assembled the -root, carries its readiness, `root::Ready`: the length of the guest agent's -build, one byte, then the build, 1 to 255 printable ASCII characters without a -space. The guest agent's build is the crate's version, `GUEST_BUILD`, which is -also the build that the host program ships unless it names another; see -[The runtime identity](#the-runtime-identity). The host sends a guest agent of -another build nothing more; see [The host's order](#the-hosts-order). - -A clone's `Ready`, its answer to the host's `Activate`, also reports its -activation after the build, as `nvx::activation::ReadyActivation::encode` -writes it: 1, then the transaction ID of the record's restore, 16 bytes. -`root::Ready::with_activation` adds it, and `root::Ready::activation` returns -it. A guest that boots cold reports none, and `root::Ready::decode` refuses -bytes after the build that are not an activation. - -The host's mapping table, of as many mappings as the root configuration -announces, travels in parts, each in a `Maps`, whose body is a `MAPS` payload -of `nvx::protocol`, as its `encode_maps_payloads` writes it: the index in the -table of the part's first mapping, then the number of its mappings, each a -little-endian `u32`, then each mapping: its flags, a little-endian `u16` that -is 1 for a read-only mapping and 0 otherwise, then the lengths of its source -and its target, each a little-endian `u16`, then the source, a path relative -to the guest's mount of the host's export, or empty for the export's root -itself, and the target, an absolute path in the root. `root::encode_maps` -splits a table into parts of at most 256 KiB, and an empty table into none. -`root::decode_maps` reads a part with -`nvx::protocol::decode_guest_maps_payload`, which takes the paths as bytes, as -Linux does, and refuses what that decoder refuses, a part without a mapping, -and a part that would end the table past 4,096 mappings. - -`Mapped` answers a part whose mappings the guest applied, and `MapFailed` one -that it refused, or one of whose mappings it failed to apply. The body of -`MapFailed`, `root::MapFailure`, holds how many of the table's mappings the -guest applied, in order, a little-endian `u32`, which is the index in the -table of the mapping that failed, then the problem, as UTF-8 text. `Start` -asks the guest to start the sandbox's workloads once it applied the table, -and `Started` confirms it. `Mapped`, `Start`, and `Started` have no body. The -guest's worker takes the table and the start in the order that -[The mapping table and the start](#the-mapping-table-and-the-start) -describes. - -### Probes - -A probe, `probe::Probe`, is a named check that the host asks the guest's -worker to run on its layered root. It is UTF-8 text: the probe's name, which -starts with a lowercase ASCII letter, and holds lowercase ASCII letters, -digits, and `-`, then each of its arguments after one space, each without -whitespace or control characters. The worker answers with the facts that the -probe found, `probe::Facts`, also UTF-8 text: one `KEY=VALUE` line per fact, -each ended by a newline, in the order in which the probe found them, and each -key at most once. A key holds lowercase ASCII letters, digits, `.`, and `-`, -and a value escapes `\`, newlines, and carriage returns as `\\`, `\n`, and -`\r`. - -The worker runs the probes of `worker::probes` while it runs no -[execution](#the-workers-executions). FILE and DIRECTORY are relative paths in -the root, of normal components, and the other probes take no arguments: - -| Probe | Facts | -| --- | --- | -| `mounts` | For the mount of each layer, by the name of its mount point, `distro` for an EROFS image, and `pN` for the layer in partition N of a GPT disk, the scratch's, `scratch`, the tmpfs's, `tmpfs`, and the overlay's, `root`: the number of mounts at its mount point, `NAME.count`, and of the topmost, its `NAME.type`, `NAME.source`, `NAME.options`, and `NAME.super`, the file system's own options | -| `contents` | The root's `os-release.id` and `root.mode`. It writes `.runtime-adapter-probe` at the top of the root, and reports what the root, `write.root`, the upper, `write.upper`, and the top layer, `write.lower`, hold at that name, and whether the top layer refuses a new file, `lower.write` | -| `copy-up FILE` | It sets the mode of FILE, a regular file of the layers, to the mode that it has, which copies the file up, and reports its size in the topmost layer that holds it, `lower.size`, and in the upper, `upper.size`; whether the upper's copy holds the file's data, `upper.data`: `copied`, `none` once the overlay copied up the file's metadata alone, or `different`; and whether the root still shows the layer's data, `root.data`: `intact` or `different` | -| `rename DIRECTORY` | It renames DIRECTORY, a directory of the layers, to `.runtime-adapter-renamed` at the top of the root, and back if it can: `outcome` is `allowed`, `refused` once the overlay refused it with `EXDEV`, or `failed: PROBLEM` | -| `fill` | It writes a file in the root until the upper is full, then removes it, and writes a file there again: the upper's size, `total`, and the bytes available in it before, `available`; the bytes that it wrote, `written`; the error that ended the writes, `error`: `ENOSPC` once the upper was full, or the problem; what the second file holds, `after`; and the guest's memory, `memory`, in bytes | -| `devices` | The guest's disks, the block devices that a device backs: `disks`, each as `NAME:SECTORS:ACCESS`, its size in 512-byte sectors and its access mode, `ro` or `rw`, in the order of their names, separated by spaces | -| `reads` | The sectors that the guest read from each of its disks, by the disk's statistics: `disks`, each as `NAME:SECTORS`, in 512-byte sectors, in the order of their names, separated by spaces | -| `read-disks` | It reads each of the guest's disks from its start to its end, through the page cache: `disks`, each as `NAME:BYTES`, the bytes that it read, in the order of their names, separated by spaces | -| `memory` | The guest's memory: the size of its memory blocks, `block-size`, how many are online, `blocks`, and their bytes, `online`, and the memory that the kernel manages, `total`, each in bytes | -| `processors` | The guest's CPUs, as the kernel lists them: those that are online, `online`, present, `present`, and possible, `possible`; and the CPUs that the restore of a clone brought online, as the time ABI's record of the restore names them, `restored`, `none` for none, or `absent` without a record | -| `time` | The guest's time state, `/run/nvx/time/state`, which NVX's time daemon keeps: each line `KEY=VALUE` as a fact, with each `_` of its key as `-`, in order, leaving out a key that no fact may have, and a repeated one | -| `first-partition` | The bytes of the first partition of a GPT disk that the assembly returned, without their trailing padding: how many, `length`, and the bytes, `bytes`, in hexadecimal, or `none` for both for an EROFS image | -| `cgroups` | The file system at the cgroups' parent, `parent.type`, and its `parent.subtree-control`; whether each cgroup exists, `agent.exists` and `workload.exists`; `agent.memory.low`, `workload.memory.max`, and `workload.pids.max`; the number of processes in each, `agent.processes` and `workload.processes`; and `user-namespaces`, the guest's limit of user namespaces, or `unavailable` without them | -| `processes` | For PID 1, `pid1`, the worker, `worker`, and PID 1's unrelated child, `sibling`: `NAME.pid`, and while the process exists, `NAME.state`, `NAME.ppid`, `NAME.root`, `NAME.cgroup`, `NAME.command`, and `NAME.namespaces`: `same` if it shares PID 1's, or the ones that differ | -| `hold` | It opens the root, and keeps it open until the worker ends, so that the teardown finds the root busy: `held`, the path that it holds | - -A value that a probe cannot read is `absent` if its file does not exist, and -otherwise `unreadable: PROBLEM`. A new check of a root is a new probe in this -table. - -The worker of a clone also answers the probe `activation`, which takes no -arguments, itself, at any time once the clone is ready, executions running or -not, with when the events of the clone's activation and order happened, in -microseconds after the repair's acknowledgement of the restore: whether the -repair acknowledged the restore, `acknowledged`, `yes`, or `none` for a restore -whose host did not gate its input until the acknowledgement; how long the worker -waited before the repair, `repair-delay`, in microseconds, as its template's -[repair delay](#template-mode) asked, for tests, or `none` without one; and -when the clone took the reset that the restore queued, `reset`, when the host's -first message came, `message`, which must be the clone's `Activate`, when the -clone was ready, `ready`, when its sandbox started, `started`, and when the -start of the host's first execution came, `execution`. Each event is `none` -until it happened, `before` if it happened before the acknowledgement, and -`unknown` without one. The worker of a VM that boots cold runs no such probe. - -The probes of `worker::mapping` inspect the mappings beneath the root, and map -from the host export, as an adapter maps the host paths of its sandboxes. `map` -inspects the mappings that the worker applied from the host's mapping table, -through the root itself, with `nvx::guest::root::Root::map`, which checks their -targets against the container root's reserved set, and the export that they -come from. `map-checks` maps from the export through mappers of the adapter's -own, `nvx::guest::mappings::Mapper`, with reserved sets of its choice, and -tears down what it mapped. The links that it plants in the root lead to -`/tmp/runtime-adapter-outside`, an empty directory outside it. - -| Probe | Facts | -| --- | --- | -| `map [NAME=PATH...]` | The export's mount at `hostfs/root` in the runtime directory: the number of mounts there, `export.count`, and of the topmost, its `export.type` and `export.options`; the mode of `hostfs`, in octal, `hostfs.mode`, and its owner, `hostfs.owner`, as `UID:GID`, or `absent` for both; and the mount points below the root, in the mount table's order, `mounts`. For each NAME, at PATH in the root, such as a mapping's target: the options of the topmost mount there, `NAME.options`, or `none`; what it shows, `NAME.kind`; a directory's entries, by name, in order, separated by commas, `NAME.entries`, or a file's contents, `NAME.read`; whether a write is `allowed` or `refused`, `NAME.write`: of a new file `written`, which holds `runtime-adapter`, in a directory, or of the file, opened for writing; and what the overlay's upper holds at PATH, `NAME.upper`, an empty directory or file where the worker created the target. Finally whether the root is latched, `latched` | -| `map-checks DIR FILE` | From the export's directory DIR, which holds a file `marker` and a directory `inner`, and in which the guest may write, and its file FILE, each check through a new mapper: a link at each component of a target, `link.a`, `link.b`, and `link.new`, and to `..`, `link.up`; a magic link of a procfs that it mounts in the root for the checks, `magic.self`, `magic.thread-self`, `magic.root`, `magic.cwd`, and `magic.fd`; a directory onto a file, `kind.directory`, and a file onto a directory, `kind.file`; a mapping onto a file that an earlier one shows, `nested.onto`, one that would create its target inside it, `nested.create`, and their teardown, `nested.teardown`; the runtime's reserved targets, `reserved.run`, `reserved.run-nvx`, `reserved.time`, and `reserved.time-below`, the container root's, `reserved.proc`, `reserved.sys`, `reserved.dev`, `reserved.agent`, `reserved.machine-id`, and `reserved.etc`, and those of a set of the adapter's, with the tree `/srv/adapter` and the path `/srv/checks/exact`, `reserved.adapter-tree`, `reserved.adapter-ancestor`, `reserved.adapter-path`, and `reserved.adapter-below`, and whether a refused one created anything, `reserved.created`; a read-only mapping of DIR attached and unmounted 200 times while another thread tries to open its `marker` for writing, `read-only.attaches`, `read-only.flags`, and `read-only.writable`; and the entries and mounts of the directory outside the root, `outside.entries` and `outside.mounts`, and the mounts that the checks left, `left` | - -A NAME holds lowercase ASCII letters, digits, and `-`, but is neither `export` -nor `hostfs`, and comes once. PATH is a relative path in the root, and DIR and -FILE are relative paths in the export, such as `2`, its third child, each of -normal components. A kind is `directory`, `file SIZE`, with the file's size in -bytes, `link`, `other`, `denied` for a path that the guest may not look up, -such as one that the export hides, or `absent`. A check's outcome is -`applied`, or why the mapper refused the mapping: `link` for a link at the -component that the check expects, and otherwise the kind of the error, such as -`reserved` or `target-kind`, with the path that it names, if any. The checks -write nothing to the export. - -### Executions - -An execution is a program that the guest's worker runs for the host, whose -standard streams and outcome NVX's execution engine, `nvx::exec`, carries. The -host's side of an execution is the engine's `Peer`, and the worker's its -`Runner`, which the worker binds to the program with -`nvx::guest::exec::binding`. Each message of the engine travels in a frame of -its own kind, which the `exec` module encodes and decodes: `exec::HostMessage` -from the host, and `exec::GuestMessage` from the guest. Each body starts with -the execution's key: its ID, then its generation, each a little-endian `u64`. - -| Kind | From | Message | Body after the key | -| --- | --- | --- | --- | -| 12 | host | `ExecStart`: starts the execution | the rest of its start | -| 13 | host | `ExecInput`: bytes of the program's standard input | the bytes, at most a chunk | -| 14 | host | `ExecInputEnd`: the end of the input, after its last byte | none | -| 15 | host | `ExecCancel`: ends the execution | the cause | -| 16 | both | `ExecCredit`: credit of an output pool, from the host, or of the input, from the guest | the pool, then the units, a `u32` | -| 17 | guest | `ExecStarted`: the program runs | none | -| 18 | guest | `ExecOutput`: bytes of the program's standard output or error | the stream, then the bytes, at most a chunk | -| 19 | guest | `ExecEnd`: the end of a stream, after its last byte | the stream | -| 20 | guest | `ExecInputClosed`: the program's input closed before the host ended it | none | -| 21 | guest | `ExecTerminal`: the execution's outcome, the guest's last message about it | the outcome | - -A start and the messages of kinds 13 and 18 hold at most 256 KiB, and the -others at most 64 bytes. A pool is 0 for the input, 1 for standard output, 2 -for standard error, and 3 for the output pool that both streams share; a stream -is 1 for standard output and 2 for standard error; and a cause is 1 for a -cancellation, 2 for a drop, 3 for a timeout, 4 for a reset, and 5 for a -disconnection, of which the host's cancellation names one of the first three. - -A start, `exec::Start`, holds the limits on which both sides agree: the chunk, -a `u32` of at most `exec::MAX_CHUNK` bytes, what the longest body holds after -the key and a stream; the input's window, then the output's, each a unit, 0 -for bytes or 1 for frames, then its size, a `u32`; and whether the streams -share one output pool, 0 or 1. It then holds the worker's timers, in -milliseconds, each a `u64`: the grace, the settlement time, and whether a -timeout follows, 0 or 1, then the timeout, or 0. Then come whether the host -sends the program's standard input, 0 or 1, and the program's arguments: their -count, a `u16`, then each argument's length, a `u32`, and its bytes. The first -argument is the program's path, which is not searched for in `PATH`, and no -argument holds a NUL byte. The start ends with how the program enters the -worker's root: - -- whether it names the program's identity, `exec::Identity`, 0 or 1, and if - it does, the user ID and the group ID, each a `u32`, the account mode, - `exec::AccountMode`, 0 for an existing account or 1 for numeric IDs, the - capabilities, a `u64` whose bit N stands for the capability numbered N, then - the supplementary groups: their count, a `u16`, then each group, a `u32`. - Exactly one source names an execution's identity: its start, or the - sandbox's workload identity, which the root configuration names; -- whether it names the program's working directory, 0 or 1, and if it does, - the directory's length, a `u32`, and its bytes: an absolute path in the - root, without a NUL byte. Otherwise the program starts in the root's `/`; -- the program's whole environment: the count of its entries, a `u16`, then - each entry's length, a `u32`, and its bytes, `NAME=VALUE`, with a name that - is not empty, and without a NUL byte. - -An outcome holds the program's status: 0 for none, 1 for an exit, or 2 for a -signal, then the exit's code or the signal's number, an `i32`, or 0; the cause, -or 0 for none; whether the launch failed, 0 or 1, then its stage, a `u32`, and -its `errno`, an `i32`, or 0 for each; and flags: 1 if standard output lost -bytes, 2 if standard error did, and 4 if the execution is reusable. Stage 0 is -the worker's refusal of the start, and a later stage is the binding's number of -the stage of `nvx::guest::spawn` that failed, such as 21 for its `exec`. - -`exec::HostExecution` is the host's side of one execution, in every -configuration: the engine's `Peer`, with a sink that takes the program's output -as it arrives, unless its caller stalls it, so that the guest gets no credit -for more, and a source that hands over the program's input, if the start says -that the host sends it, then ends it. Its caller sends the start, hands it each -of the guest's messages about the execution with `deliver`, and sends the -messages that `serve` returns, until the peer finished, serving it again once -its `deadline`, the peer's next timer, passes. The sink keeps at most -`exec::MAX_CAPTURE`, 16 MiB, of each stream: once more of a stream arrives, the -host's side cancels the execution, and its peer discards the rest once the grace -ends, so that the outcome marks the stream truncated, with the cause -`cancelled`. The host program's [driver](#host-program) and the tests of the -worker run it. - -### Root-profile entries - -The probe of `worker::entries`, which takes no arguments either, enters the -root as an adapter's root profiles do, through `nvx::guest::entry` and -`nvx::guest::spawn`: each execution in new mount, PID, and UTS namespaces, -with the root moved over `/`, a view of its own, and an identity with only the -capabilities that the entry allows. The worker copies its own executable to -`/.runtime-adapter-guest` at the top of the root, and runs the executions in a -helper, `runtime-adapter-guest entries ROOT CGROUP`, which enters a mount -namespace of its own first, as NVX's managed agent does, so that the runtime's -binds of a generated machine ID and of the time status end with it, before the -worker tears the root down. The helper places each execution in the workload -cgroup, leaks it a descriptor of the guest's own `/` without close-on-exec, -and has it run `runtime-adapter-guest inside DIRECTORY DESCRIPTOR` from the -root, which reports what the execution sees and may do, in its case's -directory below `/tmp/runtime-adapter-entries`. The helper blocks `SIGCHLD`, -as a caller that collects its children through a signal descriptor does, -which no execution inherits. - -| Probe | Facts | -| --- | --- | -| `entries` | The capabilities outside the allowlist that the entry refused, `refused.outside`, and any that it accepted, `refused.accepted`; its refusal of `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW` while listeners are published, `refused.published.net-bind-service` and `refused.published.net-raw`, of a capability for a user ID other than 0, `refused.non-root`, and of an identity without an account in existing-account mode, `refused.existing`. For each case, `CASE.launch` and `CASE.status`, and what the execution reported, as `CASE.KEY`: the descriptors open in it, `fds`, and whether the leaked one is, `leaked`; `uid`, `gid`, and `groups`; the capability sets, `cap.inh`, `cap.prm`, `cap.eff`, `cap.bnd`, and `cap.amb`, and `no-new-privs`; the signals that it blocks, `signals.blocked`; `pid`; its working directory, `cwd`; `hostname`; the `machine-id` and a mount's options at `/etc/machine-id`, `mount.machine-id`; the first line of the time status's state, `time.state`, the names in `/run/nvx/time`, `time.files`, and a mount's options there, `mount.time`; the unprivileged port start, `threshold`; the outcome of lowering it, `threshold.write`, of writing the hostname through `/proc/sys`, `hostname.write`, of opening `/proc/sysrq-trigger` for writing, `sysrq.write`, of binding UDP port 1023, a privileged source port, from which the gateway accepts listener announcements, `bind`, of opening a raw socket, which could forge such a datagram, `raw`, of changing a new file's owner, `chown`, of reading a new file of mode 0, `dac-override`, of changing the mode of a file of user ID 1000, `fowner`, and of running a program as user ID 4321, `setuid`, and as group ID 4321, `setgid`; the root's entries, `root`; and, last, a nested `chroot` escape, `escape`: `contained` if climbing with `..` from outside a `chroot` ends in the execution's own root. For the working-directory descriptors, only their distinct outcomes, sorted and separated by commas, `cwd-descriptors.launch` and `cwd-descriptors.status`. The generated machine ID, `generated.source`. Once the helper ended, the root's own machine ID, `after.machine-id`, and the mounts below the root, `after.mounts`, in the worker's mount namespace | - -The cases, as user ID 0 in numeric-account mode unless they say otherwise, are -each allowed capability alone, without published listeners, `only-NAME`, such -as `only-net-bind-service`; no capability, `none`; every allowed capability -but those that published listeners refuse, with published listeners, -`published`, which stands for both publication modes, private-network ingress -and the host's loopback, since the entry treats them alike; every allowed -capability, without published listeners, `unpublished`; user and group ID -4242, of which the image has no account, `numeric`; with every allowed -capability, without published listeners, the working-directory cases, each of -which names a working directory: the case's own, `cwd-directory`, and, through -a magic link, which the entry refuses before the program runs, the leaked -descriptor's, `cwd-leaked`, the execution's own working directory and root, -`cwd-self` and `cwd-root`, that of the first process of its namespace, -`cwd-init`, a symbolic link to its working directory, `cwd-link`, and the link -of each descriptor from 0 to 63, `cwd-descriptors`; and, once the helper has -bound the runtime's files, a generated machine ID, the time status that the -guest's time daemon publishes at `/run/nvx/time`, and the hostname -`runtime-adapter`, `generated`. The helper first writes a machine ID -in the root, as its own, which the cases before `generated` see. An attempt's -outcome is `allowed`, `refused ERRNO`, such as `refused EPERM`, or `failed -PROBLEM`. - -## Guest agent - -`guest::serve` serves the host program on a stream to the guest's end of the -control session, through `nvx::guest::nvxs`. It attaches, then: - -- acknowledges each reset of the session with the smallest receive window that - the session allows, once it dropped the earlier epoch's partial message, - because a new epoch starts with a new message; -- credits each payload back once it holds the payload, so that a message - longer than the window still arrives whole; -- answers each `Ping` with `Pong`, sends each `Echo` back, answers each - `Probe` with `ProbeFailed`, since the VM has no layered root to probe, and - answers `Stop` with `Stopped`, then returns. - -A failure of the session, a malformed message, a message about a layered root, -such as its configuration, or about an execution, which the guest agent of a -VM without a layered root does not accept, and a message that only the guest -sends end serving with an error. - -`runtime-adapter-guest serve` runs the guest agent on its standard input and -output, which connect to the guest's end of the session, such as a control -console in raw mode. It exits with 0 once the host stopped it or the broker -closed the stream, and with 1, after a diagnostic on standard error, once -serving failed. Without arguments, it reports the contract that it was built -against. - -`guest::serve_terminal` serves a tty, such as a VM's control tty, as `serve` -does. It opens the tty with `nvx::guest::terminal`, as NVX's managed agent -does: it waits up to 30 seconds while the kernel sets the tty up, and puts it -in raw mode, so that the tty passes the session's bytes as they are. Once -serving ends, it waits until the tty has sent what serving wrote, such as the -confirmation of a stop. - -`runtime-adapter-guest init TTY` runs `serve_terminal` on the tty TTY in place -of NVX's managed agent, as a VM's PID 1; see [VMs](#vms). It ends with 0 once -the host stopped it, and with 1, after a diagnostic on standard error, once -serving failed or the broker closed the tty. As PID 1, which must not exit, it -ends the VM with that status instead, through NVX's `/sbin/nvx-exit`, as the -managed agent does. - -### PID 1 and the worker of a layered root - -`runtime-adapter-guest init TTY layered` runs PID 1 of a VM with a layered -root, in place of NVX's init agent; see [VMs](#vms). PID 1, -`worker::supervise`, stays minimal: - -1. It starts an unrelated child, `runtime-adapter-guest idle`, which sleeps - until a signal ends it, and stands for a daemon that a guest starts before - its agent. -2. It starts the worker, `runtime-adapter-guest worker TTY SIBLING`, with the - process ID of the unrelated child, which the worker's probes check, since - the guest's kernel does not list a process's children. -3. It collects the status of every child, adopted processes included, until - the worker ends, and then ends the VM with the worker's status, through - `/sbin/nvx-exit`. A wait also reports to PID 1, whatever its options, each - stop of a child's thread that asked PID 1 to trace it, and PID 1 goes on - past it. A real-time signal, which `nix` decodes in no report, may end the - worker or a process that PID 1 adopted, such as one that an execution left, - or stop a thread that PID 1 traces, so PID 1 waits until a child has - something to report without taking it, then takes the worker's report - through `nvx::guest::process::Child`, which keeps every signal, and - another's by the ID that the wait gives if the report decodes. Otherwise, - PID 1 finds the threads of its children that `/proc` lists as zombies, or - as stopped while traced, whatever the names of their programs. No wait for - any child takes the worker's status. - -The worker, `worker::work`, is a process of its own, never PID 1. Through -`worker::work_on_terminal`, it opens the tty with `nvx::guest::terminal`, as -`serve_terminal` does, then opens the tty again for itself, nonblocking, and -then: - -1. attaches the session; -2. receives the host's root configuration, and checks it; -3. reads the boot layout of the root from the kernel command line, with - `nvx::protocol::root::RootLayout::parse`, and the host export, if the - command line names one, with `nvx::protocol::export::HostExport::parse`, - then bootstraps the guest's cgroups with `nvx::guest::root::bootstrap`, - which moves the guest's processes into the agent's cgroup, assembles the - root with `nvx::guest::root::assemble`, and mounts the export out of the - root with `Root::mount_export`. If the configuration names a workload - identity, it then checks the identity's account in the assembled root, in - existing-account mode, with `nvx::guest::entry`; -4. answers `Ready`, with the guest agent's build, or `RootFailed` with the - stage that failed, and then ends; -5. applies the host's [mapping table](#the-mapping-table-and-the-start) - beneath the root, from the export, in the parts that the host sends, and - answers each part with `Mapped` or `MapFailed`; -6. on `Start`, once it applied every mapping that the configuration - announced, starts the sandbox, which it confirms with `Started`; -7. serves `Ping`, `Echo`, and `Probe`, though it refuses a probe while it runs - executions, and, once the sandbox started, runs the host's - [executions](#the-workers-executions); -8. on `Stop`, drops its executions, which ends their programs, tears the root - down, reports a teardown that failed with `RootFailed`, confirms the stop - with `Stopped`, and ends. - -Before the root configuration, the worker accepts only the configuration, or a -`Stop`, which ends it without a root. It assembles an EROFS image, or the -layers of a GPT disk, whose first partition holds at most the bound of the -configuration, over an ext4 scratch or a tmpfs in RAM, in the overlay mode of -the configuration. At the layout stage, it refuses a configuration that -configures a scratch for an upper in RAM, bounds a first partition for an EROFS -image, or announces mappings while the kernel command line names no host export, -from which they come, and a command line whose tokens of the export break their -rules, as `HostExport::parse` checks them. An export that fails to mount fails -the assembly. A workload identity whose account the assembled root lacks fails -the identity stage. The worker ends with 0 once the host stopped it, after a -teardown that succeeded, with the stage's status, from 11 to 16, once a stage -failed, and with 1 once its session failed or the broker closed the tty, each -but the first after a diagnostic on standard error. Once the broker closed the -stream at a record boundary, which may close only the broker's end, the worker -drops its executions, and writes what it queued, as once the host stopped it. -Before it exits, it closes what its probes hold and the host export's root, then -unmounts a root that it still holds, as `nvx::guest::root::Root` does when it -drops, ignoring failures. A teardown that failed leaves the root mounted until -the VM ends. The worker runs the root's operations through -`worker::RootOperations`, and enters its executions through -`worker::EntryOperations`, so that tests run it against a fake root. In a -template's VM, the worker captures the VM instead, and then activates and serves -each clone that a restore of the snapshot starts, as -[Template mode](#template-mode) describes. - -#### The mapping table and the start - -The root configuration announces how many mappings the host's mapping table -holds, and the host sends the table once the root is ready, in parts, each in -a `Maps`, in order. The worker keeps its account of the table in -`nvx::guest::mappings::MappingTable`, which checks each part and hands its -mappings to the root, which applies them beneath it, in order, from the host -export, with `nvx::guest::root::Root::map`, and answers the part with `Mapped`, -or with `MapFailed` and how many of the table's mappings it applied: - -- It refuses a part as a whole, at the mapping that it expects next, if the - part does not decode, holds no mapping, or does not continue the table: if - its first mapping is not the one that the worker expects next, or its last is - past the count that the configuration announced. It also refuses a part with - a mapping of the export's root, which an empty source names: the export is - an aggregate, whose root lists its children. -- A mapping that the root refuses or fails to apply latches the root, which - keeps the mappings before it, so that the count is that mapping's index in - the table. -- Once it answered `MapFailed`, the worker refuses every later part, with - `MapFailed`, and the start: it starts no workload on a partial table, and - the host tears the VM down. - -A table of zero mappings does no work: the host sends no part, and the worker -maps nothing, though it mounts the export if the command line names one. - -`Start` starts the sandbox once the worker applied every mapping that the -configuration announced, and none failed, which the worker confirms with -`Started`. Only then does it run the host's executions, and it takes no part -of the table any more. A message about an execution before the start, a -`Start` before every announced mapping is applied, once the table failed, or -once the sandbox started, and a `Maps` once the sandbox started end the -worker's session with an error, as a message that it does not take in its -state does. The worker serves pings, echoes, probes, and the stop before the -start and after it. Its order outlives a reset of the session, as its root -does. - -#### The worker's executions - -Once the host started the sandbox, the worker runs several executions at once, -each through NVX's execution engine and its guest binding, with the limits and -timers of its start, such as frame-counted credit and one output pool that -standard output and error share, which the adapter's tests use. One loop that -never blocks serves the session and every execution: it takes the host's -messages as they arrive, steps every execution, queues their messages for the -host, and then polls the session's stream and every execution's descriptors -until one is ready or an execution's deadline passes. So a program's blocked -stream, an exhausted window, a stalled sink of the host, or a full stream to -the host holds back only the bytes that wait on it, never a timer, a -cancellation, a reset, a stop, or another execution. Nor does a probe, which -the worker runs only while it runs no execution, and otherwise answers with -`ProbeFailed` and `worker::EXECUTING`. While more than a MiB of its records -waits for the stream, the executions take no more output from their programs, -and go on otherwise, and the worker still takes the host's messages, until -more than 4 MiB wait, which only the replies to the host's messages and the -executions' messages other than output can queue: a host that sends messages -without reading the replies holds the worker's memory to that. The executions -take turns at the room on the stream: in a turn of the loop, each reads at -most an eighth of that MiB of output, and the loop serves another execution -first in each turn, so that a program that always has output, with all the -credit that it needs, never keeps the others' output from the host. - -- The host's messages reach an execution by its ID, and the execution's runner - sorts them by generation: one of an earlier generation is stale, and one of - an ID that the worker no longer runs, as after the execution's terminal - record, is late. Neither changes anything. The worker also remembers the - latest generation that the host started of up to `worker::MAX_HISTORY`, - 1024, IDs, those of the starts that it refused included, and forgets the ID - that it heard of first beyond that, so that a start of an earlier generation - is stale even after the execution's terminal record. A message that the - engine refuses, such as credit beyond the window, a body that does not - decode, or a second start of the same execution ends the session with an - error. -- A start of a later generation abandons the execution of the same ID, as a - reset of the session abandons every execution: the worker drops it, and it - runs until it ends, detached from the host, which receives nothing more - about it. -- The worker admits at most `worker::MAX_EXECUTIONS`, 8, executions at once, - abandoned ones included, and those that keep their places once they ended, - as the items below describe, and none that would hold more than - `worker::MAX_EXECUTION_MEMORY`, 1 MiB: its runner's buffers, a chunk of each - output stream and the input's window. It refuses a start beyond those, one - whose identity or entry it refuses, as the next item describes, or one whose - cgroup it cannot prepare, or whose program it cannot spawn, with a terminal - record whose launch failed at stage 0, with `EAGAIN`, `ENOMEM`, the `errno` - of the refusal, or that of the preparation or the spawn. -- Each program enters the layered root through `nvx::guest::entry`, as a - `worker::Entrance` describes it: in new mount, PID, and UTS namespaces, with - the root moved over `/`, the sandbox's hostname, the entry's default, - `nvx-sandbox`, after a cold boot, and the record's in a clone, and no time - status, and with the publication of listeners that the root configuration - names, published unless it names them unpublished. After a cold boot, it sees - the image's own machine ID and `/etc/resolv.conf`, so that the worker binds - nothing into the root, and in a clone, the machine ID that the record names, - which the clone's application bound for every execution if the record asks - for a generated one, and, if the record's network names a DNS server, the - resolver file that names it, which the application bound over the root's - `/etc/resolv.conf`. It runs as - the identity that exactly one source names: the sandbox's workload identity, - with no supplementary groups or capabilities, or, in a sandbox without one, - the identity that its start names, with its supplementary groups, its - capabilities, and its account mode. A start that names an identity in a - sandbox with a workload identity, or none in a sandbox without one, is - refused with `EINVAL`. Before the worker prepares the execution's cgroup, the - entry checks the identity, and reads its account in existing-account mode: it - refuses with `EPERM` a capability outside its allowlist, any capability of a - user ID other than 0, and `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW` while - listeners are published, with `ENOENT` an identity without the user ID's - passwd entry or the group ID's group entry, with the error of an account - database or a root that it cannot read, and with `EINVAL` otherwise. The - program starts in the working directory of its start, or in the root's `/`, - with exactly the environment of its start, none of the worker's own, and with - its standard input from the host or the null device. It leads a session of - its own, without a controlling terminal, and the process group that the - session creates, and, as PID 1 of its PID namespace, its end ends every - process of the namespace. The guest's network device, its addresses, and the - announcer of its listeners stay with NVX's `/init`, which configures them - before PID 1 starts, but for the announcer of a clone, which this temporary - adapter starts after the clone's process-free application and the worker - supervises; see [Networks](#networks) and - [Template mode](#template-mode). -- Each execution runs in a cgroup of its own below the workload cgroup of the - root configuration, `execution-N`, where N counts the cgroups that the worker - prepared. The worker prepares it with - `nvx::guest::cgroup::ExecutionCgroup::prepare`, and the program joins it as - it spawns, before it runs, so that its descendants start in it, however they - leave its process group or session. The execution settles through that - cgroup alone, which leaves the other executions running: once the program - ended, as its status, which the execution collects itself, tells, the - execution kills what is left in the cgroup, and settles once the cgroup is - empty. So no descendant outlives the program or holds its output open, and - an execution whose program ends on its own loses nothing, however slowly the - host takes its output, and is reusable. -- An execution keeps its place among `worker::MAX_EXECUTIONS` until its cgroup - is empty and the worker removed it, which the worker does once the runner - returned the terminal outcome of an execution that settled. One that did not - settle, whose outcome is fenced, keeps its place and its cgroup until the - host stops the worker, as does a start whose program the worker spawned, but - whose execution it could not have: until the worker collected the status of - such a program, the program may still join the cgroup, so that a removal - could hide it once it ended. -- Without cgroups, as where the in-process tests cannot create them, the - root's operations name none, and the program's process group stands in for - its cgroup: its execution kills the group as a whole. The worker collects the - status of every program itself then, without waiting for it, and learns that - a program ended from a pidfd of its own. Before it collects the status, it - kills what is left of the program's process group, as the settlement through - a cgroup kills what is left in the cgroup: until then, the program, a - zombie, keeps its process ID, which is the group's, from any other process. - So a descendant that stayed in the group neither outlives the program nor - holds its output open. Nothing proves that a descendant that left the group - is gone, though, so each outcome is fenced: once the program ended, a stream - that such a process may still hold has the settlement time and the grace to - close, after which the runner gives up on it, while a stream that no process - holds any more loses nothing, however slowly the host takes it. Such an - execution keeps its place until the worker collected its program's status. -- Before it hands an execution the host's cancellation, the worker gives the - execution a turn, as the loop does, which collects the status of a program - that ended, and reports the program's end to the runner. So the end of a - program that ended before its timeout while the worker did not run, as when - the guest's processor did not, stays the program's own, without the cause - of a timeout or of the cancellation, even if the cancellation comes after - the timeout's instant. -- The worker forgets an execution once its runner returned the terminal - outcome, which may come before the program's end, as once the runner gave up - on a program that a kill did not end in time: forgetting it kills the - program, and what is left in its cgroup or its process group, again, and the - worker collects the program's status once it ended. On a stop, or once its - session ended otherwise, the worker drops every execution, which kills the - program of each one that has not ended, with what is left in its cgroup or - its process group, and what is left in the cgroups of the executions that did - not settle, and waits up to a second in all for those programs to end and - those cgroups to empty, removing each cgroup that emptied once it collected - the statuses of those programs. - -The worker's `entries` probe runs two more commands of the guest agent: -`runtime-adapter-guest entries ROOT CGROUP`, its helper, which runs the probe's -executions in the root ROOT, each placed in the cgroup CGROUP, in a mount -namespace of its own, reports their facts on standard output, and exits with 0, -or with 1 after a diagnostic on standard error; and -`runtime-adapter-guest inside DIRECTORY DESCRIPTOR`, which each execution runs -from inside the root, and which reports what the execution sees and may do; -see [Probes](#probes). - -### Template mode - -A template's VM captures a snapshot of its guest before the guest assembles any -root, so that every clone that a restore of the snapshot starts attaches a root -of its own. The host opens no application session with it, and waits for no -`Ready`: a committed capture ends the VM. Its kernel command line has -`nvx_sandbox=1`, so NVX's `/init` runs the time ABI's boot step, then hands the -VM to the guest agent's PID 1 of a layered root, which starts the worker, as in -every VM with a layered root. OpenVMM adds `nvx_snapshot_tier=platform` to the -kernel command line of a VM that captures a platform snapshot. - -On x86_64, whose time ABI a capture needs, the worker reads the kernel command -line before anything else, with `template::boot`, which splits it as the kernel -does, with `nvx::protocol::cmdline::parameters`. A command line in which the -kernel finds no parameter `nvx_snapshot_tier=platform` boots cold: the worker -goes on as above, whatever else the command line holds. With it, the VM is a -template's, whose snapshot holds the platform's invariant configuration alone, -which every clone shares. So the worker accepts only the parameters of the -platform tier's allowlist, `template::is_platform_token`: OpenVMM's own check, -which the adapter compiles from the excerpts of OpenVMM's sources that NVX's -tests keep in `nvx-rs/tests/vectors/openvmm/`, so that the worker accepts -exactly what OpenVMM's capture of a platform snapshot admits, as text, byte for -byte: - -- `earlycon=xe9`, `console=hvc0`, `console=hvc1`, `reboot=t`, `panic=-1`, - `nvx_sandbox=1`, `nvx_config=0xd0010000,65536`, - `nvx_snapshot_tier=platform`, and `nvx_control_tty=hvc2`; -- `nr_cpus=`, `maxcpus=`, `virtio_mmio.device=`, `virtnet_ip=`, - `virtnet_mask=`, `virtnet_gw=`, `virtnet_dns=`, `virtnet_ip6=`, and - `virtnet_gw6=`, with any value that the host chooses. - -It refuses any other parameter, one that is not UTF-8 text among them, and a -double quote, which the kernel drops from a parameter, or `--`, after which the -kernel hands the rest of the command line to init, though init may still read -it from `/proc/cmdline`: OpenVMM refuses both in the command line of a VM with -a control console. OpenVMM checks the tokens that it finds at ASCII -whitespace, while the kernel also splits at the vertical tab and at 0xa0, so -the worker checks the parameters that the kernel finds. Otherwise, the worker, -through `worker::capture::capture_on_terminal`: - -1. opens the control tty, in raw mode, as the worker of a VM that boots cold - does; -2. bootstraps the guest for the root of every clone, with the template's own - root configuration, `worker::capture::template_config`, the default one, - since the host sends a template's VM none: it mounts the cgroup v2 - hierarchy unless it is mounted, and prepares the agent's and the workload - cgroups, with their resource values, as the worker of a VM that boots cold - does before it assembles its root. The snapshot holds them, so that a - clone assembles its root without them; -3. attaches the control session with `nvx::guest::nvxs::GuestConnection::attach`, - takes the reset with which the broker answers the attachment at once, - acknowledges it with the guest agent's receive window, and waits until the - tty has sent the acknowledgement, so that the snapshot holds the guest's - session active, in the broker's first epoch. A clone then reads no reset but - the one that the restore queues, of another instance: the restore's broker - would finish sending a reset that the template's broker began to send, and - the clone would read one that the template did not. The worker reads nothing - else from the session before the capture, and serves no application session; -4. prepares the guest for the capture with - `nvx::guest::clone::CapturePreparation`, which closes the worker's fence of - workload admission, checks that nothing is mounted from the `distro` or the - scratch block device, at the MMIO windows from which the worker assembles a - root, from an overlay, or from OpenVMM's host filesystem device, whose tag - is `microvm`, then waits for the time ABI's boot checks, runs its capture - checks, and suppresses the stall detectors; -5. requests the capture with `CaptureBarrier::capture`, with the platform - tier's request, which keeps no scratch. - -Closing the worker's fence, `worker::capture::Admission`, acquires it: a close -fails while the fence is closed, whoever closed it, and leaves it closed. The -worker of a template's VM admits no execution before the capture. The worker -returns from the capture only once the capture ended without a snapshot, or -failed, or in a clone, which a restore of the snapshot started. In a clone, the -worker repairs the restore next, with `nvx::guest::clone::repair`, behind its -fence, which stays closed: the repair brings the CPUs of the restore's -processor target online, activates the restore's memory target, reseeds the -kernel's random number generator with the restore's fresh entropy, and -finishes the restore, with the CPUs that the restore brought online and the -acknowledgement that the host awaits, which hands the deferred time work, the -restore checks, those of the new CPUs among them, and the release of the stall -detectors, to the time daemon. NVX's `/init` -started the daemon in the template, with the time ABI's boot step, before the -guest agent, so every clone holds it, and the guest agent's PID 1 collects its -status. The repair takes the time ABI in strict mode, so that a violation of -the ABI fails the repair, rather than stopping the VM, and the worker ends the -VM with the failure's status. A failure of the deferred time work stops the VM -with the time ABI's own status, since the daemon enforces the ABI. - -For tests, a template may delay the repair of each of its clones: once -`template::boot` took a template's command line, the worker reads -`template::REPAIR_DELAY_FILE`, `/etc/runtime-adapter-repair-delay`, which the -initramfs of the template's capture holds when the host program's creation -asks for a delay, and which `template::parse_repair_delay` takes as a positive -number of milliseconds, of at most `template::MAX_REPAIR_DELAY`, 30 seconds, -in decimal digits, with or without a newline. Each clone of the template then -waits that long before its repair, so that its restore stays unacknowledged, -and OpenVMM gates the host's input, while the host's attachment reaches -OpenVMM. The snapshot holds the delay, which the worker read before the -capture. The kernel command line cannot carry it: OpenVMM's capture of a -platform snapshot refuses a parameter that the tier's allowlist does not hold, -as `template::boot` does. A template without the file delays nothing, and a -file that the worker cannot read, or that holds anything else, ends the -template's VM with status 20, before the capture. - -Once it repaired a clone, the worker activates it, on the same tty, with -`worker::work_clone`, through the clone's `nvx::guest::clone::Activation`, -whose deadline ends `worker::activation::ACTIVATION_TIMEOUT`, 120 seconds, -after the repair's acknowledgement of the restore: - -1. It resumes the session that the template attached, and whose first epoch it - acknowledged, which the snapshot holds, and acknowledges the reset that the - restore queued, the only one that the clone reads, with its receive window: - it attaches no session. -2. It takes the host's `Activate`. The activation decodes its record, which - must be a clone's, with a restore section, and checks that it expects the - CPU count and the memory size that the repair found, before anything has a - side effect. -3. It applies the record with `nvx::guest::clone::apply`, with - `worker::capture::root_template` of the template's configuration and of its - NIC's IPv4 gateway, which `template::boot` read from the template's kernel - command line: its runtime directory, the `distro` and scratch block devices, - its device timeout, the tag of the host filesystem device, - `worker::capture::FILESYSTEM_TAG`, and the gateway, if the template has a - NIC. The application assembles the root, an - EROFS image or a GPT disk over a tmpfs with the record's RAM cap or the - scratch, in the record's overlay mode, waiting for each device until the activation's deadline at the - latest, so that it ends by then. The record carries no execution identity: - each execution names its own. For a GPT disk, it first asks - the kernel to reread the disk's partition table, so that the - block devices of its partitions are the clone's disk's, not the template's - placeholder's. The upper must be fresh: the assembly refuses one whose - `upper` or `work` directory exists and is not empty. If the record asks for - an export, the application mounts the host export through the - host filesystem device, read-write, at `hostfs/root` of the runtime - directory, outside the root, where only the guest's root user reaches it, as - the worker of a VM that boots cold mounts the one that its kernel command - line names. It applies the record's VM-wide view and DNS server through - `view::apply`: the hostname, generated machine ID when requested, and resolver - file, with every bind owned by the root. The library starts no process. - After successful application this temporary adapter uses the returned - publication and gateway to start its announcer, - `/sbin/nvx-ingress inherit GATEWAY_IPV4`, with its own launcher, through - `nvx::guest::spawn::Command`: it binds a UDP socket to a - privileged source port, from which alone OpenVMM's gateway takes - announcements, and runs the program with that socket as its standard input, - to the template's gateway, waiting until the program runs, but for no - announcement. The template, captured under a policy that publishes nothing, - runs no announcer of its own. -4. Once it took what the session received while the application ran, which came - before `Ready`, it completes the fence-free activation and - answers `Ready`, which reports the activation after the guest agent's build, - to the `Activate` and to each retry of it, byte for byte, that came - meanwhile, which joins the application. Anything else that came meanwhile, a - reset included, fails the activation, as before the application, so the host - waits for `Ready` before it sends the mapping table or the start. -5. It then serves the host's order, as after a cold boot: the mapping table that - the record announces, which it applies from the record's export, the start, - the executions, the probes, and the stop. A clone has no workload identity of - the whole sandbox: each execution names its own, and a start that names none - is refused, as in a sandbox whose root configuration names none. Every - execution takes the record's hostname. The host tells the workloads the - endpoint of the network's proxy, as after a cold boot: the host program's - `exec` adds it to each execution's environment, so the worker leaves each - execution's environment exactly its start's. A clone's executions enter the - root with the publication of listeners that the record's network names: - published under a policy that publishes them, which refuses - `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW`, and unpublished otherwise, a clone - without a network included. The worker polls the pidfd of the announcer that - this caller started after application with the session's stream: once the - announcer ended, before the clone was ready or after, the clone ends with status 57, as one - whose announcer could not start. On the stop, the worker stops the announcer, - closes the export's root, and tears the root down, which removes its own - view binds before unmounting it, through - `nvx::guest::clone::Applied::teardown`. A failed announcer stop is reported - as a teardown failure. It also answers the - [`activation` probe](#probes), with when the events of its activation and - order happened after the repair's acknowledgement of the restore. - -Until the clone is ready, its session takes nothing but that `Activate`: -another reset, any other message, a `Stop` and a `Ping` included, an -`Activate` that the activation refuses, and the passing of the deadline fail -the activation finally, as does an application that failed, which nothing -retries. Once the clone is ready, a retry of the same `Activate`, byte for -byte, gets the same `Ready` without a second application, and any other -`Activate` fails the activation. A failed activation ends the clone with the -status of its class, which `worker::activation::failure_status` selects, and a -broker that closes the stream before the clone is ready ends the clone with -status 1, once the worker dropped its executions, and waited for its stream, -and then the tty, to take what it wrote, for at most -`worker::UNACTIVATED_DRAIN`, a second each, whatever the stream does, so that a -stream that takes nothing, as once the broker failed, does not keep the clone -from ending. Activation owns no fence; the caller admits no workload before -`Ready`. Once the clone is ready, a broker that closes the stream ends it as it -ends the worker of a VM -that boots cold. The worker of a VM that boots cold refuses an `Activate` as -any message that it does not take in its state. The worker, and with it the VM, -ends with one of these statuses, after a diagnostic on standard error: - -| Status | When | -| --- | --- | -| 0 | In a clone, the host stopped the worker once the clone was ready, after a teardown that succeeded | -| 1 | The control tty cannot be opened or put in raw mode, the attachment or the acknowledgement of the broker's reset cannot be sent, or the broker closes the session or sends anything but a reset first, or, in a clone, its session failed or the broker closed it | -| 13 | The bootstrap of the guest for the template's root failed, before the attachment | -| 15 | In a clone, the root's teardown failed once the host stopped the worker | -| 20 | The kernel command line names the platform tier, but holds another parameter, a double quote, `--`, or a gateway that OpenVMM does not render, or the worker cannot read it, or the template's repair delay file holds no delay, or the worker cannot read it | -| 21 | The fence of workload admission cannot be closed | -| 22 | Something is mounted from the template's devices or from an overlay, or the mount table cannot be checked | -| 23 | The time ABI's boot checks did not complete | -| 24 | The time ABI's capture checks failed | -| 25 | The stall detectors cannot be suppressed | -| 26 | The VM generation ID cannot be read | -| 27 | The capture request failed, which leaves the fence closed, other than by a violation of the time ABI in a clone | -| 28 | The stall detectors' settings cannot be restored once the capture ended without a snapshot | -| 29 | A stage of the capture failed that this build of the worker does not know | -| 30 | The capture ended without a snapshot, as without a destination, or once OpenVMM rejected it, and the template continued | -| 40 | The clone's restore cannot be repaired: it holds no fresh entropy, or its capture finished it | -| 41 | The clone's online CPUs cannot be read, are not a prefix of the CPUs, exceed the restore's processor target, or the CPUs of the target cannot be brought online | -| 42 | The restore's memory target is refused, before any memory block is added | -| 43 | The activation of the restore's memory target failed | -| 44 | The reseed of the kernel's random number generator failed | -| 45 | The restore's finish failed, other than by a violation of the time ABI | -| 50 | The clone was not ready before its activation's deadline | -| 51 | The host broke the clone's session: it reset the session again, or sent another request, before the clone was ready, or an `Activate` of another transaction, or one that changes the accepted record | -| 52 | The `Activate`'s record does not decode, breaks a limit, or is a cold boot's | -| 53 | The record expects other resources than the clone has | -| 54 | The root's assembly failed, its upper not fresh among its causes, or the kernel did not reread the partition table of the record's GPT disk before it | -| 55 | The clone's workload cannot be activated: the machine ID that the record asks for cannot be generated, or the record's hostname or that machine ID cannot be set for every execution | -| 56 | The record's host export cannot be mounted | -| 57 | The record's network settings cannot be applied: the resolver file that names its DNS server cannot be written, or set for every execution, or the announcer of the listeners that its network publishes cannot start, as without the template's gateway; or that announcer ended | -| 58 | The worker's own order misused the activation: it completed an activation that did not run, or started the sandbox before the clone was ready, or again | -| 193 | A conformance check of the time ABI failed in the clone's repair, such as the finish's, which needs a running time daemon | -| 195 | The time ABI's part of the clone's repair failed, such as the validation of the restore packet, or the record of the restore | - -The statuses from 21 to 29 are `template::CAPTURE_STATUS_BASE` plus the number -of the stage that failed, in the order in which -`nvx::guest::clone::CaptureStage` declares the stages. A capture request that -fails with a violation of the time ABI, which it does in a clone whose restore -the time ABI could not repair, ends with the violation's status instead, as -`worker::capture::capture_status` tells. `nvx::protocol::status` documents -the statuses of a clone's repair, from 40 to 45, which -`nvx::guest::clone::RepairError::status` selects, the time ABI's own, from 193 -to 195, and those of a clone's activation, from 50 to 58. A stage whose -failure may leave the worker some of the permission for the time ABI's ports -fails with the time ABI's port error, and the worker then starts no thread or -process before it ends. A violation of the time ABI in enforcing mode, in the -capture's preparation or in the time daemon, stops the VM with the time ABI's -own status, from 193 to 195, instead. - -## Host program - -`host::Client` is the host program's client of the guest agent. It runs over -the host's end of the control session, through `nvx::host::nvxs`, on any -transport to the session's host endpoint, such as the one that -`nvx::host::session::connect` opens to a VM's control endpoint: - -- `Client::attach` attaches with the VM's launch capability, waits while the - broker waits for the guest, and reports the identity of the session's epoch - that the broker attached it to; -- `send` sends a message in as many of the session's records as it takes, and - `receive` returns the guest agent's next message, whether it spans records - or shares one with the next message; -- `ping`, `echo` and `stop` each send a request and check its answer: `Pong`, - an `Echo` of the same bytes, and `Stopped`. `stop` also returns the failure - of a layered root's teardown, which the guest reports before it confirms. - `stop_skipping` stops the guest agent as `stop` does, but skips the messages - before `Stopped` that its caller accepts, such as those that the guest may - still send about a driver's executions, which `Executions::is_about` - accepts; -- `configure_root` sends a root configuration, and returns how the guest - answered: `Ready`, with the guest agent's build, or the stage of the root - that failed; -- `activate` sends a clone's activation record in an `Activate`, and returns - the guest's `Ready`, with the guest agent's build and the clone's - activation. A clone whose activation failed answers nothing, but ends, and - its session with it; -- `map` sends a part of the host's mapping table, and returns how the guest - answered: `Mapped`, or `MapFailed`, with how many of the table's mappings it - applied and the problem; `start` sends `Start`, and checks that the guest - answers `Started`; -- `probe` sends a probe, and returns the facts that it found, or the problem. - -Besides the session's own buffers, the client holds at most the rest of one of -the guest's records and one partial message. A reset of the session ends the -attachment, and its partial message with it, because the next epoch starts -with a new message, in a new attachment. - -A failure of the session, a send that fails or times out, a malformed message, -an answer whose body does not decode, and a message that only the host program -sends, an execution's start, input, end of input, or cancellation, and a -clone's `Activate` included, end the client, and its later calls fail. A -receive that times out doesn't: it keeps what arrived. A request ends the -client on any failure after the framing accepted it, a timeout included, -because an answer that arrived later would answer the wrong request. - -`host::Executions` is the host program's driver of -[executions](#executions), over one attachment of the client: - -- `start` sends an execution's start, once it checks that the start decodes, - as the guest decodes it. A start that does not, such as one with a relative - working directory, and one of an ID that the driver started already, fail - before anything is sent, and leave the client usable: the guest would end its - session on either. So do a start that says that the host sends the program's - input, one whose output windows count bytes, or more than - `host::MAX_OUTPUT_FRAMES`, 32, frames, one beyond `exec::MAX_EXECUTIONS`, 8, - executions, as the guest runs no more at once, and any once the driver ran: - a driver runs one batch, whose executions it starts before it runs them; -- `run` carries the guest's messages to each execution's - `exec::HostExecution`, and theirs to the guest, until every execution - finished. One thread runs them all: each receive waits at most until the - earliest timer of the executions, which so fires in time whatever the guest - sends, and no execution's messages wait for another's. A message of the - guest about no execution, about one that the driver did not start, whose body - does not decode, or that breaks the engine's rules, ends the run, as a - deadline that passes first does: in a receive, which leaves the client - usable, or in a send. Each send waits, as the client's do, until the - transport takes the message or the run's deadline passes, and no timer fires - meanwhile, so a guest that stops reading the session holds the run until its - deadline, when the send's timeout ends the client. - -Even a guest that reads can hold a send while the host does not read: the -session's broker passes the guest's credit for the host's records on only -behind the guest's own records, and takes no more of the host's records than -that credit allows. So the driver sends little before it reads: no program's -input, which one thread could not send while it reads the programs' output; the -starts of at most 8 executions, whose answers wait for the run, as no program's -output flows before the run's credit does; and, in the run, besides a -cancellation for each execution, at most a credit for each message that it -reads, of output windows of at most 32 frames, so that the guest's messages -ahead of its own credit stay few. The guest may still send messages about the -executions of a run that gave up on one or failed, so a later batch takes an -attachment of its own, as each command of the host program does, and a stop in -the same attachment skips them, with `Client::stop_skipping` and -`Executions::is_about`, as `runtime-adapter-host run` stops its clone. - -The guest ends its session on a second start of the same key, even in a later -epoch of the session, so the keys of a driver's executions must be new to the -guest. - -### The host's order - -A VM with a layered root is ready for workloads once its guest assembled the -root, and applied the host's mapping table. The `host` module runs that order -over a client in two steps, between which the host program releases the files -that it handed OpenVMM, and its [step hook](#the-step-hook) may pause it: - -1. `host::ready` sends the root configuration, which announces how many - mappings the table holds, and waits for `Ready`, whose build must be the - one that the host ships. The host sends a guest agent of another build - nothing more, since it cannot tell what that build would do with it. -2. `host::start_sandbox` sends the table in the parts that `root::encode_maps` - splits it into, each once the guest applied the one before, then `Start`, - and waits for `Started`. A table of no mapping sends no part. - -A part that the guest fails ends the order: the host sends no more of the -table, nor `Start`, so the guest starts no workload on a partial table, and -sends `Stop`, which tears the root down, mappings first, as NVX's managed -lifecycle stops a guest whose mappings failed. The order fails with an -`host::OrderError`: the stage of the root that failed, another build, the -mapping that failed with how the stop ended, a table that does not encode, -which fails before anything is sent, or a failure of the client. The host -program [tears the VM down](#commands) after any of them. - -A clone, which a restore of a [template's](#template-mode) snapshot started, -takes the host's activation of its record instead of a root configuration: -the guest assembles the clone's root from the record, with the inputs that the -template's capture kept, as it activates the clone. Its first step is -`host::activate` instead of `host::ready`: it sends the clone's activation -record in an `Activate`, the first request of the session, and waits for -`Ready`, whose build it checks as `host::ready` does, and which reports the -clone's activation. The table and `Start` then follow as after a cold boot, in -two steps: `host::map_table` sends the table as `host::start_sandbox` does, -and ends the order in the same way at a part that the guest fails, and -`Client::start` sends `Start`. `host::CloneOrder` runs these steps as the -framing of the restore sequence of `nvx::host::template`, which checks the -activation against the record, sends the table only for a clone with an -export, and `Start` only once the guest applied every mapping, and which -replaces a session that the order takes for lost: one that a reset of the -broker, the end of its stream, or a failure of the transport ended while the -host attached or awaited `Ready`. A clone that was ready answers the same -record, byte for byte, with the same `Ready` in the new session, and one that -took the reset before it was ready has ended. OpenVMM's broker ends the stream -both to end an attachment and to refuse one, so the order takes such a refusal -for a lost session too, whose retries the sequence's limit of sessions bounds. -Any other failure is final: a refusal that the broker sends, a deadline that -passed, another build, and a message that breaks the framing. The guest's -`MapFailed` does not tell whether the request's mappings or a failure of the -guest's own failed the table, so the sequence attributes it, as any other -failure of the table, to what the host saw of it: a broken protocol, unless the -deadline passed or the session was lost. - -## VMs - -The host program runs the guest agent in VMs of NVX's artifacts, through NVX's -supervisor, `nvx::host::supervisor`, each in a sandbox store of -`nvx::host::store`. - -### The guest agent in place of NVX's agents - -NVX's initramfs hands a VM of the managed lifecycle to NVX's managed agent: its -`/init` runs -`exec /sbin/nvx-managed-agent /dev/hvc2 - nvx-managed UID GID NAME HOME` as -PID 1. A VM with a layered root, whose kernel command line has -`nvx_sandbox=1` and no old OpenVMM lifecycle token, goes to NVX's init instead: -`/init` runs `exec /sbin/nvx-init TIME_DAEMON [ANNOUNCER]`, with the helpers' -process IDs. Old-token layered launches keep `/sbin/nvx-init-agent`. -`initramfs::write` copies that initramfs, and appends to the copy a second -archive, in the `newc` format, which Linux unpacks over the first: - -- the guest agent, at `/sbin/runtime-adapter-guest`; -- a script, `/sbin/runtime-adapter-init`, that runs - `runtime-adapter-guest init "$1"`, with the control tty, the first of the - arguments that `/init` passes the managed agent; -- a symbolic link from `/sbin/nvx-managed-agent` to that script; -- a script, `/sbin/runtime-adapter-layered`, that runs - `runtime-adapter-guest init /dev/hvc2 layered`, which names the control tty - itself, and ignores the helper arguments; -- a symbolic link from `/sbin/nvx-init` to that script, so that NVX's - init agent never assembles the root in the adapter's VMs; -- for the capture of a template whose clones delay their repair, for tests, the - delay's file, `/etc/runtime-adapter-repair-delay`, in the `/etc` directory of - NVX's initramfs; see [Template mode](#template-mode). - -Linux removes a file whose type an archive changes before it creates the new -one, so each link replaces NVX's agent without writing to its file, which could -have other names. The second archive starts at a multiple of 4 bytes, as Linux -requires of an uncompressed archive, after zeros that Linux skips. NVX's own -images don't change. - -### The runtime identity - -The supervisor records the runtime identity of each launch, and its reattach -compares the recorded identity with the caller's. `runtime::Runtime::from_dir` -takes NVX's artifacts from a repository checkout or an extracted release, with -`nvx::host::artifacts::Artifacts::from_dir`, and their runtime identity from -`nvx::host::artifacts::ArtifactDir::runtime_identity`, which names each -artifact by the SHA-256 digest that the install metadata beside it records, -never from its contents: a release's `SHA256SUMS`, or the build provenance in a -checkout's `build/`, where `nvx-rs download` also installs a release's. - -The VMs boot a copy of NVX's initramfs with the guest agent in it, so the -identity of their initramfs names both: -`sha256:DIGEST+runtime-adapter-guest/BUILD`. The guest agent has no install -metadata of its own, so the caller names its build, BUILD, which is the crate's -version unless the caller names another. A host program that ships another -build of the guest agent runs another runtime, and finds a VM of this one -mismatched. - -### Staged images - -The images of a layered root are staged first, in the sandbox's store, as -`staged::Staged::stage` stages them. Staging verifies the image explicitly, -with `nvx::host::sandbox::distro::DistroBinding::verify`, which hashes the -image only when a manifest is beside it, and compares the manifest's name of -the image and digest with the image's own. It then checks the binding from -metadata, as a launch checks it, so that it stages no binding that every launch -would refuse. It binds the ext4 scratch image of an upper on scratch with -`nvx::host::sandbox::scratch::ScratchBinding::bind`, which neither reads nor -claims the image. Staging is where hashing belongs: a launch never hashes. -Staging also refuses images that no launch could attach, as the plan of a -launch checks them. - -Staging records both bindings in the sandbox's store, in -`runtime-adapter-staged.json`, in place of those staged earlier: - -```json -{"format": 1, - "distro": {"path": "/images/distro.erofs", "binding": DISTRO_BINDING}, - "scratch": {"path": "/sandboxes/one/scratch.ext4", "binding": SCRATCH_BINDING}} -``` - -Each `path` is the absolute path of a file, as a plan names it, and each -binding the serde form of NVX's `DistroBinding` or `ScratchBinding`. `scratch` -is `null` when no scratch image was staged. A reader refuses another format, -an object that lacks a member or has another, and a binding that does not -decode. A launch takes the bindings of its images from the record, by their -paths, and [checks them](#the-checked-handoff) before it spawns the VM. - -### Layered roots - -`layered::LayeredLaunch` plans a VM with a layered root with NVX's typed plan -of a cold layered launch for an adapter's own guest, -`nvx::host::sandbox::SandboxPlan::adapter`, and checks it with -`SandboxPlan::validated`. The adapter's plan holds none of the settings of -NVX's sandbox profile, such as a workload identity, since the guest agent runs -its executions itself. The plan names the images by absolute paths, since -OpenVMM runs in the sandbox's store, and the image's identity, which a launch -requires, since a binding names it. The VM's kernel command line is -`nvx_sandbox=1`, the launch's own token, with which NVX's `/init` hands the VM -to NVX's init agent, which the guest agent replaces, then the plan's boot -tokens of the root alone: the image's, such as -`nvx_layer=distro,0xd0003000,erofs:UUID`, and the upper's, such as -`nvx_scratch=0xd0006000,ext4`. OpenVMM's arguments, which NVX's -`PreparedLaunch::adapter_arguments` renders, are the plan's, which attach the -[checked files](#the-checked-handoff) of the image and the scratch as block -devices, and the VM's [network](#networks), if any, then those of the export of -the launch's [shares](#shares), if any, then those of the launch's -`nvx::host::launch::VmmLaunch`: the VM in one process, NVX's default memory, -the kernel, the initramfs, that command line, no virtio console, the control -console with the launch capability on standard input, and the outcome report. -They name no workload identity, and no managed lifecycle: OpenVMM only checks -the arguments of the managed lifecycle and labels its outcome report with it, -and only NVX's init agent reads its boot token. A VM without a layered root -keeps the managed lifecycle, with which NVX's `/init` starts the managed agent, -which the guest agent replaces. - -The command line that runs OpenVMM with them must fit the 32,767 UTF-16 code -units of a Windows process command line, on every host. As it plans, before it -touches the store, a launch checks with NVX's -`SandboxPlan::check_adapter_launch`, through `LayeredLaunch::check_command_line`, -that the plan's arguments, those of the export, and the kernel command line -leave `VmmLaunch::RESERVE` of them to the rest of the launch, so that it -refuses a network or shares that leave too little; and -`PreparedLaunch::adapter_arguments` refuses a complete command line that does -not fit, before it spawns the VMM. - -The boot tokens carry only what NVX's boot contract defines. The launch sends -the rest in the [root configuration](#root-configurations), once it attaches: -the defaults, with the runtime directory that the caller names, as it is, so -that the guest checks it, the size of each image in sectors, if it is a whole -number of them, which the guest checks against its block device, the overlay -mode and the scratch's journal policy that the caller names, if any, and the -number of the mappings of the export's table, if the launch shares anything. - -The launch records what it launched in the sandbox's store, beside NVX's -runtime records, in `runtime-adapter.json`, before it spawns the VMM: - -```json -{"format": 1, "mode": "layered", "kernelCommandLine": "nvx_sandbox=1 ...", - "plan": PLAN, "bindings": {"distro": DISTRO_BINDING, "scratch": SCRATCH_BINDING}, - "rootConfiguration": "version=1\nruntime=/run/runtime-adapter\n..."} -``` - -`mode` is `layered`, or `blockless` for a VM without a layered root, whose -`plan`, `bindings`, and `rootConfiguration` are `null`. `plan` is the persisted -form of the VM's `SandboxPlan`, whose `network` keeps the VM's -[network](#networks), or is `null` without one. `bindings` holds the bindings -that the launch checked, which the store [stages](#staged-images) for the -launch's images: the image's, and the scratch image's, `null` for an upper in -RAM. `rootConfiguration` is the root configuration that the launch sends, -whether or not the guest accepts it. - -### The checked handoff - -Before it spawns the VM, a launch checks the bindings of its images from their -metadata alone, with `LayeredLaunch::prepare`: NVX's -`DistroBinding::check` checks the image against its binding, under the distro -policy that this build accepts, and `ScratchBinding::check` checks the scratch -image of an upper on scratch against its own, and claims it for the sandbox. -Neither reads the images, and neither hashes. A launch fails before it begins, -leaving no launch journal, VMM log, or record in the store, if the store stages -no binding for the image or for the scratch image at its path, if a binding no -longer holds, as for an image that changed or was replaced after it was -staged, or if another sandbox's VM still claims the scratch image. - -The checks return the files that they opened, which NVX's -`nvx::host::sandbox::PreparedLaunch` hands OpenVMM in place of the images' -paths, so that nothing that takes a path after the checks reaches the VM. On -Linux, OpenVMM inherits the checked files' descriptors, which its arguments -name as `/proc/self/fd/N`, and nothing else of the host program's; on Windows, -its arguments name the checked files by their final paths, which the checked -files keep anything else from taking while they are open, and OpenVMM inherits -the checked files and the scratch image's claim, so that its copies hold the -paths even if the host program ends before OpenVMM has opened the images. The -launch holds the checked files until the guest has assembled the root, when -`PreparedLaunch::release` closes them, and on Windows OpenVMM's copies of them, -or until it has torn down a launch that failed. - -### Shares - -A launch may share host directories and regular files with the root, as -`sharing::HostFsSharing`: each `sharing::Share` at an absolute guest path in -the root, read-only or read-write. Neither exposes its fields: each field has a -getter and a setter, and each list of `HostFsSharing` a method that adds an -entry, as its option does. `HostFsSharing::plan` plans their export with NVX's -`nvx::host::filesystem::plan`, through an `ExportRequest` of a mapping for each -share at its target, the host paths that it hides inside the shares, the host -paths inside hidden ones that it shows again, with the directories on the way -to them listing only them, and the guest targets that the root keeps: those of -NVX's container root, `ReservedTargets::CONTAINER_ROOT`, and the trees that -the adapter keeps for itself. Each share must lie inside a host directory that -the adapter authorizes, once links are resolved: `HostFsSharing::plan` refuses -one outside them before it plans, and checks the export's directories and -files, which the planner resolves anew, again. The planner refuses what it -cannot export, such as a target that takes or covers a reserved tree, or a -read-only share inside a read-write one, which the host could not keep -read-only, a hidden path that does not exist inside a share, which nothing -would hide once a workload creates it, and a path to show again that lies -inside no hidden one. - -`LayeredLaunch::share` attaches the export to the launch, which -`LayeredLaunch::export` returns, and announces its table, of at most 4,096 -mappings, in the root configuration. NVX's adapter plan -holds no export, so the launch attaches it itself: OpenVMM's arguments export -the shares through OpenVMM's aggregate virtio-fs device, as -`ExportPlan::openvmm_arguments` renders them, and OpenVMM appends the boot -tokens with which the guest mounts the export, out of the root. Before it -spawns the VM, `LayeredLaunch::prepare` checks the export with -`ExportPlan::verify`: the host still keeps its read-only mappings read-only, -and its directories and files, and the paths that the guest could move inside -the parts that it may write, still name what they named when the export was -planned. Once the guest is ready, the launch sends the table, -`LayeredLaunch::table`, in the [host's order](#the-hosts-order): each mapping -before those inside its target. A [clone's](#clones) start takes the same -shares, whose export NVX attaches to the template's dormant filesystem slot as -an aggregate, as a launch attaches it, and whose table it sends as a launch -sends it. - -### Networks - -A layered launch may give the VM a network: an -`nvx::host::sandbox::SandboxNetwork`, the guest's address and the policy that -OpenVMM enforces there. `layered::read_network` reads it from a file, in the -serde form of a plan's network, as NVX's plan persists it, such as: - -```json -{"guestNetwork": "10.0.0.2/24", "egress": "deny", - "allow": {"name": "egress.allow", - "rules": [{"to": [{"cidr": "192.0.2.0/24", "except": []}], - "ports": [{"protocol": "tcp", "port": 443, "endPort": null}]}]}, - "deny": {"name": "egress.deny", "rules": []}, - "ingress": "deny", "hostLoopback": "deny", "hostLoopbackForwards": [], - "proxy": null} -``` - -The form requires every member and refuses any other. Reading it checks the -network as the plan does: the guest's address must leave a gateway, and the -policy must lower to OpenVMM's arguments. A launch reads its network before -anything else, so it refuses one that the plan refuses before it needs NVX's -artifacts or creates the store. - -The launch puts the network on the adapter's plan, with -`LayeredLaunch::with_network`, which renders it after the block devices, -through NVX's one lowering of a policy, `nvx::host::network::openvmm_arguments`: -the portable profile at the guest's address, the defaults of egress and -ingress, the host-loopback access, the egress rules, lowered into OpenVMM's -native ones, the host-loopback forwards, and the proxy of a proxy-only -network, which the guest reaches at its gateway, at the URL that -`SandboxPlan::proxy_url` gives. A policy that lets nothing through attaches no -network device. OpenVMM appends the guest's network settings to the kernel -command line, from which NVX's `/init` configures the network device and its -addresses, and starts the announcer of the guest's listeners if OpenVMM -publishes them, before it hands the VM to the guest agent, which leaves all -three to `/init`. The launch's [record](#layered-roots) keeps the network in -its plan, which `LaunchRecord::read_plan` reads back from the store. - -The guest's executions follow the network: - -- The [root configuration](#root-configurations) says whether the guest's - listeners are published, as OpenVMM publishes them: to the host's private - networks with ingress allowed, or on host loopback with host-loopback access - allowed, but neither with outbound host-loopback access alone nor through a - proxy, and not without a network. The executions enter the root with that - publication, so while listeners are published the entry refuses - `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW`, with which an execution could bind - the privileged port from which the announcer sends, or forge its datagrams, - at stage 0 with `EPERM`. -- `runtime-adapter-host exec` gives each of its executions the URL of a - proxy-only network's proxy in its environment, from the plan that the - launch's record keeps, with `layered::with_proxy`, since NVX gives an - execution no proxy settings, and its environment is exactly its start's: in - each of `layered::PROXY_VARIABLES`, `http_proxy`, `https_proxy`, - `HTTP_PROXY`, and `HTTPS_PROXY`, after the start's own entries. A start that - names any of them keeps its own proxy settings. - -In VMs, the [layered scenarios](#layered-scenarios) check what each kind of -network lets the guest's executions reach, and whether the host reaches the -guest's listeners, through publication or a forward. - -### The step hook - -For tests, `launch`, `reattach`, and `stop` take a step hook, `hook::StepHook`, -from the variable `RUNTIME_ADAPTER_STEP_HOOK`, which pauses the command at a -named step, so that a test can act between two steps, or exits the command -there without unwinding, as a crash would end it: - -- `STEP:pause:PATH` creates the file PATH, which holds the step's name, and - waits until the file is gone, then goes on: a launch leaves the time that it - waited out of its deadline, so that a test may pause it for as long as it - needs; -- `STEP:exit` exits at once with status 5: the command ends nothing that it - started, such as the VMM, and the system releases what it holds, such as its - descriptors or handles, and their locks. - -The steps of a launch, in order, then that of a command that reattaches, are: - -| Step | Where the command is | -| --- | --- | -| `checked` | It has checked the bindings of its images, and the export of its shares, if any, and holds the checked files, which claim the scratch image, but has begun nothing in the store, and spawned nothing. A launch without a layered root reaches it too. | -| `spawned` | The VMM runs, and the launch's journal records it, but the launch has not connected to the VM. | -| `ready` | The guest agent, of the build that the host ships, has assembled the layered root, and the launch has released the checked files, but has sent neither the mapping table nor `Start`. | -| `commit` | The guest has started the sandbox, if the VM has a layered root, the guest agent has answered, and the launch is about to commit: its journal still records the VMM, which no runtime record names yet. | -| `found` | `reattach` or `stop` has found the VM running, and holds it, under the sandbox's lock, with the session in which a clone answered the `Activate` of its record again, if the VM is a clone's, but has sent the guest agent nothing more, and stopped nothing. | - -A value that names no step or action is a usage error. Without the variable, -the hook does nothing. - -`start` takes the hook too, at `checked` alone, which it reaches once NVX's -`ClonePlan` checked the [clone](#clones) against the leased generation: the -bindings of its images, whose checked files the plan holds, and which claim its -scratch image, if it has one, its processor and memory targets, its network, -and the export of its shares, if any. The start has recorded nothing of the -clone yet, and spawned nothing, and the clone's launch checks the export again, -so a test may change a shared object there. A start refuses a hook at another -step as a usage error. - -### Templates - -The adapter's operations on templates, and on their clones, are commands of -`runtime-adapter-host`, which [Commands](#commands) details: - -| Operation | Command | What it does | -| --- | --- | --- | -| Create, refresh | `create`, `refresh` | Serves an eligible generation of the template's capture class that the store holds, or captures a new one, as `refresh` always does, and prints it once the store committed it, then evicts the generations that it supersedes; see [Eviction](#eviction) | -| List, inspect | `list`, `inspect` | Reports the generations, their keys, classes, states, eligibility and its reason, and leases, and the metadata of a capture, without loading VM state or hashing; see [Listings](#listings) | -| Remove, retire | `remove` | Tombstones a generation, so that it takes no new lease, reports busy, with the status 7, while leases remain, and deletes it once they drained, but never moves or deletes a leased generation | -| Start from a template | `start` | Starts a clone of an explicit, eligible generation, with its image, scratch, and settings, which the sandbox stages; reports a selection of no template, of an expired generation, or of an ineligible one, and a clone's failure, attributed to the request, the host, or the template, or whose teardown is unresolved; never captures, boots cold, rehashes, or falls back to another format; see [Clones](#clones) | -| Run from a template | `run` | Plans the clone as `start` does, with its network and shares, stages the clone's images in a preflight that it reports, starts a clone as `start` does, runs one execution in the clone's root, and tears the clone down; see [Running a clone](#running-a-clone) | - -`runtime-adapter-host create` serves a template of the guest agent from an NVX -template store, `nvx::host::template::Store`, with `Store::create_template`: a -generation of the template's capture class that the store holds, and that is -eligible at the host's time, or else a new one, which the store captures -through NVX's supervisor and publishes. `capture::Request` holds what the -command's options choose of the template, and `Request::capture` returns NVX's -`Capture` of it: - -- the adapter's plan of the placeholder, with the template's network, if any, - and the placeholder's binding, which the command plans and stages as `stage` - plans and stages an image. The placeholder is a trusted artifact of the - platform, whose length is the capacity of every clone's image, and the - capture refuses one that anyone else may change; -- the capture class, from which the store takes the slot that serializes the - class's captures, and the template's compatibility key: the runtime identity - of the adapter's VMs, the backend and the CPU profile, the - processors, those online at boot, 1 unless the command names more, the - fewest that the template's clones take, the RAM and its capacity, the - geometries of the placeholder and of the dormant scratch, as OpenVMM presents - a sandbox block device, which `capture::geometry` gives, the network, as - the capture attaches its NIC, which no network's policy changes, and, for a - template whose clones delay their repair, for tests, the delay, in - milliseconds, as its one invariant setting, - `runtime-adapter.repair-delay-ms`, so that the class is one of its own. The - class also records the guest agent's build, `runtime-adapter-guest/BUILD`, - which the compatibility key does not hold; -- NVX's artifacts, with the initramfs that puts the guest agent in place of - NVX's agents, and holds the file of the delay of each clone's repair, if the - template has one, which the command writes into a directory of its own in the - system's temporary directory, private to the current user, as a launch - writes it into the sandbox's private store, and removes once the creation - returns. - -For tests, `create` and `refresh` take that delay from the variable -`RUNTIME_ADAPTER_REPAIR_DELAY`, `capture::REPAIR_DELAY_VARIABLE`: a positive -number of milliseconds, of at most 30000, in decimal digits. Any other value is -a usage error, before the command creates anything, and without the variable, -the template delays nothing; see [Template mode](#template-mode). - -NVX's store launches the capture's VM from the capture's transaction, in the -class's slot, whose VMM listens there, beside the snapshot's destination, as -OpenVMM requires. On Linux, the path of that socket, -`captures/C/transaction/control.sock` in the store, must fit a Unix socket's, -so NVX refuses a store whose path is too long for it. - -The capture boots the template's VM with the kernel command line -`nvx_sandbox=1`, and for a template whose processors are not all online at -boot, `maxcpus=` with those that are, to which OpenVMM adds the platform tier's -tokens, `nvx_snapshot_tier=platform` among them, so the worker of the guest -agent captures the VM in [template mode](#template-mode) before it assembles -any root, and the capture ends the VM. OpenVMM records the processors online at -boot, from which each restore brings a clone's processor target online. The -store checks what the VM left, and publishes -the snapshot as its next generation, with the record of its capture and its -compatibility key, whose kernel command line holds the platform tier's tokens -alone: neither the placeholder's identity nor a root's tokens. The template has -a NIC exactly when its network lets something through: a network that denies -egress, ingress, and host-loopback access, without a proxy, attaches none, as -no network does. The capture attaches the NIC at the network's guest address, -under OpenVMM's complete deny-all policy, whatever the network's own: egress -denied without rules, ingress and host-loopback access denied, without a proxy -or forwards, the one policy of a snapshot that a restore may replace. So the -template's guest learns no DNS server, and starts no announcer of its -listeners, and the key holds what the capture made of the network alone: -whether there is a NIC, its addresses, and the identity of its deny-all -policy. The networks of every policy at one guest address therefore share one -class, and one template, which serves the [clones](#clones) of each of them. -The key's encoding, version 1, holds the NIC that way and has not been -released, so a template store that an earlier build created is not supported: -delete it and create a new one. There is no migration. The command refuses a -class that no capture takes, such as one of 3 processors, or -of more online at boot than in all, before it creates the store. A capture that -fails leaves the store's -generations as they were, and the command's diagnostic names the log of the VMM -of the class's latest capture, when the class's slot keeps one. - -`runtime-adapter-host refresh` takes the options of `create`, and captures a -new generation of the template's class with `Store::refresh_template`, whether -or not the store holds an eligible one. Only `create`, for a class of which the -store holds no eligible generation, and `refresh` capture templates: no other -command captures one, and no restore triggers a capture. Both print the -generation that they served only once the store has committed it. - -#### The template store - -The adapter keeps its template store where its caller says: in the directory -STORE that each template command names, as the sandbox commands name a -sandbox's store. It is a directory of its own, apart from every sandbox's -store, private to the current user, whose parent exists. `create` and -`refresh` create it if needed, while `list`, `inspect`, and `remove` only open -it, and fail if it holds no template store. On Linux, the control socket of a -capture's VMM, `captures/C/transaction/control.sock` in the store, must fit a -Unix socket's path, so NVX refuses to capture into a store whose absolute path -is longer than 56 bytes. - -#### Eviction - -The adapter keeps, of each capture class, the generation that it served last, -and none that it supersedes. Once `create` or `refresh` has served a -generation of a class, which the store held already or has just published, it -retires every other generation of the class that the store published before -it, or that a tombstone quarantines, with `generations::evict`, and prints -which: those that are gone, and those that leases keep. `remove` retires the -generation that it names. A retirement, NVX's `Store::retire`, tombstones the -generation first, so that it takes no new lease, and deletes it, with its -payload, once no lease of it remains: it never moves or deletes a generation -that a clone leases. A generation that leases keep stays tombstoned, and the -next retirement of it, by a `remove` that names it, or by the next eviction of -its class, deletes it once its clones have released them. A retirement that a -crash interrupts once it has removed its generation from the store's index -leaves the generation's tombstone alone: the generation is gone, with its -payload, its leases, and the class that its entry recorded. So each eviction -also retires every such tombstone, of a generation below the index's next one -that the index no longer lists, whatever the class that it serves, and lists it -among those that are gone. An eviction that fails leaves the generations that -it did not retire to the next one, and fails nothing that the command served, -which the store has committed. - -#### Listings - -`runtime-adapter-host list` and `inspect` report what the store records of its -generations, from its index, its tombstones, and the records of its leases -alone, with `generations::entries` and `generations::entry`: each generation's -store generation, the identities of its compatibility key and of its capture -class, its state, `available` or `tombstoned`, whether a clone may restore it at -the host's time, and why, as `Generation::eligibility` decides it, and its -leases, and, from `inspect`, each lease's sandbox and launch, and the metadata -of the capture: its storage generation, its time contract's host, boot, clock, -anchors, and lineage counter, and the lengths of its saved state and of its -RAM. They read no generation's payload, neither its snapshot nor its key, whose -decoding checks the digest of the key's kernel command line, so they load no VM -state, and hash nothing. They take no lock, so each record is as they read it. - -### Clones - -`runtime-adapter-host start` starts a clone of a generation of a template store -in a sandbox store, through NVX's public API, `nvx::host::template`. Its -library side, `clone`, holds what the command's options choose of the clone, -`clone::Request`: its image, an EROFS layer or a GPT disk no longer than the -template's placeholder, its upper, on an ext4 scratch image or in RAM, its -processor target, the template's processors online at boot unless given, its -memory target, the template's RAM unless given, a legacy workload option that -activation records do not encode, its overlay mode, its scratch's journal -policy, the bound of a GPT disk's first partition, the machine ID that its -executions see, the host paths that it shares with its root, as a launch's -[shares](#shares), and its [network](#networks), as a launch reads it, which -must be at the guest's address of the template's NIC, under a policy of its -own. -`Request::plan` plans the image over the upper, with the network, as a launch -plans a layered root, `Request::export` plans the export of the shares as a -launch plans it, -and `Request::clone_request` returns NVX's `CloneRequest` of both, with the -bindings that the sandbox's store stages for the images, as `stage` stages -them, so the start hashes nothing. The clone's activation record names the -hostname `runtime-adapter`, which every execution sees, the image's own machine -ID, unless the request asks for one that the guest generates once for the -clone, which a read-only bind holds at the root's `/etc/machine-id` for every -execution, and the guest's defaults of the other settings, unless the request -names them; for a GPT disk, the bound of its first partition is 64 KiB unless -the request names one, as a launch's guest takes it by default. For a template -with a NIC, the record names the clone's guest-applied network settings: -the DNS server that OpenVMM serves the guest under -the policy, if any, which NVX finds as OpenVMM does, since the template's guest -learned none, and which the guest writes to a resolver file, of mode 0444, that -a read-only bind holds at the root's `/etc/resolv.conf` for every execution, -beside a generated machine ID; whether the policy publishes the workload's -listeners, as it does with ingress, or with host-loopback access both ways, for -which the clone's guest starts their announcer, and refuses -`CAP_NET_BIND_SERVICE` and `CAP_NET_RAW` to its executions; and the URL of the -policy's proxy, if any, which every execution of the clone finds in its -environment, as an execution of a launch does; see -[Template mode](#template-mode). Without a DNS server, the root keeps the -image's own `/etc/resolv.conf`. -The guest of a GPT clone rereads the disk's partition table before it -assembles the root, since its kernel created the partitions of the template's -placeholder. - -The command first reads the clone's network, if given, as a launch reads it, -and refuses one that the plan refuses, then plans the export of the clone's -shares, if it has any, and -refuses those that it cannot export, as a launch refuses them, then reads the -generation from the store's index, and, unless the request names both of the -clone's targets, its compatibility key, before it needs NVX's artifacts: the -processor target is then the key's processors online at boot, all of its -processors for a template that records none, unless the request names one, and -the memory target the key's RAM, unless the request names one. A key that -cannot be read is left to NVX's plan, which reads it again, and attributes the -failure, to a malformed generation, which quarantines it, or to the host, as it -does for every clone. The store then -leases the generation to the sandbox, `Store::acquire`, at the host's time, -which it refuses for a generation that a tombstone quarantines, or that is not -eligible then, by the monotonic clock of the host's boot that captured the -template, and by UTC otherwise. For tests, the host's time may come from the -variable `RUNTIME_ADAPTER_HOST_TIME`, as a clock fixture, such as another -boot's, in the form that `clone::host_time_text` writes: -`host=HOST,boot=BOOT,clock=CLOCK,monotonic=NANOSECONDS,utc=NANOSECONDS`, with -the identities of the host and of its boot, 32 lowercase hexadecimal digits -each, the clock as a manifest spells it, `linux-boottime` or -`windows-interrupt-time`, and the readings of the monotonic clock and of UTC. -It affects the store's check alone: OpenVMM's restore samples the host's own -time. - -A selection that the store refuses ends the start before it leases anything, -with the refusal's record, in the form of a failure's, at the stage -`selection`, without a VM, and the status 10: a template store that does not -exist, and a generation that the store never published, or no longer holds, -name no template, `no-template`, by the evidence `no-store`, `unpublished`, or -`retired`; a generation whose capture the restore window of its time contract -no longer admits at the host's time is `expired`; and one that a tombstone -quarantines, or whose capture is after the host's time, by the clock that -measures its downtime, is `ineligible`, by the evidence `tombstone` or -`early`. The record's code is the code of NVX's error: `not-found`, -`template-tombstoned`, or `template-ineligible`. - -NVX's `ClonePlan` then checks the clone against the generation from the -metadata of its files alone, under the sandbox's lock, which the lease holds: -its network, which must attach a NIC exactly when the template has one, at the -guest's address of the template's NIC, under a policy of its own, which the -clone's restore installs on the NIC, or, for a template without a NIC, none, or -one that lets nothing through, so that a clone at another guest address finds -no template in it, -the bindings of its images, which no one else may change, the geometry of its -image, which must have the placeholder's block sizes, and be no longer than -the placeholder, whose length is the capacity that the restore presents, -reading zeros past the image's end, and that of its scratch image, which must -be the template's scratch's, its length included, its processor target, -1, 2, 4, or 8, from the template's processors online at boot to its capacity, -which the restore brings online, and the clone's repair verifies, its memory -target, whole blocks of 128 MiB, from the template's RAM to its capacity, and -its export, if it has one, which it checks as a cold launch's: from 1 to 256 -directories and files, the outermost shares, each of which the template's -filesystem slot, which its capture left dormant, attaches as a child of an -aggregate, with from 1 to 4,096 mappings, none of them at a target that the -container root keeps, and none of the directories and files may lie inside the -template store's directory or hold it, so that no clone reaches the payload of -a generation. A clone that -the template cannot serve fails there, before any VMM starts. The command then records the clone in the -sandbox's store, as `clone::CloneRecord`, in the format of a launch's -[record](#layered-roots), in the mode `clone`, with the template's kernel -command line, the plan, with the clone's network, and the bindings, so that -`probe` and `exec` read the plan back, and `exec` gives executions the URL of -the network's proxy, if any, as for a launch, no root configuration, and the -template store and the generation, so -that a later stop finds the store whose lease it releases. Once NVX checked -the export again, with `ExportPlan::verify`, OpenVMM restores the -generation's snapshot in place, through NVX's supervisor, as the launch that the -lease reserves, with the clone's image at the capacity of the template's -placeholder, over the scratch image, or, for a clone in RAM, with the -template's scratch absent, which OpenVMM attaches with no file, so that no -file of the host backs it, with the export on the template's filesystem slot, -which OpenVMM attaches as an aggregate, as a cold launch attaches it: each -directory and file a child, read-only or read-write, with the paths that the -export hides, shows again, or lets the guest write, and the template's NIC, if -it has one, under the policy of the clone's -network, which `--restore-network-policy` installs in place of the template's -complete deny-all policy, and the restore sequence activates the clone through -the -adapter's clone order, `host::CloneOrder`, which sends the clone's activation -record in an `Activate`, checks the guest agent's build in its `Ready`, sends -the export's mapping table, which the record announces, in order, as a launch -sends its own, and then `Start`. The record names the export's shape and mapping -count, and the table holds its mappings, each from a source that names its -child by the child's index, as a cold launch's table does. A clone without -shares has no -export: its restore leaves the slot dormant, its record announces no mapping, -and the sequence sends no table. If the guest fails a mapping, the sequence -sends no more of the table, nor `Start`, and the clone fails, once the clone -order asked the guest to stop, which tears down the root. The guest's -`MapFailed` does not tell the request's mappings from a failure of its own, so -the clone order takes it for neither, and NVX attributes the failure to what -the host saw of it: a broken protocol, unless the deadline passed or the -session was lost. - -A clone that fails, from its plan to its commit, fails with NVX's -`CloneFailure`, which attributes the failure, by the evidence that the host saw -of it, such as a length that the request supplied, or a status with which the -guest ended the VM, to the request, the staging of its files, the host, the -template, or a cause that the evidence does not show, and which, from the -clone's launch on, records it in the sandbox's store, with the stage at which -the clone failed, how far it got, and how the teardown that followed ended. The -command prints the failure's record, then releases the lease, `Store::release`, -which removes the records with it, once the failure's teardown is resolved, and -nothing of the clone runs: the clone had no VM, its VM exited, or NVX -terminated it. A failure whose teardown is unresolved, such as that of a commit, -or one that NVX could not record before it ended the clone's VM, leaves the -lease, and the records, to a later reattach. Once it released the -sandbox's lock, which comes after the template store's in NVX's lock order, the -command quarantines the generation through its tombstone, -`CloneFailure::quarantine`, for a failure of the template alone. A guest agent -that does not answer the command's `Ping` or `Echo` once the clone's sandbox -started fails the clone so, through `Started::fail`, which ends the launch, -with the evidence that NVX's restore sequence gives the failures of its own -sessions: the passing of the deadline, the loss of the session, or a failure -of the protocol. A start that fails otherwise, at a step of the adapter's own -before the clone's launch, such as the read of the images that the sandbox -stages, releases the lease, since nothing of the clone runs. - -The commands that reattach take the lock of a clone's sandbox through the path -of its lease, which they read under the lock, as a release requires. A sandbox -that holds a lease must not move: they refuse one whose lease names another -path, at which no sandbox holds the same lease, as the lease of a sandbox that -moved does, rather than take another sandbox there for it. They then reattach -to the clone with NVX's `reattach_clone`, in a new session of the adapter's -clone order, in which the guest agent answers the `Activate` of the clone's -record again, with the `Ready` that it answered once, and which they then use -as `reattach` attaches one. The reattach ends a clone that runs under another -runtime identity, without a word to its guest, or that fails the check of its -`Ready`, through its verified process identity, and finishes the record of an -earlier failure of the clone, whose teardown it resolves if it can. The -command then releases the lease of a clone whose VM ended without a failure, -and fails; ends as `start` does for a clone that failed; and keeps the lease -and the records, and fails, for a clone whose state cannot be determined. A -stop of a clone releases the lease once the store verified that the VM exited, -and the clone's records, which the release removes, told how the clone ended, -as `reattach_clone` reads them: a VM that had exited when the stop began ended -without its holder, as NVX's `Vmm::stop_unless_exited`, which then stops -nothing, tells by keeping the runtime record of its launch, and the stop ends -as the other commands do for a clone that they find so, while a VM that the -guest ended, once the stop began, with a status that is evidence of a failure, -such as its repair's, failed, which ends the stop as it ends `start`. -The commands release a lease through the template store that the clone's -record names, once the lease is of it. A sandbox whose start ended before it -recorded the clone holds a lease of a template store that no record names: it -records no clone, or an earlier clone's, which a release leaves, of another -template store perhaps. `reattach` and `stop` then release it with -`--template TEMPLATES`, through the template store TEMPLATES, in place of the -record's, which the release refuses unless the lease is of it. -A tombstone of the generation refuses new leases of it, and leaves those that -sandboxes hold to their clones, which reattach, stop, and release them as -before. - -#### Running a clone - -`runtime-adapter-host run` runs one execution in a clone of a generation, and -tears the clone down. Its preflight, which it reports with `preflight=staged`, -stages the clone's images in the sandbox's store, as `stage` stages them, -verifying the image, which hashes it only when a manifest is beside it, before -the clone's start, which hashes nothing. The command first reads the clone's -network, and plans the clone and the export of its shares, as `start` does, -then checks that the guest takes the execution's start, with the URL of the -proxy of the clone's network, if any, in its environment, as `exec` gives it to -an execution of a clone, so a network, an image, shares, or a start that it -would refuse fail the command before it stages anything. It then starts the -clone as `start` does, -at the host's time, or the one that `RUNTIME_ADAPTER_HOST_TIME` gives, and ends -as a start ends once the template store refuses the selection, or the clone -fails, with the record of the refusal or of the failure. Once the clone runs, -the command runs the execution in its root, the one that the clone's -activation assembled, through `host::Executions`, in the session of the -adapter's clone order in which the clone's sandbox started, as `exec` runs -one. The program, its arguments, and its environment reach the guest in the -execution's start, through the guest agent's execution control, never in the -template's boot tokens. The execution finds the clone's shares beneath the -root, since the restore sequence maps them before the clone's sandbox starts. -Then, whatever the execution did, the command stops the -clone as `stop` does, and releases the lease once the store verified that the -VM exited: a run leaves no clone running, and no lease held, unless its stop -timed out. The stop takes the same session, in which it skips the guest's -messages about the execution that may still arrive once the command finished -it without them, as when it gave up waiting for its outcome. It holds the -sandbox's lock, which the lease took, from the start to -the teardown, so no other command reattaches to its clone meanwhile. A run -whose execution failed to run, such as one whose session ended, still tears the -clone down, then fails, after the execution's diagnostic. Since the session may -have ended with the VM, as when its VMM crashed, the run first waits up to 10 -seconds for the VMM to exit, as long as a stop waits for one that it -terminated: its stop then finds that the VM exited without it, and ends the -clone as a reattach that finds it so does, with the record of its failure, so -the run exits with 8, or 9 while the teardown is unresolved. It exits with 4 -once the guest reported that a stage of the root failed, and with 1 otherwise. - -### Commands - -Each command of `runtime-adapter-host` but `stage`, `list`, `inspect`, and -`remove`, which need no artifacts, takes NVX's artifacts from the directory that -`NVX_ARTIFACTS_DIR` names, and the guest agent's build from -`--guest-build BUILD`, the crate's version by default: - -- `stage STORE --distro FORMAT:PATH --distro-identity UUID [--upper scratch:PATH]` - [stages](#staged-images) the images of a layered root for the launches of a - VM in the sandbox store STORE, which it creates if needed: the image PATH, - an EROFS file system, `erofs`, or a GPT disk, `gpt`, of which the guest - expects the identity UUID, and the ext4 scratch image PATH, if given. It - verifies and binds the images before it takes the sandbox's lock, which it - holds while it records them, and prints how many bytes it read through the - operating system's read calls, the cache's included, `read-bytes=BYTES`, as - `nvx::host::process::own_bytes_read` counts them, as it exits: a staging - that hashes the image reads it whole. -- `launch STORE --guest-agent PATH [--hypervisor NAME] [LAYERED]` launches a VM - in the sandbox store STORE, which it creates if needed, with the guest agent - PATH, a static `x86_64-unknown-linux-musl` build of `runtime-adapter-guest`, - on the hypervisor NAME, or else on the one that - `nvx::Hypervisor::from_env_or_default` selects. Once the launch has checked - that no VM of the sandbox runs, it writes the VM's initramfs and its - [record](#layered-roots) into the store. It connects to the VM's control - endpoint with `nvx::host::session::connect`, attaches with the launch's - capability, exchanges a `Ping` and the longest `Echo` with the guest agent, - commits the launch, and exits, leaving the VM running. A launch that fails - ends its VM. - - LAYERED, - `--distro FORMAT:PATH --distro-identity UUID --upper (scratch:PATH | ram) [--runtime-dir DIR] [--overlay-mode MODE] [--scratch-journal POLICY] [--first-partition-bound BYTES] [--workload-identity UID:GID] [--network FILE] [SHARES]`, - gives the VM a [layered root](#layered-roots). Its image is the file PATH, an - EROFS file system, `erofs`, or a GPT disk, `gpt`, of which the guest expects - the identity UUID: the file system's UUID, or the disk's GUID. Its upper is - the ext4 scratch image PATH, or RAM. The guest assembles the root below its - runtime directory DIR, `/run/runtime-adapter` by default, in the overlay mode - MODE, `trusted-lowers` or `untrusted-lowers`, checks a scratch's journal - against the policy POLICY, `allowed` or `forbidden`, which only a scratch - takes, and refuses a first partition larger than BYTES, which only a GPT disk - takes, with the guest's defaults unless given. A workload identity, the user - ID UID and the group ID GID, two numbers from 1 to 4294967294, is the identity - as which every execution of the sandbox runs: the guest checks its account in - the assembled root, without which the root's `identity` stage fails. The VM - has the [network](#networks) that FILE holds, if given, which the launch - reads before anything else: one that the plan refuses fails the launch before - it needs NVX's artifacts or creates the store. Before it spawns the VM, the - launch checks the bindings that STORE stages for the image and the scratch - image, and fails if STORE stages none for either, or one no longer holds, as - [the checked handoff](#the-checked-handoff) describes. It records the - bindings that it checked. - - SHARES, - `--share HOST:TARGET[:ro]... [--deny PATH]... [--allow PATH]... [--reserved TARGET]... [--share-root DIR]...`, - shares each host directory or regular file HOST with the root, at the - absolute guest path TARGET, which starts at the last `:/`, read-only with - `:ro`, as [Shares](#shares) describes. Each HOST must lie inside a host - directory DIR that the adapter authorizes, the export hides each host path - PATH of `--deny` inside the shares, but shows each host path PATH of - `--allow` inside a hidden one again, and no TARGET may take or cover a guest - tree that the container root keeps, or a tree TARGET that the adapter keeps. - The launch refuses shares that it cannot export before it touches STORE, and - checks the export before it spawns the VM. - - Once attached, the launch follows the [host's order](#the-hosts-order): it - sends the root configuration, checks that the guest agent that assembled the - root is of the build that the host program ships, sends the mapping table, - then `Start`, and exchanges the `Ping` and the `Echo` once the guest started - the sandbox. A guest agent of another build gets nothing more, and the launch - fails, which ends the VM. If the guest reports that a stage of the root - failed instead, the launch waits for the VM to end, abandons the launch, and - checks that the VM ended with the stage's status. If the guest fails a - mapping, the launch sends no more of the table, nor `Start`, stops the guest, - which tears down the root, waits for the VM to end, abandons the launch, and - checks that the VM ended with a success and a complete teardown. -- `create STORE --guest-agent PATH --distro FORMAT:PATH --distro-identity UUID [--hypervisor NAME] [--cpu-profile PROFILE] [--processors N] [--boot-processors B] [--memory MIB] [--memory-capacity MIB] [--scratch-length BYTES] [--network FILE]` - serves a [template](#templates) of the guest agent PATH from the template - store STORE, which it creates if needed: a generation of the template's - class that the store holds, and that is eligible at the host's time, or else - a new one, which the store captures and publishes. The placeholder is the - file PATH, an EROFS file system, `erofs`, or a GPT disk, `gpt`, with the - identity UUID. The template's VM runs on the hypervisor NAME, as a launch's - does, with the CPU profile PROFILE, `auto` by default, as OpenVMM's - `--cpu-profile` takes it, N processors, 1 by default, of which B are online - at boot, 1 by default, the fewest that its clones take, MIB mebibytes of - RAM, 256 by default, a RAM capacity of MIB mebibytes, its - RAM by default, and a dormant scratch of BYTES bytes, 64 MiB by default. It - has one NIC under the policy of the [network](#networks) FILE, unless the - policy lets nothing through, and none without a network. The command reads - the network before anything else, as a launch does, and checks the class - before it creates the store, then prints the generation once the store has - committed it, and [evicts](#eviction) the generations of the class that it - supersedes. -- `refresh STORE --guest-agent PATH --distro FORMAT:PATH --distro-identity UUID [--hypervisor NAME] [--cpu-profile PROFILE] [--processors N] [--boot-processors B] [--memory MIB] [--memory-capacity MIB] [--scratch-length BYTES] [--network FILE]` - takes the options of `create`, and captures a new generation of the - template's class in STORE, whether or not the store holds an eligible one, - then prints it once the store has committed it, and evicts the generations of - the class that it supersedes, as `create` does. -- `list STORE` reports each generation of the template store STORE, in the - order of their publication, from the store's records alone, as - [Listings](#listings) describes. -- `inspect STORE GENERATION` reports the generation of STORE whose store - generation is GENERATION, a positive number, as `list` does, with its leases - and the metadata of its capture. It fails for a generation that the store - does not hold. -- `remove STORE GENERATION` retires the generation GENERATION of STORE: it - tombstones the generation, so that it takes no new lease, and deletes it once - no lease of it remains. While leases remain, it reports them, and leaves the - generation tombstoned, with its payload, which a later `remove`, or the next - [eviction](#eviction) of its class, deletes once they drained. A generation - that the store no longer holds is retired already, and one that it never - published fails the removal. -- `reattach STORE [--template TEMPLATES]` takes the sandbox's lock, and - reattaches to its VM, which - another process launched, and which it holds, although the VM is not its - child, through its verified process identity. It reads the capability from - the store, attaches a new session, in the session's next epoch, exchanges a - `Ping`, and exits, leaving the VM running. It reattaches to a - [clone](#clones) in the session in which the clone answers the `Activate` of - its record again, and releases the lease of a clone that ended, through the - template store TEMPLATES, if given, in place of the one that the clone's - record names. -- `start STORE --template TEMPLATES --generation G --distro FORMAT:PATH --distro-identity UUID --upper (scratch:PATH | ram) [--processors N] [--memory MIB] [--workload-identity UID:GID] [--overlay-mode MODE] [--scratch-journal POLICY] [--first-partition-bound BYTES] [--machine-id (image | generated)] [--network FILE] [SHARES]` - starts a [clone](#clones) of the generation G of the template store - TEMPLATES in the sandbox store STORE, which it creates if needed, and whose - `stage` staged its images, which no one else may change: the image PATH, an - EROFS file system, `erofs`, or a GPT disk, `gpt`, of which the guest expects - the identity UUID, the file system's UUID or the disk's GUID, no longer than - the template's placeholder, whatever the placeholder's format, over the - ext4 scratch image PATH, or RAM, with a processor target of N processors, the - template's processors online at boot by default, a memory target of MIB - mebibytes, the template's RAM by default, the legacy workload option UID:GID, if - given, which activation records do not encode, the overlay mode - MODE and the scratch's journal policy POLICY, as a - launch takes them, and for a GPT disk, the bound BYTES of its first - partition, 64 KiB by default, as a launch's guest takes it. Every execution - of the clone sees the hostname `runtime-adapter`, and the machine ID that - `--machine-id` chooses: the image's own, `image`, by default, or one that the - guest generates once for the clone, `generated`. The clone's VM has the - [network](#networks) that FILE holds, if given, which the start reads before - anything else, as a launch does, and which must be the template's: NVX's plan - refuses any other before any VMM starts, as [Clones](#clones) describes. - SHARES, as `launch` takes - them, share host paths with the clone's root: the start refuses those that it - cannot export before anything but its network, and NVX's plan those that the - template's filesystem slot cannot bind, as [Clones](#clones) describes. - Once the restore sequence sent the clone's mapping table, if it has one, and - started the clone's sandbox, it exchanges a `Ping` - and the longest `Echo` with the guest agent, commits the launch, and exits, - leaving the VM running, and the sandbox holding the lease, once it printed - how many bytes it read through the operating system's read calls, - `read-bytes=BYTES`, as `stage` prints them: a start never hashes, nor copies, - the clone's images. A clone that - fails, such as one whose activation failed before it was ready, and whose VM - ended with the status of the failure's class, from 50 to 58, ends as - [Clones](#clones) describes, with the record of its failure. A selection - that the template store refuses, of no template, of an expired generation, - or of an ineligible one, ends the start before it leases anything, with the - refusal's record, as [Clones](#clones) describes. -- `stop STORE [--template TEMPLATES]` reattaches and attaches as `reattach` - does, then stops the VM: - the adapter's `Stop`, which the guest agent confirms with `Stopped` before it - ends the VM, is the stop's graceful step. It waits for the VM to exit, and - checks the VM's outcome report: a success, with status 0 and a complete - teardown. A VM whose guest agent does not confirm the stop is terminated, and - the stop fails. The worker of a layered root tears the root down before it - confirms, and reports a teardown that failed first: the stop then checks - that the VM ended with the teardown's status instead. A stop of a clone - releases its lease once the VM exited, and the clone's records told how it - ended, as [Clones](#clones) describes. -- `probe STORE PROBE...` reattaches and attaches as `reattach` does, then runs - each PROBE in the VM's layered root, in order, and prints the facts that it - found. A PROBE is the text of a [probe](#probes): its name, then its - arguments, each after one space. A probe that fails fails the command. -- `exec STORE EXECUTION [; EXECUTION]...` reattaches and attaches as - `reattach` does, then runs each EXECUTION in the VM's layered root, all at - once, over that one session, through `host::Executions`, and prints their - outputs and outcomes once every one finished. It runs at most 8 executions, - as many as the guest runs at once. An EXECUTION, - `[--identity UID:GID [--groups GID,...] [--capabilities N,...] [--numeric]] [--cwd DIR] [--env NAME=VALUE]... -- PROGRAM [ARG]...`, - runs PROGRAM with its ARGs, up to a lone `;`, which a shell takes quoted, as - `find -exec` takes them, without standard input. Without `--identity`, it - names no identity, and runs as the sandbox's workload identity, which the - guest otherwise refuses; with it, it runs as the user ID UID and the group ID - GID, with the supplementary groups GIDs and the capabilities numbered N, such - as 0 for `CAP_CHOWN`, if given, and the guest checks its account, unless - `--numeric` says that the IDs have none. It starts in the root's directory - DIR, or `/`, with exactly the environment of its `--env` entries, then, if - the VM's [network](#networks) has a proxy, the proxy's URL as the guest sees - it in `http_proxy`, `https_proxy`, `HTTP_PROXY`, and `HTTPS_PROXY`, unless - the entries name any of them. Before it starts any execution, the command - checks that the framing still carries each start with the proxy's URL, and - runs none of the executions if one no longer fits. It runs for at most a minute, with 16 KiB - chunks and the credit of 4 of them for its input and 8 for its output, which - its standard output and error share. The command keeps at most 16 MiB of each - stream, and cancels an execution whose stream would exceed them. Execution N, - from 1, has the key `(B + N, E)`: E is the epoch of the command's attachment, - and B the first 8 bytes of the broker instance's ID, as a little-endian - number. The guest refuses a key that it heard of before, and the epochs of a - broker instance only advance, while a broker restored from a snapshot has a - new instance ID, so no command repeats the key of another's execution. -- `run STORE --template TEMPLATES --generation G --distro FORMAT:PATH --distro-identity UUID --upper (scratch:PATH | ram) [--processors N] [--memory MIB] [--workload-identity UID:GID] [--overlay-mode MODE] [--scratch-journal POLICY] [--first-partition-bound BYTES] [--machine-id (image | generated)] [--network FILE] [SHARES] EXECUTION` - runs one EXECUTION, one of `exec`'s, in a [clone](#clones) of the generation - G of the template store TEMPLATES, in the sandbox store STORE, which it - creates if needed, then tears the clone down, as - [running a clone](#running-a-clone) describes. The execution's options may - come among the start's, in any order, and its PROGRAM and ARGs are every - argument after `--`. It first plans the clone as `start` does, with its - network and its shares, and checks the execution's start, with the URL of - the network's proxy, if any. Its preflight then stages the clone's images in - STORE, as `stage` stages them, and prints `preflight=staged`. It then starts - the clone as `start` does, runs the execution in the clone's root as `exec` - runs one, and prints its output and outcome once it finished. Then it stops - the clone as `stop` does, whatever the execution did, which releases the - lease once the VM exited. - -`reattach`, `stop`, `probe`, and `exec` find a VM that booted cold under -another runtime identity mismatched, and then signal nothing: the VM keeps -running. They end a clone that runs under another, as [Clones](#clones) -describes. The commands print what they find as `KEY=VALUE` lines: `reattached` -(`live`, `mismatched`, `exited`, or `indeterminate`, and of a clone, `live`, -`ended`, `failed`, or `indeterminate`), `differences` (the parts of the runtime -identity in which a mismatched VM differs), `progress` (the step that a clone, -which ended, reached, from `planned`, `launching`, `spawned`, `event`, -`acknowledged`, `activating`, `ready`, `starting`, and `started` to -`committed`), `pid` (the VMM's process ID), `instance` and -`epoch` (the identity of the session that they attached to), `root` (`ready` or -`failed`: how the guest answered the root configuration, or `failed` when a stop -finds that the root's teardown failed), `build` (the build of the guest agent -that assembled the root), `stage` and `problem` (the stage of the root that -failed, such as `assembly`, and the problem, or the problem of a mapping that -failed), `table` (`applied` or `failed`: how the guest took the mapping table), -`mapped` (how many of the table's mappings the guest applied), `sandbox` -(`started` once the guest started the sandbox), `preflight` (`staged` once -`run` staged the images of its clone), `PROBE.KEY` (the fact KEY that -the probe named PROBE found), `exec.N.KEY` (what execution N did: its `launch`, -`started`, `failed STAGE ERRNO`, or `none`; its `status`, `exit CODE`, -`signal NUMBER`, or `none`; the `cause` for which a side ended it, such as -`timed-out`, or `none`; the streams that it `truncated`, such as -`stdout,stderr`, or `none`; whether it `settled`, `reusable` or `fenced`; and -its `stdout` and `stderr`), `stopped` (`graceful`, `forced`, or `timed-out`, or -`exited` once a clone's VM, which the command found running, had exited when -the stop began, which stopped nothing), -`outcome` (such as `success 0`, or `guest-exit 14` once the assembly failed), -`teardown` (`complete` or `incomplete`: whether the VMM's teardown of the VM -completed, which the outcome report records apart from the root's), `created` -(`captured` once a capture made the template's generation, as a refresh's -always does, or `existing` once the store held it), `generation` (the -generation's number in the template store, which `create` or `refresh` served, -`inspect` reports, or `start` or `run` leased), `key` and `class` (the identities of its -compatibility key and of its capture class, in hexadecimal), `retired` and -`draining` (the generations that the eviction of a creation or a refresh -retired: those of the class that are gone, with those whose interrupted -retirement it completed, and those that leases keep, each list separated by -commas, or `none`), -`lease` (the identity of the lease that `start` or `run` took, 32 hexadecimal -digits), -`downtime` and `source` (the downtime of the leased generation's capture at the -host's time, in nanoseconds, and the clock that measured it, `monotonic` or -`utc`), `released` (the identity of the lease that a stop, a start that failed, -or the teardown of a run released), `failure.KEY` (the record of a clone's -failure, or of the selection of a start or a run that the template store -refused: its -`outcome`, what the failure is attributed to, `request`, `staging`, `host`, -`unknown`, `template`, or `repair`, or `no-template`, `expired`, or -`ineligible` for a refused selection; its `evidence`, what the host saw of it, -which attributes it, such as `supplied-length`, or `no-store`, `unpublished`, -`retired`, `expired`, `tombstone`, or `early` for a refused selection; its -`stage`, where the clone -failed, `plan`, `launch`, `restore`, `activation`, `start`, `commit`, `run`, or -`reattach`, or `selection`; its `progress`, the step that the clone reached, as -`progress` spells it, or `none` for a refused selection; its `status`, the -status with which the guest ended the VM, as the -VM's outcome report gives it, or `none`; its `teardown`, how the teardown that -followed ended, `no-vm`, `exited`, `terminated`, or `unresolved`; and its -`code`, the code of the failure's error, such as `runtime-mismatch`), -`quarantined` (whether the command quarantined the generation of a clone that -failed: `yes`, for a failure of the template, or `no`), `store` (the identity -of the template store), `generations` (the store generations that `list` -found, or `none`), `generation.G.KEY` (what `list` found of the generation G, -with the KEYs `key`, `class`, `state`, `eligibility`, `reason`, and `leases`, -which `inspect` prints of its generation), `state` (`available`, or -`tombstoned` once a tombstone quarantines the generation), `eligibility` and -`reason` (`eligible`, `early`, or `expired`, at the host's time, and NVX's -reason, with the capture's downtime), `leases` (the identities of the -generation's leases, or `none`), `lease.ID.sandbox` and `lease.ID.launch` (the -sandbox that holds the lease ID, and the launch of its clone that the lease -reserves), `storage` (the capture's storage generation), `time.host`, -`time.boot`, `time.clock`, `time.utc`, `time.monotonic`, and `time.lineage` -(the capture's time contract: the identities of the host and of its boot, the -host's monotonic clock, the capture's anchor in UTC and by that clock, in -nanoseconds, and its time lineage counter), `length.state` and `length.memory` -(the lengths of the snapshot's saved state and of its RAM, in bytes), and -`removed` (`retired`, once the generation is gone, or `draining`, with the -leases that keep it). Values -escape `\`, newlines, and carriage returns as `\\`, `\n`, and `\r`, and each -byte of a program's output that is not part of UTF-8 text as `\xHH`. The -commands exit with 0 once they did what they were asked, `exec` once every -execution finished, whatever its outcome, and `run` once its execution finished -so, and the clone stopped as `stop` stops one, with 3 when they find a VM that -booted cold mismatched, with 4 when the guest reported that a stage of the root -failed, and the VM ended with the stage's status and a complete teardown, with -5 when the [step hook](#the-step-hook) ended a command, with 6 when the guest -failed a mapping of the table, and the VM stopped gracefully, with a success -and a complete teardown, with 7 when `remove` found leases that keep the -generation, which it tombstoned, with 8 when a clone failed, once nothing of it -runs and the command released its lease, with 9 when a clone failed, and its -teardown is unresolved, so that the sandbox keeps the lease, with 10 when the -template store refused the selection of a start or a run, so that it leased -and launched nothing, and with 1, after -a diagnostic on standard error, when they -fail, as `run` does once its execution failed to run, after its teardown, -unless the teardown ended with 4, 8, or 9, which `run` then exits with, after -the diagnostic. A -value of `RUNTIME_ADAPTER_HOST_TIME` that is not a host's time is a -usage error, as is a value of `RUNTIME_ADAPTER_STEP_HOOK` that sets no step -hook, or that sets one at another step than `checked` for a start, and a value -of `RUNTIME_ADAPTER_REPAIR_DELAY` that sets no delay of a template's clones' -repair, for `create` and `refresh`, with -status 2. Without arguments, the program reports the contract that it was -built against. - -## Tests - -`tests/independent_build.rs` checks that every configuration builds as an -independent adapter's does: - -- The adapter is a workspace of its own with its own lockfile, and declares - the minimum supported Rust version of `nvx`. -- The protocol-only build depends on `nvx` alone, without features. The - host-only build enables the host side of `nvx` and not its guest side. The - guest-only build enables only the guest side, and pulls in neither Tokio nor - a dependency that only the host side of `nvx` declares. The tests read - these dependency trees from `cargo tree`, which downloads the packages of the - musl target the first time, on Windows too. -- Each binary runs and reports the contract that it was built against, and the - guest binary is a static executable: it has no program interpreter, needs no - shared library, and keeps its stack non-executable. - -The unit tests of `frame`, `root`, `probe`, `exec`, and `template` run in every -configuration. They cover each kind's encoding, messages in pieces of any size, -a message of each kind split at every byte, the headers that the framing refuses -before their body arrives, and the root configurations, readinesses, with or -without a clone's activation, parts of mapping tables, failures, probes, facts, -and messages of executions that decode, and those that don't, and the host's -side of an execution, which still sends the cancellation of one that it gives up -on in the step that ends it, without grace or settlement time, and which cancels -one whose stream outgrows the 16 MiB that it keeps. Those of `template` check -the platform tier's allowlist, token by -token, against the tokens that OpenVMM's capture of a platform snapshot allows -and refuses, through NVX's copy of OpenVMM's check, which refuses a token that -is not UTF-8 text, and admits by its prefix one that the kernel splits in two at -a vertical tab or at 0xa0, and template mode: a command -line in which the kernel finds no platform tier boots cold, whatever else it -holds, and one with it boots a template's VM if it holds the tier's tokens -alone, between any of the kernel's whitespace, and is refused otherwise, for a -parameter outside the allowlist at its start, in its middle, or at its end, or -within a platform token's value, after a vertical tab or the byte 0xa0, which -the kernel takes for whitespace and a split at ASCII whitespace does not, for a -platform token that the no-break space's first byte leaves without UTF-8 text, -for a double quote, and for `--`, and the file of a template's delay of its -clones' repair: a positive number of milliseconds, of at most 30 seconds, with -or without a newline, and the contents that it refuses. In the host-only -configuration, those of -`generations` check which generations an eviction retires: those of the served -generation's class that it supersedes, and, whatever their class, the -tombstones that interrupted retirements left alone, but not those of the -generations that the store published after the eviction read its index. In -the guest-only configuration, the unit tests of `worker::probes` -cover what the probes read: the mount table with its escapes, the kernel's -records of a process, `os-release`, and the guest's memory, and the probes and -arguments that the worker refuses. Those of -`worker::executions`, which spawn each program without entering a root, as a -fake root does, hand an execution the host's cancellation once its -timeout's instant passed, after its program ended before it, as a worker that -did not run meanwhile would, and check that the program's end stays its own. -Those of `worker` check, on x86-64, the adapter's own launcher of a clone's -announcer, which runs its program with its socket as its standard input, in a -session of its own, stops it and collects its status when stopped or dropped, -and reports a failed exec, after which no child remains; and they run PID 1's -collection alone, in a run of the test binary of -its own, with a thread of each of two children that it traces, which stop -before the worker ends, one with a signal whose report `nix` decodes and one -with a real-time signal, and check that PID 1 takes both stops and goes on -until it collects the worker's status. -Those of `worker::entries` cover the names of capabilities, the fields of a -process's status, the outcomes of attempts, the entry's refusals, and the -probe's arguments. Those of `worker::capture` check that closing the worker's -fence acquires it, the status of each stage of a capture, which no other -outcome of the worker shares, a template's devices, the template's own root -configuration, and the inputs of a clone's root that it gives, the tag of the -host filesystem device among them, and that the -worker attaches its session, takes the broker's reset, and acknowledges it, -which the stream sent, before the capture runs, without reading anything more -of the session, and keeps the session, active in the broker's first epoch, and -that no capture runs once the attachment cannot be written, the broker closes -the stream or sends anything but a reset first, or the acknowledgement cannot -be sent. Those of `worker::activation` check the status of -each class of a clone's activation's failures, that the activation's -timeout covers the wait for both block devices, and the facts of the -`activation` probe, the delay of the repair, if any, among them. - -`tests/guest_session.rs` checks, in the guest-only configuration, that the -guest agent serves the control session with the adapter's own framing. A test -broker plays the broker's end of the guest's session over a Unix socket pair -with the public record API of `nvx`, for `guest::serve` and for -`runtime-adapter-guest serve`. It covers messages that span records both ways -and fill the receive window, a record whose bytes arrive one at a time, resets -in the middle of a message, with the next epoch of the instance and with a new -instance, streams that end at and inside a record, and the records and -messages that the guest agent refuses: sequence, epoch, and instance errors, -stale and identity-less resets, record lengths, bootstrap identity, ERROR -payloads, and malformed headers. It also checks that the guest agent of a VM -without a layered root answers probes with a failure, and refuses the messages -about a layered root, such as its configuration, and executions. - -`tests/host_session.rs` checks, in the host-only configuration, that the host -program's client attaches to the control session and exchanges the adapter's -own framing with the guest agent. A test broker plays the broker's host-facing -end of the session, and the guest agent behind it, with the public record API -of `nvx`, over an in-memory transport, so that the tests run on Linux and -Windows. They cover: - -- the attachment: its bootstrap record with the launch capability, waits for - the guest, a deadline that passes meanwhile, and refusals, with an ERROR or - by the end of the stream; -- messages that span records both ways, records that end a message and start - the next, and records whose bytes arrive in pieces down to one byte; -- a receive that times out inside a message, then returns the whole message; -- a send that times out after it wrote part of a record, and a send whose - transport fails, each of which ends the client with the message that it - held; -- resets and disconnects in the middle of a message, with the next epoch of - the instance and with a new instance, and streams that end inside a record; -- the records that the client refuses: sequence, epoch, and instance errors, - stale and identity-less resets, record lengths, bootstrap identity, ERROR - payloads, a payload beyond the session's limit, which fails at its header, - and header faults; -- the messages that it refuses: malformed headers, which fail before their - body arrives, and messages that only the host program sends, a clone's - `Activate` among them; and a message beyond the framing's limit, which it - refuses to send; -- the answers that its requests check, and a request whose answer arrives too - late; -- a layered root: the configuration that the client sends, the guest's - answers, ready with the guest agent's build or the stage that failed, probes - and their facts, a stop that learns that the teardown failed, and answers - whose bodies do not decode, which end the client; -- the host's order: the root configuration that announces the table, the - guest's readiness, then each part of a table of no mapping, of one, and of - several parts, once the guest applied the one before, then `Start`, also in - records of 7 bytes; a guest agent of another build, and a root that failed, - after which the host sends nothing more; a table that does not encode, of - which the host sends nothing; a mapping that the guest fails at the first, a - middle, or the last part, after which the host sends no more of the table, - nor `Start`, but `Stop`, and learns how the stop ended, with or without a - teardown that failed, or as the session ends; and `Maps` or `Start` answered - with another kind, or failures whose bodies do not decode, which end the - client; -- a clone's order: an attachment that waits through the session's bootstrap, - the record as the first request, after which the host waits for `Ready`, - whose activation it returns, also none, then the table in parts and `Start`; - a session that a later reset, the end of the stream, or a failure of the - transport ended, which the order takes for lost, and whose successor sends - the same record first; and the failures that are final: a refusal in an - ERROR, deadlines that pass while attaching or awaiting `Ready`, another - build, after which the host sends nothing more, an `Activate` answered with - another kind or with an activation that does not decode, and a record beyond - the framing's limit, which fails before it is sent; -- the driver of executions, whose guest runs the runner of NVX's execution - engine for each start, as the worker does, without a program: two - executions at once, one with more output than its credit holds, each - byte-exact, with its outcome, and one whose launch the guest refuses, whose - records carry their messages whole or split; the starts that the guest would - not decode, a second start of an ID, a start whose input the host would - send, starts whose output windows count bytes or more than 32 frames, a - start beyond 8 executions, and a start once the driver ran, of which the - driver sends nothing, with a client that stays usable; a message about an - execution that it did not start, about none, whose body does not decode, or - that breaks the engine's rules, and the end of the stream, each of which ends - the run; a timeout that fires while the guest says nothing, after which the - execution is cancelled, then fenced; and a run whose deadline passes first, - in a receive, which leaves the client usable, or in a send that waits for a - guest that stopped reading, which ends it; and a stop, once the driver gave - up on an execution, that skips the guest's late messages about it, before - and after a teardown that failed, while `stop` takes the first for its - answer, and neither skips a message about an execution that the driver did - not start. - -Through a relay between its test broker and a pseudo-terminal, -`tests/guest_session.rs` also checks that `guest::serve_terminal` and -`runtime-adapter-guest init` serve a tty in raw mode, with an `Echo` of every -byte value, and that `init` fails at once on a path that is no tty. - -`tests/guest_session.rs` also runs the worker against a fake root, behind its -test broker: a root that the worker assembles, probes, and tears down on a -stop, across a reset of the session; an image over RAM, in the overlay mode of -the configuration; a GPT disk, with the configuration's bound of the first -partition; the host export that it mounts, the mapping table that it applies in -order, in parts, across a reset of the session, and the start; a table of zero -mappings, which does no work, with or without an export; the parts that do not -continue the table, or map the export's root, which it refuses as a whole, and -a mapping that the root refuses, which latches the root, after either of which -it refuses the later parts and the start; the executions and starts that it -refuses out of order, -before the start and after it; the failure of each stage, the tokens of a host -export that break their rules, mappings without an export, an export that fails -to mount, and a workload identity whose account the root lacks among them, which -the worker reports before it ends with the stage's status, without the -operations of the later stages; the account check of a sandbox's workload -identity, once the root is assembled, with the default hostname; a teardown -that fails; before the root, the configuration and the `Stop` that it accepts, -and the messages that it refuses, a clone's `Activate` among them; once the -root is ready, a second configuration, an `Activate`, and a broker that closes -the stream, which leave the root that it assembled to its operations; -and a broker that closes only its end of the stream while the worker probes, -which still receives the replies that the worker queued. -Through the binary, it checks that `runtime-adapter-guest init TTY layered`, -which is not PID 1 there, starts the worker, which fails at the layout stage on -a kernel command line without a layered root, and ends with its status, and -checks `worker`, `idle`, and the usage errors. Its PID 1 collects the status -of its unrelated child once a real-time signal ended it, even when the name of -its program, and so of its children, is no UTF-8, as that of a link to the -guest binary, and supervises the worker on, and ends with the status that a -shell reports for a worker that a real-time signal ended, alone or together -with the unrelated child. Each guest binary that a test starts runs in a -process group of its own, with the processes that it starts, and keeps its -process ID, and with it the group's, until the test has killed the group. A -binary that exits but leaves processes of its group running fails its test. A -test that fails, or gives up on the binary, kills the whole group, even once -the binary exited, so that none of them outlives the test or holds its output -open, which tests check too. - -`tests/guest_session.rs` also runs the worker's executions behind its test -broker, whose host side starts the sandbox, then drives each execution's -`exec::HostExecution`, with frame-counted credit and one output pool. The fake -root prepares each execution's entry, as the worker describes it, in a directory -of the tests' own whose account databases hold the accounts that the tests name, -and then spawns the program without entering a root, which needs privileges, in -the tests' own root and identity. Unless a test says otherwise, the sandbox has -a workload identity, as which every execution runs. Where the tests can create -cgroups, as root in a cgroup v2 hierarchy with the `pids` controller, the fake -root gives the worker a workload cgroup of the test's own, below which each -execution runs in a cgroup of its own, so that an execution whose program ends -on its own settles, and is reusable. Elsewhere, the fake root names no cgroups, -and each program runs in a process group alone, so that such an execution is -fenced. The tests cover: - -- three executions at once, each byte-exact: standard output beyond a MiB, 1.5 - MiB of input through `cat`, and both streams of a program that fails; -- a sink of the host that stalls, and so stalls only its own execution, while - another one streams to its end, a timeout ends a third one in time, and a - ping is answered; the stalled execution's cancellation then ends it within - its grace, with its output truncated; -- a probe while an execution runs, which the worker refuses, so that the - execution's timeout ends it in time, and runs once the execution ended; -- a stream to the host that fills up while four executions write without end, - behind which a cancellation still ends its execution's program, and a stop - the others, before the stream takes anything again; -- two executions that write without end, each with more credit than the - worker queues for the stream, which take turns at the stream; -- the starts that the worker refuses: beyond its limit of executions, and - beyond its memory for one, and a program that it cannot find; -- the messages about executions that end the session: before the root, a body - that does not decode, a second start, credit that the engine refuses, credit - of the input from the host, and a message that only the guest sends; -- the generations that the worker remembers: a start of an earlier generation - after the later one ended, and after the worker refused a later one, which - are stale, and a second start of a generation that ended or that the worker - refused, which ends the session; -- a later generation and a reset of the session, which abandon executions - whose programs end, and a stale message of an abandoned generation; -- a stop that ends the running executions and collects their programs' - statuses, and a program whose status the worker collects once it ended, - after its execution's runner gave up on it; -- programs that end while a descendant holds their output: the worker ends - the descendant, through the execution's cgroup or the program's process - group, so that the output ends at once, losing nothing, and a stop once the - worker collected such a program's status leaves no process of its group - running; -- with cgroups, a descendant in a session of its own, which holds a program's - output once the program ended, and which the execution's settlement ends, so - that the output ends at once, losing nothing, and the execution settles; - the worker removed the execution's cgroup before it sent the outcome, and - runs its limit of executions at once again; -- with cgroups, an execution that does not settle within its settlement time, - because a zombie whose parent is outside its cgroup occupies the cgroup: it - keeps its place, so that the worker runs one execution fewer than its limit, - even once the cgroup emptied, until the host stops the worker, which then - removes the cgroup; -- a program that a real-time signal ends, whose status the worker collects; -- a program that runs in the working directory of its start, with exactly the - environment of its start; -- the identities that starts name in a sandbox without a workload identity, - with their supplementary groups, capabilities, and account modes, and the - default hostname, all of which the worker's entry takes; -- the publication of listeners that the root configuration names, published - without it, with which every execution enters, and the account check of a - sandbox's workload identity too: while listeners are published, the entry - refuses `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW` at stage 0, with `EPERM`, - and while they are not, it takes them, as it takes the other capabilities - of its allowlist either way; -- the starts that the worker refuses at stage 0 for their identities: one that - names an identity in a sandbox with a workload identity, or none in a - sandbox without one, with `EINVAL`; a capability outside the entry's - allowlist, one of a user ID other than 0, and `CAP_NET_BIND_SERVICE` and - `CAP_NET_RAW` while listeners are published, with `EPERM`; an identity - without a passwd entry, or whose primary group has no group entry, with - `ENOENT`; and one whose passwd entry has another primary group, or whose - user ID the kernel reads as "unchanged", with `EINVAL`. None of them takes a - place, so the worker then runs its limit of executions at once; -- an execution whose start comes in the payload of the root's configuration - and the sandbox's start, whose timeout counts from its own start, however - long the root's assembly takes; -- executions on a tty in raw mode. - -A test of the worker's executions that fails before it stops the worker, even -while its broker pauses, ends the worker's session as it drops its host, so -that the worker drops its executions, and their programs end, which a test -checks too. - -On x86_64, `tests/guest_session.rs` also runs the worker of a clone, which a -restore of a template's snapshot started, with `worker::work_clone`, behind its -test broker, with the session that the template's VM attached, active in the -epoch of its broker's first reset, and the clone's fence-free activation, -`nvx::guest::activation::Activation::with_resources`. The fake -root applies each record as `nvx::guest::clone::apply` does, without a root: for -a GPT disk, it stands for the kernel's reread of the disk's partition table, -which a test may fail; it takes as long as a test asks, but no longer than until -the activation's deadline, as the waits for the devices do, and stands for the -generation of a machine ID that the record asks for, the -mount of the record's export, the write of the resolver file that names its -network's DNS server, and the setting of the record's hostname and machine ID -for every execution, each of which a test may fail. The tests cover: - -- the reset that the restore queued, of another instance, which the worker - acknowledges without an attachment, and the `Activate`, whose record it - applies once, with the template's inputs, and answers with the `Ready` that - reports the activation; -- retries of the `Activate`, in records of 7 bytes, after a later reset, and - once the sandbox started, which get the same `Ready` without a second - application, and the order that the worker serves once the clone is ready, - as after a cold boot; -- the messages before the `Activate`, a stop and a ping among them, and another - reset, which fail the activation with the status of the host's session, with - nothing answered or applied; -- a record that does not decode, a cold boot's, and one that expects other - resources, which fail the activation with nothing applied, and a GPT disk - whose partition table the kernel does not reread, a machine ID that cannot be - generated, a hostname or machine ID that cannot be set, an export that cannot - be mounted, and a resolver file that cannot be written, which fail its - application, each with the status of its class, and no `Ready`; -- an `Activate` whose header announces more than 1 KiB, which the framing - refuses before it takes any byte of its body, and which fails the activation - with the record's status, before the clone is ready and once it is; -- the activation's deadline, whether the restore's reset came or not, or part - of an `Activate`, which the broker sends before the worker runs, so that the - worker takes it at once, however late it runs: the deadline fails the - activation with nothing applied, and ends nothing once the clone is ready; -- an application that would outlast the activation's deadline, which gets that - deadline, and which ends the clone with the deadline's status once the - deadline passed, with no `Ready`; -- a reset, a request, and a retry of the `Activate` that the host sends while - the application runs, which the worker takes before it completes the - activation: the reset and the request fail it, with the status of the host's - session and no `Ready`, and the retry joins it, so that both get `Ready`, - after one application; -- a stream that takes nothing more from the worker, as once the broker failed, - after which the worker still ends once its activation's deadline passed, - with the deadline's status, or once the broker closed its end of the stream - before the clone was ready, with status 1, once it waited for the stream for - `worker::UNACTIVATED_DRAIN`; -- a broker that closes the stream before the clone is ready, whether the - restore's reset came or not, or part of an `Activate`, which ends the clone - with status 1, with nothing applied, and once the clone is ready, which ends - it as it ends the worker of a VM that boots cold; -- a broker that sent the restore's reset, and a ping or nothing more, and is - gone before the worker runs, so that the worker cannot write its - acknowledgement: the worker still ends with the activation's status, or with - status 1, rather than with the failure of its stream; -- an `Activate` of another transaction, or of the same transaction with - another record, once the clone is ready, or its sandbox started, which fails - the activation with the status of the host's session; -- the identity that each start names, while a start that names none is refused - with `EINVAL`, and the record's hostname, which every execution's entry takes; -- the `activation` probe, which the worker answers itself, with no root - operation, executions running or not: the restore's reset, the host's first - message, the clone's `Ready`, and its start, in that order, after the - repair's acknowledgement, and the start of the first execution after them, - and no delay of the repair, which the clone's template did not ask for; -- the mapping table of the record's export, which the host sends in parts once - the clone is ready, and which the worker applies in order, from the export - that the application mounted, before it starts the sandbox, and the endpoint - of the network's proxy, which the worker leaves to the host: an execution's - environment is exactly its start's. - -`tests/clone_start.rs` checks, in the host-only configuration, on Linux and -Windows, without a hypervisor: - -- the command lines of `start` that the host program refuses, each with status - 2: missing options, a generation that is not a positive number, a scratch's - journal policy for an upper in RAM, a first-partition bound for an EROFS - image, invalid values, an option given twice, a share's options without a - share, and the options of other commands, and the options of `start` that - other commands do not take, with a clock fixture that is not a host's time - too; -- a start whose template store does not exist, or does not hold the - generation, each with status 10 and the record of a selection that names no - template, and one whose image the plan refuses, or whose shares it cannot - export, with status 1, each before it needs NVX's artifacts or creates the - sandbox's store, and the shares before it reads the template store; -- a start whose network cannot be read, or is one that the plan refuses, with - status 1 before anything else: before its image, which does not exist, its - shares, which it cannot export, and the template store, which does not exist; -- the command lines of `run` that the host program refuses, each with status - 2: missing options, a missing `--` or program, the qualifiers of an identity - without one, an execution's start that the guest would refuse, a - first-partition bound for an EROFS image, a share's options without a share, - invalid values, an option given twice, arguments before `--`, and the options - of other commands, with a clock fixture that is not a host's time too; -- a run whose network cannot be read, or is one that the plan refuses, or whose - shares it cannot export, with status 1, before it stages anything or creates - the sandbox's store; -- a run whose image the plan refuses, with status 1, before it stages anything - or creates the sandbox's store, and runs whose template store does not exist, - or does not hold the generation, each with status 10 and the preflight's - `staged` before the record of a selection that names no template, once the - preflight staged the clone's images, before it needs NVX's artifacts, and - without a lease; -- the text form of a host's time, which `RUNTIME_ADAPTER_HOST_TIME` takes, - written and read back on both clocks, and the texts that are none; -- a clone's record: its JSON, with the plan that `layered::LaunchRecord` - reads back, the template store and the generation that a stop reads, a - launch's record that is no clone's, and records that a reader refuses; -- a clone's network: the plan of a request with a proxy-only network, NVX's - request of the clone, whose plan names the proxy's URL that NVX's activation - record of the clone takes, and the clone's record, from whose plan an - execution finds that URL in each of the proxy's variables, after its own - entries; and an open network, and none, with no proxy's settings; -- NVX's request of a clone, with the staged bindings, the processor and memory - targets, and the guest's defaults or the request's settings, the images that - the sandbox's store does not stage, which it refuses, and the export of the - request's shares: none without shares, and with them, one directory, the - root of a read-only share, holding a read-write one, mapped from an empty - source and from its path. - -`tests/initramfs.rs` and `tests/runtime.rs` check, in the host-only -configuration, on Linux and Windows: - -- the archive that puts the guest agent in place of NVX's agents: its entries, - their owners and modes, both launchers, and the padding of its headers and - data, for guest agents of every length modulo 4, and a template's, which - holds the file of the delay of its clones' repair after the launchers, for - delays of every length modulo 4; -- the copy of NVX's initramfs, which stays as it is, of every length modulo 4, - with the archive after it at a multiple of 4 bytes, a template's with the - delay's file, and the files that a failure names; -- the runtime identity of a checkout and of a release, which differs from - NVX's in the initramfs alone, another build of the guest agent, which differs - in the initramfs too, and missing install metadata, which fails as NVX - reports it. - -`tests/listener_client.rs` checks, in the host-only configuration, the real TCP -greeting and answer of the VM tests' listener client on Linux and Windows. -On Linux it also checks that a self-connection leaves the guest's publication -port bindable and that a connection reset before inspection is retried. - -`tests/sandbox_names.rs` checks, in the host-only configuration, that a test -process never gives a VM test's sandbox the name of one that it dropped, since -Windows refuses to create a directory under the name of one that another test -is still removing. - -`tests/staged_images.rs` checks, in the host-only configuration, on Linux and -Windows: - -- staging: an image without a manifest, one whose manifest matches it, and - manifests that name another image or digest, which fail; a GPT disk over - RAM; and images that no launch could attach; -- the bindings that a launch takes from the staged images, and those of other - images, which it does not; -- the record of the staged images, as written and read, and the forms that a - reader refuses; -- `runtime-adapter-host stage`, which records the staged images, and prints the - bytes that it read, at least the image's length once it hashed the image, a - staging that fails, which leaves them, and prints nothing, and a staging that - replaces them. - -`tests/layered_launch.rs` checks, in the host-only configuration, on Linux and -Windows: - -- the adapter's plan of a layered launch: its block devices and boot tokens, - with none of the sandbox profile's settings, and the sizes of the images; - and the options that it passes on as they are, for the guest to check; -- the checked handoff: OpenVMM's arguments, which name the checked files, on - Linux by the descriptors that OpenVMM inherits and on Windows by their final - paths, with no workload identity and no managed lifecycle, and what OpenVMM - inherits, over a scratch image and over RAM, and a command line too long for - Windows to start OpenVMM with, which the launch refuses on every host; -- the launches that the checks refuse: of images that the store does not - stage, of another format or identity than the staged image's, and of an - image that changed, a scratch image that another file replaced, and an image - that is gone, after they were staged; -- the scratch image's claim, which a second launch waits for, then fails to - take, until the first one releases its checked files; -- the record of a launch, with the bindings that it checked, over a scratch - image and over RAM, and of a VM without a layered root; -- shares: their text, `HOST:TARGET[:ro]`; the export of a table of shares - inside an authorized directory, which maps each at its target, parents - first, hides a denied path, and narrows the writes of a read-only directory - to the read-write share inside it, and an export that shows an allowed path - inside a hidden one again; the root configuration that announces the - table, OpenVMM's arguments that attach the export, an export whose hidden - paths leave the rest of OpenVMM's command line too little, which the launch - refuses as it is planned, a launch that shares nothing, the export that the - launch verifies before it prepares anything, and a table that the guest would - refuse; -- the shares that a launch refuses: outside the authorized directories, also - through a link on Linux, without any, with an authorized directory that is - not one, or a share that is not accessible; reserved trees that are not - canonical guest paths, targets that take or cover the adapter's or the - container root's reserved targets, a read-only share inside a read-write - one, a hidden path that does not exist inside a share, and an allowed path - that lies inside no hidden one; -- the networks that a launch reads: the OpenVMM arguments of each element of a - policy, which NVX's lowering renders after the plan's block devices, a policy - that lets nothing through, which attaches no network device, the proxy's - URL, the publication of listeners that the root configuration names for - each, and the record, which keeps the network as the file gave it; and the - files that it refuses, each for the plan's reason; -- the URL of a network's proxy in an execution's environment, as the guest - sees it, with and without a scheme, after the start's own entries, which the - start carries; a start that names its own proxy settings, which keeps them; - and a launch without a proxy, which leaves the environment as it is; -- the plan that the record of a launch gives back, from which the host finds - the proxy's URL, none for a VM without a layered root, and the records that - it refuses; -- `runtime-adapter-host launch` with shares that it cannot export, which it - refuses with status 1 before it creates the store, with artifacts that boot - nothing; -- `runtime-adapter-host launch --network`, which refuses a network that the - plan refuses, or that it cannot read, with status 1 before it needs NVX's - artifacts or creates the store, and passes one that the plan takes; -- the command lines that the host program refuses, each with status 2, those of - `exec`, of `--workload-identity`, and of `create` included. - -`tests/template_capture.rs` checks, in the host-only configuration, on Linux -and Windows: - -- the capture that `capture::Request::capture` makes of a request: the - adapter's plan of the placeholder, as the host program stages it, with the - request's network, over RAM, which no template keeps; the placeholder's - binding, the policy revision, and the artifacts, with the initramfs that it - names; and the class, with the runtime identity of the adapter's VMs and the - guest agent's build, the backend, the `auto` CPU profile, the processors and - those online at boot, which the request names, 1 by default, whatever the - capacity, and which name the class, the RAM and its capacity, the geometries - that OpenVMM presents, of 512-byte logical blocks, and the network, which - names the class: a class without it is another, as is the class of a - template whose clones delay their repair, whose one invariant setting is the - delay; -- `runtime-adapter-host create` with a network that the plan refuses, or that - it cannot read, which it refuses with status 1 before it needs NVX's - artifacts or creates the store, and with one that the plan takes, or none, - which goes on to need NVX's artifacts; -- `create` and `refresh` with a value of `RUNTIME_ADAPTER_REPAIR_DELAY` that - sets no delay, which they refuse as a usage error, with status 2, before - they create anything. - -`tests/template_operations.rs` checks the template operations, in the -host-only configuration, on Linux and Windows, without a hypervisor: - -- `list`, `inspect`, and `remove` of a directory that does not exist, and of a - private one that holds no template store, which each refuses with status 1, - and creates nothing; -- `list` of a template store without generations, which prints the store and - none, and `inspect` and `remove` of a generation that it never published, - which fail with status 1, and tombstone nothing; -- `refresh` with a network that the plan refuses, which it refuses with status - 1 before it needs NVX's artifacts or creates the store, as `create` does, and - with one that the plan takes, or none, which goes on to need them; -- the command lines of `refresh`, `list`, `inspect`, and `remove` that the host - program refuses, each with status 2: a missing store or generation, a - generation that is not a positive 64-bit number, and arguments or options - that the command does not take. - -`tests/step_hook.rs` checks, in the host-only configuration, on Linux and -Windows, the [step hook](#the-step-hook): the values of -`RUNTIME_ADAPTER_STEP_HOOK` that set it, with each step and action, a path with -colons, and those that set none, which the host program refuses with status -2 for `launch`, `reattach`, and `stop`; a start's, which the host program takes -at `checked`, and refuses at the other steps with status 2; and a pause, which -does nothing, and pauses for no time, at another step, creates its file with -its step's name at its own, waits until the file is gone, on Windows until a -pending deletion of the file completes, and says how long it paused, and which -a file that is -there already fails. - -`tests/vm_steps.rs` checks the step hook on a real hypervisor, in the -host-only configuration, with NVX's Ubuntu EROFS image over a copy of the -scratch template: - -- a launch paused at `checked`, while files of zeros, which the guest could - neither assemble nor mount, try to replace the image and the scratch image - at their paths: on Linux, they replace them, and the guest still assembles - the checked originals, and writes nothing to the replacements; on Windows, - the checked files keep the replacements from the paths, and keep the images - from being deleted or the image from being opened for writing, until the - guest is ready, after which the images can be renamed again; -- a launch that the hook ends at `checked`, with status 5, which leaves nothing - of a launch in the store, and whose exit released the scratch image's claim, - since the next launch takes it; -- a launch paused at each of its steps in turn, at which the store holds what - the step says, and the scratch image's claim is held at `checked`, and which - then goes on, and is stopped. - -`tests/vm_restart.rs` checks the restart of an orchestrator, on a real -hypervisor, in the host-only configuration: - -- `runtime-adapter-host stage` stages NVX's Ubuntu EROFS image over a copy of - the scratch template, and `runtime-adapter-host launch` launches a VM with - them as its layered root, whose guest agent, of the build that the host - program ships, starts the sandbox, and exits; -- another process's `reattach` holds the VM, which is not its child, and - attaches a new session, in a later epoch of the same broker instance; -- a third process's `stop` stops the VM: gracefully, with a successful outcome, - and with no runtime record left. - -In another test, a host program that ships another build of the guest agent -finds the VM mismatched in its initramfs, and signals nothing: the VM runs on, -under the same runtime record, until the host program of the right runtime -stops it. In a third, a launch of a host program that ships another build than -the one that it boots finds that build in the guest's readiness, sends it -neither the mapping table nor `Start`, and fails, which ends the VM, whose -scratch image the next launch claims. - -### Layered scenarios - -`tests/vm_layered.rs` launches VMs with a layered root on a real hypervisor, in -the host-only configuration, through the host program's commands, and checks -that the guest's worker, outside PID 1, assembles the root, or reports the -stage that failed, and that the VM still ends cleanly. A launch whose guest -assembled the root also follows the host's order: the guest agent is of the -build that the host program ships, applies the mapping table, which is empty -unless the scenario shares host paths, and starts the sandbox. Each scenario -stages its images, then launches them, and is a row of the table that its -`scenarios!` macro declares, and an ignored test of its own: - -| Scenario | Root | Expects | -| --- | --- | --- | -| `erofs_over_scratch` | NVX's Ubuntu EROFS image, with its own identity, over a copy of the scratch template | A root that the worker assembled: the image, read-only, under the overlay, whose upper and work directories are on the scratch, each mounted once below the runtime directory, with the options of the default mode, which trusts its lowers; the merged root's contents, with a file that reaches the upper and not the image; a copy-up of a file's metadata alone, which leaves its data in the image, and the rename of a lower directory; the image's and the scratch's disks; the agent's and the workload's cgroups, with their resource values; a worker that is PID 1's child, and PID 1's unrelated child, untouched; an empty mapping table, which does no work: the launch exports nothing, the guest mounts no export, and nothing is mounted below the root; and a teardown that unmounts the scratch cleanly | -| `erofs_over_scratch_with_untrusted_lowers` | The image over the scratch, in the untrusted-lowers mode, with a journal that the launch allows | The root, whose overlay has none of the trusting mode's options, copies up a file's data too, and refuses the rename of a lower directory with `EXDEV` | -| `erofs_over_ram` | The image over a tmpfs in RAM | The root, whose upper and work directories are on a tmpfs of the kernel's default size, whose root directory has mode 0755, with no scratch mounted, and no disk but the image's | -| `erofs_over_ram_with_untrusted_lowers` | The image over a tmpfs, in the untrusted-lowers mode | The root over the tmpfs, which copies up a file's data too, and refuses the rename | -| `gpt_over_scratch` | A GPT disk that the scenario writes, with its own GUID, over a copy of the scratch template | A root that the worker assembled: the first partition's bytes, as the disk holds them without their padding; each layer, an ext4 file system, mounted once, `ro,nosuid,nodev` with journal loading suppressed (`noload`, reported as `norecovery` by the kernel), from its partition; the overlay, with partition 2's layer at its base and partition 3's at its top, in the default mode, which copies up a file's metadata alone and renames a lower directory; a file that reaches the upper and not the top layer; and the disk's and the scratch's disks | -| `gpt_over_scratch_with_untrusted_lowers` | The disk over the scratch, in the untrusted-lowers mode | The root, which copies up a file's data too, and refuses the rename | -| `gpt_over_ram` | The disk over a tmpfs, with a bound that the first partition meets exactly | The root over the tmpfs, with no scratch mounted, and no disk but the GPT disk's | -| `gpt_over_ram_with_untrusted_lowers` | The disk over a tmpfs, in the untrusted-lowers mode | The root over the tmpfs, which copies up a file's data too, and refuses the rename | -| `full_scratch_refuses_writes` | The image over the scratch | A file that fills the scratch, through the root, until a write fails with `ENOSPC`, and a write that succeeds once the file is gone | -| `full_ram_upper_refuses_writes` | The image over a tmpfs | A file that fills the tmpfs, half of the guest's memory, until a write fails with `ENOSPC`, and a write that succeeds once the file is gone | -| `forbidden_journal_fails_the_assembly` | The image over the scratch, whose journal the launch forbids | The assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | -| `other_identity_fails_the_assembly` | The image, of which the launch expects another identity | The assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | -| `other_disk_guid_fails_the_assembly` | The GPT disk, of which the launch expects another GUID | The assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | -| `first_partition_over_its_bound_fails_the_assembly` | The GPT disk, with a bound one byte smaller than its first partition | The assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | -| `corrupt_layer_fails_the_assembly` | The GPT disk, whose top layer is corrupt and asks the kernel to panic on an error | The top layer's mount, which the kernel refuses without a panic, the assembly's failure, which the launch reports, and a VM that ends with 14 before the guest mounts the scratch | -| `cyclic_journal_is_not_loaded` | The GPT disk over a tmpfs, in the untrusted-lowers mode, whose top layer has a cyclic journal but no `needs_recovery` bit | Assembly, reads, copy-up, and teardown complete without scanning the journal | -| `recovery_required_layer_fails_the_assembly` | The same cyclic journal with the top layer's `needs_recovery` bit set | The assembly refuses recovery before mounting anything, despite `noload`, and the VM ends cleanly with 14 | -| `invalid_configuration_fails_its_stage` | A relative runtime directory | The configuration's failure, which the launch reports, and a VM that ends with 11 | -| `busy_root_fails_the_teardown` | The image over the scratch, which the `hold` probe keeps open | The teardown's failure, which the stop reports, and a VM that ends with 15, leaving the scratch mounted | -| `erofs_over_scratch_maps_the_table` | The image over the scratch, with the mapping table, from whose export the `map-checks` probe also maps | The export, mounted once, read-write, `nosuid`, and `nodev`, below a `hostfs` of mode 0700 that root owns; the table's mappings, mounted in the plan's order, by the depth of their targets, then by their targets, each with its access, `nosuid`, and `nodev`, on targets that the worker created on the overlay, but none inside another mapping; a hidden file that its directory does not list, and whose lookup the host refuses; and writes that reach the host's writable sources alone, the one inside a read-only mapping included. Through mappers of the adapter's own, from the export: a link or a magic link at each component, the wrong kinds, a target that would be created inside another mapping, and the runtime's, the container root's, and the adapter's own reserved targets refused, and a read-only mapping of a directory that the guest may write through the export, which is never writable. A teardown that unmounts the mappings, the export, then the scratch, cleanly | -| `erofs_over_ram_maps_the_table` | The image over a tmpfs, with the mapping table | The table, as over the scratch | -| `gpt_over_scratch_maps_the_table` | The GPT disk over the scratch, with the mapping table | The table, whose targets the worker created with their parents, which the disk's layers do not hold | -| `gpt_over_ram_maps_the_table` | The disk over a tmpfs, with the mapping table | The table, as over the scratch | -| `a_failed_mapping_tears_the_vm_down` | The image over the scratch, with the mapping table and `out` again at `/usr/lib/os-release`, a file of the image, which the plan orders fourth | The guest's refusal of the fourth mapping, a directory onto a file, which the launch reports with the three mappings that the guest applied; no more of the table, and no `Start`; a stop that the guest confirms once it tore the root down, a VM that ends with success and a complete teardown, an exit with 6, no runtime record, and the host's sources unchanged | -| `a_read_only_share_inside_a_writable_one_is_refused_before_launch` | The image over the scratch, with the mapping table and a read-only share inside `out` | A launch that fails before it spawns the VM, since the host could not keep the share read-only, and that leaves nothing of a launch in the store | -| `a_share_inside_a_reserved_tree_is_refused_before_launch` | The image over the scratch, with the mapping table, a tree `/srv/adapter` that the adapter reserves, and a share inside it | A launch that fails before it spawns the VM, and that leaves nothing of a launch in the store | -| `a_share_over_a_reserved_tree_is_refused_before_launch` | The image over the scratch, with the mapping table, the reserved tree, and a share at `/srv`, which would cover it | A launch that fails before it spawns the VM, and that leaves nothing of a launch in the store | -| `an_unauthorized_host_path_is_refused_before_launch` | The image over the scratch, with the mapping table and a share of a host directory beside the sources | A launch that fails before it spawns the VM, since the adapter authorizes the sources alone, and that leaves nothing of a launch in the store | -| `root_profiles_enter_the_root` | The image over the scratch, which the `entries` probe enters | Executions that start as PID 1 of their namespaces, with the root moved over `/`, only their standard streams, the leaked descriptor closed, no inheritable or ambient capability, and no new privileges, blocking no signal that their caller blocks, at the top of the root unless they name a working directory. A working directory that the execution names is entered, but none through a magic link, such as the leaked descriptor's, its own working directory's or root's, or any descriptor's, nor through a symbolic link to one: each fails the launch before the program runs. User ID 0 with each allowed capability alone holds it alone, and only its own attempt succeeds; with none, nothing does, nor for a numeric identity without an account. No execution writes `/proc/sys`, so none lowers the unprivileged port start, and a nested `chroot` stays in the root. With published listeners, user ID 0 with every other allowed capability can neither bind a privileged source port nor open a raw socket, so it cannot forge a listener announcement in either publication mode; without them, it may. The entry refuses the 51 capabilities outside its allowlist, the binding and raw-socket ones while listeners are published, capabilities for another user ID, and an identity without an account in existing-account mode. The root's own machine ID, then a generated one and the time status that the guest's time daemon publishes, read-only, in which a root workload finds no lock of the daemon's, which the runtime bound in the helper's mount namespace, and which leave the worker's namespace without mounts below the root. A teardown that unmounts the scratch cleanly | -| `unstaged_images_are_refused_before_spawn` | The image over the scratch, which the scenario does not stage | A launch that fails before it spawns the VM, since the store stages no image at its path, and that leaves no launch journal, VMM log, runtime record, or launch record; then, once the scenario stages the images, the root that the worker assembles, and a clean teardown | -| `a_changed_image_is_refused_before_spawn` | The GPT disk over the scratch, which the scenario writes in place after it stages it | A launch that fails before it spawns the VM, since the disk's modification time differs from its binding's, and that leaves nothing of a launch in the store; then, once the scenario stages the disk again, the root, and a clean teardown | -| `a_replaced_scratch_is_refused_before_spawn` | The image over the scratch, which another copy of the template replaces after the scenario stages it | A launch that fails before it spawns the VM, since the scratch image's identity differs from its binding's, and that leaves nothing of a launch in the store; then, once the scenario stages the images again, the root, and a clean teardown | -| `executions_run_in_the_root` | The image over the scratch, without a workload identity, in which `exec` runs executions that name their identities | Over one session, four executions as `nobody` that meet only while all of them run, beside one as root that writes more than a MiB to its standard output and a little to its standard error, each byte-exact, ending on its own with 0, and reusable. Over a later session, with new keys: a program that leaves a descendant in a session of its own, which the program's end, as PID 1 of its namespace, ends, so that no output is lost; `nobody`'s IDs and groups, with empty permitted, effective, and bounding sets; root with `CAP_CHOWN` alone in each of them; exactly the start's environment; the start's working directory; the entry's default hostname, `nvx-sandbox`, and PID 1; and a start without an identity, and one with `CAP_SYS_ADMIN`, which the guest refuses at stage 0 with `EINVAL` and `EPERM`, fenced and without output | -| `executions_run_as_the_workload_identity` | The image over the scratch, with the workload identity of `nobody` | Executions that name no identity, which run as `nobody`, without capabilities, with exactly the start's environment, and a start that names an identity, which the guest refuses at stage 0 with `EINVAL` | -| `accountless_workload_identity_fails_its_stage` | The image over the scratch, with a workload identity whose account the image lacks | The identity stage's failure once the guest assembled the root, which the launch reports, and a VM that ends with 16, which unmounted the scratch | -| `executions_take_the_settings_of_a_proxy_only_network` | The image over the scratch, in a proxy-only network, whose proxy is the test proxy on host loopback | A root configuration whose listeners are unpublished; executions that find the proxy's URL, `http://192.0.2.1:PORT`, at the test proxy's port, in `http_proxy`, `https_proxy`, `HTTP_PROXY`, and `HTTPS_PROXY`, after their own entries, but one that names `HTTPS_PROXY` itself, which keeps exactly its own environment; root with `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW`, which hold them; and, on Linux, a batch with a start that the framing carries alone but not with the proxy's URL, which `exec` refuses before it attaches, running none of its executions | -| `published_listeners_refuse_binding_and_raw_sockets` | The image over the scratch, in a network with ingress allowed, through which OpenVMM publishes the guest's listeners | A root configuration whose listeners are published; root with `CAP_NET_BIND_SERVICE`, and with `CAP_NET_RAW`, which the guest refuses at stage 0 with `EPERM`, fenced and without output, and with `CAP_CHOWN`, which holds it; and an execution with exactly its start's environment, without a proxy | -| `a_denied_network_lets_no_probe_through` | The image over the scratch, in a network that denies everything, without a proxy | After the positive control, no network device, and an execution that reaches neither the proxy nor any destination, none of whose traffic arrives | -| `an_allow_rule_lets_only_its_destination_through` | The image over the scratch, in a network that denies egress but for one rule, TCP to the gateway at the port of a service of host loopback, with outbound host-loopback access | After the positive control, an execution that reaches that service alone: not the proxy, nor the proxy's port over UDP, the host's own address, or host IPv6 loopback | -| `a_proxy_only_network_lets_only_its_proxy_through` | The image over the scratch, in a proxy-only network, whose proxy is the test proxy | After the positive control, an execution that reaches the test proxy through the URL in its environment, and none of the destinations, whose traffic never arrives | -| `outbound_host_loopback_publishes_no_listener` | The image over the scratch, in a network with egress allowed and outbound host-loopback access | After the positive control, an execution that reaches the proxy and every destination, and a listener of the guest that the host never reaches at the guest's port of host loopback while it listens, since OpenVMM does not publish it | -| `allowed_host_loopback_publishes_the_listeners` | The image over the scratch, in a network with host-loopback access in both directions, and egress denied | After the positive control, an execution that reaches neither the proxy nor any destination, since egress governs the guest's own connections, and a listener of the guest that the host reaches at the guest's port of host loopback, and exchanges a line each way with | -| `a_forward_reaches_its_guest_port` | The image over the scratch, in that network, with a forward of a port of host loopback to a guest port | A listener of the guest at that port, which the host reaches through the forward, and exchanges a line each way with | - -A row names the root, the image, the GPT disk's top-layer variant, the -identity that the launch expects of the image, the upper, the runtime -directory, the overlay mode, the scratch's journal policy, the bound of the -first partition, the sandbox's workload identity, the VM's network, which the -launch reads from a file that the scenario writes, and the host paths that the -launch shares, and what the scenario expects: a root that the guest assembles, -with a check that probes it or runs executions in it, a teardown that fails -after a check, the stage that fails, with its problem, a mapping of the table -that the guest fails, after how many, with its problem, shares that the host -program refuses before it spawns the VM, with the problem, or a launch that the -host program refuses before it spawns the VM, with the change that the scenario -makes after it stages the images, and the problem. Each scenario that launches a -VM checks the launch's record, with the bindings that the store stages, which -the launch checked, and the number of mappings that the root configuration -announced, and, over a scratch, whether the VM left the scratch's journal to -recover, which tells whether the guest unmounted the scratch. - -A scenario with the mapping table writes its sources in its directory, below -`sources`: a directory `dir`, with the files `marker` and `secret` and an empty -directory `inner`; a file `file.txt`; and a directory `out`, with a file -`marker` and an empty directory `inner`. The launch shares `dir` read-only at -`/srv/maps/dir`, `dir/inner` read-write inside it, `file.txt` read-only at -`/srv/maps/file.txt`, and `out` read-write at `/srv/maps/out`, hides -`dir/secret` with `--deny`, and authorizes `sources` alone with -`--share-root`. The plan exports `dir`, `file.txt`, and `out` as the export's -children `0`, `1`, and `2`. Once the `map` probe ran, the sources hold what -the scenario wrote, and the file `written` in `dir/inner` and in `out` alone; -the scenarios of a failed mapping and of refused shares leave the sources as -the scenario wrote them. - -The scenarios with a network give the guest the address `192.0.2.2/24`, in a -range reserved for documentation, which no host's own address shares, so that -the guest reaches the host's address through its gateway, `192.0.2.1`. Before -the launch, the test binds the host's side of the network, whose ports the -network names: a test proxy on host IPv4 loopback, which the guest reaches at -the same port of its gateway, and the sockets of the probes' destinations, -another port of host loopback over TCP, the proxy's port over UDP, and the -proxy's port over TCP at the host's own address, if it has one, and on host -IPv6 loopback, if it has that, which the guest reaches at its gateway's IPv6 -address. It also draws the port of a listener of the guest, which no socket -holds for TCP or UDP, on every host address or on loopback alone, as OpenVMM -needs to publish the listener on host loopback, and the port of a forward. It -draws each port at random from the dynamic range, since Windows excludes ports -per protocol, in blocks. Scenarios run at the same time, so each holds the -ports that it drew until it ends, or for good if it fails, and draws none that -another holds: OpenVMM binds the ports of a listener and of a forward only once -it runs, and again for the scenario's VM after the positive control's. - -Each VM with a network first has its guest's network devices checked, of which -OpenVMM attaches none for a network that lets nothing through, and otherwise -one, which `/init` configures, and that OpenVMM asks `/init` on the kernel -command line to publish the guest's listeners exactly where the root -configuration says that they are published. Then, as NVX's proxy-only scenario -does, an execution requests a URL through the test proxy with `wget`, while the -host serves the proxy, and tries each destination with `nc`, and the host -checks what arrived at each of its sockets. A listener of the guest, `nc` in -the image, greets the host, which tries to reach it until the listener's -execution ends. - -A scenario whose network stops traffic begins, as NVX's proxy-only scenario -does, with the positive control: a VM of an open network, with egress allowed -and host-loopback access in both directions, over the same host's side. There -the execution reaches the test proxy, which it names itself, and every -destination, whose traffic arrives at the host's socket, and the host reaches -a listener of the guest at the guest's port of host loopback. Once that VM -ended, the host discards what is left at its sockets, and the scenario's own VM -runs the same checks, so that what does not arrive there shows the network's -policy, not a broken probe. - -The scenarios use the fixtures of NVX's own layered scenarios, from NVX's -artifacts: the Ubuntu EROFS image, `ubuntu-distro.erofs`, its manifest, which -holds its UUID, and the scratch template, `ubuntu-smoke-scratch.ext4`, an -empty ext4 file system, of which each scenario boots a copy. Packages ship the -template beside the image, so `nvx-rs download` installs both into `build/`, -and the guest-artifact build makes it with the image; -`python3 scripts/nvx.py build-scratch-template` formats it on Linux, with the -parameters that the packages' template has -([`build-scratch-template`](../../doc/usage.md#build-scratch-template)). - -A scenario of a GPT disk writes the disk from the template first: a protective -MBR, and the primary and backup tables of 128 entries, with the first -partition, 4 KiB of bytes that are not UTF-8, then padding; the base layer, in -partition 2, a copy of the template; and the top layer, in partition 3, a small -file system in the ext2 format, which the kernel's ext4 driver mounts, with a -file and a directory. The test writes the disk, and the top layer itself, with -`tests/common/gpt.rs`, since `mkfs.ext4` is not available everywhere that the -tests run, and checks the disk with `nvx::gpt` before it launches the VM. For -`corrupt_layer_fails_the_assembly`, -it corrupts the top layer: the inode of the layer's top directory loses its -mode, an error that the kernel finds when it mounts the layer, and the error -policy of its superblock asks the kernel to panic on an error, which the -assembly's `errors=remount-ro` overrides. - -The journal scenarios append an internal journal to the top layer, mapped by -one extent in reserved inode 8. Its nonempty ring holds empty revoke records -of one sequence, with no commit, so a kernel journal scan would cycle forever. -With the file system's `needs_recovery` bit clear, `noload` bypasses that scan -and the root remains usable. With that bit set, the assembly refuses the -layer before any mount: `noload` is not permission to ignore required recovery. - -### Template scenarios - -`tests/vm_template.rs` creates [templates](#templates) on a real hypervisor, in -the host-only configuration, through `runtime-adapter-host create`, and runs -`refresh`, `list`, `inspect`, and `remove` on them. Each test -stages its placeholder, a copy of NVX's Ubuntu EROFS image with its manifest, -in a directory that only its user may change, as a platform stages its trusted -artifacts, and creates a template of it in a template store of its own: - -| Test | Template | Expects | -| --- | --- | --- | -| `a_template_without_a_network_has_no_nic` | 2 processors, 1 of them online at boot, 256 MiB of RAM with a capacity of 512 MiB, a 32 MiB scratch, and no network | A captured template without a NIC, whose key records its 1 processor online at boot, which `maxcpus=1` limits | -| `a_template_whose_processors_are_all_online_at_boot_limits_none` | 2 processors, both online at boot | A captured template whose key records no processors online at boot, without `maxcpus=` | -| `a_network_that_lets_nothing_through_has_no_nic` | A network that denies egress, ingress, and host-loopback access, without a proxy | A captured template without a NIC | -| `the_networks_of_every_policy_at_one_guest_s_address_share_one_template` | Host-loopback access both ways, with a forward of port 18080 of host loopback to the guest's port 8080; then, at the same guest's address, each network that lets something through: egress allowed, with outbound host-loopback access or without, one TCP rule to the gateway at port 8080, with outbound host-loopback access, a proxy at port 3128 of host loopback, ingress allowed, host-loopback access both ways, and that forward | A captured template with a NIC, under the complete deny-all policy, whose key's command line names the NIC's addresses, but neither a DNS server nor a publication of the guest's listeners; then each creation `existing`, with the same generation, key, and class, without a capture | -| `a_class_that_no_capture_takes_is_refused_before_the_store` | 3 processors, then 2 processors with 4 online at boot | NVX's refusal of each class, with status 1, before the store exists | -| `a_refresh_captures_and_retires_the_generations_that_it_supersedes` | A creation, then two refreshes, the second while a clone leases the second generation, then a creation once the clone released it | `list` and `inspect` of the first generation; each refresh captures a new generation, although the store holds an eligible one, and its eviction retires the first generation, and leaves the leased second one, tombstoned, with its payload; the creation serves the third without a capture, and its eviction deletes the second | -| `a_removal_reports_busy_while_leases_keep_the_generation` | A creation, two clones' leases of its generation, a removal, a second creation, then a removal after each release | `list` and `inspect` with the leases; the removal tombstones the generation, which keeps its payload and takes no new lease, and reports busy, with status 7 and its leases; the second creation captures a new generation, and leaves the first to its leases; a removal reports busy while a lease remains, and deletes the generation once none does; a second removal finds it retired, one of a generation that the store never published fails, and a generation that no lease keeps goes at once | -| `a_refresh_refuses_a_class_that_no_capture_takes_before_the_store` | A refresh of 3 processors | NVX's refusal of the class, with status 1, before the store exists | - -Unless a test says otherwise, a template has 1 processor, online at boot, 256 -MiB of RAM, its capacity, and a 64 MiB scratch. A captured template is the -store's one generation, whose payload holds the record of its capture, its key, -and its snapshot, and whose key holds what the test asked for: the runtime -identity of the adapter's VMs, the backend, the -processors and those online at boot, none for a template whose processors are -all online at boot, the RAM and its capacity, the geometries of the placeholder -and of the scratch, the NIC, and a kernel command line of the platform tier's -tokens alone, with the processors, `maxcpus=` with those online at boot -exactly when they are fewer, and, exactly with a NIC, the NIC's addresses, but -neither a DNS server nor a publication of the guest's listeners, which OpenVMM -names to no guest under the NIC's deny-all policy, and which names neither the -placeholder's identity nor a root's tokens. Its class's -slot then holds no transaction and records no launch, and a second creation -prints `existing`, with the same generation, key, and class, without a capture: -the slot's log of the VMM stays as the capture left it. - -The scenarios of the operations lease generations to clones through NVX's -store, as a clone's start does, but launch nothing, and release the leases as -a clone's teardown does. They check what `list` and `inspect` print against -what the store records: each generation's key, class, state, eligibility at -the host's time, by its monotonic clock, since this host and boot captured it, -and leases, and, from `inspect`, the capture's storage generation, time -contract, and lengths; and what each eviction and removal printed against the -generations and payloads that the store then holds. - -### Clone scenarios - -`tests/vm_clone.rs` starts [clones](#clones) of templates on a real hypervisor, -in the host-only configuration, through `runtime-adapter-host start` and -`run`. Each test -captures a template of its own, of 1 processor and 256 MiB of RAM with a -capacity of 1 GiB, in a template store of its own, of a copy of NVX's Ubuntu -EROFS image that a directory of the platform holds, which the clones stage as -their image, or of a GPT disk 16 MiB longer than it that the platform holds -beside it, -without a NIC unless its row names a network, whose guest is at `192.0.2.2/24`, -as in the [template scenarios](#template-scenarios), -and starts its clones, each in a sandbox store of its own, over a copy of the -scratch template, which no one else may change either, or in RAM. The GPT disks -of a template's placeholder and of the clones, which the test writes with -`tests/common/gpt.rs`, as `tests/vm_layered.rs` writes its own, hold their -first partition, then layers, empty but for one disk's regular file `etc`, at -partitions that differ from one disk to another. The host directories that -clones export lie in the template's directory, beside the template store, in a -directory that the starts authorize: - -| Test | Clones | Expects | -| --- | --- | --- | -| `one_generation_serves_clones_of_each_upper_and_memory_target` | Two at once, over a scratch image at 256 MiB, and in RAM at 512 MiB with untrusted lowers, then one over a scratch image at 1 GiB | Each started, by the host's monotonic clock, with its own lease, which the store lists with the clone's sandbox, its memory target online, its upper, overlay mode, and image's contents, and the events of its activation in order; a stop releases each lease; the generation's snapshot unchanged, and its payload with nothing but the capture's files once the clones stopped | -| `one_generation_serves_clones_of_each_processor_count_from_its_boot_online_count_to_its_capacity` | Of a template of 4 processors, 1 of them online at boot: in turn, one without a processor target, one of 2, and one of 4; then targets of 3 and 8 | Each started, with the CPUs of its target online, CPU 0 alone by default, of the 4 possible ones, and the CPUs beyond the first that the time ABI's record of the restore names as those that the restore brought online, none by default, and the events of its activation in order; then for each of the others, status 8, with NVX's reason, and the record of a failure of the request, at the plan, without a VM, by the evidence of invalid targets, which quarantines nothing; the lease released, and the generation's snapshot unchanged | -| `a_generation_refuses_clones_of_fewer_processors_than_its_template_booted_with` | Of a template of 4 processors, 2 of them online at boot: a target of 1, then in turn one without a processor target, and one of 4 | Status 8, with NVX's reason, and the record of a failure of the request, at the plan, without a VM, by the evidence of invalid targets; then each started, with CPUs 0 and 1 online by default, which the restore brought none of online, and all 4 online for the target of 4, of which the restore brought CPUs 2 and 3 online | -| `one_generation_serves_clones_of_either_format_over_either_upper` | Of a template whose placeholder is a GPT disk of two layers, longer than the EROFS image, four at once, each image shorter than the placeholder: the EROFS image over a scratch image and in RAM, a GPT disk of one layer, of the EROFS image's length, in RAM, and one of three layers, as short as its layout allows, over a scratch image, with a bound that its first partition meets exactly; then the disk of one layer, with a bound one byte smaller than its first partition | Each started, with its upper and the overlay's options; the EROFS image's root, without a first partition; each GPT disk's layers, ext4 file systems mounted read-only once each, from the partitions that its own table names, stacked in order, and the bytes of its own first partition; the image's read-only disk at the placeholder's length, and the scratch device at the template's scratch length, each of which the guest reads to its end; for a clone in RAM, no file of its sandbox as long as the template's scratch, and on Linux `--restore-absent-scratch` and no `scratch` block device among its VMM's arguments, which name one over a scratch image; then status 8, and the record of a failure of the request, at the activation, once the VM exited, by the evidence of an unusable root, with the guest's status 54; nothing quarantined, and the lease released, with the records | -| `a_start_refuses_what_the_template_cannot_serve_before_any_vmm_starts` | An EROFS image and a GPT disk 4 KiB longer, and an EROFS image 100 bytes longer, than the placeholder, a scratch image of twice the template's scratch, memory targets of 2 GiB and of 320 MiB, unstaged images, and a generation that the store does not hold, then the generation once its key is gone | For those that NVX's plan refuses, status 8, with NVX's reason, and the record of a failure of the request, at the plan, without a VM, by the evidence of a length that the request supplied, of the image's geometry, which OpenVMM cannot attach, or of invalid targets, which quarantines nothing; for unstaged images, status 1, without a record; for the generation, status 10, with the record of a selection that names no template, which the store never published, without a lease; any lease released, and no VMM's log or record in the sandbox; for the generation without its key, status 8, and the record of a failure of the template, at the plan, by the evidence of a malformed generation, which quarantines the generation, so that the store refuses a later start's selection, with status 10, as `ineligible`, by the evidence of its tombstone, without a lease | -| `a_clone_whose_activation_fails_ends_before_it_is_ready` | A scratch image of zeros, a journal that the record forbids, and, through NVX's restore sequence, a ping, a start, or the start of an execution before the `Activate` | Status 8, no `Ready`, and the record of a failure, at the activation, once the VM exited: of the request, by the evidence of an unusable root, with the guest's status 54, or, before the `Activate`, of an unknown cause, by the protocol's, with 51, which the sandbox records until the release; nothing quarantined, and the lease released, with the records | -| `a_clone_refuses_a_scratch_image_whose_upper_an_earlier_clone_used` | Over a scratch image, in whose root the `contents` probe writes a file, then over a copy of that scratch image once the first clone stopped | The first started; then status 8, since the copy's upper is not fresh, and the record of a failure of the request, at the activation, once the VM exited, by the evidence of an unusable root, with the guest's status 54; nothing quarantined, and the lease released, with the records | -| `clones_at_once_keep_their_own_uppers_and_machine_ids` | Three at once: over a scratch image and in RAM, each with a generated machine ID, and in RAM with the image's own; two executions in each, as user ID 0, that write a marker in the root, and try to write `/etc/machine-id` | In every execution, the hostname `runtime-adapter`, and no marker of another clone; a generated machine ID that differs between the clones, the same in each execution of a clone, held by one bind that is read-only, `nosuid`, and `nodev`, and that the write cannot change; with the image's own machine ID, none, since the image has none, no bind, and in the later execution, the ID that the first wrote; then each stop releases its lease | -| `a_clone_whose_view_cannot_be_set_ends_before_it_is_ready` | Of a template of the EROFS image, a GPT disk whose layer holds a regular file `etc`, with the image's own machine ID, then with a generated one | The first started, with the disk's root; then status 8, since no bind can hold the machine ID below a file, and the record of a failure of the request, at the activation, once the VM exited, by the evidence of invalid settings, with the guest's status 55; nothing quarantined, and the lease released, with the records | -| `a_clone_runs_the_executions_that_name_their_identity_once_it_is_ready` | Over a scratch image, with the legacy workload option of nobody, which no activation record encodes | An execution that names nobody's identity, which runs as nobody, one that names none refused with `EINVAL`, and the first execution after the clone's start | -| `openvmm_reads_none_of_the_host_s_input_until_the_repair_acknowledges_the_restore` | Of a template whose clones wait 8 seconds before their repair, in RAM, which the host starts through NVX's public API, with the host program's clone order, over transports that watch their attachments, each of which they write a byte at a time; then a reattach through NVX's public API | The template's key, which names the delay; the attachment written within half the delay of the launch, the broker's first answer, and the attachment of the host, only after the delay; on Linux, `SIOCOUTQ` at the control socket risen by the same step with each byte that the host wrote, then as the write left it at every record before then, and zero once the broker answered, while a listener of the test's own process sees it fall once it read the first byte of such an attachment, and on Windows no record, since a named pipe tells its writer nothing; then the reattach's attachment answered within a quarter of the delay, and on Linux read whole by then; the guest's `repair-delay` of 8 seconds, and the events of its activation in order; and a stop that releases the lease | -| `a_generation_is_eligible_by_the_clock_of_its_capture` | At the host's time, then with clock fixtures: UTC a day behind, another boot an hour after the capture by UTC, and the restore window passed by each clock, or UTC before the capture | The host's monotonic clock on the boot of the capture, whatever UTC says, UTC after a reboot, with the fixture's downtime, and, outside the window, status 10, with the record of the store's refusal of the selection, `expired` once the window passed, and `ineligible`, by the evidence `early`, before the capture, without a lease | -| `a_clone_reads_no_more_of_its_disks_than_its_root_needs` | Over a scratch image, and in RAM | Less than an eighth of each disk read once the clone is ready, by the guest's statistics, and each disk's whole length once the guest read it to its end; on the host, the staging's count of the bytes that it read at least the image's length, which it hashed, the start's less than an eighth of it, and the VMM's less than an eighth of the disks whose files back the clone's, those of the image and of a scratch image, once the clone is ready; then, once the guest read each disk to its end, the VMM's count raised by at least what the guest asked of those files on Windows, and by less than an eighth of it on Linux, whose `rchar` misses OpenVMM's reads through io_uring | -| `a_clone_s_time_settles_after_it_is_ready` | In RAM | The time daemon's restore check `ok`, after the restore's generation, without a violation, after a downtime that the host's monotonic clock measured, and a clone that still serves | -| `each_command_reattaches_to_the_clone_from_a_new_process` | Over a scratch image, then `reattach`, `probe`, `exec`, and `reattach` again, each from a new process | The launch's records, committed, with the clone's VM and no failure; each command `live`, with the clone's VMM, in a new session of the same broker, in a later epoch; the events of the activation as they were; an execution as nobody; and a stop that releases the lease, with the records | -| `a_host_program_of_another_runtime_ends_the_clone_through_its_verified_identity` | In RAM, then a `reattach` with another guest agent's build | Status 8, `reattached=failed`, without a session, and the record of a failure of the host, at the reattach, by the evidence of the runtime's mismatch, once NVX terminated the VMM, which quarantines nothing; the lease released, with the records; then another clone that starts and stops | -| `a_clone_whose_vm_exits_once_a_stop_found_it_fails_before_its_lease_is_released` | In RAM, then a `stop` that the step hook pauses at `found`, while the test terminates the clone's VMM | Status 8, `reattached=live` and `stopped=exited`, and the record of a failure of an unknown cause, at the run, by the evidence of the clone's lost status, without a status of the guest's, once the VM exited, which quarantines nothing; the lease released, with the records; then another clone that starts and stops | -| `a_tombstone_leaves_the_running_clone_of_its_generation_to_its_lease` | Over a scratch image, then a tombstone of its generation, through NVX's store | A start whose selection the store refuses, with status 10 and the record of an `ineligible` generation, by the evidence of its tombstone, without a lease; the running clone, which reattaches, probes, runs an execution, and stops, which releases its lease; and the generation's snapshot unchanged | -| `the_lease_of_a_start_that_ended_before_its_record_is_released_through_its_template_store` | In RAM, in a sandbox that NVX's store leased a generation to, as a start that ended before it recorded the clone leaves it: first moved, with another sandbox at its lease's path, which holds a lease of its own, then none; then in place, without a record of a clone, then with the record of an earlier clone, of another template | For the moved sandbox, a `stop` and a `reattach` that refuse it, since the sandbox at its lease's path does not hold its lease, and leave that sandbox's lease; in place, a stop that finds the clone `ended` at `planned`, and keeps the lease, whose template store no record names; a stop with another template store, which the release refuses; a `reattach` or a `stop` with `--template` naming the lease's own, which releases it; and a start and a stop in the sandbox after each | -| `a_clone_maps_an_aggregate_of_directories_and_files_beneath_its_root_before_its_sandbox_starts` | Over a scratch image, sharing directories and files of a host directory: a read-only directory, with a read-write directory inside it, at its target there, a read-only file, a read-write file, and a read-write directory, with a file of the first hidden, and of the last a directory hidden, but a file in it shown again | Started, with five mappings, and a record that names the export's aggregate shape and five mappings; on Linux, the VMM's `--mount-aggregate` and one `--mount-child` per directory and file, in order, the first directory read-write, each file with its access and `file`, then the hidden paths, the one shown again, and the one that the guest may write, and no `--mount`; the export mounted once, read-write, `nosuid`, and `nodev`, below a `hostfs` of mode 0700 that root owns; the mappings below the root, and nothing else, in the table's order, each with its access, `nosuid`, and `nodev`, showing what the host holds but for the hidden files, and the hidden directory, which lists the file shown again alone, with the missing targets created empty in the upper; the same from an execution as user ID 0; and writes through the writable mappings alone, which the host's directory holds, before and after the stop | -| `clones_at_once_see_only_their_own_exports` | Four at once: one with a read-write export of its own and a generated machine ID, two over the same read-only export, over a scratch image and in RAM, and one without an export | Each with an export started with one mapping, and a record that names its aggregate shape without a host plan identity, and on Linux, the VMM's `--mount-aggregate`, with one `--mount-child`, read-write or read-only, as the mapping needs; each export mounted read-write, even where its mapping is read-only; below the root, the mapping, beside the bind of a generated machine ID; each clone's own directory, with its access; without an export, no export in the record, no mapping, none of OpenVMM's options of an export on Linux, no export mounted, and nothing below the root; the `entries` probe, whose helper writes the root's machine ID, failing with `EROFS` where the generated one is bound; the host's directories with what the writable export took alone; then each stop releases its lease | -| `a_start_refuses_an_export_whose_mapped_objects_changed_before_any_vmm_starts` | Sharing a read-write directory, a directory inside it at a target of its own, and a read-only file; while the step hook pauses each start at `checked`, a directory replaces the file, then another directory the one inside the read-write directory | Status 8, with NVX's reason, and the record of a failure of the request, at the launch, without a VM, by the evidence of invalid mappings, which quarantines nothing; no VMM's log, launch journal, runtime record, or record of the clone's launch in the sandbox, and the lease released | -| `a_clone_whose_mapping_fails_ends_before_its_sandbox_starts` | Over a scratch image, sharing a host directory whose second mapping puts a directory onto a file of the image | Status 8, a `Ready`, but no start, and the record of a failure of an unknown cause, at the start, at the clone's `Ready`, by the evidence of the protocol, once the VM exited with the guest's success, or NVX terminated it, with the guest's report that the table failed after its first mapping; nothing quarantined, and the lease released, with the records; the host's directory unchanged | -| `a_template_without_a_nic_serves_clones_without_a_network` | Two at once, without a network, and with one that lets nothing through; then, once they stopped, one of each network with a NIC of the template scenarios | The first two started, with no network in their records, on Linux no argument of a network of their VMMs, no network device, and the image's own `/etc/resolv.conf`, an empty file of mode 0644, which no bind holds, and root may write; then for each of the others, status 8, with NVX's reason, and the record of a failure of the request, at the plan, without a VM, by the evidence of invalid settings, which quarantines nothing; the lease released | -| `one_template_serves_the_clones_of_every_network_policy_at_its_guest_s_address` | Of a template of host-loopback access both ways, with a forward: in turn, one of each network with a NIC, of egress allowed, with outbound host-loopback access or without, of one TCP rule to the gateway, with outbound host-loopback access, of a proxy, of ingress allowed, of host-loopback access both ways, and of that access with a forward; then one at the guest's address `192.0.2.3/24`, one of a network that lets nothing through, and one without a network | Each of the first started, with one network device, on Linux the VMM's `--restore-network-policy`, then the arguments of the clone's own policy, without `--net`, and a record that names the gateway's DNS server for the networks with egress allowed, the proxy's URL for the proxy-only one, and a publication of the guest's listeners for those with ingress or host-loopback access both ways, without a host policy identity; an execution's TCP connections to the gateway, at the ports of the host's listeners on host loopback, that reach the allow rule's alone, the proxy's alone, every one with egress and outbound host-loopback access, and none otherwise, as the host's listeners confirm; a listener of the guest, as nobody, that the host reaches at its port of host loopback, with a line each way, exactly with host-loopback access both ways, once the clone's announcer announced it, and through the forward's port; and executions as user ID 0 that ask for `CAP_NET_BIND_SERVICE` or `CAP_NET_RAW` refused with `EPERM` where the listeners are published, and started elsewhere, as one that asks for `CAP_CHOWN` everywhere; then for each of the others, status 8, with NVX's reason, and the record of a failure of the request, at the plan, without a VM, by the evidence of invalid settings, which quarantines nothing; the lease released | -| `a_forward_whose_port_another_socket_holds_fails_the_clone_as_the_host_s` | Of a template of a network with egress allowed, one of host-loopback access both ways, with a forward of a port of host loopback that a listener of the host holds; then, once the listener closed, one of the same network | Status 8, with OpenVMM's cause, that it cannot create the forward's socket, and the record of a failure of the host, at the restore, once the VMM exited before its restore-ready event, by the evidence of the forward's bind, which quarantines nothing; the lease released, with the records; then a clone that starts and stops | -| `a_clone_binds_the_resolver_of_its_policy_s_dns_server_read_only` | Of a template of a proxy-only network, two at once, of a network with egress allowed, with a generated machine ID, and with the image's own; then, once they stopped, a GPT disk whose layer holds a regular file `etc`, with the image's own machine ID | Each of the first two started, with a record that names the gateway's DNS server, one network device, and at `/etc/resolv.conf`, a file of mode 0444 that names the server, held by a bind that is read-only, `nosuid`, and `nodev`, after the bind of the generated machine ID, below the root, whose file system refuses a write as root, with nothing in the upper there; and a TCP connection to the server's DNS port, which the gateway's DNS proxy accepts without the host's resolver; then status 8, since no bind can hold the resolver configuration below a file, and the record of a failure of the request, at the activation, once the VM exited, by the evidence of invalid settings, with the guest's status 57; nothing quarantined, and the lease released, with the records | -| `a_clone_s_executions_find_the_proxy_of_its_policy` | Of a template of a network with egress allowed, one of a proxy-only network, with an execution as user ID 0 that runs `env`, then one that names its own `HTTPS_PROXY` | A record that names the proxy's URL, `http://192.0.2.1:PORT`, at the port of the proxy's listener, and no DNS server; the first execution with its own entry, then the URL in `http_proxy`, `https_proxy`, `HTTP_PROXY`, and `HTTPS_PROXY`; the second with exactly its own environment; one network device, and the image's own `/etc/resolv.conf`, which no bind holds | -| `a_clone_of_a_template_with_a_nic_maps_its_export_beside_its_resolver` | Of a template of a network with egress allowed, one with a generated machine ID, sharing a host directory read-write | Started, with one mapping, and a record that names the export and the gateway's DNS server; on Linux, the VMM's `--mount-aggregate`, with one `--mount-child`, read-write, and its `--restore-network-policy`, then the arguments of the clone's policy; below the root, the binds of the machine ID and of the resolver configuration, then the export's mapping; the export's directory, and a write through it, which the host's directory holds after the stop | -| `a_selection_that_names_no_template_leases_nothing` | A start that names a template store that does not exist, then, once `remove` retired the generation, which no lease kept, a start of it | Status 10, with the record of a selection that names no template, by the evidence `no-store`, then `retired`, without a lease or a VM; and a store that indexes no generation | -| `a_run_runs_one_execution_in_its_clone_then_tears_the_clone_down` | A run over a scratch image, whose execution names nobody's identity, its environment, and its working directory, and ends with status 3, then a run in RAM, whose execution names no identity | Each with status 0, in one command: the preflight's `staged`, the clone started as a start starts one, the execution, and a stop that releases the lease, without a VM; the first execution's output, of the Ubuntu image's root, as nobody, in its working directory, with its environment, and its status; the second refused with `EINVAL`; no lease left, and the payload with nothing but the capture's files | -| `a_run_whose_clone_does_not_start_runs_nothing` | A run that names a template store that does not exist, one with a clock fixture past the restore window, and one over a scratch image of zeros | Once the preflight staged the images, status 10, with the record of a selection that names no template, by the evidence `no-store`, then of an `expired` generation, without a lease or a VM; then status 8, with the record of a failure of the request at the activation, once the VM exited, the lease released, and no execution or stop | -| `a_run_in_a_template_of_a_network_finds_its_policy_s_proxy` | Of a template of a proxy-only network, a run of its network, whose execution, as user ID 0, runs `env` with an entry of its own, then a run of a network that lets nothing through | The first with status 0, its clone started and stopped as a run's, and the execution's own entry, then the proxy's URL, `http://192.0.2.1:PORT`, at the port of the proxy's listener, in `http_proxy`, `https_proxy`, `HTTP_PROXY`, and `HTTPS_PROXY`; then, once the preflight staged the images, status 8, with NVX's reason, and the record of a failure of the request, at the plan, without a VM, by the evidence of invalid settings, which quarantines nothing, the lease released, and no execution or stop | -| `a_run_s_execution_finds_its_shares_beneath_the_root` | A run over a scratch image, sharing a host directory read-write, whose execution, as user ID 0, reads a file of the directory and writes another | Status 0, its clone started with one mapping and stopped as a run's; the file's contents in the execution's output; and the written file in the host's directory once the run tore the clone down | -| `a_run_whose_clone_s_vm_exits_during_its_execution_ends_with_the_clone_s_failure` | A run over a scratch image, sharing a host directory read-write, whose execution, as user ID 0, writes a file there, then waits, while the test terminates the clone's VMM | Status 8, its clone started with one mapping, the diagnostic of the executions' failure, `stopped=exited`, and the record of a failure of an unknown cause, at the run, by the evidence of the clone's lost status, without a status of the guest's, once the VM exited, which quarantines nothing, and no execution; the lease released, with the records; then another clone that starts and stops | - -The scenarios' networks let nothing through, or allow egress, with outbound -host-loopback access or without, or one rule that allows TCP to the gateway at -the port of a listener of host loopback, with outbound host-loopback access, or -have a proxy at the port of another, or allow ingress, or host-loopback access -both ways, with a forward or without. The tests bind those listeners, and a -third, which no network names, on host IPv4 loopback, at ports that the host -chooses, and hold them until they end. The forward's port of host loopback, and -the guest's port of the listener that the scenarios run in the guest, to which -it forwards, are ports that no socket of the host held when the tests drew -them. A -clone's start names its network by a file that the test writes in the -template's directory. - -Two of the scenarios check from the host what the guest cannot tell it, and -each backend observes part of it alone: - -- That OpenVMM reads none of the host's input until the clone's repair - acknowledges its restore, which NVX's restore sequence relies on, and which - the guest's `activation` probe cannot show, since its worker serves the - session only once its repair returned, so input that OpenVMM let through - early would wait in the guest. On Linux, `SIOCOUTQ` of the host's end of the - control socket shows that the attachment waited there, unread, until the - repair could acknowledge the restore. `SIOCOUTQ` charges each buffer of the - socket's writes until the reader read it whole, and a read of part of a - buffer leaves it as it was, so the host writes the attachment a byte at a - time, a buffer for each byte, whose charge falls as soon as OpenVMM reads any - byte. The kernel charges the same for each byte's buffer, so the charge rises - by the same step with each byte that the host writes, while OpenVMM reads - none. A named pipe tells its writer nothing of what its reader has not read, - so on Windows the check is the timing alone: the broker, which answers a host - that it authenticated at once, answered only after the delay. -- That a clone's restore reads no large content of its disks. The host - program's own count of the bytes that it read, and the VMM's, which - `nvx::host::process::Process::bytes_read` reads through its process, count - every read call, the cache's included, and on Linux those of the children - that the process reaped, but no file that the process maps, so they leave - out the memory of the snapshot, which OpenVMM maps copy-on-write. - On Windows, OpenVMM reads the clone's disks with `ReadFile`, which - `ReadTransferCount` counts, so the guest's read of each disk to its end - raises the VMM's count by at least what it asked of the disks' files. On - Linux, OpenVMM reads them through io_uring, which `rchar` does not count, so - the VMM's count covers its read calls alone, as a hash or a copy of an image - through them would show, and not its reads of the disks: the scenario checks - that the guest's read of each disk to its end raises it by less than an - eighth of what the guest asked. - -A test that fails keeps its template's directory, with the template store and -the platform's image, as long as it keeps the directory of a clone that leases -the template, whose VM might still run. - -### Stage scenarios - -The template and clone stage's scenarios run as one suite, which -[`scenarios.txt`](scenarios.txt) names: each line names a test target of -`tests/`, then one of its tests, optionally followed by `linux` or `windows`. -Unqualified entries run on both platforms; qualified entries run only on the -named host platform. Duplicate target/test pairs, even with different -qualifiers, and unknown qualifiers are errors. CI's runtime adapter step runs -the suite on each backend, KVM, MSHV, and WHP, with -`python3 scripts/nvx.py test-runtime-adapter --backend BACKEND`, in the -host-only configuration: it runs each target's scenarios by name, the VM -scenarios, which are ignored, with the others, and checks that each of them ran -and passed. It fails once a line names a test that its target does not have, a -target that a line names has a test that no line names, the library or another -target has an ignored test, or a named test does not run and pass, so that a -scenario that is renamed, added, or gone fails it rather than drop out of the -suite. Inventory checks, execution and scenario counts all use the same -platform-selected entries. With `--check-only`, it compiles and lists the -tests to check the names, but runs no scenarios and needs no guest artifacts. - -The suite holds each operation's outcomes, from `create` to `run`, without a -hypervisor and in VMs, and every scenario of the stage's VMs: launches of -layered roots, captures of templates, and the starts and runs of their clones. - -| Target | Scenario | Proves | -| --- | --- | --- | -| `template_capture` | `a_capture_takes_its_class_and_inputs_from_the_request` | A capture's request: the plan of the placeholder over RAM, with the request's network, the placeholder's binding, the policy revision, the artifacts, and the class, which the network names, and the delay of a template's clones' repair, as its invariant setting | -| `template_capture` | `the_host_program_refuses_a_repair_delay_that_it_cannot_read` | `create` and `refresh` refuse a value of `RUNTIME_ADAPTER_REPAIR_DELAY` that sets no delay with status 2 before they create anything | -| `template_capture` | `the_host_program_refuses_a_template_before_it_creates_anything` | `create` refuses a network that the plan refuses, or that it cannot read, with status 1 before it needs NVX's artifacts or creates the store | -| `template_operations` | `a_refresh_refuses_a_template_before_it_creates_anything` | `refresh` refuses a network that the plan refuses with status 1 before it needs NVX's artifacts or creates the store, as `create` does | -| `template_operations` | `an_empty_template_store_lists_no_generation` | `list` of a store without generations prints none, and `inspect` and `remove` of a generation that it never published fail with status 1, and tombstone nothing | -| `template_operations` | `the_host_program_refuses_command_lines_of_the_operations` | The command lines of `refresh`, `list`, `inspect`, and `remove` that the host program refuses, each with status 2 | -| `template_operations` | `the_operations_open_a_template_store_and_create_none` | `list`, `inspect`, and `remove` of a directory that does not exist, or that holds no template store, fail with status 1, and create nothing | -| `clone_start` | `a_clone_s_plan_request_and_record_carry_its_network` | A clone's network reaches the adapter's plan, NVX's request of the clone, and the clone's record, from which an execution finds the URL of the network's proxy | -| `clone_start` | `a_clone_s_record_names_its_template_store_and_generation_beside_its_plan` | A clone's record names its template store and its generation, which a stop reads back, beside the plan that the other commands read | -| `clone_start` | `a_clone_s_request_takes_the_staged_bindings_and_the_guest_s_defaults` | NVX's request of a clone takes the staged bindings, the processor and memory targets, the guest's defaults or the request's settings, and the export of the request's shares, and refuses images that the store does not stage | -| `clone_start` | `a_host_s_time_keeps_its_text_form` | A host's time keeps the clock fixture's text form, on both clocks, and the texts that are none are refused | -| `clone_start` | `a_run_refuses_a_network_or_shares_that_the_plan_refuses_before_it_stages_anything` | A run refuses a network that it cannot read, or that the plan refuses, and shares that it cannot export, with status 1 before it stages anything or creates the sandbox's store | -| `clone_start` | `a_run_refuses_a_selection_that_names_no_template_once_its_preflight_staged_the_images` | A run refuses an image that the plan refuses with status 1 before it stages anything, and a selection that names no template with status 10 and the refusal's record once its preflight staged the images, without a lease | -| `clone_start` | `a_start_refuses_a_generation_that_the_template_store_does_not_hold_before_anything_else` | A start whose template store does not exist, or does not hold the generation, ends with status 10 and the refusal's record, and one whose image the plan refuses with status 1, before it needs NVX's artifacts or creates the sandbox's store | -| `clone_start` | `a_start_refuses_a_network_that_the_plan_refuses_before_anything_else` | A start refuses a network that it cannot read, or that the plan refuses, with status 1 before its image, its shares, and the template store | -| `clone_start` | `a_start_refuses_shares_that_it_cannot_export_before_anything_else` | A start refuses shares that it cannot export with status 1 before it reads the template store or creates the sandbox's store | -| `clone_start` | `the_host_program_refuses_options_that_do_not_make_a_run` | The command lines of `run` that the host program refuses, its execution's included, each with status 2, and a clock fixture that is not a host's time | -| `clone_start` | `the_host_program_refuses_options_that_do_not_make_a_start` | The command lines of `start` that the host program refuses, each with status 2, and a clock fixture that is not a host's time | -| `host_session` | `a_clone_s_order_ends_its_table_at_a_mapping_that_the_guest_fails_and_stops_the_guest` | A clone's order sends no more of its table, nor `Start`, once the guest fails a mapping, and asks the guest to stop | -| `host_session` | `a_clone_s_order_maps_its_export_s_children_by_their_indices` | A clone's order sends its table as a cold launch's, whose sources name each path by the index of the export's child that holds it | -| `host_session` | `a_clone_s_order_sends_its_record_first_then_starts_its_sandbox_as_after_a_cold_boot` | A clone's order sends its activation record first, returns the activation that `Ready` reports, then sends the table in parts and `Start`, and fails on a refusal, a deadline, another build, or an answer that does not decode | -| `host_session` | `a_clone_s_session_that_ends_after_its_bootstrap_is_lost_and_the_next_takes_the_same_record` | A clone's session that a reset, the end of its stream, or its transport ended is lost, and the next session sends the same record first | -| `host_session` | `a_mapping_that_the_guest_fails_ends_the_table_and_the_host_stops_the_guest` | A mapping that the guest fails, at the first, a middle, or the last part, ends the table without `Start`, and the host stops the guest and learns how the stop ended | -| `host_session` | `a_receive_that_times_out_keeps_the_partial_message` | A receive that times out inside a message keeps what arrived, and a later receive returns the whole message | -| `host_session` | `a_reset_or_a_disconnect_in_the_middle_of_a_message_drops_it` | A reset or a disconnect in the middle of a message drops the message, in the next epoch or with a new instance | -| `host_session` | `a_send_that_fails_or_times_out_ends_the_client_with_what_it_holds` | A send that times out after part of a record, or whose transport fails, ends the client with the message that it held | -| `host_session` | `a_stop_skips_the_guest_s_late_messages_about_the_executions_that_the_driver_gave_up_on` | A stop skips the guest's late messages about the executions that the driver gave up on, before and after a teardown that failed, and no message about an execution that it did not start | -| `host_session` | `answers_whose_bodies_do_not_decode_end_the_client` | Answers whose bodies do not decode end the client | -| `host_session` | `only_a_session_that_ended_is_lost_to_a_clone_s_order` | A clone's order takes only a session that ended for lost, and its other failures for final | -| `host_session` | `requests_check_their_answers_and_end_the_client_when_they_fail` | Requests check the kind of their answers, and end the client when they fail | -| `host_session` | `the_client_attaches_and_exchanges_messages_that_span_records` | The client attaches with the launch capability, and exchanges messages that span records both ways | -| `host_session` | `the_client_configures_a_layered_root_probes_it_and_learns_how_its_teardown_ended` | The client configures a layered root, probes it, and learns how its teardown ended once it stopped | -| `host_session` | `the_client_fails_on_records_that_the_protocol_does_not_allow` | Sequence, epoch, and instance errors, stale and identity-less resets, record lengths, bootstrap identities, ERROR payloads, and header faults end the client | -| `host_session` | `the_client_reassembles_messages_from_partial_and_shared_records` | Records that end a message and start the next, and records whose bytes arrive in pieces down to one byte, give back whole messages | -| `host_session` | `the_client_refuses_messages_that_its_framing_does_not_allow` | The client refuses malformed headers before their body arrives, messages that only the host program sends, and a message beyond the framing's limit, which it does not send | -| `host_session` | `the_client_returns_the_stage_of_the_root_that_failed` | The client returns the stage of the root that failed, with its problem | -| `host_session` | `the_client_waits_for_the_guest_and_reports_refusals_while_attaching` | The client waits for the guest while it attaches, until its deadline, and reports a refusal, in an ERROR or by the end of the stream | -| `host_session` | `the_driver_ends_its_run_on_a_message_that_breaks_the_rules` | A message about an execution that the driver did not start, about none, that does not decode, or that breaks the engine's rules, and the end of the stream, end the driver's run | -| `host_session` | `the_driver_fires_the_timers_of_its_executions_while_the_guest_says_nothing` | An execution's timeout fires while the guest says nothing, after which the driver cancels the execution, then fences it | -| `host_session` | `the_driver_refuses_starts_before_it_sends_them` | The driver refuses the starts that the guest would not decode or run, and a start once it ran, before it sends anything, with a client that stays usable | -| `host_session` | `the_driver_runs_executions_at_once_and_collects_their_output_and_outcomes` | The driver runs executions at once, and collects the output of each, byte-exact, and its outcome, a refused launch's included | -| `host_session` | `the_driver_s_sends_wait_for_a_guest_that_stops_reading_until_the_deadline_of_its_run` | A run whose deadline passes in a receive leaves the client usable, and in a send that waits for a guest that stopped reading, ends it | -| `host_session` | `the_host_orders_a_layered_root_ready_then_its_table_in_parts_then_start` | The host's order: the root configuration that announces the table, the guest's readiness, each part of the table once the guest applied the one before, then `Start` | -| `host_session` | `the_host_sends_nothing_more_to_a_guest_agent_of_another_build_or_whose_root_failed` | The host sends nothing more to a guest agent of another build, or whose root failed | -| `staged_images` | `a_launch_takes_the_bindings_that_the_store_stages_for_its_images` | A launch takes the bindings that the store staged for its images, and not those of other images | -| `staged_images` | `staging_verifies_the_image_and_binds_the_scratch` | Staging verifies an image against the manifest beside it, refuses another image's, stages a GPT disk over RAM, and refuses images that no launch could attach | -| `staged_images` | `the_host_program_stages_images_in_the_store` | `stage` records the staged images, and prints the bytes that it read, the image's once it hashed it, a staging that fails leaves them, and another replaces them | -| `staged_images` | `the_store_records_the_staged_images` | The record of the staged images, as written and read, and the forms that a reader refuses | -| `layered_launch` | `a_launch_claims_its_scratch_image_until_it_releases_its_checked_files` | A launch claims its scratch image, which a second launch waits for, then fails to take, until the first releases its checked files | -| `layered_launch` | `a_launch_reads_only_the_networks_that_the_plan_takes` | A launch refuses the network files that the plan refuses, each for the plan's reason | -| `layered_launch` | `a_launch_records_what_it_launched` | A launch's record holds the bindings that it checked, over a scratch image and over RAM, and a VM without a layered root has one too | -| `layered_launch` | `a_launch_refuses_images_that_are_not_staged_or_changed_since` | A launch refuses images that the store does not stage, of another format or identity, and images that changed, were replaced, or are gone since they were staged | -| `layered_launch` | `a_launch_refuses_shares_that_it_cannot_export` | A launch refuses shares outside the authorized directories, inaccessible ones, reserved targets, a read-only share inside a read-write one, a hidden path that does not exist, and an allowed path inside no hidden one | -| `layered_launch` | `a_launch_renders_each_element_of_its_network_through_the_plan` | OpenVMM's arguments of each element of a policy, after the plan's block devices, the proxy's URL, and the publication of listeners, as NVX's lowering renders them | -| `layered_launch` | `a_launch_shares_host_paths_inside_the_authorized_directories` | The export of shares inside an authorized directory maps each at its target, parents first, hides a denied path, shows an allowed path inside a hidden one again, and comes with the root configuration and OpenVMM's arguments that attach it; an export whose hidden paths leave the rest of the command line too little is refused as the launch is planned | -| `layered_launch` | `a_layered_launch_hands_openvmm_the_checked_images` | OpenVMM's arguments name the checked files, by the descriptors that OpenVMM inherits on Linux and by their final paths on Windows, without a workload identity or a managed lifecycle, and a command line too long for Windows is refused | -| `layered_launch` | `a_layered_launch_passes_what_the_guest_checks_as_it_is` | A launch passes on the options that the guest checks as they are | -| `layered_launch` | `a_layered_launch_plans_its_blocks_and_boot_tokens_with_the_adapters_plan` | The adapter's plan of a layered launch: its block devices and boot tokens, without the sandbox profile's settings, and the sizes of the images | -| `layered_launch` | `an_execution_finds_the_url_of_its_network_s_proxy_in_its_environment` | An execution's start carries the URL of its network's proxy after its own entries, keeps its own proxy settings, and is unchanged without a proxy | -| `layered_launch` | `shares_read_their_host_path_their_target_and_their_access` | The text of a share, `HOST:TARGET[:ro]` | -| `layered_launch` | `the_host_program_refuses_a_network_before_it_launches` | `launch --network` refuses a network that the plan refuses, or that it cannot read, with status 1 before it needs NVX's artifacts or creates the store | -| `layered_launch` | `the_host_program_refuses_options_that_do_not_make_a_command` | The command lines that the host program refuses, each with status 2, those of `exec`, `--workload-identity`, and `create` included | -| `layered_launch` | `the_host_program_refuses_shares_before_it_touches_the_store` | `launch` refuses shares that it cannot export with status 1 before it creates the store | -| `layered_launch` | `the_record_of_a_launch_gives_back_its_plan` | A launch's record gives back its plan, from which the host finds the proxy's URL, none for a VM without a layered root, and a reader refuses other records | -| `vm_restart` | `a_host_program_of_another_runtime_finds_the_vm_mismatched_and_signals_nothing` | A host program that ships another build of the guest agent finds the VM mismatched, and signals nothing, until the host program of its runtime stops it | -| `vm_restart` | `a_launch_tears_down_a_vm_whose_guest_agent_is_of_another_build` | A launch whose VM's guest agent reports another build sends it neither the table nor `Start`, and fails, which ends the VM | -| `vm_restart` | `a_restarted_host_program_reattaches_to_the_vm_and_stops_it` | Another process's `reattach` holds the VM, which is not its child, in a later epoch, and a third's `stop` stops it gracefully, with no runtime record left | -| `vm_steps` | `a_launch_that_the_hook_ends_after_its_checks_leaves_no_launch` | A launch that the step hook ends at `checked`, with status 5, leaves nothing of a launch in the store, and releases the scratch image's claim | -| `vm_steps` | `images_replaced_after_the_checks_never_reach_the_vm` | Images that files of zeros replace after the checks never reach the VM, whose guest assembles the checked originals | -| `vm_steps` | `the_hook_pauses_a_launch_at_each_of_its_steps` | The step hook pauses a launch at each of its steps, at which the store holds what the step says | -| `vm_layered` | `a_changed_image_is_refused_before_spawn` | A GPT disk written in place after it was staged is refused before the VM spawns, and boots once staged again | -| `vm_layered` | `a_denied_network_lets_no_probe_through` | A network that denies everything attaches no network device, and an execution reaches nothing | -| `vm_layered` | `a_failed_mapping_tears_the_vm_down` | A mapping that the guest refuses ends the table, without `Start`, and the VM ends cleanly once the guest tore the root down | -| `vm_layered` | `a_forward_reaches_its_guest_port` | A forward of a port of host loopback reaches a listener of the guest at its port | -| `vm_layered` | `a_proxy_only_network_lets_only_its_proxy_through` | A proxy-only network lets an execution reach its proxy, through the URL in its environment, and nothing else | -| `vm_layered` | `a_read_only_share_inside_a_writable_one_is_refused_before_launch` | A read-only share inside a writable one is refused before the VM spawns | -| `vm_layered` | `a_replaced_scratch_is_refused_before_spawn` | A scratch image that another copy replaced after it was staged is refused before the VM spawns, and boots once staged again | -| `vm_layered` | `a_share_inside_a_reserved_tree_is_refused_before_launch` | A share inside a tree that the adapter reserves is refused before the VM spawns | -| `vm_layered` | `a_share_over_a_reserved_tree_is_refused_before_launch` | A share that would cover a reserved tree is refused before the VM spawns | -| `vm_layered` | `accountless_workload_identity_fails_its_stage` | A workload identity whose account the image lacks fails its stage once the root is assembled, and the VM ends with 16 | -| `vm_layered` | `allowed_host_loopback_publishes_the_listeners` | Host-loopback access both ways publishes the guest's listeners, which the host reaches, while egress still governs the guest's own connections | -| `vm_layered` | `an_allow_rule_lets_only_its_destination_through` | An allow rule lets an execution reach its destination alone | -| `vm_layered` | `an_unauthorized_host_path_is_refused_before_launch` | A share of a host directory that the adapter does not authorize is refused before the VM spawns | -| `vm_layered` | `busy_root_fails_the_teardown` | A root that a probe keeps open fails its teardown, which the stop reports, and the VM ends with 15 | -| `vm_layered` | `corrupt_layer_fails_the_assembly` | A corrupt layer that asks the kernel to panic on an error fails the assembly without a panic, and the VM ends with 14 | -| `vm_layered` | `cyclic_journal_is_not_loaded` | A layer's cyclic journal, without `needs_recovery`, is never scanned: the root assembles, serves, and tears down | -| `vm_layered` | `erofs_over_ram` | The EROFS image over a tmpfs in RAM, without a scratch, and with no disk but the image's | -| `vm_layered` | `erofs_over_ram_maps_the_table` | The mapping table over a tmpfs, as over a scratch image | -| `vm_layered` | `erofs_over_ram_with_untrusted_lowers` | The image over a tmpfs in the untrusted-lowers mode, which copies up a file's data and refuses the rename of a lower directory | -| `vm_layered` | `erofs_over_scratch` | The EROFS image over a scratch image: the root that the worker assembled outside PID 1, its overlay in the default mode, its cgroups, an empty table, and a clean teardown | -| `vm_layered` | `erofs_over_scratch_maps_the_table` | The export, mounted once below a private `hostfs`, and the table's mappings, in the plan's order, each with its access, `nosuid`, and `nodev` | -| `vm_layered` | `erofs_over_scratch_with_untrusted_lowers` | The image over a scratch image in the untrusted-lowers mode, which copies up a file's data and refuses the rename of a lower directory | -| `vm_layered` | `executions_run_as_the_workload_identity` | Executions that name no identity run as the workload identity, without capabilities, with exactly their start's environment, and one that names an identity is refused with `EINVAL` | -| `vm_layered` | `executions_run_in_the_root` | Executions run at once in the root, as the identities that they name, byte-exact, and end on their own, reusable | -| `vm_layered` | `executions_take_the_settings_of_a_proxy_only_network` | The executions of a proxy-only network find its proxy's URL in their environment, after their own entries, unless they name their own | -| `vm_layered` | `first_partition_over_its_bound_fails_the_assembly` | A GPT disk whose first partition is larger than its bound fails the assembly, and the VM ends with 14 | -| `vm_layered` | `forbidden_journal_fails_the_assembly` | A scratch image whose journal the launch forbids fails the assembly before the guest mounts it, and the VM ends with 14 | -| `vm_layered` | `full_ram_upper_refuses_writes` | A tmpfs upper refuses writes with `ENOSPC` once full, and takes them again once freed | -| `vm_layered` | `full_scratch_refuses_writes` | A scratch upper refuses writes with `ENOSPC` once full, and takes them again once freed | -| `vm_layered` | `gpt_over_ram` | A GPT disk over a tmpfs, with a bound that its first partition meets exactly | -| `vm_layered` | `gpt_over_ram_maps_the_table` | The mapping table over a GPT disk over a tmpfs | -| `vm_layered` | `gpt_over_ram_with_untrusted_lowers` | A GPT disk over a tmpfs in the untrusted-lowers mode | -| `vm_layered` | `gpt_over_scratch` | A GPT disk over a scratch image: its first partition's bytes, and its layers, each mounted read-only once, stacked in order | -| `vm_layered` | `gpt_over_scratch_maps_the_table` | The mapping table over a GPT disk, whose targets the worker creates, with their parents | -| `vm_layered` | `gpt_over_scratch_with_untrusted_lowers` | A GPT disk over a scratch image in the untrusted-lowers mode | -| `vm_layered` | `invalid_configuration_fails_its_stage` | A relative runtime directory fails the configuration's stage, and the VM ends with 11 | -| `vm_layered` | `other_disk_guid_fails_the_assembly` | A GPT disk of another GUID than the launch expects fails the assembly, and the VM ends with 14 | -| `vm_layered` | `other_identity_fails_the_assembly` | An image of another identity than the launch expects fails the assembly, and the VM ends with 14 | -| `vm_layered` | `outbound_host_loopback_publishes_no_listener` | Egress with outbound host-loopback access reaches every destination, and publishes no listener of the guest | -| `vm_layered` | `published_listeners_refuse_binding_and_raw_sockets` | While listeners are published, the guest refuses `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW` with `EPERM`, and keeps the other capabilities of its allowlist | -| `vm_layered` | `recovery_required_layer_fails_the_assembly` | A layer whose journal needs recovery is refused before any mount, whatever `noload` says, and the VM ends with 14 | -| `vm_layered` | `root_profiles_enter_the_root` | Executions enter the root as PID 1 of their namespaces, with their standard streams alone, no inheritable or ambient capability, and no new privileges | -| `vm_layered` | `unstaged_images_are_refused_before_spawn` | Images that the store does not stage are refused before the VM spawns, with nothing of a launch left, and boot once staged | -| `vm_template` | `a_class_that_no_capture_takes_is_refused_before_the_store` | `create` of a class that no capture takes, of 3 processors, or of more online at boot than in all, fails with status 1 before the store exists | -| `vm_template` | `a_network_that_lets_nothing_through_has_no_nic` | `create` of a network that lets nothing through captures a template without a NIC | -| `vm_template` | `a_refresh_captures_and_retires_the_generations_that_it_supersedes` | `refresh` captures a new generation although the store holds an eligible one, and its eviction retires the one that it supersedes, or tombstones it while a clone leases it, as `list` and `inspect` report | -| `vm_template` | `a_refresh_refuses_a_class_that_no_capture_takes_before_the_store` | `refresh` of a class that no capture takes fails with status 1 before the store exists | -| `vm_template` | `a_removal_reports_busy_while_leases_keep_the_generation` | `remove` tombstones a leased generation, keeps its payload, and reports busy, with status 7 and its leases, until no lease remains, then deletes it | -| `vm_template` | `a_template_whose_processors_are_all_online_at_boot_limits_none` | `create` captures a template whose processors are all online at boot without `maxcpus=`, and whose key records none online at boot | -| `vm_template` | `a_template_without_a_network_has_no_nic` | `create` captures a template without a NIC, whose key holds what it asked for, its processor online at boot among it, and serves it again without a capture | -| `vm_template` | `the_networks_of_every_policy_at_one_guest_s_address_share_one_template` | `create` of a network of any policy captures one template, whose NIC the capture attaches under the complete deny-all policy, and the creations of the networks of every other policy at the same guest's address serve it without a capture | -| `vm_clone` | `a_clone_binds_the_resolver_of_its_policy_s_dns_server_read_only` | A clone of a network with egress allowed binds a resolver file that names the gateway's DNS server read-only at `/etc/resolv.conf`, and a TCP connection to the server's DNS port opens, which the gateway's DNS proxy accepts; one whose resolver cannot be bound ends with status 57 | -| `vm_clone` | `a_clone_maps_an_aggregate_of_directories_and_files_beneath_its_root_before_its_sandbox_starts` | A clone maps an aggregate of host directories and files beneath its root before its sandbox starts, each with its access, a hidden file, and a hidden directory that shows a file again, and writes reach the host's directories and file through the writable ones alone | -| `vm_clone` | `a_clone_of_a_template_with_a_nic_maps_its_export_beside_its_resolver` | A clone of a template with a NIC maps its export below the binds of its machine ID and of its resolver configuration | -| `vm_clone` | `a_clone_reads_no_more_of_its_disks_than_its_root_needs` | A clone's restore reads less than an eighth of each disk, before the guest reads each to its end, by the guest's statistics, and as the host counts the reads of the start, which hashes and copies nothing, and of the VMM, whose count on Windows rises by what the guest asked of the disks' files, and on Linux, where it misses io_uring's reads, does not | -| `vm_clone` | `a_clone_refuses_a_scratch_image_whose_upper_an_earlier_clone_used` | A scratch image whose upper an earlier clone used ends the clone before it is ready, with status 54 | -| `vm_clone` | `a_clone_runs_the_executions_that_name_their_identity_once_it_is_ready` | A clone runs the executions that name their identity once it is ready, and refuses one that names none with `EINVAL` | -| `vm_clone` | `a_clone_s_executions_find_the_proxy_of_its_policy` | The executions of a clone of a proxy-only network find its proxy's URL in their environment, and the clone keeps the image's own `/etc/resolv.conf` | -| `vm_clone` | `a_clone_s_time_settles_after_it_is_ready` | A clone's time settles once it is ready, without a violation, after the downtime that the host's monotonic clock measured | -| `vm_clone` | `a_clone_whose_activation_fails_ends_before_it_is_ready` | A clone whose activation fails ends before it is ready, with the status of the failure's class and the failure's record, quarantines nothing, and releases its lease | -| `vm_clone` | `a_clone_whose_mapping_fails_ends_before_its_sandbox_starts` | A clone whose guest fails a mapping ends before its sandbox starts, with the failure's record, and leaves the host's directory unchanged | -| `vm_clone` | `a_clone_whose_view_cannot_be_set_ends_before_it_is_ready` | A generated machine ID that no bind can hold ends the clone before it is ready, with status 55, while the image's own needs no bind | -| `vm_clone` | `a_clone_whose_vm_exits_once_a_stop_found_it_fails_before_its_lease_is_released` | A clone whose VM exits once a stop found it running fails, with status 8 and the record of a failure by the evidence of its lost status, quarantines nothing, and releases its lease | -| `vm_clone` | `a_forward_whose_port_another_socket_holds_fails_the_clone_as_the_host_s` | A forward whose port of host loopback another socket holds ends the clone before its restore-ready event, with status 8 and the record of a failure of the host, by the evidence of the forward's bind, and the template serves the network once the port is free | -| `vm_clone` | `a_generation_is_eligible_by_the_clock_of_its_capture` | The store checks eligibility by the host's monotonic clock on the boot of the capture, and by UTC after a reboot, and refuses an expired or an early generation with status 10, without a lease | -| `vm_clone` | `a_generation_refuses_clones_of_fewer_processors_than_its_template_booted_with` | NVX's plan refuses a processor target below the template's processors online at boot before any VMM starts, and the generation serves its clones from those up | -| `vm_clone` | `a_host_program_of_another_runtime_ends_the_clone_through_its_verified_identity` | A host program of another runtime ends the clone through its verified identity, without a word to its guest, with status 8, and releases its lease | -| `vm_clone` | `a_run_in_a_template_of_a_network_finds_its_policy_s_proxy` | A run's execution, in a clone of a proxy-only template, finds the proxy's URL in its environment, and a run of a network that lets nothing through runs nothing, with status 8 | -| `vm_clone` | `a_run_runs_one_execution_in_its_clone_then_tears_the_clone_down` | A run stages its images, starts a clone, runs one execution in its root, with the execution's environment and working directory, then stops the clone and releases its lease, whatever the execution did | -| `vm_clone` | `a_run_s_execution_finds_its_shares_beneath_the_root` | A run's execution reads and writes through its share, which the clone mapped beneath its root, and the host's directory keeps what it wrote | -| `vm_clone` | `a_run_whose_clone_does_not_start_runs_nothing` | A run whose selection the store refuses ends with status 10, and one whose clone fails with status 8, without an execution or a stop | -| `vm_clone` | `a_run_whose_clone_s_vm_exits_during_its_execution_ends_with_the_clone_s_failure` | A run whose clone's VM exits during its execution ends with the clone's failure, status 8, once its stop found that the VM exited, quarantines nothing, and releases its lease | -| `vm_clone` | `a_selection_that_names_no_template_leases_nothing` | A start of a template store that does not exist, or of a generation that `remove` retired, ends with status 10 and the refusal's record, without a lease or a VM | -| `vm_clone` | `a_start_refuses_an_export_whose_mapped_objects_changed_before_any_vmm_starts` | An export whose mapped file, or pinned directory, another object replaced once NVX's plan checked the clone is refused with status 8 at the clone's launch, before any VMM starts | -| `vm_clone` | `a_start_refuses_what_the_template_cannot_serve_before_any_vmm_starts` | Images, scratch images, and memory targets that the template cannot serve are refused with status 8 before any VMM starts, unstaged images with status 1, and a generation that the store does not hold with status 10 | -| `vm_clone` | `a_template_without_a_nic_serves_clones_without_a_network` | A template without a NIC serves clones without a network, or with one that lets nothing through, and refuses those of a network with a NIC with status 8 | -| `vm_clone` | `a_tombstone_leaves_the_running_clone_of_its_generation_to_its_lease` | A tombstone refuses new clones of its generation with status 10, while the running clone serves on, and releases its lease once it stops | -| `vm_clone` | `clones_at_once_keep_their_own_uppers_and_machine_ids` | Clones at once keep their own uppers, and their own machine IDs, generated, distinct, and bound read-only, or the image's own | -| `vm_clone` | `clones_at_once_see_only_their_own_exports` | Clones of one generation at once see only their own exports, and a clone without one none | -| `vm_clone` | `each_command_reattaches_to_the_clone_from_a_new_process` | `reattach`, `probe`, and `exec` each reattach to the clone from a new process, in a later epoch, without a second activation | -| `vm_clone` | `one_generation_serves_clones_of_each_processor_count_from_its_boot_online_count_to_its_capacity` | One generation serves clones of each supported processor count from the template's processors online at boot to its capacity, whose repair brings the target's CPUs online, and NVX's plan refuses an unsupported count and one beyond the capacity before any VMM starts | -| `vm_clone` | `one_generation_serves_clones_of_each_upper_and_memory_target` | One generation serves clones over a scratch image and in RAM, at each memory target up to its capacity, each with its own lease, and leaves its snapshot unchanged | -| `vm_clone` | `one_generation_serves_clones_of_either_format_over_either_upper` | A template of a GPT placeholder serves EROFS and GPT clones of other layouts, shorter than the placeholder, at its length, over either upper, those in RAM with no scratch file, and a first partition over its bound ends a clone with status 54 | -| `vm_clone` | `one_template_serves_the_clones_of_every_network_policy_at_its_guest_s_address` | One template with a NIC serves the clones of every network policy at its guest's address, whose restores install their own policies on its NIC, which lets through what each allows, publishes the guest's listeners, through the clone's announcer, and forwards ports where it says, and NVX's plan refuses another address, or no NIC, with status 8 before any VMM starts | -| `vm_clone` | `openvmm_reads_none_of_the_host_s_input_until_the_repair_acknowledges_the_restore` | OpenVMM reads none of the host's attachment until the repair of a clone, whose template delays it, acknowledges the restore: the broker answers it only after the delay, and on Linux, where the host writes it a byte at a time, `SIOCOUTQ` rises with each byte and stays as the write left it until then, while a reattach's is answered at once | -| `vm_clone` | `restore_exclusion::hard_linked_restore_views_obey_the_callers_exclusion` (linux) | Caller-locked concurrent hard-linked restores succeed, an unlocked link change during startup fails the restore, a controller killed before ready prevents completion, and link changes after the peer-verified ready connection do not fail it | -| `vm_clone` | `the_lease_of_a_start_that_ended_before_its_record_is_released_through_its_template_store` | The lease of a start that ended before it recorded its clone is released only through its own template store, which `--template` names | - -### Running the VM tests - -The VM tests, `tests/vm_restart.rs`, `tests/vm_steps.rs`, -`tests/vm_layered.rs`, `tests/vm_template.rs`, and `tests/vm_clone.rs`, share -their sandboxes, the fixtures of their layered roots, and their host listener -client through `tests/common`. The sandboxes start -the host program's commands with `nvx::host::supervisor::spawn_command`, so -that on Windows a command cannot inherit the handles of a VM that a test -launches in its own process meanwhile, such as its log and its capability's -pipe, which it would hold until it exits. A test that fails -ends the VM that it left running, and the host program that it left paused, -and keeps the sandbox's directory, whose path it prints, unless it can confirm -that the VM ended. The tests boot VMs, so they are ignored. They read the -hypervisor from `NVX_HYPERVISOR`, through -`nvx::Hypervisor::from_env_or_default`, NVX's artifacts, with their install -metadata and the fixtures, from `NVX_ARTIFACTS_DIR`, and the guest agent from -`RUNTIME_ADAPTER_GUEST`. In a checkout where `nvx-rs download` installed NVX's -artifacts, the scratch template among them, run them in this directory with: - -```bash -cargo build --locked --release --features guest \ - --target x86_64-unknown-linux-musl --bin runtime-adapter-guest -NVX_HYPERVISOR=kvm NVX_ARTIFACTS_DIR="$(cd ../.. && pwd)" \ - RUNTIME_ADAPTER_GUEST="$PWD/target/x86_64-unknown-linux-musl/release/runtime-adapter-guest" \ - cargo test --locked --features host -- --ignored -``` - -On Windows, take the guest agent from a Linux build. To run the -[stage scenarios](#stage-scenarios) by name, as CI runs them, run in the -repository's root, after the build of the guest agent above, or with -`--guest-agent PATH` naming another: - -```bash -python3 scripts/nvx.py test-runtime-adapter --backend kvm -``` - -Run the checks of each configuration in this directory. The guest-only -configuration needs Linux and the musl target, which -`rustup target add x86_64-unknown-linux-musl` installs: - -```bash -cargo fmt --all --check -cargo clippy --locked --all-targets -- -D warnings -cargo test --locked -RUSTDOCFLAGS="-D warnings" cargo doc --locked --no-deps -cargo clippy --locked --all-targets --features host -- -D warnings -cargo test --locked --features host -RUSTDOCFLAGS="-D warnings" cargo doc --locked --no-deps --features host -cargo clippy --locked --all-targets --features guest \ - --target x86_64-unknown-linux-musl -- -D warnings -cargo test --locked --features guest --target x86_64-unknown-linux-musl -RUSTDOCFLAGS="-D warnings" cargo doc --locked --no-deps --features guest \ - --target x86_64-unknown-linux-musl -``` - -## CI - -The [`check-nvx-rs`](../../.github/actions/check-nvx-rs/action.yml) action -runs these checks with `cargo build`, and `cargo check` with the minimum -supported Rust version, on Linux and Windows, where it skips the guest-only -configuration. A change to the adapter is a test input of CI's -[change classification](../../doc/ci.md#change-classification), so it runs the -NVX microVM tests. Their -[workflow](../../.github/workflows/run-nvx-microvm-tests.yml) builds the -host-only configuration on each backend, in the jobs that run the guest and -sandbox steps on KVM and MSHV, and in a job of its own on WHP, where the -scenarios take about twice as long, and runs the adapter's consolidated suite, -the [stage scenarios](#stage-scenarios), by name, with -`nvx.py test-runtime-adapter`, which names the backend in `NVX_HYPERVISOR`, the -checkout in `NVX_ARTIFACTS_DIR`, into which the job downloaded NVX's -artifacts, their provenance, and the fixtures of the layered scenarios with -the guest artifacts, as NVX's own layered scenarios take them, and the guest -agent in `RUNTIME_ADAPTER_GUEST`. The Linux jobs build the guest agent -first, and the WHP job takes the one that CI's `distro-artifacts` job -builds on Linux. diff --git a/tests/runtime-adapter/scenarios.txt b/tests/runtime-adapter/scenarios.txt deleted file mode 100644 index 5588b216c..000000000 --- a/tests/runtime-adapter/scenarios.txt +++ /dev/null @@ -1,176 +0,0 @@ -# The runtime adapter's consolidated suite: every scenario of the template and clone stage, which -# CI's runtime adapter step runs by name on each backend, with -# `python3 scripts/nvx.py test-runtime-adapter`. Each line names a test target of `tests/`, then -# one of its tests, optionally followed by linux or windows to select only that host platform. -# Unqualified lines run on both platforms, whether ignored or not. The step fails once a line -# names a test that its target does not have, a target that a line names has a test that no line -# names, the library or a target that no line names has an ignored test, or a named test does not -# run and pass. The README's "Stage scenarios" says what each scenario proves. - -template_capture a_capture_takes_its_class_and_inputs_from_the_request -template_capture the_host_program_refuses_a_repair_delay_that_it_cannot_read -template_capture the_host_program_refuses_a_template_before_it_creates_anything - -template_operations a_refresh_refuses_a_template_before_it_creates_anything -template_operations an_empty_template_store_lists_no_generation -template_operations the_host_program_refuses_command_lines_of_the_operations -template_operations the_operations_open_a_template_store_and_create_none - -clone_start a_clone_s_plan_request_and_record_carry_its_network -clone_start a_clone_s_record_names_its_template_store_and_generation_beside_its_plan -clone_start a_clone_s_request_takes_the_staged_bindings_and_the_guest_s_defaults -clone_start a_host_s_time_keeps_its_text_form -clone_start a_run_refuses_a_network_or_shares_that_the_plan_refuses_before_it_stages_anything -clone_start a_run_refuses_a_selection_that_names_no_template_once_its_preflight_staged_the_images -clone_start a_start_refuses_a_generation_that_the_template_store_does_not_hold_before_anything_else -clone_start a_start_refuses_a_network_that_the_plan_refuses_before_anything_else -clone_start a_start_refuses_shares_that_it_cannot_export_before_anything_else -clone_start the_host_program_refuses_options_that_do_not_make_a_run -clone_start the_host_program_refuses_options_that_do_not_make_a_start - -host_session a_clone_s_order_ends_its_table_at_a_mapping_that_the_guest_fails_and_stops_the_guest -host_session a_clone_s_order_maps_its_export_s_children_by_their_indices -host_session a_clone_s_order_sends_its_record_first_then_starts_its_sandbox_as_after_a_cold_boot -host_session a_clone_s_session_that_ends_after_its_bootstrap_is_lost_and_the_next_takes_the_same_record -host_session a_mapping_that_the_guest_fails_ends_the_table_and_the_host_stops_the_guest -host_session a_receive_that_times_out_keeps_the_partial_message -host_session a_reset_or_a_disconnect_in_the_middle_of_a_message_drops_it -host_session a_send_that_fails_or_times_out_ends_the_client_with_what_it_holds -host_session a_stop_skips_the_guest_s_late_messages_about_the_executions_that_the_driver_gave_up_on -host_session answers_whose_bodies_do_not_decode_end_the_client -host_session only_a_session_that_ended_is_lost_to_a_clone_s_order -host_session requests_check_their_answers_and_end_the_client_when_they_fail -host_session the_client_attaches_and_exchanges_messages_that_span_records -host_session the_client_configures_a_layered_root_probes_it_and_learns_how_its_teardown_ended -host_session the_client_fails_on_records_that_the_protocol_does_not_allow -host_session the_client_reassembles_messages_from_partial_and_shared_records -host_session the_client_refuses_messages_that_its_framing_does_not_allow -host_session the_client_returns_the_stage_of_the_root_that_failed -host_session the_client_waits_for_the_guest_and_reports_refusals_while_attaching -host_session the_driver_ends_its_run_on_a_message_that_breaks_the_rules -host_session the_driver_fires_the_timers_of_its_executions_while_the_guest_says_nothing -host_session the_driver_refuses_starts_before_it_sends_them -host_session the_driver_runs_executions_at_once_and_collects_their_output_and_outcomes -host_session the_driver_s_sends_wait_for_a_guest_that_stops_reading_until_the_deadline_of_its_run -host_session the_host_orders_a_layered_root_ready_then_its_table_in_parts_then_start -host_session the_host_sends_nothing_more_to_a_guest_agent_of_another_build_or_whose_root_failed - -staged_images a_launch_takes_the_bindings_that_the_store_stages_for_its_images -staged_images staging_verifies_the_image_and_binds_the_scratch -staged_images the_host_program_stages_images_in_the_store -staged_images the_store_records_the_staged_images - -layered_launch a_launch_claims_its_scratch_image_until_it_releases_its_checked_files -layered_launch a_launch_reads_only_the_networks_that_the_plan_takes -layered_launch a_launch_records_what_it_launched -layered_launch a_launch_refuses_images_that_are_not_staged_or_changed_since -layered_launch a_launch_refuses_shares_that_it_cannot_export -layered_launch a_launch_renders_each_element_of_its_network_through_the_plan -layered_launch a_launch_shares_host_paths_inside_the_authorized_directories -layered_launch a_layered_launch_hands_openvmm_the_checked_images -layered_launch a_layered_launch_passes_what_the_guest_checks_as_it_is -layered_launch a_layered_launch_plans_its_blocks_and_boot_tokens_with_the_adapters_plan -layered_launch an_execution_finds_the_url_of_its_network_s_proxy_in_its_environment -layered_launch shares_read_their_host_path_their_target_and_their_access -layered_launch the_host_program_refuses_a_network_before_it_launches -layered_launch the_host_program_refuses_options_that_do_not_make_a_command -layered_launch the_host_program_refuses_shares_before_it_touches_the_store -layered_launch the_record_of_a_launch_gives_back_its_plan - -vm_restart a_host_program_of_another_runtime_finds_the_vm_mismatched_and_signals_nothing -vm_restart a_launch_tears_down_a_vm_whose_guest_agent_is_of_another_build -vm_restart a_restarted_host_program_reattaches_to_the_vm_and_stops_it - -vm_steps a_launch_that_the_hook_ends_after_its_checks_leaves_no_launch -vm_steps images_replaced_after_the_checks_never_reach_the_vm -vm_steps the_hook_pauses_a_launch_at_each_of_its_steps - -vm_layered a_changed_image_is_refused_before_spawn -vm_layered a_denied_network_lets_no_probe_through -vm_layered a_failed_mapping_tears_the_vm_down -vm_layered a_forward_reaches_its_guest_port -vm_layered a_proxy_only_network_lets_only_its_proxy_through -vm_layered a_read_only_share_inside_a_writable_one_is_refused_before_launch -vm_layered a_replaced_scratch_is_refused_before_spawn -vm_layered a_share_inside_a_reserved_tree_is_refused_before_launch -vm_layered a_share_over_a_reserved_tree_is_refused_before_launch -vm_layered accountless_workload_identity_fails_its_stage -vm_layered allowed_host_loopback_publishes_the_listeners -vm_layered an_allow_rule_lets_only_its_destination_through -vm_layered an_unauthorized_host_path_is_refused_before_launch -vm_layered busy_root_fails_the_teardown -vm_layered corrupt_layer_fails_the_assembly -vm_layered cyclic_journal_is_not_loaded -vm_layered erofs_over_ram -vm_layered erofs_over_ram_maps_the_table -vm_layered erofs_over_ram_with_untrusted_lowers -vm_layered erofs_over_scratch -vm_layered erofs_over_scratch_maps_the_table -vm_layered erofs_over_scratch_with_untrusted_lowers -vm_layered executions_run_as_the_workload_identity -vm_layered executions_run_in_the_root -vm_layered executions_take_the_settings_of_a_proxy_only_network -vm_layered first_partition_over_its_bound_fails_the_assembly -vm_layered forbidden_journal_fails_the_assembly -vm_layered full_ram_upper_refuses_writes -vm_layered full_scratch_refuses_writes -vm_layered gpt_over_ram -vm_layered gpt_over_ram_maps_the_table -vm_layered gpt_over_ram_with_untrusted_lowers -vm_layered gpt_over_scratch -vm_layered gpt_over_scratch_maps_the_table -vm_layered gpt_over_scratch_with_untrusted_lowers -vm_layered invalid_configuration_fails_its_stage -vm_layered other_disk_guid_fails_the_assembly -vm_layered other_identity_fails_the_assembly -vm_layered outbound_host_loopback_publishes_no_listener -vm_layered published_listeners_refuse_binding_and_raw_sockets -vm_layered recovery_required_layer_fails_the_assembly -vm_layered root_profiles_enter_the_root -vm_layered unstaged_images_are_refused_before_spawn - -vm_template a_class_that_no_capture_takes_is_refused_before_the_store -vm_template a_network_that_lets_nothing_through_has_no_nic -vm_template a_refresh_captures_and_retires_the_generations_that_it_supersedes -vm_template a_refresh_refuses_a_class_that_no_capture_takes_before_the_store -vm_template a_removal_reports_busy_while_leases_keep_the_generation -vm_template a_template_whose_processors_are_all_online_at_boot_limits_none -vm_template a_template_without_a_network_has_no_nic -vm_template the_networks_of_every_policy_at_one_guest_s_address_share_one_template - -vm_clone a_clone_binds_the_resolver_of_its_policy_s_dns_server_read_only -vm_clone a_clone_maps_an_aggregate_of_directories_and_files_beneath_its_root_before_its_sandbox_starts -vm_clone a_clone_of_a_template_with_a_nic_maps_its_export_beside_its_resolver -vm_clone a_clone_reads_no_more_of_its_disks_than_its_root_needs -vm_clone a_clone_refuses_a_scratch_image_whose_upper_an_earlier_clone_used -vm_clone a_clone_runs_the_executions_that_name_their_identity_once_it_is_ready -vm_clone a_clone_s_executions_find_the_proxy_of_its_policy -vm_clone a_clone_s_time_settles_after_it_is_ready -vm_clone a_clone_whose_activation_fails_ends_before_it_is_ready -vm_clone a_clone_whose_mapping_fails_ends_before_its_sandbox_starts -vm_clone a_clone_whose_view_cannot_be_set_ends_before_it_is_ready -vm_clone a_clone_whose_vm_exits_once_a_stop_found_it_fails_before_its_lease_is_released -vm_clone a_forward_whose_port_another_socket_holds_fails_the_clone_as_the_host_s -vm_clone a_generation_is_eligible_by_the_clock_of_its_capture -vm_clone a_generation_refuses_clones_of_fewer_processors_than_its_template_booted_with -vm_clone a_host_program_of_another_runtime_ends_the_clone_through_its_verified_identity -vm_clone a_run_in_a_template_of_a_network_finds_its_policy_s_proxy -vm_clone a_run_runs_one_execution_in_its_clone_then_tears_the_clone_down -vm_clone a_run_s_execution_finds_its_shares_beneath_the_root -vm_clone a_run_whose_clone_does_not_start_runs_nothing -vm_clone a_run_whose_clone_s_vm_exits_during_its_execution_ends_with_the_clone_s_failure -vm_clone a_selection_that_names_no_template_leases_nothing -vm_clone a_start_refuses_an_export_whose_mapped_objects_changed_before_any_vmm_starts -vm_clone a_start_refuses_what_the_template_cannot_serve_before_any_vmm_starts -vm_clone a_template_without_a_nic_serves_clones_without_a_network -vm_clone a_tombstone_leaves_the_running_clone_of_its_generation_to_its_lease -vm_clone clones_at_once_keep_their_own_uppers_and_machine_ids -vm_clone clones_at_once_see_only_their_own_exports -vm_clone each_command_reattaches_to_the_clone_from_a_new_process -vm_clone one_generation_serves_clones_of_each_processor_count_from_its_boot_online_count_to_its_capacity -vm_clone one_generation_serves_clones_of_each_upper_and_memory_target -vm_clone one_generation_serves_clones_of_either_format_over_either_upper -vm_clone one_template_serves_the_clones_of_every_network_policy_at_its_guest_s_address -vm_clone openvmm_reads_none_of_the_host_s_input_until_the_repair_acknowledges_the_restore -vm_clone restore_exclusion::hard_linked_restore_views_obey_the_callers_exclusion linux -vm_clone the_lease_of_a_start_that_ended_before_its_record_is_released_through_its_template_store diff --git a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs b/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs deleted file mode 100644 index 305a0c01c..000000000 --- a/tests/runtime-adapter/src/bin/runtime-adapter-guest.rs +++ /dev/null @@ -1,324 +0,0 @@ -//! The runtime adapter's guest agent, a static `x86_64-unknown-linux-musl` executable. -//! -//! - `runtime-adapter-guest` reports the contract that it was built against. -//! - `runtime-adapter-guest serve` serves the host program with the adapter's own framing on its -//! standard input and output, which connect to the guest's end of the control session between -//! the host and the guest, such as a control console in raw mode. It exits with 0 once the host -//! stopped it or the broker closed the stream, and with 1, after a diagnostic on standard error, -//! once serving failed. -//! - `runtime-adapter-guest init TTY` serves the host program on the tty TTY, which it puts in raw -//! mode itself, in place of NVX's managed agent, whose initramfs runs it on the VM's control tty -//! as PID 1; see `runtime_adapter::initramfs`. It then ends with 0 once the host stopped it, and -//! with 1, after a diagnostic on standard error, once serving failed or the broker closed the -//! tty. -//! - `runtime-adapter-guest init TTY layered` runs PID 1 of a VM with a layered root, in place of -//! NVX's init agent: it starts an unrelated child, `runtime-adapter-guest idle`, and the worker, -//! `runtime-adapter-guest worker TTY SIBLING`, collects the status of every child until the -//! worker ends, and ends with the worker's status; see `runtime_adapter::worker`. -//! - `runtime-adapter-guest worker TTY [SIBLING]` runs the worker on the tty TTY: it assembles the -//! layered root that the host configures, and serves the host program until the host stops it. -//! SIBLING is the process ID of PID 1's unrelated child, which its probes check. It exits with 0 -//! once the host stopped it, with 11 to 16 once a stage of the root failed, and with 1 once its -//! session failed or the broker closed the tty, each but the first after a diagnostic on -//! standard error. In a template's VM, whose kernel command line names the platform snapshot -//! tier, the worker captures the VM before it assembles any root instead, on x86_64, as -//! `runtime_adapter::worker::capture` describes: it exits with 13 once the template's bootstrap -//! failed, or with 20 to 30. In a clone, which it repairs, then activates and serves, it exits -//! with 40 to 45, 193, or 195 once the repair failed, with 50 to 58 once the clone's activation -//! failed, as `nvx::protocol::status` documents, and, once the clone was ready, as the worker of -//! a VM that boots cold does. It exits with 1 once its session failed. Each comes after a -//! diagnostic on standard error, unless a committed capture ended the VM; see -//! `runtime_adapter::template`. -//! - `runtime-adapter-guest idle` sleeps until a signal ends it. -//! - `runtime-adapter-guest entries ROOT CGROUP` runs the executions of the worker's `entries` -//! probe in the root ROOT, each placed in the cgroup CGROUP, in a mount namespace of its own, and -//! reports their facts on standard output; see `runtime_adapter::worker::entries`. It exits with -//! 0, or with 1 after a diagnostic on standard error. -//! - `runtime-adapter-guest inside DIRECTORY DESCRIPTOR` runs in each of those executions, from -//! inside the root: it reports what the execution sees and may do on standard output, with its -//! case's directory DIRECTORY and the number of the descriptor that its caller leaked. It exits -//! with 0, or with 1 after a diagnostic on standard error. -//! -//! As PID 1, which must not exit, `init` ends the VM with its status instead, through NVX's -//! `/sbin/nvx-exit`, as NVX's managed agent does. Anything else is a usage error, with status 2. - -use std::process::ExitCode; - -#[cfg(target_os = "linux")] -fn main() -> ExitCode { - use runtime_adapter::{ - GUEST_ENTRIES_COMMAND, GUEST_IDLE_COMMAND, GUEST_INIT_COMMAND, GUEST_INSIDE_COMMAND, - GUEST_LAYERED_MODE, GUEST_SERVE_COMMAND, GUEST_WORKER_COMMAND, - }; - use std::path::Path; - - let arguments = std::env::args_os().skip(1).collect::>(); - match arguments.as_slice() { - [] => { - println!("runtime-adapter-guest: {}", runtime_adapter::contract()); - ExitCode::SUCCESS - } - [command] if command == GUEST_SERVE_COMMAND => serve(), - [command, tty] if command == GUEST_INIT_COMMAND => init(Path::new(tty)), - [command, tty, mode] if command == GUEST_INIT_COMMAND && mode == GUEST_LAYERED_MODE => { - end(runtime_adapter::worker::supervise(Path::new(tty))) - } - [command, tty, sibling @ ..] if command == GUEST_WORKER_COMMAND && sibling.len() < 2 => { - match sibling - .first() - .map(|id| id.to_str().and_then(|id| id.parse().ok())) - { - Some(None) => usage(), - Some(Some(sibling)) => worker(Path::new(tty), Some(sibling)), - None => worker(Path::new(tty), None), - } - } - [command] if command == GUEST_IDLE_COMMAND => runtime_adapter::worker::idle(), - [command, root, workload] if command == GUEST_ENTRIES_COMMAND => { - entries(Path::new(root), Path::new(workload)) - } - [command, directory, descriptor] if command == GUEST_INSIDE_COMMAND => { - let descriptor = descriptor.to_str().and_then(|number| number.parse().ok()); - inside(Path::new(directory), descriptor) - } - _ => usage(), - } -} - -/// Reports a usage error. -#[cfg(target_os = "linux")] -fn usage() -> ExitCode { - eprintln!( - "usage: runtime-adapter-guest [{} | {} TTY [{}] | {} TTY [SIBLING] | {} | {} ROOT CGROUP | \ - {} DIRECTORY DESCRIPTOR]", - runtime_adapter::GUEST_SERVE_COMMAND, - runtime_adapter::GUEST_INIT_COMMAND, - runtime_adapter::GUEST_LAYERED_MODE, - runtime_adapter::GUEST_WORKER_COMMAND, - runtime_adapter::GUEST_IDLE_COMMAND, - runtime_adapter::GUEST_ENTRIES_COMMAND, - runtime_adapter::GUEST_INSIDE_COMMAND, - ); - ExitCode::from(2) -} - -/// Serves the session on standard input and output. -#[cfg(target_os = "linux")] -fn serve() -> ExitCode { - let stdio = Stdio { - input: std::io::stdin().lock(), - output: std::io::stdout().lock(), - }; - match runtime_adapter::guest::serve(stdio) { - Ok(_) => ExitCode::SUCCESS, - Err(error) => { - eprintln!("runtime-adapter-guest: {error}"); - ExitCode::FAILURE - } - } -} - -/// NVX's program that ends the VM with an exit status, which its initramfs provides. -#[cfg(target_os = "linux")] -const NVX_EXIT: &str = "/sbin/nvx-exit"; - -/// Serves the session on the tty `tty` in place of NVX's managed agent, then ends with the status -/// of serving. -#[cfg(target_os = "linux")] -fn init(tty: &std::path::Path) -> ExitCode { - use runtime_adapter::guest::{self, Served}; - - let status: u8 = match guest::serve_terminal(tty) { - Ok(Served::Stopped) => 0, - Ok(Served::Closed) => { - eprintln!("runtime-adapter-guest: the broker closed {}", tty.display()); - 1 - } - Err(error) => { - eprintln!("runtime-adapter-guest: {error}"); - 1 - } - }; - end(status) -} - -/// Runs the worker on the tty `tty`, with the process ID of PID 1's unrelated child, if any, and -/// ends with its status. In a template's VM, the worker captures the VM instead; see -/// `capture_template`. -#[cfg(target_os = "linux")] -fn worker(tty: &std::path::Path, sibling: Option) -> ExitCode { - use runtime_adapter::worker::{self, LibraryRoot, Worked}; - - let mut operations = LibraryRoot::new(sibling); - #[cfg(target_arch = "x86_64")] - if let Some(status) = capture_template(tty, &mut operations) { - return ExitCode::from(status); - } - let worked = worker::work_on_terminal(tty, &mut operations); - let status = match worked { - Ok(worked) => { - match &worked { - Worked::Stopped => {} - Worked::Failed(failure) => eprintln!("runtime-adapter-guest: {failure}"), - Worked::Closed | Worked::Abandoned => { - eprintln!("runtime-adapter-guest: the broker closed {}", tty.display()); - } - // Only a clone's worker activates, in `capture_template`. - Worked::Unactivated { problem, .. } => { - eprintln!("runtime-adapter-guest: {problem}"); - } - } - worked.status() - } - Err(error) => { - eprintln!("runtime-adapter-guest: {error}"); - worker::SESSION_STATUS - } - }; - // Closes what the probes held, then unmounts a root that the worker still holds, ignoring - // failures, before the worker exits. - drop(operations); - ExitCode::from(status) -} - -/// Reads the kernel command line through `operations`, before anything else, and captures a -/// template's VM on the tty `tty`, with `operations`, as `runtime_adapter::worker::capture` -/// describes, then repairs a clone that a restore of its snapshot started, and serves it through -/// its activation. Returns the status with which the worker ends then, after a diagnostic on -/// standard error, or `None` in a VM that boots cold, whose worker serves the host. -/// -/// A command line that the worker cannot read, or that `runtime_adapter::template::boot` refuses, -/// ends the worker with `COMMAND_LINE_STATUS`, as does a template's repair delay, for tests, that -/// the worker cannot read, or that `runtime_adapter::template::parse_repair_delay` refuses. -#[cfg(all(target_os = "linux", target_arch = "x86_64"))] -fn capture_template( - tty: &std::path::Path, - operations: &mut runtime_adapter::worker::LibraryRoot, -) -> Option { - use runtime_adapter::template::{self, Boot, COMMAND_LINE_STATUS, REPAIR_DELAY_FILE}; - use runtime_adapter::worker::capture; - use runtime_adapter::worker::{RootOperations, SESSION_STATUS}; - - let boot = operations - .command_line() - .and_then(|command_line| template::boot(&command_line).map_err(|error| error.to_string())); - let gateway = match boot { - Ok(Boot::Cold) => return None, - Ok(Boot::Template { gateway }) => gateway, - Err(problem) => { - eprintln!("runtime-adapter-guest: {problem}"); - return Some(COMMAND_LINE_STATUS); - } - }; - let repair_delay = match std::fs::read(REPAIR_DELAY_FILE) { - Ok(contents) => match template::parse_repair_delay(&contents) { - Some(delay) => Some(delay), - None => { - eprintln!( - "runtime-adapter-guest: {REPAIR_DELAY_FILE} holds no repair delay: {}", - contents.escape_ascii() - ); - return Some(COMMAND_LINE_STATUS); - } - }, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => None, - Err(error) => { - eprintln!("runtime-adapter-guest: cannot read {REPAIR_DELAY_FILE}: {error}"); - return Some(COMMAND_LINE_STATUS); - } - }; - Some( - match capture::capture_on_terminal(tty, operations, gateway, repair_delay) { - Ok(templated) => { - eprintln!("runtime-adapter-guest: {templated}"); - templated.status() - } - Err(error) => { - eprintln!("runtime-adapter-guest: {error}"); - SESSION_STATUS - } - }, - ) -} - -/// Runs the executions of the worker's `entries` probe in the root `root`, each placed in the cgroup -/// `workload`, and reports their facts. -#[cfg(target_os = "linux")] -fn entries(root: &std::path::Path, workload: &std::path::Path) -> ExitCode { - use runtime_adapter::worker::entries; - - match entries::helper(root, workload).and_then(|facts| entries::report(&facts)) { - Ok(()) => ExitCode::SUCCESS, - Err(error) => { - eprintln!("runtime-adapter-guest: {error}"); - ExitCode::FAILURE - } - } -} - -/// Reports, from inside an execution of the worker's `entries` probe, what it sees and may do, with -/// its case's `directory` and the number of the descriptor that its caller leaked, if any. -#[cfg(target_os = "linux")] -fn inside(directory: &std::path::Path, descriptor: Option) -> ExitCode { - use runtime_adapter::worker::entries; - - match entries::report(&entries::inside(directory, descriptor)) { - Ok(()) => ExitCode::SUCCESS, - Err(error) => { - eprintln!("runtime-adapter-guest: {error}"); - ExitCode::FAILURE - } - } -} - -/// Ends with `status`: as PID 1, by ending the VM through [`NVX_EXIT`]. -#[cfg(target_os = "linux")] -fn end(status: u8) -> ExitCode { - if std::process::id() != 1 { - return ExitCode::from(status); - } - // PID 1 must not exit, or the kernel panics: `nvx-exit` replaces it and ends the VM. Should - // it fail to start, PID 1 waits, and the host stops the VM. - let error = std::os::unix::process::CommandExt::exec( - std::process::Command::new(NVX_EXIT).arg(status.to_string()), - ); - eprintln!("runtime-adapter-guest: cannot run {NVX_EXIT}: {error}"); - loop { - std::thread::sleep(std::time::Duration::from_secs(3600)); - } -} - -/// Standard input and output as one stream, which carries nothing but the session's records. -#[cfg(target_os = "linux")] -struct Stdio { - /// Standard input, from the broker. - input: std::io::StdinLock<'static>, - /// Standard output, to the broker. - output: std::io::StdoutLock<'static>, -} - -#[cfg(target_os = "linux")] -impl std::io::Read for Stdio { - fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { - self.input.read(buffer) - } -} - -#[cfg(target_os = "linux")] -impl std::io::Write for Stdio { - fn write(&mut self, bytes: &[u8]) -> std::io::Result { - self.output.write(bytes) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.output.flush() - } -} - -// The guest agent runs only in Linux guests. Elsewhere this stub fails, so that every target still -// builds with every feature. -#[cfg(not(target_os = "linux"))] -fn main() -> ExitCode { - eprintln!("runtime-adapter-guest runs only in Linux guests"); - ExitCode::FAILURE -} diff --git a/tests/runtime-adapter/src/bin/runtime-adapter-host.rs b/tests/runtime-adapter/src/bin/runtime-adapter-host.rs deleted file mode 100644 index 844c9a4f0..000000000 --- a/tests/runtime-adapter/src/bin/runtime-adapter-host.rs +++ /dev/null @@ -1,3742 +0,0 @@ -//! The runtime adapter's host program. -//! -//! - `runtime-adapter-host` reports the contract that it was built against and whether `nvx` -//! supports this host. -//! - `runtime-adapter-host stage STORE --distro FORMAT:PATH --distro-identity UUID -//! [--upper scratch:PATH]` stages an image and a scratch image for the launches of a VM with a -//! layered root in the sandbox store STORE, which it creates if needed. The image is the raw -//! file PATH, an EROFS filesystem, `erofs`, or a GPT disk, `gpt`, of which the guest expects the -//! identity UUID, and the scratch image the ext4 image PATH, if given. Staging verifies the -//! image explicitly, which hashes the image only when a manifest is beside it, binds the scratch -//! image without reading it, and records both bindings in STORE, in place of those staged -//! earlier; see `runtime_adapter::staged::Staged`. It needs no artifacts, and prints the bytes -//! that it read as it exits, `read-bytes`. -//! - `runtime-adapter-host launch STORE --guest-agent PATH [--hypervisor NAME] [LAYERED]` launches -//! a VM in the sandbox store STORE, which it creates if needed, through NVX's supervisor, with -//! the guest agent PATH, a static `x86_64-unknown-linux-musl` build of `runtime-adapter-guest`, -//! in place of NVX's agents; see `runtime_adapter::initramfs`. It records what it launches in -//! STORE; see `runtime_adapter::layered::LaunchRecord`. It attaches to the VM's control -//! session, exchanges a `Ping` and the longest `Echo` with the guest agent, commits the launch, -//! and exits, leaving the VM running. The VM runs on the hypervisor NAME, `kvm`, `mshv`, or -//! `whp`, or else on the one that `nvx::Hypervisor::from_env_or_default` selects. -//! -//! LAYERED, `--distro FORMAT:PATH --distro-identity UUID --upper (scratch:PATH | ram) -//! [--runtime-dir DIR] [--overlay-mode MODE] [--scratch-journal POLICY] -//! [--first-partition-bound BYTES] [--workload-identity UID:GID] [--network FILE] [SHARES]`, -//! gives the VM a layered root, which the guest agent's worker assembles; see -//! `runtime_adapter::layered` and `runtime_adapter::worker`. Its image is the raw file PATH, an -//! EROFS filesystem, `erofs`, or a GPT disk, `gpt`, of which the guest expects the identity UUID: -//! the file system's UUID, or the disk's GUID. Its upper is the ext4 scratch image PATH, or RAM. -//! The guest assembles the root below its runtime directory DIR, `/run/runtime-adapter` by -//! default, in the overlay mode MODE, `trusted-lowers` or `untrusted-lowers`, checks a scratch's -//! journal against the policy POLICY, `allowed` or `forbidden`, and refuses a first partition of -//! a GPT disk larger than BYTES, with the guest's defaults, `trusted-lowers`, `allowed`, and 64 -//! KiB, unless given. A workload identity, the user ID UID and the group ID GID, two numbers from -//! 1 to 4294967294, is the identity as which every execution of the sandbox runs: the guest -//! checks its account in the assembled root, without which the root's `identity` stage fails. The -//! VM has a network if FILE names one: an `nvx::host::sandbox::SandboxNetwork`, the guest's -//! address and the policy that OpenVMM enforces there, in the serde form of a plan's network, -//! which the launch reads before anything else, so that it refuses a network that the plan -//! refuses before it needs NVX's artifacts or the store, and puts on the adapter's plan, which -//! renders it; see `runtime_adapter::layered::read_network`. The root configuration says whether -//! the guest's listeners are published through the network, as OpenVMM publishes them, and -//! unpublished without one; see `runtime_adapter::layered::LayeredLaunch::with_network`. Before -//! it spawns the VM, the launch checks the bindings that STORE stages for the image and the -//! scratch image, from their metadata alone, and fails if STORE stages none for either, or one no -//! longer holds, as for an image that changed after it was staged. It hands OpenVMM the files -//! that the checks opened, which it holds until the guest has assembled the root, or until it has -//! torn down a launch that failed, and records the plan, with its network, and the bindings. -//! -//! SHARES, `--share HOST:TARGET[:ro]... [--deny PATH]... [--allow PATH]... -//! [--reserved TARGET]... [--share-root DIR]...`, shares each host directory or regular file -//! HOST with the root, at the absolute guest path TARGET, read-only with `:ro`, and read-write -//! otherwise. The guest path starts at the last `:/`. Each HOST must lie inside a host directory -//! DIR, which the adapter authorizes, once links are resolved: the launch refuses one outside -//! them before it plans the export, with NVX's `filesystem::plan`. The export hides each host -//! path PATH of `--deny` inside the shares, but shows again each host path PATH of `--allow` -//! inside a hidden one, with the directories on the way to it listing only it, and no TARGET may -//! take or cover a guest tree that the container root keeps, or a tree TARGET that `--reserved` -//! keeps for the adapter. The launch refuses a request that `filesystem::plan` refuses, such as -//! one with a read-only share inside a read-write one, before it spawns the VM, and checks the -//! export with `ExportPlan::verify` before it spawns it. OpenVMM exports the shares; see -//! `runtime_adapter::sharing::HostFsSharing`. -//! -//! Once attached, the launch follows the host's order; see `runtime_adapter::host`. It sends the -//! root configuration, which announces how many mappings the export's table holds, and waits for -//! the guest agent's `Ready`, then checks that the guest agent is of the build that the host -//! ships, sends the mapping table, in order, then `Start`, and exchanges the `Ping` and the -//! `Echo` once the guest started the sandbox. A guest agent of another build gets nothing more, -//! and the launch fails, which terminates the VM. If the guest reports that a stage of the root -//! failed instead, the launch waits for the VM to end, abandons it, and checks that the VM ended -//! with the stage's status. If the guest fails a mapping, the launch sends no more of the table, -//! nor `Start`, asks the guest agent to stop, which tears down the root, waits for the VM to -//! end, abandons it, and checks that the VM ended with success and a complete teardown. -//! - `runtime-adapter-host create STORE --guest-agent PATH --distro FORMAT:PATH --distro-identity -//! UUID [--hypervisor NAME] [--cpu-profile PROFILE] [--processors N] [--boot-processors B] -//! [--memory MIB] [--memory-capacity MIB] [--scratch-length BYTES] [--network FILE]` serves a -//! template of the guest agent PATH from the template store STORE, an -//! `nvx::host::template::Store`, which it creates if needed: a generation of the template's -//! class that the store holds, and that is eligible at the host's time, or else a new one, -//! which the store captures and publishes, with `Store::create_template`. The options choose -//! the class; see `runtime_adapter::capture`. The placeholder is the raw file PATH, an EROFS -//! filesystem, `erofs`, or a GPT disk, `gpt`, with the identity UUID, a trusted artifact of the -//! platform, whose length is the capacity of every clone's image. The template's VM runs on the -//! hypervisor NAME, as a launch's does, with the CPU profile PROFILE, `auto` by default, N -//! processors, 1 by default, of which B are online at boot, 1 by default, the fewest that its -//! clones take, since each clone's restore brings its own processor target online, up to N, MIB -//! mebibytes of RAM, 256 by default, and a RAM capacity of MIB mebibytes, its RAM by default, to -//! which a restore may expand it, and a dormant scratch of BYTES bytes, 64 MiB by default. It has -//! one NIC at the guest's address of the network FILE, which the creation reads as a launch reads -//! its network, under the complete deny-all policy, whatever the network's own, which each -//! clone's restore replaces with the clone's, unless the network's policy lets nothing through, -//! and none without a network: the networks of every policy at one guest's address share one -//! class, and so one template. The creation -//! reads the network before anything else, then stages the placeholder as `stage` stages an -//! image, verifying it explicitly, which hashes it only when a manifest is beside it, and checks -//! the class before it creates the store. A capture boots the template's VM through NVX's -//! supervisor, with the guest agent in place of NVX's agents, which captures the VM before it -//! assembles any root, in template mode; see `runtime_adapter::template`. The store checks what -//! the VM left, and publishes the snapshot as its next generation. The command writes the -//! initramfs of the guest agent into a directory of its own in the system's temporary -//! directory, private to the current user, which it removes once the creation returns, and -//! prints the generation only once the store has committed it. Once it served the generation, -//! the creation evicts the generations of the template's class that it supersedes: it retires -//! every other generation of the class that the store published before it, or that a tombstone -//! quarantines, and the tombstones that interrupted retirements left alone, whatever their -//! class; see `runtime_adapter::generations`. An eviction that fails leaves the rest for -//! the next one, and fails nothing that the creation served. -//! - `runtime-adapter-host refresh STORE` takes the options of `create`, and captures a new -//! generation of the template, as `create` captures one, with `Store::refresh_template`, whether -//! or not the store holds an eligible generation of its class, then evicts the generations of -//! the class that the new one supersedes, as `create` does. No other command captures, and no -//! restore triggers a capture. -//! - `runtime-adapter-host list STORE` reports each generation of the template store STORE, which -//! it opens, and does not create, in the order of their publication, from the store's index, -//! its tombstones, and the records of its leases alone: it reads no generation's payload, so it -//! loads no VM state, and hashes nothing, and it takes no lock. Its eligibility is at the host's -//! time; see `runtime_adapter::generations`. -//! - `runtime-adapter-host inspect STORE GENERATION` reports the generation of STORE whose store -//! generation is GENERATION, a positive number, as `list` reports it, with its leases and the -//! metadata of its capture. It fails for a generation that the store does not hold. -//! - `runtime-adapter-host remove STORE GENERATION` retires the generation GENERATION of STORE, -//! with `Store::retire`: it tombstones the generation first, so that it takes no new lease, and -//! deletes it once no lease of it remains, but never moves or deletes a generation that a clone -//! leases. While leases remain, it reports them, and leaves the generation tombstoned, which a -//! later `remove`, or the next eviction of its class, deletes once they drained. A generation -//! that the store no longer holds is retired already. -//! - `runtime-adapter-host reattach STORE` reattaches to the VM of STORE, which an earlier process -//! launched and which it holds although it is not its child, attaches a new session to it, in -//! the session's next epoch, and exchanges a `Ping`, leaving the VM running. -//! - `runtime-adapter-host start STORE --template TEMPLATES --generation G --distro FORMAT:PATH -//! --distro-identity UUID --upper (scratch:PATH | ram) [--processors N] [--memory MIB] -//! [--workload-identity UID:GID] [--overlay-mode MODE] [--scratch-journal POLICY] -//! [--first-partition-bound BYTES] [--machine-id (image | generated)] [--network FILE] [SHARES]` -//! starts a clone of the -//! generation G of the template store TEMPLATES, an `nvx::host::template::Store`, in the sandbox -//! store STORE, which it creates if needed; see `runtime_adapter::clone`. The clone's VM has the -//! network of the file FILE, if given, which the command reads first, as `launch` reads its -//! network: it must be at the guest's address of the template's NIC, under a policy of its own, -//! which the clone's restore installs on the NIC in place of the template's complete deny-all -//! policy, or for a template without a NIC none, or one that lets nothing through. SHARES, as -//! `launch` takes them, share -//! host paths with the clone's root: the command plans their export as a launch does, with -//! NVX's `filesystem::plan`, and refuses shares that it cannot export before anything but its -//! network. It then reads the generation from the store's index, and, unless N and MIB -//! mebibytes are both given, its compatibility key, before it needs NVX's artifacts: the key's -//! processors online at boot, or all of its processors for a template that records none, are -//! then the clone's processor target unless N is given, and its RAM the clone's memory target -//! unless MIB mebibytes are given. A key that cannot be read is left to NVX's plan, which reads -//! it again, and attributes the failure, to the generation or to the host, as it does for every -//! clone. The store then leases the generation to the sandbox, at the host's time, with -//! `Store::acquire`, which refuses a generation that a tombstone quarantines, or that is not -//! eligible then, and NVX's `ClonePlan` checks the clone against the generation, from the -//! metadata of its files -//! alone: its network, which must attach a NIC exactly when the template's key records one, at -//! the guest's address of the template's NIC, under a policy of its own, its image, the -//! raw file PATH, an EROFS filesystem, `erofs`, or a GPT disk, `gpt`, of -//! which the guest expects the identity UUID, no longer than the template's placeholder, -//! whatever the placeholder's format, and of its block sizes, its upper, the ext4 scratch image -//! PATH, of the geometry of the template's scratch, or RAM, which needs no file, with the -//! bindings that STORE stages for the images, as -//! `stage` stages them, its processor target, 1, 2, 4, or 8, from the template's processors -//! online at boot to its capacity, which the restore brings online, its memory target, in whole -//! blocks, from the template's RAM to its -//! capacity, and its export, if it has one, which must be one that a cold launch attaches, of -//! directories and files that neither lie inside the template store's directory nor hold it: -//! the clone's restore attaches it to the template's dormant filesystem slot as an aggregate. -//! A request that the template cannot serve fails there, before any VMM starts. The command -//! records the clone in STORE, with the template store and the generation; see -//! `runtime_adapter::clone::CloneRecord`. Once NVX checked the export again, with -//! `ExportPlan::verify`, OpenVMM restores the generation's snapshot in place, with the clone's -//! devices, through NVX's supervisor, as the launch that the lease reserves, and the restore -//! sequence of `nvx::host::template` activates the clone through the adapter's clone order, -//! `runtime_adapter::host::CloneOrder`, with the clone's activation record: the overlay mode -//! MODE and the scratch's journal policy -//! POLICY, as a launch takes them, for a GPT disk, the bound BYTES of its first partition, 64 -//! KiB unless given, as a launch's guest takes it by default, the export's shape and number of -//! mappings, and for a template with a NIC the clone's network: -//! the DNS server that OpenVMM serves the guest under the policy, if -//! any, which NVX finds as OpenVMM does, since the template's guest learned none, and whose -//! address the clone's guest writes to a resolver file that it binds read-only over the root's -//! `/etc/resolv.conf`, whether the policy publishes the workload's listeners, as it does with -//! ingress, or with host-loopback access both ways, for which the clone's guest starts their -//! announcer and refuses `CAP_NET_BIND_SERVICE` and `CAP_NET_RAW` to its executions, and the -//! URL of the policy's proxy, if any, which every execution then finds in its environment, as a -//! launch's executions do. Every -//! execution of the clone sees the hostname -//! `runtime-adapter`, and the machine ID that `--machine-id` chooses: the image's own, `image`, -//! by default, or one that the guest generates once for the clone, `generated`, which a -//! read-only bind holds at `/etc/machine-id`. The guest refuses a first partition larger than -//! BYTES, and an upper that is not fresh, whose `upper` or `work` directory on the scratch image -//! exists and is not empty, as an earlier clone's would be, before the clone is ready. Once the -//! clone is ready, the sequence sends the export's mapping table, in order, as a launch sends -//! its own; a clone without shares has no export: its restore leaves the slot dormant, and the -//! sequence sends no table. If the guest fails a mapping, the sequence sends no more of the -//! table, nor `Start`, and ends the clone, whose guest the clone order asks to stop, which tears -//! down the root, and which then fails as below. Once every mapping applied, the sequence -//! starts the clone's sandbox, and the command exchanges a `Ping` and the longest `Echo` with -//! the guest agent, commits the launch, prints the bytes that it read, `read-bytes`, and exits, -//! leaving the VM running, and the sandbox holding the lease, which a stop releases. -//! A clone that fails, from its plan to its commit, fails with NVX's -//! `nvx::host::template::CloneFailure`, which attributes the failure by its evidence, and records -//! it in STORE from the clone's launch on: the command prints the failure's record, releases the -//! lease once the failure's teardown is resolved, and nothing of the clone runs, or keeps it, and -//! the records, for a later reattach, while it is not, and quarantines the generation through -//! its tombstone, once it released the sandbox's lock, for a failure of the template. A guest -//! agent that does not answer the `Ping` or the `Echo` fails the clone so, through -//! `Started::fail`, as the restore sequence attributes the failures of its sessions. A start -//! that fails otherwise, at a step of the adapter's own before the clone's launch, releases the -//! lease. A selection that the template store refuses ends the start before it leases anything, -//! with the refusal's record, in the form of a failure's: a template store that does not exist, -//! and a generation that the store never published, or no longer holds, name no template; a -//! generation whose capture the restore window of its time contract no longer admits at the -//! host's time is expired; and one that a tombstone quarantines, or whose capture is after the -//! host's time, is ineligible. -//! - `runtime-adapter-host stop STORE` reattaches and attaches as `reattach` does, then stops the -//! VM: it asks the guest agent to stop, which first tears down a layered root, waits for the VM -//! to exit, and checks the VM's outcome report. A VM whose guest agent does not confirm the stop -//! is terminated. A stop of a clone releases the lease once the VM exited, and the clone's -//! records told how it ended, which the release removes, with the template store that the -//! clone's record names: a clone whose VM had exited when the stop began ended without its -//! holder, and one whose VM the guest ended, once the stop began, with a status that is evidence -//! of a failure, such as its repair's, failed, which ends the stop as it ends `start`. -//! - `runtime-adapter-host probe STORE PROBE...` reattaches and attaches as `reattach` does, then -//! runs each PROBE in the VM's layered root, and prints the facts that it found. A PROBE is the -//! text of a probe: its name, then its arguments, each after one space; see -//! `runtime_adapter::probe` and `runtime_adapter::worker::probes`. -//! - `runtime-adapter-host exec STORE EXECUTION [; EXECUTION]...` reattaches and attaches as -//! `reattach` does, then runs each EXECUTION in the VM's layered root, all at once, over that one -//! session, through `runtime_adapter::host::Executions`, and prints their outputs and outcomes -//! once every one finished. It runs at most 8 executions, as many as the guest runs at once; see -//! `runtime_adapter::exec::MAX_EXECUTIONS`. An EXECUTION, `[--identity UID:GID [--groups GID,...] -//! [--capabilities N,...] [--numeric]] [--cwd DIR] [--env NAME=VALUE]... -- PROGRAM [ARG]...`, -//! runs PROGRAM with its ARGs, up to a lone `;`, as `find -exec` takes them, without standard -//! input. Without `--identity`, it names no identity, and runs as the sandbox's workload -//! identity, which the guest otherwise refuses; with it, it runs as the user ID UID and the group -//! ID GID, with the supplementary groups GIDs and the capabilities numbered N, such as 0 for -//! `CAP_CHOWN`, if given, and the guest checks its account, unless `--numeric` says that the IDs -//! have none. It starts in the root's directory DIR, or `/`, with exactly the environment of its -//! `--env` entries, then, if the VM's network has a proxy, the proxy's URL as the guest sees it -//! in `http_proxy`, `https_proxy`, `HTTP_PROXY`, and `HTTPS_PROXY`, unless the entries name any -//! of them; see `runtime_adapter::layered::with_proxy`. Before it starts any execution, the -//! command checks that the framing still carries each start with the proxy's URL, and runs none -//! of the executions if one no longer fits. It runs for at most a minute, with 16 KiB chunks -//! and the credit of 4 of them for its input and 8 for its output, which its standard output and -//! error share. The command keeps at most 16 MiB of each stream, and cancels an execution whose -//! stream would exceed them; see `runtime_adapter::exec::MAX_CAPTURE`. Execution N, from 1, has -//! the key `(B + N, E)`: E is the epoch of the command's attachment, and B the first 8 bytes of -//! the broker instance's ID, as a little-endian number. The guest refuses a key that it heard of -//! before, and the epochs of a broker instance only advance, while a broker restored from a -//! snapshot has a new instance ID, so no command repeats the key of another's execution. -//! - `runtime-adapter-host run STORE --template TEMPLATES --generation G --distro FORMAT:PATH -//! --distro-identity UUID --upper (scratch:PATH | ram) [--processors N] [--memory MIB] -//! [--workload-identity UID:GID] [--overlay-mode MODE] [--scratch-journal POLICY] -//! [--first-partition-bound BYTES] [--machine-id (image | generated)] [--network FILE] [SHARES] -//! EXECUTION` runs one execution in -//! a clone of the generation G of the template store TEMPLATES, in the sandbox store STORE, -//! which it creates if needed, then tears the clone down. The EXECUTION is one of `exec`'s, whose -//! options may come among the start's, in any order, and whose PROGRAM and ARGs are every -//! argument after `--`. The command first reads the clone's network, and plans the clone and the -//! export of its shares, as `start` does, then checks that the guest takes the execution's -//! start, with the URL of the proxy of the clone's network, if any, in its environment, as -//! `exec` gives it to an execution of a clone, so that none of them fails the command once it -//! staged anything. Its preflight then stages the clone's images in STORE, as `stage` stages -//! them, which verifies the image, and hashes it only when a manifest is beside it, and prints -//! that it did, before the clone's start, which hashes nothing. It starts the clone as `start` -//! does, and ends as a start ends once the template store refuses the selection, or the clone -//! fails. It runs the execution in the clone's root, as `exec` runs one, through the guest -//! agent's execution control, in the session in which the clone's sandbox started, so that the -//! program, its arguments, and its environment reach the guest in the execution's start, never -//! in the template's boot tokens, and prints its output and outcome once it finished. The -//! execution finds the clone's shares beneath the root, since the restore sequence maps them -//! before the clone's sandbox starts. Then it stops the clone, as `stop` does, whatever the -//! execution did, which releases the lease once the VM exited. The stop takes the same session, -//! in which it skips the guest's messages about the execution that may still arrive once the -//! command finished it without them, as when it gave up waiting for its outcome. It holds the -//! sandbox's lock, which the lease took, until then, so no other command reattaches to the clone -//! meanwhile. -//! -//! Each command but `stage`, `list`, `inspect`, and `remove`, which need no artifacts, takes NVX's -//! artifacts from the directory that `NVX_ARTIFACTS_DIR` names, a repository checkout or an -//! extracted release, and the VM's runtime identity from NVX's identity of them, with the guest -//! agent's build that `--guest-build BUILD` names, this crate's version by default; see -//! `runtime_adapter::runtime`. Reattach finds a VM that booted cold under another runtime identity -//! mismatched, and then signals nothing: the VM keeps running. -//! -//! The sandbox of a clone holds the clone's lease, through whose path the commands that reattach -//! take the sandbox's lock. They reattach to a clone with `nvx::host::template::reattach_clone`, -//! in a new session of the adapter's clone order, in which the clone answers the Activate of its -//! record again, with the Ready that it answered once, and which they then use as `reattach` -//! attaches one. The reattach ends a clone that runs under another runtime identity, without a -//! word to its guest, or that fails the check of its Ready, through its verified identity, and -//! finishes the record of an earlier failure of the clone, whose teardown it resolves if it can. -//! The command then releases the lease of a clone whose VM ended without a failure, and fails; -//! ends as `start` does for a clone that failed; and keeps the lease and the records, and fails, -//! for a clone whose state cannot be determined. It releases a lease through the template store -//! that the clone's record names, once the lease is of it. A sandbox whose start ended before it -//! recorded the clone holds a lease of a template store that no record names: it records no -//! clone, or an earlier clone's, which a release leaves, of another template store perhaps. -//! `reattach STORE --template TEMPLATES` and `stop STORE --template TEMPLATES` release such a -//! lease through the template store TEMPLATES, in place of the record's, which the release refuses -//! unless the lease is of it. -//! -//! The commands print what they find as `KEY=VALUE` lines: -//! -//! - `reattached`: what reattach found: `live`, `mismatched`, `exited`, or `indeterminate`, and of -//! a clone, `live`, `ended`, `failed`, or `indeterminate`; -//! - `differences`: the parts of the runtime identity in which a mismatched VM differs, separated -//! by commas; -//! - `progress`: the step that a clone, which ended, reached: `planned`, `launching`, `spawned`, -//! `event`, `acknowledged`, `activating`, `ready`, `starting`, `started`, or `committed`; -//! - `pid`: the VMM's process ID; -//! - `instance` and `epoch`: the broker instance, in hexadecimal, and the epoch of the session -//! that the command attached to; -//! - `root`: how the guest answered the root configuration: `ready` or `failed`, or `failed` -//! once a stop finds that the root's teardown failed; -//! - `build`: the build of the guest agent that assembled the root; -//! - `stage` and `problem`: the stage of the root that failed, such as `assembly`, and the -//! problem, or the problem of a mapping that failed; -//! - `table`: how the guest took the mapping table: `applied` or `failed`; -//! - `mapped`: how many of the table's mappings the guest applied; -//! - `sandbox`: `started` once the guest started the sandbox; -//! - `preflight`: `staged` once `run` staged the images of its clone; -//! - `PROBE.KEY`: the fact KEY that the probe named PROBE found; -//! - `exec.N.KEY`: what execution N did, with these KEYs: -//! - `launch`: `started`, `failed STAGE ERRNO`, with the number of the guest's stage that failed -//! and the error number, or `none` once the guest reported neither; -//! - `status`: how the program ended: `exit CODE`, `signal NUMBER`, or `none`; -//! - `cause`: why a side ended the execution, `cancelled`, `dropped`, `timed-out`, `reset`, or -//! `disconnected`, `other` for a cause that `nvx` adds later, or `none` for an execution that -//! ended on its own; -//! - `truncated`: the streams that lost bytes, `stdout`, `stderr`, or both, separated by a -//! comma, or `none`; -//! - `settled`: `reusable` once the execution settled, or `fenced`; -//! - `stdout` and `stderr`: the program's standard output and error; -//! - `stopped`: how the stop ended: `graceful`, `forced`, or `timed-out`, or `exited` once a -//! clone's VM, which the command found running, had exited when the stop began, which stopped -//! nothing; -//! - `outcome`: the category and status code of the VM's outcome report, such as `success 0`; -//! - `teardown`: whether the VM's teardown completed: `complete` or `incomplete`; -//! - `created`: how `create` or `refresh` served the template: `captured`, with a generation that -//! a capture made and the store published, which a refresh always prints, or `existing`, with -//! one that the store held; -//! - `generation`, `key`, and `class`: the store generation of the template that `create` or -//! `refresh` served, or that `inspect` reports, and the identities of its compatibility key and -//! of its capture class, each 64 hexadecimal digits, or the store generation that `start` or -//! `run` leased; -//! - `retired` and `draining`: the generations that the eviction of `create` or `refresh` retired, -//! each list separated by commas, or `none`: those of the class that it deleted, or found gone -//! already, with those whose interrupted retirement it completed, and those that leases keep, -//! tombstoned; -//! - `lease`: the identity of the lease that `start` or `run` took, 32 hexadecimal digits; -//! - `downtime` and `source`: the downtime of the leased generation's capture at the host's time, -//! in nanoseconds, and the clock that measured it: `monotonic`, the host's monotonic clock, on -//! the host's boot that captured the template, or `utc` otherwise; -//! - `released`: the identity of the lease that a stop, a start that failed, or the teardown of a -//! run released; -//! - `failure.KEY`: the record of a clone's failure, or of the selection of a start or a run that -//! the template store refused, with these KEYs: -//! - `outcome`: what the failure is attributed to: `request`, `staging`, `host`, `unknown`, -//! `template`, or `repair`, or, for a refused selection, `no-template`, `expired`, or -//! `ineligible`; -//! - `evidence`: what the host saw of it, which attributes it, such as `supplied-length`, or, -//! for a refused selection, `no-store`, `unpublished`, or `retired`, of no template, -//! `expired`, or `tombstone` or `early`, of an ineligible generation; -//! - `stage`: where the clone failed: `plan`, `launch`, `restore`, `activation`, `start`, -//! `commit`, `run`, or `reattach`, or `selection` for a refused selection; -//! - `progress`: the step that the clone reached, as `progress` spells it, or `none` for a -//! refused selection; -//! - `status`: the status with which the guest ended the VM, as OpenVMM's outcome report gives -//! it, or `none`; -//! - `teardown`: how the teardown that followed ended: `no-vm`, `exited`, `terminated`, or -//! `unresolved`; -//! - `code`: the code of the failure's error, such as `runtime-mismatch`, or `not-found`, -//! `template-tombstoned`, or `template-ineligible` for a refused selection; -//! - `quarantined`: whether the command quarantined the generation of a clone that failed: -//! `yes`, for a failure of the template, or `no`; -//! - `store`: the identity of the template store that `list` or `inspect` read, 32 hexadecimal -//! digits; -//! - `generations`: the store generations that `list` found, separated by commas, or `none`; -//! - `generation.G.KEY`: what `list` found of the generation G, with the KEYs `key`, `class`, -//! `state`, `eligibility`, `reason`, and `leases`, which `inspect` prints of its generation; -//! - `state`: `available`, or `tombstoned` once a tombstone quarantines the generation; -//! - `eligibility` and `reason`: whether a clone may restore the generation at the host's time, -//! `eligible`, `early`, or `expired`, and NVX's reason, with the downtime of its capture; -//! - `leases`: the identities of the generation's leases that the store's records list, each 32 -//! hexadecimal digits, separated by commas, or `none`; -//! - `lease.ID.sandbox` and `lease.ID.launch`: the directory of the sandbox that holds the lease ID, -//! with its links resolved, and the launch of its clone that the lease reserves; -//! - `storage`: the storage generation of the capture, 32 hexadecimal digits; -//! - `time.host`, `time.boot`, `time.clock`, `time.utc`, `time.monotonic`, and `time.lineage`: the -//! time contract of the capture: the identities of the host and of its boot that captured it, -//! each 32 hexadecimal digits, the host's monotonic clock, `linux-boottime` or -//! `windows-interrupt-time`, the anchor of the capture in UTC, in nanoseconds since the Unix -//! epoch, and by that clock, in nanoseconds, and its time lineage counter; -//! - `length.state` and `length.memory`: the lengths of the snapshot's saved state and of its RAM, -//! in bytes; -//! - `removed`: what `remove` did: `retired`, once the generation is gone, or `draining`, once it -//! tombstoned the generation, which the leases that it reports keep; -//! - `read-bytes`: how many bytes `stage`, or a `start` that started its clone, read through the -//! operating system's read calls, the cache's included, as it exits, as -//! `nvx::host::process::own_bytes_read` counts them. -//! -//! Values escape `\`, newlines, and carriage returns as `\\`, `\n`, and `\r`, and each byte of a -//! program's output that is not part of UTF-8 text as `\xHH`, with two hexadecimal digits. -//! -//! For tests, `launch`, `reattach`, and `stop` take a step hook from the variable -//! `RUNTIME_ADAPTER_STEP_HOOK`, which pauses the command at a named step, for a time that a launch -//! leaves out of its deadline, or exits it there without unwinding, with status 5; see -//! `runtime_adapter::hook`. `start` takes one at `checked` alone: once NVX's `ClonePlan` checked -//! the clone against the leased generation, and holds the files that its checks opened, before the -//! start records the clone, and before the clone's launch checks its export again. `start` and -//! `run` take the host's time, at which the store checks the generation's eligibility, from the -//! variable `RUNTIME_ADAPTER_HOST_TIME`, if it is set, as a clock fixture, such as another boot's; -//! see -//! `runtime_adapter::clone::HOST_TIME_VARIABLE`. `create` and `refresh` take the delay of the -//! repair of each clone of the template that a capture makes from the variable -//! `RUNTIME_ADAPTER_REPAIR_DELAY`, in milliseconds, if it is set, which they write into the -//! capture's initramfs, and name as the class's invariant setting; see -//! `runtime_adapter::capture::REPAIR_DELAY_VARIABLE`. -//! -//! They exit with 0 once they did what they were asked, `exec` once every execution finished, -//! whatever its outcome, and `run` once its execution finished so, and the clone stopped, as -//! `stop` stops one, with 3 when reattach finds a VM that booted cold mismatched, with 4 when -//! the guest reported that a stage of the root failed, and the VM ended with the stage's status -//! and a complete teardown, with 5 when the step hook ends a command, with 6 when the guest failed -//! a mapping of the table, and the VM stopped gracefully, with success and a complete teardown, -//! with 7 when `remove` finds leases that keep the generation, which it tombstoned, -//! with 8 when a clone failed, once nothing of it runs and the command released its lease, with 9 -//! when a clone failed, and its teardown is unresolved, so that the sandbox keeps the lease, with -//! 10 when the template store refused the selection of a start or a run, so that it leased and -//! launched nothing, and with 1, after a diagnostic on standard error, when they fail, as `run` -//! does once its execution failed to run, after the teardown, unless the teardown ended with 4, 8, -//! or 9, as when the clone's VM exited during the execution, which `run` then exits with, after -//! the diagnostic. Anything else is a usage error, with status 2, as is a value of -//! `RUNTIME_ADAPTER_STEP_HOOK` that sets no step hook, or a step hook of a start at another step -//! than `checked`, a value of `RUNTIME_ADAPTER_HOST_TIME` that is not a host's time, or a value of -//! `RUNTIME_ADAPTER_REPAIR_DELAY` that sets no delay. - -use std::env; -use std::error::Error; -use std::ffi::OsString; -use std::fs; -use std::io; -use std::num::NonZeroU64; -use std::path::{Path, PathBuf}; -use std::process::ExitCode; -use std::time::{Duration, Instant}; - -use nvx::activation::{MachineIdMode, StoreGeneration}; -use nvx::exec::{Cause, Key, Limits, Outcome, Status, Stream, Timers, Window}; -use nvx::host::artifacts::{Artifacts, absolute}; -use nvx::host::filesystem::ExportPlan; -use nvx::host::instance::generate_capability; -use nvx::host::launch::{DEFAULT_MEMORY_MIB, LaunchSpec}; -use nvx::host::report::{self, Report}; -use nvx::host::sandbox::SandboxPlan; -use nvx::host::session; -use nvx::host::store::{CallerFiles, IdentityPart, LOG_NAME, LaunchId, LockedStore, SandboxStore}; -use nvx::host::supervisor::{self, Reattached, Starting, StopOutcome, Vmm, VmmCommand}; -use nvx::host::template::{ - Acquired, CAPTURES_NAME, Capture, CloneControl, CloneFailure, ClonePlan, Created, - DowntimeSource, Eligibility, Evidence, FailureRecord, Generation, HostTime, Lease, - ReattachedClone, RestoreDeadlines, Retirement, Store, TRANSACTION_NAME, reattach_clone, -}; -use nvx::protocol::CAPABILITY_LEN; -use nvx::protocol::root::ImageFormat; -use nvx::uuid::Uuid; -use nvx::{ErrorCode, ErrorKind, Hypervisor}; -use runtime_adapter::GUEST_BUILD; -use runtime_adapter::capture::{ - DEFAULT_BOOT_PROCESSORS, DEFAULT_CPU_PROFILE, DEFAULT_PROCESSORS, DEFAULT_SCRATCH_LENGTH, - Request, repair_delay_from_env, -}; -use runtime_adapter::clone::{CloneRecord, host_time_from_env}; -use runtime_adapter::exec::{self, AccountMode, HostExecution, Start}; -use runtime_adapter::frame::{MAX_BODY, Message}; -use runtime_adapter::generations::{self, Entry}; -use runtime_adapter::hook::{STEP_HOOK, Step, StepHook}; -use runtime_adapter::host::{self, Client, ClientError, CloneOrder, Executions, OrderError}; -use runtime_adapter::initramfs; -use runtime_adapter::layered::{Bindings, LaunchRecord, LayeredLaunch, read_network, with_proxy}; -use runtime_adapter::probe::{Probe, ProbeReply, escape}; -use runtime_adapter::root::{JournalPolicy, OverlayMode, RootFailure}; -use runtime_adapter::runtime::Runtime; -use runtime_adapter::sharing::{HostFsSharing, Share}; -use runtime_adapter::staged::Staged; - -/// How long a command waits for the sandbox's lock, and `remove` and an eviction for the template -/// store's lock to retire a generation. -const LOCK_TIMEOUT: Duration = Duration::from_secs(10); -/// How long a launch waits for the VM to boot, its guest agent to assemble a layered root, and to -/// answer. -const START_TIMEOUT: Duration = Duration::from_secs(60); -/// How long reattach, stop, and probe wait to attach a session and for the guest agent's answers. -const SESSION_TIMEOUT: Duration = Duration::from_secs(30); -/// How long a stop waits for the guest agent's confirmation and for the VM to exit, and how long -/// a launch whose root failed waits for the VM to exit. -const STOP_TIMEOUT: Duration = Duration::from_secs(30); -/// How long a stop waits for a VM that it terminated to exit. -const FORCED_TIMEOUT: Duration = Duration::from_secs(10); -/// The name of the initramfs that boots the guest agent, which a launch writes into the sandbox's -/// store, and a creation into a private directory of its own. -const INITRAMFS_NAME: &str = "runtime-adapter.cpio.gz"; -/// How long a creation waits for the slot of the template's class and for the template store's -/// lock, and for a capture, which boots the template's VM and writes its RAM. -const CAPTURE_TIMEOUT: Duration = Duration::from_secs(120); -/// The prefix of the name of the directory, private to the current user, into which a creation -/// writes the initramfs of a capture's VM, in the system's temporary directory. -const CAPTURE_DIRECTORY_PREFIX: &str = "runtime-adapter-template-"; -/// The hexadecimal digits of the identity of a capture class that name its slot in a template -/// store, as the store documents its layout. -const SLOT_NAME_LEN: usize = 16; -/// Bytes in a mebibyte, the unit of `create`'s `--memory` and `--memory-capacity`. -const MIB: u64 = 1024 * 1024; -/// The exit status of a command that finds the VM mismatched. -const MISMATCHED: u8 = 3; -/// The exit status of a command whose guest reported that a stage of the root failed, once the VM -/// ended with the stage's status. -const ROOT_FAILED: u8 = 4; -/// The exit status of a launch whose guest failed a mapping of the table, once the VM stopped -/// gracefully, with success and a complete teardown. -const TABLE_FAILED: u8 = 6; -/// The exit status of a removal that found leases of the generation, which keep it, tombstoned, -/// until their clones release them. -const BUSY: u8 = 7; -/// The exit status of a command whose clone failed, as NVX attributes and records the failure, -/// once nothing of the clone runs: the failure's teardown is resolved, and the command released -/// the clone's lease. -const CLONE_FAILED: u8 = 8; -/// The exit status of a command whose clone failed, once the failure's teardown is unresolved: a -/// VM of the clone may still run, or NVX could not record the failure before it ended the VM, and -/// the sandbox keeps the clone's lease, and NVX's records of its launch, for a later reattach. -const CLONE_UNRESOLVED: u8 = 9; -/// The exit status of a start whose template store refused its selection: the store does not -/// exist, or does not hold the generation, or the generation is expired or ineligible at the -/// host's time, so that the start leased and launched nothing. -const SELECTION_REFUSED: u8 = 10; -// The timers and limits of `exec` below are values chosen for tests that make a valid -// configuration, within the limits that the guest and the host allow: chunks of at most -// `exec::MAX_CHUNK` bytes, at most `worker::MAX_EXECUTION_MEMORY` bytes held in the guest for -// each execution, and output windows of at most `host::MAX_OUTPUT_FRAMES` frames. -/// How long an execution of `exec` may run before its sides end it. -const EXEC_TIMEOUT: Duration = Duration::from_secs(60); -/// How long the pending output of an execution of `exec` still flows once a side ended it. -const EXEC_GRACE: Duration = Duration::from_secs(1); -/// How long an execution of `exec` may take to settle, once its program or a side ended it. -const EXEC_SETTLEMENT: Duration = Duration::from_secs(5); -/// The most bytes of a chunk of an execution of `exec`. -const EXEC_CHUNK: u32 = 16 * 1024; -/// The chunks of an execution's input that the guest may hold for `exec`. -const EXEC_INPUT_FRAMES: u32 = 4; -/// The chunks of an execution's output, of both streams, that `exec` may hold. -const EXEC_OUTPUT_FRAMES: u32 = 8; -/// The argument that ends an execution's arguments, after which another execution may follow. -const EXEC_END: &str = ";"; -/// The options of an execution, which `run` takes among those of a start. -const EXECUTION_OPTIONS: [&str; 6] = [ - "--identity", - "--groups", - "--capabilities", - "--numeric", - "--cwd", - "--env", -]; -/// The command lines that the program accepts. -const USAGE: &str = concat!( - "usage: runtime-adapter-host\n", - " runtime-adapter-host stage STORE --distro FORMAT:PATH --distro-identity UUID", - " [--upper scratch:PATH]\n", - " runtime-adapter-host launch STORE --guest-agent PATH [--hypervisor NAME]", - " [--guest-build BUILD]\n", - " [--distro FORMAT:PATH --distro-identity UUID", - " --upper (scratch:PATH | ram) [--runtime-dir DIR]\n", - " [--overlay-mode MODE] [--scratch-journal POLICY]", - " [--first-partition-bound BYTES]\n", - " [--workload-identity UID:GID] [--network FILE]\n", - " [--share HOST:TARGET[:ro]... [--deny PATH]... [--allow PATH]...\n", - " [--reserved TARGET]... [--share-root DIR]...]]\n", - " runtime-adapter-host (create | refresh) STORE --guest-agent PATH --distro FORMAT:PATH", - " --distro-identity UUID\n", - " [--hypervisor NAME] [--guest-build BUILD] [--cpu-profile PROFILE]", - " [--processors N]\n", - " [--boot-processors N] [--memory MIB] [--memory-capacity MIB]", - " [--scratch-length BYTES]\n", - " [--network FILE]\n", - " runtime-adapter-host list STORE\n", - " runtime-adapter-host (inspect | remove) STORE GENERATION\n", - " runtime-adapter-host (reattach | stop) STORE [--guest-build BUILD]", - " [--template TEMPLATES]\n", - " runtime-adapter-host start STORE --template TEMPLATES --generation G", - " --distro FORMAT:PATH\n", - " --distro-identity UUID --upper (scratch:PATH | ram) [--guest-build BUILD]", - " [--processors N]\n", - " [--memory MIB] [--workload-identity UID:GID] [--overlay-mode MODE]", - " [--scratch-journal POLICY]\n", - " [--first-partition-bound BYTES] [--machine-id (image | generated)]", - " [--network FILE]\n", - " [--share HOST:TARGET[:ro]... [--deny PATH]... [--allow PATH]...\n", - " [--reserved TARGET]... [--share-root DIR]...]\n", - " runtime-adapter-host run STORE --template TEMPLATES --generation G", - " --distro FORMAT:PATH\n", - " --distro-identity UUID --upper (scratch:PATH | ram) [--guest-build BUILD]", - " [--processors N]\n", - " [--memory MIB] [--workload-identity UID:GID] [--overlay-mode MODE]", - " [--scratch-journal POLICY]\n", - " [--first-partition-bound BYTES] [--machine-id (image | generated)]", - " [--network FILE]\n", - " [--share HOST:TARGET[:ro]... [--deny PATH]... [--allow PATH]...\n", - " [--reserved TARGET]... [--share-root DIR]...] EXECUTION\n", - " runtime-adapter-host probe STORE PROBE... [--guest-build BUILD]\n", - " runtime-adapter-host exec STORE [--guest-build BUILD] EXECUTION [';' EXECUTION]...\n", - " EXECUTION: [--identity UID:GID [--groups GID,...] [--capabilities N,...]", - " [--numeric]]\n", - " [--cwd DIR] [--env NAME=VALUE]... -- PROGRAM [ARG]...", -); - -/// What the program was asked to do. -enum Command { - /// Report the contract that the program was built against. - Report, - /// Stage the images of a layered root. - Stage { - /// The sandbox's store. - store: PathBuf, - /// The image. - distro: PathBuf, - /// The image's format. - format: ImageFormat, - /// The identity that the guest expects of the image. - identity: Uuid, - /// The ext4 scratch image, if any. - scratch: Option, - }, - /// Launch a VM of the guest agent. - Launch { - /// The sandbox's store. - store: PathBuf, - /// The guest agent's executable. - guest_agent: PathBuf, - /// The hypervisor, unless `nvx::Hypervisor::from_env_or_default` selects it. - hypervisor: Option, - /// The guest agent's build. - guest_build: String, - /// The VM's layered root, if it has one. - layered: Option>, - /// The step hook, for tests, if `RUNTIME_ADAPTER_STEP_HOOK` sets one. - hook: Option, - }, - /// Serve a template from a template store, which captures one if it holds none, or which a - /// refresh captures anyway. - Create { - /// The template store. - store: PathBuf, - /// The guest agent's executable. - guest_agent: PathBuf, - /// The hypervisor, unless `nvx::Hypervisor::from_env_or_default` selects it. - hypervisor: Option, - /// The guest agent's build. - guest_build: String, - /// The template. - template: Box