diff --git a/.github/workflows/run-nvx-microvm-tests.yml b/.github/workflows/run-nvx-microvm-tests.yml index 082567b1d..0bfbd703b 100644 --- a/.github/workflows/run-nvx-microvm-tests.yml +++ b/.github/workflows/run-nvx-microvm-tests.yml @@ -239,6 +239,8 @@ jobs: --scenario sandbox-ram-upper --scenario sandbox-gpt-layer --scenario vm-lifecycle + --scenario snapshot-core + --scenario snapshot-tiers --output-dir "build/test-results/public-exec-${{ inputs.backend }}" - name: Run public managed sandbox acceptance on Windows @@ -253,6 +255,8 @@ jobs: --scenario sandbox-ram-upper --scenario sandbox-gpt-layer --scenario vm-lifecycle + --scenario snapshot-core + --scenario snapshot-tiers --output-dir "build/test-results/public-exec-${{ inputs.backend }}" - name: Run Ubuntu guest smoke tests on Linux @@ -331,7 +335,6 @@ jobs: --scenario managed-lifecycle --scenario network-snapshot --scenario smp - --scenario snapshot-core --scenario workload-identity --output-dir "build/test-results/ubuntu-kvm-extended" diff --git a/doc/acceptance-case-matrix.md b/doc/acceptance-case-matrix.md index 620e518d7..70c4d4f42 100644 --- a/doc/acceptance-case-matrix.md +++ b/doc/acceptance-case-matrix.md @@ -13,9 +13,10 @@ docs, and each microVM scenario, one fate, by name: SL5.1b (#1118) added the smoke tests on real VMs of a cold boot: `test-microvm --scenario vm-lifecycle` creates, spawns, reattaches to and stops a VM through the public API, and the CLI's 12 cold cells, EROFS and GPT images, each over a scratch or a RAM upper, run on KVM, MSHV and WHP, -as [Cold boots](#cold-boots) lists. The later slice is SL5.1c (#1118), whose smoke tests capture a +as [Cold boots](#cold-boots) lists. SL5.1c (#1118) added those of snapshots, which capture a snapshot, open it and restore it with nvx-init as the guest, run concurrent restores of one -snapshot under the CLI's lock, and check that a failed restore reports its attribution. +snapshot under the CLI's lock, and check that a failed restore reports its attribution, as +[Snapshots](#snapshots) lists. The inventory is fixed: it lists the tests as they are where this matrix was added, from `tests/runtime-adapter/scenarios.txt` and the adapter's sources, `MICROVM_TEST_SCENARIOS` and @@ -29,7 +30,7 @@ its own in turn. | --- | ---: | ---: | ---: | ---: | | [The adapter's scenarios](#the-adapters-scenarios) | 157 | 0 | 114 | 43 | | [The adapter's other tests](#the-adapters-other-tests) | 169 | 0 | 9 | 160 | -| [MicroVM scenarios](#microvm-scenarios) | 54 | 50 | 3 | 1 | +| [MicroVM scenarios](#microvm-scenarios) | 54 | 51 | 2 | 1 | | [CI](#ci) | 69 | 43 | 3 | 23 | | [Docs](#docs) | 36 | 35 | 1 | 0 | @@ -211,13 +212,13 @@ The adapter's README says what each proves. | `a_removal_reports_busy_while_leases_keep_the_generation` | retired | Removals, leases, tombstones and generations leave with the old clone path (#1121) | | `a_template_whose_processors_are_all_online_at_boot_limits_none` | rewritten | `nvx-rs/src/host/snapshot/tests.rs::snapshot_accepts_treats_zero_boot_processors_as_all_capacity_online`; `nvx-rs/src/host/config/tests.rs::fresh_captures_pass_with_supported_processors_and_aligned_memory`; `test-microvm --scenario smp-snapshot` | | `a_template_without_a_network_has_no_nic` | rewritten | `nvx-rs/src/host/vm/tests.rs::capture_creation_records_sparse_disks_devices_and_a_complete_serde_context`; `nvx-rs/src/host/snapshot/tests.rs::snapshot_accepts_refuses_both_nic_presence_mismatches_and_each_named_address_mismatch`. A store that serves it again retires (#1121) | -| `the_networks_of_every_policy_at_one_guest_s_address_share_one_template` | rewritten | `nvx-rs/src/host/network/tests.rs::templates_attach_their_nic_under_the_complete_deny_all_policy`; `test-microvm --scenario untiered-network-policy`, which SL5.1c rewrites. A cached template retires (#1121) | +| `the_networks_of_every_policy_at_one_guest_s_address_share_one_template` | rewritten | `nvx-rs/src/host/network/tests.rs::templates_attach_their_nic_under_the_complete_deny_all_policy`; `test-microvm --scenario untiered-network-policy`, which SL3.7c (#1114) rewrites on a strict snapshot with a NIC. A cached template retires (#1121) | ### `vm_clone` (36) | Scenario | Fate | Coverage, or why it retires | | --- | --- | --- | -| `a_clone_binds_the_resolver_of_its_policy_s_dns_server_read_only` | rewritten | `test-microvm --scenario untiered-network-policy`, which SL5.1c rewrites; `test-microvm --scenario network-snapshot`; `nvx-rs/src/guest/root/tests/view.rs::actual_resolver_failures_keep_the_network_class_and_root_owned_bind` | +| `a_clone_binds_the_resolver_of_its_policy_s_dns_server_read_only` | rewritten | `test-microvm --scenario untiered-network-policy`, which SL3.7c (#1114) rewrites on a strict snapshot with a NIC; `test-microvm --scenario network-snapshot`; `nvx-rs/src/guest/root/tests/view.rs::actual_resolver_failures_keep_the_network_class_and_root_owned_bind` | | `a_clone_maps_an_aggregate_of_directories_and_files_beneath_its_root_before_its_sandbox_starts` | rewritten | `test-microvm --scenario platform-aggregate-export`; `nvx-rs/src/guest/mappings/table/tests.rs::mappings_keep_the_paths_and_the_access_of_their_entries` | | `a_clone_of_a_template_with_a_nic_maps_its_export_beside_its_resolver` | rewritten | `nvx-rs/src/guest/activation/state/apply/tests.rs::the_record_s_view_is_applied_once_with_its_own_dns_and_returned_machine_id`; `test-microvm --scenario filesystem-snapshot` | | `a_clone_reads_no_more_of_its_disks_than_its_root_needs` | rewritten | A restore reads no content of its disks or snapshot: `nvx-rs/src/host/snapshot/tests.rs::snapshot_accepts_never_reopens_snapshot_paths_or_reads_runtime_artifacts`; `nvx-rs/src/host/sandbox/distro/tests.rs::the_staged_check_reads_no_content`; `nvx-rs/src/host/sandbox/scratch/tests.rs::checks_claim_the_image_without_reading_it`. The adapter's in-guest read statistics retire | @@ -225,7 +226,7 @@ The adapter's README says what each proves. | `a_clone_runs_the_executions_that_name_their_identity_once_it_is_ready` | retired | Executions that name their identities leave with the exec engine (#1087) | | `a_clone_s_executions_find_the_proxy_of_its_policy` | rewritten | `nvx-rs/src/guest/managed_agent/launch/tests.rs::restored_proxy_defaults_replace_template_values_before_exec_overrides`; `test-microvm --scenario platform-network-policy` | | `a_clone_s_time_settles_after_it_is_ready` | rewritten | `test-microvm --scenario time-host-correction`; `test-microvm --scenario restore-downtime` | -| `a_clone_whose_activation_fails_ends_before_it_is_ready` | rewritten | `nvx-rs/src/host/restore/tests.rs::restore_failures_attribute_the_existing_policy_free_evidence_classes`; `nvx-rs/src/host/restore/tests.rs::restore_exit_reports_keep_guest_status_classification_and_actual_details`; SL5.1c's failed restore. Quarantine and leases retire (#1121) | +| `a_clone_whose_activation_fails_ends_before_it_is_ready` | rewritten | `nvx-rs/src/host/restore/tests.rs::restore_failures_attribute_the_existing_policy_free_evidence_classes`; `nvx-rs/src/host/restore/tests.rs::restore_exit_reports_keep_guest_status_classification_and_actual_details`; `test-microvm --scenario snapshot-core`, whose restore over a zeroed scratch image ends its VM before its workload runs. Quarantine and leases retire (#1121) | | `a_clone_whose_mapping_fails_ends_before_its_sandbox_starts` | rewritten | `nvx-rs/src/guest/mappings/mapper/tests.rs::a_failure_at_any_step_leaves_nothing_mounted`; `nvx-rs/src/guest/mappings/table/tests.rs::a_failed_mapping_keeps_those_before_it_and_latches_the_table` | | `a_clone_whose_view_cannot_be_set_ends_before_it_is_ready` | rewritten | `nvx-rs/src/guest/activation/state/apply/tests.rs::the_record_s_view_is_applied_once_with_its_own_dns_and_returned_machine_id`; `nvx-rs/src/guest/root/tests/view.rs::a_later_bind_failure_stays_owned_and_its_teardown_removes_the_earlier_bind` | | `a_clone_whose_vm_exits_once_a_stop_found_it_fails_before_its_lease_is_released` | retired | Clone records and leases leave with the old clone path (#1121). A stop that finds an exit: `nvx-rs/src/host/vm/tests.rs::stop_preserves_report_results_after_a_real_self_exit` | @@ -233,8 +234,8 @@ The adapter's README says what each proves. | `a_generation_is_eligible_by_the_clock_of_its_capture` | rewritten | `nvx-rs/src/host/snapshot/eligibility/tests.rs::within_one_boot_the_downtime_is_the_host_monotonic_clocks`; `nvx-rs/src/host/snapshot/tests.rs::snapshot_accepts_reports_early_and_expired_time_as_unavailable` | | `a_generation_refuses_clones_of_fewer_processors_than_its_template_booted_with` | rewritten | `nvx-rs/src/host/snapshot/tests.rs::snapshot_accepts_refuses_a_blockless_root_and_each_processor_mismatch`; `test-microvm --scenario restore-processors` | | `a_host_program_of_another_runtime_ends_the_clone_through_its_verified_identity` | rewritten | `nvx-rs/tests/security_checklist.rs::a_saved_vm_runs_until_its_process_exits_and_another_runtime_is_refused_without_a_signal`; `nvx-rs/src/host/vm/tests.rs::stop_forces_only_its_verified_process_after_grace`. Leases retire (#1121) | -| `a_run_in_a_template_of_a_network_finds_its_policy_s_proxy` | rewritten | `test-microvm --scenario platform-network-policy`; SL5.1c's restore smoke tests. The template's `run` retires (#1121) | -| `a_run_runs_one_execution_in_its_clone_then_tears_the_clone_down` | rewritten | SL5.1c's restore smoke tests. The template's `run` and leases retire (#1121) | +| `a_run_in_a_template_of_a_network_finds_its_policy_s_proxy` | rewritten | `test-microvm --scenario platform-network-policy`; `test-microvm --scenario snapshot-core`, whose runs restore a snapshot without a NIC until SL3.7c (#1114). The template's `run` retires (#1121) | +| `a_run_runs_one_execution_in_its_clone_then_tears_the_clone_down` | rewritten | `test-microvm --scenario snapshot-core`, whose `run --restore-snapshot` runs one workload in its restored VM and removes its sandbox. The template's `run` and leases retire (#1121) | | `a_run_s_execution_finds_its_shares_beneath_the_root` | rewritten | `test-microvm --scenario platform-aggregate-export` | | `a_run_whose_clone_does_not_start_runs_nothing` | retired | Template selections and the template's `run` leave with the old clone path (#1121) | | `a_run_whose_clone_s_vm_exits_during_its_execution_ends_with_the_clone_s_failure` | retired | The template's `run`, clone records and leases leave with the old clone path (#1121) | @@ -243,15 +244,15 @@ The adapter's README says what each proves. | `a_start_refuses_what_the_template_cannot_serve_before_any_vmm_starts` | rewritten | `nvx-rs/src/host/snapshot/tests.rs::snapshot_accepts_refuses_an_image_longer_than_its_placeholder`; `nvx-rs/src/host/snapshot/tests.rs::snapshot_accepts_refuses_each_scratch_geometry_mismatch`; `nvx-rs/src/host/snapshot/tests.rs::snapshot_accepts_refuses_each_memory_mismatch_without_rounding_or_overflow` | | `a_template_without_a_nic_serves_clones_without_a_network` | rewritten | `nvx-rs/src/host/snapshot/tests.rs::snapshot_accepts_refuses_both_nic_presence_mismatches_and_each_named_address_mismatch` | | `a_tombstone_leaves_the_running_clone_of_its_generation_to_its_lease` | retired | Tombstones, generations and leases leave with the old clone path (#1121) | -| `clones_at_once_keep_their_own_uppers_and_machine_ids` | rewritten | SL5.1c's concurrent restores of one snapshot; `test-microvm --scenario scratch-snapshot` | -| `clones_at_once_see_only_their_own_exports` | rewritten | SL5.1c's concurrent restores of one snapshot; `test-microvm --scenario platform-aggregate-export` | +| `clones_at_once_keep_their_own_uppers_and_machine_ids` | rewritten | `test-microvm --scenario snapshot-core`, whose concurrent restores of one snapshot each keep their own upper and machine ID; `test-microvm --scenario scratch-snapshot` | +| `clones_at_once_see_only_their_own_exports` | rewritten | `test-microvm --scenario snapshot-core`, whose concurrent restores of one snapshot each see their own share alone; `test-microvm --scenario platform-aggregate-export` | | `each_command_reattaches_to_the_clone_from_a_new_process` | rewritten | `nvx-rs/src/host/vm/tests/reattach.rs::reattaching_returns_the_held_owned_child_and_authenticated_connection`; `nvx-rs/tests/vm_lifecycle.rs::a_cold_vm_is_reattached_from_saved_values_and_stopped`. The adapter's probes retire | | `one_generation_serves_clones_of_each_processor_count_from_its_boot_online_count_to_its_capacity` | rewritten | `test-microvm --scenario restore-processors`; `test-microvm --scenario platform-restore-processors` | | `one_generation_serves_clones_of_each_upper_and_memory_target` | rewritten | `test-microvm --scenario scratch-snapshot`; `test-microvm --scenario restore-memory`. Leases retire (#1121) | -| `one_generation_serves_clones_of_either_format_over_either_upper` | rewritten | `test-microvm --scenario snapshot-tiers`, which SL5.1c rewrites; `nvx-rs/src/host/snapshot/tests.rs::snapshot_accepts_matching_real_disk_geometry_and_every_resource_boundary` | -| `one_template_serves_the_clones_of_every_network_policy_at_its_guest_s_address` | rewritten | `test-microvm --scenario untiered-network-policy`, which SL5.1c rewrites; `test-microvm --scenario platform-network-policy` | +| `one_generation_serves_clones_of_either_format_over_either_upper` | rewritten | `test-microvm --scenario snapshot-tiers`; `nvx-rs/src/host/snapshot/tests.rs::snapshot_accepts_matching_real_disk_geometry_and_every_resource_boundary` | +| `one_template_serves_the_clones_of_every_network_policy_at_its_guest_s_address` | rewritten | `test-microvm --scenario untiered-network-policy`, which SL3.7c (#1114) rewrites on a strict snapshot with a NIC; `test-microvm --scenario platform-network-policy` | | `openvmm_reads_none_of_the_host_s_input_until_the_repair_acknowledges_the_restore` | retired | It watches OpenVMM's own gating of the host's input through the repair delay of the adapter's templates, which leaves with them. NVX's side stays: `nvx-rs/src/guest/activation/state/tests.rs::the_deadline_starts_at_the_acknowledgement_and_retries_never_extend_it`; `nvx-rs/src/guest/init/boot/tests.rs::restored_sessions_keep_the_deadline_of_the_actual_repair_acknowledgement` | -| `restore_exclusion::hard_linked_restore_views_obey_the_callers_exclusion` (linux) | rewritten | SL5.1c's concurrent restores of one snapshot under the CLI's lock. The view that a restore hard-links, which `vm::create` documents: `nvx-rs/src/host/snapshot/tests.rs::a_restore_view_hard_links_only_the_fixed_openvmm_inventory`; `nvx-rs/src/host/vm/tests.rs::snapshot_creation_owns_the_fixed_view_and_uses_captured_nic_with_requested_policy`; `nvx-rs/src/host/vm/tests.rs::snapshot_creation_refuses_a_true_cross_filesystem_view_without_copying`; `nvx-rs/src/host/vm/tests.rs::snapshot_creation_never_replaces_existing_outputs_views_or_listeners`; `nvx-rs/src/host/vm/tests.rs::the_same_prebound_listener_keeps_its_queue_through_both_launch_paths_without_inheritance`. OpenVMM's own checks before its restore-ready connection, which fail a restore whose view changed or whose caller died, and which `vm::create` documents too, leave NVX's tests | +| `restore_exclusion::hard_linked_restore_views_obey_the_callers_exclusion` (linux) | rewritten | `test-microvm --scenario snapshot-core`, whose concurrent restores of one snapshot take turns at the CLI's lock and leave no link to its files. The view that a restore hard-links, which `vm::create` documents: `nvx-rs/src/host/snapshot/tests.rs::a_restore_view_hard_links_only_the_fixed_openvmm_inventory`; `nvx-rs/src/host/vm/tests.rs::snapshot_creation_owns_the_fixed_view_and_uses_captured_nic_with_requested_policy`; `nvx-rs/src/host/vm/tests.rs::snapshot_creation_refuses_a_true_cross_filesystem_view_without_copying`; `nvx-rs/src/host/vm/tests.rs::snapshot_creation_never_replaces_existing_outputs_views_or_listeners`; `nvx-rs/src/host/vm/tests.rs::the_same_prebound_listener_keeps_its_queue_through_both_launch_paths_without_inheritance`. OpenVMM's own checks before its restore-ready connection, which fail a restore whose view changed or whose caller died, and which `vm::create` documents too, leave NVX's tests | | `the_lease_of_a_start_that_ended_before_its_record_is_released_through_its_template_store` | retired | Leases and clone records leave with the old clone path (#1121) | ## The adapter's other tests @@ -296,7 +297,8 @@ check the adapter's own code, and CI runs in its configurations of the adapter. `nvx.py test-microvm` runs these scenarios, which use no adapter: CI runs those of the default suite in its shards, and the public sandbox scenarios in its guest and sandbox jobs. The public `sandbox-gpt-layer` and `vm-lifecycle` scenarios came with SL5.1b, after this matrix; -[Cold boots](#cold-boots) says what they check. +[Cold boots](#cold-boots) says what they check. SL5.1c rewrote `snapshot-core` and +`snapshot-tiers` as public scenarios; [Snapshots](#snapshots) says what they check. | Scenario | Fate | Note | | --- | --- | --- | @@ -343,13 +345,13 @@ suite in its shards, and the public sandbox scenarios in its guest and sandbox j | `smp` | kept | | | `smp-lapic` | kept | Runs only when a run names it | | `smp-snapshot` | kept | | -| `snapshot-core` | rewritten | SL5.1c rewrites it on strict snapshots, keeping its DNS, egress, forwards and publication; only its raw-snapshot parts retire | +| `snapshot-core` | rewritten | On a named snapshot, as [Snapshots](#snapshots) says. Its checks of raw snapshots retire: the guest's capture with `nvx-snapshot` and its terminal boundary; a snapshot request without a destination, which OpenVMM releases, and which no scenario sends any longer, though `snapshot-rollback` keeps the checks of the source after a released request, which follows the capture that it rolls back; the CPU clocks of a sleeping task, an armed timer's wait, the restore packet's entropy file and `nvx-reseed`, the VM generation ID, and the time ABI's port access. It had no network | | `snapshot-rollback` | kept | | -| `snapshot-tiers` | rewritten | SL5.1c rewrites it on strict snapshots, keeping its DNS, egress, forwards and publication; only its raw-snapshot parts retire | +| `snapshot-tiers` | rewritten | On a named snapshot of the platform tier, as [Snapshots](#snapshots) says. Its checks of raw snapshots retire: the workload-start and instance-checkpoint tiers, a restore of the wrong tier, the restore's input gate and its timeout, a restore of an instance checkpoint without `--machine microvm`, and a second claim of one, and its run on the debug kernel. It checked nothing of its NIC | | `structured-outcome` | kept | | | `time-abi-conformance` | kept | | | `time-host-correction` | kept | | -| `untiered-network-policy` | rewritten | SL5.1c rewrites it on strict snapshots, keeping its DNS, egress, forwards and publication; only its raw-snapshot parts retire | +| `untiered-network-policy` | kept | On a raw snapshot, until a strict snapshot can carry a NIC: SL3.7c (#1114) rewrites it on one. It keeps the coverage of a restore's DNS, egress, forwards and listener publication until then | | `virtio-features` | kept | | | `virtio-net` | kept | | | `vm-lifecycle` | kept | A [cold boot](#cold-boots) through the public API: `nvx-rs/tests/vm_lifecycle.rs::a_cold_vm_is_reattached_from_saved_values_and_stopped` | @@ -378,8 +380,29 @@ directory. `test-microvm --scenario vm-lifecycle` cold-boots a VM through the pu `nvx-rs/tests/vm_lifecycle.rs::a_cold_vm_is_reattached_from_saved_values_and_stopped` creates and spawns it, with nvx-init over the EROFS layer and a scratch upper, lets the launch go, reattaches to it twice from the values that it saved, and stops it through its guest, which must exit without -a forced termination and report a complete teardown. No cell or smoke test captures or restores a -snapshot: SL5.1c adds those. +a forced termination and report a complete teardown. No cell captures or restores a snapshot: +[Snapshots](#snapshots) lists the smoke tests that do. + +### Snapshots + +`test-microvm --scenario snapshot-core` and `test-microvm --scenario snapshot-tiers`, which CI's +guest and sandbox job of each backend runs, capture a named snapshot with `nvx-rs snapshot +capture`, whose VM nvx-init runs, and restore it with `nvx-rs snapshot restore` and `nvx-rs run +--restore-snapshot`, which open it as a strict snapshot under the CLI's lock of the snapshot, as +[Run](run.md#named-snapshots) describes. + +| Smoke test | Where | +| --- | --- | +| A real VM is captured, opened as a strict snapshot and restored with nvx-init | `snapshot-core`: two one-off restores and three managed ones, whose guests keep the snapshot's uptime, read the host's clock, and take fresh random values and machine IDs. `snapshot-tiers`: a managed restore of each image format, EROFS and GPT, over each upper, scratch and RAM | +| Concurrent restores of one snapshot are serialized by the CLI's lock | `snapshot-core`: three restores at once, whose workloads end only when the scenario releases them. While one holds the lock, no other links its view of the snapshot, and a restore that cannot wait for the lock fails with `sandbox-busy` before its workload runs. The three then run at once, each with its own share, upper and machine ID | +| The selected failed restore reports its attribution | `snapshot-core`: a scratch image that the snapshot cannot serve is refused before any VM starts, with an error that names the lengths of the requested scratch and of the snapshot's, which attributes the failure to the request; the managed sandbox of the refused restore holds its configuration and its lock alone, with no trace of a launch. A restore that fails in its guest ends its VM, frees the lock, and reports `guest-exit status 125` | + +The CLI prints no attribution of a restore that fails in its guest beyond the guest's exit +status: nvx-init's status for each class of failure, and the CLI's report of the restore's +attribution, are a follow-up on #1118. A snapshot that `snapshot capture` captures has no NIC, so +its restores take no network option: `untiered-network-policy` keeps the coverage of a restore's +DNS, egress, forwards and listener publication on raw snapshots until SL3.7c (#1114) lets a +capture keep a NIC. ## CI @@ -421,9 +444,9 @@ their own. The tests that a step runs have rows of their own where they concern | `.github/workflows/ci.yml` | crate-release: Check the nvx package | kept | | | `.github/workflows/ci.yml` | nvx-microvm-adapter-whp: the runtime adapter's job on WHP | retired | The adapter's scenarios, whose fates are above | | `.github/workflows/run-nvx-microvm-tests.yml` | Run NVX microVM tests on Linux and on Windows | kept | The default suite's scenarios, by shard and on the debug kernel | -| `.github/workflows/run-nvx-microvm-tests.yml` | Run public managed sandbox acceptance on Linux and on Windows | kept | Runs `sandbox-gpt-layer` and `vm-lifecycle` too, since SL5.1b: see [Cold boots](#cold-boots) | +| `.github/workflows/run-nvx-microvm-tests.yml` | Run public managed sandbox acceptance on Linux and on Windows | kept | Runs `sandbox-gpt-layer` and `vm-lifecycle` too, since SL5.1b: see [Cold boots](#cold-boots); and `snapshot-core` and `snapshot-tiers`, since SL5.1c: see [Snapshots](#snapshots) | | `.github/workflows/run-nvx-microvm-tests.yml` | Run Ubuntu and Azure Linux guest smoke tests on Linux and on Windows | kept | | -| `.github/workflows/run-nvx-microvm-tests.yml` | Run extended Ubuntu guest tests on Linux/KVM | kept | | +| `.github/workflows/run-nvx-microvm-tests.yml` | Run extended Ubuntu guest tests on Linux/KVM | kept | No longer runs `snapshot-core`, which needs the Alpine control guest since SL5.1c rewrote it | | `.github/workflows/run-nvx-microvm-tests.yml` | Run the Ubuntu sandbox layer, live-share, managed live-share, caller-owned live-share, concurrent shares and share access policy tests on Linux and on Windows | kept | The layer smoke test is the `run` of a [cold cell](#cold-boots) | | `.github/workflows/run-nvx-microvm-tests.yml` | Build the runtime adapter's guest agent on Linux; Download the runtime adapter's guest agent on Windows | retired | The adapter's guest agent | | `.github/workflows/run-nvx-microvm-tests.yml` | Run runtime adapter scenarios on Linux and on Windows | retired | The adapter's scenarios, whose fates are above | diff --git a/doc/ci.md b/doc/ci.md index 3ea335e33..70ce313f1 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -234,21 +234,18 @@ a cold boot that declares that profile, and passing boot checks. It then captures the guest and restores it with `auto` and no hook: the restore must take the snapshot's host profile without another warning, and pass the warp probe and its restore checks. -The `snapshot-core` scenario first sends a snapshot request to a guest that -OpenVMM launched without a snapshot destination. OpenVMM releases the request, -so it returns in the source, which must continue exactly once. Before the -request, the guest's snapshot agent saved and overrode the stall detectors' -settings, and it must restore them when the request returns -(`nvx-time cancel-capture`). Afterwards, `nvx-time status` must exit 0 with a -passing boot line at generation 0 and no restore line, and -`/sys/module/rcupdate/parameters/rcu_cpu_stall_suppress` must read 0. The `snapshot-rollback` scenario fails a capture after its quiesce. The guest's RAM has a file of its own, so the snapshot destination's parent holds nothing, and it disappears after boot: the capture quiesces and saves the VM, then fails to publish, and OpenVMM rolls it back. The request must return in the source with status 0, after OpenVMM's capture profile records exactly one quiesce and one save but no publication, and the source must pass the checks -that `snapshot-core` runs after its released request. Once the parent is back, +of a released request. Before the request, the guest's snapshot agent saved +and overrode the stall detectors' settings, and it must restore them when the +request returns (`nvx-time cancel-capture`): `nvx-time status` must then exit 0 +with a passing boot line at generation 0 and no restore line, and +`/sys/module/rcupdate/parameters/rcu_cpu_stall_suppress` must read 0. Once the +parent is back, a second request of the same source must publish a snapshot and end the source with status 0, leaving only the snapshot in the parent, and the snapshot's restore must continue past that request exactly once. @@ -350,8 +347,8 @@ line and adds it to the GitHub job summary, because CI keeps the guest logs only for failed jobs: the number of warp probe runs with their worst `max_abs_offset_ns` and `max_backward_ns`, the count and `elapsed_us` range of the newest check that each `nvx-time status` query reported (boot after a cold -boot, restore after a restore, and capture after `snapshot-core`'s released -request), the exhaustive check's summary, and the +boot, restore after a restore, and capture after `snapshot-rollback`'s +rolled-back request), the exhaustive check's summary, and the `restore-downtime` stall counts. Where the guest reports a check's CPU time (`cpu_us`), the line also gives its range and how many checks exceed the backend's CPU-time budget for their phase. The budgets come from guest @@ -367,8 +364,8 @@ The `nvx-microvm-debug-{kvm,mshv,whp}` jobs run `test-microvm --debug-kernel` on the CI debug kernel (`build/vmlinux-debug`, built from `kernel/config-microvm-debug`), whose soft-lockup and hung-task detectors production kernels leave out. It selects the same-host restore scenarios -`smp`, `smp-snapshot`, `restore-processors`, `restore-downtime`, and -`snapshot-tiers`. The jobs skip every other test step, because those boot the +`smp`, `smp-snapshot`, `restore-processors`, and `restore-downtime`. The jobs +skip every other test step, because those boot the production kernel, which the `nvx-microvm-guests-{kvm,mshv,whp}` jobs and WHP's runtime adapter job already cover: the public managed execution configuration, the Ubuntu and Azure Linux guest tests, the Ubuntu sandbox @@ -376,11 +373,9 @@ layer and live-share smoke tests, the caller-owned live-share test, and the runtime adapter scenarios. Any RCU stall, soft lockup, or hung task makes the guest's time ABI watcher power off with status 194, which fails the run. Every guest those scenarios restore -asks `nvx-time status` before it exits -(`snapshot-tiers` after its tier assertions), which waits for the restore's -deferred checks and must report a passing restore line, so no restore leaves -its checks pending. The one exception is `snapshot-tiers`' gate-timeout check, -whose guest OpenVMM stops at the restore gate on purpose. The harness refuses +asks `nvx-time status` before it exits, which waits for the restore's deferred +checks and must report a passing restore line, so no restore leaves its checks +pending. The harness refuses a kernel whose `vmlinux-debug.config` lacks the detectors, because the guest's boot check of those detectors passes vacuously. To bound the cost, pull requests run the debug kernel on MSHV only, rather than on KVM, whose general runners @@ -560,7 +555,14 @@ tmpfs in guest memory, with no scratch image, and sandboxes whose root is a GPT disk, over either upper, so that the three backends run the CLI's twelve cold cells (see the [case matrix](acceptance-case-matrix.md#cold-boots)). The public `vm-lifecycle` scenario then creates, spawns, reattaches to, and stops -a VM through the public API of `nvx`. +a VM through the public API of `nvx`. The public `snapshot-core` and +`snapshot-tiers` scenarios capture a named snapshot with +`nvx-rs snapshot capture`, with nvx-init as the guest, and restore it with +`snapshot restore` and `run --restore-snapshot` (see +[Run](run.md#named-snapshots)): restores of each image format over each upper, +repeated restores with fresh entropy and machine IDs, clocks that follow the +host's across the downtime, concurrent restores that take turns at the +snapshot's lock, failed restores, and snapshots that no restore changes. Their bounded evidence is part of the backend's failure logs. The same entrypoint then verifies a live virtio-fs share inside the container: a read-write `/workspace` share with a denied diff --git a/doc/run.md b/doc/run.md index 921e359f5..0f9ae62ad 100644 --- a/doc/run.md +++ b/doc/run.md @@ -1113,6 +1113,86 @@ reports of each run and stop. With `sandbox-lifecycle` and `sandbox-ram-upper`, CI runs it on KVM, MSHV, and WHP: the CLI's twelve cold cells, which the [case matrix](acceptance-case-matrix.md#cold-boots) lists. +### Named snapshots + +`nvx-rs snapshot capture` captures a VM that nvx-init runs, before any +workload, with a placeholder image, a dormant scratch, and no NIC, into a named +snapshot of a store, which `nvx-rs snapshot restore` and +`nvx-rs run --restore-snapshot` restore under the snapshot's lock, with an +image and an upper of their own; see +[`snapshot capture`](usage.md#snapshot-capture) and +[`snapshot restore`](usage.md#snapshot-restore). The public `snapshot-core` and +`snapshot-tiers` scenarios of `test-microvm --backend BACKEND` are their +acceptance, with the Ubuntu layer and the scratch template that +`nvx-rs download` installs. The workload of each restore is a probe, a +script that `/bin/sh -c` runs, which reports the guest's clock and uptime, +fresh random values, its machine ID, host name, and identity, the lower and +upper directories of its root's overlay, its block devices and network +interfaces, the distribution of its image, and the marker file of its host +share at `/data`, if it has one. + +The `snapshot-core` scenario captures one snapshot, whose directory must hold +exactly `key.bin`, `snapshot.json`, and `openvmm/`, each file with one link, +while no directory or OpenVMM process of the capture's VM remains. After a +downtime longer than the capture, two `run --restore-snapshot` restores of the +Ubuntu EROFS layer, in RAM as `65534:65534` and over a fresh copy of the +scratch template as `1:1`, each with a host name of its own, must run as the +restore's identity with its host name, the layer as the root's lower directory +over the upper, the snapshot's two block devices, and no NIC, with a clock +that reads the host's time and an uptime that kept the time before the capture +and counts the downtime, and must report different random values, kernel +UUIDs, machine IDs, and temporary file names. A `snapshot restore` over a +scratch image of half the dormant scratch's length is refused before any VM +starts: its error names the lengths of the requested scratch and of the +dormant one, which attributes the failure to the request, and its sandbox, +which a failed start keeps provisioned, must hold `config.json` and +`sandbox.lock` alone, where a launch leaves OpenVMM's log and outcome report +and the view of the snapshot, until `sandbox deprovision` removes it. A +`run --restore-snapshot` over a zeroed scratch image of the dormant scratch's +length, which the guest cannot mount as its upper, ends its VM before its +workload runs and frees the snapshot's lock, which the restores that follow +take, and the run reports the guest's exit: +`OpenVMM exited during startup: guest-exit status 125`. + +Then three `snapshot restore` restores start at once, each with a host name +and a read-only host share of its own, the last over a scratch upper and the +others in RAM, and a workload that ends only once the scenario creates a +`release` file in its share, which the guest reads live. While the first holds +the snapshot's lock, a `run --restore-snapshot` restore that may wait one +second for the lock fails with `sandbox-busy` before its workload runs, and +neither of the other two may link its view of the snapshot, which a restore +does only once it holds the lock. Once the scenario releases the first, one of +the other two takes the lock, and the last may link its view only once the +scenario releases that one. The three sandboxes then run at once: each guest +must read the marker of its own share, and a file that a `sandbox exec` writes +in one sandbox's `/tmp`, in its upper, must appear in no other. Each must stop with an outcome report of a successful restored VM +without a network policy and a complete teardown, and be deprovisioned. No +restore may change the snapshot's files, leave a link to them, or leave an +OpenVMM process. + +The `snapshot-tiers` scenario captures one snapshot of the platform tier, the +one tier that `snapshot capture` captures, and restores it with +`snapshot restore` into a managed sandbox of each image format over each +upper: the Ubuntu EROFS layer, and a GPT disk of its tree that a guest of the +Alpine control initramfs writes, as `sandbox-gpt-layer` does, each over a +fresh copy of the scratch template and in RAM. Each guest must have its image +as its root's lower directory, `/run/nvx/layers/distro` or +`/run/nvx/layers/p2`, over its upper, and a `sandbox exec` in its sandbox must +see the same root of the same guest. Each sandbox must stop cleanly and be +deprovisioned, and the restores change neither image nor the snapshot. + +Both scenarios journal the OpenVMM processes of their managed restores, as +the other scenarios of the `snapshot-isolation` +[adversarial campaign](adversarial-testing.md) journal theirs. Their evidence, +with a `snapshot-core-` or `snapshot-tiers-` prefix, holds the bounded command +observations, the output of each command, and the outcome reports of the +managed sandboxes, and that of `snapshot-tiers` the console of the guest that +wrote its GPT disk. CI runs them on KVM, MSHV, and WHP with the public sandbox +scenarios. A snapshot that `snapshot capture` captures has no NIC, so its +restores take no network option, and the raw `untiered-network-policy` +scenario keeps the coverage of a restore's DNS, egress, forwards, and listener +publication until a capture can keep a NIC. + ### Live host shares `sandbox run` and `sandbox provision` accept repeatable diff --git a/doc/usage.md b/doc/usage.md index 00c3265a8..d0ebf4914 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -1254,7 +1254,9 @@ which runs only when named: it repeats `smp` and also asserts that every CPU uses the one-shot counting LAPIC. `managed-exec-config`, `sandbox-lifecycle`, `sandbox-proxy-only`, `sandbox-ram-upper`, and `sandbox-gpt-layer`, the public sandbox scenarios, drive the public `nvx-rs sandbox` lifecycle commands (see -[Run](run.md#managed-lifecycle)), and `vm-lifecycle` drives a VM through the +[Run](run.md#managed-lifecycle)), `snapshot-core` and `snapshot-tiers` capture +and restore named snapshots with `nvx-rs snapshot` and `nvx-rs run` (see +[Run](run.md#named-snapshots)), and `vm-lifecycle` drives a VM through the public API of `nvx`; they also read the Ubuntu distro layer, its manifest, and the sandbox scratch template, which [`download`](#download) installs and the guest-artifact build makes; a run @@ -1263,16 +1265,16 @@ files is missing. Alpine remains the default. Ubuntu and Azure Linux cannot act as sandbox control, so they reject the Alpine-control-only `interrupt-map`, `sandbox-blocks`, `scratch-snapshot`, and `virtio-features` scenarios and -the sandbox-control-dependent `snapshot-tiers`, `platform-restore-processors`, -`platform-zero-fill`, `platform-filesystem`, `platform-aggregate-export`, -`platform-network-policy`, `managed-exec-config`, `sandbox-lifecycle`, -`sandbox-proxy-only`, `sandbox-ram-upper`, `sandbox-gpt-layer`, and -`vm-lifecycle` scenarios. Ubuntu also rejects +the sandbox-control-dependent `snapshot-core`, `snapshot-tiers`, +`platform-restore-processors`, `platform-zero-fill`, `platform-filesystem`, +`platform-aggregate-export`, `platform-network-policy`, `managed-exec-config`, +`sandbox-lifecycle`, `sandbox-proxy-only`, `sandbox-ram-upper`, +`sandbox-gpt-layer`, and `vm-lifecycle` scenarios. Ubuntu also rejects the Alpine-prompt-specific `console-snapshot` scenario. `--debug-kernel` boots the CI debug kernel, `build/vmlinux-debug`, whose soft-lockup and hung-task detectors the guest's time ABI watcher reports; without `--scenario`, it runs only the same-host restore scenarios `smp`, `smp-snapshot`, -`restore-processors`, `restore-downtime`, and `snapshot-tiers`. +`restore-processors`, and `restore-downtime`. `--shard INDEX/COUNT`, such as `1/2`, runs shard INDEX of the COUNT shards of the default Alpine suite on the default kernel, the scenarios of a run without `--scenario` but the public sandbox scenarios, which CI runs in its guest and diff --git a/nvx-rs/src/host/platform/linux/endpoint.rs b/nvx-rs/src/host/platform/linux/endpoint.rs index b088a9e40..427dc7139 100644 --- a/nvx-rs/src/host/platform/linux/endpoint.rs +++ b/nvx-rs/src/host/platform/linux/endpoint.rs @@ -1,10 +1,10 @@ //! The Unix sockets of OpenVMM's endpoints: its control endpoint and the transport over it, and //! the listener of a clone's restore-ready event. -use std::fs::{self, OpenOptions, Permissions}; +use std::fs::{self, File, Permissions}; use std::io::{self, Read}; use std::os::fd::{AsFd, AsRawFd}; -use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; +use std::os::unix::fs::PermissionsExt; use std::os::unix::net::{UnixListener, UnixStream}; use std::path::Path; use std::thread; @@ -227,11 +227,16 @@ fn set_ready_socket_mode(socket_path: &Path) -> io::Result<()> { use std::os::unix::fs::FileTypeExt; use nix::fcntl::OFlag; + use nix::sys::stat::Mode; - let socket = OpenOptions::new() - .read(true) - .custom_flags((OFlag::O_PATH | OFlag::O_NOFOLLOW).bits()) - .open(socket_path)?; + // `nix` passes the flags on as they are, where `OpenOptionsExt::custom_flags` masks out the + // bits of `O_ACCMODE`, which on musl include `O_PATH`: a musl build would open the socket + // itself, which the kernel refuses with `ENXIO`. + let socket = File::from(nix::fcntl::open( + socket_path, + OFlag::O_PATH | OFlag::O_NOFOLLOW | OFlag::O_CLOEXEC, + Mode::empty(), + )?); if !socket.metadata()?.file_type().is_socket() { return Err(io::Error::other(format!( "{} is not the restore-ready socket that the launch bound", diff --git a/scripts/nvx_tools/adversarial_cases/snapshot-isolation.json b/scripts/nvx_tools/adversarial_cases/snapshot-isolation.json index 184316305..16ce8b90a 100644 --- a/scripts/nvx_tools/adversarial_cases/snapshot-isolation.json +++ b/scripts/nvx_tools/adversarial_cases/snapshot-isolation.json @@ -124,11 +124,11 @@ { "id": "snapshot-tier-boundary", "scenario": "snapshot-tiers", - "description": "Exercise policy-tier dispatch and cross-instance state isolation across restores.", + "description": "Restore one platform-tier snapshot into sandboxes of each image format over each upper, each with a root of its own.", "coverage": [ "snapshot-tiers", - "cross-instance-isolation", - "tier-validation" + "image-formats", + "cross-instance-isolation" ], "repetitions": 1, "timeout_multiplier": 6 diff --git a/scripts/nvx_tools/microvm_shards.json b/scripts/nvx_tools/microvm_shards.json index d5002a80b..e5f4e4bc5 100644 --- a/scripts/nvx_tools/microvm_shards.json +++ b/scripts/nvx_tools/microvm_shards.json @@ -60,9 +60,7 @@ "scratch-snapshot": 3.4, "smp": 21.2, "smp-snapshot": 26.1, - "snapshot-core": 13.4, "snapshot-rollback": 1.5, - "snapshot-tiers": 6.8, "structured-outcome": 0.6, "time-abi-conformance": 0.7, "time-host-correction": 14.0, @@ -73,12 +71,11 @@ }, "shards": [ { - "seconds": 267.8, + "seconds": 258.5, "scenarios": [ "command-line-limit", "console-snapshot", "directional-network-policy", - "endpoint-policy-snapshot", "filesystem-comma-paths", "filesystem-snapshot", "lifecycle", @@ -88,20 +85,20 @@ "restore-downtime", "restore-memory", "restore-processors", + "smp", "smp-snapshot", - "snapshot-core", - "snapshot-tiers", "time-abi-conformance", "time-host-correction", "virtio-features" ] }, { - "seconds": 269.3, + "seconds": 258.4, "scenarios": [ "console-exit", "cpu-profile-fallback", "denied-filesystem-paths", + "endpoint-policy-snapshot", "filesystem-owner", "filesystem-policy", "filesystem-shares", @@ -120,7 +117,6 @@ "private-network-ingress", "sandbox-blocks", "scratch-snapshot", - "smp", "snapshot-rollback", "structured-outcome", "untiered-network-policy", @@ -168,9 +164,7 @@ "scratch-snapshot": 3.2, "smp": 22.6, "smp-snapshot": 28.5, - "snapshot-core": 12.9, "snapshot-rollback": 1.5, - "snapshot-tiers": 6.3, "structured-outcome": 0.5, "time-abi-conformance": 0.8, "time-host-correction": 14.0, @@ -181,7 +175,7 @@ }, "shards": [ { - "seconds": 250.3, + "seconds": 239.8, "scenarios": [ "command-line-limit", "console-snapshot", @@ -190,23 +184,21 @@ "filesystem-shares", "filesystem-snapshot", "interrupt-map", + "ipv6-egress-policy", "platform-filesystem", "platform-restore-processors", "platform-zero-fill", "restore-downtime", - "restore-processors", "scratch-snapshot", "smp", "smp-snapshot", - "snapshot-core", "snapshot-rollback", - "snapshot-tiers", "time-host-correction", "untiered-network-policy" ] }, { - "seconds": 248.7, + "seconds": 240.0, "scenarios": [ "console-exit", "cpu-profile-fallback", @@ -217,7 +209,6 @@ "guest-boot", "guest-identity", "host-loopback-policy", - "ipv6-egress-policy", "l3-l4-egress-policy", "l3-l4-egress-port-ranges", "l3-l4-egress-protocols", @@ -228,6 +219,7 @@ "platform-network-policy", "private-network-ingress", "restore-memory", + "restore-processors", "sandbox-blocks", "structured-outcome", "time-abi-conformance", @@ -276,9 +268,7 @@ "scratch-snapshot": 8.8, "smp": 30.9, "smp-snapshot": 46.6, - "snapshot-core": 17.2, "snapshot-rollback": 1.5, - "snapshot-tiers": 12.0, "structured-outcome": 0.8, "time-abi-conformance": 1.6, "time-host-correction": 14.0, @@ -289,14 +279,14 @@ }, "shards": [ { - "seconds": 354.1, + "seconds": 339.3, "scenarios": [ "denied-filesystem-paths", "directional-network-policy", "endpoint-policy-snapshot", "filesystem-policy", + "filesystem-shares", "host-loopback-policy", - "interrupt-map", "network-snapshot", "platform-aggregate-export", "platform-filesystem", @@ -307,12 +297,11 @@ "scratch-snapshot", "smp", "smp-snapshot", - "snapshot-tiers", "untiered-network-policy" ] }, { - "seconds": 353.7, + "seconds": 339.3, "scenarios": [ "command-line-limit", "console-exit", @@ -320,10 +309,10 @@ "cpu-profile-fallback", "filesystem-comma-paths", "filesystem-owner", - "filesystem-shares", "filesystem-snapshot", "guest-boot", "guest-identity", + "interrupt-map", "ipv6-egress-policy", "l3-l4-egress-policy", "l3-l4-egress-port-ranges", @@ -334,7 +323,6 @@ "platform-zero-fill", "private-network-ingress", "sandbox-blocks", - "snapshot-core", "snapshot-rollback", "structured-outcome", "time-abi-conformance", diff --git a/scripts/nvx_tools/microvm_test_scripts/snapshot-core.sh b/scripts/nvx_tools/microvm_test_scripts/snapshot-core.sh deleted file mode 100644 index 4708517aa..000000000 --- a/scripts/nvx_tools/microvm_test_scripts/snapshot-core.sh +++ /dev/null @@ -1,64 +0,0 @@ -set -eu -fail() { - code="$1" - echo "NVX-SNAPSHOT-CORE-FAIL code=$code" - nvx-exit "$code" - exit "$code" -} - -generation_id_before="$(/sbin/nvx-time generation-id)" || fail 48 -[ "${#generation_id_before}" -eq 32 ] || fail 49 -sleep 3600 & sleeping_pid=$! -sleep 5 & timer_pid=$! -sleep 1 -[ -r "/proc/$sleeping_pid/stat" ] || fail 47 -wall_before="$(date +%s)" -uptime_before_raw="$(cut -d' ' -f1 /proc/uptime)" -uptime_before=${uptime_before_raw%%.*} -uptime_before_cs="$(echo "$uptime_before_raw" | tr -d .)" -process_cpu_before="$(awk '{ print $14 + $15 }' "/proc/$sleeping_pid/stat")" -thread_cpu_before="$(awk '{ print $14 + $15 }' "/proc/$sleeping_pid/task/$sleeping_pid/stat")" -nvx-snapshot -wall_restored="$(date +%s)" -uptime_restored_raw="$(cut -d' ' -f1 /proc/uptime)" -uptime_restored=${uptime_restored_raw%%.*} -uptime_restored_cs="$(echo "$uptime_restored_raw" | tr -d .)" -process_cpu_restored="$(awk '{ print $14 + $15 }' "/proc/$sleeping_pid/stat")" -thread_cpu_restored="$(awk '{ print $14 + $15 }' "/proc/$sleeping_pid/task/$sleeping_pid/stat")" -kill "$sleeping_pid" -timer_wait_before="$(cut -d. -f1 /proc/uptime)" -wait "$timer_pid" -timer_wait_after="$(cut -d. -f1 /proc/uptime)" -echo NVX-SNAPSHOT-CORE-CONTINUED -echo "NVX-SNAPSHOT-DOWNTIME-$((wall_restored - wall_before))-$((uptime_restored - uptime_before))" -echo "NVX-SNAPSHOT-UPTIME-CS-$((uptime_restored_cs - uptime_before_cs))" -echo "NVX-SNAPSHOT-CPU-$((process_cpu_restored - process_cpu_before))-$((thread_cpu_restored - thread_cpu_before))" -echo "NVX-SNAPSHOT-TIMER-WAIT-$((timer_wait_after - timer_wait_before))" -generation_id_after="$(/sbin/nvx-time generation-id)" || fail 50 -[ "$generation_id_before" != "$generation_id_after" ] || fail 51 -# nvx-time capture consumed the restore packet and, for an untiered restore -# without targets, left its 64 entropy bytes to the caller. The generation ID -# is their first 16 bytes. -entropy=/run/nvx/restore-entropy -[ "$(wc -c <"$entropy")" -eq 64 ] || fail 44 -generation_id_packet="$( - head -c 16 "$entropy" | od -An -tx1 -v | tr -d '[:space:]' -)" -[ "$generation_id_after" = "$generation_id_packet" ] || fail 52 -/sbin/nvx-reseed "$entropy" || fail 45 -rng="$(/sbin/nvx-reseed --sample)" || fail 53 -# nvx-time holds the port permission only for each port transaction, so a -# process that it starts afterwards cannot reach the ports. -port_access="$(/sbin/nvx-time test port-access)" || fail 55 -uuid="$(cat /proc/sys/kernel/random/uuid)" -temp_path="$(mktemp /tmp/nvx-clone.XXXXXX)" || fail 54 -temp_id=${temp_path##*/} -rm -f "$temp_path" -echo "NVX-SNAPSHOT-RNG-$rng" -echo "NVX-SNAPSHOT-GENERATION-ID-$generation_id_after" -echo "NVX-SNAPSHOT-UUID-$uuid" -echo "NVX-SNAPSHOT-TEMP-ID-$temp_id" -echo "NVX-SNAPSHOT-PORT-ACCESS-$port_access" -dd if=/dev/zero of=/tmp/nvx-snapshot-dirty bs=1M count=32 2>/dev/null -echo NVX-SNAPSHOT-CORE-OK -nvx-exit 0 diff --git a/scripts/nvx_tools/microvm_test_scripts/snapshot-tier.sh.in b/scripts/nvx_tools/microvm_test_scripts/snapshot-tier.sh.in deleted file mode 100644 index 3f9051ae0..000000000 --- a/scripts/nvx_tools/microvm_test_scripts/snapshot-tier.sh.in +++ /dev/null @@ -1,34 +0,0 @@ -set -eu -if /sbin/nvx-snapshot --tier @MISMATCHED_TIER@ >/run/nvx-tier-mismatch.log 2>&1; then - nvx-exit 69 - exit 69 -fi -grep -q 'requested tier does not match host snapshot tier' /run/nvx-tier-mismatch.log || { - cat /run/nvx-tier-mismatch.log - nvx-exit 68 - exit 68 -} -echo @MISMATCH_MARKER@ -@PAIRED_SETUP@ -echo @SETUP_MARKER@ -@RUNTIME_HOOK@ -echo @HOOK_MARKER@ -@PRE_CAPTURE_ACTION@ -stty -F /dev/hvc1 raw -echo -(dd if=/dev/hvc1 bs=1 count=1 >/dev/null 2>&1; echo @INPUT_MARKER@) & -gate_reader=$! -mkdir -p /run/nvx -echo "$gate_reader" >/run/nvx/gate-reader.pid -sleep 0.1 -echo @CAPTURE_MARKER@ -@CAPTURE_ACTION@ -@REPAIR_ACTION@ -wait "$gate_reader" -echo @RELEASED_MARKER@ -blockdev --flushbufs /dev/vda -value="$(dd if=/dev/vda bs=1 count=1 2>/dev/null | od -An -tu1 | tr -d '[:space:]')" -echo "@LAYER_MARKER@$value" -set +e -@STATUS_QUERY@ -dd if=/dev/hvc1 bs=1 count=1 >/dev/null 2>&1 -nvx-exit 0 diff --git a/scripts/nvx_tools/microvm_tests.py b/scripts/nvx_tools/microvm_tests.py index 6b6ae5585..caaac5933 100644 --- a/scripts/nvx_tools/microvm_tests.py +++ b/scripts/nvx_tools/microvm_tests.py @@ -6,6 +6,7 @@ import hashlib import ipaddress import json +import math import os import queue import random @@ -22,7 +23,8 @@ import time import uuid import zlib -from collections.abc import Callable, Collection, Sequence +from collections.abc import Callable, Collection, Mapping, Sequence +from concurrent.futures import Future, ThreadPoolExecutor from contextlib import ExitStack, suppress from dataclasses import dataclass from pathlib import Path @@ -43,6 +45,7 @@ parse_snapshot_profile_line, positive_float, positive_int, + record_adversarial_openvmm_pid, smp_probe_script, snapshot_restore_command, workload_boot_command, @@ -68,6 +71,7 @@ from .guests import GUEST_NAMES, GuestDescriptor, guest_descriptor from .managed_exec_tests import ( WorkloadArtifacts, + bounded_text, load_workload_artifacts, run_managed_exec_configuration, workload_artifact_paths, @@ -84,12 +88,16 @@ OUTCOME, PROBE, PROVISIONED_ENTRIES, + RUNTIME, STOPPED_ENTRIES, LifecycleAcceptance, bounded_evidence, check_managed_report, + kill_openvmm, managed_report, openvmm_processes, + read_runtime, + recorded_pid, run_acceptance, run_sandbox_lifecycle, terminate_process, @@ -195,16 +203,17 @@ "sandbox-proxy-only", "sandbox-ram-upper", "scratch-snapshot", + "snapshot-core", "snapshot-tiers", "virtio-features", "vm-lifecycle", ) ) -# The public scenarios, of `nvx-rs sandbox` and of the public API's VM -# lifecycle, which also read the Ubuntu distro layer and the sandbox scratch -# template that `nvx-rs download` installs. A default run runs them, but CI -# runs them in its guest and sandbox jobs, which take those Ubuntu artifacts, -# rather than in the shards of the default suite. +# The public scenarios, of `nvx-rs sandbox`, of its named snapshots, and of +# the public API's VM lifecycle, which also read the Ubuntu distro layer and +# the sandbox scratch template that `nvx-rs download` installs. A default run +# runs them, but CI runs them in its guest and sandbox jobs, which take those +# Ubuntu artifacts, rather than in the shards of the default suite. PUBLIC_SANDBOX_SCENARIOS = frozenset( ( "managed-exec-config", @@ -212,6 +221,8 @@ "sandbox-lifecycle", "sandbox-proxy-only", "sandbox-ram-upper", + "snapshot-core", + "snapshot-tiers", "vm-lifecycle", ) ) @@ -223,7 +234,6 @@ "smp-snapshot", "restore-processors", "restore-downtime", - "snapshot-tiers", ) # Scenarios that run only when named, never in a default suite. The time ABI # hides TSC-deadline on every backend, so `smp` already runs on the one-shot @@ -378,8 +388,6 @@ class MicrovmVirtioDevice(NamedTuple): PRIVATE_INGRESS_UDP_REQUEST = b"NVX-PRIVATE-INGRESS-UDP-REQUEST\n" PRIVATE_INGRESS_UDP_REPLY = b"NVX-PRIVATE-INGRESS-UDP-REPLY\n" SANDBOX_BLOCK_SIZE = 8 * 1024 * 1024 -SNAPSHOT_CORE_CONTINUED_MARKER = b"NVX-SNAPSHOT-CORE-CONTINUED" -SNAPSHOT_CORE_COMPLETION_MARKER = b"NVX-SNAPSHOT-CORE-OK" CONSOLE_BINARY_MARKER = b"\0\r\n\x7f\xffNVX-CONSOLE-BINARY" CONSOLE_RX_READY_MARKER = b"NVX-CONSOLE-RX-READY" CONSOLE_RX_QUEUED_MARKER = b"NVX-CONSOLE-RX-QUEUED" @@ -1063,17 +1071,6 @@ def _single_framed_marker_value(output: bytes, prefix: bytes, suffix: bytes) -> return values[0] -def _parse_marker_pair(output: bytes, prefix: bytes) -> tuple[int, int]: - value = _single_marker_value(output, prefix) - parts = value.split(b"-") - if len(parts) != 2: - raise RuntimeError(f"malformed {prefix!r} marker: {value!r}") - try: - return int(parts[0]), int(parts[1]) - except ValueError as error: - raise RuntimeError(f"malformed {prefix!r} marker: {value!r}") from error - - def _available_tcp_address() -> tuple[str, int]: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as listener: listener.bind(("127.0.0.1", 0)) @@ -1793,6 +1790,56 @@ def header(own: int, alternate: int, array: int) -> bytes: return str(guid) +def write_gpt_layer( + disk: Path, + distro: Path, + *, + executable: Path, + kernel: Path, + initrd: Path, + backend: str, + timeout: float, + log_path: Path, +) -> str: + """Write a new GPT disk at ``disk`` whose layer partition holds an ext4 + copy of the tree of the EROFS layer ``distro``, and return its GUID. + + The layer partition takes GPT_LAYER_EXPANSION times the compressed EROFS + layer, at least GPT_LAYER_MIN_BYTES, in whole MiB. A guest of the control + initramfs, which ``executable``, ``kernel``, and ``initrd`` boot on + ``backend``, formats it and copies the tree, with the EROFS layer as its + distro block device and the disk as its scratch, and its console goes to + ``log_path``. + """ + layer_bytes = max(GPT_LAYER_MIN_BYTES, GPT_LAYER_EXPANSION * distro.stat().st_size) + layer_bytes += -layer_bytes % (1 << 20) + guid = write_gpt_disk(disk, layer_bytes) + command = workload_boot_command( + executable, + backend, + kernel, + initrd, + GPT_LAYER_MEMORY_MIB, + "quiet loglevel=0", + ) + command.extend( + ( + "--microvm-sandbox-block", + _block_arg("distro", distro, read_only=True), + "--microvm-sandbox-block", + _block_arg("scratch", disk, read_only=False), + ) + ) + run_guest_script( + command, + GPT_LAYER_SCRIPT, + GPT_LAYER_READY_MARKER, + timeout=timeout, + log_path=log_path, + ) + return guid + + class GptLayerAcceptance(LifecycleAcceptance): """The CLI's cold cells of a GPT disk on the selected backend: over a scratch upper and a RAM upper, a `sandbox run`, and a managed sandbox's @@ -1836,35 +1883,15 @@ def run(self) -> None: self.require_processes((), "after the GPT sandboxes") def write_disk(self) -> None: - """Writes the GPT disk, then has a guest of the control initramfs - format its layer partition as ext4 and copy the EROFS layer's tree - into it.""" - layer_bytes = max( - GPT_LAYER_MIN_BYTES, - GPT_LAYER_EXPANSION * self.artifacts.distro.stat().st_size, - ) - layer_bytes += -layer_bytes % (1 << 20) - self.guid = write_gpt_disk(self.disk, layer_bytes) - command = workload_boot_command( - self.executable, - self.backend, - self.kernel, - self.initrd, - GPT_LAYER_MEMORY_MIB, - "quiet loglevel=0", - ) - command.extend( - ( - "--microvm-sandbox-block", - _block_arg("distro", self.artifacts.distro, read_only=True), - "--microvm-sandbox-block", - _block_arg("scratch", self.disk, read_only=False), - ) - ) - run_guest_script( - command, - GPT_LAYER_SCRIPT, - GPT_LAYER_READY_MARKER, + """Writes the GPT disk, whose layer partition a guest of the control + initramfs fills with the tree of the EROFS layer.""" + self.guid = write_gpt_layer( + self.disk, + self.artifacts.distro, + executable=self.executable, + kernel=self.kernel, + initrd=self.initrd, + backend=self.backend, timeout=self.timeout, log_path=self.output_dir / f"{self.evidence_prefix}disk.log", ) @@ -2048,6 +2075,1184 @@ def run_sandbox_gpt_layer( ) +# The public snapshot scenarios, snapshot-core and snapshot-tiers, capture a +# named snapshot with `nvx-rs snapshot capture`, whose VM nvx-init runs without +# a NIC, and restore it with `snapshot restore` and `run --restore-snapshot`, +# under the CLI's lock of the snapshot (#1118). The workload of each restore is +# the snapshot probe, which reports on one line what the restored guest sees: +# its clock and uptime, fresh random values, its machine ID, host name and +# identity, the lower and upper directories of its root's overlay, its block +# devices and network interfaces, the distribution of its image, and the +# marker of the host share at SNAPSHOT_SHARE_TARGET, or none without one. +SNAPSHOT_PROBE_PREFIX = b"NVX-SNAPSHOT-PROBE " +SNAPSHOT_PROBE_FIELDS: frozenset[str] = frozenset( + ( + "label", + "wall", + "uptime", + "uuid", + "rng", + "machine", + "host", + "id", + "temp", + "lower", + "upper", + "devices", + "net", + "release", + "share", + ) +) +SNAPSHOT_SHARE_TARGET = "/data" +# The file whose creation in its share ends the workload of a restore that +# holds the snapshot's lock until the scenario releases it. +SNAPSHOT_RELEASE = "release" +SNAPSHOT_PROBE_SCRIPT = r"""set -eu +line=NVX-SNAPSHOT-PROBE +report() { + line="$line $1=$2" +} +report label @LABEL@ +report wall "$(date +%s.%N)" +report uptime "$(cut -d' ' -f1 /proc/uptime)" +report uuid "$(cat /proc/sys/kernel/random/uuid)" +report rng "$(head -c 32 /dev/urandom | od -An -v -tx1 | tr -d ' \n')" +report machine "$(cat /etc/machine-id)" +report host "$(cat /proc/sys/kernel/hostname)" +report id "$(id -u):$(id -g)" +temp=$(mktemp) +rm -f "$temp" +report temp "${temp##*/}" +root=$(grep ' / / ' /proc/self/mountinfo) +report lower "$(printf '%s\n' "$root" | grep -o 'lowerdir=[^,]*' | cut -d= -f2)" +report upper "$(printf '%s\n' "$root" | grep -o 'upperdir=[^,]*' | cut -d= -f2)" +report devices "$(ls /sys/block | grep '^vd' | paste -sd, -)" +report net "$(ls /sys/class/net | paste -sd, -)" +. /etc/os-release +report release "$ID" +share=none +if [ -r @SHARE@/marker ]; then + share=$(cat @SHARE@/marker) +fi +report share "$share" +printf '%s\n' "$line" +""" +# What a probe that holds its restore runs after its report: it waits until +# the host creates SNAPSHOT_RELEASE in its share, which the guest reads live. +SNAPSHOT_PROBE_HOLD = r"""while [ ! -e @SHARE@/@RELEASE@ ]; do + sleep 0.1 +done +""" +# What a sandbox's root holds, in the marker of its share and in the files +# that execs wrote in its upper, while the restored sandboxes run at once. +SNAPSHOT_VIEW_SCRIPT = ( + r"""printf 'share=%s\n' "$(cat @SHARE@/marker)" +for name in /tmp/nvx-upper-*; do + printf 'file=%s\n' "${name##*/}" +done +""" +).replace("@SHARE@", SNAPSHOT_SHARE_TARGET) +# The entries of a named snapshot's directory, which `snapshot capture` +# publishes. +SNAPSHOT_ENTRIES: frozenset[str] = frozenset(("key.bin", "snapshot.json", "openvmm")) +# The block devices of a VM that a snapshot of `snapshot capture` restores, +# which the snapshot fixes: its image, in place of the placeholder, and its +# dormant scratch, which a scratch upper's image takes, whatever the upper. +SNAPSHOT_DEVICES = "vda,vdb" +# The upper directory of a restored root's overlay, by its upper. +SNAPSHOT_UPPERS = {"ram": "/run/nvx/tmpfs/upper", "scratch": "/run/nvx/scratch/upper"} +# The guest memory of a restore: the boot memory of `snapshot capture`, which +# is its capacity too. +SNAPSHOT_MEMORY_MIB = 256 +# The host name and the workload identity of a restored sandbox by default. +SNAPSHOT_HOSTNAME = "nvx-sandbox" +SNAPSHOT_IDENTITY = "65534:65534" +# How far the clock of a restored guest may be from the host's. +SNAPSHOT_CLOCK_TOLERANCE_SECONDS = 2.0 +# The outcome of a restored VM that `sandbox stop` stopped. +RESTORED_SUCCESS = {"operation": "restore", "category": "success", "status_code": 0} +# The launch journal of the VM of a capture, in the store's `capture` +# directory until the capture ends. +CAPTURE_JOURNAL = "launch.json" + + +def snapshot_probe_script(label: str, *, hold: bool = False) -> str: + """Return the snapshot probe, which reports as ``label``, and with + ``hold`` then waits until the host creates SNAPSHOT_RELEASE in its share. + """ + if not re.fullmatch(r"[a-z0-9][a-z0-9-]*", label): + raise ValueError(f"invalid snapshot probe label {label!r}") + script = SNAPSHOT_PROBE_SCRIPT + (SNAPSHOT_PROBE_HOLD if hold else "") + return ( + script.replace("@LABEL@", label) + .replace("@SHARE@", SNAPSHOT_SHARE_TARGET) + .replace("@RELEASE@", SNAPSHOT_RELEASE) + ) + + +def snapshot_probe_workload(label: str, *, hold: bool = False) -> list[str]: + """Return the workload options of a restore whose shell runs the snapshot + probe, as snapshot_probe_script describes it.""" + return [ + "--entrypoint", + "/bin/sh", + "--arg=-c", + f"--arg={snapshot_probe_script(label, hold=hold)}", + ] + + +def snapshot_probe_report(output: bytes, label: str) -> dict[str, str]: + """Return the fields of the one report in ``output`` of the snapshot + probe that ran as ``label``.""" + prefix = SNAPSHOT_PROBE_PREFIX + reports: list[dict[str, str]] = [] + for line in _output_lines(output): + if not line.startswith(prefix): + continue + malformed = RuntimeError( + f"the snapshot probe printed a malformed report: {line!r}" + ) + fields: dict[str, str] = {} + for field in line[len(prefix) :].decode("ascii", "replace").split(" "): + key, separator, value = field.partition("=") + if not separator or key in fields: + raise malformed + fields[key] = value + if frozenset(fields) != SNAPSHOT_PROBE_FIELDS: + raise malformed + if fields["label"] == label: + reports.append(fields) + if len(reports) != 1: + raise RuntimeError( + f"the snapshot probe {label} printed {len(reports)} reports, not 1" + ) + return reports[0] + + +def require_distinct_restores(reports: Sequence[dict[str, str]], context: str) -> None: + """Require each of the restores of one snapshot whose probes printed + ``reports`` to have taken fresh random values and a machine ID of its own: + the kernel's random UUID, bytes of /dev/urandom, the machine ID, and the + name of a temporary file.""" + for key, what in ( + ("uuid", "a kernel random UUID"), + ("rng", "random bytes"), + ("machine", "a machine ID"), + ("temp", "a temporary file name"), + ): + values = [report[key] for report in reports] + if len(set(values)) != len(values): + raise RuntimeError(f"{context} reused {what}: {values}") + + +def snapshot_directory(store: Path, name: str) -> Path: + """Return the directory of the snapshot ``name`` in the named snapshot + store ``store``, which spells the name in lowercase hexadecimal.""" + return store / "snapshots" / name.encode("ascii").hex() + + +def named_snapshot_files(directory: Path) -> dict[str, tuple[str, int]]: + """Return the digest and the link count of each file of the named + snapshot in ``directory``, by its path there, once the directory holds + exactly SNAPSHOT_ENTRIES.""" + entries = sorted(path.name for path in directory.iterdir()) + if frozenset(entries) != SNAPSHOT_ENTRIES: + raise RuntimeError( + f"the named snapshot holds {entries}, expected {sorted(SNAPSHOT_ENTRIES)}" + ) + return { + path.relative_to(directory).as_posix(): ( + sha256_file(path), + path.stat().st_nlink, + ) + for path in sorted(directory.rglob("*")) + if path.is_file() + } + + +def check_restored_report(report: dict[str, Any], backend: str, context: str) -> None: + """Check the outcome report of a restored sandbox that `sandbox stop` + stopped: a successful restored VM of ``backend``, without a network policy, + since the snapshot has no NIC, and a complete teardown.""" + if report["outcome"] != RESTORED_SUCCESS: + raise RuntimeError(f"{context} reported the outcome {report['outcome']!r}") + if report["backend"] != backend: + raise RuntimeError(f"{context} reported the backend {report['backend']!r}") + if report["network_policy"]["status"] != "not-requested": + raise RuntimeError( + f"{context} reported the network policy {report['network_policy']!r}" + ) + if not all(report["teardown"].values()): + raise RuntimeError( + f"{context} reported an incomplete teardown: {report['teardown']!r}" + ) + + +class SnapshotAcceptance(LifecycleAcceptance): + """A public snapshot scenario's fixture: a named snapshot store, in which + `nvx-rs snapshot capture` captures ``snapshot_name`` with nvx-init as the + guest, and the restores of that snapshot, each in a state directory of its + own in ``self.state``, which cleanup stops and deprovisions.""" + + snapshot_name = "snapshot" + + def __init__( + self, + backend: str, + *, + timeout: float, + output_dir: Path, + artifacts: WorkloadArtifacts, + root: Path, + ) -> None: + super().__init__( + backend, + timeout=timeout, + output_dir=output_dir, + artifacts=artifacts, + root=root, + ) + self.store = root / "store" + # run_acceptance keeps the fixture if cleanup leaves this directory, + # which cleanup removes once it has deprovisioned every sandbox in it. + self.state = root / "sandboxes" + # When the capture ran, by the host's clock. + self.captured = (0.0, 0.0) + # The files of the named snapshot that the capture published. + self.files: dict[str, tuple[str, int]] = {} + # When each command ran, by its label. + self.timings: dict[str, tuple[float, float]] = {} + + @property + def snapshot(self) -> Path: + return snapshot_directory(self.store, self.snapshot_name) + + def nvx( + self, + label: str, + *arguments: str, + state: Path | None = None, + expected: int = 0, + code: str | None = None, + message: str | None = None, + ) -> subprocess.CompletedProcess[bytes]: + """Runs `nvx-rs ARGUMENTS`, records its bounded observations, its + output, and when it ran, as ``label``, and requires the status + ``expected``, and the error ``code`` whose message holds ``message``, + when given.""" + started = time.time() + result = subprocess.run( + nvx_rs.command_line(arguments), + cwd=BuildConstants.REPO_ROOT, + capture_output=True, + timeout=2 * self.timeout + 30, + ) + self.timings[label] = (started, time.time()) + self.record(label, result, state=state, expected=expected) + self.save_evidence(f"{label}.log", result.stdout + result.stderr) + reported = nvx_rs.reported_error(result.stderr) + if ( + result.returncode != expected + or ( + code is not None + and ( + reported is None + or reported.code != code + or (message is not None and message not in reported.message) + ) + ) + or (code is None and expected == 0 and reported is not None) + ): + wanted = "" if code is None else f" with error[{code}]" + raise RuntimeError( + f"{label}: nvx-rs {' '.join(arguments[:2])} returned " + f"{result.returncode}, expected {expected}{wanted}; " + f"stdout={bounded_text(result.stdout)!r}; " + f"stderr={bounded_text(result.stderr)!r}" + ) + return result + + def capture(self) -> None: + """Captures the named snapshot, with a dormant scratch as long as the + scratch template, and requires the store to hold exactly the + snapshot's SNAPSHOT_ENTRIES, each of whose files has one link, and no + directory or OpenVMM process of the capture's VM.""" + scratch_bytes = self.artifacts.scratch_template.stat().st_size + if not scratch_bytes or scratch_bytes % (1 << 20): + raise RuntimeError("the scratch template is not a whole number of MiB") + result = self.nvx( + "capture", + "snapshot", + "capture", + self.snapshot_name, + "--store-dir", + str(self.store), + "--hypervisor", + self.backend, + "--scratch-mib", + str(scratch_bytes >> 20), + "--timeout", + str(math.ceil(self.timeout)), + ) + self.captured = self.timings["capture"] + if _output_lines(result.stdout) != [b"Snapshot captured"]: + raise RuntimeError( + f"the capture printed {bounded_text(result.stdout)!r}, not that " + "it captured the snapshot" + ) + if (self.store / "capture").exists(): + raise RuntimeError("the capture left its VM's directory in the store") + self.files = named_snapshot_files(self.snapshot) + if any(links != 1 for _, links in self.files.values()): + raise RuntimeError( + f"the captured snapshot's files are linked: {self.files}" + ) + self.require_processes((), "after the capture") + + def check_snapshot_unchanged(self, context: str) -> None: + """Requires the named snapshot's files to be as the capture published + them, each with one link, once no view of a restore holds them.""" + if named_snapshot_files(self.snapshot) != self.files: + raise RuntimeError( + f"{context} changed the named snapshot, or left links to its files" + ) + + def restore_options( + self, + label: str, + image: Path, + identity: str, + *, + image_format: str, + upper: str, + ) -> list[str]: + """Returns the options of a restore of ``image``, whose identity is + ``identity``, in the format ``image_format``, over ``upper``: with a + new copy of the scratch template, whose upper no root wrote, for a + scratch upper.""" + options = [ + "--hypervisor", + self.backend, + "--layer", + f"distro,{image},{identity}", + "--memory-mib", + str(SNAPSHOT_MEMORY_MIB), + ] + if image_format != "erofs": + options += ["--layer-format", image_format] + if upper == "ram": + return [*options, "--upper", "ram"] + scratch = self.root / f"{label}-scratch.ext4" + shutil.copyfile(self.artifacts.scratch_template, scratch) + return [*options, "--scratch", str(scratch)] + + def one_off( + self, + label: str, + *options: str, + timeout: float | None = None, + expected: int = 0, + code: str | None = None, + message: str | None = None, + ) -> subprocess.CompletedProcess[bytes]: + """Restores the snapshot once with `run --restore-snapshot`, whose + sandbox's state directory the run must remove, also after a failure.""" + state = self.state / label + result = self.nvx( + label, + "run", + "--restore-snapshot", + self.snapshot_name, + "--store-dir", + str(self.store), + "--state-dir", + str(state), + "--timeout", + str(self.timeout if timeout is None else timeout), + *options, + state=state, + expected=expected, + code=code, + message=message, + ) + if state.exists(): + raise RuntimeError(f"the {label} run left its sandbox's state directory") + return result + + def restore( + self, label: str, *options: str + ) -> tuple[Path, subprocess.CompletedProcess[bytes]]: + """Restores the snapshot into a new managed sandbox with `snapshot + restore`, which leaves it running once its workload has exited with + status 0, and records its OpenVMM process for an adversarial + campaign.""" + state = self.state / label + result = self.nvx( + label, + "snapshot", + "restore", + self.snapshot_name, + "--store-dir", + str(self.store), + "--state-dir", + str(state), + "--timeout", + str(self.timeout), + *options, + state=state, + ) + record_adversarial_openvmm_pid( + recorded_pid(read_runtime(state)), dict(os.environ) + ) + return state, result + + def stop_and_deprovision(self, state: Path) -> None: + """Stops a restored sandbox, which must stop cleanly, and deprovisions + it, which removes its state and its view of the snapshot.""" + label = state.name + self.nvx( + f"{label}-stop", + "sandbox", + "stop", + "--state-dir", + str(state), + "--timeout", + str(self.timeout), + state=state, + ) + self.copy_evidence(state / OUTCOME, f"{label}-outcome.json") + check_restored_report( + _read_outcome_report(state / OUTCOME), + self.backend, + f"the {label} sandbox", + ) + self.nvx( + f"{label}-deprovision", + "sandbox", + "deprovision", + "--state-dir", + str(state), + "--timeout", + str(self.timeout), + state=state, + ) + if state.exists(): + raise RuntimeError(f"deprovision kept the {label} sandbox's state") + + def check_guest( + self, + report: dict[str, str], + label: str, + *, + host: str, + identity: str, + layer: str, + upper: str, + share: str = "none", + ) -> None: + """Checks what the probe of the restore ``label`` reports: the host + name and the identity of the restore, the image's layer ``layer`` as + its root's lower directory over ``upper``, the snapshot's block devices, + no NIC, the Ubuntu image, and the marker ``share`` of its host share; a + clock that reads the host's time while the restore ran; and an uptime + that kept the time before the capture and counts the time since, so + that the guest booted while the capture ran, by its clock and uptime.""" + context = f"the {label} restore" + expected = { + "host": host, + "id": identity, + "lower": f"/run/nvx/layers/{layer}", + "upper": SNAPSHOT_UPPERS[upper], + "devices": SNAPSHOT_DEVICES, + "net": "lo", + "release": "ubuntu", + "share": share, + } + for key, value in expected.items(): + if report[key] != value: + raise RuntimeError( + f"{context}'s guest reports {key}={report[key]}, expected {value}" + ) + try: + wall, uptime = float(report["wall"]), float(report["uptime"]) + except ValueError as error: + raise RuntimeError( + f"{context}'s guest reports a malformed clock" + ) from error + tolerance = SNAPSHOT_CLOCK_TOLERANCE_SECONDS + before, after = self.timings[label] + if not before - tolerance <= wall <= after + tolerance: + raise RuntimeError( + f"{context}'s guest clock reads {wall}, while the restore ran " + f"from {before} to {after}" + ) + captured_before, captured_after = self.captured + if ( + not captured_before - tolerance + <= wall - uptime + <= captured_after + tolerance + ): + raise RuntimeError( + f"{context}'s guest booted at {wall - uptime} by its clock and " + f"uptime, which the capture from {captured_before} to " + f"{captured_after} does not hold" + ) + + def clean_up(self) -> list[Exception]: + """Stops and deprovisions the sandboxes that a failed scenario left, + ends the capture's VM and every other OpenVMM process of the fixture, + and then, unless any of that failed, removes the directory of the + sandboxes once it is empty, so that run_acceptance keeps the fixture, + with the store's recovery records, for a VM that may have survived.""" + errors: list[Exception] = [] + sandboxes = sorted(self.state.iterdir()) if self.state.is_dir() else [] + for state in sandboxes: + try: + if (state / RUNTIME).is_file(): + runtime = read_runtime(state) + self.run_cli("stop", state=state) + kill_openvmm(runtime, self.timeout, self.root.name) + except Exception as error: + errors.append(error) + # The launch journal of a capture that has not ended names its VM by + # the identity that a sandbox's runtime record has, which tells when + # its last thread has exited too. + journal = self.store / "capture" / CAPTURE_JOURNAL + try: + if journal.is_file(): + record = json.loads(journal.read_text(encoding="utf-8")) + if record.get("process") is not None: + kill_openvmm(record, self.timeout, self.root.name) + except Exception as error: + errors.append(error) + # Every OpenVMM process whose arguments name the fixture is the + # scenario's own, the capture's VM among them. + try: + for pid in openvmm_processes(self.root.name): + with suppress(OSError): + terminate_process(pid) + deadline = time.monotonic() + self.timeout + while strays := openvmm_processes(self.root.name): + if time.monotonic() >= deadline: + raise RuntimeError(f"OpenVMM processes {list(strays)} survived") + time.sleep(0.05) + except Exception as error: + errors.append(error) + for state in sandboxes: + if not state.exists(): + continue + try: + self.copy_evidence(state / LOG, f"{state.name}-openvmm.log") + # Deprovision ends a start that was interrupted, and removes a + # restore's view of the snapshot under the snapshot's lock. + result = self.run_cli("deprovision", state=state) + if result.returncode != 0: + raise RuntimeError( + f"cannot deprovision {state}: " + f"{bounded_text(result.stderr).strip()}" + ) + except Exception as error: + errors.append(error) + if not errors: + with suppress(OSError): + self.state.rmdir() + return errors + + +# The seconds for which a one-off restore that cannot have the snapshot's lock +# may wait for it, and for which snapshot-core watches, while one restore holds +# the lock, for the view of the snapshot of another that must wait for it. +SNAPSHOT_CORE_BUSY_TIMEOUT_SECONDS = 1 +SNAPSHOT_CORE_EXCLUSION_SECONDS = 2.0 +# How much longer than the capture, with the clocks' tolerance on either side, +# the downtime before snapshot-core's first restore is. +SNAPSHOT_CORE_DOWNTIME_MARGIN_SECONDS = 1.0 + + +class SnapshotCoreAcceptance(SnapshotAcceptance): + """snapshot-core: a named snapshot that nvx-init's VM captured, restored + again and again. + + The capture publishes exactly the snapshot's entries. Two one-off restores + of the EROFS layer, in RAM and over a scratch image, with host names and + identities of their own, show a guest whose clock reads the host's time, + whose uptime counts the time since the capture, and which took fresh + random values and a machine ID of its own. A scratch image that the + snapshot cannot serve is refused before any VM starts, which leaves the + managed sandbox of the restore as provisioned, with an error that names + the requested scratch, which attributes the failure to the request; + a zeroed scratch image, which the guest cannot mount as its upper, ends + its VM before its workload runs, and the CLI reports the guest's exit. + Three managed restores at once, each with a host share of its own, take + turns at the snapshot's lock, and a one-off restore that cannot wait for it + fails; the three then run at once, each with its own share, machine ID, and + upper. No restore changes the snapshot or leaves an OpenVMM process or a + link to the snapshot's files. + """ + + evidence_prefix = "snapshot-core-" + snapshot_name = "core" + + def run(self) -> None: + self.state.mkdir() + self.capture() + # A downtime longer than the capture, with the clocks' tolerance on + # either side, tells an uptime that counts it from one that does not. + before, after = self.captured + time.sleep( + max( + 0.0, + 2 * after + - before + + 2 * SNAPSHOT_CLOCK_TOLERANCE_SECONDS + + SNAPSHOT_CORE_DOWNTIME_MARGIN_SECONDS + - time.time(), + ) + ) + repeated = [ + self.check_repeated_restore( + "repeat-ram", upper="ram", host="core-one", identity="65534:65534" + ), + self.check_repeated_restore( + "repeat-scratch", upper="scratch", host="core-two", identity="1:1" + ), + ] + require_distinct_restores(repeated, "the repeated restores") + self.check_failed_restores() + concurrent = self.check_concurrent_restores() + require_distinct_restores([*repeated, *concurrent], "the restores") + self.check_snapshot_unchanged("the restores") + self.require_processes((), "after the restores") + + def check_repeated_restore( + self, label: str, *, upper: str, host: str, identity: str + ) -> dict[str, str]: + """Restores the snapshot once with `run --restore-snapshot`, over + ``upper``, as ``identity``, with the host name ``host``, and checks + what its guest reports.""" + options = self.restore_options( + label, + self.artifacts.distro, + self.artifacts.layer_uuid, + image_format="erofs", + upper=upper, + ) + result = self.one_off( + label, + *options, + "--hostname", + host, + "--workload-user", + identity, + *snapshot_probe_workload(label), + ) + report = snapshot_probe_report(result.stdout, label) + self.check_guest( + report, label, host=host, identity=identity, layer="distro", upper=upper + ) + self.check_snapshot_unchanged(f"the {label} restore") + self.require_processes((), f"after the {label} restore") + return report + + def check_failed_restores(self) -> None: + """Checks the failed restores of the snapshot. + + A managed restore over a scratch image of half the dormant scratch's + length, which the snapshot cannot serve, is refused before any VM + starts: its error names the lengths of the requested scratch and of + the dormant one, which attributes the failure to the request, and its + sandbox, which a failed start keeps provisioned, holds its + configuration and its lock alone, where a launch leaves OpenVMM's log + and outcome report and the view of the snapshot. A one-off restore + over a zeroed scratch image of the dormant scratch's length, which the + guest cannot mount as its upper, ends its VM before its workload runs, + which the CLI reports as the guest's exit with status 125; the + restores that follow take the snapshot's lock, which it freed. + """ + size = self.artifacts.scratch_template.stat().st_size + short, zeroed = "short-scratch", "zeroed-scratch" + + def options(label: str, length: int) -> list[str]: + scratch = self.root / f"{label}.raw" + with scratch.open("xb") as image: + image.truncate(length) + return [ + "--hypervisor", + self.backend, + "--layer", + f"distro,{self.artifacts.distro},{self.artifacts.layer_uuid}", + "--memory-mib", + str(SNAPSHOT_MEMORY_MIB), + "--scratch", + str(scratch), + *snapshot_probe_workload(label), + ] + + state = self.state / short + refused = self.nvx( + short, + "snapshot", + "restore", + self.snapshot_name, + "--store-dir", + str(self.store), + "--state-dir", + str(state), + "--timeout", + str(self.timeout), + *options(short, size // 2), + state=state, + expected=1, + code="unsupported", + message="differs from the dormant scratch's", + ) + if SNAPSHOT_PROBE_PREFIX in refused.stdout: + raise RuntimeError(f"the {short} restore ran its workload") + reported = nvx_rs.reported_error(refused.stderr) + assert reported is not None + if not all( + text in reported.message + for text in ( + "snapshot scratch geometry", + f"length: {size // 2},", + f"length: {size},", + ) + ): + raise RuntimeError( + "the refusal of the short scratch image does not name the " + f"lengths of both scratches: {reported.message!r}" + ) + entries = frozenset(path.name for path in state.iterdir()) + if entries != PROVISIONED_ENTRIES: + raise RuntimeError( + f"the {short} restore left {sorted(entries)} in its sandbox's " + "state, not its configuration and its lock alone: it launched " + "a VM before it refused the scratch image" + ) + self.require_processes((), f"after the {short} restore") + self.nvx( + f"{short}-deprovision", + "sandbox", + "deprovision", + "--state-dir", + str(state), + "--timeout", + str(self.timeout), + state=state, + ) + if state.exists(): + raise RuntimeError(f"deprovision kept the {short} sandbox's state") + failed = self.one_off( + zeroed, + *options(zeroed, size), + expected=1, + code="exited", + message="OpenVMM exited during startup: guest-exit status 125", + ) + if SNAPSHOT_PROBE_PREFIX in failed.stdout: + raise RuntimeError(f"the {zeroed} restore ran its workload") + self.require_processes((), f"after the {zeroed} restore") + + def check_concurrent_restores(self) -> list[dict[str, str]]: + """Restores the snapshot into three managed sandboxes at once, which + take turns at the snapshot's lock, and refuses a one-off restore that + cannot wait for it; returns what the three guests reported. + + A restore takes the lock before it links the snapshot's files into its + view, and keeps it until its workload has ended, which the workload of + each of these restores does only once the scenario creates + SNAPSHOT_RELEASE in its share. While the first restore holds the lock, + a one-off restore that may wait SNAPSHOT_CORE_BUSY_TIMEOUT_SECONDS for + it fails with the CLI's error of a busy snapshot before its workload + runs, and neither of the two other restores, which wait for the lock, + may link its view. Once the scenario releases the first, one of them + takes the lock, and the other may link its view only once the scenario + releases that one. Each sandbox has a host share of its own, and the + last a scratch upper where the others have theirs in RAM. The three + restored sandboxes then run at once, each with its own share, machine + ID, and upper, until each stops cleanly and is deprovisioned. + """ + holder = "hold" + waiters = ("wait-one", "wait-two") + labels = (holder, *waiters) + uppers = {holder: "ram", "wait-one": "ram", "wait-two": "scratch"} + shares = {label: self.root / f"{label}-share" for label in labels} + for label, share in shares.items(): + share.mkdir() + (share / "marker").write_text(label, encoding="ascii") + + def release(label: str) -> None: + (shares[label] / SNAPSHOT_RELEASE).touch() + + futures: dict[str, Future[tuple[Path, subprocess.CompletedProcess[bytes]]]] + futures = {} + with ThreadPoolExecutor(max_workers=len(labels)) as pool: + try: + for label in labels: + futures[label] = pool.submit( + self.restore, + label, + *self.restore_options( + label, + self.artifacts.distro, + self.artifacts.layer_uuid, + image_format="erofs", + upper=uppers[label], + ), + "--hostname", + label, + "--mount", + f"{SNAPSHOT_SHARE_TARGET},{shares[label]},ro", + *snapshot_probe_workload(label, hold=True), + ) + if label == holder: + self.wait_for_view((holder,), futures) + # A one-off restore takes no share. + try: + refused = self.one_off( + "busy", + *self.restore_options( + "busy", + self.artifacts.distro, + self.artifacts.layer_uuid, + image_format="erofs", + upper="ram", + ), + *snapshot_probe_workload("busy"), + timeout=SNAPSHOT_CORE_BUSY_TIMEOUT_SECONDS, + expected=1, + code="sandbox-busy", + message=( + "another command still holds the lock of the snapshot " + f"{self.snapshot_name}" + ), + ) + except RuntimeError: + require_holding(holder, futures[holder]) + raise + if SNAPSHOT_PROBE_PREFIX in refused.stdout: + raise RuntimeError("the busy restore ran its workload") + self.require_no_views(waiters, holder, futures[holder]) + release(holder) + first = self.wait_for_view(waiters, futures) + (second,) = (label for label in waiters if label != first) + self.require_no_views((second,), first, futures[first]) + release(first) + self.wait_for_view((second,), futures) + release(second) + restored = {label: futures[label].result() for label in labels} + finally: + # No restore's workload waits for the scenario any longer. + for label in labels: + with suppress(OSError): + release(label) + reports: dict[str, dict[str, str]] = {} + for label, (_, result) in restored.items(): + reports[label] = snapshot_probe_report(result.stdout, label) + self.check_guest( + reports[label], + label, + host=label, + identity=SNAPSHOT_IDENTITY, + layer="distro", + upper=uppers[label], + share=label, + ) + pids = tuple( + sorted(recorded_pid(read_runtime(state)) for state, _ in restored.values()) + ) + self.require_processes(pids, "while the restored sandboxes run") + self.check_isolation({label: state for label, (state, _) in restored.items()}) + for state, _ in restored.values(): + self.stop_and_deprovision(state) + return [reports[label] for label in labels] + + def wait_for_view( + self, + labels: Sequence[str], + futures: Mapping[str, Future[tuple[Path, subprocess.CompletedProcess[bytes]]]], + ) -> str: + """Waits until one of the restores ``labels``, whose ``futures`` run + them, has linked its view of the snapshot, which it does only once it + holds the snapshot's lock, and returns its label; fails with the + failure of a restore that ends without one, or once each could have + failed by itself.""" + limit = 2 * self.timeout + 30 + deadline = time.monotonic() + limit + while True: + for label in labels: + if (self.state / label / "openvmm").is_dir(): + return label + if futures[label].done(): + futures[label].result() + if not (self.state / label / "openvmm").is_dir(): + raise RuntimeError( + f"the {label} restore ended without a view of the snapshot" + ) + if time.monotonic() >= deadline: + raise RuntimeError( + f"no restore of {list(labels)} linked a view of the snapshot " + f"within {limit} seconds" + ) + time.sleep(0.05) + + def require_no_views( + self, + labels: Sequence[str], + holder: str, + holding: Future[tuple[Path, subprocess.CompletedProcess[bytes]]], + ) -> None: + """Requires none of the restores ``labels``, which wait for the + snapshot's lock, to link its view of the snapshot for + SNAPSHOT_CORE_EXCLUSION_SECONDS, while the restore ``holder``, which + ``holding`` runs, holds the lock.""" + deadline = time.monotonic() + SNAPSHOT_CORE_EXCLUSION_SECONDS + while True: + require_holding(holder, holding) + for label in labels: + if (self.state / label / "openvmm").exists(): + raise RuntimeError( + f"the {label} restore linked its view of the snapshot " + f"while the {holder} restore held the snapshot's lock" + ) + if time.monotonic() >= deadline: + return + time.sleep(0.05) + + def check_isolation(self, sandboxes: dict[str, Path]) -> None: + """Has an exec in each of the running ``sandboxes`` write a file of its + own in the root's /tmp, in its upper, and then requires each to read + the marker of its own share and to find its own file there and no + other's.""" + for label, state in sandboxes.items(): + self.nvx( + f"{label}-write", + "sandbox", + "exec", + "--state-dir", + str(state), + "--timeout", + str(self.timeout), + "--entrypoint", + "/bin/touch", + f"--arg=/tmp/nvx-upper-{label}", + state=state, + ) + for label, state in sandboxes.items(): + viewed = self.nvx( + f"{label}-view", + "sandbox", + "exec", + "--state-dir", + str(state), + "--timeout", + str(self.timeout), + "--entrypoint", + "/bin/sh", + "--arg=-c", + f"--arg={SNAPSHOT_VIEW_SCRIPT}", + state=state, + ) + lines = [ + line.decode("ascii", "replace") for line in _output_lines(viewed.stdout) + ] + share = [line[6:] for line in lines if line.startswith("share=")] + if share != [label]: + raise RuntimeError( + f"the {label} sandbox reads the share markers {share} while the " + "restored sandboxes run, not its own" + ) + found = sorted(line[5:] for line in lines if line.startswith("file=")) + if found != [f"nvx-upper-{label}"]: + raise RuntimeError( + f"the {label} sandbox finds {found} in its upper, not its own " + "file alone" + ) + + +def require_holding( + label: str, future: Future[tuple[Path, subprocess.CompletedProcess[bytes]]] +) -> None: + """Requires the restore ``label``, which ``future`` runs and whose + workload waits until the scenario releases it, to be still running, and + raises its failure otherwise.""" + if future.done(): + future.result() + raise RuntimeError(f"the {label} restore ended before the scenario released it") + + +def run_snapshot_core(backend: str, *, timeout: float, output_dir: Path) -> None: + """Capture a named snapshot with nvx-init as the guest and restore it, as + SnapshotCoreAcceptance describes.""" + artifacts = load_workload_artifacts() + output_dir.mkdir(parents=True, exist_ok=True) + root = Path(tempfile.mkdtemp(prefix="nvx-snapshot-core-")) + run_acceptance( + SnapshotCoreAcceptance( + backend, + timeout=timeout, + output_dir=output_dir, + artifacts=artifacts, + root=root, + ) + ) + + +class SnapshotTiersAcceptance(SnapshotAcceptance): + """snapshot-tiers: a named snapshot of the platform tier, whose + placeholder image and dormant scratch nvx-init's capture VM had, restored + into a managed sandbox of each image format over each upper: the EROFS + layer and a GPT disk of its tree, each over a fresh scratch image and in + RAM. + + Each restored guest must have the restore's image as its root's lower + directory over the upper, and an exec in its sandbox the same root; each + sandbox must stop cleanly and be deprovisioned. The restores change + neither image nor the snapshot. + """ + + evidence_prefix = "snapshot-tiers-" + snapshot_name = "tiers" + + def __init__( + self, + backend: str, + *, + timeout: float, + output_dir: Path, + artifacts: WorkloadArtifacts, + root: Path, + executable: Path, + kernel: Path, + initrd: Path, + ) -> None: + super().__init__( + backend, + timeout=timeout, + output_dir=output_dir, + artifacts=artifacts, + root=root, + ) + self.executable = executable + self.kernel = kernel + self.initrd = initrd + self.disk = root / "distro.gpt" + + def run(self) -> None: + self.state.mkdir() + guid = write_gpt_layer( + self.disk, + self.artifacts.distro, + executable=self.executable, + kernel=self.kernel, + initrd=self.initrd, + backend=self.backend, + timeout=self.timeout, + log_path=self.output_dir / f"{self.evidence_prefix}disk.log", + ) + # Each image format, with its image, its identity, and the layer of + # the image that the guest's root has as its lower directory. + images = { + "erofs": (self.artifacts.distro, self.artifacts.layer_uuid, "distro"), + "gpt": (self.disk, guid, "p2"), + } + digests = {name: sha256_file(image) for name, (image, _, _) in images.items()} + self.capture() + for name, (image, identity, layer) in images.items(): + for upper in ("scratch", "ram"): + self.check_restore(name, image, identity, layer=layer, upper=upper) + for name, (image, _, _) in images.items(): + if sha256_file(image) != digests[name]: + raise RuntimeError(f"the restores modified the {name} image") + self.check_snapshot_unchanged("the restores") + self.require_processes((), "after the restores") + + def check_restore( + self, + image_format: str, + image: Path, + identity: str, + *, + layer: str, + upper: str, + ) -> None: + """Restores the snapshot into a managed sandbox of ``image``, in the + format ``image_format``, over ``upper``, checks what its guest and an + exec in it report, and stops and deprovisions it.""" + label = f"{image_format}-{upper}" + state, result = self.restore( + label, + *self.restore_options( + label, image, identity, image_format=image_format, upper=upper + ), + *snapshot_probe_workload(label), + ) + report = snapshot_probe_report(result.stdout, label) + self.check_guest( + report, + label, + host=SNAPSHOT_HOSTNAME, + identity=SNAPSHOT_IDENTITY, + layer=layer, + upper=upper, + ) + # The restored sandbox runs as a started one does: an exec runs in the + # root of the same guest. + executed = self.nvx( + f"{label}-exec", + "sandbox", + "exec", + "--state-dir", + str(state), + "--timeout", + str(self.timeout), + "--entrypoint", + "/bin/sh", + "--arg=-c", + f"--arg={snapshot_probe_script(f'{label}-exec')}", + state=state, + ) + view = snapshot_probe_report(executed.stdout, f"{label}-exec") + for key in ("machine", "host", "id", "lower", "upper", "devices", "net"): + if view[key] != report[key]: + raise RuntimeError( + f"an exec in the {label} sandbox reports {key}={view[key]}, " + f"but its restore's workload {key}={report[key]}" + ) + self.stop_and_deprovision(state) + + +def run_snapshot_tiers( + executable: Path, + kernel: Path, + initrd: Path, + backend: str, + *, + timeout: float, + output_dir: Path, +) -> None: + """Restore a named snapshot of each image format over each upper, as + SnapshotTiersAcceptance describes, with a GPT disk that a guest of + ``executable``, ``kernel``, and ``initrd`` writes from the Ubuntu EROFS + layer.""" + artifacts = load_workload_artifacts() + output_dir.mkdir(parents=True, exist_ok=True) + root = Path(tempfile.mkdtemp(prefix="nvx-snapshot-tiers-")) + run_acceptance( + SnapshotTiersAcceptance( + backend, + timeout=timeout, + output_dir=output_dir, + artifacts=artifacts, + root=root, + executable=executable, + kernel=kernel, + initrd=initrd, + ) + ) + + def run_structured_outcome( executable: Path, kernel: Path, @@ -8657,284 +9862,98 @@ def run_restore_memory( memory_path = require_file(snapshot_path / "memory.bin", "snapshot memory.bin") if memory_path.stat().st_size != base_mib * 1024 * 1024: raise RuntimeError( - "memory expansion snapshot does not retain exact base RAM" - ) - - for target_mib in targets_mib: - log_path = output_dir / f"restore-memory-{target_mib}.log" - measure_once( - snapshot_restore_command( - executable, - backend, - snapshot_path, - restore_memory_mib=target_mib, - ), - environment=_restore_environment(), - timeout=timeout, - marker=b"NVX-RESTORE-MEMORY-WORKLOAD-OK", - guest_exit_prequeued=True, - log_path=log_path, - ) - expected_added = (target_mib - base_mib) * 1024 * 1024 - output = log_path.read_bytes() - marker = f"NVX-MEMORY-ONLINE-OK: added_bytes={expected_added} ".encode() - if marker not in output: - raise RuntimeError( - f"restore target {target_mib} MiB did not online the expected memory" - ) - if _snapshot_fingerprint(snapshot_path) != fingerprint: - raise RuntimeError( - f"restore memory target {target_mib} modified snapshot artifacts" - ) - - -CANCELED_CAPTURE_RCU_PREFIX = b"NVX-CANCELED-CAPTURE-RCU " -CANCELED_CAPTURE_DONE_MARKER = b"NVX-CANCELED-CAPTURE-DONE" -RCU_STALL_SUPPRESS_PATH = "/sys/module/rcupdate/parameters/rcu_cpu_stall_suppress" - - -def canceled_capture_checks() -> str: - """Return the guest checks after a snapshot request that OpenVMM released: - the time ABI status, then whether RCU stall detection is still suppressed. - - They form one console line, which the shell reads whole before it runs - any of it, so that the terminal echoes none of it into the status lines - (#785). - """ - rcu = CANCELED_CAPTURE_RCU_PREFIX.decode() - done = CANCELED_CAPTURE_DONE_MARKER.decode() - # A quote pair splits each marker, so the console's echo of the command, - # which the tty may wrap, never contains it. - return ( - status_script().removesuffix("\n") - + f'; echo "{rcu[:4]}""{rcu[4:]}$(cat {RCU_STALL_SUPPRESS_PATH})"; ' - + f'echo "{done[:4]}""{done[4:]}"\n' - ) - - -def check_canceled_capture( - output: bytes, - command: Sequence[str], - *, - context: str = "after the released snapshot request", -) -> None: - """Check that a released snapshot request left the source as it was. - - The snapshot agent saves and overrides the stall detectors' settings - before the request; when the request returns in the source, it removes its - barriers and restores the saved values (nvx-time cancel-capture). The - source then reports a passing status at generation 0 with no restore, and - RCU stall detection is no longer suppressed. - """ - monitor = TimeAbiMonitor(command) - try: - monitor.feed(output) - monitor.finish() - monitor.require_status("its checks finished") - monitor.require_boot("its checks finished") - except TimeAbiFailure as error: - raise RuntimeError(f"{context}: {error}") from error - if monitor.restores: - raise RuntimeError( - f"{context}: nvx-time status reported a restore at " - f"generation={monitor.restores[-1].get('generation')}, but the " - "request returned in the source" - ) - try: - value = _single_marker_value(output, CANCELED_CAPTURE_RCU_PREFIX) - except RuntimeError as error: - raise RuntimeError(f"{context}: {error}") from error - if value != b"0": - raise RuntimeError( - f"{context}: rcu_cpu_stall_suppress is {value.decode(errors='replace')!r}, " - "not 0: the snapshot agent did not restore the value it saved" - ) - - -def run_snapshot_core( - executable: Path, - kernel: Path, - initrd: Path, - backend: str, - *, - memory_mib: int, - timeout: float, - output_dir: Path, -) -> None: - no_destination_command = workload_boot_command( - executable, - backend, - kernel, - initrd, - memory_mib, - "quiet loglevel=0", - ) - no_destination_marker = b"NVX-SNAPSHOT-NO-DESTINATION-OK" - with OpenvmmProcess( - no_destination_command, - output_dir / "snapshot-core-no-destination.log", - ) as process: - process.wait_for(BOOT_MARKER, timeout) - process.send_line("nvx-snapshot; echo NVX-SNAPSHOT-NO-DESTINATION-OK") - process.wait_for_line(no_destination_marker, timeout) - checks_start = len(process.output) - process.send_bytes(canceled_capture_checks().encode()) - process.wait_for_line( - CANCELED_CAPTURE_DONE_MARKER, max(timeout, STATUS_TIMEOUT_SECONDS) - ) - check_canceled_capture(process.output[checks_start:], no_destination_command) - process.send_line("nvx-exit 0") - result = process.wait(timeout) - if result.returncode != 0: - raise RuntimeError(f"no-destination OpenVMM exited with {result.returncode}") - if _output_lines(result.output).count(no_destination_marker) != 1: - raise RuntimeError( - "snapshot request without a destination did not continue once" - ) - - with tempfile.TemporaryDirectory(prefix="nvx-snapshot-core-") as temporary: - snapshot_path = Path(temporary) / "snapshot" - capture_command = [ - *workload_boot_command( - executable, - backend, - kernel, - initrd, - memory_mib, - "quiet loglevel=0", - ), - "--snapshot-destination", - str(snapshot_path), - ] - with OpenvmmProcess( - capture_command, - output_dir / "snapshot-core-capture.log", - ) as process: - process.wait_for(BOOT_MARKER, timeout) - _stage_script( - process, - "/tmp/nvx-snapshot-core", - "NVX_SNAPSHOT_CORE", - _read_script("snapshot-core.sh"), - ) - source = process.wait(timeout) - if source.returncode != 0: - raise RuntimeError( - f"snapshot source OpenVMM exited with {source.returncode}" - ) - if SNAPSHOT_CORE_CONTINUED_MARKER in _output_lines(source.output): - raise RuntimeError("snapshot source crossed its terminal capture boundary") - fingerprint = _snapshot_fingerprint(snapshot_path) - capture_wall_time = time.time() - rng_hashes: list[bytes] = [] - generation_ids: list[bytes] = [] - uuids: list[bytes] = [] - temp_ids: list[bytes] = [] - - for restore_index in range(2): - time.sleep(5) - minimum_downtime_centiseconds = int((time.time() - capture_wall_time) * 100) - with OpenvmmProcess( - snapshot_restore_command(executable, backend, snapshot_path), - output_dir / f"snapshot-core-restore-{restore_index}.log", - ) as process: - process.wait_for(SNAPSHOT_CORE_COMPLETION_MARKER, timeout) - restored = process.wait(timeout) - if restored.returncode != 0: - raise RuntimeError( - f"snapshot restore {restore_index} exited with {restored.returncode}" - ) - lines = _output_lines(restored.output) - if lines.count(SNAPSHOT_CORE_CONTINUED_MARKER) != 1: - raise RuntimeError( - f"snapshot restore {restore_index} did not continue exactly once" - ) - timer_wait = int( - _single_marker_value(restored.output, b"NVX-SNAPSHOT-TIMER-WAIT-") - ) - if timer_wait > 1: - raise RuntimeError( - f"snapshot restore {restore_index} waited {timer_wait}s for an expired timer" - ) - wall_delta, uptime_delta = _parse_marker_pair( - restored.output, - b"NVX-SNAPSHOT-DOWNTIME-", - ) - if max(wall_delta, uptime_delta) < 4 or abs(wall_delta - uptime_delta) > 1: - raise RuntimeError( - f"snapshot restore {restore_index} clock mismatch: " - f"wall={wall_delta}s uptime={uptime_delta}s" - ) - uptime_centiseconds = int( - _single_marker_value(restored.output, b"NVX-SNAPSHOT-UPTIME-CS-") - ) - if uptime_centiseconds + 5 < minimum_downtime_centiseconds: - raise RuntimeError( - f"snapshot restore {restore_index} discarded captured uptime: " - f"guest={uptime_centiseconds}cs host={minimum_downtime_centiseconds}cs" - ) - process_cpu, thread_cpu = _parse_marker_pair( - restored.output, - b"NVX-SNAPSHOT-CPU-", - ) - if process_cpu != 0 or thread_cpu != 0: - raise RuntimeError( - f"snapshot restore {restore_index} advanced sleeping-task CPU clocks" - ) - rng_hash = _single_marker_value(restored.output, b"NVX-SNAPSHOT-RNG-") - if len(rng_hash) != 64 or not all( - byte in b"0123456789abcdefABCDEF" for byte in rng_hash - ): - raise RuntimeError( - f"snapshot restore {restore_index} emitted a malformed RNG digest" - ) - rng_hashes.append(rng_hash) - generation_id = _single_marker_value( - restored.output, b"NVX-SNAPSHOT-GENERATION-ID-" - ) - if len(generation_id) != 32 or not all( - byte in b"0123456789abcdefABCDEF" for byte in generation_id - ): - raise RuntimeError( - f"snapshot restore {restore_index} emitted a malformed generation ID" - ) - generation_ids.append(generation_id) - restored_uuid = _single_marker_value(restored.output, b"NVX-SNAPSHOT-UUID-") - try: - uuid.UUID(restored_uuid.decode("ascii")) - except (UnicodeDecodeError, ValueError) as error: - raise RuntimeError( - f"snapshot restore {restore_index} emitted a malformed UUID" - ) from error - uuids.append(restored_uuid) - temp_id = _single_marker_value(restored.output, b"NVX-SNAPSHOT-TEMP-ID-") - if not temp_id or any(byte in b" \t\r\n/" for byte in temp_id): - raise RuntimeError( - f"snapshot restore {restore_index} emitted a malformed temp ID" - ) - temp_ids.append(temp_id) - port_access = _single_marker_value( - restored.output, b"NVX-SNAPSHOT-PORT-ACCESS-" + "memory expansion snapshot does not retain exact base RAM" + ) + + for target_mib in targets_mib: + log_path = output_dir / f"restore-memory-{target_mib}.log" + measure_once( + snapshot_restore_command( + executable, + backend, + snapshot_path, + restore_memory_mib=target_mib, + ), + environment=_restore_environment(), + timeout=timeout, + marker=b"NVX-RESTORE-MEMORY-WORKLOAD-OK", + guest_exit_prequeued=True, + log_path=log_path, ) - if port_access != b"denied": + expected_added = (target_mib - base_mib) * 1024 * 1024 + output = log_path.read_bytes() + marker = f"NVX-MEMORY-ONLINE-OK: added_bytes={expected_added} ".encode() + if marker not in output: raise RuntimeError( - f"snapshot restore {restore_index} started a process that reached " - f"OpenVMM's ports: {port_access.decode(errors='replace')}" + f"restore target {target_mib} MiB did not online the expected memory" ) if _snapshot_fingerprint(snapshot_path) != fingerprint: raise RuntimeError( - f"snapshot restore {restore_index} modified snapshot artifacts" + f"restore memory target {target_mib} modified snapshot artifacts" ) - if rng_hashes[0] == rng_hashes[1]: - raise RuntimeError( - "fresh restore entropy did not diversify guest RNG output" - ) - if generation_ids[0] == generation_ids[1]: - raise RuntimeError("restored clones reused the VM generation ID") - if uuids[0] == uuids[1]: - raise RuntimeError("restored clones reused a kernel UUID") - if temp_ids[0] == temp_ids[1]: - raise RuntimeError("restored clones reused a temporary-file ID") + + +CANCELED_CAPTURE_RCU_PREFIX = b"NVX-CANCELED-CAPTURE-RCU " +CANCELED_CAPTURE_DONE_MARKER = b"NVX-CANCELED-CAPTURE-DONE" +RCU_STALL_SUPPRESS_PATH = "/sys/module/rcupdate/parameters/rcu_cpu_stall_suppress" + + +def canceled_capture_checks() -> str: + """Return the guest checks after a snapshot request that OpenVMM released: + the time ABI status, then whether RCU stall detection is still suppressed. + + They form one console line, which the shell reads whole before it runs + any of it, so that the terminal echoes none of it into the status lines + (#785). + """ + rcu = CANCELED_CAPTURE_RCU_PREFIX.decode() + done = CANCELED_CAPTURE_DONE_MARKER.decode() + # A quote pair splits each marker, so the console's echo of the command, + # which the tty may wrap, never contains it. + return ( + status_script().removesuffix("\n") + + f'; echo "{rcu[:4]}""{rcu[4:]}$(cat {RCU_STALL_SUPPRESS_PATH})"; ' + + f'echo "{done[:4]}""{done[4:]}"\n' + ) + + +def check_canceled_capture( + output: bytes, + command: Sequence[str], + *, + context: str = "after the released snapshot request", +) -> None: + """Check that a released snapshot request left the source as it was. + + The snapshot agent saves and overrides the stall detectors' settings + before the request; when the request returns in the source, it removes its + barriers and restores the saved values (nvx-time cancel-capture). The + source then reports a passing status at generation 0 with no restore, and + RCU stall detection is no longer suppressed. + """ + monitor = TimeAbiMonitor(command) + try: + monitor.feed(output) + monitor.finish() + monitor.require_status("its checks finished") + monitor.require_boot("its checks finished") + except TimeAbiFailure as error: + raise RuntimeError(f"{context}: {error}") from error + if monitor.restores: + raise RuntimeError( + f"{context}: nvx-time status reported a restore at " + f"generation={monitor.restores[-1].get('generation')}, but the " + "request returned in the source" + ) + try: + value = _single_marker_value(output, CANCELED_CAPTURE_RCU_PREFIX) + except RuntimeError as error: + raise RuntimeError(f"{context}: {error}") from error + if value != b"0": + raise RuntimeError( + f"{context}: rcu_cpu_stall_suppress is {value.decode(errors='replace')!r}, " + "not 0: the snapshot agent did not restore the value it saved" + ) SNAPSHOT_ROLLBACK_STATUS_PREFIX = b"NVX-SNAPSHOT-ROLLBACK-STATUS-" @@ -8987,8 +10006,8 @@ def run_snapshot_rollback( The snapshot destination's parent disappears after boot, so the capture quiesces and saves the VM, and then fails to publish, which rolls it back: - the guest's request returns in the source, as for a request that OpenVMM - releases (``snapshot-core``), and the capture's profile records show the + the guest's request returns in the source, as one that OpenVMM releases + for want of a destination does, and the capture's profile records show the quiesce and the save, but no publication. Once the parent is back, a second request of the same source publishes a snapshot, whose restore continues past that request. @@ -10802,442 +11821,6 @@ def fresh_restore_command(scratch: Path | None) -> list[str]: ) -def _snapshot_tier_kinds(tier: str) -> tuple[bool, bool, bool]: - if tier not in ("platform", "workload-start", "instance-checkpoint"): - raise ValueError(f"unsupported snapshot tier {tier!r}") - return tier == "platform", tier == "workload-start", tier == "instance-checkpoint" - - -def _snapshot_tier_script(tier: str) -> str: - platform, workload_start, instance_checkpoint = _snapshot_tier_kinds(tier) - prefix = f"NVX-TIER-{tier.upper()}" - workload_marker = f"{prefix}-WORKLOAD-RAN" - paired_setup = "" - if not platform: - paired_setup = f"""mkdir -p /run/nvx/scratch /sys/fs/cgroup -mountpoint -q /sys/fs/cgroup || mount -t cgroup2 none /sys/fs/cgroup -mkdir -p /sys/fs/cgroup/container -mkfs.ext4 -F /dev/vdb >/dev/null -mount -t ext4 /dev/vdb /run/nvx/scratch -printf 'captured-workload-id\\n' >/run/nvx/workload-machine-id -barrier=/run/nvx/test-container-start -mkfifo "$barrier" -(IFS= read -r start <"$barrier" -[ "$start" = start ] -exec unshare --mount --uts --fork --kill-child sh -c 'mount --make-rprivate /; mkdir -p /etc; : >/etc/machine-id; mount --bind /run/nvx/workload-machine-id /etc/machine-id; hostname captured-workload; while [ ! -e /run/nvx/restore-active ]; do sleep 0.01; done; echo {workload_marker}; : >/run/nvx/workload-ran; while :; do sleep 60; done') & -workload_pid=$! -echo "$workload_pid" >/sys/fs/cgroup/container/cgroup.procs -printf 'start\\n' >"$barrier" -rm -f "$barrier" -echo "$workload_pid" >/run/nvx/container.pid -tries=0 -while [ "$(nsenter -t "$workload_pid" -u hostname 2>/dev/null || true)" != captured-workload ] && [ "$tries" -lt 200 ]; do - sleep 0.01 - tries=$((tries + 1)) -done -[ "$tries" -lt 200 ] || {{ nvx-exit 67; exit 67; }}""" - - repair_marker = f"{prefix}-REPAIR" - premature_marker = f"{prefix}-PREMATURE-INPUT" - runtime_hook = ":" - if platform: - runtime_hook = f"""cat >/run/nvx/runtime-post-restore <<'NVX_TIER_HOOK' -#!/bin/sh -set -eu -case "${{NVX_VM_GENERATION_ID:-}}" in - '' | *[!0-9a-f]*) exit 71 ;; -esac -[ "${{#NVX_VM_GENERATION_ID}}" -eq 32 ] || exit 71 -echo {repair_marker} -sleep 1 -[ "$(date -u +%Y)" -ge 2025 ] -grep -Eq '^[0-9a-f]{{32}}$' /etc/machine-id -[ "$(hostname)" = nvx-sandbox ] -if ! kill -0 "$(cat /run/nvx/gate-reader.pid)" 2>/dev/null; then - echo {premature_marker} - exit 1 -fi -NVX_TIER_HOOK -chmod +x /run/nvx/runtime-post-restore""" - elif workload_start: - runtime_hook = f"""cat >/run/nvx/runtime-post-restore <<'NVX_TIER_HOOK' -#!/bin/sh -set -eu -case "${{NVX_VM_GENERATION_ID:-}}" in - '' | *[!0-9a-f]*) exit 71 ;; -esac -[ "${{#NVX_VM_GENERATION_ID}}" -eq 32 ] || exit 71 -: >/run/nvx/restore-active -echo {repair_marker} -sleep 1 -grep -Eq '^[0-9a-f]{{32}}$' /etc/machine-id -[ "$(cat /run/nvx/workload-machine-id)" = "$(cat /etc/machine-id)" ] -[ "$(nsenter -t "$(cat /run/nvx/container.pid)" -m -r cat /etc/machine-id)" = "$(cat /etc/machine-id)" ] -[ "$(nsenter -t "$(cat /run/nvx/container.pid)" -u hostname)" = restored-workload-start ] -if ! kill -0 "$(cat /run/nvx/gate-reader.pid)" 2>/dev/null; then - echo {premature_marker} - exit 1 -fi -NVX_TIER_HOOK -chmod +x /run/nvx/runtime-post-restore""" - - repair_action = ":" - if workload_start: - repair_action = """tries=0 -while [ ! -e /run/nvx/workload-ran ] && [ "$tries" -lt 200 ]; do - sleep 0.01 - tries=$((tries + 1)) -done -[ -e /run/nvx/workload-ran ] || { nvx-exit 70; exit 70; }""" - elif instance_checkpoint: - repair_action = f"""[ "$(cat /etc/machine-id)" = captured-machine-id ] -[ "$(cat /run/nvx/workload-machine-id)" = captured-workload-id ] -[ "$(nsenter -t "$(cat /run/nvx/container.pid)" -m -r cat /etc/machine-id)" = captured-workload-id ] -[ "$(nsenter -t "$(cat /run/nvx/container.pid)" -u hostname)" = captured-workload ] -printf 'restored-direct-claimed\\n' >/run/nvx/scratch/direct-claimed -sync -echo {repair_marker}""" - - pre_capture_action = ( - "date -u -s 200001010000.00 >/dev/null\n" - "printf 'captured-machine-id\\n' >/etc/machine-id" - if platform - else "hostname captured-host\nprintf 'captured-machine-id\\n' >/etc/machine-id" - ) - capture_action = ( - "/sbin/nvx-snapshot" - if instance_checkpoint - else f"/sbin/nvx-snapshot --tier {tier}" - ) - return _render_script( - "snapshot-tier.sh.in", - MISMATCHED_TIER="workload-start" if platform else "platform", - MISMATCH_MARKER=f"{prefix}-MISMATCH-REJECTED", - PAIRED_SETUP=paired_setup, - SETUP_MARKER=f"{prefix}-SETUP-READY", - RUNTIME_HOOK=runtime_hook, - HOOK_MARKER=f"{prefix}-HOOK-READY", - PRE_CAPTURE_ACTION=pre_capture_action, - INPUT_MARKER=f"{prefix}-INPUT", - CAPTURE_MARKER=f"{prefix}-CAPTURE", - CAPTURE_ACTION=capture_action, - REPAIR_ACTION=repair_action, - RELEASED_MARKER=f"{prefix}-RELEASED", - LAYER_MARKER=f"{prefix}-LAYER-", - # After the tier's assertions, the restored guest asks nvx-time status, - # which waits for the restore's deferred checks, the debug kernel's - # watchdogs among them, and prints the restore line; the script turns - # off set -e first, so a failing status still reports its exit status. - # It then waits for one byte from the host before nvx-exit, so the - # query's lines reach the virtio console before the VM stops. - STATUS_QUERY=status_script().rstrip("\n"), - ) - - -def _snapshot_storage_policy(tier: str) -> tuple[str, ...]: - if tier == "platform": - return () - if tier == "workload-start": - generation = "00112233445566778899aabbccddee01" - restore_mode = "private-copy" - elif tier == "instance-checkpoint": - generation = "00112233445566778899aabbccddee02" - restore_mode = "direct-claimed" - else: - raise ValueError(f"unsupported snapshot tier {tier!r}") - return ( - "--snapshot-block-identity", - "generation", - "--snapshot-generation-id", - generation, - "--snapshot-scratch-restore-mode", - restore_mode, - ) - - -def _run_snapshot_tier( - tier: str, - executable: Path, - kernel: Path, - initrd: Path, - backend: str, - *, - memory_mib: int, - timeout: float, - output_dir: Path, -) -> None: - platform, workload_start, instance_checkpoint = _snapshot_tier_kinds(tier) - - prefix = f"NVX-TIER-{tier.upper()}" - capture_marker = f"{prefix}-CAPTURE".encode() - repair_marker = f"{prefix}-REPAIR".encode() - input_marker = f"{prefix}-INPUT".encode() - released_marker = f"{prefix}-RELEASED".encode() - premature_marker = f"{prefix}-PREMATURE-INPUT".encode() - mismatch_marker = f"{prefix}-MISMATCH-REJECTED".encode() - workload_marker = f"{prefix}-WORKLOAD-RAN".encode() - layer_marker = f"{prefix}-LAYER-".encode() - - with tempfile.TemporaryDirectory(prefix=f"nvx-snapshot-tier-{tier}-") as temporary: - root = Path(temporary) - snapshot = root / "snapshot" - source_layer = root / "source.erofs" - replacement_layer = root / "replacement.erofs" - source_scratch = root / "source-scratch.raw" - restore_scratch = root / "restore-scratch.raw" - _write_pattern(source_layer, 1024 * 1024, 0x3C) - _write_pattern(replacement_layer, 1024 * 1024, 0xC3) - _write_pattern(source_scratch, 1024 * 1024, 0xA5) - _write_pattern(restore_scratch, 1024 * 1024, 0x5A) - address = _available_tcp_address() - - capture_command = workload_boot_command( - executable, - backend, - kernel, - initrd, - memory_mib, - "" if platform else f"nvx_hostname=restored-{tier}", - network="10.0.0.2/24", - boot_console=False, - ) - capture_command.extend( - ( - "--allow-host", - "10.0.0.1", - "--virtio-console", - f"listen=tcp:{address[0]}:{address[1]}", - "--snapshot-destination", - str(snapshot), - "--snapshot-tier", - tier, - *_snapshot_storage_policy(tier), - "--microvm-sandbox-block", - _block_arg("distro", source_layer, read_only=True), - "--microvm-sandbox-block", - _block_arg("scratch", source_scratch, read_only=False), - ) - ) - - source_console_path = output_dir / f"snapshot-tier-{tier}-capture-console.log" - source_console = b"" - console: TcpConsole | None = None - with OpenvmmProcess( - capture_command, - output_dir / f"snapshot-tier-{tier}-capture-process.log", - ) as process: - try: - # The guest's shell is on this virtio console, so it answers - # the cold boot's status query there. - console = TcpConsole.connect( - address, - timeout, - monitor=TimeAbiMonitor(capture_command), - time_abi_status=True, - ) - console.wait_for(BOOT_MARKER, timeout) - script = _snapshot_tier_script(tier) - console.send_bytes( - b"cat >/tmp/nvx-snapshot-tier <<'NVX_SNAPSHOT_TIER'\n" - + script.encode() - + b"NVX_SNAPSHOT_TIER\nsh /tmp/nvx-snapshot-tier\n" - ) - console.wait_for(capture_marker, timeout) - source = process.wait(timeout) - source_console = console.finish() - console = None - finally: - source_console = _persist_console_log( - console, - source_console, - source_console_path, - ) - if source.returncode != 0: - raise RuntimeError( - f"{tier} snapshot capture exited with {source.returncode}" - ) - source_lines = _output_lines(source_console) - if source_lines.count(mismatch_marker) != 1 or any( - marker in source_lines - for marker in (repair_marker, input_marker, released_marker) - ): - raise RuntimeError(f"{tier} source crossed its terminal capture boundary") - fingerprint = _scratch_snapshot_fingerprint(snapshot) - - restore_layer = replacement_layer if platform else source_layer - restore_command = snapshot_restore_command( - executable, - backend, - snapshot, - network_profile="portable", - boot_console=False, - ) - restore_command.extend( - ( - "--allow-host", - "10.0.0.1", - "--microvm-sandbox-block", - _block_arg("distro", restore_layer, read_only=True), - ) - ) - if platform: - restore_command.extend( - ( - "--microvm-sandbox-block", - _block_arg("scratch", restore_scratch, read_only=False), - ) - ) - - if instance_checkpoint: - wrong_profile = [ - str(executable), - "--single-process", - "--hypervisor", - backend, - "--restore-snapshot", - str(snapshot), - ] - _expect_process_failure( - wrong_profile, - output_dir / f"snapshot-tier-{tier}-wrong-profile.log", - timeout, - b"microVM snapshot restore requires --machine microvm", - ) - if (snapshot / "resume.claim").exists(): - raise RuntimeError("wrong-profile restore consumed instance checkpoint") - - expected_layer = layer_marker + (b"195" if platform else b"60") - restore_console_path = output_dir / f"snapshot-tier-{tier}-restore-console.log" - restore_console = b"" - console = None - with OpenvmmProcess( - restore_command, - output_dir / f"snapshot-tier-{tier}-restore-process.log", - ) as process: - try: - console = TcpConsole.connect( - address, timeout, monitor=TimeAbiMonitor(restore_command) - ) - # A status query that one of the milestone reads below consumes - # must still count for the wait after them. - status_baseline = console.status_queries - console.send_bytes(b"Z") - console.wait_for(repair_marker, timeout) - if workload_start: - console.wait_for(workload_marker, timeout) - console.wait_for(released_marker, timeout) - console.wait_for(expected_layer, timeout) - # The guest's status query waits for the restore's deferred - # checks. It then waits for one byte from here before nvx-exit, - # so the query's lines reach the console before the VM stops. - console.wait_for_time_abi_status( - STATUS_TIMEOUT_SECONDS, since=status_baseline - ) - console.send_bytes(b"Z") - restored = process.wait(timeout) - restore_console = console.finish() - console = None - finally: - restore_console = _persist_console_log( - console, - restore_console, - restore_console_path, - ) - if restored.returncode != 0: - raise RuntimeError(f"{tier} restore exited with {restored.returncode}") - restore_lines = _output_lines(restore_console) - if restore_lines.count(input_marker) != 1 or premature_marker in restore_lines: - raise RuntimeError(f"{tier} input crossed the restore gate") - if restore_lines.count(expected_layer) != 1: - raise RuntimeError(f"{tier} restore observed the wrong layer binding") - # The restored guest's status query, after the tier's assertions, must - # report a passing restore line, so the deferred restore checks ran. - _check_restore_status( - restore_console, - restore_command, - processors=int(restore_command[restore_command.index("--processors") + 1]), - context=f"{tier} restore", - ) - restored_fingerprint = _scratch_snapshot_fingerprint(snapshot) - if restored_fingerprint[:3] != fingerprint[:3]: - raise RuntimeError(f"{tier} restore modified snapshot payloads") - if instance_checkpoint: - if restored_fingerprint[3] == fingerprint[3]: - raise RuntimeError( - "instance-checkpoint restore did not modify direct-claimed scratch" - ) - elif restored_fingerprint != fingerprint: - raise RuntimeError(f"{tier} restore modified reusable snapshot scratch") - - if workload_start: - timeout_command = [ - *restore_command, - "--restore-gate-timeout-ms", - "500", - ] - timeout_console: TcpConsole | None = None - timeout_console_path = ( - output_dir / f"snapshot-tier-{tier}-timeout-console.log" - ) - timeout_output = b"" - with OpenvmmProcess( - timeout_command, - output_dir / f"snapshot-tier-{tier}-timeout-process.log", - ) as process: - try: - timeout_console = TcpConsole.connect( - address, timeout, monitor=TimeAbiMonitor(timeout_command) - ) - timeout_console.send_bytes(b"Z") - timed_out = process.wait(timeout) - timeout_output = timeout_console.finish() - timeout_console = None - finally: - timeout_output = _persist_console_log( - timeout_console, - timeout_output, - timeout_console_path, - ) - timeout_lines = _output_lines(timeout_output) - if timed_out.returncode == 0 or any( - marker in timeout_lines - for marker in (input_marker, released_marker, workload_marker) - ): - raise RuntimeError("workload-start timeout released gated guest state") - - if instance_checkpoint: - _expect_process_failure( - restore_command, - output_dir / f"snapshot-tier-{tier}-duplicate.log", - timeout, - b"resume snapshot has already been claimed", - ) - - -def run_snapshot_tiers( - executable: Path, - kernel: Path, - initrd: Path, - backend: str, - *, - memory_mib: int, - timeout: float, - output_dir: Path, - tiers: tuple[str, ...] = ("platform", "workload-start", "instance-checkpoint"), -) -> None: - for tier in tiers: - _run_snapshot_tier( - tier, - executable, - kernel, - initrd, - backend, - memory_mib=memory_mib, - timeout=timeout, - output_dir=output_dir, - ) - - def require_debug_kernel(config: Path) -> None: """Fail unless a kernel config enables the debug kernel's detectors. @@ -11985,17 +12568,9 @@ def _run_scenarios( if "snapshot-core" in scenarios: report.start( "snapshot-core", - f"Running microVM snapshot-core correctness on OpenVMM/{args.backend}", - ) - run_snapshot_core( - executable, - kernel, - initrd, - args.backend, - memory_mib=args.memory_mib, - timeout=args.timeout, - output_dir=output_dir, + f"Running public named snapshot restores on OpenVMM/{args.backend}", ) + run_snapshot_core(args.backend, timeout=args.timeout, output_dir=output_dir) if "snapshot-rollback" in scenarios: report.start( "snapshot-rollback", @@ -12013,14 +12588,14 @@ def _run_scenarios( if "snapshot-tiers" in scenarios: report.start( "snapshot-tiers", - f"Running microVM snapshot-tier correctness on OpenVMM/{args.backend}", + "Running public named snapshot restores of each image and upper " + f"on OpenVMM/{args.backend}", ) run_snapshot_tiers( executable, kernel, initrd, args.backend, - memory_mib=args.memory_mib, timeout=args.timeout, output_dir=output_dir, ) diff --git a/scripts/test_microvm_tests.py b/scripts/test_microvm_tests.py index b6f5fcb2d..9c2cb664c 100644 --- a/scripts/test_microvm_tests.py +++ b/scripts/test_microvm_tests.py @@ -6,7 +6,6 @@ import errno import functools import hashlib -import inspect import io import ipaddress import itertools @@ -3521,6 +3520,903 @@ def test_the_scenario_needs_cargo_before_it_starts_anything(self): self.assertEqual(test.calls, []) +# The length of the fake scratch template, and so of the dormant scratch of +# the fake snapshots, in MiB. +_SNAPSHOT_SCRATCH_MIB = 1 + + +def _probe_label(script: str) -> str: + """Returns the label of the snapshot probe ``script``.""" + labelled = re.search(r"^report label (\S+)$", script, re.MULTILINE) + assert labelled is not None + return labelled[1] + + +class _FakeSnapshotCli: + """Emulates the commands of `nvx-rs` that the public snapshot scenarios + run: `snapshot capture` and `snapshot restore`, `run --restore-snapshot`, + and `sandbox exec`, `stop`, and `deprovision` of a restored sandbox, with + the snapshot's lock, and the guests of the restores, which run the + snapshot probe. + + Each fault names a regression of the capture, of the restores, or of + their guests that the scenarios must detect. + """ + + def __init__(self, *faults: str) -> None: + self.faults = set(faults) + # The snapshot's lock, which the scenarios' one snapshot takes. + self.lock = threading.Lock() + self.records = threading.Lock() + self.booted = 0.0 + self.running: dict[int, bool] = {} + self.guests: dict[Path, dict[str, str]] = {} + self.backends: dict[Path, str] = {} + # The files that execs wrote in each sandbox's upper, and its share, + # by its state, and the state of the newest managed restore. + self.uppers: dict[Path, set[str]] = {} + self.shares: dict[Path, Path] = {} + self.newest: Path | None = None + # The VMs that no termination ends, and those that cleanup ended + # through their records. + self.unkillable: set[int] = set() + self.killed: list[int] = [] + self.next_pid = 7000 + self.commands: list[list[str]] = [] + + def __call__( + self, command: list[str], **_: object + ) -> subprocess.CompletedProcess[bytes]: + arguments = command[command.index("--root") + 2 :] + with self.records: + self.commands.append(arguments) + returncode, stdout, stderr = self._dispatch(arguments) + return subprocess.CompletedProcess(command, returncode, stdout, stderr) + + @staticmethod + def _options(words: list[str]) -> tuple[dict[str, str], list[str]]: + values: dict[str, str] = {} + arguments: list[str] = [] + index = 0 + while index < len(words): + if words[index].startswith("--arg="): + arguments.append(words[index].removeprefix("--arg=")) + index += 1 + else: + values[words[index]] = words[index + 1] + index += 2 + return values, arguments + + @staticmethod + def _error(code: str, message: str) -> _CliResult: + return 1, b"", f"error[{code}]: {message}\n".encode() + + def _dispatch(self, arguments: list[str]) -> _CliResult: + if arguments[:2] == ["snapshot", "capture"]: + return self._capture(arguments[2], self._options(arguments[3:])[0]) + if arguments[:2] in (["snapshot", "restore"], ["run", "--restore-snapshot"]): + values, workload = self._options(arguments[3:]) + return self._restore( + arguments[2], values, workload, one_off=arguments[0] == "run" + ) + assert arguments[0] == "sandbox" + values, workload = self._options(arguments[2:]) + handlers: dict[str, Callable[[Path, dict[str, str], list[str]], _CliResult]] + handlers = { + "exec": self._exec, + "stop": self._stop, + "deprovision": self._deprovision, + } + return handlers[arguments[1]](Path(values["--state-dir"]), values, workload) + + def _capture(self, name: str, values: dict[str, str]) -> _CliResult: + assert values["--scratch-mib"] == str(_SNAPSHOT_SCRATCH_MIB) + assert values["--timeout"].isdigit() + if {"capture-vm-runs", "capture-vm-survives"} & self.faults: + # The capture fails, and its launch journal keeps naming its VM, + # which still runs, and outlives every attempt to end it, too. + pid = self._spawn() + if "capture-vm-survives" in self.faults: + self.unkillable.add(pid) + journal = Path(values["--store-dir"]) / "capture" + journal.mkdir(parents=True) + (journal / microvm_tests.CAPTURE_JOURNAL).write_text( + json.dumps({"process": {"pid": pid, "startTime": pid * 7}}), + encoding="utf-8", + ) + return self._error("timed-out", "the capture's VM did not exit in time") + store = Path(values["--store-dir"]) + directory = microvm_tests.snapshot_directory(store, name) + (directory / "openvmm").mkdir(parents=True) + for entry in ( + "key.bin", + "snapshot.json", + "openvmm/manifest.bin", + "openvmm/memory.bin", + "openvmm/state.bin", + ): + (directory / entry).write_bytes(entry.encode()) + if "capture-leaves-its-vm" in self.faults: + (store / "capture").mkdir() + if "capture-publishes-more" in self.faults: + (directory / "scratch.img").write_bytes(b"scratch") + self.booted = time.time() + return 0, b"Snapshot captured\n", b"" + + def _restore( + self, name: str, values: dict[str, str], workload: list[str], *, one_off: bool + ) -> _CliResult: + state = Path(values["--state-dir"]) + timeout = float(values["--timeout"]) + busy = self._error( + "sandbox-busy", + f"another command still holds the lock of the snapshot {name}", + ) + # A restore opens the snapshot under its lock before it provisions. + unlocked = "unserialized-restores" in self.faults or ( + "waiters-skip-the-lock" in self.faults and state.name.startswith("wait-") + ) + if not unlocked: + if not self.lock.acquire(timeout=timeout): + return busy + self.lock.release() + state.mkdir() + for entry in ("config.json", "sandbox.lock"): + (state / entry).write_text("{}", encoding="utf-8") + result = self._start( + name, state, values, workload, one_off=one_off, unlocked=unlocked + ) + if one_off and "run-keeps-its-sandbox" not in self.faults: + shutil.rmtree(state) + return result + + def _start( + self, + name: str, + state: Path, + values: dict[str, str], + workload: list[str], + *, + one_off: bool, + unlocked: bool, + ) -> _CliResult: + scratch = values.get("--scratch") + zeroed = False + if scratch is not None: + image = Path(scratch).read_bytes() + dormant = _SNAPSHOT_SCRATCH_MIB << 20 + geometry = "BlockGeometry {{ length: {}, logical_block_size: 512 }}" + if len(image) != dormant and "serves-any-scratch" not in self.faults: + if "launches-before-refusing" in self.faults: + # A VM starts and exits before the refusal, and leaves its + # log and outcome report in the sandbox's state. + self.terminate(self._spawn()) + for name in ("openvmm.log", "outcome.json"): + (state / name).write_text("{}", encoding="utf-8") + return self._error( + "unsupported", + f"snapshot scratch geometry {geometry.format(len(image))} differs " + f"from the dormant scratch's {geometry.format(dormant)}", + ) + zeroed = not any(image) and "mounts-zeroed-scratch" not in self.faults + if not unlocked and not self.lock.acquire(timeout=float(values["--timeout"])): + return self._error( + "sandbox-busy", + f"another command still holds the lock of the snapshot {name}", + ) + locked = not unlocked + if "waiters-share-the-lock" in self.faults and state.name.startswith("wait-"): + # The waiters wait for the first restore, but not for each other. + self.lock.release() + locked = False + try: + (state / "openvmm").mkdir() + directory = microvm_tests.snapshot_directory( + Path(values["--store-dir"]), name + ) + if one_off and "restore-changes-snapshot" in self.faults: + with (directory / "openvmm" / "memory.bin").open("ab") as memory: + memory.write(b"changed") + if one_off and "leaks-a-link" in self.faults: + leftover = state.parent.parent / "leftover.bin" + if not leftover.exists(): + os.link(directory / "openvmm" / "memory.bin", leftover) + if zeroed: + # The guest cannot mount the zeroed scratch as its upper, so its + # VM exits before the workload runs, and frees the lock. + self.terminate(self._spawn()) + if "failure-keeps-the-lock" in self.faults: + locked = False + return self._error( + "exited", "OpenVMM exited during startup: guest-exit status 125" + ) + guest = self._guest(values) + self.guests[state] = guest + self.backends[state] = values["--hypervisor"] + assert values["--entrypoint"] == "/bin/sh" and workload[0] == "-c" + script = workload[1] + stdout = self._probe(script, guest) + label = _probe_label(script) + held = script == microvm_tests.snapshot_probe_script(label, hold=True) + if not held: + assert script == microvm_tests.snapshot_probe_script(label) + # A held workload keeps the lock until its share releases it, but + # for the first one's that ends at once. + if held and not (label == "hold" and "holder-ends-early" in self.faults): + share = self._share_path(values) + assert share is not None + release = share / microvm_tests.SNAPSHOT_RELEASE + deadline = time.monotonic() + float(values["--timeout"]) + while not release.exists(): + if time.monotonic() >= deadline: + return self._error("exited", "the workload timed out") + time.sleep(0.01) + finally: + if locked: + self.lock.release() + if not one_off: + share = self._share_path(values) + if share is not None: + self.shares[state] = share + self._launch(state) + return 0, stdout, b"" + + def _guest(self, values: dict[str, str]) -> dict[str, str]: + """Returns what the guest of a restore of ``values`` keeps.""" + gpt = values.get("--layer-format") == "gpt" + ram = values.get("--upper") == "ram" or "upper-in-ram" in self.faults + return { + "machine": ( + "0" * 32 if "shared-machine-id" in self.faults else uuid.uuid4().hex + ), + "host": ( + "nvx-sandbox" + if "ignores-hostname" in self.faults + else values.get("--hostname", "nvx-sandbox") + ), + "id": values.get("--workload-user", "65534:65534"), + "lower": ( + "/run/nvx/layers/p2" + if gpt and "gpt-root-is-distro" not in self.faults + else "/run/nvx/layers/distro" + ), + "upper": ("/run/nvx/tmpfs/upper" if ram else "/run/nvx/scratch/upper"), + "devices": "vda,vdb", + "net": "eth0,lo" if "has-nic" in self.faults else "lo", + "release": "ubuntu", + "share": self._share(values), + } + + @staticmethod + def _share_path(values: dict[str, str]) -> Path | None: + """Returns the host directory of the share of a restore of + ``values``, if it has one.""" + if "--mount" not in values: + return None + target, path, mode = values["--mount"].rsplit(",", 2) + assert (target, mode) == (microvm_tests.SNAPSHOT_SHARE_TARGET, "ro") + return Path(path) + + def _share(self, values: dict[str, str]) -> str: + """Returns the marker of the share of a restore of ``values``, or + none.""" + path = self._share_path(values) + if path is None: + return "none" + if "shares-one-export" in self.faults: + path = path.with_name("hold-share") + return (path / "marker").read_text(encoding="ascii") + + def _probe(self, script: str, guest: dict[str, str]) -> bytes: + """Returns what the snapshot probe ``script`` prints in ``guest``.""" + label = _probe_label(script) + wall = self.booted if "stale-clock" in self.faults else time.time() + reused = "reused-entropy" in self.faults + fields = { + "label": label, + "wall": f"{wall:.9f}", + "uptime": ( + "0.01" + if "uptime-restarts" in self.faults + else f"{wall - self.booted:.2f}" + ), + "uuid": "1" * 8 + "-1111-1111-1111-" + "1" * 12 + if reused + else str(uuid.uuid4()), + "rng": "ab" * 32 if reused else os.urandom(32).hex(), + "temp": f"tmp.{os.urandom(5).hex()}", + **guest, + } + output = " ".join(f"{key}={value}" for key, value in fields.items()) + return f"NVX-SNAPSHOT-PROBE {output}\n".encode() + + def _spawn(self) -> int: + """Returns the process ID of a new running VM.""" + with self.records: + pid = self.next_pid + self.next_pid += 1 + self.running[pid] = True + return pid + + def _launch(self, state: Path) -> None: + pid = self._spawn() + with self.records: + self.newest = state + capability = os.urandom(32) + (state / "control.capability").write_bytes(capability) + (state / "runtime.json").write_text( + json.dumps( + { + "capability": list(capability), + "vmDir": str(state.resolve()), + "process": {"pid": pid, "startTime": pid * 7, "boot": "b" * 32}, + "endpoint": str(state / "control.sock"), + "runtime": {}, + } + ), + encoding="utf-8", + ) + + def _pid(self, state: Path) -> int | None: + path = state / "runtime.json" + if not path.is_file(): + return None + return json.loads(path.read_text(encoding="utf-8"))["process"]["pid"] + + def _exec( + self, state: Path, values: dict[str, str], workload: list[str] + ) -> _CliResult: + pid = self._pid(state) + if pid is None or not self.running[pid]: + return self._error("sandbox-not-running", "sandbox is not running") + # Without uppers of their own, the sandboxes' /tmp is the first's. + upper = ( + next(iter(self.uppers.values())) + if "shares-one-upper" in self.faults and self.uppers + else self.uppers.setdefault(state, set()) + ) + if values["--entrypoint"] == "/bin/touch": + upper.add(Path(workload[0]).name) + return 0, b"", b"" + if workload == ["-c", microvm_tests.SNAPSHOT_VIEW_SCRIPT]: + # Rebound shares all show the newest restore's. + share = ( + self.shares[self.newest] + if "rebinds-shares" in self.faults and self.newest is not None + else self.shares[state] + ) + marker = (share / "marker").read_text(encoding="ascii") + files = sorted(upper) or ["nvx-upper-*"] + lines = [f"share={marker}", *(f"file={name}" for name in files)] + return 0, "".join(f"{line}\n" for line in lines).encode(), b"" + guest = dict(self.guests[state]) + if "exec-sees-another-root" in self.faults: + guest["upper"] = "/run/nvx/other/upper" + assert values["--entrypoint"] == "/bin/sh" and workload[0] == "-c" + return 0, self._probe(workload[1], guest), b"" + + def _stop(self, state: Path, _: dict[str, str], __: list[str]) -> _CliResult: + pid = self._pid(state) + if pid is None or not self.running[pid]: + return self._error("sandbox-not-running", "sandbox is not running") + self.terminate(pid) + for name in ("runtime.json", "control.capability"): + (state / name).unlink() + outcome = ( + {"operation": "restore", "category": "guest-exit", "status_code": 1} + if "stop-reports-failure" in self.faults + else microvm_tests.RESTORED_SUCCESS + ) + (state / "outcome.json").write_text( + json.dumps( + { + "schema_version": 1, + "instance_id": "33" * 16, + "backend": self.backends[state], + "outcome": outcome, + "network_policy": { + "status": "not-requested", + "status_code": 0, + "mode": "none", + "allow_rule_count": 0, + "deny_rule_count": 0, + "host_loopback": "none", + }, + "teardown": dict.fromkeys( + microvm_tests.OUTCOME_TEARDOWN_FIELDS, True + ), + } + ), + encoding="utf-8", + ) + return 0, b"", b"" + + def _deprovision( + self, state: Path, values: dict[str, str], _: list[str] + ) -> _CliResult: + pid = self._pid(state) + if pid is not None and self.running[pid]: + return self._error( + "sandbox-running", "sandbox must be stopped before deprovision" + ) + # Deprovision removes a restore's view under the snapshot's lock. + if (state / "openvmm").is_dir(): + if not self.lock.acquire(timeout=float(values["--timeout"])): + return self._error("sandbox-busy", "the snapshot's lock is busy") + self.lock.release() + shutil.rmtree(state) + return 0, b"", b"" + + def processes(self, _: str) -> tuple[int, ...]: + with self.records: + return tuple(sorted(pid for pid, up in self.running.items() if up)) + + def terminate(self, pid: int) -> None: + with self.records: + if pid not in self.unkillable: + self.running[pid] = False + + def kill(self, runtime: dict[str, Any], _timeout: float, _marker: str) -> None: + pid = runtime["process"]["pid"] + self.killed.append(pid) + self.terminate(pid) + if self.running[pid]: + raise RuntimeError(f"OpenVMM process {pid} survived its kill") + + +class PublicSnapshotTests(unittest.TestCase): + """The public snapshot scenarios, snapshot-core and snapshot-tiers, with + a fake of the CLI and of the guests of its restores.""" + + def _run( + self, root: Path, fake: _FakeSnapshotCli, scenario: str = "snapshot-core" + ) -> tuple[Path, MagicMock]: + distro = root / "ubuntu-distro.erofs" + distro.write_bytes(b"distro") + template = root / "ubuntu-smoke-scratch.ext4" + template.write_bytes(b"\x01" * (_SNAPSHOT_SCRATCH_MIB << 20)) + artifacts = managed_exec_tests.WorkloadArtifacts( + distro, "12345678-1234-1234-1234-123456789abc", template + ) + output_dir = root / "results" + mkdtemp = tempfile.mkdtemp + + def fixture(prefix: str) -> str: + return mkdtemp(prefix=prefix, dir=root) + + constants = { + "SNAPSHOT_CORE_BUSY_TIMEOUT_SECONDS": 0.05, + "SNAPSHOT_CORE_EXCLUSION_SECONDS": 0.2, + "SNAPSHOT_CORE_DOWNTIME_MARGIN_SECONDS": 0.1, + "SNAPSHOT_CLOCK_TOLERANCE_SECONDS": 0.1, + "GPT_LAYER_MIN_BYTES": 1 << 20, + } + with ExitStack() as stack: + for name, value in constants.items(): + stack.enter_context(patch.object(microvm_tests, name, value)) + stack.enter_context( + patch.object( + microvm_tests, "load_workload_artifacts", return_value=artifacts + ) + ) + stack.enter_context( + patch.object(microvm_tests.tempfile, "mkdtemp", side_effect=fixture) + ) + stack.enter_context( + patch.object(microvm_tests.subprocess, "run", side_effect=fake) + ) + stack.enter_context( + patch.object(nvx_rs, "require_binary", return_value=_FAKE_NVX_RS) + ) + for module in (microvm_tests, sandbox_lifecycle_tests): + stack.enter_context( + patch.object( + module, "openvmm_processes", side_effect=fake.processes + ) + ) + stack.enter_context( + patch.object( + module, "terminate_process", side_effect=fake.terminate + ) + ) + stack.enter_context( + patch.object(microvm_tests, "kill_openvmm", side_effect=fake.kill) + ) + guest = stack.enter_context(patch.object(microvm_tests, "run_guest_script")) + if scenario == "snapshot-core": + microvm_tests.run_snapshot_core("kvm", timeout=2, output_dir=output_dir) + else: + microvm_tests.run_snapshot_tiers( + Path("openvmm"), + Path("vmlinux"), + Path("initramfs.cpio.gz"), + "kvm", + timeout=2, + output_dir=output_dir, + ) + return output_dir, guest + + @staticmethod + def _option(arguments: list[str], name: str) -> str | None: + return arguments[arguments.index(name) + 1] if name in arguments else None + + def test_snapshot_core_captures_and_restores_one_snapshot_again_and_again(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + journal = root / "pids.jsonl" + fake = _FakeSnapshotCli() + with patch.dict( + os.environ, {"NVX_ADVERSARIAL_OPENVMM_PID_JOURNAL": str(journal)} + ): + output_dir, guest = self._run(root, fake) + recorded = [ + json.loads(line)["pid"] + for line in journal.read_text(encoding="utf-8").splitlines() + ] + evidence = {path.name for path in output_dir.iterdir()} + fixtures = list(root.glob("nvx-snapshot-core-*")) + + self.assertEqual(fixtures, []) + self.assertFalse(any(fake.running.values())) + guest.assert_not_called() + # The capture, two one-off restores, the managed restore that refuses + # the short scratch and its deprovision, the one-off restore over the + # zeroed scratch, then three managed restores at once and a one-off + # one that cannot wait for the lock, two execs in each managed one, + # and its stop and its deprovision. + labels = [arguments[1] for arguments in fake.commands] + self.assertEqual( + labels[:6], + [ + "capture", + "--restore-snapshot", + "--restore-snapshot", + "restore", + "deprovision", + "--restore-snapshot", + ], + ) + self.assertEqual( + sorted(labels[6:10]), + ["--restore-snapshot", "restore", "restore", "restore"], + ) + self.assertEqual(labels[10:], ["exec"] * 6 + ["stop", "deprovision"] * 3) + short = fake.commands[3] + self.assertEqual( + Path(str(self._option(short, "--state-dir"))).name, "short-scratch" + ) + self.assertEqual( + self._option(fake.commands[4], "--state-dir"), + self._option(short, "--state-dir"), + ) + # The three managed restores at once each take a share of their own, + # and the last a scratch upper. + managed = [ + arguments + for arguments in fake.commands + if arguments[:2] == ["snapshot", "restore"] and "--mount" in arguments + ] + self.assertEqual( + sorted( + ( + self._option(arguments, "--hostname"), + Path(str(self._option(arguments, "--mount")).split(",")[1]).name, + self._option(arguments, "--upper"), + ) + for arguments in managed + ), + [ + ("hold", "hold-share", "ram"), + ("wait-one", "wait-one-share", "ram"), + ("wait-two", "wait-two-share", None), + ], + ) + runs = [arguments for arguments in fake.commands if arguments[0] == "run"] + repeat_ram, repeat_scratch, zeroed = runs[:3] + self.assertEqual( + [ + self._option(repeat_ram, name) + for name in ("--upper", "--hostname", "--workload-user") + ], + ["ram", "core-one", "65534:65534"], + ) + self.assertIsNone(self._option(repeat_ram, "--scratch")) + self.assertEqual( + [ + self._option(repeat_scratch, name) + for name in ("--upper", "--hostname", "--workload-user") + ], + [None, "core-two", "1:1"], + ) + # Each scratch upper's image is a new copy of the template. + scratches = [ + self._option(arguments, "--scratch") + for arguments in (repeat_scratch, short, zeroed) + ] + self.assertEqual(len(set(scratches)), 3) + busy = next(arguments for arguments in runs[3:]) + self.assertEqual(self._option(busy, "--timeout"), "0.05") + # The VM of the restore over the zeroed scratch image, 7000, exited + # by itself, and only the managed restores record theirs. + self.assertEqual(sorted(recorded), [7001, 7002, 7003]) + names = { + "capture", + "repeat-ram", + "repeat-scratch", + "short-scratch", + "short-scratch-deprovision", + "zeroed-scratch", + "busy", + "checks.json", + } + for label in ("hold", "wait-one", "wait-two"): + names |= { + label, + f"{label}-write", + f"{label}-view", + f"{label}-stop", + f"{label}-deprovision", + f"{label}-outcome.json", + } + self.assertEqual( + evidence, + { + f"snapshot-core-{name}" + if name.endswith(".json") + else f"snapshot-core-{name}.log" + for name in names + }, + ) + + def test_snapshot_core_detects_regressions_of_snapshots_and_their_restores(self): + regressions = { + "capture-leaves-its-vm": "the capture left its VM's directory", + "capture-publishes-more": "the named snapshot holds ['key.bin', 'openvmm'", + "stale-clock": "the repeat-ram restore's guest clock reads", + "uptime-restarts": "the repeat-ram restore's guest booted at", + "ignores-hostname": "reports host=nvx-sandbox, expected core-one", + "has-nic": "reports net=eth0,lo, expected lo", + "run-keeps-its-sandbox": "the repeat-ram run left its sandbox's state", + "restore-changes-snapshot": "the repeat-ram restore changed the named", + "leaks-a-link": "or left links to its files", + "reused-entropy": "the repeated restores reused a kernel random UUID", + "shared-machine-id": "the repeated restores reused a machine ID", + # The guest then cannot mount the short scratch image either. + "serves-any-scratch": ( + "short-scratch: nvx-rs snapshot restore returned 1, expected 1 " + "with error[unsupported]; stdout=''; stderr='error[exited]" + ), + "launches-before-refusing": ( + "the short-scratch restore left ['config.json', 'openvmm.log', " + "'outcome.json', 'sandbox.lock'] in its sandbox's state" + ), + "mounts-zeroed-scratch": ( + "zeroed-scratch: nvx-rs run --restore-snapshot returned 0, expected 1 " + "with error[exited]" + ), + "unserialized-restores": ( + "busy: nvx-rs run --restore-snapshot returned 0, expected 1 " + "with error[sandbox-busy]" + ), + "waiters-skip-the-lock": ( + "restore linked its view of the snapshot while the hold restore " + "held the snapshot's lock" + ), + # The first restore's workload does not wait for the scenario. + "holder-ends-early": ( + "the hold restore ended before the scenario released it" + ), + "waiters-share-the-lock": ( + "restore linked its view of the snapshot while the wait-" + ), + # The next restore cannot have the lock. + "failure-keeps-the-lock": ( + "hold: nvx-rs snapshot restore returned 1, expected 0" + ), + "shares-one-export": "the wait-one restore's guest reports share=hold", + "rebinds-shares": "the hold sandbox reads the share markers ['wait-", + "shares-one-upper": ( + "the hold sandbox finds ['nvx-upper-hold', 'nvx-upper-wait-one', " + "'nvx-upper-wait-two']" + ), + "stop-reports-failure": "the hold sandbox reported the outcome", + } + for fault, message in regressions.items(): + with ( + self.subTest(fault=fault), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + fake = _FakeSnapshotCli(fault) + with self.assertRaisesRegex(RuntimeError, re.escape(message)): + self._run(root, fake) + # Cleanup stops and deprovisions every restored sandbox, and + # removes the fixture. + self.assertFalse(any(fake.running.values())) + self.assertEqual(list(root.glob("nvx-snapshot-core-*")), []) + self.assertTrue( + (root / "results" / "snapshot-core-checks.json").is_file() + ) + + def test_snapshot_core_ends_the_vm_of_a_failed_capture_through_its_journal( + self, + ): + # Cleanup ends the VM that the capture's launch journal names, as it + # ends a sandbox's, and removes the fixture once that VM has exited. + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + fake = _FakeSnapshotCli("capture-vm-runs") + with self.assertRaisesRegex(RuntimeError, "capture: nvx-rs snapshot"): + self._run(root, fake) + self.assertEqual(fake.killed, [7000]) + self.assertFalse(any(fake.running.values())) + self.assertEqual(list(root.glob("nvx-snapshot-core-*")), []) + + def test_snapshot_core_keeps_its_fixture_while_a_vm_may_survive(self): + # Cleanup cannot end the VM of a failed capture, so the fixture, whose + # store holds what recovers that VM, stays for recovery. + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + fake = _FakeSnapshotCli("capture-vm-survives") + with self.assertRaisesRegex( + RuntimeError, + r"OpenVMM process 7000 survived its kill.*" + r"fixture preserved for recovery", + ): + self._run(root, fake) + fixtures = list(root.glob("nvx-snapshot-core-*")) + self.assertEqual(len(fixtures), 1) + self.assertTrue((fixtures[0] / "sandboxes").is_dir()) + self.assertTrue((fixtures[0] / "store" / "capture").is_dir()) + + def test_snapshot_tiers_restores_each_image_format_over_each_upper(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + journal = root / "pids.jsonl" + fake = _FakeSnapshotCli() + with patch.dict( + os.environ, {"NVX_ADVERSARIAL_OPENVMM_PID_JOURNAL": str(journal)} + ): + output_dir, guest = self._run(root, fake, "snapshot-tiers") + recorded = journal.read_text(encoding="utf-8").splitlines() + fixtures = list(root.glob("nvx-snapshot-tiers-*")) + checks = json.loads( + (output_dir / "snapshot-tiers-checks.json").read_text(encoding="utf-8") + ) + + self.assertEqual(fixtures, []) + self.assertFalse(any(fake.running.values())) + self.assertEqual(len(recorded), 4) + # A guest of the control initramfs writes the GPT disk from the EROFS + # layer. + guest.assert_called_once() + self.assertEqual( + guest.call_args.args[1:], + (microvm_tests.GPT_LAYER_SCRIPT, microvm_tests.GPT_LAYER_READY_MARKER), + ) + self.assertEqual( + [record["operation"] for record in checks], + [ + "capture", + *( + f"{image}-{upper}{step}" + for image in ("erofs", "gpt") + for upper in ("scratch", "ram") + for step in ("", "-exec", "-stop", "-deprovision") + ), + ], + ) + restores = [ + arguments + for arguments in fake.commands + if arguments[:2] == ["snapshot", "restore"] + ] + self.assertEqual( + [ + ( + self._option(arguments, "--layer-format"), + self._option(arguments, "--upper"), + self._option(arguments, "--scratch") is not None, + ) + for arguments in restores + ], + [ + (None, None, True), + (None, "ram", False), + ("gpt", None, True), + ("gpt", "ram", False), + ], + ) + + def test_snapshot_tiers_detects_regressions_of_the_restored_roots(self): + regressions = { + "gpt-root-is-distro": ( + "the gpt-scratch restore's guest reports lower=/run/nvx/layers/" + "distro, expected /run/nvx/layers/p2" + ), + "upper-in-ram": ( + "the erofs-scratch restore's guest reports upper=/run/nvx/tmpfs/" + "upper, expected /run/nvx/scratch/upper" + ), + "exec-sees-another-root": ( + "an exec in the erofs-scratch sandbox reports " + "upper=/run/nvx/other/upper" + ), + } + for fault, message in regressions.items(): + with ( + self.subTest(fault=fault), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + fake = _FakeSnapshotCli(fault) + with self.assertRaisesRegex(RuntimeError, re.escape(message)): + self._run(root, fake, "snapshot-tiers") + self.assertFalse(any(fake.running.values())) + self.assertEqual(list(root.glob("nvx-snapshot-tiers-*")), []) + + def test_snapshot_probes_report_before_they_hold_and_take_plain_labels(self): + hold_loop = "while [ ! -e /data/release ]; do\n sleep 0.1\ndone\n" + for hold in (False, True): + with self.subTest(hold=hold): + script = microvm_tests.snapshot_probe_script("hold", hold=hold) + self.assertTrue(script.startswith("set -eu\n")) + self.assertIn("report label hold\n", script) + self.assertIn("if [ -r /data/marker ]; then\n", script) + # A held probe waits for the host after its report. + self.assertEqual( + script.endswith(f"printf '%s\\n' \"$line\"\n{hold_loop}"), hold + ) + self.assertEqual(hold_loop in script, hold) + for label in ("", "Hold", "wait one", "-hold", 'a"b'): + with ( + self.subTest(label=label), + self.assertRaisesRegex(ValueError, "invalid snapshot probe label"), + ): + microvm_tests.snapshot_probe_script(label) + + def test_snapshot_probe_reports_each_field_once_in_a_linux_shell(self): + if sys.platform != "linux": + self.skipTest("the probe reads Linux's /proc and /sys") + result = subprocess.run( + ["sh", "-c", microvm_tests.snapshot_probe_script("host")], + capture_output=True, + timeout=30, + check=True, + ) + report = microvm_tests.snapshot_probe_report(result.stdout, "host") + self.assertEqual(report["id"], f"{os.getuid()}:{os.getgid()}") + self.assertEqual(report["host"], socket.gethostname()) + self.assertIn("lo", report["net"].split(",")) + self.assertRegex(report["rng"], r"^[0-9a-f]{64}$") + uuid.UUID(report["uuid"]) + self.assertLessEqual(abs(float(report["wall"]) - time.time()), 30) + + def test_snapshot_probe_reports_are_one_well_formed_line_by_label(self): + fields = " ".join( + f"{key}={'hold' if key == 'label' else 'x'}" + for key in sorted(microvm_tests.SNAPSHOT_PROBE_FIELDS) + ) + report = f"NVX-SNAPSHOT-PROBE {fields}" + other = report.replace("label=hold", "label=wait-one") + output = f"noise\r\n{report}\r\n{other}\n".encode() + self.assertEqual( + microvm_tests.snapshot_probe_report(output, "hold")["label"], "hold" + ) + for broken, message in ( + (b"", "printed 0 reports, not 1"), + (f"{report}\n{report}\n".encode(), "printed 2 reports, not 1"), + (f"{report} uptime=1\n".encode(), "malformed report"), + (report.replace(" temp=x", "").encode(), "malformed report"), + (f"{report} extra\n".encode(), "malformed report"), + ): + with ( + self.subTest(message=message), + self.assertRaisesRegex(RuntimeError, message), + ): + microvm_tests.snapshot_probe_report(broken, "hold") + + class GuestIdentityScriptTests(unittest.TestCase): def test_guest_identity_checks_fail_before_success_markers(self): descriptor = microvm_tests.guest_descriptor("ubuntu") @@ -7690,9 +8586,8 @@ def test_every_tcp_console_is_monitored_and_cold_boot_shells_are_queried(self): if isinstance(node, ast.Call) and ast.unparse(node.func) == "TcpConsole.connect" ] - # The console-snapshot capture and restore, and the snapshot-tier - # capture, restore, and gate-timeout restore. - self.assertEqual(len(connects), 5) + # The console-snapshot capture and restore. + self.assertEqual(len(connects), 2) for keywords in connects: self.assertRegex( keywords.get("monitor", ""), r"^TimeAbiMonitor\(\w*command\)$" @@ -7718,80 +8613,9 @@ def test_every_tcp_console_is_monitored_and_cold_boot_shells_are_queried(self): # init starts the managed agent instead of a shell. self.assertEqual( [k["monitor"] for k in connects if k.get("time_abi_status") == "True"], - ["TimeAbiMonitor(capture_command)"] * 2, + ["TimeAbiMonitor(capture_command)"], ) - def test_snapshot_core_script_handles_no_clocksource(self): - # The time ABI fixes the clocksource on every backend, so snapshot-core - # neither selects nor waits for one. - script = microvm_tests._read_script("snapshot-core.sh") - - for text in ("clocksource", "kvm-clock", "tsc-early", "@SELECT", "@VALIDATE"): - self.assertNotIn(text, script) - self.assertIn("/sbin/nvx-reseed", script) - self.assertIn("/sbin/nvx-reseed --sample", script) - self.assertIn("NVX-SNAPSHOT-GENERATION-ID-", script) - self.assertIn("NVX-SNAPSHOT-UUID-", script) - self.assertIn("NVX-SNAPSHOT-TEMP-ID-", script) - - def test_snapshot_core_checks_that_restored_processes_cannot_reach_the_ports(self): - # nvx-time holds the port permission only for each port transaction, so - # the processes that it starts on a restored VM cannot reach the ports. - script = microvm_tests._read_script("snapshot-core.sh") - - self.assertIn( - 'port_access="$(/sbin/nvx-time test port-access)" || fail 55', script - ) - self.assertIn('echo "NVX-SNAPSHOT-PORT-ACCESS-$port_access"', script) - self.assertEqual(script.count("|| fail 55"), 1) - - def test_snapshot_core_reads_the_entropy_that_nvx_time_kept(self): - # The restore packet is consumed by nvx-time capture, which leaves the - # entropy of an untiered restore in /run/nvx/restore-entropy. - shell = _posix_shell() - if shell is None: - self.skipTest("POSIX shell is unavailable") - script = microvm_tests._read_script("snapshot-core.sh") - self.assertNotIn("OPENVMM_ENTROPY_V1", script) - self.assertNotIn("/dev/port", script) - section = ( - "generation_id_after=" - + script.split("generation_id_after=", 1)[1].split("rng=", 1)[0] - ) - generation = bytes(range(16)) - cases = ( - (generation + bytes(48), "00" * 16, "reseeded", 0), - (generation + bytes(47), "00" * 16, "FAIL 44", 44), - (bytes(16) + bytes(48), "00" * 16, "FAIL 52", 52), - (generation + bytes(48), generation.hex(), "FAIL 51", 51), - ) - for entropy, before, expected, status in cases: - with self.subTest(expected=expected): - with tempfile.TemporaryDirectory() as temporary: - path = Path(temporary) / "restore-entropy" - path.write_bytes(entropy) - body = ( - section.replace("/run/nvx/restore-entropy", path.as_posix()) - .replace("/sbin/nvx-time", "nvx_time") - .replace("/sbin/nvx-reseed", "nvx_reseed") - ) - result = subprocess.run( - [shell], - input=( - "set -eu\n" - 'fail() { echo "FAIL $1"; exit "$1"; }\n' - f"nvx_time() {{ echo {generation.hex()}; }}\n" - 'nvx_reseed() { echo "reseeded"; }\n' - f"generation_id_before={before}\n" + body - ), - text=True, - capture_output=True, - timeout=10, - check=False, - ) - self.assertEqual(result.returncode, status, result.stderr) - self.assertIn(expected, result.stdout) - def test_smp_worker_requires_bounded_local_timer_progress(self): shell = _posix_shell() if shell is None: @@ -7854,83 +8678,6 @@ def test_smp_worker_requires_bounded_local_timer_progress(self): ["1", "101", "1"], ) - def test_snapshot_core_waits_for_no_destination_marker_line_before_exit(self): - events: list[tuple[str, bytes | str | None]] = [] - marker = b"NVX-SNAPSHOT-NO-DESTINATION-OK" - command = ["openvmm", "--hypervisor", "whp", "--kernel", "vmlinux"] - - class StopAfterNoDestination(Exception): - pass - - class FakeProcess: - def __init__(self) -> None: - self.console = bytearray(b"NVX-SNAPSHOT-NO-DESTINATION-OK\r\n") - - def __enter__(self): - return self - - def __exit__( - self, - _exception_type: type[BaseException] | None, - _exception: BaseException | None, - _traceback: object | None, - ) -> None: - return None - - @property - def output(self) -> bytes: - return bytes(self.console) - - def wait_for(self, expected: bytes, _timeout: float) -> None: - events.append(("wait_for", expected)) - - def wait_for_line(self, expected: bytes, _timeout: float) -> None: - events.append(("wait_for_line", expected)) - - def send_line(self, line: str) -> None: - events.append(("send_line", line)) - - def send_bytes(self, data: bytes) -> None: - events.append(("send_bytes", data.decode())) - self.console.extend(_canceled_capture_output("whp")) - - def wait(self, _timeout: float) -> openvmm_process.OpenvmmProcessResult: - events.append(("wait", None)) - return openvmm_process.OpenvmmProcessResult(0, marker + b"\n") - - with ( - patch.object(microvm_tests, "workload_boot_command", return_value=command), - patch.object(microvm_tests, "OpenvmmProcess", return_value=FakeProcess()), - patch.object( - microvm_tests.tempfile, - "TemporaryDirectory", - side_effect=StopAfterNoDestination, - ), - self.assertRaises(StopAfterNoDestination), - ): - microvm_tests.run_snapshot_core( - Path("openvmm"), - Path("vmlinux"), - Path("initramfs"), - "whp", - memory_mib=128, - timeout=1, - output_dir=Path("logs"), - ) - - self.assertEqual( - events, - [ - ("wait_for", microvm_tests.BOOT_MARKER), - ("send_line", "nvx-snapshot; echo NVX-SNAPSHOT-NO-DESTINATION-OK"), - ("wait_for_line", marker), - ("send_bytes", microvm_tests.canceled_capture_checks()), - ("wait_for_line", microvm_tests.CANCELED_CAPTURE_DONE_MARKER), - ("send_line", "nvx-exit 0"), - ("wait", None), - ], - ) - def test_canceled_capture_checks_query_status_and_rcu_suppression(self): script = microvm_tests.canceled_capture_checks() query = time_abi.status_script().removesuffix("\n") + "; " @@ -8278,10 +9025,6 @@ def test_snapshot_marker_parsers_require_single_well_formed_values(self): ), b"17", ) - self.assertEqual( - microvm_tests._parse_marker_pair(output, b"PAIR-"), - (4, 5), - ) with self.assertRaisesRegex(RuntimeError, "exactly one"): microvm_tests._single_marker_value(b"X-1\nX-2\n", b"X-") with self.assertRaisesRegex(RuntimeError, "exactly one"): @@ -8296,8 +9039,6 @@ def test_snapshot_marker_parsers_require_single_well_formed_values(self): b"FRAME-", b"-END", ) - with self.assertRaisesRegex(RuntimeError, "malformed"): - microvm_tests._parse_marker_pair(b"PAIR-4\n", b"PAIR-") def test_console_snapshot_script_preserves_backend_specific_rx_and_tx(self): kvm, kvm_count, kvm_rx = microvm_tests._console_snapshot_script("kvm") @@ -8375,114 +9116,6 @@ def test_network_snapshot_script_renders_host_ports(self): self.assertNotIn("@HTTP_PORT@", script) self.assertNotIn("@UDP_PORT@", script) - def test_snapshot_tier_scripts_preserve_tier_specific_policy(self): - platform = microvm_tests._snapshot_tier_script("platform") - workload = microvm_tests._snapshot_tier_script("workload-start") - checkpoint = microvm_tests._snapshot_tier_script("instance-checkpoint") - - self.assertIn("/sbin/nvx-snapshot --tier platform", platform) - self.assertIn("date -u -s 200001010000.00", platform) - self.assertNotIn("mkfs.ext4", platform) - self.assertIn("/sbin/nvx-snapshot --tier workload-start", workload) - self.assertIn("mkfs.ext4 -F /dev/vdb", workload) - self.assertIn("runtime-post-restore", workload) - self.assertIn(": >/run/nvx/workload-ran", workload) - self.assertIn("[ -e /run/nvx/workload-ran ]", workload) - self.assertIn("/sbin/nvx-snapshot\n", checkpoint) - self.assertIn("captured-workload-id", checkpoint) - self.assertIn( - "printf 'restored-direct-claimed\\n' " - ">/run/nvx/scratch/direct-claimed\nsync", - checkpoint, - ) - self.assertNotIn("@CAPTURE_ACTION@", checkpoint) - - def test_restored_tier_guests_report_their_restore_checks_before_exiting(self): - # The debug-kernel lane gates on snapshot-tiers, so every restored tier - # guest waits for its deferred restore checks before it exits. - status = microvm_tests.status_script().rstrip("\n") - for tier in ("platform", "workload-start", "instance-checkpoint"): - with self.subTest(tier=tier): - script = microvm_tests._snapshot_tier_script(tier) - self.assertNotIn("@STATUS_QUERY@", script) - self.assertLess( - script.index(f"NVX-TIER-{tier.upper()}-LAYER-"), - script.index(f"set +e\n{status}\n"), - ) - # The guest waits for the host's byte, so the query's lines - # reach the console before nvx-exit stops the VM. - self.assertTrue( - script.endswith( - f"{status}\ndd if=/dev/hvc1 bs=1 count=1 >/dev/null 2>&1\n" - "nvx-exit 0\n" - ) - ) - runner = inspect.getsource(microvm_tests._run_snapshot_tier) - self.assertIn("_check_restore_status(\n restore_console,", runner) - # The status baseline is taken before the milestone reads, so a query - # one of them consumes still counts for the wait. - self.assertLess( - runner.index("status_baseline = console.status_queries"), - runner.index("console.wait_for(released_marker, timeout)"), - ) - self.assertLess( - runner.index("console.wait_for_time_abi_status("), - runner.index('console.send_bytes(b"Z")\n restored = '), - ) - self.assertIn("since=status_baseline", runner) - - def test_snapshot_tier_entry_points_reject_unsupported_tiers(self): - with self.assertRaisesRegex(ValueError, "unsupported snapshot tier 'invalid'"): - microvm_tests._snapshot_tier_script("invalid") - with self.assertRaisesRegex(ValueError, "unsupported snapshot tier 'invalid'"): - microvm_tests._run_snapshot_tier( - "invalid", - Path("openvmm"), - Path("vmlinux"), - Path("initrd"), - "kvm", - memory_mib=128, - timeout=60, - output_dir=Path("logs"), - ) - - def test_snapshot_tier_runner_dispatches_all_tiers(self): - with patch.object(microvm_tests, "_run_snapshot_tier") as run_tier: - microvm_tests.run_snapshot_tiers( - Path("openvmm"), - Path("vmlinux"), - Path("initrd"), - "whp", - memory_mib=128, - timeout=60, - output_dir=Path("logs"), - ) - - self.assertEqual( - [entry.args[0] for entry in run_tier.call_args_list], - ["platform", "workload-start", "instance-checkpoint"], - ) - - def test_snapshot_storage_policy_selects_generation_and_materialization(self): - self.assertEqual(microvm_tests._snapshot_storage_policy("platform"), ()) - workload = microvm_tests._snapshot_storage_policy("workload-start") - checkpoint = microvm_tests._snapshot_storage_policy("instance-checkpoint") - - self.assertEqual( - workload, - ( - "--snapshot-block-identity", - "generation", - "--snapshot-generation-id", - "00112233445566778899aabbccddee01", - "--snapshot-scratch-restore-mode", - "private-copy", - ), - ) - self.assertEqual(checkpoint[-1], "direct-claimed") - with self.assertRaisesRegex(ValueError, "unsupported snapshot tier"): - microvm_tests._snapshot_storage_policy("other") - def test_lifecycle_uses_one_vcpu_linux_guest(self): with ( patch.object( @@ -15872,6 +16505,55 @@ def require(path: Path, _description: str) -> Path: ) gpt_layer.assert_not_called() + def test_runner_dispatches_public_snapshot_scenarios(self): + with tempfile.TemporaryDirectory() as temporary: + output_dir = Path(temporary) + args = nvx.parse_args( + [ + "test-microvm", + "--backend", + "whp", + "--scenario", + "snapshot-core", + "--scenario", + "snapshot-tiers", + "--output-dir", + str(output_dir), + ] + ) + + def require(path: Path, _description: str) -> Path: + return path + + with ( + patch.object(microvm_tests, "validate_openvmm_test_backend"), + patch.object(microvm_tests, "require_file", side_effect=require), + patch.object(microvm_tests, "run_snapshot_core") as core, + patch.object(microvm_tests, "run_snapshot_tiers") as tiers, + patch.object(microvm_tests, "run_vm_lifecycle") as vm_lifecycle, + ): + self.assertEqual(microvm_tests.run(args), 0) + core.assert_called_once_with( + "whp", timeout=args.timeout, output_dir=output_dir + ) + # The guest that writes the GPT disk boots the run's artifacts. + tiers.assert_called_once_with( + microvm_tests.openvmm_binary_path(), + microvm_tests.artifact_path("vmlinux"), + microvm_tests.artifact_path("initramfs.cpio.gz"), + "whp", + timeout=args.timeout, + output_dir=output_dir, + ) + vm_lifecycle.assert_not_called() + # CI runs them in its guest and sandbox jobs, which take the Ubuntu + # artifacts, and not on the debug kernel, which the CLI never boots. + for scenario in ("snapshot-core", "snapshot-tiers"): + self.assertIn(scenario, microvm_tests.PUBLIC_SANDBOX_SCENARIOS) + self.assertIn(scenario, microvm_tests.SANDBOX_CONTROL_SCENARIOS) + self.assertNotIn(scenario, microvm_tests.default_suite()) + self.assertNotIn(scenario, microvm_tests.DEBUG_KERNEL_SCENARIOS) + def test_a_default_run_runs_the_public_sandbox_scenarios(self): def require(path: Path, _description: str) -> Path: return path @@ -16194,7 +16876,6 @@ def require(path: Path, _description: str) -> Path: "smp-snapshot": "run_smp_snapshot", "restore-processors": "run_restore_processors", "restore-downtime": "run_restore_downtime", - "snapshot-tiers": "run_snapshot_tiers", } self.assertEqual(tuple(dispatch), microvm_tests.DEBUG_KERNEL_SCENARIOS) with tempfile.TemporaryDirectory() as temporary: diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py index 6bef5161f..bdf9b2407 100644 --- a/scripts/test_nvx_tools.py +++ b/scripts/test_nvx_tools.py @@ -12814,6 +12814,8 @@ def test_ci_runs_the_public_sandbox_acceptance_on_every_backend(self): self.assertIn("--scenario sandbox-ram-upper", acceptance) self.assertIn("--scenario sandbox-gpt-layer", acceptance) self.assertIn("--scenario vm-lifecycle", acceptance) + self.assertIn("--scenario snapshot-core", acceptance) + self.assertIn("--scenario snapshot-tiers", acceptance) self.assertIn( '--output-dir "build/test-results/public-exec-${{ inputs.backend }}"', acceptance,