diff --git a/doc/usage.md b/doc/usage.md index 12eb06dd9..84074e9ba 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -1322,8 +1322,10 @@ Runs the consolidated suite of the [runtime adapter](../tests/runtime-adapter/RE the [stage scenarios](../tests/runtime-adapter/README.md#stage-scenarios) that `tests/runtime-adapter/scenarios.txt` names, against a real hypervisor. Each line of that file names one of the adapter's test targets, then one of its -tests. The command first checks that the lines name exactly the tests of their -targets, and that neither the adapter's library nor another target has an +tests, optionally followed by `linux` or `windows` to select that host platform. +Unqualified entries run on both. The command first checks that the selected +lines name exactly the tests of their targets, and that neither the adapter's +library nor another target has an ignored test, as the adapter's VM scenarios are, so that a scenario that is renamed, added, or gone fails it rather than drop out of the suite. It then runs each target's scenarios by @@ -1333,8 +1335,10 @@ directory that `NVX_ARTIFACTS_DIR` names, from the current directory if it is relative, or this checkout unless it names one, and the adapter's guest agent from `--guest-agent` (default `tests/runtime-adapter/target/x86_64-unknown-linux-musl/release/runtime-adapter-guest`), -and fails unless each of them ran and passed. `--check-only` checks the names -alone, and needs neither a hypervisor nor the guest agent. +and fails unless each of them ran and passed. Execution and counts use the same +platform-selected entries as the inventory check. `--check-only` compiles and +lists the tests but runs no scenarios, and needs neither a hypervisor nor the +guest agent. ### `test-adversarial` diff --git a/scripts/nvx_tools/runtime_adapter_tests.py b/scripts/nvx_tools/runtime_adapter_tests.py index f8295a674..03ed56608 100644 --- a/scripts/nvx_tools/runtime_adapter_tests.py +++ b/scripts/nvx_tools/runtime_adapter_tests.py @@ -1,9 +1,10 @@ """The runtime adapter's consolidated suite, run by name on a real hypervisor. `tests/runtime-adapter/scenarios.txt` names the suite's scenarios, each a test of one of -the adapter's test targets. The command checks that the scenarios name exactly the tests -of their targets, and that neither the library nor another target has an ignored test, -as the scenarios that boot VMs are, so that a scenario that is renamed, added, or gone +the adapter's test targets, optionally limited to Linux or Windows. The command checks +that the host's scenarios name exactly the tests of their targets, and that neither the +library nor another target has an ignored test, as the scenarios that boot VMs are, so +that a scenario that is renamed, added, or gone fails the command rather than drop out of it. It then runs each target's scenarios by name, the ignored ones with the others, and checks that each of them ran and passed. """ @@ -77,25 +78,32 @@ def configure_parser(parser: argparse.ArgumentParser) -> None: def read_scenarios(path: Path = SCENARIOS) -> list[Target]: - """Returns the targets that the scenarios of `path` name, in the order of their - first lines, each with its tests in the order of their lines. + """Returns the host's scenarios of `path`, grouped by target in the order of their + first selected lines, each with its tests in the order of their lines. - A line names a target, then one of its tests; `#` starts a comment. Raises - `ScriptError` for another line, and a scenario that two lines name. + A line names a target, then one of its tests, optionally followed by `linux` or + `windows`; `#` starts a comment. Raises `ScriptError` for another line, an unknown + platform, and a scenario that two lines name, even on different platforms. """ + host_platform = "windows" if sys.platform == "win32" else sys.platform tests: dict[str, list[str]] = {} + seen: set[tuple[str, str]] = set() for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): fields = line.split("#", 1)[0].split() if not fields: continue where = f"{path}:{number}" - if len(fields) != 2: + if len(fields) not in (2, 3): raise ScriptError(f"{where}: expected a target and a test, not {line!r}") - target, test = fields - named = tests.setdefault(target, []) - if test in named: + target, test = fields[:2] + platform = fields[2] if len(fields) == 3 else None + if platform not in (None, "linux", "windows"): + raise ScriptError(f"{where}: unknown scenario platform {platform!r}") + if (target, test) in seen: raise ScriptError(f"{where}: {target} {test} is named twice") - named.append(test) + seen.add((target, test)) + if platform is None or platform == host_platform: + tests.setdefault(target, []).append(test) if not tests: raise ScriptError(f"{path} names no scenario") return [Target(name, tuple(named)) for name, named in tests.items()] diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py index c984b57b1..db8e5dfc2 100644 --- a/scripts/test_nvx_tools.py +++ b/scripts/test_nvx_tools.py @@ -21038,30 +21038,86 @@ def test_scenarios_name_targets_and_tests_in_the_order_of_their_lines(self): def test_scenarios_refuse_other_lines(self): for text, problem in ( ("vm_a\n", r":1: expected a target and a test, not 'vm_a'"), - ("unit x\nvm_a a b\n", r":2: expected a target and a test"), + ("unit x\nvm_a a linux extra\n", r":2: expected a target and a test"), + ("unit x\nvm_a a b\n", r":2: unknown scenario platform 'b'"), + ("vm_a a Linux\n", r":1: unknown scenario platform 'Linux'"), ("vm_a a\nvm_a a\n", r":2: vm_a a is named twice"), + ("vm_a a linux\nvm_a a windows\n", r":2: vm_a a is named twice"), + ("vm_a a linux\nvm_a a\n", r":2: vm_a a is named twice"), + ("vm_a a windows\nvm_a a windows\n", r":2: vm_a a is named twice"), ("# Nothing yet.\n", r"names no scenario"), + ): + for platform in ("linux", "win32"): + with ( + self.subTest(text=text, platform=platform), + patch.object(sys, "platform", platform), + self.assertRaisesRegex(common.ScriptError, problem), + ): + runtime_adapter_tests.read_scenarios(self._scenarios(text)) + + def test_scenarios_select_the_host_platform_in_line_order(self): + path = self._scenarios( + "vm_a a windows\nunit x\nvm_a b linux # Linux only\nvm_a common\n" + ) + for platform, expected in ( + ( + "linux", + [ + runtime_adapter_tests.Target("unit", ("x",)), + runtime_adapter_tests.Target("vm_a", ("b", "common")), + ], + ), + ( + "win32", + [ + runtime_adapter_tests.Target("vm_a", ("a", "common")), + runtime_adapter_tests.Target("unit", ("x",)), + ], + ), ): with ( - self.subTest(text=text), - self.assertRaisesRegex(common.ScriptError, problem), + self.subTest(platform=platform), + patch.object(sys, "platform", platform), ): - runtime_adapter_tests.read_scenarios(self._scenarios(text)) + self.assertEqual(runtime_adapter_tests.read_scenarios(path), expected) + + with ( + patch.object(sys, "platform", "win32"), + self.assertRaisesRegex(common.ScriptError, "names no scenario"), + ): + runtime_adapter_tests.read_scenarios(self._scenarios("vm_a a linux\n")) def test_the_adapter_s_readme_lists_its_scenarios(self): - targets = runtime_adapter_tests.read_scenarios() - named = [(target.name, test) for target in targets for test in target.tests] - self.assertLessEqual( - {"vm_clone", "vm_template", "vm_layered", "template_operations"}, - {target.name for target in targets}, - ) readme = (runtime_adapter_tests.ADAPTER_DIRECTORY / "README.md").read_text( encoding="utf-8" ) section = re.split(r"\n#+ ", readme.split("\n### Stage scenarios\n", 1)[1])[0] - rows = re.findall(r"^\| `(\w+)` \| `(\w+)` \|", section, re.MULTILINE) - - self.assertEqual(rows, named) + rows = re.findall( + r"^\| `(\w+)` \| `([^`]+)`(?: \((linux|windows)\))? \|", + section, + re.MULTILINE, + ) + for platform, host in (("linux", "linux"), ("win32", "windows")): + with ( + self.subTest(platform=platform), + patch.object(sys, "platform", platform), + ): + targets = runtime_adapter_tests.read_scenarios() + named = [ + (target.name, test) for target in targets for test in target.tests + ] + self.assertLessEqual( + {"vm_clone", "vm_template", "vm_layered", "template_operations"}, + {target.name for target in targets}, + ) + self.assertEqual( + [ + (target, test) + for target, test, on in rows + if not on or on == host + ], + named, + ) def test_libtest_s_terse_list_names_the_tests(self): self.assertEqual( @@ -21189,6 +21245,68 @@ def test_the_handler_takes_a_relative_directory_of_artifacts_from_the_caller(sel self.assertEqual(status, 0, stderr) self.assertEqual(Path(runs[0][1]["NVX_ARTIFACTS_DIR"]), expected) + def test_the_handler_checks_and_runs_the_same_platform_scenarios(self): + path = self._scenarios( + "vm_a common\nvm_a linux_case linux\nunit windows_case windows\n" + ) + for platform, selected in ( + ("linux", [("vm_a", ("common", "linux_case"))]), + ("win32", [("vm_a", ("common",)), ("unit", ("windows_case",))]), + ): + listings = { + (target, "--list"): "".join(f"{test}: test\n" for test in tests) + for target, tests in selected + } + listings.update( + ((target, "--ignored"), "") for target in ("lib", "unit", "other") + ) + for check_only in (False, True): + with ( + self.subTest(platform=platform, check_only=check_only), + patch.object(sys, "platform", platform), + ): + status, runs, stdout, stderr = self._run( + path, listings, check_only=check_only + ) + self.assertEqual(status, 0, stderr) + expected = [ + runtime_adapter_tests.cargo_test( + "cargo", target, "--include-ignored", "--exact", *tests + ) + for target, tests in selected + ] + self.assertEqual( + [command for command, _ in runs], [] if check_only else expected + ) + if check_only: + self.assertIn( + f"2 scenarios name exactly the tests of {len(selected)} targets", + stdout, + ) + else: + self.assertIn( + f"the 2 scenarios of {len(selected)} targets passed", stdout + ) + + def test_platform_selection_keeps_unlisted_and_ignored_test_errors(self): + path = self._scenarios( + "vm_a common\nvm_a linux_case linux\nother hidden linux\n" + ) + listings = { + ("vm_a", "--list"): "common: test\nlinux_case: test\n", + ("lib", "--ignored"): "", + ("unit", "--ignored"): "", + ("other", "--ignored"): "hidden: test\n", + } + with patch.object(sys, "platform", "win32"): + status, runs, _, stderr = self._run(path, listings) + + self.assertEqual((status, runs), (1, [])) + self.assertIn("vm_a has the test linux_case, which no scenario names", stderr) + self.assertIn( + "other has the ignored test hidden, which no scenario names", stderr + ) + def test_the_handler_runs_nothing_once_the_names_are_not_the_tests(self): path = self._scenarios("vm_a a\nvm_a gone\ngone x\n") listings = { diff --git a/tests/runtime-adapter/README.md b/tests/runtime-adapter/README.md index a2b7e8c65..42eca7760 100644 --- a/tests/runtime-adapter/README.md +++ b/tests/runtime-adapter/README.md @@ -3050,8 +3050,11 @@ the template, whose VM might still run. The template and clone stage's scenarios run as one suite, which [`scenarios.txt`](scenarios.txt) names: each line names a test target of -`tests/`, then one of its tests. CI's runtime adapter step runs the suite on -each backend, KVM, MSHV, and WHP, with +`tests/`, then one of its tests, optionally followed by `linux` or `windows`. +Unqualified entries run on both platforms; qualified entries run only on the +named host platform. Duplicate target/test pairs, even with different +qualifiers, and unknown qualifiers are errors. CI's runtime adapter step runs +the suite on each backend, KVM, MSHV, and WHP, with `python3 scripts/nvx.py test-runtime-adapter --backend BACKEND`, in the host-only configuration: it runs each target's scenarios by name, the VM scenarios, which are ignored, with the others, and checks that each of them ran @@ -3059,7 +3062,9 @@ and passed. It fails once a line names a test that its target does not have, a target that a line names has a test that no line names, the library or another target has an ignored test, or a named test does not run and pass, so that a scenario that is renamed, added, or gone fails it rather than drop out of the -suite. With `--check-only`, it checks the names alone, and runs nothing. +suite. Inventory checks, execution and scenario counts all use the same +platform-selected entries. With `--check-only`, it compiles and lists the +tests to check the names, but runs no scenarios and needs no guest artifacts. The suite holds each operation's outcomes, from `create` to `run`, without a hypervisor and in VMs, and every scenario of the stage's VMs: launches of @@ -3222,6 +3227,7 @@ layered roots, captures of templates, and the starts and runs of their clones. | `vm_clone` | `one_generation_serves_clones_of_either_format_over_either_upper` | A template of a GPT placeholder serves EROFS and GPT clones of other layouts, shorter than the placeholder, at its length, over either upper, those in RAM with no scratch file, and a first partition over its bound ends a clone with status 54 | | `vm_clone` | `one_template_serves_the_clones_of_every_network_policy_at_its_guest_s_address` | One template with a NIC serves the clones of every network policy at its guest's address, whose restores install their own policies on its NIC, which lets through what each allows, publishes the guest's listeners, through the clone's announcer, and forwards ports where it says, and NVX's plan refuses another address, or no NIC, with status 8 before any VMM starts | | `vm_clone` | `openvmm_reads_none_of_the_host_s_input_until_the_repair_acknowledges_the_restore` | OpenVMM reads none of the host's attachment until the repair of a clone, whose template delays it, acknowledges the restore: the broker answers it only after the delay, and on Linux, where the host writes it a byte at a time, `SIOCOUTQ` rises with each byte and stays as the write left it until then, while a reattach's is answered at once | +| `vm_clone` | `restore_exclusion::hard_linked_restore_views_obey_the_callers_exclusion` (linux) | Caller-locked concurrent hard-linked restores succeed, an unlocked link change during startup fails the restore, a controller killed before ready prevents completion, and link changes after the peer-verified ready connection do not fail it | | `vm_clone` | `the_lease_of_a_start_that_ended_before_its_record_is_released_through_its_template_store` | The lease of a start that ended before it recorded its clone is released only through its own template store, which `--template` names | ### Running the VM tests diff --git a/tests/runtime-adapter/scenarios.txt b/tests/runtime-adapter/scenarios.txt index 476988d64..5588b216c 100644 --- a/tests/runtime-adapter/scenarios.txt +++ b/tests/runtime-adapter/scenarios.txt @@ -1,7 +1,8 @@ # The runtime adapter's consolidated suite: every scenario of the template and clone stage, which # CI's runtime adapter step runs by name on each backend, with # `python3 scripts/nvx.py test-runtime-adapter`. Each line names a test target of `tests/`, then -# one of its tests, which the step runs whether it is ignored or not. The step fails once a line +# one of its tests, optionally followed by linux or windows to select only that host platform. +# Unqualified lines run on both platforms, whether ignored or not. The step fails once a line # names a test that its target does not have, a target that a line names has a test that no line # names, the library or a target that no line names has an ignored test, or a named test does not # run and pass. The README's "Stage scenarios" says what each scenario proves. @@ -171,4 +172,5 @@ vm_clone one_generation_serves_clones_of_each_upper_and_memory_target vm_clone one_generation_serves_clones_of_either_format_over_either_upper vm_clone one_template_serves_the_clones_of_every_network_policy_at_its_guest_s_address vm_clone openvmm_reads_none_of_the_host_s_input_until_the_repair_acknowledges_the_restore +vm_clone restore_exclusion::hard_linked_restore_views_obey_the_callers_exclusion linux vm_clone the_lease_of_a_start_that_ended_before_its_record_is_released_through_its_template_store diff --git a/tests/runtime-adapter/tests/vm_clone.rs b/tests/runtime-adapter/tests/vm_clone.rs index 46f447124..c7e6ab682 100644 --- a/tests/runtime-adapter/tests/vm_clone.rs +++ b/tests/runtime-adapter/tests/vm_clone.rs @@ -5370,3 +5370,513 @@ fn a_run_whose_clone_s_vm_exits_during_its_execution_ends_with_the_clone_s_failu another.stop(); assert_payload(&template, false); } + +#[cfg(target_os = "linux")] +mod restore_exclusion { + use super::*; + use std::io::Write as _; + use std::os::unix::fs::{MetadataExt as _, OpenOptionsExt as _}; + use std::os::unix::net::{UnixListener, UnixStream}; + use std::process::{Child, Command, ExitStatus, Stdio}; + + use nix::fcntl::OFlag; + use nix::sys::socket::{getsockopt, sockopt::PeerCredentials}; + use nix::sys::stat::Mode; + use nix::unistd::{mkfifo, pipe2}; + use nvx::activation::{ + ExpectedResources, Image as ActivationImage, OverlayMode, RestoreSection, TransactionId, + Upper as ActivationUpper, View, + }; + + const TEST: &str = "restore_exclusion::hard_linked_restore_views_obey_the_callers_exclusion"; + const ROLE: &str = "RUNTIME_ADAPTER_RESTORE_ROLE"; + const DIRECTORY: &str = "RUNTIME_ADAPTER_RESTORE_DIRECTORY"; + const CAPABILITY: [u8; CAPABILITY_LEN] = [0x51; CAPABILITY_LEN]; + const TIMEOUT: Duration = Duration::from_secs(60); + type Case = (&'static str, fn(&Template, &Path)); + + fn lock(path: &Path) -> File { + let file = File::options() + .create(true) + .truncate(false) + .read(true) + .write(true) + .open(path) + .unwrap(); + file.lock().unwrap(); + file + } + + struct OwnedChild(Child); + + impl OwnedChild { + fn wait(&mut self) -> ExitStatus { + let deadline = Instant::now() + TIMEOUT; + loop { + if let Some(status) = self.0.try_wait().unwrap() { + eprintln!("SL2.15 reaped pid={} status={status}", self.0.id()); + return status; + } + assert!( + Instant::now() < deadline, + "pid {} did not exit", + self.0.id() + ); + thread::sleep(Duration::from_millis(10)); + } + } + } + + impl Drop for OwnedChild { + fn drop(&mut self) { + if self.0.try_wait().unwrap().is_none() { + self.0.kill().unwrap(); + self.wait(); + } + } + } + + struct RestoreView { + sandbox: Sandbox, + lock_path: PathBuf, + exclusion: Option, + listener: Option, + prepare_gate: Option, + controller: Option, + vmm: Option, + start_time: u64, + } + + impl RestoreView { + fn start(template: &Template, lock_path: &Path, controller: bool) -> Self { + let exclusion = lock(lock_path); + let sandbox = Sandbox::new("restore-view"); + SandboxStore::create(&sandbox.store).unwrap(); + let mut view = Self { + sandbox, + lock_path: lock_path.to_owned(), + exclusion: Some(exclusion), + listener: None, + prepare_gate: None, + controller: None, + vmm: None, + start_time: 0, + }; + fs::create_dir(view.snapshot()).unwrap(); + let source = template.payload().join(SNAPSHOT_NAME); + for entry in fs::read_dir(source).unwrap() { + let entry = entry.unwrap(); + assert!(entry.file_type().unwrap().is_file()); + let target = view.snapshot().join(entry.file_name()); + fs::hard_link(entry.path(), &target).unwrap(); + let (original, linked) = ( + fs::metadata(entry.path()).unwrap(), + fs::metadata(target).unwrap(), + ); + assert_eq!( + (original.dev(), original.ino()), + (linked.dev(), linked.ino()) + ); + } + if controller { + view.controller = Some(view.helper("controller")); + view.until(|view| view.sandbox.store.join("controller-bound").exists()); + } else { + let listener = UnixListener::bind(view.ready_path()).unwrap(); + listener.set_nonblocking(true).unwrap(); + view.listener = Some(listener); + } + let artifacts = env::var_os(Artifacts::ENV_DIR).unwrap(); + let runtime = Runtime::from_dir(artifacts, GUEST_BUILD).unwrap(); + let store = template.store(); + let key = store.read_key(&template.generation()).unwrap(); + let log = File::create(view.log_path()).unwrap(); + let gate = view.sandbox.store.join("prepare.fifo"); + mkfifo(&gate, Mode::S_IRUSR | Mode::S_IWUSR).unwrap(); + let (input, output) = pipe2(OFlag::O_CLOEXEC).unwrap(); + let mut output = File::from(output); + output.write_all(&CAPABILITY).unwrap(); + drop(output); + let mut command = Command::new(runtime.artifacts.openvmm); + command + .args( + in_ram(template) + .plan() + .unwrap() + .openvmm_arguments() + .unwrap(), + ) + .args(["--single-process", "--restore-absent-scratch"]) + .args(["--hypervisor", key.hypervisor().as_str()]) + .args(["--cpu-profile", &key.cpu_profile()]) + .args(["--processors", &key.processors().to_string()]) + .arg("--restore-snapshot") + .arg(view.snapshot()); + if key.boot_processors() != 0 { + command.args(["--restore-processors", &key.boot_processors().to_string()]); + } + let child = command + .args(["--restore-memory", &format!("{}M", key.memory() / MIB)]) + .arg("--restore-ready-path") + .arg(view.ready_path()) + .args(["--virtio-console", "none", "--microvm-control-console"]) + .arg(format!("listen={}", view.endpoint().display())) + .arg("--microvm-control-auth-stdin") + .arg("--microvm-report") + .arg(view.sandbox.store.join("outcome.json")) + .arg("--log-file") + .arg(gate) + .current_dir(&view.sandbox.store) + .env("OPENVMM_STARTUP_PROFILE", "1") + .stdin(Stdio::from(input)) + .stdout(log.try_clone().unwrap()) + .stderr(log) + .spawn() + .unwrap(); + view.vmm = Some(OwnedChild(child)); + let pid = view.vmm.as_ref().unwrap().0.id(); + view.start_time = nvx::host::platform::process_start_time(pid) + .unwrap() + .expect("the owned VMM runs"); + eprintln!( + "SL2.15 spawned pid={pid} start={} vm_dir={}", + view.start_time, + view.sandbox.store.display() + ); + // After retaining memory's ctime, OpenVMM opens --log-file before prepare/ready. + // The FIFO blocks that open until proceed supplies a reader. --single-process + // discards this log handle, so no worker can block on an undrained log. + view.until(|view| { + view.log() + .contains("OPENVMM_SNAPSHOT_PROFILE_V1 operation=restore phase=artifact_open ") + }); + assert!(!view.log().contains("phase=artifact_prepare ")); + view + } + + fn snapshot(&self) -> PathBuf { + self.sandbox.store.join("openvmm") + } + + fn ready_path(&self) -> PathBuf { + self.sandbox.store.join("ready.sock") + } + + fn endpoint(&self) -> PathBuf { + self.sandbox.store.join("control.sock") + } + + fn log_path(&self) -> PathBuf { + self.sandbox.store.join("vmm.log") + } + + fn log(&self) -> String { + fs::read_to_string(self.log_path()).unwrap() + } + + fn until(&mut self, condition: impl Fn(&Self) -> bool) { + let deadline = Instant::now() + TIMEOUT; + while !condition(self) { + if let Some(vmm) = &mut self.vmm { + assert!( + vmm.0.try_wait().unwrap().is_none(), + "VMM exited before the required boundary:\n{}", + self.log() + ); + } + assert!(Instant::now() < deadline, "boundary timed out"); + thread::sleep(Duration::from_millis(10)); + } + } + + fn helper(&self, role: &str) -> OwnedChild { + let log = File::create(self.sandbox.store.join(format!("{role}.log"))).unwrap(); + let child = Command::new(env::current_exe().unwrap()) + .args(["--exact", TEST, "--ignored", "--nocapture"]) + .env(ROLE, role) + .env(DIRECTORY, &self.sandbox.store) + .stdin(Stdio::piped()) + .stdout(log.try_clone().unwrap()) + .stderr(log) + .spawn() + .unwrap(); + let start = nvx::host::platform::process_start_time(child.id()) + .unwrap() + .expect("the unreaped helper has its original process identity"); + eprintln!("SL2.15 helper role={role} pid={} start={start}", child.id()); + OwnedChild(child) + } + + fn change_link_without_lock(&self) { + let memory = self.snapshot().join("memory.bin"); + let before = fs::metadata(&memory).unwrap(); + let mut helper = self.helper("link"); + drop(helper.0.stdin.take()); + assert!(helper.wait().success()); + let after = fs::metadata(memory).unwrap(); + assert_eq!(after.nlink(), before.nlink() + 1); + assert_eq!( + (after.len(), after.mtime(), after.mtime_nsec()), + (before.len(), before.mtime(), before.mtime_nsec()) + ); + assert_ne!( + (after.ctime(), after.ctime_nsec()), + (before.ctime(), before.ctime_nsec()) + ); + } + + fn proceed(&mut self) { + self.prepare_gate = Some( + File::options() + .read(true) + .custom_flags(OFlag::O_NONBLOCK.bits()) + .open(self.sandbox.store.join("prepare.fifo")) + .unwrap(), + ); + } + + fn connected(&mut self) -> UnixStream { + let deadline = Instant::now() + TIMEOUT; + let stream = loop { + match self.listener.as_ref().unwrap().accept() { + Ok((stream, _)) => break stream, + Err(error) if error.kind() == io::ErrorKind::WouldBlock => {} + Err(error) => panic!("restore-ready accept: {error}"), + } + assert!( + self.vmm.as_mut().unwrap().0.try_wait().unwrap().is_none(), + "VMM exited before connecting restore-ready:\n{}", + self.log() + ); + assert!(Instant::now() < deadline, "restore-ready connect timed out"); + thread::sleep(Duration::from_millis(10)); + }; + // This unreaped Child's PID cannot be reused while the peer is checked. + let peer = getsockopt(&stream, PeerCredentials).unwrap(); + assert_eq!(peer.pid() as u32, self.vmm.as_ref().unwrap().0.id()); + assert!(self.vmm.as_mut().unwrap().0.try_wait().unwrap().is_none()); + stream.set_read_timeout(Some(TIMEOUT)).unwrap(); + stream + } + + fn ready(&mut self, mut stream: UnixStream) { + let mut event = Vec::new(); + stream.read_to_end(&mut event).unwrap(); + nvx::protocol::restore_ready::parse_ready_event(&event).unwrap(); + drop(self.exclusion.take()); + } + + fn finish(&mut self, template: &Template) { + let deadline = Instant::now() + TIMEOUT; + let child = &self.vmm.as_ref().unwrap().0; + let transport = nvx::host::session::connect( + self.endpoint().to_str().unwrap(), + child.id(), + self.start_time, + deadline, + ) + .unwrap(); + let mut client = Client::attach(transport, &CAPABILITY, deadline).unwrap(); + let record = ActivationRecord::new( + Some(RestoreSection { + transaction: TransactionId::from_bytes([0x15; 16]), + }), + Configuration { + image: ActivationImage::Erofs { + uuid: Some(Uuid::parse(&template.identity).unwrap()), + }, + upper: ActivationUpper::Tmpfs { + max_memory_percent: 50, + }, + overlay: OverlayMode::TrustedLowers, + export: None, + mappings: 0, + network: None, + view: View { + hostname: CLONE_HOSTNAME.as_bytes().to_vec(), + machine_id: MachineIdMode::Image, + }, + resources: ExpectedResources { + cpus: 1, + memory: TEMPLATE_MEMORY * MIB, + }, + }, + ) + .unwrap(); + let ready = client.activate(&record.encode(), deadline).unwrap(); + assert_eq!(ready.build(), GUEST_BUILD); + assert_eq!(ready.activation(), Some(ReadyActivation::of(&record))); + client.start(deadline).unwrap(); + client.ping(deadline).unwrap(); + assert_eq!(client.stop(deadline).unwrap(), None); + assert!( + self.vmm.as_mut().unwrap().wait().success(), + "{}", + self.log() + ); + } + } + + impl Drop for RestoreView { + fn drop(&mut self) { + drop(self.controller.take()); + drop(self.vmm.take()); + drop(self.prepare_gate.take()); + if self.log_path().is_file() { + eprintln!( + "SL2.15 native-log {}:\n{}", + self.log_path().display(), + self.log() + ); + } + let _exclusion = self + .exclusion + .take() + .unwrap_or_else(|| lock(&self.lock_path)); + let extra = self.sandbox.store.join("extra-memory-link"); + if extra.exists() { + fs::remove_file(extra).unwrap(); + } + fs::remove_dir_all(self.snapshot()).unwrap(); + } + } + + #[test] + #[ignore = "boots VMs; see the module documentation for what they need"] + fn hard_linked_restore_views_obey_the_callers_exclusion() { + if let Some(role) = env::var_os(ROLE) { + let directory = PathBuf::from(env::var_os(DIRECTORY).unwrap()); + match role.to_str().unwrap() { + "controller" => { + let _listener = UnixListener::bind(directory.join("ready.sock")).unwrap(); + fs::write(directory.join("controller-bound"), b"bound").unwrap(); + io::stdin().read_exact(&mut [0]).unwrap(); + } + "link" => { + assert_eq!(io::stdin().read(&mut [0]).unwrap(), 0); + fs::hard_link( + directory.join("openvmm/memory.bin"), + directory.join("extra-memory-link"), + ) + .unwrap(); + } + role => panic!("unknown restore helper role {role}"), + } + return; + } + + let template = Template::create("restore-exclusion"); + let snapshot = template.payload().join(SNAPSHOT_NAME); + let before = files_below(&snapshot); + let lock_path = template.root().join("snapshot.lock"); + let cases: [Case; 4] = [ + ("locked-concurrent-restores", |template, lock_path| { + let mut restores = thread::scope(|scope| { + let workers = (0..3) + .map(|_| { + scope.spawn(|| { + let mut restore = RestoreView::start(template, lock_path, false); + restore.proceed(); + let stream = restore.connected(); + restore.ready(stream); + restore + }) + }) + .collect::>(); + workers + .into_iter() + .map(|worker| worker.join().unwrap()) + .collect::>() + }); + for restore in &mut restores { + assert!( + restore + .vmm + .as_mut() + .unwrap() + .0 + .try_wait() + .unwrap() + .is_none() + ); + } + let mut first = restores.remove(0); + first.finish(template); + drop(first); + let mut replacement = RestoreView::start(template, lock_path, false); + replacement.proceed(); + let stream = replacement.connected(); + replacement.ready(stream); + restores.push(replacement); + for restore in &mut restores { + restore.finish(template); + } + }), + ("unlocked-link-during-start", |template, lock_path| { + let mut restore = RestoreView::start(template, lock_path, false); + restore.change_link_without_lock(); + restore.proceed(); + assert!(!restore.vmm.as_mut().unwrap().wait().success()); + assert!( + restore + .log() + .contains("snapshot memory generation changed after it was opened"), + "{}", + restore.log() + ); + assert_eq!( + restore + .listener + .as_ref() + .unwrap() + .accept() + .unwrap_err() + .kind(), + io::ErrorKind::WouldBlock + ); + }), + ("dead-controller-before-connect", |template, lock_path| { + use std::os::unix::process::ExitStatusExt as _; + + let mut restore = RestoreView::start(template, lock_path, true); + let controller = restore.controller.as_mut().unwrap(); + controller.0.kill().unwrap(); + assert_eq!(controller.wait().signal(), Some(9)); + restore.proceed(); + assert!(!restore.vmm.as_mut().unwrap().wait().success()); + assert!( + restore + .log() + .contains("failed to connect restore readiness endpoint"), + "{}", + restore.log() + ); + }), + ("link-after-ready-connect", |template, lock_path| { + let mut restore = RestoreView::start(template, lock_path, false); + restore.proceed(); + let stream = restore.connected(); + restore.change_link_without_lock(); + restore.ready(stream); + restore.finish(template); + }), + ]; + let mut failures = Vec::new(); + for (name, case) in cases { + let result = std::panic::catch_unwind(|| case(&template, &lock_path)); + eprintln!( + "SL2.15 case={name} result={}", + if result.is_ok() { "PASS" } else { "FAIL" } + ); + if result.is_err() { + failures.push(name); + } + } + assert_eq!(files_below(&snapshot), before); + assert_payload(&template, false); + assert!( + failures.is_empty(), + "blocking restore gate failures: {failures:?}" + ); + } +}