From 9b816f7e5235555135b84e888a53221239b5111d Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Wed, 11 Nov 2026 11:25:38 -0800 Subject: [PATCH] cli: start a capture test's fake VMM before its deadline (SL3.7b follow-up) The feature branch's push CI failed on Windows at 66fd43145ce (run 43071545182) in the capture test a_capture_ack_and_successful_output_without_timely_exit_are_stopped_but_never_finished: the fake VMM's ack.json was missing after the capture timed out. Four slice agents saw it fail on their Windows machines too, more often under CPU load; Linux hasn't shown it. The test captures with a 1 s timeout and a fake VMM that acknowledges the capture, writes successful output and never exits, to check that the command stops the VM and never finishes the capture. --timeout bounds launch, capture and exit, as the command's help says, so the deadline also covers the fake's startup, a new process that runs the test binary again. On a loaded Windows host that startup alone can outlast the deadline: the capture then timed out while it connected to the fake's control endpoint, before the fake had acknowledged anything, and the test checked a phase other than the one it means to check. The production code applies the deadline as documented. The test now starts its fake VMM before the capture, as an owned child, as the interrupted-capture tests start theirs. The fake waits for its capability on its standard input, which the capture's spawn step hands over, as spawn_vm does, and then enters its VM directory, so its startup never runs against the deadline. The child's guard ends the fake only after the test has checked that the capture stopped it. The test also checks that the error is the exit wait's own timeout, "capture VM did not exit before its deadline", so that a timeout in another phase fails with its cause, and it gives the two processes' handshake 5 s, which took up to 2.4 s under heavy CPU contention. The other capture tests keep spawning their fake through spawn_vm. Evidence, on a Windows 11 host (Intel i9-12900H, 20 logical processors) at 3340bbd7ee4, with 16 CPU-bound processes pinned to 2 logical processors and each run pinned to the same 2, interleaving four builds for 25 rounds: - the base test, 1 s: 23 of 25 runs failed, with the CI symptom; - the fake started first, 1 s: 2 of 25 failed, connecting; - the base test, 5 s: 4 of 25 failed, with the CI symptom; - the fake started first, 5 s: none of 25 failed. A second interleaving of the base test and this commit failed 6 and 0 of 25 runs. With a fake VMM that takes 6 s to start, the base test with a 5 s timeout failed 6 of 6 runs with the CI symptom, and this change passed 6 of 6. A capture that no longer stops its VM fails the test. On nvx-kvm-test (Linux, KVM), the test passed 20 of 20 runs, and 20 of 20 with 8 busy loops on its 2 processors, as the base test did, and the crate's unit tests fail the same 28 guest tests as at the base. Tests: the test keeps its assertions and also checks which phase timed out. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../src/host/cli/snapshots/capture/tests.rs | 83 +++++++++++++++++-- 1 file changed, 75 insertions(+), 8 deletions(-) diff --git a/nvx-rs/src/host/cli/snapshots/capture/tests.rs b/nvx-rs/src/host/cli/snapshots/capture/tests.rs index dc9b04c87..98848a887 100644 --- a/nvx-rs/src/host/cli/snapshots/capture/tests.rs +++ b/nvx-rs/src/host/cli/snapshots/capture/tests.rs @@ -10,7 +10,7 @@ use serde_json::{Value, json}; use super::*; use crate::host::platform; -use crate::host::process::{Opened, ProcessRef}; +use crate::host::process::{BootId, Opened, ProcessRef}; use crate::host::snapshot::tests::{ManifestSpec, SnapshotFixture, class}; use crate::host::snapshot::{MANIFEST_NAME, Snapshot}; use crate::host::spawn::{self, VmmCommand}; @@ -21,6 +21,9 @@ use crate::protocol::{ const HELPER: &str = "host::cli::snapshots::capture::tests::capture_helper"; const COMMAND_HELPER: &str = "host::cli::snapshots::capture::tests::command_helper"; +/// Names the VM directory that a fake VMM started before its capture enters once its capability +/// arrives. +const STARTED_VM_DIR: &str = "NVX_CAPTURE_STARTED_VM_DIR"; type Inventory = Vec<(PathBuf, (u64, u128), u64, std::time::SystemTime)>; @@ -47,6 +50,43 @@ impl Fixture { }) } + /// Captures with a fake VMM that starts before the capture and waits for the capability that + /// its launch hands over, so that the capture's timeout, which bounds launch too, never runs + /// while the VMM's process starts. The returned guard ends the fake only once dropped, so that + /// the caller can check first that the capture stopped it. + fn capture_started(&self, name: &str, behavior: &str, timeout: u32) -> (Result, OwnedCommand) { + let mut command = Command::new(std::env::current_exe().unwrap()); + command + .args(["--exact", HELPER, "--ignored", "--nocapture"]) + .env(STARTED_VM_DIR, self.vm_dir()) + .current_dir(self.root.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + let mut vmm = OwnedCommand(crate::host::tests::spawn_locked(&mut command).unwrap()); + let reference = ProcessRef { + pid: vmm.0.id(), + start_time: platform::process_start_time(vmm.0.id()).unwrap().unwrap(), + boot: BootId::current().unwrap(), + }; + record(self.root.path(), &reference); + let Opened::Running(process) = Process::open(&reference).unwrap() else { + panic!("the fake VMM exited before its capture"); + }; + wait_file(&self.root.path().join("started")); + let mut stdin = vmm.0.stdin.take().unwrap(); + let mut options = self.options(name); + options.timeout = timeout; + let result = capture(&options, &runtime(self.root.path()), |prepared| { + write_recipe(self.root.path(), &prepared, behavior); + // As `spawn_vm` does, this hands the VMM its capability on its standard input. + stdin.write_all(prepared.capability().as_bytes()).unwrap(); + drop(stdin); + Ok(process) + }); + (result, vmm) + } + fn store(&self) -> Store { Store::create(&self.root.path().join("store")).unwrap() } @@ -158,7 +198,8 @@ fn runtime(root: &Path) -> Runtime { } } -fn fake_spawn(root: &Path, prepared: Prepared, behavior: &str) -> Process { +/// Checks the durable barrier that a spawn follows, and writes the fake VMM's recipe. +fn write_recipe(root: &Path, prepared: &Prepared, behavior: &str) { let dir = prepared.outputs().log.parent().unwrap(); let journal: Value = serde_json::from_slice(&fs::read(dir.join("launch.json")).unwrap()).unwrap(); @@ -212,12 +253,16 @@ fn fake_spawn(root: &Path, prepared: Prepared, behavior: &str) -> Process { serde_json::to_vec(&recipe).unwrap(), ) .unwrap(); +} + +fn fake_spawn(root: &Path, prepared: Prepared, behavior: &str) -> Process { + write_recipe(root, &prepared, behavior); let command = VmmCommand::new( std::env::current_exe().unwrap(), ["--exact", HELPER, "--ignored", "--nocapture"] .map(Into::into) .to_vec(), - dir, + prepared.outputs().log.parent().unwrap(), ); let child = spawn::spawn_vm( &command, @@ -226,17 +271,22 @@ fn fake_spawn(root: &Path, prepared: Prepared, behavior: &str) -> Process { ) .unwrap(); let process = child.process().unwrap(); + record(root, process.reference()); + child.detach(); + drop(prepared); + process +} + +/// Records an owned fake VMM, which the fixture ends if a test leaves it running. +fn record(root: &Path, reference: &ProcessRef) { let path = root.join("processes.json"); let mut references: Vec = if path.exists() { serde_json::from_slice(&fs::read(&path).unwrap()).unwrap() } else { Vec::new() }; - references.push(process.reference().clone()); + references.push(reference.clone()); fs::write(path, serde_json::to_vec(&references).unwrap()).unwrap(); - child.detach(); - drop(prepared); - process } fn wait_file(path: &Path) { @@ -354,8 +404,16 @@ fn duplicate_names_and_a_failed_new_capture_never_change_an_existing_snapshot() #[test] fn a_capture_ack_and_successful_output_without_timely_exit_are_stopped_but_never_finished() { let fixture = Fixture::new(); - let error = fixture.capture("late", "hang", 1).unwrap_err(); + // The timeout bounds launch too: the fake VMM starts before it, and the timeout leaves the + // handshake of two processes room on a loaded host, so that only the exit wait times out. + let (result, _vmm) = fixture.capture_started("late", "hang", 5); + let error = result.unwrap_err(); assert_eq!(error.code(), Some(ErrorCode::TimedOut), "{error:?}"); + assert_eq!( + error.message(), + Some("capture VM did not exit before its deadline"), + "only the wait for the acknowledged capture's exit may time out" + ); assert!(fixture.root.path().join("ack.json").exists()); assert!(fixture.vm_dir().join(OPENVMM_DIR).exists()); assert!(fixture.vm_dir().join("outcome.json").exists()); @@ -648,8 +706,17 @@ fn respond(stream: &mut (impl Read + Write), capability: &[u8], refused: bool) { #[test] #[ignore = "authenticated fake VMM subprocess; not real-VM capture proof"] fn capture_helper() { + // Started before its capture, the fake reports that it runs, and enters its VM directory, + // as a spawned VMM starts in it, once its capability arrives. + let started = std::env::var_os(STARTED_VM_DIR); + if started.is_some() { + fs::write("started", b"").unwrap(); + } let mut capability = Vec::new(); std::io::stdin().read_to_end(&mut capability).unwrap(); + if let Some(dir) = started { + std::env::set_current_dir(dir).unwrap(); + } assert_eq!(capability.len(), CAPABILITY_LEN); assert_eq!(fs::read("control.capability").unwrap(), capability); let directory = std::env::current_dir().unwrap();