From f3f2cef27ba0e801315309adc78de27e5aee5fc1 Mon Sep 17 00:00:00 2001 From: Pedro Henrique Penna Date: Sun, 8 Nov 2026 18:58:54 -0800 Subject: [PATCH] test-microvm: have the control console interrupt with input The interrupt-map scenario's control console sometimes never raised its interrupt: the guest wrote 16 one-byte buffers to /dev/hvc2, IRQ 23 never fired, and OpenVMM exited with status 37 after NVX-INTERRUPT-MAP-QUIET. It failed in about one run in ten on GitHub-hosted runners, on KVM and WHP and on AMD and Intel CPUs, and in 3 merge groups on NVX CI's Intel MSHV runners. No interrupt was lost: the device never sent one. The console's driver waits for each write by polling the used ring of the console's output, takes the buffer back as soon as the device returns it, and moves its used event past the buffer. With VIRTIO_F_EVENT_IDX, the device notifies the driver of a buffer only if the used event still names the buffer when the device reads the event after publishing it, as virtio 1.2 (2.7.10) requires, so a write interrupts only when the device reads the event first. Runs of an instrumented OpenVMM recorded the event that the device read at each completion: at every skipped notification of either console, the driver had already moved the event to the next buffer. How often the driver wins this race varies by boot. In 320 boots on AMD runners under KVM, the control console's first write interrupted in 205, while 22 boots used up the scenario's 16 writes (5 of 140 on EPYC 9V74 runners, 17 of 60 on EPYC 9V45, and none of 120 on EPYC 7763), 2 of them with none of 43 writes interrupting. The boot console's driver won at most 15% of its races, so its 16 writes still suffice. So the control console now interrupts with input, which nothing waits for: the guest sends the console's control-session broker the bootstrap GuestAttach record, and the device writes the broker's answer, a Reset record, into a buffer of the console's input, which the driver takes back from its interrupt. The guest holds the console open, raw, and silent until the interrupt, so that the terminal sends the record unchanged and echoes no answer back, at which the broker would stop as at any malformed record, never completing the echo's write. The scenario's check is unchanged: IRQ 23 must fire with the console's own traffic. On GitHub-hosted runners, with the release of 7714de163 and each job alternating the two scenarios, the previous one failed in 64 of 640 runs (31 of 320 on KVM, 33 of 320 on WHP) and this one in 1 of 640, a WHP run on an Intel Xeon 8573C runner that left no evidence of its cause. With an instrumented OpenVMM, this one passed 960 runs on KVM and 960 on WHP. A unit test checks that the script writes OpenVMM's GUEST_ATTACH vector to /dev/hvc2, once, held open, raw, and silent; it fails with the previous script. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> (cherry picked from commit 64975689e1444d6f2912661a017c559fa425c7e6) --- doc/ci.md | 13 +++++- .../microvm_test_scripts/interrupt-map.sh.in | 46 +++++++++++-------- scripts/nvx_tools/microvm_tests.py | 35 ++++++++++---- scripts/test_microvm_tests.py | 42 +++++++++++++++++ 4 files changed, 106 insertions(+), 30 deletions(-) diff --git a/doc/ci.md b/doc/ci.md index c382c3093..e044ba784 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -480,8 +480,17 @@ the six, the guest reports, for each, the base of its MMIO window and the interrupt that its driver requested, as `/sys/kernel/irq` describes it: its controller and input, its trigger, its handlers, and how many times it fired. It then drives each device, pinging the gateway, reading the share, writing to -both consoles, reading the `distro` block and writing the `scratch` block past -the page cache, and reports again. Each device must interrupt on the IOAPIC +the boot console, attaching to the control console's session broker, reading +the `distro` block and writing the `scratch` block past the page cache, and +reports again. A console's driver waits for each write's completion by polling +the used ring of the console's output, and takes the buffer back as soon as the +device returns it, so a write interrupts only when the device reads the +driver's used event before the driver moves it, a race that the driver can win +at every write (#1400). The guest therefore writes to the boot console until a +write interrupts, and has the control console interrupt with input instead, +which nothing waits for: it sends the broker the bootstrap `GuestAttach` +record, and the device writes the broker's answer, a `Reset` record, into a +buffer of the console's input. Each device must interrupt on the IOAPIC input of its window, 16 through 23, edge-triggered and with no other handler, its interrupt must have fired, and `/proc/interrupts` must count no APIC error or spurious PIC interrupt (`ERR`), no level-triggered interrupt that the IOAPIC diff --git a/scripts/nvx_tools/microvm_test_scripts/interrupt-map.sh.in b/scripts/nvx_tools/microvm_test_scripts/interrupt-map.sh.in index 25c79e466..49561a0a3 100644 --- a/scripts/nvx_tools/microvm_test_scripts/interrupt-map.sh.in +++ b/scripts/nvx_tools/microvm_test_scripts/interrupt-map.sh.in @@ -85,14 +85,15 @@ report() { } # Makes each virtio device interrupt, one device at a time: the NIC with an -# echo of its gateway, the host filesystem with a read of its share, the -# consoles with output, and the block devices with a read of the read-only -# one and a flushed write to the writable one, past the page cache. Each -# device's traffic must raise the device's own interrupt before the next -# device's starts, so two devices whose interrupts were crossed cannot each -# raise the other's: the consoles' drivers poll the used ring for their -# output, so a console's writes complete even when its interrupt reaches -# another handler. +# echo of its gateway, the host filesystem with a read of its share, the boot +# console with output, the control console with input that the guest asks its +# session broker for, and the block devices with a read of the read-only one +# and a flushed write to the writable one, past the page cache. Each device's +# traffic must raise the device's own interrupt before the next device's +# starts, so two devices whose interrupts were crossed cannot each raise the +# other's: the boot console's driver polls the used ring for its output, so +# the console's writes complete even when its interrupt reaches another +# handler. traffic() { for device in /sys/bus/virtio/devices/virtio*; do name=${device##*/} @@ -104,9 +105,10 @@ traffic() { d0000000) ping -c 2 -W 2 10.0.0.1 >/dev/null || fail 30 ;; d0001000) cat /mnt/share/seed >/dev/null || fail 31 ;; d0002000) - # When a console's driver takes its buffer back before the + # When the console's driver takes its buffer back before the # device reads the used event, the device rightly skips the - # interrupt, so write until one interrupts. + # interrupt, so write until one interrupts. In #1400's + # measurements, the driver won at most 15% of these races. while [ "$(irq_count "$irq")" -le "$fired" ] && [ "$writes" -lt 16 ]; do echo NVX-INTERRUPT-MAP-CONSOLE writes=$((writes + 1)) @@ -124,15 +126,21 @@ traffic() { fail 34 ;; d0007000) - # As for the boot console, a byte at a time, each in a buffer - # of its own: fewer bytes than the 44-byte header of the - # control-session protocol, which the console parses its - # input as. - while [ "$(irq_count "$irq")" -le "$fired" ] && [ "$writes" -lt 16 ]; do - printf . >/dev/hvc2 || fail 35 - writes=$((writes + 1)) - await_irq "$irq" "$fired" 10 - done + # The console's driver polls the used ring for its output and + # takes each buffer back as soon as the device returns it, so + # a write interrupts only if the device reads the used event + # before the driver moves it past the buffer, a race that the + # driver can win at every write (#1400). Nothing waits for + # the console's input, whose buffers the driver takes back + # from its interrupt, so attach to the control-session broker + # that the console runs: the device writes the broker's + # answer, a Reset record, into one of them. The terminal stays + # open, raw, and silent until the interrupt, so that it sends + # the record unchanged and echoes no answer to the broker, + # which would stop at the echo as at any malformed record and + # never complete the write. + { stty raw -echo <&3 && printf '@GUEST_ATTACH@' >&3 && + await_irq "$irq" "$fired"; } 3<>/dev/hvc2 || fail 35 ;; *) fail 36 ;; esac diff --git a/scripts/nvx_tools/microvm_tests.py b/scripts/nvx_tools/microvm_tests.py index caaac5933..b07015b4b 100644 --- a/scripts/nvx_tools/microvm_tests.py +++ b/scripts/nvx_tools/microvm_tests.py @@ -275,6 +275,13 @@ INTERRUPT_MAP_ERROR_COUNTS = ("err", "mis", "spu") INTERRUPT_MAP_NETWORK = "10.0.0.2/24" INTERRUPT_MAP_SHARE_SEED = "NVX-INTERRUPT-MAP-SHARE" +# The record with which the interrupt-map scenario's guest attaches to the +# control console's session broker, so that the broker answers with a Reset +# record (4.4.3): the bootstrap GuestAttach record, whose 44-byte header holds +# the magic NVXS, version 1, type 1, and zero flags, instance, epoch, +# sequence, and payload length, as the GUEST_ATTACH vector of OpenVMM's +# control-session protocol gives it. +INTERRUPT_MAP_GUEST_ATTACH = b"NVXS" + bytes((1, 0, 1, 0)) + bytes(36) # The devices of the interrupt-map scenario's snapshot, by the bases of their # windows: the NIC, the host filesystem with a share, and the boot console. INTERRUPT_MAP_SNAPSHOT_DEVICES = (0xD000_0000, 0xD000_1000, 0xD000_2000) @@ -7122,6 +7129,21 @@ def check_interrupt_map( ) +def _interrupt_map_script(devices: int, *, capture: str) -> str: + """Return the interrupt-map scenario's guest script for a guest with + ``devices`` virtio devices, which runs ``capture`` between its boot report + and its traffic. + + The script writes INTERRUPT_MAP_GUEST_ATTACH to the control console with + printf, whose format gives each of its bytes as an octal escape.""" + return _render_script( + "interrupt-map.sh.in", + DEVICES=str(devices), + CAPTURE=capture, + GUEST_ATTACH="".join(f"\\{byte:03o}" for byte in INTERRUPT_MAP_GUEST_ATTACH), + ) + + def run_interrupt_map( executable: Path, kernel: Path, @@ -7171,11 +7193,7 @@ def run_interrupt_map( ) result = run_guest_script( command, - _render_script( - "interrupt-map.sh.in", - DEVICES=str(len(MICROVM_VIRTIO_DEVICES)), - CAPTURE="", - ), + _interrupt_map_script(len(MICROVM_VIRTIO_DEVICES), capture=""), INTERRUPT_MAP_COMPLETION_MARKER, timeout=timeout, log_path=output_dir / "interrupt-map.log", @@ -7208,10 +7226,9 @@ def run_interrupt_map( process, "/tmp/nvx-interrupt-map", "NVX_INTERRUPT_MAP", - _render_script( - "interrupt-map.sh.in", - DEVICES=str(len(INTERRUPT_MAP_SNAPSHOT_DEVICES)), - CAPTURE=f"echo {capture_marker}\nnvx-snapshot\nreport restore", + _interrupt_map_script( + len(INTERRUPT_MAP_SNAPSHOT_DEVICES), + capture=f"echo {capture_marker}\nnvx-snapshot\nreport restore", ), ) source = process.wait(timeout) diff --git a/scripts/test_microvm_tests.py b/scripts/test_microvm_tests.py index 6ad80c10f..ee395bba2 100644 --- a/scripts/test_microvm_tests.py +++ b/scripts/test_microvm_tests.py @@ -12486,6 +12486,48 @@ def wait(self, _timeout: float) -> openvmm_process.OpenvmmProcessResult: output_dir=Path(temporary), ) + def test_interrupt_map_attaches_the_control_console_to_its_broker(self): + # The control console's driver can take back every buffer of its + # output before the device reads its used event, which skips the + # interrupt of every write (#1400). The guest therefore makes the + # console interrupt with input: the broker's answer to the bootstrap + # GuestAttach record, whose bytes are OpenVMM's vector. + vectors = ( + BuildConstants.REPO_ROOT + / "nvx-rs" + / "tests" + / "vectors" + / "openvmm" + / "control_session_protocol_vectors.txt" + ) + (guest_attach,) = ( + bytes.fromhex(line.split("|")[1]) + for line in vectors.read_text(encoding="utf-8").splitlines() + if line.startswith("GUEST_ATTACH|") + ) + self.assertEqual(microvm_tests.INTERRUPT_MAP_GUEST_ATTACH, guest_attach) + + for devices in ( + len(microvm_tests.MICROVM_VIRTIO_DEVICES), + len(microvm_tests.INTERRUPT_MAP_SNAPSHOT_DEVICES), + ): + script = microvm_tests._interrupt_map_script(devices, capture="") + self.assertNotIn("@", script) + # The control console's step writes the record, once, to the + # console held open, raw, and silent until its interrupt. + steps = re.findall( + r"\{ stty raw -echo <&3 && printf '([^']*)' >&3 &&\n" + r' +await_irq "\$irq" "\$fired"; \} 3<>/dev/hvc2 \|\| fail 35\n', + script, + ) + self.assertEqual(len(steps), 1) + self.assertRegex(steps[0], r"^(\\[0-7]{3})+$") + written = bytes( + int(escape, 8) for escape in re.findall(r"\\([0-7]{3})", steps[0]) + ) + self.assertEqual(written, guest_attach) + self.assertEqual(script.count("/dev/hvc2"), 1) + def test_smp_snapshot_runs_the_warp_probe_after_each_restore(self): def measure(command: list[str], **kwargs: object) -> None: processors = int(command[command.index("--processors") + 1])