test-microvm: have the control console interrupt with input (port of #1410)

doc/ci.md

@@ -480,8 +480,17 @@ the six, the guest reports, for each, the base of its MMIO window and the
 interrupt that its driver requested, as `/sys/kernel/irq` describes it: its
 controller and input, its trigger, its handlers, and how many times it fired.
 It then drives each device, pinging the gateway, reading the share, writing to
-both consoles, reading the `distro` block and writing the `scratch` block past
-the page cache, and reports again. Each device must interrupt on the IOAPIC
+the boot console, attaching to the control console's session broker, reading
+the `distro` block and writing the `scratch` block past the page cache, and
+reports again. A console's driver waits for each write's completion by polling
+the used ring of the console's output, and takes the buffer back as soon as the
+device returns it, so a write interrupts only when the device reads the
+driver's used event before the driver moves it, a race that the driver can win
+at every write (#1400). The guest therefore writes to the boot console until a
+write interrupts, and has the control console interrupt with input instead,
+which nothing waits for: it sends the broker the bootstrap `GuestAttach`
+record, and the device writes the broker's answer, a `Reset` record, into a
+buffer of the console's input. Each device must interrupt on the IOAPIC
 input of its window, 16 through 23, edge-triggered and with no other handler,
 its interrupt must have fired, and `/proc/interrupts` must count no APIC error
 or spurious PIC interrupt (`ERR`), no level-triggered interrupt that the IOAPIC

scripts/nvx_tools/microvm_test_scripts/interrupt-map.sh.in

@@ -85,14 +85,15 @@ report() {
 }
 
 # Makes each virtio device interrupt, one device at a time: the NIC with an
-# echo of its gateway, the host filesystem with a read of its share, the
-# consoles with output, and the block devices with a read of the read-only
-# one and a flushed write to the writable one, past the page cache. Each
-# device's traffic must raise the device's own interrupt before the next
-# device's starts, so two devices whose interrupts were crossed cannot each
-# raise the other's: the consoles' drivers poll the used ring for their
-# output, so a console's writes complete even when its interrupt reaches
-# another handler.
+# echo of its gateway, the host filesystem with a read of its share, the boot
+# console with output, the control console with input that the guest asks its
+# session broker for, and the block devices with a read of the read-only one
+# and a flushed write to the writable one, past the page cache. Each device's
+# traffic must raise the device's own interrupt before the next device's
+# starts, so two devices whose interrupts were crossed cannot each raise the
+# other's: the boot console's driver polls the used ring for its output, so
+# the console's writes complete even when its interrupt reaches another
+# handler.
 traffic() {
     for device in /sys/bus/virtio/devices/virtio*; do
         name=${device##*/}
@@ -104,9 +105,10 @@ traffic() {
             d0000000) ping -c 2 -W 2 10.0.0.1 >/dev/null || fail 30 ;;
             d0001000) cat /mnt/share/seed >/dev/null || fail 31 ;;
             d0002000)
-                # When a console's driver takes its buffer back before the
+                # When the console's driver takes its buffer back before the
                 # device reads the used event, the device rightly skips the
-                # interrupt, so write until one interrupts.
+                # interrupt, so write until one interrupts. In #1400's
+                # measurements, the driver won at most 15% of these races.
                 while [ "$(irq_count "$irq")" -le "$fired" ] && [ "$writes" -lt 16 ]; do
                     echo NVX-INTERRUPT-MAP-CONSOLE
                     writes=$((writes + 1))
@@ -124,15 +126,21 @@ traffic() {
                     fail 34
                 ;;
             d0007000)
-                # As for the boot console, a byte at a time, each in a buffer
-                # of its own: fewer bytes than the 44-byte header of the
-                # control-session protocol, which the console parses its
-                # input as.
-                while [ "$(irq_count "$irq")" -le "$fired" ] && [ "$writes" -lt 16 ]; do
-                    printf . >/dev/hvc2 || fail 35
-                    writes=$((writes + 1))
-                    await_irq "$irq" "$fired" 10
-                done
+                # The console's driver polls the used ring for its output and
+                # takes each buffer back as soon as the device returns it, so
+                # a write interrupts only if the device reads the used event
+                # before the driver moves it past the buffer, a race that the
+                # driver can win at every write (#1400). Nothing waits for
+                # the console's input, whose buffers the driver takes back
+                # from its interrupt, so attach to the control-session broker
+                # that the console runs: the device writes the broker's
+                # answer, a Reset record, into one of them. The terminal stays
+                # open, raw, and silent until the interrupt, so that it sends
+                # the record unchanged and echoes no answer to the broker,
+                # which would stop at the echo as at any malformed record and
+                # never complete the write.
+                { stty raw -echo <&3 && printf '@GUEST_ATTACH@' >&3 &&
+                    await_irq "$irq" "$fired"; } 3<>/dev/hvc2 || fail 35
                 ;;
             *) fail 36 ;;
         esac

scripts/nvx_tools/microvm_tests.py

@@ -275,6 +275,13 @@
 INTERRUPT_MAP_ERROR_COUNTS = ("err", "mis", "spu")
 INTERRUPT_MAP_NETWORK = "10.0.0.2/24"
 INTERRUPT_MAP_SHARE_SEED = "NVX-INTERRUPT-MAP-SHARE"
+# The record with which the interrupt-map scenario's guest attaches to the
+# control console's session broker, so that the broker answers with a Reset
+# record (4.4.3): the bootstrap GuestAttach record, whose 44-byte header holds
+# the magic NVXS, version 1, type 1, and zero flags, instance, epoch,
+# sequence, and payload length, as the GUEST_ATTACH vector of OpenVMM's
+# control-session protocol gives it.
+INTERRUPT_MAP_GUEST_ATTACH = b"NVXS" + bytes((1, 0, 1, 0)) + bytes(36)
 # The devices of the interrupt-map scenario's snapshot, by the bases of their
 # windows: the NIC, the host filesystem with a share, and the boot console.
 INTERRUPT_MAP_SNAPSHOT_DEVICES = (0xD000_0000, 0xD000_1000, 0xD000_2000)
@@ -7122,6 +7129,21 @@ def check_interrupt_map(
         )
 
 
+def _interrupt_map_script(devices: int, *, capture: str) -> str:
+    """Return the interrupt-map scenario's guest script for a guest with
+    ``devices`` virtio devices, which runs ``capture`` between its boot report
+    and its traffic.
+
+    The script writes INTERRUPT_MAP_GUEST_ATTACH to the control console with
+    printf, whose format gives each of its bytes as an octal escape."""
+    return _render_script(
+        "interrupt-map.sh.in",
+        DEVICES=str(devices),
+        CAPTURE=capture,
+        GUEST_ATTACH="".join(f"\\{byte:03o}" for byte in INTERRUPT_MAP_GUEST_ATTACH),
+    )
+
+
 def run_interrupt_map(
     executable: Path,
     kernel: Path,
@@ -7171,11 +7193,7 @@ def run_interrupt_map(
             )
         result = run_guest_script(
             command,
-            _render_script(
-                "interrupt-map.sh.in",
-                DEVICES=str(len(MICROVM_VIRTIO_DEVICES)),
-                CAPTURE="",
-            ),
+            _interrupt_map_script(len(MICROVM_VIRTIO_DEVICES), capture=""),
             INTERRUPT_MAP_COMPLETION_MARKER,
             timeout=timeout,
             log_path=output_dir / "interrupt-map.log",
@@ -7208,10 +7226,9 @@ def run_interrupt_map(
                 process,
                 "/tmp/nvx-interrupt-map",
                 "NVX_INTERRUPT_MAP",
-                _render_script(
-                    "interrupt-map.sh.in",
-                    DEVICES=str(len(INTERRUPT_MAP_SNAPSHOT_DEVICES)),
-                    CAPTURE=f"echo {capture_marker}\nnvx-snapshot\nreport restore",
+                _interrupt_map_script(
+                    len(INTERRUPT_MAP_SNAPSHOT_DEVICES),
+                    capture=f"echo {capture_marker}\nnvx-snapshot\nreport restore",
                 ),
             )
             source = process.wait(timeout)

scripts/test_microvm_tests.py

@@ -12486,6 +12486,48 @@ def wait(self, _timeout: float) -> openvmm_process.OpenvmmProcessResult:
                 output_dir=Path(temporary),
             )
 
+    def test_interrupt_map_attaches_the_control_console_to_its_broker(self):
+        # The control console's driver can take back every buffer of its
+        # output before the device reads its used event, which skips the
+        # interrupt of every write (#1400). The guest therefore makes the
+        # console interrupt with input: the broker's answer to the bootstrap
+        # GuestAttach record, whose bytes are OpenVMM's vector.
+        vectors = (
+            BuildConstants.REPO_ROOT
+            / "nvx-rs"
+            / "tests"
+            / "vectors"
+            / "openvmm"
+            / "control_session_protocol_vectors.txt"
+        )
+        (guest_attach,) = (
+            bytes.fromhex(line.split("|")[1])
+            for line in vectors.read_text(encoding="utf-8").splitlines()
+            if line.startswith("GUEST_ATTACH|")
+        )
+        self.assertEqual(microvm_tests.INTERRUPT_MAP_GUEST_ATTACH, guest_attach)
+
+        for devices in (
+            len(microvm_tests.MICROVM_VIRTIO_DEVICES),
+            len(microvm_tests.INTERRUPT_MAP_SNAPSHOT_DEVICES),
+        ):
+            script = microvm_tests._interrupt_map_script(devices, capture="")
+            self.assertNotIn("@", script)
+            # The control console's step writes the record, once, to the
+            # console held open, raw, and silent until its interrupt.
+            steps = re.findall(
+                r"\{ stty raw -echo <&3 && printf '([^']*)' >&3 &&\n"
+                r' +await_irq "\$irq" "\$fired"; \} 3<>/dev/hvc2 \|\| fail 35\n',
+                script,
+            )
+            self.assertEqual(len(steps), 1)
+            self.assertRegex(steps[0], r"^(\\[0-7]{3})+$")
+            written = bytes(
+                int(escape, 8) for escape in re.findall(r"\\([0-7]{3})", steps[0])
+            )
+            self.assertEqual(written, guest_attach)
+            self.assertEqual(script.count("/dev/hvc2"), 1)
+
     def test_smp_snapshot_runs_the_warp_probe_after_each_restore(self):
         def measure(command: list[str], **kwargs: object) -> None:
             processors = int(command[command.index("--processors") + 1])