diff --git a/mk/tests.mk b/mk/tests.mk index 724bcf03..eb8a24df 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -55,7 +55,7 @@ ELFUSE_HOST_NOFILE_MIN ?= $(shell bash "$(CURDIR)/tests/test-config.sh" --host-n test-sysroot-pathmax test-sysroot-corpus \ test-sysroot-name-soak check-soak \ check-name-caseexact test-sysroot-path-matrix \ - test-usage-synopsis \ + test-usage-synopsis test-qemu-runner \ probe-volume-naming perf ## Build and run the assembly hello world test @@ -344,6 +344,7 @@ check: $(ELFUSE_BIN) $(TEST_DEPS) check-syscall-coverage check-eintr-contract ch $(call run-lane,test-rosetta-cli,rosetta CLI gating) $(call run-lane,test-bench-guardrail,hot-syscall guardrail) $(call run-lane,test-sharun,sharun launcher and probe) + $(call run-lane,test-qemu-runner,qemu-runner start and stop checks) ## Hot-syscall performance guardrail: ensure getpid, libc clock_gettime, ## and 1-byte /dev/urandom reads stay under their TODO ns/op ceilings. @@ -1170,6 +1171,10 @@ test-usage-synopsis: $(ELFUSE_BIN) test-gdbstub-host: $(BUILD_DIR)/test-gdbstub-host $(BUILD_DIR)/test-gdbstub-host +## Check qemu-runner.sh start reporting and stop identity, against stand-ins +test-qemu-runner: + @bash tests/test-qemu-runner.sh + ## Run GDB stub integration tests (LLDB <-> elfuse gdbstub) test-gdbstub: $(ELFUSE_BIN) $(TEST_DIR)/test-hello $(BUILD_DIR)/test-gdbstub-host $(call run-host-unit,test-gdbstub-host,buffered GDB session regression) diff --git a/tests/lib/qemu-ssh.sh b/tests/lib/qemu-ssh.sh new file mode 100644 index 00000000..9ea7a40b --- /dev/null +++ b/tests/lib/qemu-ssh.sh @@ -0,0 +1,24 @@ +# Shared ssh argv for the qemu test VM. +# +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 +# shellcheck shell=bash +# timeout(1) cannot wrap a shell function, so what the callers share is the +# argv: qemu_ssh_opts fills QEMU_SSH_OPTS from QEMU_SSH_KEY and QEMU_PORT at +# call time, and each caller builds its own ssh command line around it. + +# shellcheck disable=SC2034 # Consumed by the sourcing script. +qemu_ssh_opts() +{ + QEMU_SSH_OPTS=( + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o LogLevel=ERROR + -o BatchMode=yes + -o ConnectTimeout=10 + -o ServerAliveInterval=10 + -o ServerAliveCountMax=6 + -i "$QEMU_SSH_KEY" + -p "$QEMU_PORT" + ) +} diff --git a/tests/qemu-runner.sh b/tests/qemu-runner.sh index e5837fe4..c35a95d5 100755 --- a/tests/qemu-runner.sh +++ b/tests/qemu-runner.sh @@ -19,6 +19,9 @@ _QR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]:-$0}")/.." && pwd)" _QR_FIX="${_QR_DIR}/externals/test-fixtures" +# shellcheck source=tests/lib/qemu-ssh.sh +source "${_QR_DIR}/tests/lib/qemu-ssh.sh" + QEMU_BIN="${QEMU_BIN:-qemu-system-aarch64}" QEMU_PORT="${QEMU_PORT:-2222}" QEMU_MEM="${QEMU_MEM:-2048}" @@ -38,6 +41,7 @@ QEMU_SHARE_PATH="${QEMU_SHARE_PATH:-${_QR_DIR}}" _QR_PIDFILE="" _QR_LOG="" +_QR_ERR="" _QR_CTL="" # Fixture path inside the VM (always /mnt/host/). @@ -98,6 +102,21 @@ qemu_pick_cpu() esac } +# qemu_stop removes the rundir, so what explains a failed start has to be +# printed before it: the harness sees only this output. +qemu_fail_start() +{ + echo "qemu-runner: $1" >&2 + local f + for f in "$_QR_ERR" "$_QR_LOG"; do + if [ -s "$f" ]; then + echo "qemu-runner: tail of ${f##*/}:" >&2 + tail -n 20 "$f" >&2 + fi + done + qemu_stop +} + qemu_start() { qemu_ensure_fixtures || return 1 @@ -119,6 +138,7 @@ qemu_start() rundir="$(mktemp -d -t elfuse-qemu.XXXXXX)" _QR_PIDFILE="${rundir}/qemu.pid" _QR_LOG="${rundir}/qemu-serial.log" + _QR_ERR="${rundir}/qemu.log" _QR_CTL="${rundir}/ssh-ctl" QEMU_PORT="$(qemu_pick_port)" @@ -135,7 +155,7 @@ qemu_start() -nographic -display none -no-reboot -monitor none \ -serial "file:${_QR_LOG}" \ -pidfile "$_QR_PIDFILE" \ - > /dev/null 2>&1 & + > "$_QR_ERR" 2>&1 & disown # Wait for ssh port to come up. @@ -147,8 +167,7 @@ qemu_start() sleep 1 done if ! (echo > /dev/tcp/127.0.0.1/"$QEMU_PORT") 2> /dev/null; then - echo "qemu-runner: VM did not boot within ${QEMU_BOOT_TIMEOUT}s" >&2 - qemu_stop + qemu_fail_start "VM did not boot within ${QEMU_BOOT_TIMEOUT}s" return 1 fi @@ -165,7 +184,10 @@ qemu_start() # a dedicated tmpfs, as any regular system has, so paths under /tmp map to a # resolvable st_dev. Guarded so a repeated qemu_start against a running VM # does not stack mounts. - _qemu_ssh_raw 'grep -q " /tmp tmpfs " /proc/mounts || mount -t tmpfs tmpfs /tmp' + if ! _qemu_ssh_raw 'grep -q " /tmp tmpfs " /proc/mounts || mount -t tmpfs tmpfs /tmp'; then + qemu_fail_start "could not prepare /tmp in the guest" + return 1 + fi } # Each call opens a fresh ssh connection. Avoids ControlMaster pitfalls (master @@ -174,16 +196,8 @@ qemu_start() # the suite's tolerance. _qemu_ssh_raw() { - ssh -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o BatchMode=yes \ - -o ConnectTimeout=10 \ - -o ServerAliveInterval=10 \ - -o ServerAliveCountMax=6 \ - -i "$QEMU_SSH_KEY" \ - -p "$QEMU_PORT" \ - root@127.0.0.1 "$@" + qemu_ssh_opts + ssh "${QEMU_SSH_OPTS[@]}" root@127.0.0.1 "$@" } # Run a command in the VM. Any argument that is an absolute path under the host @@ -210,6 +224,14 @@ qemu_stop() if [ -n "$_QR_PIDFILE" ] && [ -s "$_QR_PIDFILE" ]; then local pid pid=$(cat "$_QR_PIDFILE" 2> /dev/null) + + # A state file outlives its VM, so the pid it names may since have been + # recycled. qemu_start gives qemu this pidfile, and mktemp makes the + # path unique, so the argv is what proves the process is ours. + case " $(ps -o command= -p "${pid:-0}" 2> /dev/null) " in + *" -pidfile $_QR_PIDFILE "*) ;; + *) pid="" ;; + esac if [ -n "$pid" ] && kill -0 "$pid" 2> /dev/null; then kill "$pid" 2> /dev/null # give qemu time to exit cleanly; force-kill if it lingers @@ -218,7 +240,15 @@ qemu_stop() kill -0 "$pid" 2> /dev/null || break sleep 1 done - kill -0 "$pid" 2> /dev/null && kill -9 "$pid" 2> /dev/null + if kill -0 "$pid" 2> /dev/null; then + kill -9 "$pid" 2> /dev/null + sleep 1 + # Keep the pidfile and state so a later stop can retry. + if kill -0 "$pid" 2> /dev/null; then + echo "qemu-runner: pid $pid survived SIGKILL" >&2 + return 1 + fi + fi fi fi if [ -n "$_QR_PIDFILE" ]; then @@ -226,33 +256,70 @@ qemu_stop() fi _QR_PIDFILE="" _QR_LOG="" + _QR_ERR="" _QR_CTL="" } -# When sourced, register a cleanup trap that does not clobber the caller's -# existing trap chain. When executed directly, the EXIT trap fires on script -# exit. -trap 'qemu_stop' EXIT +qemu_write_state() +{ + mkdir -p "$(dirname "$1")" || return 1 + printf 'port=%s\nkey=%s\npidfile=%s\n' \ + "$QEMU_PORT" "$QEMU_SSH_KEY" "$_QR_PIDFILE" > "$1.tmp" && mv -f "$1.tmp" "$1" +} + +qemu_read_state() +{ + [ -s "$1" ] || { + echo "qemu-runner: no state file $1" >&2 + return 1 + } + _QR_PIDFILE="$(sed -n 's/^pidfile=//p' "$1")" + + # Restrict cleanup to the directory shape created by mktemp. + case "$_QR_PIDFILE" in + */elfuse-qemu.*/qemu.pid) ;; + *) + echo "qemu-runner: $1 names no qemu-runner pidfile: $_QR_PIDFILE; remove the file once the VM is gone" >&2 + return 1 + ;; + esac +} -# CLI driver: when run directly, support 'qemu-runner.sh start|exec|stop'. if [ "${BASH_SOURCE[0]:-$0}" = "$0" ]; then cmd="${1:-help}" shift || true + state_file="" + if [ "$cmd" != exec ] && [ "${1:-}" = "--state-file" ]; then + state_file="${2:?--state-file needs a path}" + shift 2 + fi case "$cmd" in start) - qemu_start + if [ -n "$state_file" ] && [ -e "$state_file" ]; then + echo "qemu-runner: $state_file names a live VM; run stop first" >&2 + exit 1 + fi + trap 'qemu_stop' EXIT + qemu_start || exit 1 + if [ -n "$state_file" ]; then + qemu_write_state "$state_file" || exit 1 + trap - EXIT + fi echo "PORT=$QEMU_PORT KEY=$QEMU_SSH_KEY" ;; exec) + trap 'qemu_stop' EXIT qemu_start qemu_exec "$@" ;; stop) - qemu_stop + [ -z "$state_file" ] || qemu_read_state "$state_file" || exit 1 + qemu_stop || exit 1 + [ -z "$state_file" ] || rm -f "$state_file" ;; *) cat << EOF -Usage: $0 +Usage: $0 Boots qemu-system-aarch64 with the test fixtures and exposes ssh. The host repo is shared into the VM at /mnt/host (read-only). diff --git a/tests/test-matrix.sh b/tests/test-matrix.sh index 6ddea218..3903904d 100755 --- a/tests/test-matrix.sh +++ b/tests/test-matrix.sh @@ -73,6 +73,8 @@ source "${REPO_ROOT}/tests/test-config.sh" TEST_LABEL_WIDTH=45 # shellcheck source=tests/lib/test-runner.sh source "${REPO_ROOT}/tests/lib/test-runner.sh" +# shellcheck source=tests/lib/qemu-ssh.sh +source "${REPO_ROOT}/tests/lib/qemu-ssh.sh" # Globals (test-runner.sh seeds pass/fail/skip; test-matrix.sh resets them per # mode and tracks no extra counters). @@ -192,15 +194,8 @@ run_qemu() if [ "${#args[@]}" -gt 0 ]; then printf -v quoted '%q ' "${args[@]}" fi - timeout 60 ssh \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -o LogLevel=ERROR \ - -o BatchMode=yes \ - -o ConnectTimeout=10 \ - -o ServerAliveInterval=15 \ - -o ServerAliveCountMax=4 \ - -i "$QEMU_SSH_KEY" -p "$QEMU_PORT" \ + qemu_ssh_opts + timeout 60 ssh "${QEMU_SSH_OPTS[@]}" \ root@127.0.0.1 "cd /mnt/host && ${quoted}" 2> /dev/null } @@ -1278,11 +1273,12 @@ run_suite() # shellcheck disable=SC1091 . "${REPO_ROOT}/tests/qemu-runner.sh" printf "Booting qemu-system-aarch64 (Alpine minirootfs)\n" + _qemu_active=1 qemu_start || { + _qemu_active=0 echo "qemu boot failed" return 1 } - _qemu_active=1 runner="run_qemu" dyn_runner="run_qemu" ;; diff --git a/tests/test-qemu-runner.sh b/tests/test-qemu-runner.sh new file mode 100755 index 00000000..562934ea --- /dev/null +++ b/tests/test-qemu-runner.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash + +# test-qemu-runner.sh -- Pin qemu-runner.sh start and stop against stand-ins +# +# Copyright 2026 elfuse contributors +# SPDX-License-Identifier: Apache-2.0 +# +# Usage: tests/test-qemu-runner.sh +# +# No VM boots here: a stub stands in for qemu. stop must leave a recycled pid +# alone yet terminate a process whose argv names the run's pidfile, and a failed +# start must report what the VM said before its run directory goes. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +RUNNER="$SCRIPT_DIR/qemu-runner.sh" +# shellcheck source=tests/lib/report.sh +. "$SCRIPT_DIR/lib/report.sh" + +work="$(mktemp -d)" +victims=() +cleanup() +{ + local p + for p in ${victims[@]+"${victims[@]}"}; do + kill "$p" 2> /dev/null || true + done + rm -rf "$work" +} +trap cleanup EXIT + +# qemu_read_state accepts only the directory shape mktemp gives a run. +rundir="$work/elfuse-qemu.test" +pidfile="$rundir/qemu.pid" +state="$work/qemu.state" + +fake_start() +{ + mkdir -p "$rundir" + printf '%s\n' "$1" > "$pidfile" + printf 'port=1\nkey=/dev/null\npidfile=%s\n' "$pidfile" > "$state" +} + +check() +{ + local label="$1" want="$2" got="$3" + if [ "$want" = "$got" ]; then + report_pass "$label" + else + report_fail "$label (got $got, want $want)" + fi +} + +alive() +{ + kill -0 "$1" 2> /dev/null && echo alive || echo dead +} + +present() +{ + [ -e "$1" ] && echo present || echo gone +} + +has() +{ + case "$1" in + *"$2"*) echo yes ;; + *) echo no ;; + esac +} + +# A recycled pid: a sleep whose argv never mentions the pidfile. +sleep 300 & +bystander=$! +victims+=("$bystander") +fake_start "$bystander" +rc=0 +bash "$RUNNER" stop --state-file "$state" > /dev/null 2>&1 || rc=$? +check "stop returns 0 for a recycled pid" 0 "$rc" +check "a recycled pid survives stop" alive "$(alive "$bystander")" +check "stop removes the stale state file" gone "$(present "$state")" +check "stop removes the stale run directory" gone "$(present "$rundir")" + +# The run's own process: its argv carries the pidfile, and it exits on TERM. +loop='trap "exit 0" TERM; while :; do sleep 1; done' +bash -c "$loop" bash -pidfile "$pidfile" & +own=$! +victims+=("$own") +fake_start "$own" +rc=0 +bash "$RUNNER" stop --state-file "$state" > /dev/null 2>&1 || rc=$? +check "stop returns 0 for the run's own process" 0 "$rc" +check "the run's own process is terminated" dead "$(alive "$own")" +check "stop removes the state file after a kill" gone "$(present "$state")" + +# A failed start: a stub qemu that never opens the port. +stub="$work/qemu-stub" +cat > "$stub" << 'STUB' +#!/usr/bin/env bash +serial="" +while [ $# -gt 0 ]; do + [ "$1" = -serial ] && serial="${2#file:}" + [ "$1" = -pidfile ] && echo "$2" > "$(dirname "$0")/pidfile-arg" + shift +done +[ -z "$serial" ] || echo SERIAL-MARKER > "$serial" +echo STDERR-MARKER >&2 +STUB +chmod +x "$stub" +fixture="$work/fixture" +echo fixture > "$fixture" +rm -f "$state" +rc=0 +out="$(QEMU_BIN="$stub" QEMU_ACCEL=tcg QEMU_BOOT_TIMEOUT=1 \ + QEMU_KERNEL="$fixture" QEMU_INITRD="$fixture" QEMU_SSH_KEY="$fixture" \ + bash "$RUNNER" start --state-file "$state" 2>&1)" || rc=$? +check "start fails when the VM never boots" 1 "$rc" +check "the failure names the timeout" yes "$(has "$out" "did not boot")" +check "the serial console reaches the caller" yes "$(has "$out" SERIAL-MARKER)" +check "qemu's own output reaches the caller" yes "$(has "$out" STDERR-MARKER)" +check "a failed start writes no state file" gone "$(present "$state")" +failed_rundir="$(dirname "$(cat "$work/pidfile-arg")")" +check "a failed start removes its run directory" gone "$(present "$failed_rundir")" + +report_summary +[ "$fail" -eq 0 ]