Skip to content

Stacked runners: Forward host vsock connections to guest Unix sockets #1754

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Sangho Lee (sangho2) wants to merge 1 commit into from
Closed

Stacked runners: Forward host vsock connections to guest Unix sockets #1754

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -157,16 +157,26 @@ jobs:
run: |
QEMU_ACCEL=kvm CARGO_BUILD="$PWD/.github/tools/github_actions_run_cargo build" \
./dev_tools/run_optee_on_vm_userland.sh -u -t concurrent-sessions -t ta-death -t hello-ta
- name: Enable vhost-vsock
# For the vsock test of run_linux_on_vm_userland.sh, which
# REQUIRE_VSOCK makes fail rather than skip without it.
run: |
sudo modprobe vhost_vsock
echo 'KERNEL=="vhost-vsock", MODE="0666", OPTIONS+="static_node=vhost-vsock"' | sudo tee /etc/udev/rules.d/99-vhost-vsock.rules
sudo udevadm control --reload-rules
sudo udevadm trigger --name-match=vhost-vsock
sudo udevadm settle
test -r /dev/vhost-vsock -a -w /dev/vhost-vsock
- name: Run static and dynamic Linux programs in a QEMU VM with stacked runners (KVM)
# Also builds the kernel and userland runners.
run: |
QEMU_ACCEL=kvm CARGO_BUILD="$PWD/.github/tools/github_actions_run_cargo build" \
REQUIRE_VSOCK=1 QEMU_ACCEL=kvm CARGO_BUILD="$PWD/.github/tools/github_actions_run_cargo build" \
./dev_tools/run_linux_on_vm_userland.sh
- name: Run Linux programs in a QEMU VM with stacked runners on 2 CPUs (KVM)
# Processes run on every CPU; the suite also checks that the other CPU
# ran some.
run: |
QEMU_SMP=2 QEMU_ACCEL=kvm CARGO_BUILD="$PWD/.github/tools/github_actions_run_cargo build" \
REQUIRE_VSOCK=1 QEMU_SMP=2 QEMU_ACCEL=kvm CARGO_BUILD="$PWD/.github/tools/github_actions_run_cargo build" \
./dev_tools/run_linux_on_vm_userland.sh
- name: Run OP-TEE TAs in a QEMU VM with stacked runners on 2 CPUs (KVM)
run: |
Expand Down
96 changes: 79 additions & 17 deletions dev_tools/qemu_test_lib.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,21 @@ default_tas() {
}

WORK=$(mktemp -d)
trap 'rm -rf "$WORK"' EXIT
# run_vm's background jobs still running (`timeout`, which passes the signal
# on to QEMU, and the host client): stopped and reaped on exit.
background=()
cleanup() {
local pid
set +e
for pid in "${background[@]}"; do
# A client's processes, then the subshell running it.
pkill -P "$pid" 2>/dev/null
kill "$pid" 2>/dev/null
done
[[ ${#background[@]} -eq 0 ]] || wait "${background[@]}" 2>/dev/null
rm -rf "$WORK"
}
trap cleanup EXIT

pass=0
fail=0
Expand All @@ -63,37 +77,85 @@ skipped=0

# run_vm NAME KERNEL PAYLOAD [QEMU_ARG]...: boots KERNEL with PAYLOAD as its
# initrd and records the outcome; with $verbose set, prints passing logs too.
# If $check names a function, a guest pass also needs `$check NAME` to
# succeed; it prints why not otherwise.
# If $client names a function, `$client NAME` runs alongside as the host
# client, and a pass needs it to exit 0: a failing client stops the VM, and
# one still running 5s after the VM ends is stopped. If $check names a
# function, a guest pass also needs `$check NAME` to succeed; it prints why
# not otherwise.
run_vm() {
local name=$1 kernel=$2 payload=$3 log="$WORK/$1.log" status why
local name=$1 kernel=$2 payload=$3 log="$WORK/$1.log" status why qemu cpid
local client_status=0 stopped=0
shift 3
set +e
# shellcheck disable=SC2086 # QEMU is a command line
timeout "$TIMEOUT" $QEMU \
-machine q35 -accel "$QEMU_ACCEL" -cpu "$CPU" \
-m "$QEMU_MEM" -smp "$QEMU_SMP" -nic none \
-kernel "$kernel" -initrd "$payload" -append "litebox.log=${LITEBOX_LOG:-info}" \
-serial stdio -display none -no-reboot \
-device isa-debug-exit,iobase=0xf4,iosize=0x04 \
"$@" \
</dev/null >"$log" 2>&1
status=$?
# shellcheck disable=SC2206 # QEMU is a command line
local vm=(timeout "$TIMEOUT" $QEMU
-machine q35 -accel "$QEMU_ACCEL" -cpu "$CPU"
-m "$QEMU_MEM" -smp "$QEMU_SMP" -nic none
-kernel "$kernel" -initrd "$payload" -append "litebox.log=${LITEBOX_LOG:-info}"
-serial stdio -display none -no-reboot
-device "isa-debug-exit,iobase=0xf4,iosize=0x04"
"$@")
if [[ -z ${client:-} ]]; then
"${vm[@]}" </dev/null >"$log" 2>&1
status=$?
else
"${vm[@]}" </dev/null >"$log" 2>&1 &
qemu=$!
background=("$qemu")
$client "$name" >"$WORK/$name.client" 2>&1 &
cpid=$!
background+=("$cpid")
# Until either ends (before bash 5.1, `wait -n` takes no PIDs).
while kill -0 "$qemu" 2>/dev/null && kill -0 "$cpid" 2>/dev/null; do
wait -n "$qemu" "$cpid"
done
if kill -0 "$qemu" 2>/dev/null; then
background=("$qemu")
wait "$cpid"
client_status=$?
if [[ $client_status -ne 0 ]]; then
kill "$qemu" 2>/dev/null && stopped=1
fi
else
background=("$cpid")
for _ in {1..50}; do
kill -0 "$cpid" 2>/dev/null || break
sleep 0.1
done
if kill -0 "$cpid" 2>/dev/null; then
pkill -P "$cpid" 2>/dev/null || kill "$cpid" 2>/dev/null
fi
wait "$cpid"
client_status=$?
fi
wait "$qemu"
status=$?
background=()
fi
set -e
# isa-debug-exit: QEMU exits with (value << 1) | 1; 33 = pass.
if [[ $status -eq 33 ]] && { [[ -z ${check:-} ]] || why=$($check "$name"); }; then
if [[ $status -eq 33 && $client_status -eq 0 ]] &&
{ [[ -z ${check:-} ]] || why=$($check "$name"); }; then
echo "PASS $name"
pass=$((pass + 1))
[[ ${verbose:-0} -eq 0 ]] || cat "$log"
if [[ ${verbose:-0} -ne 0 ]]; then
cat "$log"
[[ -z ${client:-} ]] || cat "$WORK/$name.client"
fi
else
case $status in
33) ;;
65) why="guest reported failure" ;;
124) why="timed out after ${TIMEOUT}s" ;;
*) why="exit status $status" ;;
*) if [[ $stopped -ne 0 ]]; then why="stopped"; else why="exit status $status"; fi ;;
esac
[[ $client_status -eq 0 ]] || why="${why:+$why, }host client exit status $client_status"
echo "FAIL $name ($why)"
sed 's/^/ | /' "$log" | tail -40
if [[ -n ${client:-} ]]; then
echo " host client:"
sed 's/^/ > /' "$WORK/$name.client" | tail -20
fi
fail=$((fail + 1))
failed+=("$name")
fi
Expand Down
50 changes: 49 additions & 1 deletion dev_tools/run_linux_on_vm_userland.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,12 @@
# A fresh host directory per test, with greeting.txt, is mounted read-only at
# /mnt/host over virtio-9p (its mount tag).
#
# A test with a host client, <test>.host.py, also gets a vhost-vsock device
# with guest CID $VSOCK_CID, and runs `python3 <test>.host.py CID CONSOLE`
# alongside the VM (CONSOLE: the console output's file), which must exit 0
# too. Without a usable /dev/vhost-vsock, or python3 AF_VSOCK, it is skipped,
# or, with REQUIRE_VSOCK set, fails.
#
# With QEMU_SMP above 1, the CPUs but the boot CPU must have run programs too:
# the kernel prints the slices each CPU ran, and the run fails if, over the
# passing tests with concurrent runs, the other CPUs ran none (a smoke check,
Expand All @@ -60,6 +66,10 @@
# QEMU_SMP guest CPUs, as QEMU's -smp (default: 1)
# TIMEOUT per-run timeout in seconds (default: 120, or 600 with tcg)
# LITEBOX_LOG guest log level, e.g. debug (default: info)
# REQUIRE_VSOCK if set, tests with a host client fail where they cannot run,
# instead of being skipped (CI sets it)
# VSOCK_CID guest CID of tests with a host client (default: 3 plus
# this script's PID, unique among running scripts)
# CARGO_BUILD command that builds the runners, run in their directories
# (default: "cargo build"; CI uses
# ".github/tools/github_actions_run_cargo build")
Expand All @@ -74,6 +84,11 @@ TESTS_DIR="$KERNEL_DIR/tests/linux"
HOSTED_TESTS_DIR="$REPO_DIR/litebox_runner_linux_userland/tests"
SHARED=(fork_threads_parent.c fork_interrupt_parent.c fork_event_parent.c fork_epoll_parent.c
fork_shared_futex_parent.c)
VSOCK_CID=${VSOCK_CID:-$((3 + $$))}
if [[ ! $VSOCK_CID =~ ^[1-9][0-9]{0,9}$ ]] || ((VSOCK_CID < 3 || VSOCK_CID > 4294967294)); then
echo "error: VSOCK_CID '$VSOCK_CID' must be a decimal integer from 3 to 4294967294" >&2
exit 2
fi
# shellcheck source=dev_tools/qemu_test_lib.sh
source "$SCRIPT_DIR/qemu_test_lib.sh"

Expand Down Expand Up @@ -173,6 +188,21 @@ share_args() {
-device virtio-9p-pci,fsdev=fs0,mount_tag=/mnt/host,disable-legacy=on
}

# Why host clients cannot run here, if they cannot.
host_client_unusable() {
if [[ ! -r /dev/vhost-vsock || ! -w /dev/vhost-vsock ]]; then
echo "/dev/vhost-vsock is not readable and writable"
elif ! python3 -c 'import socket; socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM).close()' \
2>/dev/null; then
echo "python3 cannot open AF_VSOCK sockets"
fi
}

# Test $1's host client: see the header.
run_host_client() {
TIMEOUT=$TIMEOUT python3 "$TESTS_DIR/$1.host.py" "$VSOCK_CID" "$WORK/$1.stdout"
}

# The kernel checks the output; this only shows it.
show_console() {
if [[ ${verbose:-0} -ne 0 ]]; then
Expand Down Expand Up @@ -236,6 +266,23 @@ for t in "${tests[@]}"; do
skipped=$((skipped + 1))
continue
fi
client=
vsock=()
if [[ -f "$TESTS_DIR/$t.host.py" ]]; then
why=$(host_client_unusable)
if [[ -n $why && -n ${REQUIRE_VSOCK:-} ]]; then
echo "FAIL $t ($why)"
fail=$((fail + 1))
failed+=("$t")
continue
elif [[ -n $why ]]; then
echo "SKIP $t ($why)"
skipped=$((skipped + 1))
continue
fi
client=run_host_client
vsock=(-device "vhost-vsock-pci,guest-cid=$VSOCK_CID,disable-legacy=on")
fi
mkdir -p "$WORK/$t"
cp "$USERLAND" "$WORK/$t/runner.elf"
cp "$WORK/rootfs.tar" "$WORK/$t/rootfs.tar"
Expand All @@ -253,7 +300,8 @@ for t in "${tests[@]}"; do
feeder=$!
fi
# shellcheck disable=SC2046 # these print separate arguments
run_vm "$t" "$KERNEL" "$WORK/$t.tar" $(console_args "$t") $(network_args) $(share_args "$t")
run_vm "$t" "$KERNEL" "$WORK/$t.tar" $(console_args "$t") $(network_args) $(share_args "$t") \
"${vsock[@]}"
if [[ -n $feeder ]]; then
kill "$feeder" 2>/dev/null || true
wait "$feeder" 2>/dev/null || true
Expand Down
3 changes: 3 additions & 0 deletions litebox_broker_vm_kernel/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,5 +18,8 @@ litebox_platform_vm_kernel = { path = "../litebox_platform_vm_kernel", version =
log = { version = "0.4", default-features = false }
spin = { version = "0.10.0", default-features = false, features = ["spin_mutex"] }

[dev-dependencies]
litebox_broker_core = { path = "../litebox_broker_core", version = "0.1.0", features = ["test-support"] }

[lints]
workspace = true
57 changes: 43 additions & 14 deletions litebox_broker_vm_kernel/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,11 @@
//! Timers are on the kernel clock. IPv4 TCP and UDP
//! sockets are `litebox_broker_smoltcp`'s, which mediates their traffic:
//! between runner processes directly, and, with the runner's [`Network`]
//! device ([`Config::network`]), to the outside on smoltcp. Readiness (e.g., of
//! standard input, a timer's expiry, or sockets) goes to the association's
//! notification ring.
//! device ([`Config::network`]), to the outside on smoltcp. With the runner's
//! socket device ([`Config::vsock`]), the [`vsock`] forwarder connects the
//! host's `AF_VSOCK` connections to guests' Unix stream sockets. Readiness
//! (e.g., of standard input, a timer's expiry, or sockets) goes to the
//! association's notification ring.
//!
//! As in the userland broker, requests are served apart from the runner's
//! calls ([`Association::serve`]), and a wait blocks only its process: the
Expand Down Expand Up @@ -60,6 +62,7 @@ mod readiness;
pub mod share;
pub mod stdio;
mod timers;
pub mod vsock;

use alloc::boxed::Box;
use alloc::sync::Arc;
Expand Down Expand Up @@ -131,6 +134,10 @@ pub struct Config {
pub shares: Vec<share::Share>,
/// The network sockets reach beyond the guest, if any.
pub network: Option<&'static dyn Network>,
/// The socket device the host's `AF_VSOCK` connections arrive on, if
/// any (see [`vsock`]); ignored without
/// [`Self::allow_process_duplication`] (programs).
pub vsock: Option<&'static dyn vsock::Vsock>,
/// Whether processes may start child processes (process duplication), as
/// the userland broker's `allow_process_duplication`; the kernel then
/// creates their runner processes (see [`PendingRunnerAssociation`]). Roots
Expand All @@ -149,6 +156,8 @@ pub struct Broker {
sockets: Arc<SmoltcpSocketProvider>,
/// The virtual init, with [`Config::allow_process_duplication`].
init: Option<Arc<BrokerProcess>>,
/// With [`Config::vsock`].
vsock: Option<Arc<vsock::Forwarder>>,
}

impl Broker {
Expand Down Expand Up @@ -192,6 +201,18 @@ impl Broker {
.expect("a running process can change its place in the tree");
init
});
// Only programs' guests listen on vsock; after init, which must be
// process 1.
if config.vsock.is_some() && init.is_none() {
log::warn!("vsock device ignored without process duplication: host connections hang");
}
let platform = config.platform;
let vsock = config.vsock.filter(|_| init.is_some()).and_then(|device| {
vsock::Forwarder::new(&core, device, Box::new(move || platform.now()))
.inspect_err(|error| log::warn!("no vsock forwarder: {error}"))
.ok()
.map(Arc::new)
});
Self {
core,
platform: config.platform,
Expand All @@ -200,6 +221,7 @@ impl Broker {
timers,
sockets,
init,
vsock,
}
}

Expand Down Expand Up @@ -294,17 +316,24 @@ impl Broker {
}
}

/// Looks at what changes outside requests: standard input, timers, and
/// the network.
/// Looks at what changes outside requests: standard input, timers, the
/// network, and the socket device.
fn poll_sources(&self) {
self.stdio.poll();
self.timers.fire();
self.sockets.poll();
if let Some(vsock) = &self.vsock {
vsock.poll();
}
}

/// When the network stack must next run, if ever.
fn next_network_deadline(&self) -> Option<Instant> {
self.sockets.next_deadline().map(timers::instant)
/// When the network stack, or the vsock forwarder, must next run, if
/// ever.
fn next_sources_deadline(&self) -> Option<Instant> {
timers::earliest(
self.sockets.next_deadline().map(timers::instant),
self.vsock.as_ref().and_then(|vsock| vsock.next_deadline()),
)
}
}

Expand Down Expand Up @@ -457,15 +486,15 @@ fn frame(bytes: &[u8]) -> Result<WireFrame, Status> {
impl Association {
/// When, on the kernel clock, [`Self::poll_events`] next has something
/// to do: the earliest armed broker timer's deadline, or the network
/// stack's (e.g., to retransmit).
/// stack's (e.g., to retransmit), or the vsock forwarder's.
pub fn next_timer_deadline(&self) -> Option<Instant> {
timers::earliest(
self.broker.timers.next_deadline(),
self.broker.next_network_deadline(),
self.broker.next_sources_deadline(),
)
}

/// Looks at the devices (standard input, the network) and fires the
/// Looks at the devices (standard input, the network, vsock) and fires the
/// broker timers that are due, publishing readiness, as
/// [`RingFutex::Wait`] would; for a process that is running instead
/// (e.g., an interrupt stopped it). A notification ring this corrupts
Expand Down Expand Up @@ -789,14 +818,14 @@ impl Association {
if wait.deadline.is_none() && !wait.notifications && !self.submitted.get() {
return Err(EnterError::Status(Status::Stalled));
}
// Only notifications see timers; the network stack runs at its
// deadlines, whatever the wait.
// Only notifications see timers; the network stack and the vsock
// forwarder run at their deadlines, whatever the wait.
let wake = if wait.notifications {
timers::earliest(wait.deadline, self.broker.timers.next_deadline())
} else {
wait.deadline
};
let wake = timers::earliest(wake, self.broker.next_network_deadline());
let wake = timers::earliest(wake, self.broker.next_sources_deadline());
Ok(WaitState::Blocked { wake })
})
.unwrap_or_else(|error| WaitState::Ended(Err(error)))
Expand Down
Loading
Loading