diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 8f67835..3105ffc 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -316,6 +316,9 @@ jobs: name: rootfiles-binary path: . - name: Tenant lifecycle on the runner's Docker and systemd + env: + ROOTFILES_TENANT_SCENARIO_CI: '1' + ROOTFILES_TENANT_REMOUNT_CYCLES: '25' run: | chmod +x rootfiles ROOTFILES_TEST_BINARY="$PWD/rootfiles" bash tests/scenarios/tenant-lifecycle.sh diff --git a/docs/specs/tenant.md b/docs/specs/tenant.md index 48824f8..7a65b2b 100644 --- a/docs/specs/tenant.md +++ b/docs/specs/tenant.md @@ -121,3 +121,9 @@ Single-port SSH (sshpiper) ## Open issues - Not tracked: the 2026-10-20 design stored GPU UUIDs and checked OCI `linux.devices`; the code stores indices and verifies with `nvidia-smi -L`. + +## Scenario mount restarts + +- Tenant lifecycle runs only through the disposable hosted tenant-scenario job. Bounded trials retain mount/dependency/namespace diagnostics and restore temporary systemd logging on exit; pacing respects the configured start-limit window without changing unit settings or counters. +- The CI scenario stops the mount, waits for udev and the old loop association/device to be released, then starts it; production ensureMountUnit only starts unmounted units and never restarts them. +- Each trial verifies the existing image inode/size, filesystem UUID and data marker. Failure to confirm release or a bare mountpoint fails immediately; the scenario never formats or populates it as a workaround. Production mount-unit generation/options remain unchanged; native/GPU rollout validation stays separate. diff --git a/tests/scenarios/tenant-lifecycle.sh b/tests/scenarios/tenant-lifecycle.sh index 40497bd..bd8271f 100755 --- a/tests/scenarios/tenant-lifecycle.sh +++ b/tests/scenarios/tenant-lifecycle.sh @@ -6,6 +6,16 @@ # GPU steps (runtime=nvidia, reload safety) need a GPU host and are not run. set -euo pipefail +if [[ "${GITHUB_ACTIONS:-}" != true || "${GITHUB_JOB:-}" != tenant-scenario || "${ROOTFILES_TENANT_SCENARIO_CI:-}" != 1 ]]; then + echo "tenant-lifecycle changes host services and volumes; run it only through the disposable CI tenant-scenario job" >&2 + exit 1 +fi +REMOUNT_CYCLES="${ROOTFILES_TENANT_REMOUNT_CYCLES:-1}" +if [[ ! "$REMOUNT_CYCLES" =~ ^[1-9][0-9]?$ ]] || (( REMOUNT_CYCLES < 1 || REMOUNT_CYCLES > 50 )); then + echo "invalid remount cycle count; set ROOTFILES_TENANT_REMOUNT_CYCLES to 1..50 in the CI job" >&2 + exit 1 +fi + REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" source "$REPO_ROOT/tests/integration/assert.sh" @@ -19,7 +29,11 @@ command -v sshpass >/dev/null || sudo apt-get install -y -qq sshpass >/dev/null TMP="$(mktemp -d)" KH="$TMP/known_hosts" +SYSTEMD_LOG_LEVEL_BEFORE="" cleanup() { + if [ -n "$SYSTEMD_LOG_LEVEL_BEFORE" ]; then + sudo systemd-analyze log-level "$SYSTEMD_LOG_LEVEL_BEFORE" || echo "could not restore systemd log level; inspect systemd-analyze log-level on this disposable runner" >&2 + fi docker rm -f tenant-t1 tenant-t1-prev tenant-t2 tenant-t3 legacy >/dev/null 2>&1 || true } trap cleanup EXIT @@ -139,32 +153,143 @@ check "container kept running through growth" test "$started" = "$(docker inspec check "online shrink refused" bash -c "! $RF tenant volume resize t1 200M >/dev/null 2>&1" docker exec tenant-t1 rm -f /data/fill +# Read-only evidence, also on the host-bare failure branch. Do not create +# data/home/tmp or repair a mount here: those would hide the failure. +mount_diagnostics() { + local phase=$1 service daemon_pid + echo "--- mount diagnostics: $phase ---" + sudo readlink /proc/1/ns/mnt "/proc/$$/ns/mnt" || true + sudo nsenter -t 1 -m -- findmnt --mountpoint /srv/tenants/t1/vol || true + sudo systemctl show srv-tenants-t1-vol.mount -p ActiveState -p SubState -p Result -p What -p Where -p Job -p BindsTo -p Requires -p After -p StopPropagatedFrom -p StateChangeTimestampMonotonic || true + sudo journalctl -u srv-tenants-t1-vol.mount --no-pager -n 160 -o short-monotonic || true + sudo findmnt --mountpoint /srv/tenants/t1/vol -o TARGET,SOURCE,MAJ:MIN,FSTYPE,PROPAGATION || true + sudo findmnt -T /srv/tenants/t1/vol/data || true + sudo losetup -j /srv/tenants/t1/volume.img || true + sudo blkid /srv/tenants/t1/volume.img || true + for service in docker containerd; do + daemon_pid=$(systemctl show "$service.service" -p MainPID --value 2>/dev/null || true) + echo "$service MainPID=${daemon_pid:-unknown}" + if [[ "$daemon_pid" =~ ^[1-9][0-9]*$ ]]; then + sudo readlink "/proc/$daemon_pid/ns/mnt" || true + sudo nsenter -t "$daemon_pid" -m -- findmnt --mountpoint /srv/tenants/t1/vol || true + sudo nsenter -t "$daemon_pid" -m -- stat /srv/tenants/t1/vol/data || true + fi + done +} + +# The old loop association and systemd device state must both disappear +# before reattaching the image. This never repairs or populates the mountpoint. +wait_for_loop_release() { + local image=$1 device device_unit device_state associations + local deadline=$((SECONDS + 30)) remaining released + shift + while :; do + remaining=$((deadline - SECONDS)) + if (( remaining <= 0 )); then return 1; fi + if ! sudo udevadm settle --timeout="$remaining"; then return 1; fi + if ! associations=$(sudo losetup -j "$image"); then return 1; fi + released=1 + for device in "$@"; do + device_unit=$(systemd-escape --path --suffix=device "$device") || return 1 + device_state=$(systemctl show "$device_unit" -p ActiveState --value) || return 1 + case "$device_state" in + inactive|failed) ;; + active|activating|deactivating|reloading|maintenance|refreshing) released=0 ;; + *) echo "unknown state for old loop device $device_unit; inspect systemctl show before rerunning CI" >&2; return 1 ;; + esac + done + if [ -z "$associations" ] && (( released )); then + echo "Old loop association is empty and its device units are inactive after udev settle" + return 0 + fi + if (( SECONDS >= deadline )); then return 1; fi + sleep 1 + done +} + echo "--- AC8: volume comes back after a unit restart ---" check "mount unit ordered before docker" bash -c 'systemctl show -p Before srv-tenants-t1-vol.mount | grep -q docker.service' -docker stop tenant-t1 >/dev/null -sudo systemctl restart srv-tenants-t1-vol.mount -# A successful systemctl job alone does not prove that the image is mounted -# in the namespace Docker uses. Never create /vol/data on a bare mountpoint. -if ! sudo mountpoint -q /srv/tenants/t1/vol || ! sudo test -d /srv/tenants/t1/vol/data; then - sudo systemctl show srv-tenants-t1-vol.mount -p ActiveState -p SubState -p Result -p What -p Where - sudo journalctl -u srv-tenants-t1-vol.mount --no-pager -n 60 - sudo findmnt -T /srv/tenants/t1/vol || true - sudo losetup -j /srv/tenants/t1/volume.img || true - fail "tenant volume is not mounted with its data directory after restart" +image_identity=$(sudo stat -c '%i:%s' /srv/tenants/t1/volume.img) +image_uuid=$(sudo blkid -s UUID -o value /srv/tenants/t1/volume.img) +check "existing filesystem UUID recorded" test -n "$image_uuid" +SYSTEMD_LOG_LEVEL_BEFORE=$(sudo systemd-analyze log-level) +if [[ ! "$SYSTEMD_LOG_LEVEL_BEFORE" =~ ^(emerg|alert|crit|err|warning|notice|info|debug)$ ]]; then + echo "cannot capture the systemd log level; inspect systemd-analyze log-level before rerunning the CI scenario" >&2 + SYSTEMD_LOG_LEVEL_BEFORE="" exit 1 fi -if ! docker start tenant-t1 >/dev/null; then - sudo systemctl show srv-tenants-t1-vol.mount -p ActiveState -p SubState -p Result -p What -p Where - sudo findmnt -T /srv/tenants/t1/vol/data || true - docker_pid=$(pgrep -xo dockerd || true) - if [ -n "$docker_pid" ]; then - sudo nsenter -t "$docker_pid" -m -- findmnt -T /srv/tenants/t1/vol/data || true - sudo nsenter -t "$docker_pid" -m -- stat /srv/tenants/t1/vol/data || true - fi - fail "Docker cannot see the remounted tenant data volume" +# Transient manager logging in this disposable runner only, restored by cleanup. +sudo systemd-analyze log-level debug +mount_diagnostics before-restarts +# Pacing avoids making a new start-limit failure with the stress harness. +# Keep the first post-growth restart immediate, preserving the original race. +unit_object=$(busctl call org.freedesktop.systemd1 /org/freedesktop/systemd1 org.freedesktop.systemd1.Manager GetUnit s srv-tenants-t1-vol.mount | python3 -c 'import shlex,sys; value=shlex.split(sys.stdin.read()); assert len(value)==2 and value[0]=="o"; print(value[1])') +read -r interval_type interval_us < <(busctl get-property org.freedesktop.systemd1 "$unit_object" org.freedesktop.systemd1.Unit StartLimitIntervalUSec) +if [[ "$interval_type" != t || ! "$interval_us" =~ ^[0-9]{1,9}$ ]] || (( interval_us > 30000000 )); then + fail "cannot bound the mount start-limit window; inspect the unit properties before rerunning CI" exit 1 fi -check "data intact after remount" docker exec tenant-t1 test -f /data/hello +pause_seconds=$(( (interval_us + 999999) / 1000000 + 1 )) +sudo systemctl show srv-tenants-t1-vol.mount -p StartLimitIntervalUSec -p StartLimitBurst +echo "Pacing later trials by ${pause_seconds}s without changing the unit or resetting failed state" +for ((cycle=1; cycle<=REMOUNT_CYCLES; cycle++)); do + if (( cycle > 1 )); then sleep "$pause_seconds"; fi + echo "--- remount trial $cycle/$REMOUNT_CYCLES ---" + if ! old_associations=$(sudo losetup -j /srv/tenants/t1/volume.img) || [ -z "$old_associations" ]; then + mount_diagnostics "old-loop-unknown-trial-$cycle" + fail "cannot identify the mounted image's old loop device; inspect its association before continuing" + exit 1 + fi + old_loop_devices=() + while IFS=: read -r device _; do + if [[ ! "$device" =~ ^/dev/loop[0-9]+$ ]]; then + mount_diagnostics "old-loop-invalid-trial-$cycle" + fail "unexpected loop association; inspect losetup output before continuing" + exit 1 + fi + old_loop_devices+=("$device") + done <<< "$old_associations" + echo "Old loop devices for trial $cycle: ${old_loop_devices[*]}" + docker stop tenant-t1 >/dev/null + if ! sudo systemctl stop srv-tenants-t1-vol.mount; then + mount_diagnostics "stop-job-failed-trial-$cycle" + fail "mount stop job failed; inspect the unit and dependency journal above" + exit 1 + fi + if ! wait_for_loop_release /srv/tenants/t1/volume.img "${old_loop_devices[@]}"; then + mount_diagnostics "loop-release-failed-trial-$cycle" + fail "could not confirm old loop release within 30 seconds; inspect udev, loop associations and device states above" + exit 1 + fi + if ! sudo systemctl start srv-tenants-t1-vol.mount; then + mount_diagnostics "start-job-failed-trial-$cycle" + fail "mount start job failed; inspect the unit and dependency journal above" + exit 1 + fi + # A completed systemctl job is insufficient: a later device/dependency + # event may have stopped the mount again. Preserve the first failure. + if ! sudo mountpoint -q /srv/tenants/t1/vol || ! sudo test -d /srv/tenants/t1/vol/data; then + mount_diagnostics "host-mount-missing-trial-$cycle" + fail "tenant volume disappeared after restart; inspect the mount dependency journal above" + exit 1 + fi + if ! docker start tenant-t1 >/dev/null; then + mount_diagnostics "container-start-failed-trial-$cycle" + fail "Docker cannot see the mounted tenant data; inspect the daemon namespace evidence above" + exit 1 + fi + if ! outputs kept docker exec tenant-t1 cat /data/hello || + [ "$image_identity" != "$(sudo stat -c '%i:%s' /srv/tenants/t1/volume.img)" ] || + [ "$image_uuid" != "$(sudo blkid -s UUID -o value /srv/tenants/t1/volume.img)" ]; then + mount_diagnostics "data-identity-changed-trial-$cycle" + fail "existing image identity, UUID or data changed; preserve the image and inspect the evidence above" + exit 1 + fi + pass "existing data, image identity and filesystem UUID intact after remount trial $cycle" +done +mount_diagnostics after-restarts +sudo systemd-analyze log-level "$SYSTEMD_LOG_LEVEL_BEFORE" +SYSTEMD_LOG_LEVEL_BEFORE="" echo "--- AC6: operator delegation ---" out2=$($RF tenant create t2 --image rootfiles-tenant-test:1 --ssh-port 2202 --size 64M --account carol:2100)