From 71e580cc70b051b2f7cdf84aafc87b837cc37d23 Mon Sep 17 00:00:00 2001 From: Sayan- <1415138+Sayan-@users.noreply.github.com> Date: Thu, 3 Sep 2026 16:38:44 +0000 Subject: [PATCH 1/2] Sweep leaked test networks at job start Tests create a bridge and route per VM and clean up on success, but runs that die mid-test leak them, and cancel-in-progress made mid-test death routine. The pile eventually collides with test subnets and every run fails on SUBNET CONFLICT, which has taken the fleet red twice in eight days. Each job now deletes interfaces that are DOWN with no attached ports across two snapshots, skipping the settle delay when the host is healthy. Co-Authored-By: Claude Fable 5 --- .github/workflows/test.yml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c2d1eba05..d5b81d44d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -54,6 +54,35 @@ jobs: -e lib/ingress/binaries/ \ -e bin/ + # Tests create a bridge and route per VM and clean up on success, but a + # run that dies mid-test (crash, timeout, superseded-run cancellation) + # leaks them, and enough leaked routes fail new tests on subnet + # conflicts. Each job sweeps before it starts: only interfaces DOWN + # with no attached ports across two snapshots are deleted, so anything + # a live job is mid-setup on drops out by the second look. Healthy + # hosts skip the settle delay, and nothing here ever fails the job. + - name: Clean leaked test networks + run: | + candidates() { + ip -br link show | awk '$1 ~ /^(h[0-9a-f]{8}|hi[0-9a-f]+|hm[0-9a-f]+|hype-[a-z0-9]+)$/ && $2 == "DOWN" {print $1}' | + while read -r ifc; do + if [ -z "$(ls "/sys/class/net/$ifc/brif" 2>/dev/null)" ]; then echo "$ifc"; fi + done + } + candidates | sort > "$RUNNER_TEMP/net-sweep-a" + n=$(wc -l < "$RUNNER_TEMP/net-sweep-a") + if [ "$n" -lt 25 ]; then + echo "no leak pile ($n candidates); skipping" + exit 0 + fi + sleep 10 + candidates | sort > "$RUNNER_TEMP/net-sweep-b" + comm -12 "$RUNNER_TEMP/net-sweep-a" "$RUNNER_TEMP/net-sweep-b" > "$RUNNER_TEMP/net-sweep-dead" + while read -r ifc; do + sudo ip link del "$ifc" 2>/dev/null || true + done < "$RUNNER_TEMP/net-sweep-dead" + echo "swept $(wc -l < "$RUNNER_TEMP/net-sweep-dead") leaked interfaces" + - name: Set up Go uses: actions/setup-go@v6 with: From 7774e822c1274017fe91bcd78387f02735117b59 Mon Sep 17 00:00:00 2001 From: Sayan- <1415138+Sayan-@users.noreply.github.com> Date: Thu, 3 Sep 2026 17:48:27 +0000 Subject: [PATCH 2/2] Gate the network sweep on interface age, not settle time Bugbot caught a real hole in the double-snapshot design: a live bridge sits DOWN with no ports from Initialize until the TAP attaches after the rootfs build, which can exceed any settle window, so under enough concurrency the sweep could delete live bridges. Age discriminates where liveness cannot: a live bridge waits at most one go-test timeout for its TAP, while leaks are hours old. Sweep only interfaces older than an hour, read from the sysfs dir timestamp, sparing anything whose age cannot be read. This also deletes the snapshot, sleep, and threshold machinery outright. Co-Authored-By: Claude Fable 5 --- .github/workflows/test.yml | 36 ++++++++++++++++-------------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d5b81d44d..7ccb6f999 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,31 +57,27 @@ jobs: # Tests create a bridge and route per VM and clean up on success, but a # run that dies mid-test (crash, timeout, superseded-run cancellation) # leaks them, and enough leaked routes fail new tests on subnet - # conflicts. Each job sweeps before it starts: only interfaces DOWN - # with no attached ports across two snapshots are deleted, so anything - # a live job is mid-setup on drops out by the second look. Healthy - # hosts skip the settle delay, and nothing here ever fails the job. + # conflicts. Each job sweeps before it starts. A live bridge can sit + # DOWN with no ports for minutes (Initialize creates it, the TAP only + # attaches after the rootfs build), so liveness alone cannot identify + # a leak; age can: a live bridge waits at most one go-test timeout for + # its TAP, while leaks are hours old. The sysfs dir timestamp is the + # interface's creation time, and a failed stat spares the interface. + # Nothing here ever fails the job. - name: Clean leaked test networks run: | - candidates() { - ip -br link show | awk '$1 ~ /^(h[0-9a-f]{8}|hi[0-9a-f]+|hm[0-9a-f]+|hype-[a-z0-9]+)$/ && $2 == "DOWN" {print $1}' | - while read -r ifc; do - if [ -z "$(ls "/sys/class/net/$ifc/brif" 2>/dev/null)" ]; then echo "$ifc"; fi - done - } - candidates | sort > "$RUNNER_TEMP/net-sweep-a" - n=$(wc -l < "$RUNNER_TEMP/net-sweep-a") - if [ "$n" -lt 25 ]; then - echo "no leak pile ($n candidates); skipping" - exit 0 - fi - sleep 10 - candidates | sort > "$RUNNER_TEMP/net-sweep-b" - comm -12 "$RUNNER_TEMP/net-sweep-a" "$RUNNER_TEMP/net-sweep-b" > "$RUNNER_TEMP/net-sweep-dead" + cutoff=$(( $(date +%s) - 3600 )) + ip -br link show | awk '$1 ~ /^(h[0-9a-f]{8}|hi[0-9a-f]+|hm[0-9a-f]+|hype-[a-z0-9]+)$/ && $2 == "DOWN" {print $1}' | + while read -r ifc; do + if [ -z "$(ls "/sys/class/net/$ifc/brif" 2>/dev/null)" ] \ + && [ "$(stat -c %Y "/sys/class/net/$ifc" 2>/dev/null || echo "$cutoff")" -lt "$cutoff" ]; then + echo "$ifc" + fi + done > "$RUNNER_TEMP/net-sweep-dead" while read -r ifc; do sudo ip link del "$ifc" 2>/dev/null || true done < "$RUNNER_TEMP/net-sweep-dead" - echo "swept $(wc -l < "$RUNNER_TEMP/net-sweep-dead") leaked interfaces" + echo "swept $(wc -l < "$RUNNER_TEMP/net-sweep-dead") interfaces (DOWN, portless, older than 1h)" - name: Set up Go uses: actions/setup-go@v6