From e051c4ebd839b67fd7043382423cfd43b9af34e5 Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Tue, 7 Jul 2026 22:44:05 -0700 Subject: [PATCH 1/2] ci: dump diagnostics when a test run stalls When a test hangs, prove produces no further output, so the CI job simply idles until the platform's hard timeout (six hours on GitHub) and is then cancelled, with nothing in the log to say which test stalled. Diagnosing such a hang currently means correlating a healthy run's test list against the truncated one to find the test that never reported, which is slow and only possible after the fact. Start a watchdog at the beginning of the build-and-test run that, if it sees no progress for GIT_TEST_STALL_TIMEOUT seconds (two hours by default, well beyond a healthy run), dumps state that points at the culprit: - the tests still in flight: test-lib removes a test's trash directory only once it passes, so a surviving one has not finished; - the tail of each such test's --verbose-log trace, i.e. the command it is stuck on (our CI already runs with --verbose-log -x); - the process table sorted by resident memory, plus the machine's memory and kernel log, to spot a memory or OOM cascade. The watchdog only reports; it neither fails nor kills anything, so it cannot cause a spurious failure even if it fires on an unusually slow run. It counts down in short sleeps rather than one long "sleep" so that when it is stopped on the happy path, no long-lived sleep lingers holding the job's output open. The logic lives in the shared ci/ scripts, so both the GitHub and GitLab pipelines benefit. Signed-off-by: Michael Montalbo --- ci/lib.sh | 37 +++++++++++++++++++++++++++++++++++++ ci/run-build-and-tests.sh | 21 +++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/ci/lib.sh b/ci/lib.sh index b939110a6eefcf..68259c9828d7ce 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -57,6 +57,43 @@ group () { return $res } +# Dump diagnostics about a test run that appears stalled: which tests are +# still in flight (test-lib only removes a test's trash directory once it +# passes, so a surviving one has not finished), the tail of each such test's +# --verbose-log trace (i.e. the command it is stuck on), the process table +# sorted by memory use, and the machine's memory and kernel log. A hung test +# otherwise produces no output at all, so the job merely idles until the CI +# platform's hard timeout with no hint of the culprit. +dump_stalled_test_state () { + # This is best-effort diagnostics run under "set -e"; never let a + # missing tool or file abort the dump halfway through. + set +e + results="${TEST_OUTPUT_DIRECTORY:-t}/test-results" + + echo "===================================================================" + echo "No progress for ${1}s; the test run looks stalled. Diagnostics follow." + echo "===================================================================" + + for trash in "${TEST_OUTPUT_DIRECTORY:-t}"/'trash directory.'* + do + test -d "$trash" || continue + name="${trash##*/trash directory.}" + echo "### still running: $name ###" + tail -n 20 "$results/$name.out" 2>/dev/null || + echo " (no --verbose-log trace captured)" + done + + echo "### processes, sorted by resident set size ###" + ps -eo pid,ppid,etimes,rss,stat,args 2>/dev/null | sort -k 4 -nr | head -n 40 + + echo "### memory and load ###" + free -m 2>/dev/null || head -n 3 /proc/meminfo 2>/dev/null + cat /proc/loadavg 2>/dev/null + + echo "### kernel log tail (the OOM killer records here) ###" + dmesg 2>/dev/null | tail -n 30 || echo " (dmesg unavailable)" +} + begin_group "CI setup via $(basename $0)" # Set 'exit on error' for all CI scripts to let the caller know that diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 1d9a0a736d255b..03b2bea318ca85 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -50,6 +50,25 @@ linux-reftable|linux-reftable-leaks|osx-reftable) esac +# A hung test produces no output, so the job idles until the CI platform's +# hard timeout with no hint which test stalled. Start a watchdog that dumps +# diagnostics (see dump_stalled_test_state) if the run makes no progress for +# GIT_TEST_STALL_TIMEOUT seconds. Count down in short steps rather than one +# long "sleep" so that when we stop it on the happy path, no long-lived sleep +# is left holding our output open and delaying the job from ending. +stall_timeout="${GIT_TEST_STALL_TIMEOUT:-7200}" +{ + set +x + remaining=$stall_timeout + while test 0 -lt "$remaining" + do + sleep 5 || exit + remaining=$((remaining - 5)) + done + dump_stalled_test_state "$stall_timeout" +} & +stall_watchdog_pid=$! + case "$jobname" in *-meson) group "Configure" meson setup build . \ @@ -72,5 +91,7 @@ case "$jobname" in ;; esac +kill "$stall_watchdog_pid" 2>/dev/null || : + check_unignored_build_artifacts save_good_tree From c9a38b437fe3648d516ff67b96ca67ea0fc6091c Mon Sep 17 00:00:00 2001 From: Michael Montalbo Date: Tue, 7 Jul 2026 22:44:39 -0700 Subject: [PATCH 2/2] ci: bound the GitHub test jobs so a stall fails before the 6h limit The previous commit makes a stalled test run dump diagnostics, but the job itself still idles until GitHub's six-hour hard limit before it is cancelled. Give the test jobs an explicit timeout-minutes so that a hang fails the job within a few hours instead, shortly after the ci/ stall watchdog has printed its diagnostics, and GitHub tears down the job's process tree cleanly. Three hours leaves ample margin over a healthy run (well under two hours) while still saving half of the wasted budget. Signed-off-by: Michael Montalbo --- .github/workflows/main.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cf341d74dbff21..ce35a28aeafabd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -340,6 +340,9 @@ jobs: CI_JOB_IMAGE: ${{matrix.vector.pool}} TEST_OUTPUT_DIRECTORY: ${{github.workspace}}/t runs-on: ${{matrix.vector.pool}} + # A stalled run otherwise idles until GitHub's 6h hard limit; fail + # sooner, shortly after the ci/ stall watchdog has dumped diagnostics. + timeout-minutes: 180 steps: - uses: actions/checkout@v6 - run: ci/install-dependencies.sh @@ -420,6 +423,9 @@ jobs: CI_JOB_IMAGE: ${{matrix.vector.image}} CUSTOM_PATH: /custom runs-on: ubuntu-latest + # A stalled run otherwise idles until GitHub's 6h hard limit; fail + # sooner, shortly after the ci/ stall watchdog has dumped diagnostics. + timeout-minutes: 180 container: ${{matrix.vector.image}} steps: - name: prepare libc6 for actions