diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cf341d74dbff21..ce35a28aeafabd 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -340,6 +340,9 @@ jobs: CI_JOB_IMAGE: ${{matrix.vector.pool}} TEST_OUTPUT_DIRECTORY: ${{github.workspace}}/t runs-on: ${{matrix.vector.pool}} + # A stalled run otherwise idles until GitHub's 6h hard limit; fail + # sooner, shortly after the ci/ stall watchdog has dumped diagnostics. + timeout-minutes: 180 steps: - uses: actions/checkout@v6 - run: ci/install-dependencies.sh @@ -420,6 +423,9 @@ jobs: CI_JOB_IMAGE: ${{matrix.vector.image}} CUSTOM_PATH: /custom runs-on: ubuntu-latest + # A stalled run otherwise idles until GitHub's 6h hard limit; fail + # sooner, shortly after the ci/ stall watchdog has dumped diagnostics. + timeout-minutes: 180 container: ${{matrix.vector.image}} steps: - name: prepare libc6 for actions diff --git a/ci/lib.sh b/ci/lib.sh index b939110a6eefcf..68259c9828d7ce 100755 --- a/ci/lib.sh +++ b/ci/lib.sh @@ -57,6 +57,43 @@ group () { return $res } +# Dump diagnostics about a test run that appears stalled: which tests are +# still in flight (test-lib only removes a test's trash directory once it +# passes, so a surviving one has not finished), the tail of each such test's +# --verbose-log trace (i.e. the command it is stuck on), the process table +# sorted by memory use, and the machine's memory and kernel log. A hung test +# otherwise produces no output at all, so the job merely idles until the CI +# platform's hard timeout with no hint of the culprit. +dump_stalled_test_state () { + # This is best-effort diagnostics run under "set -e"; never let a + # missing tool or file abort the dump halfway through. + set +e + results="${TEST_OUTPUT_DIRECTORY:-t}/test-results" + + echo "===================================================================" + echo "No progress for ${1}s; the test run looks stalled. Diagnostics follow." + echo "===================================================================" + + for trash in "${TEST_OUTPUT_DIRECTORY:-t}"/'trash directory.'* + do + test -d "$trash" || continue + name="${trash##*/trash directory.}" + echo "### still running: $name ###" + tail -n 20 "$results/$name.out" 2>/dev/null || + echo " (no --verbose-log trace captured)" + done + + echo "### processes, sorted by resident set size ###" + ps -eo pid,ppid,etimes,rss,stat,args 2>/dev/null | sort -k 4 -nr | head -n 40 + + echo "### memory and load ###" + free -m 2>/dev/null || head -n 3 /proc/meminfo 2>/dev/null + cat /proc/loadavg 2>/dev/null + + echo "### kernel log tail (the OOM killer records here) ###" + dmesg 2>/dev/null | tail -n 30 || echo " (dmesg unavailable)" +} + begin_group "CI setup via $(basename $0)" # Set 'exit on error' for all CI scripts to let the caller know that diff --git a/ci/run-build-and-tests.sh b/ci/run-build-and-tests.sh index 1d9a0a736d255b..03b2bea318ca85 100755 --- a/ci/run-build-and-tests.sh +++ b/ci/run-build-and-tests.sh @@ -50,6 +50,25 @@ linux-reftable|linux-reftable-leaks|osx-reftable) esac +# A hung test produces no output, so the job idles until the CI platform's +# hard timeout with no hint which test stalled. Start a watchdog that dumps +# diagnostics (see dump_stalled_test_state) if the run makes no progress for +# GIT_TEST_STALL_TIMEOUT seconds. Count down in short steps rather than one +# long "sleep" so that when we stop it on the happy path, no long-lived sleep +# is left holding our output open and delaying the job from ending. +stall_timeout="${GIT_TEST_STALL_TIMEOUT:-7200}" +{ + set +x + remaining=$stall_timeout + while test 0 -lt "$remaining" + do + sleep 5 || exit + remaining=$((remaining - 5)) + done + dump_stalled_test_state "$stall_timeout" +} & +stall_watchdog_pid=$! + case "$jobname" in *-meson) group "Configure" meson setup build . \ @@ -72,5 +91,7 @@ case "$jobname" in ;; esac +kill "$stall_watchdog_pid" 2>/dev/null || : + check_unignored_build_artifacts save_good_tree