Mm/ci diagnose hangs

.github/workflows/main.yml

@@ -340,6 +340,9 @@ jobs:
       CI_JOB_IMAGE: ${{matrix.vector.pool}}
       TEST_OUTPUT_DIRECTORY: ${{github.workspace}}/t
     runs-on: ${{matrix.vector.pool}}
+    # A stalled run otherwise idles until GitHub's 6h hard limit; fail
+    # sooner, shortly after the ci/ stall watchdog has dumped diagnostics.
+    timeout-minutes: 180
     steps:
     - uses: actions/checkout@v6
     - run: ci/install-dependencies.sh
@@ -420,6 +423,9 @@ jobs:
       CI_JOB_IMAGE: ${{matrix.vector.image}}
       CUSTOM_PATH: /custom
     runs-on: ubuntu-latest
+    # A stalled run otherwise idles until GitHub's 6h hard limit; fail
+    # sooner, shortly after the ci/ stall watchdog has dumped diagnostics.
+    timeout-minutes: 180
     container: ${{matrix.vector.image}}
     steps:
     - name: prepare libc6 for actions

ci/lib.sh

@@ -57,6 +57,43 @@ group () {
 	return $res
 }
 
+# Dump diagnostics about a test run that appears stalled: which tests are
+# still in flight (test-lib only removes a test's trash directory once it
+# passes, so a surviving one has not finished), the tail of each such test's
+# --verbose-log trace (i.e. the command it is stuck on), the process table
+# sorted by memory use, and the machine's memory and kernel log. A hung test
+# otherwise produces no output at all, so the job merely idles until the CI
+# platform's hard timeout with no hint of the culprit.
+dump_stalled_test_state () {
+	# This is best-effort diagnostics run under "set -e"; never let a
+	# missing tool or file abort the dump halfway through.
+	set +e
+	results="${TEST_OUTPUT_DIRECTORY:-t}/test-results"
+
+	echo "==================================================================="
+	echo "No progress for ${1}s; the test run looks stalled. Diagnostics follow."
+	echo "==================================================================="
+
+	for trash in "${TEST_OUTPUT_DIRECTORY:-t}"/'trash directory.'*
+	do
+		test -d "$trash" || continue
+		name="${trash##*/trash directory.}"
+		echo "### still running: $name ###"
+		tail -n 20 "$results/$name.out" 2>/dev/null ||
+			echo "  (no --verbose-log trace captured)"
+	done
+
+	echo "### processes, sorted by resident set size ###"
+	ps -eo pid,ppid,etimes,rss,stat,args 2>/dev/null | sort -k 4 -nr | head -n 40
+
+	echo "### memory and load ###"
+	free -m 2>/dev/null || head -n 3 /proc/meminfo 2>/dev/null
+	cat /proc/loadavg 2>/dev/null
+
+	echo "### kernel log tail (the OOM killer records here) ###"
+	dmesg 2>/dev/null | tail -n 30 || echo "  (dmesg unavailable)"
+}
+
 begin_group "CI setup via $(basename $0)"
 
 # Set 'exit on error' for all CI scripts to let the caller know that

ci/run-build-and-tests.sh

@@ -50,6 +50,25 @@ linux-reftable|linux-reftable-leaks|osx-reftable)
 
 esac
 
+# A hung test produces no output, so the job idles until the CI platform's
+# hard timeout with no hint which test stalled. Start a watchdog that dumps
+# diagnostics (see dump_stalled_test_state) if the run makes no progress for
+# GIT_TEST_STALL_TIMEOUT seconds. Count down in short steps rather than one
+# long "sleep" so that when we stop it on the happy path, no long-lived sleep
+# is left holding our output open and delaying the job from ending.
+stall_timeout="${GIT_TEST_STALL_TIMEOUT:-7200}"
+{
+	set +x
+	remaining=$stall_timeout
+	while test 0 -lt "$remaining"
+	do
+		sleep 5 || exit
+		remaining=$((remaining - 5))
+	done
+	dump_stalled_test_state "$stall_timeout"
+} &
+stall_watchdog_pid=$!
+
 case "$jobname" in
 *-meson)
 	group "Configure" meson setup build . \
@@ -72,5 +91,7 @@ case "$jobname" in
 	;;
 esac
 
+kill "$stall_watchdog_pid" 2>/dev/null || :
+
 check_unignored_build_artifacts
 save_good_tree