Start the flake proof's display on a display Xvfb picks, behind a cookie

.github/workflows/flake-proof.yml

@@ -200,6 +200,9 @@ jobs:
       matrix:
         include: ${{ fromJSON(needs.plan.outputs.matrix) }}
     steps:
+      - name: Note when the job started
+        run: echo "JOB_STARTED=$(date +%s)" >> "$GITHUB_ENV"
+
       - name: Check out the tested revision
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
@@ -266,10 +269,10 @@ jobs:
             npm run "$script"
           done
 
-      - name: Install the keyring on GitHub's runner
+      - name: Install the virtual display, the session bus and the keyring on GitHub's runner
         if: inputs.display && runner.os == 'Linux' && runner.environment == 'github-hosted'
         timeout-minutes: 5
-        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends gnome-keyring
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends xvfb xauth dbus-daemon dbus-bin gnome-keyring
 
       - name: Run the tests
         env:
@@ -280,17 +283,25 @@ jobs:
           RUNS: ${{ matrix.runs }}
           TARGET: ${{ matrix.target }}
           JOB: ${{ matrix.job }}
+          JOB_MINUTES: ${{ inputs.timeout-minutes }}
         run: |
           set -uo pipefail
           read -r -a filters <<< "$FILTER"
           read -r -a reports <<< "$REPORTS"
           command=(npm run "$SCRIPT" -- ${filters[@]+"${filters[@]}"})
           keyring=false
+          xvfb=
+          trap 'if [ -n "$xvfb" ]; then kill "$xvfb" 2>/dev/null || true; fi' EXIT
           if [ "$VIRTUAL_DISPLAY" = true ] && [ "$RUNNER_OS" = Linux ]; then
             command -v gnome-keyring-daemon >/dev/null || { echo "::error::gnome-keyring isn't installed on this runner; display mode needs it."; exit 1; }
+            for program in Xvfb xauth dbus-run-session; do
+              command -v "$program" >/dev/null || { echo "::error::This runner has no $program, which display mode runs each run under."; exit 1; }
+            done
             keyring=true
-            command=(xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' dbus-run-session -- sh -c 'echo | XDG_DATA_HOME="$KEYRING/data" gnome-keyring-daemon --unlock --components=secrets --control-directory="$KEYRING/control" >/dev/null && exec "$@"' sh "${command[@]}")
+            command=(dbus-run-session -- sh -c 'echo | XDG_DATA_HOME="$KEYRING/data" gnome-keyring-daemon --unlock --components=secrets --control-directory="$KEYRING/control" >/dev/null && exec "$@"' sh "${command[@]}")
           fi
+          # Each run may take an even share of the job's time still left, keeping three minutes for the artifact, so a run that hangs is named and counted as failed
+          # before the job's timeout-minutes end the job unnamed. A watchdog in this shell bounds it, on every runner, since not every runner has GNU timeout.
           mkdir -p _build/flake-proof
           passed=0
           failed=0
@@ -301,15 +312,66 @@ jobs:
               mkdir -p "$KEYRING/data/keyrings"
               mkdir -m 700 "$KEYRING/control"
               printf login > "$KEYRING/data/keyrings/default"
+              # Each run gets a new Xvfb on a display it picks itself with -displayfd, so jobs sharing a machine never race for one,
+              # open only to a new cookie. Xvfb takes every cookie in its -auth file whatever display the entry names,
+              # so the cookie goes in under :0 before the display is known, and under the display Xvfb picks once it is.
+              export XAUTHORITY="$KEYRING/xauthority"
+              touch "$XAUTHORITY"
+              cookie=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n')
+              xauth -q add :0 . "$cookie"
+              mkfifo "$KEYRING/display"
+              Xvfb -displayfd 3 -auth "$XAUTHORITY" -screen 0 1920x1080x24 -nolisten tcp 3> "$KEYRING/display" &
+              xvfb=$!
+              display=
+              read -r -t 30 display < "$KEYRING/display" || true
+              if [ -z "$display" ]; then
+                echo "::error::Xvfb didn't start for run $run, so the run has no display."
+                exit 1
+              fi
+              xauth -q add ":$display" . "$cookie"
+              export DISPLAY=":$display"
+            fi
+            limit=$(( (JOB_MINUTES * 60 - ($(date +%s) - JOB_STARTED) - 180) / (RUNS - run + 1) - 30 ))
+            if [ "$limit" -lt 60 ]; then
+              echo "::error::The job's timeout-minutes leave run $run $limit seconds; raise timeout-minutes or lower runs."
+              exit 1
             fi
-            if "${command[@]}" > "$log" 2>&1; then
+            reached="$RUNNER_TEMP/flake-proof-limit-$JOB-$run"
+            status=0
+            "${command[@]}" > "$log" 2>&1 &
+            pid=$!
+            # The watchdog sends TERM at the limit and KILL 30 seconds later; its own sleeps go when it is stopped, so nothing outlives the run.
+            (
+              trap 'kill "$nap" 2>/dev/null; exit 0' TERM
+              sleep "$limit" & nap=$!
+              wait "$nap"
+              touch "$reached"
+              kill -TERM "$pid" 2>/dev/null
+              sleep 30 & nap=$!
+              wait "$nap"
+              kill -KILL "$pid" 2>/dev/null
+            ) > /dev/null 2>&1 &
+            watchdog=$!
+            wait "$pid" || status=$?
+            kill -TERM "$watchdog" 2>/dev/null || true
+            wait "$watchdog" 2>/dev/null || true
+            if [ "$status" = 0 ] && [ ! -e "$reached" ]; then
               passed=$(( passed + 1 ))
               echo "Run $run passed."
+            elif [ -e "$reached" ]; then
+              failed=$(( failed + 1 ))
+              echo "::error::Run $run reached its limit of $limit seconds and was stopped; its log is kept in the job's artifact."
+              tail -n 60 "$log"
             else
               failed=$(( failed + 1 ))
               echo "::error::Run $run failed; its log is kept in the job's artifact."
               tail -n 60 "$log"
             fi
+            if [ -n "$xvfb" ]; then
+              kill "$xvfb" 2>/dev/null || true
+              wait "$xvfb" 2>/dev/null || true
+              xvfb=
+            fi
             for folder in ${reports[@]+"${reports[@]}"}; do
               if [ -e "$folder" ]; then
                 mkdir -p "_build/flake-proof/run-$run/$(dirname "$folder")"

README.md

@@ -385,13 +385,14 @@ jobs:
 - A repository proves an end-to-end test with four more inputs, which its caller adds to its own `workflow_dispatch` and passes on:
   - `script`: the npm script each run runs, such as `test:e2e`.
   - `prepare`: npm scripts each job runs once, after `npm run build` and before the runs, separated by spaces, such as an app's install and build. They run with the build's token and with no package's install scripts.
-  - `display`: a boolean, off by default. On Linux it runs each run under `xvfb-run`, on a 1920×1080 screen, and `dbus-run-session`, for a test that opens windows, with an unlocked `gnome-keyring` secret service on a new empty keyring, for a test whose app keeps a secret; it installs `gnome-keyring` on a GitHub-hosted runner and fails on a self-hosted one without it. Windows and macOS runners have a display and a secret store of their own.
+  - `display`: a boolean, off by default. On Linux it starts a new Xvfb for each run, on a 1920×1080 screen, on a free display Xvfb picks itself with `-displayfd`, so jobs sharing a machine don't race for one, open only to clients holding a new `xauth` cookie, which the run finds through `XAUTHORITY`, and runs the run inside `dbus-run-session`, for a test that opens windows, with an unlocked `gnome-keyring` secret service on a new empty keyring, for a test whose app keeps a secret. It installs `xvfb`, `xauth`, `dbus-daemon`, `dbus-bin` and `gnome-keyring` on a GitHub-hosted runner, fails on a self-hosted one that lacks Xvfb, `xauth`, `dbus-run-session` or `gnome-keyring-daemon`, and fails when Xvfb names no display within 30 seconds: in the trials Xvfb named its display in under a second, so 30 seconds only guards against a hung Xvfb, with a wide margin. Windows and macOS runners have a display and a secret store of their own.
   - `reports`: the folders in the checkout the script writes its report and results to, separated by spaces, such as `_build/ui/report _build/ui/results`. After each run, each folder that exists moves into the artifact as `run-<n>/<folder>`, so every run keeps its own report beside its log; the plan job fails on a folder outside the checkout, inside `_build/flake-proof`, or holding it, such as `_build`.
 
   For one end-to-end spec, the caller passes the spec and its runner's options in `filter`, which go after `--`: with `script` `test:ui`, `filter` `tests/e2e/window.spec.ts:42 --retries 0` runs `npm run test:ui -- tests/e2e/window.spec.ts:42 --retries 0`, so the runner retries nothing and a flaky run fails.
 
   Before it installs anything, each job checks that `script` and every `prepare` script are npm scripts in the package's `package.json`, and fails otherwise.
 - A test job may take `timeout-minutes`, 120 by default and up to 360, which a caller with a slow suite raises.
+  Each run may take an even share of the job's time still left, keeping three minutes for the artifact, so a run that hangs is stopped, named with `::error` as "Run <n> reached its limit of <seconds> seconds" and counted as failed before the job's timeout ends the job without naming it; the job fails at once when a run's share is under a minute. A watchdog in the job's own shell bounds each run on every runner, sending `TERM` at the limit and `KILL` 30 seconds later, since not every runner has GNU coreutils' `timeout`.
 - The jobs install from the registries and run on the runners [Shared settings](#shared-settings) names; a private repository's jobs read the organization's private packages from GitHub's registry with the run's own token, which `packages: read` allows, and a public repository's jobs hand npm no token.
 - The caller grants `contents: read` and `packages: read`, since a called job gets no permission its caller didn't grant.