diff --git a/.github/workflows/flake-proof.yml b/.github/workflows/flake-proof.yml index 27cea7b..79864b8 100644 --- a/.github/workflows/flake-proof.yml +++ b/.github/workflows/flake-proof.yml @@ -200,6 +200,9 @@ jobs: matrix: include: ${{ fromJSON(needs.plan.outputs.matrix) }} steps: + - name: Note when the job started + run: echo "JOB_STARTED=$(date +%s)" >> "$GITHUB_ENV" + - name: Check out the tested revision uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -266,10 +269,10 @@ jobs: npm run "$script" done - - name: Install the keyring on GitHub's runner + - name: Install the virtual display, the session bus and the keyring on GitHub's runner if: inputs.display && runner.os == 'Linux' && runner.environment == 'github-hosted' timeout-minutes: 5 - run: sudo apt-get update && sudo apt-get install -y --no-install-recommends gnome-keyring + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends xvfb xauth dbus-daemon dbus-bin gnome-keyring - name: Run the tests env: @@ -280,17 +283,25 @@ jobs: RUNS: ${{ matrix.runs }} TARGET: ${{ matrix.target }} JOB: ${{ matrix.job }} + JOB_MINUTES: ${{ inputs.timeout-minutes }} run: | set -uo pipefail read -r -a filters <<< "$FILTER" read -r -a reports <<< "$REPORTS" command=(npm run "$SCRIPT" -- ${filters[@]+"${filters[@]}"}) keyring=false + xvfb= + trap 'if [ -n "$xvfb" ]; then kill "$xvfb" 2>/dev/null || true; fi' EXIT if [ "$VIRTUAL_DISPLAY" = true ] && [ "$RUNNER_OS" = Linux ]; then command -v gnome-keyring-daemon >/dev/null || { echo "::error::gnome-keyring isn't installed on this runner; display mode needs it."; exit 1; } + for program in Xvfb xauth dbus-run-session; do + command -v "$program" >/dev/null || { echo "::error::This runner has no $program, which display mode runs each run under."; exit 1; } + done keyring=true - command=(xvfb-run --auto-servernum --server-args='-screen 0 1920x1080x24' dbus-run-session -- sh -c 'echo | XDG_DATA_HOME="$KEYRING/data" gnome-keyring-daemon --unlock --components=secrets --control-directory="$KEYRING/control" >/dev/null && exec "$@"' sh "${command[@]}") + command=(dbus-run-session -- sh -c 'echo | XDG_DATA_HOME="$KEYRING/data" gnome-keyring-daemon --unlock --components=secrets --control-directory="$KEYRING/control" >/dev/null && exec "$@"' sh "${command[@]}") fi + # Each run may take an even share of the job's time still left, keeping three minutes for the artifact, so a run that hangs is named and counted as failed + # before the job's timeout-minutes end the job unnamed. A watchdog in this shell bounds it, on every runner, since not every runner has GNU timeout. mkdir -p _build/flake-proof passed=0 failed=0 @@ -301,15 +312,66 @@ jobs: mkdir -p "$KEYRING/data/keyrings" mkdir -m 700 "$KEYRING/control" printf login > "$KEYRING/data/keyrings/default" + # Each run gets a new Xvfb on a display it picks itself with -displayfd, so jobs sharing a machine never race for one, + # open only to a new cookie. Xvfb takes every cookie in its -auth file whatever display the entry names, + # so the cookie goes in under :0 before the display is known, and under the display Xvfb picks once it is. + export XAUTHORITY="$KEYRING/xauthority" + touch "$XAUTHORITY" + cookie=$(od -An -N16 -tx1 /dev/urandom | tr -d ' \n') + xauth -q add :0 . "$cookie" + mkfifo "$KEYRING/display" + Xvfb -displayfd 3 -auth "$XAUTHORITY" -screen 0 1920x1080x24 -nolisten tcp 3> "$KEYRING/display" & + xvfb=$! + display= + read -r -t 30 display < "$KEYRING/display" || true + if [ -z "$display" ]; then + echo "::error::Xvfb didn't start for run $run, so the run has no display." + exit 1 + fi + xauth -q add ":$display" . "$cookie" + export DISPLAY=":$display" + fi + limit=$(( (JOB_MINUTES * 60 - ($(date +%s) - JOB_STARTED) - 180) / (RUNS - run + 1) - 30 )) + if [ "$limit" -lt 60 ]; then + echo "::error::The job's timeout-minutes leave run $run $limit seconds; raise timeout-minutes or lower runs." + exit 1 fi - if "${command[@]}" > "$log" 2>&1; then + reached="$RUNNER_TEMP/flake-proof-limit-$JOB-$run" + status=0 + "${command[@]}" > "$log" 2>&1 & + pid=$! + # The watchdog sends TERM at the limit and KILL 30 seconds later; its own sleeps go when it is stopped, so nothing outlives the run. + ( + trap 'kill "$nap" 2>/dev/null; exit 0' TERM + sleep "$limit" & nap=$! + wait "$nap" + touch "$reached" + kill -TERM "$pid" 2>/dev/null + sleep 30 & nap=$! + wait "$nap" + kill -KILL "$pid" 2>/dev/null + ) > /dev/null 2>&1 & + watchdog=$! + wait "$pid" || status=$? + kill -TERM "$watchdog" 2>/dev/null || true + wait "$watchdog" 2>/dev/null || true + if [ "$status" = 0 ] && [ ! -e "$reached" ]; then passed=$(( passed + 1 )) echo "Run $run passed." + elif [ -e "$reached" ]; then + failed=$(( failed + 1 )) + echo "::error::Run $run reached its limit of $limit seconds and was stopped; its log is kept in the job's artifact." + tail -n 60 "$log" else failed=$(( failed + 1 )) echo "::error::Run $run failed; its log is kept in the job's artifact." tail -n 60 "$log" fi + if [ -n "$xvfb" ]; then + kill "$xvfb" 2>/dev/null || true + wait "$xvfb" 2>/dev/null || true + xvfb= + fi for folder in ${reports[@]+"${reports[@]}"}; do if [ -e "$folder" ]; then mkdir -p "_build/flake-proof/run-$run/$(dirname "$folder")" diff --git a/README.md b/README.md index c0d37fc..3184eea 100644 --- a/README.md +++ b/README.md @@ -385,13 +385,14 @@ jobs: - A repository proves an end-to-end test with four more inputs, which its caller adds to its own `workflow_dispatch` and passes on: - `script`: the npm script each run runs, such as `test:e2e`. - `prepare`: npm scripts each job runs once, after `npm run build` and before the runs, separated by spaces, such as an app's install and build. They run with the build's token and with no package's install scripts. - - `display`: a boolean, off by default. On Linux it runs each run under `xvfb-run`, on a 1920×1080 screen, and `dbus-run-session`, for a test that opens windows, with an unlocked `gnome-keyring` secret service on a new empty keyring, for a test whose app keeps a secret; it installs `gnome-keyring` on a GitHub-hosted runner and fails on a self-hosted one without it. Windows and macOS runners have a display and a secret store of their own. + - `display`: a boolean, off by default. On Linux it starts a new Xvfb for each run, on a 1920×1080 screen, on a free display Xvfb picks itself with `-displayfd`, so jobs sharing a machine don't race for one, open only to clients holding a new `xauth` cookie, which the run finds through `XAUTHORITY`, and runs the run inside `dbus-run-session`, for a test that opens windows, with an unlocked `gnome-keyring` secret service on a new empty keyring, for a test whose app keeps a secret. It installs `xvfb`, `xauth`, `dbus-daemon`, `dbus-bin` and `gnome-keyring` on a GitHub-hosted runner, fails on a self-hosted one that lacks Xvfb, `xauth`, `dbus-run-session` or `gnome-keyring-daemon`, and fails when Xvfb names no display within 30 seconds: in the trials Xvfb named its display in under a second, so 30 seconds only guards against a hung Xvfb, with a wide margin. Windows and macOS runners have a display and a secret store of their own. - `reports`: the folders in the checkout the script writes its report and results to, separated by spaces, such as `_build/ui/report _build/ui/results`. After each run, each folder that exists moves into the artifact as `run-/`, so every run keeps its own report beside its log; the plan job fails on a folder outside the checkout, inside `_build/flake-proof`, or holding it, such as `_build`. For one end-to-end spec, the caller passes the spec and its runner's options in `filter`, which go after `--`: with `script` `test:ui`, `filter` `tests/e2e/window.spec.ts:42 --retries 0` runs `npm run test:ui -- tests/e2e/window.spec.ts:42 --retries 0`, so the runner retries nothing and a flaky run fails. Before it installs anything, each job checks that `script` and every `prepare` script are npm scripts in the package's `package.json`, and fails otherwise. - A test job may take `timeout-minutes`, 120 by default and up to 360, which a caller with a slow suite raises. + Each run may take an even share of the job's time still left, keeping three minutes for the artifact, so a run that hangs is stopped, named with `::error` as "Run reached its limit of seconds" and counted as failed before the job's timeout ends the job without naming it; the job fails at once when a run's share is under a minute. A watchdog in the job's own shell bounds each run on every runner, sending `TERM` at the limit and `KILL` 30 seconds later, since not every runner has GNU coreutils' `timeout`. - The jobs install from the registries and run on the runners [Shared settings](#shared-settings) names; a private repository's jobs read the organization's private packages from GitHub's registry with the run's own token, which `packages: read` allows, and a public repository's jobs hand npm no token. - The caller grants `contents: read` and `packages: read`, since a called job gets no permission its caller didn't grant.