From e1ed21da323270d7ee97ce3c42836d2fe8f2c1e7 Mon Sep 17 00:00:00 2001 From: "fizzy-macos[bot]" <347071198+fizzy-macos[bot]@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:37:25 -0500 Subject: [PATCH] ci: Windows merge group tolerates its listed known failures The Windows row's Run tests step, on merge_group only, runs through scripts/ci-known-failing.sh with .github/known-failing-windows.txt. The step passes on a failing run only when every failure is a listed `: ` pair; anything else (an unlisted test, a compile error, a runner error, no Build Summary) keeps the command's status. A listed test that passed is reported in the job summary so the list shrinks. scripts/test-ci-known-failing.sh checks it on the real output of run 40559487600 and variants, on Linux and on Windows' own awk. Agent-Team: macos Worker: claude Co-Authored-By: Claude Opus 5.5 --- .github/known-failing-windows.txt | 5 + .github/workflows/ci.yml | 19 +- scripts/ci-known-failing.sh | 178 ++++++++++++++++ .../ci-known-failing/windows-crash.txt | 197 ++++++++++++++++++ scripts/test-ci-known-failing.sh | 92 ++++++++ 5 files changed, 490 insertions(+), 1 deletion(-) create mode 100644 .github/known-failing-windows.txt create mode 100755 scripts/ci-known-failing.sh create mode 100644 scripts/fixtures/ci-known-failing/windows-crash.txt create mode 100755 scripts/test-ci-known-failing.sh diff --git a/.github/known-failing-windows.txt b/.github/known-failing-windows.txt new file mode 100644 index 000000000..361586b79 --- /dev/null +++ b/.github/known-failing-windows.txt @@ -0,0 +1,5 @@ +# Tests that fail intermittently on Windows, main included, and so may fail Windows' merge-group +# run without failing the job (scripts/ci-known-failing.sh). One per line, as the runner names +# it: `: | work#N`, the issue that owns the fix. A listed test that +# passes is reported in the job summary: remove its line then. +fizzy-control-server-tests: Server.test.an upgraded connection carries bytes both ways until the app closes it | work#549 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5be910324..6b569bc21 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,8 @@ jobs: - name: lint # Not `zig build lint`: the root build fetches the app's URL dependencies. run: zig run scripts/lint.zig + - name: scripts/ci-known-failing (Windows' merge-group list) + run: bash scripts/test-ci-known-failing.sh - name: scripts/zb (build lock) run: sh scripts/test-zb.sh - name: scripts/check-kickoff @@ -240,12 +242,27 @@ jobs: - name: Compile tests shell: bash run: bash scripts/ci-retry.sh zig build check --summary all + # The merge queue's known-failing wrapper on the awk it runs with there (Git Bash's). + - name: scripts/ci-known-failing on Windows + if: matrix.name == 'Windows' + shell: bash + run: bash scripts/test-ci-known-failing.sh - name: Run tests shell: bash # The step the hang is in, bounded on its own: the job's 30 minutes also cover a cold # cache, which a healthy run may spend most of before this step starts. timeout-minutes: 15 - run: bash scripts/ci-retry.sh zig build test --summary all + # In the merge queue, Windows tolerates the intermittent failures listed in + # `.github/known-failing-windows.txt`, which fail on main too and ejected passing PRs; + # anything not listed, by binary and test name, still fails (scripts/ci-known-failing.sh). + env: + KNOWN_FAILING: ${{ github.event_name == 'merge_group' && matrix.name == 'Windows' && '.github/known-failing-windows.txt' || '' }} + run: | + if [ -n "$KNOWN_FAILING" ]; then + bash scripts/ci-retry.sh bash scripts/ci-known-failing.sh "$KNOWN_FAILING" zig build test --summary all + else + bash scripts/ci-retry.sh zig build test --summary all + fi # Not on Windows: the wasm it compiles is the same from any host, Linux and macOS compile it # too, and Windows is the matrix's longest job. diff --git a/scripts/ci-known-failing.sh b/scripts/ci-known-failing.sh new file mode 100755 index 000000000..913d00d8c --- /dev/null +++ b/scripts/ci-known-failing.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Run `zig build test --summary all` (or a wrapper of it), and let it fail only for a test that is +# not on a list of known failures. +# +# ci-known-failing.sh +# +# The list holds one known failure per line, `: | work#N`, the issue +# that owns its fix; `#` starts a comment. CI uses it for Windows' merge-group run, where one +# intermittent crash that also fails on main ejected PRs from the queue (fizzyedit/work#549). +# +# Fail-closed: the command's own status stands unless every failure in its output is attributed, +# by exact binary and exact test name, to a listed line. That needs all of: +# - a `Build Summary:` line (a fetch failure, a kill or a dead build runner has none); +# - its `(K failed)` equal to the number of `run test ` steps in the summary tree that +# report a failing test, so a compile error or any other failed step fails the run; +# - every `error: '' ...` line in such a step's failure block listed for that binary, +# at least one per failing test, and no other `error:` line in the block; +# - the summary's failed, crashed and timed-out test totals equal to those steps' sum. +# The runner's generic `error:` and `failed command:` lines elsewhere (Windows prints a +# pkg-config one on every run, green or not) are neither matched nor needed. +# +# A listed test whose binary ran clean is reported as passing (a notice and a line in +# $GITHUB_STEP_SUMMARY), so the list shrinks; a tolerated failure is a warning naming its owner. +set -uo pipefail + +if [ "$#" -lt 2 ]; then + echo "usage: $0 " >&2 + exit 2 +fi +list=$1 +shift +if [ ! -r "$list" ]; then + echo "::error::known-failing list $list is missing" + exit 2 +fi + +log="$(mktemp)" +trap 'rm -f "$log"' EXIT + +"$@" 2>&1 | tee "$log" +status="${PIPESTATUS[0]}" + +summary="${GITHUB_STEP_SUMMARY:-/dev/null}" + +# Prints `::notice::`/`::warning::`/`::error::` lines and the step summary's Markdown (prefixed +# `md:`); exits 0 when the run may pass, 1 when the command's status must stand. +verdict=$(LC_ALL=C awk -v status="$status" ' + function trim(s) { sub(/^[ \t]+/, "", s); sub(/[ \t]+$/, "", s); return s } + function count(s, word, m) { + if (match(s, "[0-9]+ " word)) { m = substr(s, RSTART, RLENGTH); sub(/ .*/, "", m); return m + 0 } + return 0 + } + # The test name of a test runner error line, or "" when the line names no test. + function test_name(line, kws, n, i, at, best) { + if (substr(line, 1, 8) != "error: '\''") return "" + n = split("failed|errored|exited with|terminated with|timed out|leaked|logged", kws, "|") + best = 0 + for (i = 1; i <= n; i++) { + at = index(line, "'\'' " kws[i]) + if (at > 9 && (best == 0 || at < best)) best = at + } + return best ? substr(line, 9, best - 9) : "" + } + + # The list. + FILENAME == ARGV[1] { + line = $0; sub(/\r$/, "", line) + if (line ~ /^[ \t]*(#|$)/) next + bar = 0 + for (i = length(line); i > 0; i--) if (substr(line, i, 3) == " | ") { bar = i; break } + owner = bar ? trim(substr(line, bar + 3)) : "" + entry = bar ? trim(substr(line, 1, bar - 1)) : trim(line) + colon = index(entry, ": ") + if (owner !~ /^work#[0-9]+$/ || colon < 2) { + print "::error::" ARGV[1] ": \"" line "\" is not `: | work#N`" + bad_list = 1 + next + } + listed[entry] = owner + listed_bin[entry] = substr(entry, 1, colon - 1) + next + } + + # The command output. + { + line = $0 + sub(/\r$/, "", line) + gsub(/\033\[[0-9;]*m/, "", line) + } + line ~ /^Build Summary: / { + have_summary = 1; in_tree = 1 + steps = line; sub(/;.*/, "", steps) + tests = line; sub(/^[^;]*;?/, "", tests) + summary_failed_steps = count(steps, "failed") + summary_failed_tests = count(tests, "failed") + count(tests, "crashed") + count(tests, "timed out") + next + } + !in_tree { + if (line ~ /^[ |]*\+- /) { + leaf = line; sub(/^[ |]*\+- /, "", leaf) + leaf_bin = "" + if (leaf ~ /^run test [^ ]+/) { split(leaf, f, " "); leaf_bin = f[3] } + next + } + if (leaf_bin != "" && substr(line, 1, 7) == "error: ") { + name = test_name(line) + if (name == "") { stray[leaf_bin] = stray[leaf_bin] " " line; next } + key = leaf_bin ": " name + if (!(key in named)) { named[key] = 1; named_count[leaf_bin]++ } + } + next + } + line ~ /^[ |]*\+- run test [^ ]+/ && line !~ /\(reused\)/ { + step = line; sub(/^[ |]*\+- /, "", step) + split(step, f, " "); bin = f[3] + ran[bin] = 1 + bad = count(step, "fail") + count(step, "crash") + count(step, "timeout") + flagged = count(step, "leaks") + count(step, "error logs") + if (bad + flagged > 0) { + failing_steps++ + failing[bin] = 1 + want[bin] += bad > 0 ? bad : 1 + tree_failed_tests += bad + } + } + + END { + for (key in listed) { + if (!(listed_bin[key] in ran)) continue + if (listed_bin[key] in failing && key in named) continue + print "::notice::known failure passed: " key " (" listed[key] "); remove it from " ARGV[1] + passed_md = passed_md "md:- `" key "` (" listed[key] ")\n" + } + if (passed_md != "") printf "md:### Listed as failing on this OS, passed this run\n%s", passed_md + + if (bad_list) exit 1 + if (status == 0) exit 0 + if (!have_summary) { print "::notice::no Build Summary: the failure is not a test"; exit 1 } + if (failing_steps == 0 || summary_failed_steps != failing_steps) { + print "::notice::" summary_failed_steps " failed steps, " failing_steps + 0 " of them failing tests: not only known failures" + exit 1 + } + if (summary_failed_tests != tree_failed_tests) { + print "::notice::the summary counts " summary_failed_tests " failed tests, the steps " tree_failed_tests + 0 + exit 1 + } + ok = 1 + for (bin in failing) { + if (bin in stray) { print "::notice::" bin " failed outside a named test:" stray[bin]; ok = 0 } + if (named_count[bin] < want[bin]) { + print "::notice::" bin ": " want[bin] " failing tests, " named_count[bin] + 0 " named" + ok = 0 + } + } + for (key in named) { + if (key in listed) continue + print "::notice::not a known failure: " key + ok = 0 + } + if (!ok) exit 1 + for (key in named) { + print "::warning::known failure, not failing the job: " key " (" listed[key] ")" + failed_md = failed_md "md:- `" key "` (" listed[key] ")\n" + } + printf "md:### Known failures tolerated this run\n%s", failed_md + exit 0 + } +' "$list" "$log") +allowed=$? + +printf '%s\n' "$verdict" | grep -v '^md:' +printf '%s\n' "$verdict" | sed -n 's/^md://p' >>"$summary" + +if [ "$allowed" -ne 0 ]; then + if [ "$status" -eq 0 ]; then exit 1; fi + exit "$status" +fi +exit 0 diff --git a/scripts/fixtures/ci-known-failing/windows-crash.txt b/scripts/fixtures/ci-known-failing/windows-crash.txt new file mode 100644 index 000000000..964ce7f7b --- /dev/null +++ b/scripts/fixtures/ci-known-failing/windows-crash.txt @@ -0,0 +1,197 @@ +test ++- run test rehome-fixture + +- compile test rehome-fixture debug native + +- translate-c dvui-c + +- compile exe translate-c debug native failure +Can't find C:\Strawberry\perl\bin\pkg-config.BAT on PATH, '.' not in PATH. +error: Can't find C:\Strawberry\perl\bin\pkg-config.BAT on PATH, '.' not in PATH. + +failed command: pkg-config --list-all + +test ++- run test rehome-fixture + +- compile test rehome-fixture debug native failure +Can't find C:\Strawberry\perl\bin\pkg-config.BAT on PATH, '.' not in PATH. +error: Can't find C:\Strawberry\perl\bin\pkg-config.BAT on PATH, '.' not in PATH. + +error: Can't find C:\Strawberry\perl\bin\pkg-config.BAT on PATH, '.' not in PATH. + +failed command: pkg-config --list-all + +test ++- run test fizzy-control-server-tests 18 pass, 1 crash (19 total) +error: 'Server.test.an upgraded connection carries bytes both ways until the app closes it' exited with code 3 with stderr: + fizzy-control-server-tests: test.an upgraded connection carries bytes both ways until the app closes it still running after 60 s: a client or the server is blocked; aborting +failed command: ".\\.zig-cache\\o\\63a90a2b571b2854b481898f8a5839fe\\fizzy-control-server-tests.exe" "--cache-dir=.\\.zig-cache" --seed=0x50e66707 --listen=- + +Build Summary: 183/185 steps succeeded (1 failed); 872/891 tests passed (18 skipped, 1 crashed) +test transitive failure ++- run test fizzy-lint-tests 16 pass (16 total) 30ms MaxRSS:4M +| +- compile test fizzy-lint-tests debug native cached 46ms MaxRSS:18M ++- run test fizzy-direction-tests 7 pass (7 total) 15ms MaxRSS:3M +| +- compile test fizzy-direction-tests debug native cached 46ms MaxRSS:42M ++- run test fizzy-easing-tests 5 pass (5 total) 16ms MaxRSS:3M +| +- compile test fizzy-easing-tests debug native cached 35ms MaxRSS:43M ++- run test fizzy-layout-anchor-tests 7 pass (7 total) 14ms MaxRSS:3M +| +- compile test fizzy-layout-anchor-tests debug native cached 49ms MaxRSS:43M ++- run test fizzy-plugin-store-tests 7 pass (7 total) 15ms MaxRSS:4M +| +- compile test fizzy-plugin-store-tests debug native cached 47ms MaxRSS:42M ++- run test fizzy-paths-tests 2 pass, 3 skip (5 total) 16ms MaxRSS:3M +| +- compile test fizzy-paths-tests debug native cached 46ms MaxRSS:18M ++- run test fizzy-viz-scale-tests 6 pass (6 total) 12ms MaxRSS:3M +| +- compile test fizzy-viz-scale-tests debug native cached 49ms MaxRSS:43M ++- run test fizzy-profile-lookback-tests 6 pass (6 total) 45ms MaxRSS:3M +| +- compile test fizzy-profile-lookback-tests debug native cached 57ms MaxRSS:43M ++- run test fizzy-profile-spans-tests 4 pass (4 total) 50ms MaxRSS:7M +| +- compile test fizzy-profile-spans-tests debug native cached 51ms MaxRSS:18M ++- run test fizzy-profile-causes-tests 2 pass (2 total) 28ms MaxRSS:3M +| +- compile test fizzy-profile-causes-tests debug native cached 56ms MaxRSS:42M ++- run test fizzy-shell-env-tests 3 pass (3 total) 26ms MaxRSS:4M +| +- compile test fizzy-shell-env-tests debug native cached 73ms MaxRSS:18M ++- run test fizzy-darwin-spawn-tests 0 pass, 1 skip (1 total) 20ms MaxRSS:3M +| +- compile test fizzy-darwin-spawn-tests debug native cached 60ms MaxRSS:42M ++- run test fizzy-secrets-tests 1 pass (1 total) 26ms MaxRSS:6M +| +- compile test fizzy-secrets-tests debug native cached 68ms MaxRSS:18M ++- run test fizzy-lsp-protocol-tests 4 pass (4 total) 29ms MaxRSS:4M +| +- compile test fizzy-lsp-protocol-tests debug native cached 64ms MaxRSS:18M ++- run test fizzy-lsp-uri-tests 2 pass, 1 skip (3 total) 42ms MaxRSS:3M +| +- compile test fizzy-lsp-uri-tests debug native cached 49ms MaxRSS:43M ++- run test fizzy-settings-plugins-zon-tests 24 pass (24 total) 27ms MaxRSS:4M +| +- compile test fizzy-settings-plugins-zon-tests debug native cached 41ms MaxRSS:43M ++- run test fizzy-sdk-manifest-tests 8 pass (8 total) 66ms MaxRSS:4M +| +- compile test fizzy-sdk-manifest-tests debug native cached 54ms MaxRSS:18M ++- run test fizzy-sdk-wikilink-tests 18 pass (18 total) 23ms MaxRSS:3M +| +- compile test fizzy-sdk-wikilink-tests debug native cached 55ms MaxRSS:43M ++- run test fizzy-textcore-tests 94 pass (94 total) 293ms MaxRSS:4M +| +- compile test fizzy-textcore-tests debug native cached 54ms MaxRSS:43M ++- run test fizzy-text-doc-state-tests 2 pass (2 total) 19ms MaxRSS:3M +| +- compile test fizzy-text-doc-state-tests debug native cached 54ms MaxRSS:18M ++- run test fizzy-profile-tests 2 pass (2 total) 16ms MaxRSS:3M +| +- compile test fizzy-profile-tests debug native cached 49ms MaxRSS:43M ++- run test fizzy-quiet-tests 1 pass (1 total) 23ms MaxRSS:3M +| +- compile test fizzy-quiet-tests debug native cached 51ms MaxRSS:18M ++- run test fizzy-md-html-images-tests 11 pass (11 total) 19ms MaxRSS:3M +| +- compile test fizzy-md-html-images-tests debug native cached 49ms MaxRSS:43M ++- run test fizzy-md-url-join-tests 4 pass (4 total) 17ms MaxRSS:3M +| +- compile test fizzy-md-url-join-tests debug native cached 52ms MaxRSS:18M ++- run test fizzy-md-image-format-tests 5 pass (5 total) 15ms MaxRSS:3M +| +- compile test fizzy-md-image-format-tests debug native cached 36ms MaxRSS:43M ++- run test fizzy-md-block-heights-tests 50 pass (50 total) 18ms MaxRSS:4M +| +- compile test fizzy-md-block-heights-tests debug native cached 51ms MaxRSS:18M ++- run test fizzy-chat-composer-tests 5 pass (5 total) 14ms MaxRSS:3M +| +- compile test fizzy-chat-composer-tests debug native cached 59ms MaxRSS:43M ++- run test fizzy-md-code-language-tests 5 pass (5 total) 15ms MaxRSS:3M +| +- compile test fizzy-md-code-language-tests debug native cached 48ms MaxRSS:18M ++- run test fizzy-md-supermd-tests 8 pass (8 total) 17ms MaxRSS:3M +| +- compile test fizzy-md-supermd-tests debug native cached 93ms MaxRSS:18M ++- run test fizzy-reveal-tests 4 pass (4 total) 32ms MaxRSS:3M +| +- compile test fizzy-reveal-tests debug native cached 53ms MaxRSS:43M ++- run test fizzy-form-factor-tests 9 pass (9 total) 16ms MaxRSS:3M +| +- compile test fizzy-form-factor-tests debug native cached 41ms MaxRSS:18M ++- run test fizzy-crossfade-tests 8 pass (8 total) 22ms MaxRSS:3M +| +- compile test fizzy-crossfade-tests debug native cached 69ms MaxRSS:18M ++- run test fizzy-frame-clock-tests 3 pass (3 total) 18ms MaxRSS:3M +| +- compile test fizzy-frame-clock-tests debug native cached 41ms MaxRSS:42M ++- run test fizzy-folder-events-tests 31 pass, 3 skip (34 total) 272ms MaxRSS:6M +| +- compile test fizzy-folder-events-tests debug native cached 50ms MaxRSS:18M ++- run test fizzy-install-location-tests 10 pass (10 total) 22ms MaxRSS:3M +| +- compile test fizzy-install-location-tests debug native cached 53ms MaxRSS:43M ++- run test fizzy-bounded-join-tests 2 pass (2 total) 115ms MaxRSS:3M +| +- compile test fizzy-bounded-join-tests debug native cached 47ms MaxRSS:43M ++- run test fizzy-recent-commands-tests 2 pass (2 total) 18ms MaxRSS:3M +| +- compile test fizzy-recent-commands-tests debug native cached 59ms MaxRSS:43M ++- run test fizzy-plugin-folders-tests 3 pass (3 total) 21ms MaxRSS:5M +| +- compile test fizzy-plugin-folders-tests debug native cached 55ms MaxRSS:18M ++- run test fizzy-float-rules-tests 13 pass (13 total) 27ms MaxRSS:3M +| +- compile test fizzy-float-rules-tests debug native cached 39ms MaxRSS:18M ++- run test fizzy-placement-tests 3 pass (3 total) 22ms MaxRSS:3M +| +- compile test fizzy-placement-tests debug native cached 75ms MaxRSS:18M ++- run test fizzy-state-delta-tests 9 pass (9 total) 535ms MaxRSS:13M +| +- compile test fizzy-state-delta-tests debug native cached 64ms MaxRSS:18M ++- run test fizzy-state-pack-tests 3 pass (3 total) 265ms MaxRSS:21M +| +- compile test fizzy-state-pack-tests debug native cached 53ms MaxRSS:43M ++- run test fizzy-diff-tests 21 pass (21 total) 139ms MaxRSS:4M +| +- compile test fizzy-diff-tests debug native cached 42ms MaxRSS:43M ++- run test fizzy-history-transport-tests 12 pass (12 total) 13ms MaxRSS:3M +| +- compile test fizzy-history-transport-tests debug native cached 56ms MaxRSS:18M ++- run test fizzy-feed-lift-tests 4 pass (4 total) 13ms MaxRSS:3M +| +- compile test fizzy-feed-lift-tests debug native cached 76ms MaxRSS:43M ++- run test fizzy-dome-tests 7 pass (7 total) 19ms MaxRSS:3M +| +- compile test fizzy-dome-tests debug native cached 47ms MaxRSS:43M ++- run test fizzy-control-json-tests 3 pass (3 total) 16ms MaxRSS:4M +| +- compile test fizzy-control-json-tests debug native cached 50ms MaxRSS:43M ++- run test fizzy-mcp-names-tests 4 pass (4 total) 26ms MaxRSS:3M +| +- compile test fizzy-mcp-names-tests debug native cached 62ms MaxRSS:18M ++- run test fizzy-control-server-tests 18 pass, 1 crash (19 total) +| +- compile test fizzy-control-server-tests debug native cached 76ms MaxRSS:18M ++- run test fizzy-control-screen-diff-tests 4 pass (4 total) 18ms MaxRSS:4M +| +- compile test fizzy-control-screen-diff-tests debug native cached 57ms MaxRSS:18M ++- run test fizzy-glass-look-tests 12 pass (12 total) 20ms MaxRSS:3M +| +- compile test fizzy-glass-look-tests debug native cached 54ms MaxRSS:42M ++- run test fizzy-material-tests 3 pass (3 total) 19ms MaxRSS:3M +| +- compile test fizzy-material-tests debug native cached 56ms MaxRSS:43M ++- run test fizzy-orb-geometry-tests 5 pass (5 total) 16ms MaxRSS:3M +| +- compile test fizzy-orb-geometry-tests debug native cached 50ms MaxRSS:43M ++- run test fizzy-liquid-tests 5 pass (5 total) 24ms MaxRSS:3M +| +- compile test fizzy-liquid-tests debug native cached 68ms MaxRSS:43M ++- run test fizzy-menu-order-tests 1 pass (1 total) 14ms MaxRSS:3M +| +- compile test fizzy-menu-order-tests debug native cached 49ms MaxRSS:18M ++- run test fizzy-tape-tests 79 pass (79 total) 1s MaxRSS:8M +| +- compile test fizzy-tape-tests debug native cached 58ms MaxRSS:18M ++- run test fizzy-issues-url-tests 1 pass (1 total) 17ms MaxRSS:3M +| +- compile test fizzy-issues-url-tests debug native cached 48ms MaxRSS:42M ++- run test fizzy-bundle-id-tests 1 pass (1 total) 12ms MaxRSS:3M +| +- compile test fizzy-bundle-id-tests debug native cached 51ms MaxRSS:42M ++- run test backend-window-layout-tests 20 pass (20 total) 17ms MaxRSS:3M +| +- compile test backend-window-layout-tests debug native cached 60ms MaxRSS:43M ++- run test backend-titlebar-tests 3 pass (3 total) 14ms MaxRSS:3M +| +- compile test backend-titlebar-tests debug native cached 40ms MaxRSS:43M ++- run test backend-materials-tests 5 pass (5 total) 14ms MaxRSS:3M +| +- compile test backend-materials-tests debug native cached 50ms MaxRSS:42M ++- run test backend-blur-region-tests 5 pass (5 total) 17ms MaxRSS:3M +| +- compile test backend-blur-region-tests debug native cached 54ms MaxRSS:18M ++- run test backend-viewport-map-tests 14 pass (14 total) 21ms MaxRSS:3M +| +- compile test backend-viewport-map-tests debug native cached 42ms MaxRSS:42M ++- run test backend-health-tests 5 pass (5 total) 87ms MaxRSS:4M +| +- compile test backend-health-tests debug native cached 60ms MaxRSS:43M ++- run test backend-touch-slots-tests 5 pass (5 total) 13ms MaxRSS:3M +| +- compile test backend-touch-slots-tests debug native cached 51ms MaxRSS:43M ++- run test backend-winsys-tests 60 pass (60 total) 1s MaxRSS:4M +| +- compile test backend-winsys-tests debug native cached 48ms MaxRSS:43M ++- run test backend-gpu-tests 7 pass (7 total) 16ms MaxRSS:4M +| +- compile test backend-gpu-tests debug native cached 58ms MaxRSS:18M ++- run test fizzy-singleton-unix-tests 0 pass, 1 skip (1 total) 53ms MaxRSS:4M +| +- compile test fizzy-singleton-unix-tests debug native cached 81ms MaxRSS:18M ++- run test fizzy-crash-tests 16 pass (16 total) 16ms MaxRSS:5M +| +- compile test fizzy-crash-tests debug native cached 59ms MaxRSS:18M ++- run test fizzy-keymap-tests 31 pass (31 total) 28ms MaxRSS:4M +| +- compile test fizzy-keymap-tests debug native cached 42ms MaxRSS:42M ++- run test fizzy-fuzzy-tests 12 pass (12 total) 33ms MaxRSS:3M +| +- compile test fizzy-fuzzy-tests debug native cached 40ms MaxRSS:43M ++- run test fizzy-file-table-tests 47 pass (47 total) 29s MaxRSS:403M +| +- compile test fizzy-file-table-tests debug native cached 50ms MaxRSS:43M ++- run test fizzy-native-transport-tests 21 pass (21 total) 114ms MaxRSS:11M +| +- compile test fizzy-native-transport-tests debug native cached 39ms MaxRSS:42M ++- run test fizzy-work-tests 4 pass (4 total) 173ms MaxRSS:3M +| +- compile test fizzy-work-tests debug native cached 53ms MaxRSS:43M ++- run test fizzy-remote-tests 27 pass, 9 skip (36 total) 126ms MaxRSS:5M +| +- compile test fizzy-remote-tests debug native cached 61ms MaxRSS:43M +| +- options cached +| +- compile exe fake-fizzy debug native cached 53ms MaxRSS:43M ++- run test rehome-fixture 1 pass (1 total) 83ms MaxRSS:9M + +- compile test rehome-fixture debug native success 9s MaxRSS:304M + +- WriteFile freetype success + +- compile lib freetype debug native success 10s MaxRSS:34M + +- WriteFile . success + +- compile lib tree-sitter debug native success 4s MaxRSS:22M + +- compile lib freetype debug native (reused) + +- compile lib tree-sitter debug native (reused) + +- translate-c dvui-c success 540ms + | +- compile exe translate-c debug native success 23s MaxRSS:882M + | +- options success + | +- options success + | +- options success + +- options success + +- options success + +error: the following build command exited with code 1: diff --git a/scripts/test-ci-known-failing.sh b/scripts/test-ci-known-failing.sh new file mode 100755 index 000000000..92625c758 --- /dev/null +++ b/scripts/test-ci-known-failing.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash +# Checks scripts/ci-known-failing.sh on the real Windows output of run 40559487600 (a listed crash +# in fizzy-control-server-tests) and variants of it: only a failure that is listed, by binary and +# test name, may pass; everything else keeps the command's status. +set -uo pipefail +cd "$(dirname "$0")" || exit 1 +T=$(mktemp -d) +trap 'rm -f -r "$T"' EXIT + +crash=fixtures/ci-known-failing/windows-crash.txt +listed='fizzy-control-server-tests: Server.test.an upgraded connection carries bytes both ways until the app closes it' +printf '# known failures\n\n%s | work#549\n' "$listed" >"$T/list" + +failures=0 +# expect [list]: run the wrapper on a command that prints +# the log and exits with the status. +expect() { + local name=$1 want=$2 log=$3 code=$4 list=${5:-$T/list} got + : >"$T/summary" + GITHUB_STEP_SUMMARY=$T/summary bash ci-known-failing.sh "$list" sh -c 'cat "$1"; exit "$2"' sh "$log" "$code" >"$T/out" 2>&1 + got=$? + if [ "$got" -ne "$want" ]; then + echo "FAIL: $name: exit $got, want $want" + grep '^::' "$T/out" + failures=$((failures + 1)) + else + echo "ok: $name" + fi +} +# variant : the crash log, edited. +variant() { sed "$2" "$crash" >"$T/$1"; } + +expect "the listed crash passes" 0 "$crash" 1 +grep -q "^::warning::known failure, not failing the job: $listed (work#549)" "$T/out" || + { echo "FAIL: no warning naming the tolerated failure"; failures=$((failures + 1)); } +grep -q "Known failures tolerated" "$T/summary" || + { echo "FAIL: the step summary lacks the tolerated failure"; failures=$((failures + 1)); } + +variant other-name.log "s/an upgraded connection carries bytes/a different test/" +expect "an unlisted test in the same binary fails" 1 "$T/other-name.log" 1 + +variant other-bin.log "s/fizzy-control-server-tests/fizzy-control-json-tests/g" +expect "the listed name in another binary fails" 1 "$T/other-bin.log" 1 + +# A second test of the same binary fails beside the listed one. +variant second.log "/^error: 'Server.test.an upgraded/i\\ +error: 'Server.test.another one' failed: +s/18 pass, 1 crash (19 total)/17 pass, 1 fail, 1 crash (19 total)/ +s/(18 skipped, 1 crashed)/(18 skipped, 1 failed, 1 crashed)/" +expect "a second failure beside the listed one fails" 1 "$T/second.log" 1 + +# The same counts with only the listed test named: a failure the parser cannot name. +variant unnamed.log "s/18 pass, 1 crash (19 total)/17 pass, 1 fail, 1 crash (19 total)/ +s/(18 skipped, 1 crashed)/(18 skipped, 1 failed, 1 crashed)/" +expect "a failing test with no name fails" 1 "$T/unnamed.log" 1 + +variant runner.log "/^failed command: .*control-server-tests/i\\ +error: the test runner failed to respond for 60 seconds" +expect "a runner error in the listed binary's block fails" 1 "$T/runner.log" 1 + +variant compile.log "s/(1 failed)/(2 failed)/ +s/^+- run test fizzy-lint-tests .*/+- run test fizzy-lint-tests 3 errors/" +expect "a compile error beside the listed crash fails" 1 "$T/compile.log" 1 + +variant leak.log "s/(19 total)\$/(19 total); 1 leaks/ +/^failed command: .*control-server-tests/i\\ +error: 'Server.test.leaky' leaked 1 allocations:" +expect "a leak in an unlisted test fails" 1 "$T/leak.log" 1 + +grep -v '^Build Summary' "$crash" >"$T/no-summary.log" +expect "no Build Summary fails" 1 "$T/no-summary.log" 1 + +expect "the command's own status stands" 3 "$T/other-name.log" 3 + +printf '%s\n' "$listed" >"$T/ownerless" +expect "a list line with no owner fails" 1 "$crash" 1 "$T/ownerless" +expect "a list line with no owner fails a green run too" 1 "$T/no-summary.log" 0 "$T/ownerless" + +variant green.log "/^test\$/,/^\$/d +s/^Build Summary: .*/Build Summary: 185\/185 steps succeeded; 873\/891 tests passed (18 skipped)/ +s/18 pass, 1 crash (19 total)/19 pass (19 total) 1s MaxRSS:5M/" +expect "a green run passes" 0 "$T/green.log" 0 +grep -q "^::notice::known failure passed: $listed (work#549)" "$T/out" || + { echo "FAIL: a listed test that passed is not reported"; failures=$((failures + 1)); } +grep -q "passed this run" "$T/summary" || + { echo "FAIL: the step summary lacks the passing listed test"; failures=$((failures + 1)); } + +if [ "$failures" -ne 0 ]; then + echo "$failures failed" + exit 1 +fi +echo "all passed"