From df2afc8348b0285d4ca8c7193b08d32c05ca2e46 Mon Sep 17 00:00:00 2001 From: Chris Purcell <168346341+chrisdpurcell@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:24:06 -0400 Subject: [PATCH] feat(verify): fail fast on a red battery and size --full compat for the worker `--fail-fast` skips every remaining serial lane once one has come back red and is the default for `--full`; `--keep-going` restores run-every-lane and stays the fast gate's default, since its three lanes are already running when the first red appears. A cut lane keeps its row in the lane table as `skipped (--fail-fast)` so it can never be mistaken for a lane the mode does not have. `VERIFY_FULL_COMPAT_WORKERS` defaults to 16, matching the 40-core rexec worker that actually runs the gate rather than the retired 21-core workstation. Refs #236 --- CHANGELOG.md | 4 ++ scripts/verify.sh | 109 +++++++++++++++++++++++------ tests/test_repository_test_gate.py | 80 ++++++++++++++++++++- 3 files changed, 171 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0199e766..6a593a1d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,10 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Version ## [Unreleased] +### Changed + +- **`scripts/verify.sh` stops a battery at the first red lane and sizes the `--full` compatibility lane for the machine that runs it** ([#236](https://github.com/L3DigitalNet/project-standards/issues/236)). `--fail-fast` skips every remaining serial lane once one has come back red and is the default for `--full`, where roughly 35 minutes of compatibility matrix ran after the ordinary lane had already failed on the 2026-09-01 train; `--keep-going` restores the run-every-lane behaviour and stays the default for the fast gate, whose three lanes are already running when the first red appears. A lane cut short is reported in the lane table as `skipped (--fail-fast)`, never omitted. `VERIFY_FULL_COMPAT_WORKERS` now defaults to `16` instead of a literal tuned for the retired 21-core workstation. Repository tooling only: no package, payload, or consumer-visible byte changes. + ## [5.28.0] — 2026-09-01 ### Added diff --git a/scripts/verify.sh b/scripts/verify.sh index 90f0cee4..9181e520 100755 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -9,10 +9,10 @@ # then `coverage combine` + `coverage report`. # # --full runs the legacy serial sequence instead (statics, serial ordinary -# coverage run, compatibility at -n 4, performance, report). That is the -# release-prep cross-check: it is the configuration the coverage baseline was -# established under, so a disagreement between it and the fast gate is a real -# signal rather than a parallelism artifact. +# coverage run, compatibility, performance, report). That is the release-prep +# cross-check: it is the configuration the coverage baseline was established +# under, so a disagreement between it and the fast gate is a real signal +# rather than a parallelism artifact. # # Every command runs from `.venv/bin` rather than through `uv run`. Concurrent # `uv run` invocations contend on the uv cache (the 2026-07-29 failure class); @@ -43,6 +43,16 @@ # scripts/verify.sh fast gate (default) # scripts/verify.sh --full legacy serial battery / release-prep cross-check # +# --fail-fast stops the run at the first red lane and is the default for --full; +# --keep-going restores run-every-lane and is the default for the fast gate. The +# split follows what each mode is for: a serial battery that already knows it is +# red spends its remaining lanes proving nothing (on the 2026-09-01 train roughly +# 35 minutes of compatibility matrix ran after the ordinary lane had failed, +# issue #236 C6), while the fast gate's three lanes start together, so there is +# nothing left to save and a complete picture of every finding is worth more. +# Lanes cut short are reported in the summary as `skipped`, never omitted: a lane +# missing from the table would read as a lane that was never part of the mode. +# # TEST-ONLY: VERIFY_SMOKE=1 shrinks every pytest lane to a token selection so # the lane orchestration itself can be exercised in seconds. It proves plumbing, # never correctness — a smoke run is not a gate run. @@ -51,10 +61,10 @@ # too broadly; a nested fixture using the production root would collide with the # parent gate whose orchestration it is validating. # -# Worker counts are tuned for the 21-core workstation the spike measured and are -# overridable while controlled-condition benchmarking is still open: +# Worker counts are sized for the machine that actually runs this gate — the +# 40-core / 64 GiB rexec worker — and stay overridable: # VERIFY_ORDINARY_WORKERS (16), VERIFY_COMPAT_WORKERS (8), -# VERIFY_FULL_COMPAT_WORKERS (4) +# VERIFY_FULL_COMPAT_WORKERS (16) set -u set -o pipefail @@ -67,10 +77,13 @@ RELEASE_WHEEL_DIR="$REPO_ROOT/build/release-wheel" ORDINARY_WORKERS="${VERIFY_ORDINARY_WORKERS:-16}" COMPAT_WORKERS="${VERIFY_COMPAT_WORKERS:-8}" -FULL_COMPAT_WORKERS="${VERIFY_FULL_COMPAT_WORKERS:-4}" +FULL_COMPAT_WORKERS="${VERIFY_FULL_COMPAT_WORKERS:-16}" SMOKE="${VERIFY_SMOKE:-0}" MODE="fast" +# Empty until argument parsing finishes, so an explicit flag can be told apart +# from the mode-derived default resolved below. +FAIL_FAST="" die() { printf 'verify: %s\n' "$1" >&2 @@ -79,17 +92,22 @@ die() { usage() { cat <<'EOF' -Usage: scripts/verify.sh [--full] [--help] - - (default) fast gate: statics + ordinary + compatibility concurrently, - then performance alone, then coverage combine + report - --full legacy serial battery (release-prep cross-check) +Usage: scripts/verify.sh [--full] [--fail-fast | --keep-going] [--help] + + (default) fast gate: statics + ordinary + compatibility concurrently, + then performance alone, then coverage combine + report + --full legacy serial battery (release-prep cross-check) + --fail-fast stop at the first red lane; remaining lanes report as skipped + (default with --full) + --keep-going run every lane even after one is red (default for the fast gate) EOF } while [[ $# -gt 0 ]]; do case "$1" in --full) MODE="full" ;; + --fail-fast) FAIL_FAST=1 ;; + --keep-going) FAIL_FAST=0 ;; -h | --help) usage exit 0 @@ -102,6 +120,14 @@ while [[ $# -gt 0 ]]; do shift done +# --full is the release-prep cross-check, where a lane after the first red one +# proves nothing and costs tens of minutes; the fast gate's lanes are already +# running when the first red appears, so it defaults to the full picture. An +# explicit flag in either direction wins over the mode. +if [[ -z "$FAIL_FAST" ]]; then + if [[ "$MODE" == "full" ]]; then FAIL_FAST=1; else FAIL_FAST=0; fi +fi + # ── Preflight ───────────────────────────────────────────────────────────── cd "$REPO_ROOT" || die "cannot enter $REPO_ROOT" @@ -259,6 +285,43 @@ serial_lane() { run_lane "$@" } +# Answers "has any lane recorded so far come back red?" from $RESULT_DIR rather +# than from a shell variable: concurrent lanes run in background subshells, so a +# status set there can never reach this shell. Unrecorded counts as red, matching +# the summary's rule — a lane that died before writing its result is a failure. +gate_has_red() { + local lane status + for lane in "${LANE_ORDER[@]}"; do + status="" + if [[ -r "$RESULT_DIR/$lane" ]]; then + IFS=$'\t' read -r status _ <"$RESULT_DIR/$lane" || true + fi + [[ "$status" == "0" ]] || return 0 + done + return 1 +} + +# Records a lane the run deliberately did not execute. It still joins LANE_ORDER +# and still gets a log and a result file, so the summary shows it as `skipped` +# instead of dropping it: an absent row is indistinguishable from a lane the mode +# never had, which is exactly the confusion a fail-fast run must not create. +skip_lane() { + LANE_ORDER+=("$1") + printf 'skipped: an earlier lane failed and --fail-fast is in effect\n' >"$LOG_DIR/$1.log" + printf 'skipped\t\n' >"$RESULT_DIR/$1" +} + +# The fail-fast gate for every serial lane. Concurrent lanes are started before +# any result exists and so are never guarded; under --fail-fast the cut therefore +# begins at the first serial lane, which in fast mode is the performance tail. +serial_lane_unless_red() { + if [[ "$FAIL_FAST" == "1" ]] && gate_has_red; then + skip_lane "$1" + return 0 + fi + serial_lane "$@" +} + # ── Lanes ───────────────────────────────────────────────────────────────── # The statics lane runs every step even after one fails: a single pass should # surface all style findings, not just the first. @@ -358,7 +421,7 @@ lane_coverage_report() { } # ── Run ─────────────────────────────────────────────────────────────────── -printf 'verify: mode=%s tmp=%s (%s)\n' "$MODE" "$TMP_ROOT" "$TMP_KIND" +printf 'verify: mode=%s fail-fast=%s tmp=%s (%s)\n' "$MODE" "$FAIL_FAST" "$TMP_ROOT" "$TMP_KIND" [[ "$SMOKE" == "1" ]] && printf 'verify: VERIFY_SMOKE=1 — token selections, NOT a gate run\n' printf 'verify: PYTHONPATH=%s\n\n' "$PYTHONPATH" printf 'verify: PROJECT_STANDARDS_COMPATIBILITY_WHEEL=%s\n\n' "$PROJECT_STANDARDS_COMPATIBILITY_WHEEL" @@ -390,16 +453,16 @@ if [[ "$MODE" == "fast" ]]; then wait # Timing-sensitive: never concurrent with another lane. - serial_lane performance lane_performance - serial_lane coverage-combine lane_coverage_combine + serial_lane_unless_red performance lane_performance + serial_lane_unless_red coverage-combine lane_coverage_combine else - serial_lane statics lane_statics - serial_lane ordinary lane_ordinary_serial - serial_lane compatibility lane_compatibility "$FULL_COMPAT_WORKERS" - serial_lane performance lane_performance + serial_lane_unless_red statics lane_statics + serial_lane_unless_red ordinary lane_ordinary_serial + serial_lane_unless_red compatibility lane_compatibility "$FULL_COMPAT_WORKERS" + serial_lane_unless_red performance lane_performance fi -serial_lane coverage-report lane_coverage_report +serial_lane_unless_red coverage-report lane_coverage_report GATE_SECONDS="$(($(date +%s) - GATE_START))" @@ -436,6 +499,10 @@ for lane in "${LANE_ORDER[@]}"; do fi if [[ "$lane_status" == "0" ]]; then verdict="ok" + elif [[ "$lane_status" == "skipped" ]]; then + # Not a failure of its own: the lane that tripped --fail-fast is the one + # that sets the exit status, and double-counting would hide it. + verdict="skipped (--fail-fast)" elif [[ -z "$lane_status" ]]; then # Unrecorded is failed: the lane never reached the line that writes its # result, so treating it as anything else would report a green gate for diff --git a/tests/test_repository_test_gate.py b/tests/test_repository_test_gate.py index 321f3fcd..74c48c29 100644 --- a/tests/test_repository_test_gate.py +++ b/tests/test_repository_test_gate.py @@ -84,6 +84,9 @@ def _gate_fixture( fi if [[ "${VERIFY_ASSERT_FAST_START:-0}" == "1" && "$1" == "combine" ]]; then touch "$VERIFY_COVERAGE_COMBINE_FILE" +fi +if [[ "$1" == "run" ]]; then + exit "${VERIFY_ORDINARY_EXIT:-0}" fi""", ) for tool in ("prettier", "markdownlint-cli2"): @@ -412,6 +415,81 @@ def test_verify_gate__full_mode__preserves_serial_lane_order(tmp_path: Path) -> assert invocations[3].startswith( "coverage run --source=project_standards -m pytest -m not performance and not compatibility" ) - assert invocations[4].startswith("pytest -m compatibility -n 4") + assert invocations[4].startswith("pytest -m compatibility -n 16") assert invocations[5].startswith("pytest -m performance") assert invocations[6] == "coverage report --fail-under=0" + + +def test_verify_gate__full_mode__red_ordinary_lane__skips_the_later_lanes( + tmp_path: Path, +) -> None: + """--full is fail-fast by default, and a cut lane is reported, not dropped. + + The compatibility lane is the expensive one: on the 2026-09-01 train it ran + for roughly 35 minutes after the ordinary lane had already gone red (#236 C6). + """ + repo, log, environment = _gate_fixture(tmp_path, wheel_count=1) + environment["VERIFY_ORDINARY_EXIT"] = "13" + + completed = _run_gate(repo, environment, "--full") + + assert completed.returncode == 1 + invocations = log.read_text(encoding="utf-8").splitlines() + assert not any(invocation.startswith("pytest -m compatibility") for invocation in invocations) + assert not any(invocation.startswith("pytest -m performance") for invocation in invocations) + summary = completed.stdout[completed.stdout.index("════ summary ════") :] + assert "ordinary" in summary + assert "FAILED (exit 13)" in summary + for lane in ("compatibility", "performance", "coverage-report"): + assert lane in summary + line = next(row for row in summary.splitlines() if row.strip().startswith(lane)) + assert "skipped (--fail-fast)" in line + + +def test_verify_gate__full_mode__keep_going__runs_every_lane_after_a_red( + tmp_path: Path, +) -> None: + """--keep-going preserves the pre-#236 behaviour of running every lane.""" + repo, log, environment = _gate_fixture(tmp_path, wheel_count=1) + environment["VERIFY_ORDINARY_EXIT"] = "13" + + completed = _run_gate(repo, environment, "--full", "--keep-going") + + assert completed.returncode == 1 + invocations = log.read_text(encoding="utf-8").splitlines() + assert any(invocation.startswith("pytest -m compatibility") for invocation in invocations) + assert any(invocation.startswith("pytest -m performance") for invocation in invocations) + summary = completed.stdout[completed.stdout.index("════ summary ════") :] + assert "FAILED (exit 13)" in summary + assert "skipped" not in summary + + +def test_verify_gate__fast_mode__fail_fast__cuts_the_serial_tail(tmp_path: Path) -> None: + """The fast gate keeps run-every-lane by default; --fail-fast opts in. + + Its three lanes are already running when the first red appears, so the cut + can only begin at the serial tail. + """ + repo, log, environment = _gate_fixture(tmp_path, wheel_count=1) + environment["VERIFY_ORDINARY_EXIT"] = "13" + + kept_going = _run_gate(repo, environment) + fail_fast = _run_gate(repo, environment, "--fail-fast") + + invocations = log.read_text(encoding="utf-8").splitlines() + performance_runs = [ + invocation for invocation in invocations if invocation.startswith("pytest -m performance") + ] + assert kept_going.returncode == 1 + assert fail_fast.returncode == 1 + # Both runs share one log; only the default run reaches the performance lane. + assert len(performance_runs) == 1 + assert "skipped" not in kept_going.stdout[kept_going.stdout.index("════ summary ════") :] + fail_fast_summary = fail_fast.stdout[fail_fast.stdout.index("════ summary ════") :] + for lane in ("performance", "coverage-combine", "coverage-report"): + line = next(row for row in fail_fast_summary.splitlines() if row.strip().startswith(lane)) + assert "skipped (--fail-fast)" in line + # The concurrently started lanes are never cut: they are already running. + for lane in ("statics", "ordinary", "compatibility"): + line = next(row for row in fail_fast_summary.splitlines() if row.strip().startswith(lane)) + assert "skipped" not in line