From e90825e483458c28f505027730bc1670df4e197e Mon Sep 17 00:00:00 2001 From: Joshua Temple Date: Thu, 9 Jul 2026 14:57:51 -0400 Subject: [PATCH] ci(fleet): retry transient lane failures and fix the self-repin version SIGPIPE The fleet fan-out lanes primary, dependents, and remainder ran with no retry, so a transient GitHub hosted-runner non-acquisition (the job not acquired by a runner, cancelled after fifteen minutes) or a broken pipe hard-failed the whole fleet with no second chance. Set retry-attempts to two on those lanes to match the heavy lane; the dispatch-suite loop already treats a cancelled run as retryable. Also fix an intermittent broken-pipe failure in the fleet-repin self-repin step: it read the installed version with cascade version piped into head, and because cascade version prints three lines from separate writes, head closing the pipe after the first line races the later writes into a broken pipe that pipefail promotes to a step failure. Capture the output first and split it without a pipe. Signed-off-by: Joshua Temple --- .github/actions/dispatch-suite/action.yaml | 4 ++-- .github/actions/fleet-repin/action.yaml | 10 +++++++++- .github/workflows/fleet-e2e.yaml | 20 ++++++++++++++++++-- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/actions/dispatch-suite/action.yaml b/.github/actions/dispatch-suite/action.yaml index d5173e44..f5b3d7a6 100644 --- a/.github/actions/dispatch-suite/action.yaml +++ b/.github/actions/dispatch-suite/action.yaml @@ -52,8 +52,8 @@ inputs: retry-attempts: description: >- Total dispatch attempts for this suite. Default 1 keeps the historical - behavior (a single dispatch, no retry). Set to 2 or more on the heaviest, - most race-prone lane so a whole-suite failure re-dispatches a fresh run. + behavior (a single dispatch, no retry). Set to 2 or more on a fan-out lane + so a whole-suite failure re-dispatches a fresh run. This is safe because every scenario-suite run begins with its own "reset and seed dev" step, so a re-dispatch starts from a clean slate. Only a non-success conclusion (or a run that never became visible) diff --git a/.github/actions/fleet-repin/action.yaml b/.github/actions/fleet-repin/action.yaml index 1c08dc0b..2f8fea1f 100644 --- a/.github/actions/fleet-repin/action.yaml +++ b/.github/actions/fleet-repin/action.yaml @@ -105,7 +105,15 @@ runs: install -m 0755 "$TMPDIR/cascade" /usr/local/bin/cascade rm -rf "$TMPDIR" - INSTALLED=$(cascade version 2>/dev/null | head -n 1 | awk '{print $2}') + # `cascade version` prints three lines via separate writes. Piping it + # straight into `head -n 1` lets head close the pipe after the first + # line while the binary is still writing the rest, which delivers + # SIGPIPE (exit 141) to the binary and, under `set -o pipefail`, reds + # this step intermittently. Capture the output into a variable first, + # then take field two of the first line without a pipe, so no producer + # can be signalled. This mirrors the setup-cli action's capture pattern. + INSTALLED_RAW=$(cascade version 2>/dev/null || true) + read -r _ INSTALLED _ <<<"$INSTALLED_RAW" || true # Tolerate a leading v in the self-report so the check tracks the release # tag rather than a future ldflags formatting choice. echo "fleet-repin: installed cascade version ${INSTALLED} (expected ${RC_BARE})" diff --git a/.github/workflows/fleet-e2e.yaml b/.github/workflows/fleet-e2e.yaml index 66f37da2..ac87ce91 100644 --- a/.github/workflows/fleet-e2e.yaml +++ b/.github/workflows/fleet-e2e.yaml @@ -369,6 +369,11 @@ jobs: token: ${{ secrets.CASCADE_STATE_TOKEN }} cascade_version: ${{ needs.resolve.outputs.cascade_version }} cascade_version_sha: ${{ needs.resolve.outputs.cascade_version_sha }} + # Allow one whole-suite re-dispatch so a transient runner + # non-acquisition, a mid-poll API blip, or a SIGPIPE does not red the + # lane on a single unlucky attempt. Safe because each scenario-suite + # run self-resets on its first step, so a re-dispatch starts clean. + retry-attempts: '2' # Lane 1, stage 2: dependents of primary (mutate primary's shared external # state), so they only start after primary is green. Two repos run together, @@ -396,6 +401,11 @@ jobs: token: ${{ secrets.CASCADE_STATE_TOKEN }} cascade_version: ${{ needs.resolve.outputs.cascade_version }} cascade_version_sha: ${{ needs.resolve.outputs.cascade_version_sha }} + # Allow one whole-suite re-dispatch so a transient runner + # non-acquisition, a mid-poll API blip, or a SIGPIPE does not red the + # lane on a single unlucky attempt. Safe because each scenario-suite + # run self-resets on its first step, so a re-dispatch starts clean. + retry-attempts: '2' # Lane 2: 4env alone. It is the heaviest and most fragile repo, so it runs in # its own dedicated job with nothing beside it. Sequenced AFTER the dependents @@ -424,8 +434,9 @@ jobs: # plus chained multi-env hotfix, conflict, rollback, and merge_queue # against one live repo). Allow one whole-suite re-dispatch so a live # GitHub eventual-consistency blip does not red the lane. Safe because - # each suite run self-resets on its first step. Other lanes keep the - # default (no retry) so a retry stays a scoped concession here. + # each suite run self-resets on its first step. Every fan-out lane + # carries the same single re-dispatch so a transient failure on any of + # them re-dispatches once rather than redding the whole fleet. retry-attempts: '2' # Lane 3: the light remainder, capped at two repos in flight. Sequenced AFTER @@ -454,6 +465,11 @@ jobs: token: ${{ secrets.CASCADE_STATE_TOKEN }} cascade_version: ${{ needs.resolve.outputs.cascade_version }} cascade_version_sha: ${{ needs.resolve.outputs.cascade_version_sha }} + # Allow one whole-suite re-dispatch so a transient runner + # non-acquisition, a mid-poll API blip, or a SIGPIPE does not red the + # lane on a single unlucky attempt. Safe because each scenario-suite + # run self-resets on its first step, so a re-dispatch starts clean. + retry-attempts: '2' # Fan-in: this job's conclusion is the rc fleet gate. It needs EVERY lane so a # green gate means every selected repo passed; auto-promote keys off this