diff --git a/.github/workflows/content-pipeline-watchdog.yml b/.github/workflows/content-pipeline-watchdog.yml index 504795d27..c230a0c79 100644 --- a/.github/workflows/content-pipeline-watchdog.yml +++ b/.github/workflows/content-pipeline-watchdog.yml @@ -50,12 +50,18 @@ jobs: run: | set -uo pipefail + # Every API call is `|| true`: the step shell is `bash -e`, so an + # unguarded 403 aborts mid-script and surfaces as a bare red run + # indistinguishable from a real content alert. That happened once + # (actions:read 403). Collect what we can, then decide explicitly. + # ---- hop 3: what production is serving ----------------------- - LIVE=$(GH_TOKEN=$UI_TOKEN gh api "repos/$REPO/contents/src/content?ref=main" --jq '.sha') + LIVE=$(GH_TOKEN=$UI_TOKEN gh api "repos/$REPO/contents/src/content?ref=main" --jq '.sha' || true) # ---- hop 2: what the mirror has published -------------------- - TIP=$(GH_TOKEN=$CONTENT_TOKEN gh api repos/peanutprotocol/peanut-content/commits/main --jq '.sha') - TIP_AT=$(GH_TOKEN=$CONTENT_TOKEN gh api repos/peanutprotocol/peanut-content/commits/main --jq '.commit.committer.date') + TIP=$(GH_TOKEN=$CONTENT_TOKEN gh api repos/peanutprotocol/peanut-content/commits/main --jq '.sha' || true) + TIP_MSG=$(GH_TOKEN=$CONTENT_TOKEN gh api repos/peanutprotocol/peanut-content/commits/main --jq '.commit.message' 2>/dev/null | head -1 || true) + TIP_AT=$(GH_TOKEN=$CONTENT_TOKEN gh api repos/peanutprotocol/peanut-content/commits/main --jq '.commit.committer.date' || true) # ---- hop 1: is the mirror itself healthy? -------------------- # Deliberately NOT a SHA comparison against the "mirror: sync from @@ -66,19 +72,43 @@ jobs: # minutes forever. That is exactly what happened on the first live # run (d9a33de, a docs edit to content/_system/ARCHITECTURE.md). # - # Ask the question that has a real yes/no answer instead: did the - # mirror's last completed run succeed? A mirror that ran and - # published nothing is healthy; one that errored is not. - MIRROR=$(GH_TOKEN=$MONO_TOKEN gh api \ - "repos/peanutprotocol/mono/actions/workflows/mirror-to-peanut-content.yml/runs?per_page=1&status=completed" \ - --jq '.workflow_runs[0].conclusion // "none"') - - echo "mirror last run: $MIRROR | peanut-content ${TIP:0:7} | live ${LIVE:0:7}" + # Instead, find the newest mono commit that touched a path the + # mirror actually publishes, and compare THAT against the stamp. + # Reading the Actions API for the mirror's run status would be + # simpler but MONO_READ_TOKEN has contents:read only — it 403s on + # actions:read, which is how the first attempt at this fix failed. + EXPECTED="" + for SHA in $(GH_TOKEN=$MONO_TOKEN gh api \ + "repos/peanutprotocol/mono/commits?path=content&per_page=15" --jq '.[].sha' || true); do + FILES=$(GH_TOKEN=$MONO_TOKEN gh api "repos/peanutprotocol/mono/commits/$SHA" --jq '.files[].filename' || true) + # Published: content/** except content/_system/**, plus content/_system/generated/** + PUBLISHED=$( { printf '%s\n' "$FILES" | grep '^content/' | grep -v '^content/_system/' + printf '%s\n' "$FILES" | grep '^content/_system/generated/'; } | head -1 ) + if [ -n "$PUBLISHED" ]; then EXPECTED="$SHA"; break; fi + done + + MIRRORED=$(printf '%s' "$TIP_MSG" | sed -nE 's/.*mono@([0-9a-f]+).*/\1/p') + + echo "expected mono@${EXPECTED:0:7} | mirrored @${MIRRORED:-none} | peanut-content ${TIP:0:7} | live ${LIVE:0:7}" + + # Cannot evaluate == broken watchdog, which must NOT look like a + # broken pipeline. Label it distinctly so nobody debugs the wrong + # thing, but still be loud — a watchdog that can't watch is an + # outage of the thing that tells us about outages. + if [ -z "$EXPECTED" ] || [ -z "$MIRRORED" ] || [ -z "$LIVE" ] || [ -z "$TIP" ]; then + echo "::error::WATCHDOG DEGRADED — could not evaluate the pipeline (expected=${EXPECTED:0:7} mirrored=${MIRRORED:-none} live=${LIVE:0:7} tip=${TIP:0:7}). This is a watchdog fault, not necessarily a content fault." + DEGRADED=true + else + DEGRADED=false + fi STALE="" MIRROR_BROKEN=false - if [ "$MIRROR" != "success" ] && [ "$MIRROR" != "none" ]; then - STALE="the mirror workflow's last run concluded \`$MIRROR\` — content is not leaving mono" + if [ "$DEGRADED" = true ]; then + STALE="**the watchdog itself could not evaluate the pipeline** — check its permissions before assuming content is stuck" + MIRROR_BROKEN=true + elif [ "${EXPECTED:0:7}" != "${MIRRORED:0:7}" ]; then + STALE="the mirror is behind mono (expected \`${EXPECTED:0:7}\`, published \`${MIRRORED:0:7}\`)" MIRROR_BROKEN=true elif [ "$LIVE" != "$TIP" ]; then STALE="production is behind peanut-content (live \`${LIVE:0:7}\`, latest \`${TIP:0:7}\`)" @@ -89,18 +119,18 @@ jobs: exit 0 fi - AGE=$(( ( $(date -u +%s) - $(date -u -d "$TIP_AT" +%s) ) / 60 )) + AGE=$(( ( $(date -u +%s) - $(date -u -d "${TIP_AT:-now}" +%s) ) / 60 )) # ---- which hop, and is it already doomed? -------------------- NUM=$(GH_TOKEN=$UI_TOKEN gh pr list --repo "$REPO" --state open --base main \ --json number,headRefName \ - --jq '[.[] | select((.headRefName | startswith("auto/update-content-")) or (.headRefName | startswith("content/publish-to-main")))] | first | .number // empty') + --jq '[.[] | select((.headRefName | startswith("auto/update-content-")) or (.headRefName | startswith("content/publish-to-main")))] | first | .number // empty' || true) # A failed mirror will not fix itself, so skip the grace period. DOOMED=$MIRROR_BROKEN if [ -n "$NUM" ]; then - SHA=$(GH_TOKEN=$UI_TOKEN gh api "repos/$REPO/pulls/$NUM" --jq '.head.sha') - CI=$(GH_TOKEN=$UI_TOKEN gh api "repos/$REPO/commits/$SHA/check-runs?per_page=100" \ + PR_SHA=$(GH_TOKEN=$UI_TOKEN gh api "repos/$REPO/pulls/$NUM" --jq '.head.sha' || true) + CI=$(GH_TOKEN=$UI_TOKEN gh api "repos/$REPO/commits/$PR_SHA/check-runs?per_page=100" \ --jq '[.check_runs[] | select(.name=="ci-success")] | if length == 0 then "pending" elif all(.conclusion == "success") then "green" else "RED" end') WHERE="publish PR #$NUM is open, ci-success=$CI" [ "$CI" = "RED" ] && DOOMED=true