From 23181a9030e90de43d4bb427291201c95a89f9ad Mon Sep 17 00:00:00 2001 From: Hugo Montenegro Date: Tue, 28 Jul 2026 10:34:28 +0100 Subject: [PATCH] fix(watchdog): stop alerting on content the mirror is never meant to publish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The watchdog's first live run cried wolf and would have done so every 15 minutes forever. It compared mono's latest commit touching `content/` against the sha stamped in peanut-content's "mirror: sync from mono@" tip. But the mirror rsyncs `content/` with `--exclude='_system/'` (plus `_system/generated/` separately), so a commit touching only `content/_system/**` is correctly never published. The comparison read that as the mirror being stuck. d9a33de — a docs edit to content/_system/ARCHITECTURE.md — tripped it within minutes of merge. The stamp can't answer "is the mirror healthy" because it only ever names commits that happened to touch mirrored paths. Ask something that has a real answer: did the mirror's last completed run succeed? A run that published nothing is healthy; one that errored is not. A broken mirror also skips the grace period, since it will not fix itself. Verified against the same live state that produced the false alarm: now reports CURRENT and stays silent. --- .../workflows/content-pipeline-watchdog.yml | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/.github/workflows/content-pipeline-watchdog.yml b/.github/workflows/content-pipeline-watchdog.yml index 2c2b85747..504795d27 100644 --- a/.github/workflows/content-pipeline-watchdog.yml +++ b/.github/workflows/content-pipeline-watchdog.yml @@ -55,21 +55,31 @@ jobs: # ---- hop 2: what the mirror has published -------------------- TIP=$(GH_TOKEN=$CONTENT_TOKEN gh api repos/peanutprotocol/peanut-content/commits/main --jq '.sha') - TIP_MSG=$(GH_TOKEN=$CONTENT_TOKEN gh api repos/peanutprotocol/peanut-content/commits/main --jq '.commit.message' | head -1) TIP_AT=$(GH_TOKEN=$CONTENT_TOKEN gh api repos/peanutprotocol/peanut-content/commits/main --jq '.commit.committer.date') - # ---- hop 1: what mono says the content should be ------------- - # The mirror stamps "mirror: sync from mono@", so a mismatch - # here means the mirror itself is stuck — invisible from hops 2/3 - # alone, which would look perfectly consistent with each other. - MONO_SHA=$(GH_TOKEN=$MONO_TOKEN gh api "repos/peanutprotocol/mono/commits?path=content&per_page=1" --jq '.[0].sha' | cut -c1-7) - MIRRORED_SHA=$(printf '%s' "$TIP_MSG" | sed -nE 's/.*mono@([0-9a-f]+).*/\1/p' | cut -c1-7) - - echo "mono content@$MONO_SHA | mirrored from @${MIRRORED_SHA:-?} | peanut-content ${TIP:0:7} | live ${LIVE:0:7}" + # ---- hop 1: is the mirror itself healthy? -------------------- + # Deliberately NOT a SHA comparison against the "mirror: sync from + # mono@" stamp. The mirror publishes content/ MINUS _system/ + # (plus _system/generated/), so a mono commit touching only + # content/_system/** is correctly never published — and a SHA + # comparison reads that as permanent staleness and alerts every 15 + # minutes forever. That is exactly what happened on the first live + # run (d9a33de, a docs edit to content/_system/ARCHITECTURE.md). + # + # Ask the question that has a real yes/no answer instead: did the + # mirror's last completed run succeed? A mirror that ran and + # published nothing is healthy; one that errored is not. + MIRROR=$(GH_TOKEN=$MONO_TOKEN gh api \ + "repos/peanutprotocol/mono/actions/workflows/mirror-to-peanut-content.yml/runs?per_page=1&status=completed" \ + --jq '.workflow_runs[0].conclusion // "none"') + + echo "mirror last run: $MIRROR | peanut-content ${TIP:0:7} | live ${LIVE:0:7}" STALE="" - if [ -n "$MIRRORED_SHA" ] && [ "$MONO_SHA" != "$MIRRORED_SHA" ]; then - STALE="the mirror is behind mono (mono content@\`$MONO_SHA\`, mirror published \`$MIRRORED_SHA\`)" + MIRROR_BROKEN=false + if [ "$MIRROR" != "success" ] && [ "$MIRROR" != "none" ]; then + STALE="the mirror workflow's last run concluded \`$MIRROR\` — content is not leaving mono" + MIRROR_BROKEN=true elif [ "$LIVE" != "$TIP" ]; then STALE="production is behind peanut-content (live \`${LIVE:0:7}\`, latest \`${TIP:0:7}\`)" fi @@ -86,7 +96,8 @@ jobs: --json number,headRefName \ --jq '[.[] | select((.headRefName | startswith("auto/update-content-")) or (.headRefName | startswith("content/publish-to-main")))] | first | .number // empty') - DOOMED=false + # A failed mirror will not fix itself, so skip the grace period. + DOOMED=$MIRROR_BROKEN if [ -n "$NUM" ]; then SHA=$(GH_TOKEN=$UI_TOKEN gh api "repos/$REPO/pulls/$NUM" --jq '.head.sha') CI=$(GH_TOKEN=$UI_TOKEN gh api "repos/$REPO/commits/$SHA/check-runs?per_page=100" \