chore: back-merge main into dev (keeps the CSP fix)

.github/workflows/content-pipeline-watchdog.yml

@@ -0,0 +1,165 @@
+name: Content pipeline watchdog
+
+# Content publishes to production with no human in the loop. The failure mode
+# that creates is silence: a broken internal link fails `validate-links`, the
+# publish PR never merges, and nothing anywhere goes red — the content simply
+# never appears and nobody finds out until someone notices the site is stale.
+#
+# This asserts the end-to-end invariant instead of instrumenting each failure:
+#
+#     is production's content the same as the content on mono `main`?
+#
+# One question, every failure mode — mirror died, dispatch never fired,
+# update-content errored, CI red, PR conflicted, and whatever we haven't
+# thought of yet.
+#
+# It lives here rather than in mono because peanut-ui already holds tokens to
+# read all three repos (MONO_READ_TOKEN, SUBMODULE_TOKEN); mono has no secrets.
+#
+# Loud by construction: a stuck pipeline ALWAYS fails the run (red in Actions),
+# and additionally posts to Discord when the webhook is configured. A missing
+# webhook degrades the alert; it never silences it.
+
+on:
+    schedule:
+        - cron: '*/15 * * * *'
+    workflow_dispatch:
+
+permissions:
+    contents: read
+    pull-requests: read
+
+# How long the pipeline may legitimately be mid-flight. Mirror ~1min + CI ~5min
+# + Vercel ~8min lands around 15-20min in practice; 45 leaves room for a queue
+# without letting a real stall hide for long.
+env:
+    GRACE_MINUTES: 45
+
+jobs:
+    check:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Is production's content current?
+              env:
+                  UI_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                  CONTENT_TOKEN: ${{ secrets.SUBMODULE_TOKEN }}
+                  MONO_TOKEN: ${{ secrets.MONO_READ_TOKEN }}
+                  DISCORD_WEBHOOK: ${{ secrets.CONTENT_PIPELINE_DISCORD_WEBHOOK }}
+                  REPO: ${{ github.repository }}
+                  RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+              run: |
+                  set -uo pipefail
+
+                  # Every API call is `|| true`: the step shell is `bash -e`, so an
+                  # unguarded 403 aborts mid-script and surfaces as a bare red run
+                  # indistinguishable from a real content alert. That happened once
+                  # (actions:read 403). Collect what we can, then decide explicitly.
+
+                  # ---- hop 3: what production is serving -----------------------
+                  LIVE=$(GH_TOKEN=$UI_TOKEN gh api "repos/$REPO/contents/src/content?ref=main" --jq '.sha' || true)
+
+                  # ---- hop 2: what the mirror has published --------------------
+                  TIP=$(GH_TOKEN=$CONTENT_TOKEN gh api repos/peanutprotocol/peanut-content/commits/main --jq '.sha' || true)
+                  TIP_MSG=$(GH_TOKEN=$CONTENT_TOKEN gh api repos/peanutprotocol/peanut-content/commits/main --jq '.commit.message' 2>/dev/null | head -1 || true)
+                  TIP_AT=$(GH_TOKEN=$CONTENT_TOKEN gh api repos/peanutprotocol/peanut-content/commits/main --jq '.commit.committer.date' || true)
+
+                  # ---- hop 1: is the mirror itself healthy? --------------------
+                  # Deliberately NOT a SHA comparison against the "mirror: sync from
+                  # mono@<sha>" stamp. The mirror publishes content/ MINUS _system/
+                  # (plus _system/generated/), so a mono commit touching only
+                  # content/_system/** is correctly never published — and a SHA
+                  # comparison reads that as permanent staleness and alerts every 15
+                  # minutes forever. That is exactly what happened on the first live
+                  # run (d9a33de, a docs edit to content/_system/ARCHITECTURE.md).
+                  #
+                  # Instead, find the newest mono commit that touched a path the
+                  # mirror actually publishes, and compare THAT against the stamp.
+                  # Reading the Actions API for the mirror's run status would be
+                  # simpler but MONO_READ_TOKEN has contents:read only — it 403s on
+                  # actions:read, which is how the first attempt at this fix failed.
+                  EXPECTED=""
+                  for SHA in $(GH_TOKEN=$MONO_TOKEN gh api \
+                        "repos/peanutprotocol/mono/commits?path=content&per_page=15" --jq '.[].sha' || true); do
+                    FILES=$(GH_TOKEN=$MONO_TOKEN gh api "repos/peanutprotocol/mono/commits/$SHA" --jq '.files[].filename' || true)
+                    # Published: content/** except content/_system/**, plus content/_system/generated/**
+                    PUBLISHED=$( { printf '%s\n' "$FILES" | grep '^content/' | grep -v '^content/_system/'
+                                   printf '%s\n' "$FILES" | grep '^content/_system/generated/'; } | head -1 )
+                    if [ -n "$PUBLISHED" ]; then EXPECTED="$SHA"; break; fi
+                  done
+
+                  MIRRORED=$(printf '%s' "$TIP_MSG" | sed -nE 's/.*mono@([0-9a-f]+).*/\1/p')
+
+                  echo "expected mono@${EXPECTED:0:7} | mirrored @${MIRRORED:-none} | peanut-content ${TIP:0:7} | live ${LIVE:0:7}"
+
+                  # Cannot evaluate == broken watchdog, which must NOT look like a
+                  # broken pipeline. Label it distinctly so nobody debugs the wrong
+                  # thing, but still be loud — a watchdog that can't watch is an
+                  # outage of the thing that tells us about outages.
+                  if [ -z "$EXPECTED" ] || [ -z "$MIRRORED" ] || [ -z "$LIVE" ] || [ -z "$TIP" ]; then
+                    echo "::error::WATCHDOG DEGRADED — could not evaluate the pipeline (expected=${EXPECTED:0:7} mirrored=${MIRRORED:-none} live=${LIVE:0:7} tip=${TIP:0:7}). This is a watchdog fault, not necessarily a content fault."
+                    DEGRADED=true
+                  else
+                    DEGRADED=false
+                  fi
+
+                  STALE=""
+                  MIRROR_BROKEN=false
+                  if [ "$DEGRADED" = true ]; then
+                    STALE="**the watchdog itself could not evaluate the pipeline** — check its permissions before assuming content is stuck"
+                    MIRROR_BROKEN=true
+                  elif [ "${EXPECTED:0:7}" != "${MIRRORED:0:7}" ]; then
+                    STALE="the mirror is behind mono (expected \`${EXPECTED:0:7}\`, published \`${MIRRORED:0:7}\`)"
+                    MIRROR_BROKEN=true
+                  elif [ "$LIVE" != "$TIP" ]; then
+                    STALE="production is behind peanut-content (live \`${LIVE:0:7}\`, latest \`${TIP:0:7}\`)"
+                  fi
+
+                  if [ -z "$STALE" ]; then
+                    echo "production content is current."
+                    exit 0
+                  fi
+
+                  AGE=$(( ( $(date -u +%s) - $(date -u -d "${TIP_AT:-now}" +%s) ) / 60 ))
+
+                  # ---- which hop, and is it already doomed? --------------------
+                  NUM=$(GH_TOKEN=$UI_TOKEN gh pr list --repo "$REPO" --state open --base main \
+                          --json number,headRefName \
+                          --jq '[.[] | select((.headRefName | startswith("auto/update-content-")) or (.headRefName | startswith("content/publish-to-main")))] | first | .number // empty' || true)
+
+                  # A failed mirror will not fix itself, so skip the grace period.
+                  DOOMED=$MIRROR_BROKEN
+                  if [ -n "$NUM" ]; then
+                    PR_SHA=$(GH_TOKEN=$UI_TOKEN gh api "repos/$REPO/pulls/$NUM" --jq '.head.sha' || true)
+                    CI=$(GH_TOKEN=$UI_TOKEN gh api "repos/$REPO/commits/$PR_SHA/check-runs?per_page=100" \
+                           --jq '[.check_runs[] | select(.name=="ci-success")] | if length == 0 then "pending" elif all(.conclusion == "success") then "green" else "RED" end')
+                    WHERE="publish PR #$NUM is open, ci-success=$CI"
+                    [ "$CI" = "RED" ] && DOOMED=true
+                  else
+                    WHERE="no publish PR is open — the mirror, the dispatch, or update-content.yml failed"
+                  fi
+
+                  # Red CI never self-resolves, so don't wait out the grace period.
+                  if [ "$DOOMED" = false ] && [ "$AGE" -lt "$GRACE_MINUTES" ]; then
+                    echo "$STALE — ${AGE}m old, within ${GRACE_MINUTES}m grace ($WHERE). Not alerting yet."
+                    exit 0
+                  fi
+
+                  echo "::error::$STALE — stuck ${AGE}m — $WHERE"
+
+                  if [ -n "${DISCORD_WEBHOOK:-}" ]; then
+                    export ALERT=":rotating_light: **Content is not reaching production.**
+                  $STALE
+                  Stuck for **${AGE}m** — $WHERE
+                  Runbook: \`skills/publish-content/scripts/pipeline-status.sh\`
+                  $RUN_URL"
+                    # The User-Agent is load-bearing, not decoration: Cloudflare 403s
+                    # Discord webhook posts sent with python-urllib's default UA.
+                    # Verified 2026-07-28 — without it this silently never alerts.
+                    python3 -c "import json,os,urllib.request; d=json.dumps({'content':os.environ['ALERT']}).encode(); urllib.request.urlopen(urllib.request.Request(os.environ['DISCORD_WEBHOOK'],data=d,headers={'Content-Type':'application/json','User-Agent':'peanut-content-watchdog/1.0'}),timeout=15); print('alerted Discord')" \
+                      || echo "::warning::Discord post failed — the failed run below is still the alert."
+                  else
+                    echo "::warning::CONTENT_PIPELINE_DISCORD_WEBHOOK is not set — alerting via this failed run only."
+                  fi
+
+                  # Always loud, webhook or not.
+                  exit 1

.github/workflows/content-publish-automerge.yml

@@ -12,21 +12,36 @@ name: Content publish auto-merge
 # in force for real code. eslint is already advisory (not in `ci-success`), so
 # "green tests" here means the real test/typecheck/build jobs, not lint.
 #
+# TWO PATHS TO MERGE, because the approver can't always approve:
+#
+#   1. `approve-and-merge` (pull_request_target) — approve, then `--auto --merge`.
+#      GitHub enforces `ci-success` before auto-merge fires. This is the path for
+#      any publish authored by someone other than CONTENT_BOT_TOKEN's user.
+#
+#   2. `merge-on-green` (workflow_run) — for publishes that CONTENT_BOT_TOKEN's
+#      own user authored, where GitHub forbids self-approval so path 1 can never
+#      complete. Since `update-content.yml` now opens its PRs as that user, this
+#      is the common path, not the exception. It merges via the org-admin bypass,
+#      which skips the review requirement (intended — the diff guard replaces it)
+#      AND the required status check (NOT intended). So this job re-establishes
+#      green `ci-success` ITSELF before merging. Never merge on the bypass alone.
+#
 # Security model:
-#   - `pull_request_target` runs THIS logic from the base branch (main), so a PR
-#     cannot tamper with the guard or the approve step. We never check out or run
-#     PR code — only `gh` API calls by PR number — so pull_request_target is safe
-#     here (no untrusted-code execution).
-#   - Same-repo branches only (fork guard on the job).
-#   - The approval is submitted by CONTENT_BOT_TOKEN's user (Hugo0). GitHub
-#     forbids approving your own PR, so this auto-approves anyone's content
-#     publish EXCEPT one that same user authored — those fall back to a manual
-#     merge (org-admin bypass). Konrad-authored publishes (the target case) work.
+#   - `pull_request_target` and `workflow_run` both run THIS logic from the
+#     default branch (main), so a PR cannot tamper with the guard or the merge
+#     step. We never check out or run PR code — only `gh` API calls by PR number
+#     — so both triggers are safe here (no untrusted-code execution).
+#   - Same-repo branches only (fork guards on both jobs).
+#   - Both paths require the diff to be EXACTLY `src/content` and are fail-closed:
+#     any extra file, any `gh` error, any missing check → no merge.
 
 on:
     pull_request_target:
         types: [opened, synchronize, reopened, ready_for_review]
         branches: [main]
+    workflow_run:
+        workflows: ['Tests']
+        types: [completed]
 
 permissions:
     contents: read
@@ -35,7 +50,7 @@ permissions:
 jobs:
     approve-and-merge:
         # Same-repo PRs only — never a fork.
-        if: github.event.pull_request.head.repo.full_name == github.repository
+        if: github.event_name == 'pull_request_target' && github.event.pull_request.head.repo.full_name == github.repository
         runs-on: ubuntu-latest
         steps:
             - name: Guard — diff must be exactly the content submodule pointer
@@ -64,13 +79,73 @@ jobs:
                   REPO: ${{ github.repository }}
               run: |
                   set -euo pipefail
-                  # Approve — satisfies the 1-review rule. Fails only when the bot
-                  # user authored the PR (GitHub blocks self-approval); in that
-                  # case leave it for a manual merge.
+                  # Approve — satisfies the 1-review rule. Fails when the bot user
+                  # authored the PR (GitHub blocks self-approval); `merge-on-green`
+                  # picks those up once CI is green.
                   if gh pr review "$PR" --repo "$REPO" --approve; then
                     # main ruleset allows merge commits only (not squash/rebase).
                     gh pr merge "$PR" --repo "$REPO" --auto --merge \
                       || echo "::warning::auto-merge not enabled/permitted — merge manually."
                   else
-                    echo "::warning::Could not approve (bot may be the PR author) — merge manually."
+                    echo "::notice::Could not approve (bot authored this PR) — merge-on-green will handle it once ci-success is green."
                   fi
+
+    merge-on-green:
+        # Self-approval fallback. Fires when `Tests` finishes for any commit; the
+        # job itself works out whether there's a content publish to merge.
+        if: github.event_name == 'workflow_run'
+        runs-on: ubuntu-latest
+        steps:
+            - name: Merge content-only publishes once ci-success is green
+              env:
+                  GH_TOKEN: ${{ secrets.CONTENT_BOT_TOKEN }}
+                  SHA: ${{ github.event.workflow_run.head_sha }}
+                  REPO: ${{ github.repository }}
+              run: |
+                  set -euo pipefail
+
+                  # Key off the `ci-success` check run, NOT `workflow_run.conclusion`.
+                  # `ci-success` is the single check the "Protect Prod" ruleset gates
+                  # on; the run conclusion also folds in advisory jobs. Today eslint is
+                  # `continue-on-error: true` so it happens not to move the conclusion —
+                  # but that's an eslint config detail, not a guarantee. Gate on exactly
+                  # what the ruleset gates on.
+                  #
+                  # `Tests` runs on both `push` and `pull_request`, so a commit usually
+                  # carries TWO ci-success check runs. Require at least one and ALL of
+                  # them green — a single red run must block the merge.
+                  RUNS="repos/$REPO/commits/$SHA/check-runs?per_page=100"
+                  TOTAL=$(gh api "$RUNS" --jq '[.check_runs[] | select(.name=="ci-success")] | length')
+                  GREEN=$(gh api "$RUNS" --jq '[.check_runs[] | select(.name=="ci-success" and .conclusion=="success")] | length')
+                  if [ "$TOTAL" -eq 0 ] || [ "$TOTAL" -ne "$GREEN" ]; then
+                    echo "ci-success not green for $SHA ($GREEN/$TOTAL) — nothing to merge."
+                    exit 0
+                  fi
+
+                  # Open, non-draft, same-repo PRs into main still sitting on this exact
+                  # commit. The head.sha match stops us merging a PR that has since
+                  # moved on to an untested commit.
+                  PRS=$(gh api "repos/$REPO/commits/$SHA/pulls" --jq "
+                    .[]
+                    | select(.state == \"open\")
+                    | select(.draft == false)
+                    | select(.base.ref == \"main\")
+                    | select(.head.repo.full_name == \"$REPO\")
+                    | select(.head.sha == \"$SHA\")
+                    | .number")
+
+                  if [ -z "$PRS" ]; then
+                    echo "No open content-publish candidates on $SHA."
+                    exit 0
+                  fi
+
+                  for PR in $PRS; do
+                    FILES=$(gh pr diff "$PR" --repo "$REPO" --name-only)
+                    if [ "$FILES" != "src/content" ]; then
+                      echo "::notice::#$PR is not content-only — normal review gate stays in place."
+                      continue
+                    fi
+                    echo "Merging #$PR — diff is exactly src/content, ci-success green ($GREEN/$TOTAL) on $SHA."
+                    gh pr merge "$PR" --repo "$REPO" --merge \
+                      || echo "::warning::merge failed for #$PR (already merged, or bypass unavailable) — merge manually."
+                  done