release: staging control plane, /api/health, and the backend_tag fix

.env.example

@@ -73,3 +73,13 @@ MOLLIE_WEBHOOK_URL=
 # (never by the app, which reads MOLLIE_API_KEY). The suite hard-refuses anything
 # without a test_ prefix.
 MOLLIE_API_KEY_TEST=
+
+# QA admin for the DEPLOYED staging control plane, read by tests/e2e/staging-live.spec.ts
+# (`npm run test:e2e:staging`). Not used by the app or the local suite.
+#
+# The single quotes are REQUIRED. scripts/e2e-suite.sh sources this file's real counterpart
+# with `set -a && . ./.env`, and bash reads an unquoted {email: a@b.com, password: x} as an
+# assignment followed by the command `a@b.com,` — under the script's `set -euo pipefail`
+# that kills the entire unmocked E2E suite before it starts Postgres, and reports the
+# failure as an email address, which names nothing you would think to look at.
+STAGING_ADMIN='{email: , password: }'

.github/workflows/build-image.yml

@@ -8,7 +8,10 @@ name: build-image
 
 on:
   push:
-    branches: [main]
+    # `develop` was added with the staging bake: the build-staging job below is gated on
+    # that ref, and a job gate is useless if the WORKFLOW never runs on the branch. The
+    # tag gating inside each meta step is what keeps :latest and :migrate main-only.
+    branches: [main, develop]
     tags: ['v*']
 
 concurrency:
@@ -22,6 +25,9 @@ permissions:
 jobs:
   build-push:
     name: build & push (GHCR)
+    # main + tags only. On develop the staging bake below is the one that runs; letting
+    # this job run too would publish a :sha and a migrate-<sha> image nothing pulls.
+    if: github.ref != 'refs/heads/develop'
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
@@ -35,6 +41,13 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - id: stamp
+        # Wall-clock build time, stamped here rather than taken from a github.event
+        # field: the event payloads differ per trigger (push vs tag vs dispatch), and
+        # repository.updated_at answers "when did the repo change", which is not the
+        # question /api/health is asked.
+        run: echo "built_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT"
+
       - id: meta
         uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
         with:
@@ -55,11 +68,32 @@ jobs:
           push: true
           build-args: |
             NEXT_PUBLIC_SITE_URL=${{ vars.SITE_URL || 'https://sofrapiwas.com' }}
+            BUILD_SHA=${{ github.sha }}
+            BUILD_TIME=${{ steps.stamp.outputs.built_at }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
+      - name: Verify the build identity actually reached the image
+        # Without this, nothing in CI ever proves BUILD_SHA is wired: the Playwright smoke
+        # runs `next build` directly and never touches the Dockerfile, so `version` is always
+        # the "unknown" fallback there and the local spec's toHaveProperty passes regardless.
+        # Drop the build-arg during an unrelated edit and every gate stays green — the break
+        # would surface only when a human remembered to run the staging suite by hand.
+        # Reads the env straight out of the pushed image; no server boot needed.
+        env:
+          IMAGE: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}
+        run: |
+          set -euo pipefail
+          got=$(docker run --rm --entrypoint node "$IMAGE" \
+                  -e 'process.stdout.write(process.env.BUILD_SHA || "MISSING")')
+          echo "image reports BUILD_SHA=$got"
+          [ "$got" = "${GITHUB_SHA}" ] || {
+            echo "::error::image BUILD_SHA is '$got', expected '${GITHUB_SHA}' — /api/health would misreport what is deployed"
+            exit 1
+          }
+
       # One-off DB tooling image (prisma migrate deploy + admin seed) — pulled
       # on the box only when a release ships migrations. See DEPLOYMENT.md.
       # Published as a TAG of the main (public) package — a separate
@@ -84,3 +118,104 @@ jobs:
           labels: ${{ steps.meta-migrate.outputs.labels }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
+
+  # ── staging bake ──────────────────────────────────────────────────────────────
+  # A SEPARATE build, not just an extra tag on the one above: NEXT_PUBLIC_SITE_URL is a
+  # BUILD arg, so the same image cannot serve two hosts. Baked with the staging URL it
+  # emits staging canonicals — and, via lib/seo.ts IS_CANONICAL_SITE, a robots.txt that
+  # refuses every crawler. A staging twin of the marketing site left crawlable would
+  # compete with the real one for the citations the AEO work exists to win.
+  #
+  # develop only. `:staging` must never carry main's code, or rolling staging would
+  # silently deploy production build output to it.
+  build-staging:
+    name: build & push staging (GHCR)
+    if: github.ref == 'refs/heads/develop'
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+
+      - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - id: stamp
+        # Wall-clock build time, stamped here rather than taken from a github.event
+        # field: the event payloads differ per trigger (push vs tag vs dispatch), and
+        # repository.updated_at answers "when did the repo change", which is not the
+        # question /api/health is asked.
+        run: echo "built_at=$(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$GITHUB_OUTPUT"
+
+      - id: meta-staging
+        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
+        with:
+          images: ghcr.io/${{ github.repository }}
+          # `:staging` moves; `:staging-<sha>` is immutable, for rollback.
+          tags: |
+            type=raw,value=staging
+            type=sha,format=long,prefix=staging-
+
+      - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: ./Dockerfile
+          target: runner
+          push: true
+          build-args: |
+            NEXT_PUBLIC_SITE_URL=${{ vars.STAGING_SITE_URL || 'https://staging.sofrapiwas.com' }}
+            BUILD_SHA=${{ github.sha }}
+            BUILD_TIME=${{ steps.stamp.outputs.built_at }}
+          tags: ${{ steps.meta-staging.outputs.tags }}
+          labels: ${{ steps.meta-staging.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Verify the build identity actually reached the image
+        # Without this, nothing in CI ever proves BUILD_SHA is wired: the Playwright smoke
+        # runs `next build` directly and never touches the Dockerfile, so `version` is always
+        # the "unknown" fallback there and the local spec's toHaveProperty passes regardless.
+        # Drop the build-arg during an unrelated edit and every gate stays green — the break
+        # would surface only when a human remembered to run the staging suite by hand.
+        # Reads the env straight out of the pushed image; no server boot needed.
+        env:
+          IMAGE: ghcr.io/${{ github.repository }}:staging-${{ github.sha }}
+        run: |
+          set -euo pipefail
+          got=$(docker run --rm --entrypoint node "$IMAGE" \
+                  -e 'process.stdout.write(process.env.BUILD_SHA || "MISSING")')
+          echo "image reports BUILD_SHA=$got"
+          [ "$got" = "${GITHUB_SHA}" ] || {
+            echo "::error::image BUILD_SHA is '$got', expected '${GITHUB_SHA}' — /api/health would misreport what is deployed"
+            exit 1
+          }
+
+      # The DB-tooling image for staging. It has to be built HERE, in the job that runs
+      # on develop — putting the tag on `build-push`'s meta-migrate step made it dead
+      # code the moment that job started skipping on develop, and the tag simply never
+      # existed. Found by pulling it on the box.
+      #
+      # Staging needs its own: the migrate image carries prisma/migrations, so main's
+      # copy would apply main's schema to a database that tracks develop.
+      - id: meta-migrate-staging
+        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
+        with:
+          images: ghcr.io/${{ github.repository }}
+          tags: |
+            type=raw,value=migrate-staging
+            type=sha,format=long,prefix=migrate-staging-
+
+      - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0
+        with:
+          context: .
+          file: ./Dockerfile
+          target: migrate
+          push: true
+          tags: ${{ steps.meta-migrate-staging.outputs.tags }}
+          labels: ${{ steps.meta-migrate-staging.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/indexing-monitor.yml

@@ -0,0 +1,171 @@
+name: indexing monitor
+
+# Guards the INDEXING POSTURE of every public Sofra property: production must stay
+# crawlable, and every non-canonical copy must stay invisible.
+#
+# Why a monitor and not a test. This assertion used to live in the deployed-staging e2e
+# suite, which was the wrong home twice over. It only ran when someone happened to touch
+# staging — and the failure it guards against is a slow burn: a staging twin quietly
+# accumulating index coverage for weeks, competing with the real site for exactly the
+# citations the AEO work exists to win. It also turned a *production* fault red inside a
+# suite named for staging, which sends you to look at the wrong environment.
+#
+# `app/robots.ts` decides between allow-all and disallow-all from `NEXT_PUBLIC_SITE_URL`,
+# which is a BUILD arg (lib/seo.ts) — so `/robots.txt` is prerendered into each image and
+# both directions are one bake away from flipping. Nothing in CI can see it, because the
+# served value depends on which image landed on which host. Note the mechanism: it is not
+# a runtime env read, so this can NOT be corrected by editing the box `.env` — it takes a
+# rebuild. Someone debugging at 3am will try the env first.
+#
+# Daily, deliberately. This changes only when something is deployed, so the ~5-minute
+# cadence of uptime.yml would buy nothing and re-alert 288 times a day on one regression.
+#
+# Alerting reuses uptime.yml's secrets; it stays silent (with a warning) until both exist:
+#   gh secret set TELEGRAM_BOT_TOKEN -R piwas-21/sofra
+#   gh secret set TELEGRAM_CHAT_ID  -R piwas-21/sofra
+
+on:
+  schedule:
+    - cron: "17 6 * * *" # daily ~06:17 UTC (off the hour — cron load is spiky on the hour)
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+
+concurrency:
+  group: indexing-monitor
+  cancel-in-progress: false
+
+jobs:
+  probe:
+    name: robots posture
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Check robots posture and alert on regression
+        env:
+          TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }}
+          TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }}
+        run: |
+          set -uo pipefail
+
+          problems=()
+          # Scratch files go in a temp dir, not the working directory: the runner has a
+          # clean workspace, but anyone running this body locally to debug it would
+          # otherwise drop prod.txt/hidden.txt into the repo root — where a `git add -A`
+          # sweeps them into a commit. That happened on this very PR.
+          work=$(mktemp -d)
+          trap 'rm -rf "$work"' EXIT
+
+          # Fetch to a file rather than piping into a matcher: on failure the body can be
+          # echoed into the log, which is the difference between "robots.txt is wrong" and
+          # "robots.txt is wrong AND here is what it actually said".
+          fetch() { # fetch <url> <outfile> -> prints http code
+            # `code=...; || code=000` rather than `|| echo 000`: curl PRINTS 000 on a
+            # connection failure AND exits non-zero, so appending would emit "000000" and the
+            # alert would read `HTTP 000000`. uptime.yml hit this and documents the same fix.
+            local code
+            code=$(curl -sS -o "$2" -w '%{http_code}' --max-time 20 \
+              --retry 2 --retry-delay 5 --retry-all-errors "$1" 2>/dev/null) || code=000
+            [ -z "$code" ] && code=000
+            echo "$code"
+          }
+
+          # `x-robots-tag: …` for a URL, or empty. Lowercased so the caller matches once.
+          robots_header() {
+            curl -sSI --max-time 20 "$1" 2>/dev/null | tr -d '\r' | tr 'A-Z' 'a-z' \
+              | grep '^x-robots-tag:' || true
+          }
+
+          # --- production MUST stay crawlable -------------------------------------------
+          code=$(fetch "https://sofrapiwas.com/robots.txt" "$work/prod.txt")
+          if [ "$code" != "200" ]; then
+            problems+=("production robots.txt unreachable — HTTP $code")
+          else
+            # `Allow: /` with a capital A: Next writes `Disallow: /` for the deny case, so a
+            # case-insensitive match would find the substring "allow: /" inside it and pass
+            # on precisely the regression this exists to catch.
+            grep -qE '^Allow:[[:space:]]*/' "$work/prod.txt" \
+              || problems+=("production robots.txt no longer allows crawling")
+            grep -q 'GPTBot' "$work/prod.txt" \
+              || problems+=("production robots.txt no longer invites AI crawlers (AEO)")
+          fi
+
+          # Production must ALSO carry no noindex header. Checked because robots.txt and the
+          # header are set in different places and can disagree: the Caddyfile's production
+          # and staging blocks are near-identical and sit ~15 lines apart, so copying the
+          # staging `header X-Robots-Tag "noindex, …"` line into the production block is a
+          # plausible edit. That deindexes the live site while robots.txt still says Allow —
+          # every content check above passes, and the monitor would report "crawlable ✅"
+          # while the site silently leaves every index. This is the revenue-costing direction.
+          prod_hdr=$(robots_header "https://sofrapiwas.com/en")
+          case "$prod_hdr" in
+            *noindex*|*none*) problems+=("production carries a de-indexing header: ${prod_hdr}") ;;
+          esac
+
+          # --- every non-canonical copy of THIS app MUST stay hidden ---------------------
+          # url|label. Scope is deliberate: copies of the sofra marketing site, which is what
+          # competes with sofrapiwas.com for citations. `staging.fooderist.com` is NOT here
+          # despite uptime.yml labelling it "Sofra staging" — that label is stale, the host
+          # serves the RUMI tenant frontend, and it has no robots.txt at all (404). Its
+          # indexing posture is a question for the frontend repo, not this one.
+          hidden=(
+            "https://staging.sofrapiwas.com|sofra staging"
+          )
+          for entry in "${hidden[@]}"; do
+            url="${entry%%|*}"; label="${entry##*|}"
+            code=$(fetch "$url/robots.txt" "$work/hidden.txt")
+            if [ "$code" != "200" ]; then
+              problems+=("$label robots.txt unreachable — HTTP $code")
+              continue
+            fi
+            if ! grep -qE '^Disallow:[[:space:]]*/[[:space:]]*$' "$work/hidden.txt"; then
+              problems+=("$label is INDEXABLE — robots.txt no longer disallows")
+              # Bounded, and `::` stripped: GitHub parses ::workflow commands from step
+              # stdout, and this is a remote response body we do not control.
+              echo "--- $label robots.txt ---"; head -c 2000 "$work/hidden.txt" | sed 's/^:://'
+            fi
+            grep -q 'GPTBot' "$work/hidden.txt" \
+              && problems+=("$label invites AI crawlers — it must not")
+
+            # The header too, for crawlers that never fetch robots.txt. Caddy adds it, so
+            # this catches a reverse-proxy regression the app itself cannot see.
+            hdr=$(robots_header "$url/en")
+            case "$hdr" in
+              *noindex*) ;;
+              *) problems+=("$label missing X-Robots-Tag: noindex (got: ${hdr:-none})") ;;
+            esac
+          done
+
+          {
+            echo "### Indexing posture"
+            if [ ${#problems[@]} -eq 0 ]; then
+              echo "Production crawlable, non-canonical copies hidden ✅"
+            else
+              echo "Problems:"; for p in "${problems[@]}"; do echo "- $p"; done
+            fi
+          } >> "$GITHUB_STEP_SUMMARY"
+
+          if [ ${#problems[@]} -eq 0 ]; then echo "Indexing posture correct."; exit 0; fi
+
+          for p in "${problems[@]}"; do echo "PROBLEM: $p"; done
+
+          run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
+          msg=$'\xF0\x9F\x94\x8D Indexing posture regression:\n'
+          for p in "${problems[@]}"; do msg+="• ${p}"$'\n'; done
+          msg+=$'\n'"${run_url}"
+
+          if [ -z "${TELEGRAM_BOT_TOKEN}" ] || [ -z "${TELEGRAM_CHAT_ID}" ]; then
+            echo "::warning::Regression found but TELEGRAM_BOT_TOKEN/TELEGRAM_CHAT_ID not set — no alert sent."
+          else
+            curl -sS --max-time 20 \
+              -X POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \
+              --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \
+              --data-urlencode "text=${msg}" \
+              -o /dev/null -w 'telegram sendMessage: %{http_code}\n' \
+              || echo "::warning::Telegram send failed"
+          fi
+
+          # Fail the run as well as alerting: the Actions UI is the durable record, and a
+          # green history next to a Telegram message nobody kept is worse than neither.
+          exit 1