ptrlrd · ptrlrd · Aug 1, 2026
diff --git a/.github/workflows/runs-db-backup.yml b/.github/workflows/runs-db-backup.yml
@@ -1,81 +1,58 @@
-name: Runs DB backup
-
-# Daily mongodump of the live runs store onto the prod host under
-# ~/backups/spire-codex-mongo/. The old SQLite snapshot this replaces
-# targeted the frozen legacy runs.db; Mongo has been the source of
-# truth since the migration, so this is the backup that matters.
+# Freshness check for the daily on-box Mongo backup (installed by
+# infrastructure/ansible/playbooks/install-backup.yml, which also offloads
+# each verified archive to a private R2 bucket).
 #
-# The dump runs through the official mongo image (the host has no
-# mongo tools installed), reading MONGO_URL from the box's .env, and
-# is verified with a dry-run restore before it counts. Keeps 14 daily
-# archives. Still single-host risk — ship to R2 later if the corpus
-# becomes irreplaceable.
+# This job no longer SSHes to prod — that dependency broke every time the
+# box's authorized keys changed, and the backups silently stopped. Instead
+# it lists the R2 backup prefix and FAILS if the newest archive is older
+# than 48 hours, so a dead cron surfaces as a red run within a day.
+#
+# Required repo secrets (read-only R2 credentials for the backup bucket):
+#   R2_BACKUP_ENDPOINT, R2_BACKUP_BUCKET, R2_BACKUP_KEY_ID, R2_BACKUP_SECRET
+# Unset secrets fail the run with a clear message on purpose: a check that
+# passes while unconfigured is worse than no check.
+
+name: Runs DB backup
 
 on:
-  workflow_dispatch: {}
   schedule:
-    # 03:15 UTC every day — off-peak for traffic, well clear of the
-    # hourly news-refresh cron so the two don't stomp on each other.
-    - cron: "15 3 * * *"
-
-concurrency:
-  group: runs-db-backup
-  cancel-in-progress: false
+    - cron: "30 7 * * *" # 07:30 UTC, ~80 min after the on-box 06:10 dump
+  workflow_dispatch:
 
 jobs:
-  backup:
-    name: mongodump on prod
-    runs-on: self-hosted
+  freshness:
+    name: verify newest backup in R2
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.R2_BACKUP_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_BACKUP_SECRET }}
+      R2_ENDPOINT: ${{ secrets.R2_BACKUP_ENDPOINT }}
+      R2_BUCKET: ${{ secrets.R2_BACKUP_BUCKET }}
     steps:
-      - name: SSH — dump + verify + rotate
-        # SHA-pinned: this third-party action receives the prod SSH key.
-        uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5
-        with:
-          host: ${{ secrets.SSH_HOST }}
-          username: ${{ secrets.SSH_USER }}
-          key: ${{ secrets.SSH_PRIVATE_KEY }}
-          # A full dump can take a while as the corpus grows; don't let the
-          # action's 10m default kill it mid-archive.
-          command_timeout: 60m
-          script: |
-            set -euo pipefail
-            BACKUP_DIR="$HOME/backups/spire-codex-mongo"
-            STAMP=$(date -u +%Y-%m-%dT%H-%M-%SZ)
-            DEST="$BACKUP_DIR/mongo-${STAMP}.archive.gz"
-
-            mkdir -p "$BACKUP_DIR"
-
-            MONGO_URL=$(grep -E '^MONGO_URL=' /var/www/spire-codex/.env | head -1 | cut -d= -f2-)
-            if [ -z "$MONGO_URL" ]; then
-              echo "ERROR: MONGO_URL not found in /var/www/spire-codex/.env" >&2
-              exit 1
-            fi
-
-            TMP=$(mktemp "$BACKUP_DIR/.mongo.XXXXXX")
-            trap 'rm -f "$TMP"' EXIT
-
-            # --network host so localhost and public-IP URIs both work.
-            docker run --rm --network host mongo:7.0 \
-              mongodump --uri="$MONGO_URL" --archive --gzip --quiet > "$TMP"
-
-            if [ ! -s "$TMP" ]; then
-              echo "ERROR: mongodump produced an empty archive" >&2
-              exit 1
-            fi
-
-            # Verify the archive actually restores before retaining it.
-            docker run --rm -i --network none mongo:7.0 \
-              mongorestore --archive --gzip --dryRun --quiet < "$TMP" \
-              || { echo "ERROR: archive failed dry-run restore" >&2; exit 1; }
-
-            mv "$TMP" "$DEST"
-            trap - EXIT
-            echo "wrote $DEST ($(du -h "$DEST" | cut -f1))"
-
-            # Retention — keep the last 14 daily archives. mtime-based so a
-            # run that accidentally happens twice in a day doesn't double
-            # up the count.
-            find "$BACKUP_DIR" -maxdepth 1 -name 'mongo-*.archive.gz' -mtime +14 -print -delete
-
-            echo "retained:"
-            ls -1h "$BACKUP_DIR" | tail -10
+      - name: Check newest archive age
+        run: |
+          set -euo pipefail
+          if [ -z "$R2_BUCKET" ] || [ -z "$R2_ENDPOINT" ]; then
+            echo "::error::R2_BACKUP_* secrets are not configured. The on-box" \
+                 "backup may still be running (check /var/log/spire-codex-backup.log" \
+                 "on the box), but this check cannot verify it." >&2
+            exit 1
+          fi
+          newest=$(aws s3 ls "s3://$R2_BUCKET/mongo/" --endpoint-url "$R2_ENDPOINT" \
+                   | sort | tail -1)
+          if [ -z "$newest" ]; then
+            echo "::error::No archives found under s3://$R2_BUCKET/mongo/" >&2
+            exit 1
+          fi
+          echo "newest: $newest"
+          key=$(echo "$newest" | awk '{print $4}')
+          stamp="${key#mongo-}"; stamp="${stamp%.archive.gz}"
+          # Stamps are UTC like 2026-08-01T06-10-02Z; normalize to epoch.
+          iso=$(echo "$stamp" | sed 's/T\([0-9]*\)-\([0-9]*\)-\([0-9]*\)Z/T\1:\2:\3Z/')
+          age=$(( $(date -u +%s) - $(date -u -d "$iso" +%s) ))
+          echo "age: $((age / 3600))h"
+          if [ "$age" -gt $((48 * 3600)) ]; then
+            echo "::error::Newest backup is older than 48h ($key)" >&2
+            exit 1
+          fi
diff --git a/infrastructure/ansible/files/mongo-backup.sh b/infrastructure/ansible/files/mongo-backup.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+# Daily mongodump of the live runs store, run ON the box by cron (installed
+# via playbooks/install-backup.yml). Same dump/verify/rotate logic the old
+# GHA workflow used, minus the GHA->prod SSH dependency that kept breaking.
+#
+# When /etc/spire-codex/backup-r2.env exists (rendered by the playbook), the
+# verified archive is also offloaded to a private R2 bucket, which removes
+# the single-host risk and gives the SSH-free freshness check in
+# .github/workflows/runs-db-backup.yml something to look at.
+set -euo pipefail
+
+BACKUP_DIR="/data/backups/spire-codex-mongo"
+STAMP=$(date -u +%Y-%m-%dT%H-%M-%SZ)
+DEST="$BACKUP_DIR/mongo-${STAMP}.archive.gz"
+R2_ENV="/etc/spire-codex/backup-r2.env"
+
+mkdir -p "$BACKUP_DIR"
+
+MONGO_URL=$(grep -E '^MONGO_URL=' /var/www/spire-codex/.env | head -1 | cut -d= -f2-)
+if [ -z "$MONGO_URL" ]; then
+  echo "ERROR: MONGO_URL not found in /var/www/spire-codex/.env" >&2
+  exit 1
+fi
+
+TMP=$(mktemp "$BACKUP_DIR/.mongo.XXXXXX")
+trap 'rm -f "$TMP"' EXIT
+
+# --network host so localhost and public-IP URIs both work.
+docker run --rm --network host mongo:7.0 \
+  mongodump --uri="$MONGO_URL" --archive --gzip --quiet > "$TMP"
+
+if [ ! -s "$TMP" ]; then
+  echo "ERROR: mongodump produced an empty archive" >&2
+  exit 1
+fi
+
+# Verify the archive actually restores before retaining it.
+docker run --rm -i --network none mongo:7.0 \
+  mongorestore --archive --gzip --dryRun --quiet < "$TMP" \
+  || { echo "ERROR: archive failed dry-run restore" >&2; exit 1; }
+
+mv "$TMP" "$DEST"
+trap - EXIT
+echo "$(date -u +%FT%TZ) wrote $DEST ($(du -h "$DEST" | cut -f1))"
+
+# Retention — keep the last 14 daily archives locally.
+find "$BACKUP_DIR" -maxdepth 1 -name 'mongo-*.archive.gz' -mtime +14 -print -delete
+
+# Optional R2 offload. The env file provides R2_ENDPOINT, R2_BUCKET,
+# AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY. Offload failures are loud but
+# don't fail the run — the local archive already exists and rotates.
+if [ -f "$R2_ENV" ]; then
+  set -a; . "$R2_ENV"; set +a
+  if docker run --rm --network host \
+      -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \
+      -v "$BACKUP_DIR:/backups:ro" \
+      amazon/aws-cli s3 cp "/backups/$(basename "$DEST")" \
+      "s3://$R2_BUCKET/mongo/$(basename "$DEST")" \
+      --endpoint-url "$R2_ENDPOINT" --only-show-errors; then
+    echo "$(date -u +%FT%TZ) offloaded to r2://$R2_BUCKET/mongo/"
+    # Remote retention mirrors local: prune R2 copies older than 14 days.
+    CUTOFF=$(date -u -d '14 days ago' +%Y-%m-%dT%H-%M-%SZ)
+    docker run --rm --network host \
+      -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \
+      amazon/aws-cli s3 ls "s3://$R2_BUCKET/mongo/" --endpoint-url "$R2_ENDPOINT" \
+      | awk '{print $4}' | while read -r key; do
+        [ -n "$key" ] || continue
+        stamp="${key#mongo-}"; stamp="${stamp%.archive.gz}"
+        if [[ "$stamp" < "$CUTOFF" ]]; then
+          docker run --rm --network host \
+            -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \
+            amazon/aws-cli s3 rm "s3://$R2_BUCKET/mongo/$key" \
+            --endpoint-url "$R2_ENDPOINT" --only-show-errors
+        fi
+      done
+  else
+    echo "WARNING: R2 offload failed; local archive retained" >&2
+  fi
+fi
+
+echo "retained locally:"
+ls -1h "$BACKUP_DIR" | tail -10
diff --git a/infrastructure/ansible/files/spire-codex-backup.cron b/infrastructure/ansible/files/spire-codex-backup.cron
@@ -0,0 +1,10 @@
+# Spire Codex daily Mongo backup. Dumps, dry-run-verifies, rotates 14 days,
+# and offloads to R2 when /etc/spire-codex/backup-r2.env is present. See
+# /usr/local/bin/spire-codex-mongo-backup and /var/log/spire-codex-backup.log.
+#
+# 06:10 UTC: after the nightly stats quiet period, offset from the :03
+# autodeploy and anything firing on the hour.
+SHELL=/bin/bash
+PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+10 6 * * * root /usr/local/bin/spire-codex-mongo-backup >> /var/log/spire-codex-backup.log 2>&1
diff --git a/infrastructure/ansible/playbooks/install-backup.yml b/infrastructure/ansible/playbooks/install-backup.yml
@@ -0,0 +1,85 @@
+---
+# Install the daily on-box Mongo backup cron on prod origins. Replaces the
+# GHA runs-db-backup SSH job, which broke every time the box's authorized
+# keys changed. Run once per box (and re-run after any change to
+# mongo-backup.sh / the cron schedule):
+#
+#   cd infrastructure/ansible
+#   ./bin/do-ansible playbooks/install-backup.yml
+#
+# Optional R2 offload (recommended — removes the single-host risk): put the
+# backup bucket credentials in 1Password and export R2_BACKUP_ENDPOINT,
+# R2_BACKUP_BUCKET, R2_BACKUP_KEY_ID, R2_BACKUP_SECRET when running the
+# playbook; it renders them into /etc/spire-codex/backup-r2.env (0600,
+# root-only). Without them the backup still runs, local-only.
+
+- name: Install daily Mongo backup cron
+  hosts: prod_origins
+  gather_facts: false
+  vars:
+    backup_script: /usr/local/bin/spire-codex-mongo-backup
+    cron_file: /etc/cron.d/spire-codex-backup
+    env_dir: /etc/spire-codex
+    r2_env_file: /etc/spire-codex/backup-r2.env
+  tasks:
+    - name: Copy backup script
+      become: true
+      copy:
+        src: ../files/mongo-backup.sh
+        dest: "{{ backup_script }}"
+        mode: "0755"
+        owner: root
+        group: root
+
+    - name: Install cron entry
+      become: true
+      copy:
+        src: ../files/spire-codex-backup.cron
+        dest: "{{ cron_file }}"
+        mode: "0644"
+        owner: root
+        group: root
+
+    - name: Ensure env dir exists
+      become: true
+      file:
+        path: "{{ env_dir }}"
+        state: directory
+        mode: "0700"
+        owner: root
+        group: root
+
+    - name: Render R2 offload env (only when creds are provided)
+      become: true
+      copy:
+        content: |
+          # Auto-generated by install-backup.yml. Do not edit by hand —
+          # update 1Password and re-run the playbook.
+          R2_ENDPOINT="{{ lookup('env', 'R2_BACKUP_ENDPOINT') }}"
+          R2_BUCKET="{{ lookup('env', 'R2_BACKUP_BUCKET') }}"
+          AWS_ACCESS_KEY_ID="{{ lookup('env', 'R2_BACKUP_KEY_ID') }}"
+          AWS_SECRET_ACCESS_KEY="{{ lookup('env', 'R2_BACKUP_SECRET') }}"
+        dest: "{{ r2_env_file }}"
+        mode: "0600"
+        owner: root
+        group: root
+      when: lookup('env', 'R2_BACKUP_BUCKET') | length > 0
+      no_log: true
+
+    - name: Run the backup once now (proves it works before trusting cron)
+      become: true
+      command: "{{ backup_script }}"
+      register: first_run
+      changed_when: true
+
+    - name: First-run output
+      debug:
+        msg: "{{ first_run.stdout_lines[-5:] }}"
+
+    - name: Summary
+      debug:
+        msg: |
+          ✓ Daily Mongo backup installed. Fires 06:10 UTC.
+          Manual trigger: ssh box && sudo /usr/local/bin/spire-codex-mongo-backup
+          Archives:       /data/backups/spire-codex-mongo/ (14-day rotation)
+          Logs:           /var/log/spire-codex-backup.log