diff --git a/.github/workflows/runs-db-backup.yml b/.github/workflows/runs-db-backup.yml index 3949314e..4bdc1beb 100644 --- a/.github/workflows/runs-db-backup.yml +++ b/.github/workflows/runs-db-backup.yml @@ -1,81 +1,58 @@ -name: Runs DB backup - -# Daily mongodump of the live runs store onto the prod host under -# ~/backups/spire-codex-mongo/. The old SQLite snapshot this replaces -# targeted the frozen legacy runs.db; Mongo has been the source of -# truth since the migration, so this is the backup that matters. +# Freshness check for the daily on-box Mongo backup (installed by +# infrastructure/ansible/playbooks/install-backup.yml, which also offloads +# each verified archive to a private R2 bucket). # -# The dump runs through the official mongo image (the host has no -# mongo tools installed), reading MONGO_URL from the box's .env, and -# is verified with a dry-run restore before it counts. Keeps 14 daily -# archives. Still single-host risk — ship to R2 later if the corpus -# becomes irreplaceable. +# This job no longer SSHes to prod — that dependency broke every time the +# box's authorized keys changed, and the backups silently stopped. Instead +# it lists the R2 backup prefix and FAILS if the newest archive is older +# than 48 hours, so a dead cron surfaces as a red run within a day. +# +# Required repo secrets (read-only R2 credentials for the backup bucket): +# R2_BACKUP_ENDPOINT, R2_BACKUP_BUCKET, R2_BACKUP_KEY_ID, R2_BACKUP_SECRET +# Unset secrets fail the run with a clear message on purpose: a check that +# passes while unconfigured is worse than no check. + +name: Runs DB backup on: - workflow_dispatch: {} schedule: - # 03:15 UTC every day — off-peak for traffic, well clear of the - # hourly news-refresh cron so the two don't stomp on each other. - - cron: "15 3 * * *" - -concurrency: - group: runs-db-backup - cancel-in-progress: false + - cron: "30 7 * * *" # 07:30 UTC, ~80 min after the on-box 06:10 dump + workflow_dispatch: jobs: - backup: - name: mongodump on prod - runs-on: self-hosted + freshness: + name: verify newest backup in R2 + runs-on: ubuntu-latest + timeout-minutes: 5 + env: + AWS_ACCESS_KEY_ID: ${{ secrets.R2_BACKUP_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_BACKUP_SECRET }} + R2_ENDPOINT: ${{ secrets.R2_BACKUP_ENDPOINT }} + R2_BUCKET: ${{ secrets.R2_BACKUP_BUCKET }} steps: - - name: SSH — dump + verify + rotate - # SHA-pinned: this third-party action receives the prod SSH key. - uses: appleboy/ssh-action@0ff4204d59e8e51228ff73bce53f80d53301dee2 # v1.2.5 - with: - host: ${{ secrets.SSH_HOST }} - username: ${{ secrets.SSH_USER }} - key: ${{ secrets.SSH_PRIVATE_KEY }} - # A full dump can take a while as the corpus grows; don't let the - # action's 10m default kill it mid-archive. - command_timeout: 60m - script: | - set -euo pipefail - BACKUP_DIR="$HOME/backups/spire-codex-mongo" - STAMP=$(date -u +%Y-%m-%dT%H-%M-%SZ) - DEST="$BACKUP_DIR/mongo-${STAMP}.archive.gz" - - mkdir -p "$BACKUP_DIR" - - MONGO_URL=$(grep -E '^MONGO_URL=' /var/www/spire-codex/.env | head -1 | cut -d= -f2-) - if [ -z "$MONGO_URL" ]; then - echo "ERROR: MONGO_URL not found in /var/www/spire-codex/.env" >&2 - exit 1 - fi - - TMP=$(mktemp "$BACKUP_DIR/.mongo.XXXXXX") - trap 'rm -f "$TMP"' EXIT - - # --network host so localhost and public-IP URIs both work. - docker run --rm --network host mongo:7.0 \ - mongodump --uri="$MONGO_URL" --archive --gzip --quiet > "$TMP" - - if [ ! -s "$TMP" ]; then - echo "ERROR: mongodump produced an empty archive" >&2 - exit 1 - fi - - # Verify the archive actually restores before retaining it. - docker run --rm -i --network none mongo:7.0 \ - mongorestore --archive --gzip --dryRun --quiet < "$TMP" \ - || { echo "ERROR: archive failed dry-run restore" >&2; exit 1; } - - mv "$TMP" "$DEST" - trap - EXIT - echo "wrote $DEST ($(du -h "$DEST" | cut -f1))" - - # Retention — keep the last 14 daily archives. mtime-based so a - # run that accidentally happens twice in a day doesn't double - # up the count. - find "$BACKUP_DIR" -maxdepth 1 -name 'mongo-*.archive.gz' -mtime +14 -print -delete - - echo "retained:" - ls -1h "$BACKUP_DIR" | tail -10 + - name: Check newest archive age + run: | + set -euo pipefail + if [ -z "$R2_BUCKET" ] || [ -z "$R2_ENDPOINT" ]; then + echo "::error::R2_BACKUP_* secrets are not configured. The on-box" \ + "backup may still be running (check /var/log/spire-codex-backup.log" \ + "on the box), but this check cannot verify it." >&2 + exit 1 + fi + newest=$(aws s3 ls "s3://$R2_BUCKET/mongo/" --endpoint-url "$R2_ENDPOINT" \ + | sort | tail -1) + if [ -z "$newest" ]; then + echo "::error::No archives found under s3://$R2_BUCKET/mongo/" >&2 + exit 1 + fi + echo "newest: $newest" + key=$(echo "$newest" | awk '{print $4}') + stamp="${key#mongo-}"; stamp="${stamp%.archive.gz}" + # Stamps are UTC like 2026-08-01T06-10-02Z; normalize to epoch. + iso=$(echo "$stamp" | sed 's/T\([0-9]*\)-\([0-9]*\)-\([0-9]*\)Z/T\1:\2:\3Z/') + age=$(( $(date -u +%s) - $(date -u -d "$iso" +%s) )) + echo "age: $((age / 3600))h" + if [ "$age" -gt $((48 * 3600)) ]; then + echo "::error::Newest backup is older than 48h ($key)" >&2 + exit 1 + fi diff --git a/infrastructure/ansible/files/mongo-backup.sh b/infrastructure/ansible/files/mongo-backup.sh new file mode 100644 index 00000000..eb0d2657 --- /dev/null +++ b/infrastructure/ansible/files/mongo-backup.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Daily mongodump of the live runs store, run ON the box by cron (installed +# via playbooks/install-backup.yml). Same dump/verify/rotate logic the old +# GHA workflow used, minus the GHA->prod SSH dependency that kept breaking. +# +# When /etc/spire-codex/backup-r2.env exists (rendered by the playbook), the +# verified archive is also offloaded to a private R2 bucket, which removes +# the single-host risk and gives the SSH-free freshness check in +# .github/workflows/runs-db-backup.yml something to look at. +set -euo pipefail + +BACKUP_DIR="/data/backups/spire-codex-mongo" +STAMP=$(date -u +%Y-%m-%dT%H-%M-%SZ) +DEST="$BACKUP_DIR/mongo-${STAMP}.archive.gz" +R2_ENV="/etc/spire-codex/backup-r2.env" + +mkdir -p "$BACKUP_DIR" + +MONGO_URL=$(grep -E '^MONGO_URL=' /var/www/spire-codex/.env | head -1 | cut -d= -f2-) +if [ -z "$MONGO_URL" ]; then + echo "ERROR: MONGO_URL not found in /var/www/spire-codex/.env" >&2 + exit 1 +fi + +TMP=$(mktemp "$BACKUP_DIR/.mongo.XXXXXX") +trap 'rm -f "$TMP"' EXIT + +# --network host so localhost and public-IP URIs both work. +docker run --rm --network host mongo:7.0 \ + mongodump --uri="$MONGO_URL" --archive --gzip --quiet > "$TMP" + +if [ ! -s "$TMP" ]; then + echo "ERROR: mongodump produced an empty archive" >&2 + exit 1 +fi + +# Verify the archive actually restores before retaining it. +docker run --rm -i --network none mongo:7.0 \ + mongorestore --archive --gzip --dryRun --quiet < "$TMP" \ + || { echo "ERROR: archive failed dry-run restore" >&2; exit 1; } + +mv "$TMP" "$DEST" +trap - EXIT +echo "$(date -u +%FT%TZ) wrote $DEST ($(du -h "$DEST" | cut -f1))" + +# Retention — keep the last 14 daily archives locally. +find "$BACKUP_DIR" -maxdepth 1 -name 'mongo-*.archive.gz' -mtime +14 -print -delete + +# Optional R2 offload. The env file provides R2_ENDPOINT, R2_BUCKET, +# AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY. Offload failures are loud but +# don't fail the run — the local archive already exists and rotates. +if [ -f "$R2_ENV" ]; then + set -a; . "$R2_ENV"; set +a + if docker run --rm --network host \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ + -v "$BACKUP_DIR:/backups:ro" \ + amazon/aws-cli s3 cp "/backups/$(basename "$DEST")" \ + "s3://$R2_BUCKET/mongo/$(basename "$DEST")" \ + --endpoint-url "$R2_ENDPOINT" --only-show-errors; then + echo "$(date -u +%FT%TZ) offloaded to r2://$R2_BUCKET/mongo/" + # Remote retention mirrors local: prune R2 copies older than 14 days. + CUTOFF=$(date -u -d '14 days ago' +%Y-%m-%dT%H-%M-%SZ) + docker run --rm --network host \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ + amazon/aws-cli s3 ls "s3://$R2_BUCKET/mongo/" --endpoint-url "$R2_ENDPOINT" \ + | awk '{print $4}' | while read -r key; do + [ -n "$key" ] || continue + stamp="${key#mongo-}"; stamp="${stamp%.archive.gz}" + if [[ "$stamp" < "$CUTOFF" ]]; then + docker run --rm --network host \ + -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY \ + amazon/aws-cli s3 rm "s3://$R2_BUCKET/mongo/$key" \ + --endpoint-url "$R2_ENDPOINT" --only-show-errors + fi + done + else + echo "WARNING: R2 offload failed; local archive retained" >&2 + fi +fi + +echo "retained locally:" +ls -1h "$BACKUP_DIR" | tail -10 diff --git a/infrastructure/ansible/files/spire-codex-backup.cron b/infrastructure/ansible/files/spire-codex-backup.cron new file mode 100644 index 00000000..09e41743 --- /dev/null +++ b/infrastructure/ansible/files/spire-codex-backup.cron @@ -0,0 +1,10 @@ +# Spire Codex daily Mongo backup. Dumps, dry-run-verifies, rotates 14 days, +# and offloads to R2 when /etc/spire-codex/backup-r2.env is present. See +# /usr/local/bin/spire-codex-mongo-backup and /var/log/spire-codex-backup.log. +# +# 06:10 UTC: after the nightly stats quiet period, offset from the :03 +# autodeploy and anything firing on the hour. +SHELL=/bin/bash +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + +10 6 * * * root /usr/local/bin/spire-codex-mongo-backup >> /var/log/spire-codex-backup.log 2>&1 diff --git a/infrastructure/ansible/playbooks/install-backup.yml b/infrastructure/ansible/playbooks/install-backup.yml new file mode 100644 index 00000000..ef3ef11a --- /dev/null +++ b/infrastructure/ansible/playbooks/install-backup.yml @@ -0,0 +1,85 @@ +--- +# Install the daily on-box Mongo backup cron on prod origins. Replaces the +# GHA runs-db-backup SSH job, which broke every time the box's authorized +# keys changed. Run once per box (and re-run after any change to +# mongo-backup.sh / the cron schedule): +# +# cd infrastructure/ansible +# ./bin/do-ansible playbooks/install-backup.yml +# +# Optional R2 offload (recommended — removes the single-host risk): put the +# backup bucket credentials in 1Password and export R2_BACKUP_ENDPOINT, +# R2_BACKUP_BUCKET, R2_BACKUP_KEY_ID, R2_BACKUP_SECRET when running the +# playbook; it renders them into /etc/spire-codex/backup-r2.env (0600, +# root-only). Without them the backup still runs, local-only. + +- name: Install daily Mongo backup cron + hosts: prod_origins + gather_facts: false + vars: + backup_script: /usr/local/bin/spire-codex-mongo-backup + cron_file: /etc/cron.d/spire-codex-backup + env_dir: /etc/spire-codex + r2_env_file: /etc/spire-codex/backup-r2.env + tasks: + - name: Copy backup script + become: true + copy: + src: ../files/mongo-backup.sh + dest: "{{ backup_script }}" + mode: "0755" + owner: root + group: root + + - name: Install cron entry + become: true + copy: + src: ../files/spire-codex-backup.cron + dest: "{{ cron_file }}" + mode: "0644" + owner: root + group: root + + - name: Ensure env dir exists + become: true + file: + path: "{{ env_dir }}" + state: directory + mode: "0700" + owner: root + group: root + + - name: Render R2 offload env (only when creds are provided) + become: true + copy: + content: | + # Auto-generated by install-backup.yml. Do not edit by hand — + # update 1Password and re-run the playbook. + R2_ENDPOINT="{{ lookup('env', 'R2_BACKUP_ENDPOINT') }}" + R2_BUCKET="{{ lookup('env', 'R2_BACKUP_BUCKET') }}" + AWS_ACCESS_KEY_ID="{{ lookup('env', 'R2_BACKUP_KEY_ID') }}" + AWS_SECRET_ACCESS_KEY="{{ lookup('env', 'R2_BACKUP_SECRET') }}" + dest: "{{ r2_env_file }}" + mode: "0600" + owner: root + group: root + when: lookup('env', 'R2_BACKUP_BUCKET') | length > 0 + no_log: true + + - name: Run the backup once now (proves it works before trusting cron) + become: true + command: "{{ backup_script }}" + register: first_run + changed_when: true + + - name: First-run output + debug: + msg: "{{ first_run.stdout_lines[-5:] }}" + + - name: Summary + debug: + msg: | + ✓ Daily Mongo backup installed. Fires 06:10 UTC. + Manual trigger: ssh box && sudo /usr/local/bin/spire-codex-mongo-backup + Archives: /data/backups/spire-codex-mongo/ (14-day rotation) + Logs: /var/log/spire-codex-backup.log