fix: make pre-migration backups schema-aware

docs/disaster-recovery.md

@@ -57,12 +57,22 @@ migrations are the authoritative schema/security definition and are replayed
 first on any recovery target. `schema_migrations` is rebuilt by the runner.
 
 **The dump is allowlist-only.** `pg_dump` is invoked with repeated
-`--table=public.<name>` for exactly `BACKUP_TABLES` — never a whole-database
-dump. Supabase warns that raw `pg_dump` can include platform-internal
-schemas/objects; the allowlist guarantees an unknown or newly added
-schema/table is *not* backed up until explicitly classified in
+`--table=public.<name>` for the schema-aware subset of `BACKUP_TABLES` —
+never a whole-database dump. Supabase warns that raw `pg_dump` can include
+platform-internal schemas/objects; the allowlist guarantees an unknown or
+newly added schema/table is *not* backed up until explicitly classified in
 `scripts/backup-scope.mjs` (and CI fails loudly on any unclassified table).
 
+**The dump is migration-aware.** Before `pg_dump` runs, `db:backup` reads the
+target's `schema_migrations` and requests only the allowlisted tables the
+*applied* migrations actually created (`resolveBackupTables()`). This keeps
+the mandatory pre-migration backup safe when the target lags repo main —
+e.g. backing up before applying `012_ownership.sql` simply excludes the
+not-yet-created ownership tables. It fails closed on a broken schema: a
+table the applied migrations expect but the DB lacks, or an allowlisted
+table present without a recorded migration, aborts the backup rather than
+producing a silently partial artifact.
+
 ### Table classification (scripts/backup-scope.mjs is the code-level truth)
 
 | Table | Class | In backup? | Reason |

scripts/backup-db.mjs

@@ -24,7 +24,7 @@ import { fileURLToPath } from 'node:url'
 import postgres from 'postgres'
 import { loadOperatorEnv, databaseUrl, pgEnv } from './operator-env.mjs'
 import { runPgTool } from './pg-tools.mjs'
-import { BACKUP_TABLES } from './backup-scope.mjs'
+import { BACKUP_TABLES, migrationFiles, resolveBackupTables } from './backup-scope.mjs'
 
 const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
 
@@ -38,14 +38,41 @@ async function main() {
   const env = pgEnv(url)
 
   const sql = postgres(url, { onnotice: () => {} })
-  let serverMajor
+  let serverMajor, applied, existing
   try {
     const [{ server_version_num }] = await sql`show server_version_num`
     serverMajor = Math.floor(Number(server_version_num) / 10000)
+    // Migration-aware backup scope: the target may lag repo main (a fresh
+    // pre-migration backup is taken BEFORE the pending migration runs), so
+    // the allowlist must be resolved against the applied schema, not
+    // blindly requested. Missing/unrecorded schema state fails closed.
+    try {
+      applied = (await sql`select filename from public.schema_migrations`).map((r) => r.filename)
+    } catch {
+      throw new Error('schema_migrations is missing — the target was never migrated; refusing to back up')
+    }
+    // BASE TABLE only — a view must never satisfy a required backup table.
+    existing = (await sql`
+      select table_name from information_schema.tables
+      where table_schema = 'public'
+        and table_type = 'BASE TABLE'`).map((r) => r.table_name)
   } finally {
     await sql.end()
   }
 
+  const resolved = resolveBackupTables({ applied, existing })
+  if (resolved.length === 0) {
+    throw new Error('resolved backup set is empty — target schema predates any backed-up table')
+  }
+  const latest = [...applied].sort().at(-1)
+  const repoCount = migrationFiles().length
+  const behind = applied.length < repoCount
+  console.log(`schema: ${applied.length} migrations applied (latest: ${latest || 'none'}), repo has ${repoCount}${behind ? ' — target is BEHIND repo main' : ''}`)
+  const skipped = BACKUP_TABLES.filter((t) => !resolved.includes(t))
+  if (skipped.length) {
+    console.log(`note: schema-aware resolution excludes not-yet-created table(s) from this backup: ${skipped.join(', ')}`)
+  }
+
   const dir = backupDir()
   fs.mkdirSync(dir, { recursive: true })
   const stamp = new Date().toISOString().replace(/[:.]/g, '-').replace('T', '_').slice(0, 19) + 'Z'
@@ -58,7 +85,7 @@ async function main() {
     '--data-only',
     '--no-owner',
     '--no-privileges',
-    ...BACKUP_TABLES.map((t) => `--table=public.${t}`),
+    ...resolved.map((t) => `--table=public.${t}`),
   ], { binaryOut: true })
   fs.writeFileSync(file, stdout)
 
@@ -72,7 +99,7 @@ async function main() {
   console.log(`  bytes:    ${fs.statSync(file).size}`)
   console.log(`  sha256:   ${sha}`)
   console.log(`  postgres: server major ${serverMajor}`)
-  console.log(`  tables:   ${BACKUP_TABLES.length} allowlisted (${BACKUP_TABLES.join(', ')})`)
+  console.log(`  tables:   ${resolved.length} of ${BACKUP_TABLES.length} allowlisted (${resolved.join(', ')})`)
   console.log('store this artifact ONLY in an encrypted vault / encrypted disk / encrypted off-site location')
 }
 

scripts/backup-scope.mjs

@@ -1,13 +1,21 @@
 // Backup scope — the single classification of every public table.
 //
 // The backup is ALLOWLIST-ONLY: pg_dump receives repeated
-// --table=public.<name> for exactly BACKUP_TABLES. Supabase warns that raw
-// pg_dump can pick up platform-internal schemas/objects, so the safe
-// property here is "unknown/new schema or table → NOT backed up until
-// explicitly classified". backup-db.mjs, recovery-drill.mjs, the tests and
-// the docs all derive from these lists; a new table added by a migration
-// but not classified fails CI loudly instead of silently entering (or
-// silently missing) the dump.
+// --table=public.<name> for exactly the tables resolveBackupTables()
+// returns — a subset of BACKUP_TABLES valid for the target's applied
+// migration state. Supabase warns that raw pg_dump can pick up
+// platform-internal schemas/objects, so the safe property here is
+// "unknown/new schema or table → NOT backed up until explicitly
+// classified". backup-db.mjs, recovery-drill.mjs, the tests and the docs
+// all derive from these lists; a new table added by a migration but not
+// classified fails CI loudly instead of silently entering (or silently
+// missing) the dump.
+
+import fs from 'node:fs'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
 
 // CORE — irreplaceable business data. Losing these means real user loss.
 export const CORE_TABLES = [
@@ -48,5 +56,125 @@ export const EPHEMERAL_TABLES = [
   'schema_migrations',     // rebuilt by the migration runner, not restored
 ]
 
-// The canonical dump allowlist — the ONLY tables pg_dump is asked for.
+// The canonical dump allowlist — the ONLY tables pg_dump may ever be asked
+// for. The set actually requested for a given target is computed by
+// resolveBackupTables().
 export const BACKUP_TABLES = [...CORE_TABLES, ...OPERATIONAL_TABLES]
+
+// ── migration-aware resolution ───────────────────────────────────────────
+//
+// A target database may lag repository main (e.g. a mandatory pre-migration
+// backup taken while migrations/012_ownership.sql is committed but not yet
+// applied). Requesting --table for an absent table makes pg_dump fail —
+// but silently skipping absent allowlisted tables would mask a broken
+// schema. So resolution is:
+//
+//   backup set = BACKUP_TABLES ∩ tables the APPLIED migrations create
+//
+// plus fail-closed validation:
+//   * a table the applied schema expects but the DB lacks → broken schema
+//   * an allowlisted table physically present whose migration is NOT
+//     recorded in schema_migrations → bookkeeping inconsistency
+//
+// Unknown/unclassified tables in the database never enter the result.
+
+// filename -> { created, dropped } Sets of tables that migration touches,
+// parsed from the repo migration files — the schema source of truth.
+export function migrationTables() {
+  const dir = path.join(root, 'migrations')
+  const map = new Map()
+  for (const f of fs.readdirSync(dir).filter((x) => x.endsWith('.sql')).sort()) {
+    const text = fs.readFileSync(path.join(dir, f), 'utf8')
+    const created = new Set()
+    const dropped = new Set()
+    for (const m of text.matchAll(/create table(?:\s+if not exists)?\s+(?:public\.)?(\w+)/gi)) created.add(m[1])
+    for (const m of text.matchAll(/drop table(?:\s+if exists)?\s+(?:public\.)?(\w+)/gi)) dropped.add(m[1])
+    map.set(f, { created, dropped })
+  }
+  return map
+}
+
+export function migrationFiles() {
+  return [...migrationTables().keys()]
+}
+
+// The applied set must be EXACTLY a prefix of this checkout's ordered
+// migration list:
+//   * an applied filename unknown to this checkout → the target schema is
+//     ahead of the code — an older backup tool cannot know whether its
+//     allowlist is still complete, so it must fail closed
+//   * a skipped known migration (a gap) → broken migration bookkeeping
+// Lagging repo main is legitimate and valid.
+export function validateAppliedMigrationPrefix(applied) {
+  const repoFiles = migrationFiles()
+  const repoSet = new Set(repoFiles)
+  const seen = new Set()
+  for (const f of applied) {
+    if (!repoSet.has(f)) {
+      throw new Error(
+        `target database schema is ahead of this repository checkout ` +
+        `(unknown applied migration ${f}); update the repository before creating a backup`
+      )
+    }
+    seen.add(f)
+  }
+  const prefix = repoFiles.slice(0, seen.size)
+  const missing = prefix.filter((f) => !seen.has(f))
+  if (missing.length) {
+    throw new Error(
+      `applied migrations are not a contiguous prefix of the repo list — ` +
+      `missing ${missing.join(', ')}; refusing to reason about a gapped schema`
+    )
+  }
+  return true
+}
+
+// Tables the applied migration set is expected to have created, replayed
+// in filename order so drops in later migrations (e.g. 002 dropping 001's
+// sync_state/daily_activity) actually remove them.
+export function expectedTables(applied) {
+  const byMig = migrationTables()
+  const appliedSet = new Set(applied)
+  const expected = new Set()
+  for (const [file, { created, dropped }] of byMig) {
+    if (!appliedSet.has(file)) continue
+    for (const t of dropped) expected.delete(t)
+    for (const t of created) expected.add(t)
+  }
+  return expected
+}
+
+// Exact safe table list for the target DB.
+//   applied  — iterable of filenames recorded in public.schema_migrations
+//   existing — iterable of BASE TABLE names present in the public schema
+// Returns BACKUP_TABLES filtered to the applied-schema table set, or throws.
+export function resolveBackupTables({ applied, existing }) {
+  // The applied ledger must be a contiguous prefix of this checkout's
+  // migration list before any resolution happens.
+  const appliedList = [...applied]
+  validateAppliedMigrationPrefix(appliedList)
+  const appliedSet = new Set(appliedList)
+  const existingSet = new Set(existing)
+  const expected = expectedTables(appliedSet)
+
+  // fail closed: applied schema claims tables the DB does not have
+  const missing = [...expected].filter((t) => !existingSet.has(t))
+  if (missing.length) {
+    throw new Error(
+      `applied schema expects table(s) that do not exist: ${missing.join(', ')} — refusing to produce a partial backup`
+    )
+  }
+
+  // fail closed: an allowlisted table exists but its migration is not
+  // recorded — the schema and schema_migrations disagree.
+  const unrecorded = [...existingSet].filter(
+    (t) => BACKUP_TABLES.includes(t) && !expected.has(t)
+  )
+  if (unrecorded.length) {
+    throw new Error(
+      `allowlisted table(s) present without a recorded migration: ${unrecorded.join(', ')} — refusing to guess backup scope`
+    )
+  }
+
+  return BACKUP_TABLES.filter((t) => expected.has(t))
+}