feat: incremental db push (baseline + row-delta) — #17

CHANGELOG.md

@@ -1,6 +1,19 @@
 # Changelog
 
-## 0.0.1-beta.25 (2026-06-25)
+## Unreleased
+
+### Added — incremental `db push` (#17)
+
+- **`db push --incremental`** ships only the **row-level delta** since the last push instead of the whole DB: it diffs the dump against a per-site baseline (`~/.instawp/baselines/<id>/`) and applies a minimal `REPLACE`/`DELETE` set (no `DROP`/`CREATE`). First run, a schema/DDL change, or **`--full`** do a normal full push and refresh the baseline. Requires a per-row dump (`mysqldump --skip-extended-insert --order-by-primary`; extended-insert dumps are rejected). Reuses the existing safety machinery — remote backup first, prefix/role-key remap, scoped URL search-replace, `--verify`. **The full `db push` path is unchanged; incremental is purely additive.**
+- MVP scope: tables with a single-column primary key (WP core + most plugin tables); anything without one — or any DDL change — auto-falls-back to a full push. The diff engine (`lib/db-delta.ts`) and baseline store (`lib/db-baseline.ts`) are pure and unit-tested.
+- **Memory-bounded:** the baseline is stored as a compact per-row **hash manifest** (PK → content hash), not the full dump, and the current dump is **streamed** line-by-line (gz-decompressed on the fly) — so a large DB diffs in a few hundred MB instead of loading both dumps into the heap (a naive 2× full-dump diff OOM'd at ~4 GB on a 166 MB DB).
+
+### Added — db push readiness/scoping + backup retention (large-DB feedback #16)
+
+- **`db push --verify`** polls the site URL until it answers HTTP after the import (a large import can briefly return `000` right after import/flush); reported as `verified` in the summary.
+- **`db push --sr-tables <table...>`** scopes `--search-replace` to specific (prefixed) tables instead of all tables — faster on big DBs whose bulk rows have no URLs. Default stays `--all-tables`.
+- **`db backups list <site>`** and **`db backups prune <site> [--keep <n>] [--older-than <days>] [--force]`** manage the `~/db-backup-*.sql.gz` files `db push` leaves behind (they previously accumulated with no way to view/clean them).
+- Internal: extracted the HTTP-readiness helper to `lib/http-ready.ts` (now shared by `sites create` and `db push --verify`).
 
 ### Improved — db push transport & progress (large-DB path)
 

src/__tests__/db-backups.test.ts

@@ -0,0 +1,69 @@
+import { describe, it, expect } from 'vitest';
+import { parseBackupList, selectBackupsToPrune } from '../lib/db-backups.js';
+
+describe('parseBackupList', () => {
+  it('parses tab-separated stat output, newest first', () => {
+    const out = [
+      '/home/u/db-backup-2026-06-20.sql.gz\t1048576\t1718870400',
+      '/home/u/db-backup-2026-06-25.sql.gz\t2097152\t1719273600',
+    ].join('\n');
+    const list = parseBackupList(out);
+    expect(list).toHaveLength(2);
+    expect(list[0].file).toBe('/home/u/db-backup-2026-06-25.sql.gz'); // newest first
+    expect(list[0].sizeBytes).toBe(2097152);
+    expect(list[1].mtime).toBe(1718870400);
+  });
+
+  it('tolerates an SSH banner/MOTD and skips non-matching lines', () => {
+    const out = [
+      'Welcome to Ubuntu 22.04',
+      'Last login: ...',
+      '/home/u/db-backup-a.sql.gz\t100\t1719273600',
+      'some junk line without tabs',
+      '/home/u/notanumber.sql.gz\tNaN\tNaN',
+    ].join('\n');
+    const list = parseBackupList(out);
+    expect(list).toHaveLength(1);
+    expect(list[0].file).toBe('/home/u/db-backup-a.sql.gz');
+  });
+
+  it('returns [] for empty input', () => {
+    expect(parseBackupList('')).toEqual([]);
+  });
+});
+
+describe('selectBackupsToPrune', () => {
+  const now = 1_719_273_600; // fixed "now"
+  const day = 86400;
+  // newest → oldest
+  const backups = [
+    { file: 'd', sizeBytes: 1, mtime: now - 1 * day },
+    { file: 'c', sizeBytes: 1, mtime: now - 3 * day },
+    { file: 'b', sizeBytes: 1, mtime: now - 10 * day },
+    { file: 'a', sizeBytes: 1, mtime: now - 30 * day },
+  ];
+
+  it('--keep N keeps the newest N, deletes the rest', () => {
+    const { toDelete, toKeep } = selectBackupsToPrune(backups, { keep: 2 }, now);
+    expect(toKeep.map((b) => b.file)).toEqual(['d', 'c']);
+    expect(toDelete.map((b) => b.file)).toEqual(['b', 'a']);
+  });
+
+  it('--older-than D deletes backups older than D days', () => {
+    const { toDelete, toKeep } = selectBackupsToPrune(backups, { olderThanDays: 5 }, now);
+    expect(toDelete.map((b) => b.file)).toEqual(['b', 'a']); // 10d, 30d old
+    expect(toKeep.map((b) => b.file)).toEqual(['d', 'c']);   // 1d, 3d
+  });
+
+  it('combines criteria with OR (deleted if beyond keep OR too old)', () => {
+    const { toDelete } = selectBackupsToPrune(backups, { keep: 3, olderThanDays: 5 }, now);
+    // beyond keep:3 → 'a'; older than 5d → 'b','a'  → union {b,a}
+    expect(toDelete.map((b) => b.file).sort()).toEqual(['a', 'b']);
+  });
+
+  it('deletes nothing when no criteria are given', () => {
+    const { toDelete, toKeep } = selectBackupsToPrune(backups, {}, now);
+    expect(toDelete).toHaveLength(0);
+    expect(toKeep).toHaveLength(4);
+  });
+});

src/__tests__/db-baseline.test.ts

@@ -0,0 +1,41 @@
+import { describe, it, expect, afterEach } from 'vitest';
+import { tmpdir } from 'node:os';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { baselineDir, loadBaseline, saveBaseline } from '../lib/db-baseline.js';
+import { buildManifest } from '../lib/db-delta.js';
+
+const dirs: string[] = [];
+function tmpBase(): string {
+  const d = mkdtempSync(join(tmpdir(), 'iwp-baseline-'));
+  dirs.push(d);
+  return d;
+}
+afterEach(() => { for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); });
+
+const DDL = `CREATE TABLE \`wp_options\` (
+  \`option_id\` bigint(20) NOT NULL AUTO_INCREMENT,
+  \`option_name\` varchar(191) NOT NULL,
+  PRIMARY KEY (\`option_id\`)
+) ENGINE=InnoDB;`;
+
+describe('db-baseline manifest store', () => {
+  it('round-trips a manifest per site', async () => {
+    const base = tmpBase();
+    expect(loadBaseline(42, base)).toBeNull();
+    const m = await buildManifest([DDL, "INSERT INTO `wp_options` VALUES (1,'siteurl','http://x');"].join('\n').split('\n'));
+    saveBaseline(42, m, base);
+    const got = loadBaseline(42, base);
+    expect(got?.fingerprint).toBe(m.fingerprint);
+    expect(got?.single.get('wp_options')?.rows.get('1')).toBe(m.single.get('wp_options')?.rows.get('1'));
+  });
+
+  it('scopes baselines per site id', async () => {
+    const base = tmpBase();
+    const m = await buildManifest([DDL, "INSERT INTO `wp_options` VALUES (1,'a','b');"].join('\n').split('\n'));
+    saveBaseline(1, m, base);
+    expect(loadBaseline(1, base)?.fingerprint).toBe(m.fingerprint);
+    expect(loadBaseline(2, base)).toBeNull();
+    expect(baselineDir(1, base)).not.toBe(baselineDir(2, base));
+  });
+});

src/__tests__/db-delta.test.ts

@@ -0,0 +1,161 @@
+import { describe, it, expect } from 'vitest';
+import {
+  splitSqlTuple,
+  parseInsertStatement,
+  prefixFromTableNames,
+  buildManifest,
+  diffAgainstManifest,
+  serializeManifest,
+  deserializeManifest,
+  ExtendedInsertError,
+} from '../lib/db-delta.js';
+
+const OPTIONS_DDL = (autoInc = 120) => `CREATE TABLE \`wp_options\` (
+  \`option_id\` bigint(20) unsigned NOT NULL AUTO_INCREMENT,
+  \`option_name\` varchar(191) NOT NULL DEFAULT '',
+  \`option_value\` longtext NOT NULL,
+  PRIMARY KEY (\`option_id\`)
+) ENGINE=InnoDB AUTO_INCREMENT=${autoInc} DEFAULT CHARSET=utf8mb4;`;
+
+const TR_DDL = `CREATE TABLE \`wp_term_relationships\` (
+  \`object_id\` bigint(20) NOT NULL,
+  \`term_taxonomy_id\` bigint(20) NOT NULL,
+  PRIMARY KEY (\`object_id\`,\`term_taxonomy_id\`)
+) ENGINE=InnoDB;`;
+
+const lines = (s: string) => s.split('\n');
+const dump = (ddl: string, rows: string[]) => lines([ddl, ...rows].join('\n'));
+const opts = { dumpPrefix: 'wp_', remotePrefix: 'iwpa4c7_' };
+
+describe('splitSqlTuple', () => {
+  it('respects commas/parens/quotes inside strings', () => {
+    expect(splitSqlTuple("5,'a, b (c)',7")).toEqual(['5', "'a, b (c)'", '7']);
+    expect(splitSqlTuple("1,'it\\'s ok',2")).toEqual(['1', "'it\\'s ok'", '2']);
+  });
+});
+
+describe('parseInsertStatement', () => {
+  it('parses a single-row insert and flags extended', () => {
+    expect(parseInsertStatement("INSERT INTO `t` VALUES (1,'a');")?.tuples).toEqual(["1,'a'"]);
+    expect(parseInsertStatement("INSERT INTO `t` VALUES (1,'a'),(2,'b');")?.tuples.length).toBe(2);
+  });
+});
+
+describe('prefixFromTableNames', () => {
+  it('derives the table prefix from core table names', () => {
+    expect(prefixFromTableNames(['iwpa4c7_postmeta', 'iwpa4c7_options'])).toBe('iwpa4c7_');
+    expect(prefixFromTableNames(['wp_users'])).toBe('wp_');
+    expect(prefixFromTableNames(['random_table'])).toBeNull();
+  });
+});
+
+describe('buildManifest', () => {
+  it('captures single-PK row hashes, composite aggregates, and a schema fingerprint', async () => {
+    const m = await buildManifest(dump(OPTIONS_DDL(), [
+      "INSERT INTO `wp_options` VALUES (1,'siteurl','http://x');",
+      "INSERT INTO `wp_options` VALUES (2,'blogname','S');",
+    ]).concat(lines(TR_DDL), ['INSERT INTO `wp_term_relationships` VALUES (5,9);']));
+    expect(m.single.get('wp_options')?.rows.size).toBe(2);
+    expect(m.single.get('wp_options')?.pkCol).toBe('option_id');
+    expect(m.composite.get('wp_term_relationships')?.count).toBe(1);
+    expect(m.fingerprint).toMatch(/^[0-9a-f]{64}$/);
+  });
+
+  it('fingerprint is stable across AUTO_INCREMENT drift and "CREATE TABLE" in row data (B1)', async () => {
+    const adversarial = (v: string) => dump(OPTIONS_DDL(v === 'v1' ? 120 : 988), [
+      `INSERT INTO \`wp_options\` VALUES (1,'doc','says CREATE TABLE \`evil\` (x int); ${v}');`,
+    ]);
+    const a = await buildManifest(adversarial('v1'));
+    const b = await buildManifest(adversarial('v2'));
+    expect(a.fingerprint).toBe(b.fingerprint);
+  });
+
+  it('throws on an extended-insert dump', async () => {
+    await expect(buildManifest(dump(OPTIONS_DDL(), ["INSERT INTO `wp_options` VALUES (1,'a','b'),(2,'c','d');"])))
+      .rejects.toBeInstanceOf(ExtendedInsertError);
+  });
+});
+
+describe('diffAgainstManifest', () => {
+  const base = dump(OPTIONS_DDL(), [
+    "INSERT INTO `wp_options` VALUES (1,'siteurl','http://localhost:10115');",
+    "INSERT INTO `wp_options` VALUES (2,'blogname','Old Name');",
+    "INSERT INTO `wp_options` VALUES (3,'gone','x');",
+  ]);
+
+  it('emits nothing when nothing changed', async () => {
+    const m = await buildManifest(base);
+    const r = await diffAgainstManifest(base, m, { dumpPrefix: 'wp_', remotePrefix: 'wp_' });
+    expect(r.mode).toBe('delta');
+    expect(r.sql).toBe('');
+    expect(r.stats).toEqual({ tablesChanged: 0, replaces: 0, deletes: 0 });
+  });
+
+  it('REPLACE for changed/new rows, DELETE for removed, remapping the prefix', async () => {
+    const m = await buildManifest(base);
+    const cur = dump(OPTIONS_DDL(), [
+      "INSERT INTO `wp_options` VALUES (1,'siteurl','http://localhost:10115');", // unchanged
+      "INSERT INTO `wp_options` VALUES (2,'blogname','New Name');",             // changed
+      // row 3 removed
+      "INSERT INTO `wp_options` VALUES (4,'new_opt','v');",                      // new
+    ]);
+    const r = await diffAgainstManifest(cur, m, opts);
+    expect(r.mode).toBe('delta');
+    expect(r.stats).toEqual({ tablesChanged: 1, replaces: 2, deletes: 1 });
+    expect(r.sql).toContain("REPLACE INTO `iwpa4c7_options` VALUES (2,'blogname','New Name');");
+    expect(r.sql).toContain("REPLACE INTO `iwpa4c7_options` VALUES (4,'new_opt','v');");
+    expect(r.sql).toContain('DELETE FROM `iwpa4c7_options` WHERE `option_id`=3;');
+    expect(r.sql).not.toContain('siteurl');
+    expect(r.sql).not.toContain('`wp_options`');
+  });
+
+  it('falls back to full when the schema changed', async () => {
+    const m = await buildManifest(base);
+    const cur = dump(OPTIONS_DDL().replace('`option_value` longtext NOT NULL,', '`option_value` longtext NOT NULL,\n  `autoload` varchar(20) NOT NULL DEFAULT \'yes\','), [
+      "INSERT INTO `wp_options` VALUES (1,'siteurl','http://x','yes');",
+    ]);
+    const r = await diffAgainstManifest(cur, m, opts);
+    expect(r.mode).toBe('full');
+    expect(r.reason).toMatch(/schema/i);
+  });
+
+  // B2 regression: unchanged composite-PK table must not force a full push.
+  it('ignores an UNCHANGED composite-PK table and still deltas the changed single-PK table', async () => {
+    const mk = (name: string) => dump(OPTIONS_DDL(), [`INSERT INTO \`wp_options\` VALUES (1,'blogname','${name}');`])
+      .concat(lines(TR_DDL), ['INSERT INTO `wp_term_relationships` VALUES (5,9);']);
+    const m = await buildManifest(mk('Old'));
+    const r = await diffAgainstManifest(mk('New'), m, opts);
+    expect(r.mode).toBe('delta');
+    expect(r.stats).toEqual({ tablesChanged: 1, replaces: 1, deletes: 0 });
+    expect(r.sql).toContain('REPLACE INTO `iwpa4c7_options`');
+    expect(r.sql).not.toContain('term_relationships');
+  });
+
+  it('falls back to full when a composite-PK table actually changed', async () => {
+    const m = await buildManifest(dump(TR_DDL, ['INSERT INTO `wp_term_relationships` VALUES (1,2);']));
+    const r = await diffAgainstManifest(dump(TR_DDL, ['INSERT INTO `wp_term_relationships` VALUES (1,3);']), m, { dumpPrefix: 'wp_', remotePrefix: 'wp_' });
+    expect(r.mode).toBe('full');
+    expect(r.reason).toMatch(/no single-column primary key/i);
+  });
+
+  it('builds a fresh newManifest each diff (for re-basing)', async () => {
+    const m = await buildManifest(base);
+    const cur = dump(OPTIONS_DDL(), ["INSERT INTO `wp_options` VALUES (1,'siteurl','http://x');"]);
+    const r = await diffAgainstManifest(cur, m, opts);
+    expect(r.newManifest.single.get('wp_options')?.rows.size).toBe(1);
+  });
+});
+
+describe('serialize/deserialize manifest', () => {
+  it('round-trips single + composite tables', async () => {
+    const m = await buildManifest(dump(OPTIONS_DDL(), ["INSERT INTO `wp_options` VALUES (1,'a','b');"]).concat(lines(TR_DDL), ['INSERT INTO `wp_term_relationships` VALUES (5,9);']));
+    const back = deserializeManifest(serializeManifest(m));
+    expect(back.fingerprint).toBe(m.fingerprint);
+    expect(back.single.get('wp_options')?.rows.get('1')).toBe(m.single.get('wp_options')?.rows.get('1'));
+    expect(back.composite.get('wp_term_relationships')?.count).toBe(1);
+    // a diff using the deserialized manifest behaves identically (no false change)
+    const r = await diffAgainstManifest(dump(OPTIONS_DDL(), ["INSERT INTO `wp_options` VALUES (1,'a','b');"]).concat(lines(TR_DDL), ['INSERT INTO `wp_term_relationships` VALUES (5,9);']), back, { dumpPrefix: 'wp_', remotePrefix: 'wp_' });
+    expect(r.mode).toBe('delta');
+    expect(r.sql).toBe('');
+  });
+});

src/__tests__/http-ready.test.ts

@@ -0,0 +1,22 @@
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import { waitForHttp } from '../lib/http-ready.js';
+
+afterEach(() => {
+  vi.unstubAllGlobals();
+});
+
+describe('waitForHttp', () => {
+  it('returns true as soon as the URL answers (any HTTP response)', async () => {
+    const fetchMock = vi.fn().mockResolvedValue({ status: 200 });
+    vi.stubGlobal('fetch', fetchMock);
+    await expect(waitForHttp('https://x.test', 5000)).resolves.toBe(true);
+    expect(fetchMock).toHaveBeenCalledOnce();
+  });
+
+  it('returns false without calling fetch when the budget is already exhausted', async () => {
+    const fetchMock = vi.fn();
+    vi.stubGlobal('fetch', fetchMock);
+    await expect(waitForHttp('https://x.test', 0)).resolves.toBe(false);
+    expect(fetchMock).not.toHaveBeenCalled();
+  });
+});