diff --git a/CHANGELOG.md b/CHANGELOG.md index 54785cb..4fb8a8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ # Changelog -## 0.0.1-beta.25 (2026-06-25) +## Unreleased + +### Added — incremental `db push` (#17) + +- **`db push --incremental`** ships only the **row-level delta** since the last push instead of the whole DB: it diffs the dump against a per-site baseline (`~/.instawp/baselines//`) and applies a minimal `REPLACE`/`DELETE` set (no `DROP`/`CREATE`). First run, a schema/DDL change, or **`--full`** do a normal full push and refresh the baseline. Requires a per-row dump (`mysqldump --skip-extended-insert --order-by-primary`; extended-insert dumps are rejected). Reuses the existing safety machinery — remote backup first, prefix/role-key remap, scoped URL search-replace, `--verify`. **The full `db push` path is unchanged; incremental is purely additive.** +- MVP scope: tables with a single-column primary key (WP core + most plugin tables); anything without one — or any DDL change — auto-falls-back to a full push. The diff engine (`lib/db-delta.ts`) and baseline store (`lib/db-baseline.ts`) are pure and unit-tested. +- **Memory-bounded:** the baseline is stored as a compact per-row **hash manifest** (PK → content hash), not the full dump, and the current dump is **streamed** line-by-line (gz-decompressed on the fly) — so a large DB diffs in a few hundred MB instead of loading both dumps into the heap (a naive 2× full-dump diff OOM'd at ~4 GB on a 166 MB DB). + +### Added — db push readiness/scoping + backup retention (large-DB feedback #16) + +- **`db push --verify`** polls the site URL until it answers HTTP after the import (a large import can briefly return `000` right after import/flush); reported as `verified` in the summary. +- **`db push --sr-tables `** scopes `--search-replace` to specific (prefixed) tables instead of all tables — faster on big DBs whose bulk rows have no URLs. Default stays `--all-tables`. +- **`db backups list `** and **`db backups prune [--keep ] [--older-than ] [--force]`** manage the `~/db-backup-*.sql.gz` files `db push` leaves behind (they previously accumulated with no way to view/clean them). +- Internal: extracted the HTTP-readiness helper to `lib/http-ready.ts` (now shared by `sites create` and `db push --verify`). ### Improved — db push transport & progress (large-DB path) diff --git a/src/__tests__/db-backups.test.ts b/src/__tests__/db-backups.test.ts new file mode 100644 index 0000000..ad07616 --- /dev/null +++ b/src/__tests__/db-backups.test.ts @@ -0,0 +1,69 @@ +import { describe, it, expect } from 'vitest'; +import { parseBackupList, selectBackupsToPrune } from '../lib/db-backups.js'; + +describe('parseBackupList', () => { + it('parses tab-separated stat output, newest first', () => { + const out = [ + '/home/u/db-backup-2026-06-20.sql.gz\t1048576\t1718870400', + '/home/u/db-backup-2026-06-25.sql.gz\t2097152\t1719273600', + ].join('\n'); + const list = parseBackupList(out); + expect(list).toHaveLength(2); + expect(list[0].file).toBe('/home/u/db-backup-2026-06-25.sql.gz'); // newest first + expect(list[0].sizeBytes).toBe(2097152); + expect(list[1].mtime).toBe(1718870400); + }); + + it('tolerates an SSH banner/MOTD and skips non-matching lines', () => { + const out = [ + 'Welcome to Ubuntu 22.04', + 'Last login: ...', + '/home/u/db-backup-a.sql.gz\t100\t1719273600', + 'some junk line without tabs', + '/home/u/notanumber.sql.gz\tNaN\tNaN', + ].join('\n'); + const list = parseBackupList(out); + expect(list).toHaveLength(1); + expect(list[0].file).toBe('/home/u/db-backup-a.sql.gz'); + }); + + it('returns [] for empty input', () => { + expect(parseBackupList('')).toEqual([]); + }); +}); + +describe('selectBackupsToPrune', () => { + const now = 1_719_273_600; // fixed "now" + const day = 86400; + // newest → oldest + const backups = [ + { file: 'd', sizeBytes: 1, mtime: now - 1 * day }, + { file: 'c', sizeBytes: 1, mtime: now - 3 * day }, + { file: 'b', sizeBytes: 1, mtime: now - 10 * day }, + { file: 'a', sizeBytes: 1, mtime: now - 30 * day }, + ]; + + it('--keep N keeps the newest N, deletes the rest', () => { + const { toDelete, toKeep } = selectBackupsToPrune(backups, { keep: 2 }, now); + expect(toKeep.map((b) => b.file)).toEqual(['d', 'c']); + expect(toDelete.map((b) => b.file)).toEqual(['b', 'a']); + }); + + it('--older-than D deletes backups older than D days', () => { + const { toDelete, toKeep } = selectBackupsToPrune(backups, { olderThanDays: 5 }, now); + expect(toDelete.map((b) => b.file)).toEqual(['b', 'a']); // 10d, 30d old + expect(toKeep.map((b) => b.file)).toEqual(['d', 'c']); // 1d, 3d + }); + + it('combines criteria with OR (deleted if beyond keep OR too old)', () => { + const { toDelete } = selectBackupsToPrune(backups, { keep: 3, olderThanDays: 5 }, now); + // beyond keep:3 → 'a'; older than 5d → 'b','a' → union {b,a} + expect(toDelete.map((b) => b.file).sort()).toEqual(['a', 'b']); + }); + + it('deletes nothing when no criteria are given', () => { + const { toDelete, toKeep } = selectBackupsToPrune(backups, {}, now); + expect(toDelete).toHaveLength(0); + expect(toKeep).toHaveLength(4); + }); +}); diff --git a/src/__tests__/db-baseline.test.ts b/src/__tests__/db-baseline.test.ts new file mode 100644 index 0000000..6e7e420 --- /dev/null +++ b/src/__tests__/db-baseline.test.ts @@ -0,0 +1,41 @@ +import { describe, it, expect, afterEach } from 'vitest'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { baselineDir, loadBaseline, saveBaseline } from '../lib/db-baseline.js'; +import { buildManifest } from '../lib/db-delta.js'; + +const dirs: string[] = []; +function tmpBase(): string { + const d = mkdtempSync(join(tmpdir(), 'iwp-baseline-')); + dirs.push(d); + return d; +} +afterEach(() => { for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); }); + +const DDL = `CREATE TABLE \`wp_options\` ( + \`option_id\` bigint(20) NOT NULL AUTO_INCREMENT, + \`option_name\` varchar(191) NOT NULL, + PRIMARY KEY (\`option_id\`) +) ENGINE=InnoDB;`; + +describe('db-baseline manifest store', () => { + it('round-trips a manifest per site', async () => { + const base = tmpBase(); + expect(loadBaseline(42, base)).toBeNull(); + const m = await buildManifest([DDL, "INSERT INTO `wp_options` VALUES (1,'siteurl','http://x');"].join('\n').split('\n')); + saveBaseline(42, m, base); + const got = loadBaseline(42, base); + expect(got?.fingerprint).toBe(m.fingerprint); + expect(got?.single.get('wp_options')?.rows.get('1')).toBe(m.single.get('wp_options')?.rows.get('1')); + }); + + it('scopes baselines per site id', async () => { + const base = tmpBase(); + const m = await buildManifest([DDL, "INSERT INTO `wp_options` VALUES (1,'a','b');"].join('\n').split('\n')); + saveBaseline(1, m, base); + expect(loadBaseline(1, base)?.fingerprint).toBe(m.fingerprint); + expect(loadBaseline(2, base)).toBeNull(); + expect(baselineDir(1, base)).not.toBe(baselineDir(2, base)); + }); +}); diff --git a/src/__tests__/db-delta.test.ts b/src/__tests__/db-delta.test.ts new file mode 100644 index 0000000..1e174a1 --- /dev/null +++ b/src/__tests__/db-delta.test.ts @@ -0,0 +1,161 @@ +import { describe, it, expect } from 'vitest'; +import { + splitSqlTuple, + parseInsertStatement, + prefixFromTableNames, + buildManifest, + diffAgainstManifest, + serializeManifest, + deserializeManifest, + ExtendedInsertError, +} from '../lib/db-delta.js'; + +const OPTIONS_DDL = (autoInc = 120) => `CREATE TABLE \`wp_options\` ( + \`option_id\` bigint(20) unsigned NOT NULL AUTO_INCREMENT, + \`option_name\` varchar(191) NOT NULL DEFAULT '', + \`option_value\` longtext NOT NULL, + PRIMARY KEY (\`option_id\`) +) ENGINE=InnoDB AUTO_INCREMENT=${autoInc} DEFAULT CHARSET=utf8mb4;`; + +const TR_DDL = `CREATE TABLE \`wp_term_relationships\` ( + \`object_id\` bigint(20) NOT NULL, + \`term_taxonomy_id\` bigint(20) NOT NULL, + PRIMARY KEY (\`object_id\`,\`term_taxonomy_id\`) +) ENGINE=InnoDB;`; + +const lines = (s: string) => s.split('\n'); +const dump = (ddl: string, rows: string[]) => lines([ddl, ...rows].join('\n')); +const opts = { dumpPrefix: 'wp_', remotePrefix: 'iwpa4c7_' }; + +describe('splitSqlTuple', () => { + it('respects commas/parens/quotes inside strings', () => { + expect(splitSqlTuple("5,'a, b (c)',7")).toEqual(['5', "'a, b (c)'", '7']); + expect(splitSqlTuple("1,'it\\'s ok',2")).toEqual(['1', "'it\\'s ok'", '2']); + }); +}); + +describe('parseInsertStatement', () => { + it('parses a single-row insert and flags extended', () => { + expect(parseInsertStatement("INSERT INTO `t` VALUES (1,'a');")?.tuples).toEqual(["1,'a'"]); + expect(parseInsertStatement("INSERT INTO `t` VALUES (1,'a'),(2,'b');")?.tuples.length).toBe(2); + }); +}); + +describe('prefixFromTableNames', () => { + it('derives the table prefix from core table names', () => { + expect(prefixFromTableNames(['iwpa4c7_postmeta', 'iwpa4c7_options'])).toBe('iwpa4c7_'); + expect(prefixFromTableNames(['wp_users'])).toBe('wp_'); + expect(prefixFromTableNames(['random_table'])).toBeNull(); + }); +}); + +describe('buildManifest', () => { + it('captures single-PK row hashes, composite aggregates, and a schema fingerprint', async () => { + const m = await buildManifest(dump(OPTIONS_DDL(), [ + "INSERT INTO `wp_options` VALUES (1,'siteurl','http://x');", + "INSERT INTO `wp_options` VALUES (2,'blogname','S');", + ]).concat(lines(TR_DDL), ['INSERT INTO `wp_term_relationships` VALUES (5,9);'])); + expect(m.single.get('wp_options')?.rows.size).toBe(2); + expect(m.single.get('wp_options')?.pkCol).toBe('option_id'); + expect(m.composite.get('wp_term_relationships')?.count).toBe(1); + expect(m.fingerprint).toMatch(/^[0-9a-f]{64}$/); + }); + + it('fingerprint is stable across AUTO_INCREMENT drift and "CREATE TABLE" in row data (B1)', async () => { + const adversarial = (v: string) => dump(OPTIONS_DDL(v === 'v1' ? 120 : 988), [ + `INSERT INTO \`wp_options\` VALUES (1,'doc','says CREATE TABLE \`evil\` (x int); ${v}');`, + ]); + const a = await buildManifest(adversarial('v1')); + const b = await buildManifest(adversarial('v2')); + expect(a.fingerprint).toBe(b.fingerprint); + }); + + it('throws on an extended-insert dump', async () => { + await expect(buildManifest(dump(OPTIONS_DDL(), ["INSERT INTO `wp_options` VALUES (1,'a','b'),(2,'c','d');"]))) + .rejects.toBeInstanceOf(ExtendedInsertError); + }); +}); + +describe('diffAgainstManifest', () => { + const base = dump(OPTIONS_DDL(), [ + "INSERT INTO `wp_options` VALUES (1,'siteurl','http://localhost:10115');", + "INSERT INTO `wp_options` VALUES (2,'blogname','Old Name');", + "INSERT INTO `wp_options` VALUES (3,'gone','x');", + ]); + + it('emits nothing when nothing changed', async () => { + const m = await buildManifest(base); + const r = await diffAgainstManifest(base, m, { dumpPrefix: 'wp_', remotePrefix: 'wp_' }); + expect(r.mode).toBe('delta'); + expect(r.sql).toBe(''); + expect(r.stats).toEqual({ tablesChanged: 0, replaces: 0, deletes: 0 }); + }); + + it('REPLACE for changed/new rows, DELETE for removed, remapping the prefix', async () => { + const m = await buildManifest(base); + const cur = dump(OPTIONS_DDL(), [ + "INSERT INTO `wp_options` VALUES (1,'siteurl','http://localhost:10115');", // unchanged + "INSERT INTO `wp_options` VALUES (2,'blogname','New Name');", // changed + // row 3 removed + "INSERT INTO `wp_options` VALUES (4,'new_opt','v');", // new + ]); + const r = await diffAgainstManifest(cur, m, opts); + expect(r.mode).toBe('delta'); + expect(r.stats).toEqual({ tablesChanged: 1, replaces: 2, deletes: 1 }); + expect(r.sql).toContain("REPLACE INTO `iwpa4c7_options` VALUES (2,'blogname','New Name');"); + expect(r.sql).toContain("REPLACE INTO `iwpa4c7_options` VALUES (4,'new_opt','v');"); + expect(r.sql).toContain('DELETE FROM `iwpa4c7_options` WHERE `option_id`=3;'); + expect(r.sql).not.toContain('siteurl'); + expect(r.sql).not.toContain('`wp_options`'); + }); + + it('falls back to full when the schema changed', async () => { + const m = await buildManifest(base); + const cur = dump(OPTIONS_DDL().replace('`option_value` longtext NOT NULL,', '`option_value` longtext NOT NULL,\n `autoload` varchar(20) NOT NULL DEFAULT \'yes\','), [ + "INSERT INTO `wp_options` VALUES (1,'siteurl','http://x','yes');", + ]); + const r = await diffAgainstManifest(cur, m, opts); + expect(r.mode).toBe('full'); + expect(r.reason).toMatch(/schema/i); + }); + + // B2 regression: unchanged composite-PK table must not force a full push. + it('ignores an UNCHANGED composite-PK table and still deltas the changed single-PK table', async () => { + const mk = (name: string) => dump(OPTIONS_DDL(), [`INSERT INTO \`wp_options\` VALUES (1,'blogname','${name}');`]) + .concat(lines(TR_DDL), ['INSERT INTO `wp_term_relationships` VALUES (5,9);']); + const m = await buildManifest(mk('Old')); + const r = await diffAgainstManifest(mk('New'), m, opts); + expect(r.mode).toBe('delta'); + expect(r.stats).toEqual({ tablesChanged: 1, replaces: 1, deletes: 0 }); + expect(r.sql).toContain('REPLACE INTO `iwpa4c7_options`'); + expect(r.sql).not.toContain('term_relationships'); + }); + + it('falls back to full when a composite-PK table actually changed', async () => { + const m = await buildManifest(dump(TR_DDL, ['INSERT INTO `wp_term_relationships` VALUES (1,2);'])); + const r = await diffAgainstManifest(dump(TR_DDL, ['INSERT INTO `wp_term_relationships` VALUES (1,3);']), m, { dumpPrefix: 'wp_', remotePrefix: 'wp_' }); + expect(r.mode).toBe('full'); + expect(r.reason).toMatch(/no single-column primary key/i); + }); + + it('builds a fresh newManifest each diff (for re-basing)', async () => { + const m = await buildManifest(base); + const cur = dump(OPTIONS_DDL(), ["INSERT INTO `wp_options` VALUES (1,'siteurl','http://x');"]); + const r = await diffAgainstManifest(cur, m, opts); + expect(r.newManifest.single.get('wp_options')?.rows.size).toBe(1); + }); +}); + +describe('serialize/deserialize manifest', () => { + it('round-trips single + composite tables', async () => { + const m = await buildManifest(dump(OPTIONS_DDL(), ["INSERT INTO `wp_options` VALUES (1,'a','b');"]).concat(lines(TR_DDL), ['INSERT INTO `wp_term_relationships` VALUES (5,9);'])); + const back = deserializeManifest(serializeManifest(m)); + expect(back.fingerprint).toBe(m.fingerprint); + expect(back.single.get('wp_options')?.rows.get('1')).toBe(m.single.get('wp_options')?.rows.get('1')); + expect(back.composite.get('wp_term_relationships')?.count).toBe(1); + // a diff using the deserialized manifest behaves identically (no false change) + const r = await diffAgainstManifest(dump(OPTIONS_DDL(), ["INSERT INTO `wp_options` VALUES (1,'a','b');"]).concat(lines(TR_DDL), ['INSERT INTO `wp_term_relationships` VALUES (5,9);']), back, { dumpPrefix: 'wp_', remotePrefix: 'wp_' }); + expect(r.mode).toBe('delta'); + expect(r.sql).toBe(''); + }); +}); diff --git a/src/__tests__/http-ready.test.ts b/src/__tests__/http-ready.test.ts new file mode 100644 index 0000000..1e22601 --- /dev/null +++ b/src/__tests__/http-ready.test.ts @@ -0,0 +1,22 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { waitForHttp } from '../lib/http-ready.js'; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('waitForHttp', () => { + it('returns true as soon as the URL answers (any HTTP response)', async () => { + const fetchMock = vi.fn().mockResolvedValue({ status: 200 }); + vi.stubGlobal('fetch', fetchMock); + await expect(waitForHttp('https://x.test', 5000)).resolves.toBe(true); + expect(fetchMock).toHaveBeenCalledOnce(); + }); + + it('returns false without calling fetch when the budget is already exhausted', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + await expect(waitForHttp('https://x.test', 0)).resolves.toBe(false); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/commands/db.ts b/src/commands/db.ts index 5c794ae..e3b2a8a 100644 --- a/src/commands/db.ts +++ b/src/commands/db.ts @@ -1,7 +1,8 @@ import { Command } from 'commander'; import { join, dirname, basename } from 'node:path'; -import { existsSync, mkdirSync, statSync, createReadStream, createWriteStream, unlinkSync } from 'node:fs'; +import { existsSync, mkdirSync, statSync, createReadStream, createWriteStream, unlinkSync, writeFileSync } from 'node:fs'; import { createGunzip } from 'node:zlib'; +import { createInterface } from 'node:readline'; import { pipeline } from 'node:stream/promises'; import { randomBytes } from 'node:crypto'; import chalk from 'chalk'; @@ -11,7 +12,12 @@ import { ensureSshAccess } from '../lib/ssh-keys.js'; import { execViaSsh, execViaSshToFile, scpUpload } from '../lib/ssh-connection.js'; import { parseTablePrefix } from '../lib/local-instance.js'; import { detectDumpPrefix, readSqlHead, rewriteDumpPrefix } from '../lib/sql-dump.js'; -import { success, error, spinner, info, isJsonMode } from '../lib/output.js'; +import { waitForHttp } from '../lib/http-ready.js'; +import { parseBackupList, selectBackupsToPrune, type RemoteBackup } from '../lib/db-backups.js'; +import { buildManifest, diffAgainstManifest, prefixFromTableNames, ExtendedInsertError, type Manifest } from '../lib/db-delta.js'; +import { loadBaseline, saveBaseline } from '../lib/db-baseline.js'; +import { success, error, spinner, info, table, isJsonMode } from '../lib/output.js'; +import type { SshConnection } from '../types.js'; /** Timestamp like `2026-05-23T12-34-56` (filename-safe — `:` is illegal on Windows). */ function isoTimestamp(): string { @@ -50,6 +56,203 @@ function isShellSafeUrl(u: string): boolean { return /^https?:\/\/[^\s'"\\$`]+$/.test(u); } +/** Resolve a site and open an SSH connection (shared by the backups subcommands). */ +async function resolveAndConnect(siteIdentifier: string): Promise<{ site: any; conn: SshConnection }> { + const rspin = spinner('Resolving site...'); + rspin.start(); + let site: any; + try { + site = await resolveSite(siteIdentifier); + rspin.succeed(`Site: ${site.name || site.sub_domain} (ID: ${site.id})`); + } catch { + rspin.fail('Site resolution failed'); + process.exit(1); + } + const conn = await ensureSshAccess(site.id); + return { site, conn }; +} + +/** List `~/db-backup-*.sql.gz` on the remote (the files `db push` writes), newest first. */ +function listRemoteBackups(conn: SshConnection): RemoteBackup[] { + // GNU stat interprets the \t escapes; the for-loop guard avoids a literal glob + // when nothing matches. parseBackupList tolerates any MOTD/banner lines. + const cmd = `for f in ~/db-backup-*.sql.gz; do [ -e "$f" ] && stat -c '%n\\t%s\\t%Y' "$f"; done`; + const res = execViaSsh(conn, cmd); + return parseBackupList(res.stdout || ''); +} + +function isGzipped(file: string): boolean { + return file.endsWith('.gz') || file.endsWith('.gzip'); +} + +/** Stream a dump file's lines (decompressing a .gz on the fly). Bounded memory. */ +function openDumpLines(file: string): AsyncIterable { + const raw = createReadStream(file); + const input = isGzipped(file) ? raw.pipe(createGunzip()) : raw; + return createInterface({ input, crlfDelay: Infinity }); +} + +/** + * #17 — incremental push decision. Returns done=true if it fully handled the + * push (delta applied, no-op, or cancelled); done=false to fall through to the + * full push, carrying the baseline to persist after it succeeds. Additive: never + * mutates the full-push path. + */ +async function prepareIncremental(p: { + file: string; site: any; conn: SshConnection; wpPath: string; remoteHome: string; opts: any; + srFrom?: string; srTo?: string; srTables: string[]; +}): Promise<{ done: boolean; baseline?: Manifest }> { + const { file, site, conn, wpPath, remoteHome, opts, srFrom, srTo, srTables } = p; + try { + const baseline = opts.full ? null : loadBaseline(site.id); + + // First run / --full / no baseline → build a manifest from the current dump + // (streamed) and fall through to the full push, which refreshes the baseline. + if (!baseline) { + const why = opts.full ? '--full requested' : 'no baseline yet (first incremental push)'; + info(`Full push (${why}); the incremental baseline will be refreshed afterwards.`); + const mSpin = spinner('Indexing the dump for the baseline...'); + mSpin.start(); + const manifest = await buildManifest(openDumpLines(file)); + mSpin.succeed('Dump indexed'); + return { done: false, baseline: manifest }; + } + + // Diff the current dump (streamed) against the stored baseline manifest. + const pfxRes = execViaSsh(conn, `cd ${wpPath} && wp config get table_prefix`); + const remotePrefix = parseTablePrefix(pfxRes.exitCode === 0 ? pfxRes.stdout : '', 'wp_'); + const dumpPrefix = prefixFromTableNames([...baseline.single.keys(), ...baseline.composite.keys()]) ?? 'wp_'; + + const diffSpin = spinner('Computing the delta vs the baseline...'); + diffSpin.start(); + const result = await diffAgainstManifest(openDumpLines(file), baseline, { dumpPrefix, remotePrefix }); + diffSpin.stop(); + + if (result.mode === 'full') { + info(`Full push (${result.reason}); the incremental baseline will be refreshed afterwards.`); + return { done: false, baseline: result.newManifest }; + } + + if (!result.sql) { + info('No changes since the last push.'); + saveBaseline(site.id, result.newManifest); + success('Incremental push complete', { site_id: site.id, changed: false, replaces: 0, deletes: 0, tables_changed: 0 }); + return { done: true }; + } + + if (!opts.force && !isJsonMode()) { + console.log(`\nIncremental push to ${chalk.bold(conn.domain)}: ${chalk.bold(String(result.stats!.replaces))} change(s), ${chalk.bold(String(result.stats!.deletes))} deletion(s) across ${result.stats!.tablesChanged} table(s). A backup is taken first.`); + const ok = await promptYesNo('Apply this delta? (y/N) '); + if (!ok) { info('Cancelled.'); return { done: true }; } + } + + await applyDelta({ conn, wpPath, remoteHome, site, deltaSql: result.sql, stats: result.stats!, takeBackup: opts.backup !== false, remapFrom: dumpPrefix, remotePrefix, srFrom, srTo, srTables, verify: !!opts.verify }); + saveBaseline(site.id, result.newManifest); + return { done: true }; + } catch (e) { + if (e instanceof ExtendedInsertError) { + error('--incremental needs a per-row dump. Re-export with: mysqldump --skip-extended-insert --order-by-primary (or `wp db export` with those flags).'); + process.exit(1); + } + throw e; + } +} + +/** Apply a computed delta to the remote (backup → upload → import → remap → search-replace → verify). */ +async function applyDelta(p: { + conn: SshConnection; wpPath: string; remoteHome: string; site: any; + deltaSql: string; stats: { tablesChanged: number; replaces: number; deletes: number }; + takeBackup: boolean; remapFrom: string; remotePrefix: string; srFrom?: string; srTo?: string; srTables: string[]; verify: boolean; +}): Promise { + const { conn, wpPath, remoteHome, site, deltaSql, stats, takeBackup, remapFrom, remotePrefix, srFrom, srTo, srTables, verify } = p; + const startedAt = Date.now(); + const backupFilename = `db-backup-${isoTimestamp()}.sql.gz`; + const backupRemotePath = `${remoteHome}/${backupFilename}`; + + if (takeBackup) { + const s = spinner(`Backing up remote database to ~/${backupFilename}...`); + s.start(); + const r = execViaSsh(conn, `cd ${wpPath} && wp db export --single-transaction - | gzip > ${backupRemotePath}`); + if (r.exitCode !== 0) { s.fail('Backup failed — aborting delta'); if (r.stderr) error(r.stderr.trim()); process.exit(1); } + s.succeed(`Backup saved: ~/${backupFilename}`); + } else { + info('Skipping backup (--no-backup)'); + } + + const remoteTemp = `/tmp/db-delta-${randomBytes(6).toString('hex')}.sql`; + const localTemp = join(process.env.TMPDIR || '/tmp', `instawp-db-delta-${randomBytes(6).toString('hex')}.sql`); + writeFileSync(localTemp, deltaSql, 'utf-8'); + const up = spinner('Uploading delta...'); + up.start(); + const scpExit = scpUpload(conn, localTemp, remoteTemp); + try { unlinkSync(localTemp); } catch { /* ignore */ } + if (scpExit !== 0) { up.fail(`Upload failed (scp exit ${scpExit})`); if (takeBackup) info(`Remote backup preserved: ~/${backupFilename}`); process.exit(1); } + up.succeed('Delta uploaded'); + + const imp = spinner(`Applying delta (${stats.replaces} change, ${stats.deletes} delete) on ${conn.domain}...`); + imp.start(); + const ir = execViaSsh(conn, `cd ${wpPath} && wp db import ${remoteTemp}`); + execViaSsh(conn, `rm -f ${remoteTemp}`); + if (ir.exitCode !== 0) { + imp.fail('Delta apply failed'); + if (ir.stderr) error(ir.stderr.trim()); else if (ir.stdout) error(ir.stdout.trim()); + if (takeBackup) { + console.log(''); + info(`Remote backup preserved at: ~/${backupFilename}`); + console.log(` ssh ${conn.username}@${conn.host} 'cd ${wpPath} && gunzip -c ${backupRemotePath} | wp db import -'`); + } + process.exit(1); + } + imp.succeed('Delta applied'); + + // REPLACE'd role/cap rows carry the dump prefix in their key VALUES — remap (idempotent). + if (remapFrom && remapFrom !== remotePrefix) { + const cs = spinner('Remapping user roles/capabilities to the remote prefix...'); + cs.start(); + const um = `${remotePrefix}usermeta`; + const opt = `${remotePrefix}options`; + const stmts = [ + `UPDATE ${um} SET meta_key='${remotePrefix}capabilities' WHERE meta_key='${remapFrom}capabilities'`, + `UPDATE ${um} SET meta_key='${remotePrefix}user_level' WHERE meta_key='${remapFrom}user_level'`, + `UPDATE ${opt} SET option_name='${remotePrefix}user_roles' WHERE option_name='${remapFrom}user_roles'`, + ]; + let ok = true; + for (const st of stmts) { const r = execViaSsh(conn, `cd ${wpPath} && wp db query "${st}"`); if (r.exitCode !== 0) { ok = false; if (r.stderr) error(r.stderr.trim()); } } + if (ok) cs.succeed('Roles/capabilities remapped'); else cs.fail('Could not remap roles/capabilities — wp-admin access may need a manual fix'); + } + + if (srFrom && srTo) { + const scope = srTables.length ? srTables.join(' ') : '--all-tables'; + const s = spinner(`Rewriting URLs (${srFrom} -> ${srTo})...`); + s.start(); + const r = execViaSsh(conn, `cd ${wpPath} && wp search-replace '${srFrom}' '${srTo}' ${scope} --skip-columns=guid --report-changed-only`); + if (r.exitCode === 0) { s.succeed('URLs rewritten'); if (!isJsonMode() && r.stdout.trim()) console.log(r.stdout.trim()); } + else { s.fail('URL search-replace failed (delta applied; run it manually if links are wrong)'); if (r.stderr) error(r.stderr.trim()); } + } + + let verified: string | null = null; + if (verify) { + const url = String(site.url || `https://${conn.domain}`).replace(/\/+$/, ''); + const s = spinner(`Verifying ${url} responds...`); + s.start(); + const okv = await waitForHttp(url, 90000); + if (okv) { s.succeed('Site is responding'); verified = 'ok'; } + else { s.fail('Site did not respond within 90s (large changes can need a moment)'); verified = 'timeout'; } + } + + const elapsedSec = (Date.now() - startedAt) / 1000; + success('Incremental push complete', { + site_id: site.id, + backup_path: takeBackup ? backupRemotePath : null, + changed: true, + replaces: stats.replaces, + deletes: stats.deletes, + tables_changed: stats.tablesChanged, + elapsed: `${elapsedSec < 10 ? elapsedSec.toFixed(1) : Math.round(elapsedSec)}s`, + ...(verified ? { verified } : {}), + }); +} + export function registerDbCommand(program: Command): void { const db = program .command('db') @@ -136,6 +339,10 @@ export function registerDbCommand(program: Command): void { .option('--no-backup', 'Skip taking a remote backup before overwrite (DANGEROUS)') .option('--rewrite-prefix', "Rewrite the dump's table prefix to match the remote site's prefix (e.g. wp_ → iwpa4c7_)") .option('--search-replace ', 'After import, run wp search-replace across all tables (serialization-safe, skips guid). Pass exactly two URLs.') + .option('--sr-tables ', 'Scope --search-replace to these (prefixed) tables instead of all tables — faster on big DBs whose bulk has no URLs') + .option('--verify', 'After import, poll the site URL until it responds (large imports can briefly return 000)') + .option('--incremental', 'Push only the row-level delta since the last push (auto-fulls on first run or schema change). Needs a per-row dump: mysqldump --skip-extended-insert --order-by-primary') + .option('--full', 'Force a full push and refresh the incremental baseline') .addHelpText('after', ` Notes: - Always takes a remote backup first unless --no-backup is passed. @@ -146,10 +353,19 @@ Notes: - URLs: after a cross-domain push, remap URLs with: --search-replace (or run 'instawp wp search-replace ' yourself). + --search-replace scans all tables by default; pass --sr-tables to limit it. + - --verify confirms the site answers HTTP after the import (reported in the summary). + - --incremental ships only the row-delta since the last push (baseline stored + per site under ~/.instawp/baselines/). First run, a schema change, or --full + do a normal full push and refresh the baseline. Requires a per-row dump + (mysqldump --skip-extended-insert --order-by-primary). The full push is + untouched — incremental is purely additive. Examples: - $ instawp db push my-site dump.sql --rewrite-prefix + $ instawp db push my-site dump.sql --rewrite-prefix --verify $ instawp db push my-site dump.sql --search-replace http://localhost:10115 https://my-site.instawp.site + $ instawp db push my-site dump.sql --search-replace http://old https://new --sr-tables iwpa4c7_options iwpa4c7_posts iwpa4c7_postmeta + $ instawp db push my-site dump.sql --incremental # delta vs baseline (full push + baseline on first run) `) .action(async (siteIdentifier: string, file: string, opts: any) => { requireAuth(); @@ -181,6 +397,20 @@ Examples: } } + // Parse/validate --sr-tables (table names go into a shell command — keep identifier-safe) + const srTables: string[] = Array.isArray(opts.srTables) ? opts.srTables : []; + if (srTables.length) { + const bad = srTables.filter((t) => !/^[A-Za-z0-9_]+$/.test(t)); + if (bad.length) { + error(`--sr-tables: invalid table name(s): ${bad.join(', ')} (use plain prefixed table names)`); + process.exit(1); + } + if (!srFrom) { + error('--sr-tables only applies with --search-replace'); + process.exit(1); + } + } + // In JSON mode, can't prompt — require --force if (isJsonMode() && !opts.force) { error('--force is required when using --json (cannot prompt for confirmation)'); @@ -207,6 +437,16 @@ Examples: const backupRemotePath = `${remoteHome}/${backupFilename}`; const takeBackup = opts.backup !== false; + // #17 — incremental delta push (additive). May fully handle the push and + // return; otherwise falls through to the full push below (unchanged) and + // refreshes the baseline once it succeeds. + let baselineToSave: Manifest | null = null; + if (opts.incremental || opts.full) { + const decision = await prepareIncremental({ file, site, conn, wpPath, remoteHome, opts, srFrom, srTo, srTables }); + if (decision.done) return; + baselineToSave = decision.baseline ?? null; + } + // Confirmation if (!opts.force) { const backupLine = takeBackup @@ -399,9 +639,11 @@ Examples: // Skip `guid` — post GUIDs are permanent identifiers, not links, and WP // best practice is to never rewrite them on a domain change. if (srFrom && srTo) { + // Scope: explicit (prefixed) tables if --sr-tables given, else all tables. + const scope = srTables.length ? srTables.join(' ') : '--all-tables'; const srSpin = spinner(`Rewriting URLs (${srFrom} -> ${srTo})...`); srSpin.start(); - const srRes = execViaSsh(conn, `cd ${wpPath} && wp search-replace '${srFrom}' '${srTo}' --all-tables --skip-columns=guid --report-changed-only`); + const srRes = execViaSsh(conn, `cd ${wpPath} && wp search-replace '${srFrom}' '${srTo}' ${scope} --skip-columns=guid --report-changed-only`); if (srRes.exitCode === 0) { srSpin.succeed('URLs rewritten'); if (!isJsonMode() && srRes.stdout.trim()) console.log(srRes.stdout.trim()); @@ -421,6 +663,18 @@ Examples: cleanupSpin.succeed('Cleanup complete'); } + // Step 6: Optional readiness check. A large import can briefly leave the + // site returning 000 right after import/flush; poll until it answers. + let verified: string | null = null; + if (opts.verify) { + const siteUrl = String(site.url || `https://${conn.domain}`).replace(/\/+$/, ''); + const vSpin = spinner(`Verifying ${siteUrl} responds...`); + vSpin.start(); + const ok = await waitForHttp(siteUrl, 90000); + if (ok) { vSpin.succeed('Site is responding'); verified = 'ok'; } + else { vSpin.fail('Site did not respond within 90s (large imports can need a moment to warm up)'); verified = 'timeout'; } + } + const elapsedSec = (Date.now() - startedAt) / 1000; const rate = elapsedSec > 0 ? `${formatBytes(localSize / elapsedSec)}/s` : 'n/a'; const elapsedStr = `${elapsedSec < 10 ? elapsedSec.toFixed(1) : Math.round(elapsedSec)}s (${rate})`; @@ -433,10 +687,103 @@ Examples: rewrote_prefix: remapFromPrefix ? `${remapFromPrefix} -> ${remotePrefix}` : null, search_replaced: srFrom && srTo ? `${srFrom} -> ${srTo}` : null, elapsed: elapsedStr, + ...(verified ? { verified } : {}), }); if (!isJsonMode() && takeBackup) { console.log(`\n ${chalk.dim('Backup:')} ~/${backupFilename} ${chalk.dim('(on remote)')}`); } + + // #17 — record the baseline after a successful full (re)base, so the next + // --incremental push can diff against it. + if (baselineToSave) { + saveBaseline(site.id, baselineToSave); + } + }); + + // db backups list/prune — manage the ~/db-backup-*.sql.gz files db push leaves behind + const backups = db + .command('backups') + .description('List or prune db push backups (~/db-backup-*.sql.gz) on a site'); + + backups + .command('list ') + .description('List the db-backup-*.sql.gz files on the remote site (newest first)') + .action(async (siteIdentifier: string) => { + requireAuth(); + const { conn } = await resolveAndConnect(siteIdentifier); + const list = listRemoteBackups(conn); + + if (isJsonMode()) { + console.log(JSON.stringify({ + success: true, + data: { backups: list.map((b) => ({ file: b.file, size_bytes: b.sizeBytes, modified: new Date(b.mtime * 1000).toISOString() })) }, + })); + return; + } + if (!list.length) { info('No backups found (~/db-backup-*.sql.gz).'); return; } + table(['File', 'Size', 'Modified'], list.map((b) => ({ + file: b.file.replace(/^.*\//, ''), + size: formatBytes(b.sizeBytes), + modified: new Date(b.mtime * 1000).toISOString().replace('T', ' ').slice(0, 16), + }))); + const total = list.reduce((n, b) => n + b.sizeBytes, 0); + info(`${list.length} backup(s), ${formatBytes(total)} total`); + }); + + backups + .command('prune ') + .description('Delete old db-backup-*.sql.gz files (keep newest N and/or drop older than D days)') + .option('--keep ', 'Keep the newest N backups, delete the rest', (v) => parseInt(v, 10)) + .option('--older-than ', 'Delete backups older than this many days', (v) => parseInt(v, 10)) + .option('--force', 'Skip confirmation prompt') + .action(async (siteIdentifier: string, opts: any) => { + requireAuth(); + if (opts.keep === undefined && opts.olderThan === undefined) { + error('Specify --keep and/or --older-than (refusing to prune without a selector)'); + process.exit(1); + } + if ((opts.keep !== undefined && (!Number.isInteger(opts.keep) || opts.keep < 0)) || + (opts.olderThan !== undefined && (!Number.isInteger(opts.olderThan) || opts.olderThan < 0))) { + error('--keep / --older-than must be non-negative integers'); + process.exit(1); + } + if (isJsonMode() && !opts.force) { + error('--force is required when using --json (cannot prompt for confirmation)'); + process.exit(1); + } + + const { conn } = await resolveAndConnect(siteIdentifier); + const list = listRemoteBackups(conn); + const { toDelete, toKeep } = selectBackupsToPrune( + list, + { keep: opts.keep, olderThanDays: opts.olderThan }, + Math.floor(Date.now() / 1000), + ); + + if (!toDelete.length) { + if (isJsonMode()) { console.log(JSON.stringify({ success: true, data: { deleted: [], kept: toKeep.length } })); } + else { info(`Nothing to prune (${toKeep.length} backup(s) kept).`); } + return; + } + + if (!isJsonMode()) { + const freed = formatBytes(toDelete.reduce((n, b) => n + b.sizeBytes, 0)); + console.log(`Will delete ${chalk.bold.red(String(toDelete.length))} backup(s) (${freed}), keeping ${toKeep.length}:`); + for (const b of toDelete) console.log(` ${chalk.dim('-')} ${b.file.replace(/^.*\//, '')}`); + if (!opts.force) { + const ok = await promptYesNo('Delete these backups? (y/N) '); + if (!ok) { info('Cancelled.'); return; } + } + } + + const quoted = toDelete.map((b) => `'${b.file.replace(/'/g, "'\\''")}'`).join(' '); + const rm = execViaSsh(conn, `rm -f ${quoted}`); + if (rm.exitCode !== 0) { + error('Failed to delete some backups'); + if (rm.stderr) error(rm.stderr.trim()); + process.exit(1); + } + success('Backups pruned', { deleted: toDelete.length, kept: toKeep.length }); }); } diff --git a/src/commands/sites.ts b/src/commands/sites.ts index 139fd7e..d5428ae 100644 --- a/src/commands/sites.ts +++ b/src/commands/sites.ts @@ -3,6 +3,7 @@ import chalk from 'chalk'; import { requireAuth, getClient } from '../lib/api.js'; import { getApiUrl } from '../lib/config.js'; import { resolveSite } from '../lib/site-resolver.js'; +import { waitForHttp } from '../lib/http-ready.js'; import { success, error, table, spinner, info, isJsonMode } from '../lib/output.js'; export function registerSitesCommand(program: Command): void { @@ -390,30 +391,6 @@ export function registerSitesCommand(program: Command): void { }); } -/** - * Poll a URL until it answers (any HTTP status = DNS resolved + server up), or - * the budget runs out. A fresh site's DNS/edge lags task completion by 30–120s, - * so without this "Ready" lies and callers hand-roll curl-retry gates. - */ -async function waitForHttp(url: string, maxMs: number): Promise { - const deadline = Date.now() + Math.min(Math.max(maxMs, 0), 180000); - while (Date.now() < deadline) { - try { - const ctrl = new AbortController(); - const timer = setTimeout(() => ctrl.abort(), 8000); - try { - await fetch(url, { signal: ctrl.signal, redirect: 'follow' }); - return true; // any HTTP response means it's reachable - } finally { - clearTimeout(timer); - } - } catch { - // DNS/connection not ready yet — back off and retry. - } - await new Promise((r) => setTimeout(r, 3000)); - } - return false; -} // Shared create action used by both `sites create` and top-level `create` async function createSiteAction(opts: any): Promise { diff --git a/src/lib/db-backups.ts b/src/lib/db-backups.ts new file mode 100644 index 0000000..ad03ea4 --- /dev/null +++ b/src/lib/db-backups.ts @@ -0,0 +1,53 @@ +// Helpers for `db backups list/prune` — parsing the remote backup listing and +// deciding which `~/db-backup-*.sql.gz` files to prune. Kept pure (no I/O) so +// the listing command and the prune selection are unit-testable. + +export interface RemoteBackup { + file: string; + sizeBytes: number; + mtime: number; // epoch seconds +} + +/** + * Parse the output of `stat -c '%n\t%s\t%Y' ` — tab-separated + * path / size-bytes / mtime-epoch, one per line. Tolerant of an SSH login + * banner/MOTD and any non-matching lines (silently skipped). Returns + * newest-first. + */ +export function parseBackupList(stdout: string): RemoteBackup[] { + const out: RemoteBackup[] = []; + for (const line of stdout.split('\n')) { + const parts = line.split('\t'); + if (parts.length !== 3) continue; + const file = parts[0].trim(); + const sizeBytes = Number(parts[1]); + const mtime = Number(parts[2]); + if (!file || !Number.isFinite(sizeBytes) || !Number.isFinite(mtime)) continue; + out.push({ file, sizeBytes, mtime }); + } + return out.sort((a, b) => b.mtime - a.mtime); +} + +/** + * Decide which backups to delete. A backup is selected if it matches ANY given + * criterion: it falls beyond the newest `keep`, OR it is older than + * `olderThanDays`. The caller must supply at least one criterion (selecting all + * is never implicit). Returns newest-first partitions. + */ +export function selectBackupsToPrune( + backups: RemoteBackup[], + opts: { keep?: number; olderThanDays?: number }, + nowEpoch: number, +): { toDelete: RemoteBackup[]; toKeep: RemoteBackup[] } { + const sorted = [...backups].sort((a, b) => b.mtime - a.mtime); + const cutoff = opts.olderThanDays != null ? nowEpoch - opts.olderThanDays * 86400 : null; + const toDelete: RemoteBackup[] = []; + const toKeep: RemoteBackup[] = []; + sorted.forEach((b, i) => { + const beyondKeep = opts.keep != null && i >= opts.keep; + const tooOld = cutoff != null && b.mtime < cutoff; + if (beyondKeep || tooOld) toDelete.push(b); + else toKeep.push(b); + }); + return { toDelete, toKeep }; +} diff --git a/src/lib/db-baseline.ts b/src/lib/db-baseline.ts new file mode 100644 index 0000000..131dfcc --- /dev/null +++ b/src/lib/db-baseline.ts @@ -0,0 +1,30 @@ +// Per-site baseline store for `db push --incremental`. The baseline is a compact +// per-row hash MANIFEST (not the full dump) so a large DB doesn't blow up memory. +// Stored under ~/.instawp/baselines//manifest.json. +import { homedir } from 'node:os'; +import { join } from 'node:path'; +import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; +import { serializeManifest, deserializeManifest, type Manifest } from './db-delta.js'; + +/** Root dir for baselines; `base` is injectable for tests (defaults to ~/.instawp). */ +export function baselineDir(siteId: number | string, base = join(homedir(), '.instawp')): string { + return join(base, 'baselines', String(siteId)); +} + +/** Load the stored manifest for a site, or null if none / unreadable. */ +export function loadBaseline(siteId: number | string, base?: string): Manifest | null { + const p = join(baselineDir(siteId, base), 'manifest.json'); + if (!existsSync(p)) return null; + try { + return deserializeManifest(readFileSync(p, 'utf-8')); + } catch { + return null; + } +} + +/** Save (overwrite) the manifest for a site. */ +export function saveBaseline(siteId: number | string, manifest: Manifest, base?: string): void { + const dir = baselineDir(siteId, base); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, 'manifest.json'), serializeManifest(manifest), 'utf-8'); +} diff --git a/src/lib/db-delta.ts b/src/lib/db-delta.ts new file mode 100644 index 0000000..d23f84b --- /dev/null +++ b/src/lib/db-delta.ts @@ -0,0 +1,340 @@ +// Memory-bounded engine for `db push --incremental` (#17). +// +// The naive approach (load both 181 MB dumps as strings → split → per-row Maps +// for both) blew past Node's ~4 GB heap (B3). Instead: +// - the BASELINE is stored as a compact per-row hash MANIFEST (PK → content +// hash) — not the full SQL; +// - the CURRENT dump is STREAMED line-by-line (gz decompressed on the fly), +// each row hashed and compared to the manifest; only CHANGED rows are +// materialized into the delta. +// Peak memory ≈ manifest size (tens of MB), independent of dump size. +// +// A line-oriented state machine also makes the B1 class of bug structural: only +// a line *starting* with `CREATE TABLE` is schema; "CREATE TABLE …" inside a +// single-line INSERT's row data is always a row, never DDL. +// +// Input requirement: a per-row dump (`mysqldump --skip-extended-insert +// --order-by-primary`). An extended-insert dump throws ExtendedInsertError. +import { createHash } from 'node:crypto'; + +export class ExtendedInsertError extends Error {} + +export interface TableSchema { + columns: string[]; + pkCol: string | null; + pkIndex: number; +} + +export interface Manifest { + version: number; + fingerprint: string; + // single-column-PK tables: PK literal → row content hash + single: Map }>; + // composite / no-PK tables: order-independent aggregate (can only detect change) + composite: Map; +} + +export interface DiffResult { + mode: 'delta' | 'full'; + reason?: string; + sql?: string; + stats?: { tablesChanged: number; replaces: number; deletes: number }; + newManifest: Manifest; +} + +const CORE_SUFFIXES = ['options', 'users', 'posts', 'postmeta', 'usermeta', 'comments', 'commentmeta', 'term_taxonomy', 'term_relationships', 'termmeta', 'terms', 'links']; + +/** Derive the table prefix (e.g. `wp_`) from a set of table names, or null. */ +export function prefixFromTableNames(names: Iterable): string | null { + const arr = [...names]; + for (const suf of CORE_SUFFIXES) { + for (const n of arr) if (n.endsWith(suf)) return n.slice(0, n.length - suf.length); + } + return null; +} + +/** Fast 53-bit string hash (cyrb53) — for row content change-detection, not crypto. */ +export function cyrb53(str: string, seed = 0): number { + let h1 = 0xdeadbeef ^ seed; + let h2 = 0x41c6ce57 ^ seed; + for (let i = 0; i < str.length; i++) { + const ch = str.charCodeAt(i); + h1 = Math.imul(h1 ^ ch, 2654435761); + h2 = Math.imul(h2 ^ ch, 1597334677); + } + h1 = Math.imul(h1 ^ (h1 >>> 16), 2246822507); + h1 ^= Math.imul(h2 ^ (h2 >>> 13), 3266489909); + h2 = Math.imul(h2 ^ (h2 >>> 16), 2246822507); + h2 ^= Math.imul(h1 ^ (h1 >>> 13), 3266489909); + return 4294967296 * (2097151 & h2) + (h1 >>> 0); +} + +/** Split a VALUES tuple body (without outer parens) into top-level fields. */ +export function splitSqlTuple(inner: string): string[] { + const out: string[] = []; + let cur = ''; + let inStr = false; + for (let i = 0; i < inner.length; i++) { + const c = inner[i]; + if (inStr) { + if (c === '\\') { cur += c + (inner[i + 1] ?? ''); i++; continue; } + cur += c; + if (c === "'") inStr = false; + continue; + } + if (c === "'") { inStr = true; cur += c; continue; } + if (c === ',') { out.push(cur.trim()); cur = ''; continue; } + cur += c; + } + out.push(cur.trim()); + return out; +} + +/** Parse one INSERT statement into its table + tuple bodies (>1 tuple = extended). */ +export function parseInsertStatement(stmt: string): { table: string; tuples: string[] } | null { + const m = stmt.match(/^INSERT INTO\s+`([^`]+)`\s+(?:\([^)]*\)\s+)?VALUES\s*/i); + if (!m) return null; + const rest = stmt.slice(m[0].length); + const tuples: string[] = []; + let i = 0; + while (i < rest.length) { + if (rest[i] !== '(') { i++; continue; } + let depth = 0; + let inStr = false; + let tuple = ''; + let j = i; + for (; j < rest.length; j++) { + const c = rest[j]; + if (inStr) { + if (c === '\\') { tuple += c + (rest[j + 1] ?? ''); j++; continue; } + tuple += c; + if (c === "'") inStr = false; + continue; + } + if (c === "'") { inStr = true; tuple += c; continue; } + if (c === '(') { depth++; if (depth === 1) continue; tuple += c; continue; } + if (c === ')') { depth--; if (depth === 0) break; tuple += c; continue; } + tuple += c; + } + tuples.push(tuple); + i = j + 1; + } + return { table: m[1], tuples }; +} + +/** Parse a single CREATE TABLE block → ordered columns + single-column PK. */ +function parseCreateBlock(block: string): { table: string; schema: TableSchema } | null { + const m = block.match(/^CREATE TABLE\s+`([^`]+)`\s*\(([\s\S]*)\n\)/i); + if (!m) return null; + const body = m[2]; + const columns: string[] = []; + for (const line of body.split('\n')) { + const cm = line.trim().match(/^`([^`]+)`/); + if (cm) columns.push(cm[1]); + } + let pkCol: string | null = null; + const pk = body.match(/PRIMARY KEY\s*\(([^)]+)\)/i); + if (pk) { + const cols = pk[1].split(',').map((s) => s.trim().replace(/`/g, '')); + if (cols.length === 1) pkCol = cols[0]; + } + return { table: m[1], schema: { columns, pkCol, pkIndex: pkCol ? columns.indexOf(pkCol) : -1 } }; +} + +function normalizeDdl(ddl: string): string { + return ddl.replace(/AUTO_INCREMENT=\d+/gi, 'AUTO_INCREMENT=').replace(/\s+/g, ' ').trim(); +} + +function fingerprintFromBlocks(blocks: string[]): string { + return createHash('sha256').update([...blocks].sort().join('\n')).digest('hex'); +} + +type DumpEvent = + | { kind: 'schema'; table: string; ddl: string; schema: TableSchema } + | { kind: 'row'; table: string; tuple: string } + | { kind: 'extended' }; + +/** Stream a dump's lines into schema/row events (line-oriented state machine). */ +async function* parseDumpStream(lines: AsyncIterable | Iterable): AsyncGenerator { + let block: string[] | null = null; + for await (const raw of lines) { + if (block !== null) { + block.push(raw); + if (/^\)/.test(raw)) { + const ddl = block.join('\n'); + block = null; + const parsed = parseCreateBlock(ddl); + if (parsed) yield { kind: 'schema', table: parsed.table, ddl, schema: parsed.schema }; + } + continue; + } + if (/^CREATE TABLE/i.test(raw)) { block = [raw]; continue; } + if (/^\s*INSERT INTO/i.test(raw)) { + const stmt = parseInsertStatement(raw.trim()); + if (!stmt) continue; + if (stmt.tuples.length !== 1) { yield { kind: 'extended' }; continue; } + yield { kind: 'row', table: stmt.table, tuple: stmt.tuples[0] }; + } + } +} + +function emptyManifest(): Manifest { + return { version: 1, fingerprint: '', single: new Map(), composite: new Map() }; +} + +/** Build a hash manifest from a (streamed) per-row dump. Throws on extended insert. */ +export async function buildManifest(lines: AsyncIterable | Iterable): Promise { + const schemas = new Map(); + const ddlBlocks: string[] = []; + const single = new Map }>(); + const composite = new Map(); + + for await (const ev of parseDumpStream(lines)) { + if (ev.kind === 'extended') throw new ExtendedInsertError(); + if (ev.kind === 'schema') { schemas.set(ev.table, ev.schema); ddlBlocks.push(normalizeDdl(ev.ddl)); continue; } + const sch = schemas.get(ev.table); + const h = cyrb53(ev.tuple); + if (sch && sch.pkCol != null && sch.pkIndex >= 0) { + const pk = splitSqlTuple(ev.tuple)[sch.pkIndex]; + let t = single.get(ev.table); + if (!t) { t = { pkCol: sch.pkCol, pkIndex: sch.pkIndex, rows: new Map() }; single.set(ev.table, t); } + t.rows.set(pk, h); + } else { + let agg = composite.get(ev.table); + if (!agg) { agg = { count: 0, sum: 0n }; composite.set(ev.table, agg); } + agg.count++; + agg.sum += BigInt(h); + } + } + + const m = emptyManifest(); + m.fingerprint = fingerprintFromBlocks(ddlBlocks); + m.single = single; + m.composite = new Map([...composite].map(([t, a]) => [t, { count: a.count, sum: a.sum.toString() }])); + return m; +} + +/** + * Stream the current dump and diff it against the baseline manifest. Builds the + * NEW manifest as it goes (so the caller can re-base without another pass). + * Returns mode:'full' (with the new manifest) when a delta can't be expressed + * (schema/DDL change, or a composite-PK table actually changed). + */ +export async function diffAgainstManifest( + lines: AsyncIterable | Iterable, + baseline: Manifest, + opts: { dumpPrefix: string; remotePrefix: string }, +): Promise { + const { dumpPrefix, remotePrefix } = opts; + const remap = (t: string) => (t.startsWith(dumpPrefix) ? remotePrefix + t.slice(dumpPrefix.length) : t); + + const schemas = new Map(); + const ddlBlocks: string[] = []; + // working copy of baseline single-PK rows → remaining ones = DELETEs + const remaining = new Map>(); + for (const [t, info] of baseline.single) remaining.set(t, new Map(info.rows)); + // new manifest accumulators + const newSingle = new Map }>(); + const newComposite = new Map(); + + const stmts: string[] = []; + const changed = new Set(); + let replaces = 0; + let deletes = 0; + + for await (const ev of parseDumpStream(lines)) { + if (ev.kind === 'extended') throw new ExtendedInsertError(); + if (ev.kind === 'schema') { schemas.set(ev.table, ev.schema); ddlBlocks.push(normalizeDdl(ev.ddl)); continue; } + const sch = schemas.get(ev.table); + const h = cyrb53(ev.tuple); + if (sch && sch.pkCol != null && sch.pkIndex >= 0) { + const pk = splitSqlTuple(ev.tuple)[sch.pkIndex]; + let nt = newSingle.get(ev.table); + if (!nt) { nt = { pkCol: sch.pkCol, pkIndex: sch.pkIndex, rows: new Map() }; newSingle.set(ev.table, nt); } + nt.rows.set(pk, h); + + const baseRows = remaining.get(ev.table); + if (baseRows && baseRows.has(pk)) { + if (baseRows.get(pk) !== h) { + stmts.push(`REPLACE INTO \`${remap(ev.table)}\` VALUES (${ev.tuple});`); + replaces++; + changed.add(ev.table); + } + baseRows.delete(pk); // seen → not a DELETE + } else { + stmts.push(`REPLACE INTO \`${remap(ev.table)}\` VALUES (${ev.tuple});`); + replaces++; + changed.add(ev.table); + } + } else { + let agg = newComposite.get(ev.table); + if (!agg) { agg = { count: 0, sum: 0n }; newComposite.set(ev.table, agg); } + agg.count++; + agg.sum += BigInt(h); + } + } + + const newManifest = emptyManifest(); + newManifest.fingerprint = fingerprintFromBlocks(ddlBlocks); + newManifest.single = newSingle; + newManifest.composite = new Map([...newComposite].map(([t, a]) => [t, { count: a.count, sum: a.sum.toString() }])); + + if (newManifest.fingerprint !== baseline.fingerprint) { + return { mode: 'full', reason: 'schema/DDL changed since the baseline', newManifest }; + } + + // composite / no-PK tables: detect change via aggregate; can't row-diff. + const compTables = new Set([...newComposite.keys(), ...baseline.composite.keys()]); + for (const t of compTables) { + const cur = newComposite.get(t); + const curCount = cur ? cur.count : 0; + const curSum = cur ? cur.sum : 0n; + const base = baseline.composite.get(t); + const baseCount = base ? base.count : 0; + const baseSum = base ? BigInt(base.sum) : 0n; + if (curCount !== baseCount || curSum !== baseSum) { + return { mode: 'full', reason: `table \`${t}\` changed but has no single-column primary key`, newManifest }; + } + } + + // DELETEs: baseline single-PK rows not present in current. + for (const [t, rows] of remaining) { + if (!rows.size) continue; + const pkCol = baseline.single.get(t)?.pkCol ?? schemas.get(t)?.pkCol; + if (!pkCol) continue; + const remTable = remap(t); + for (const pk of rows.keys()) { + stmts.push(`DELETE FROM \`${remTable}\` WHERE \`${pkCol}\`=${pk};`); + deletes++; + changed.add(t); + } + } + + const stats = { tablesChanged: changed.size, replaces, deletes }; + const sql = stmts.length + ? `SET FOREIGN_KEY_CHECKS=0;\nSET sql_mode='NO_AUTO_VALUE_ON_ZERO';\n${stmts.join('\n')}\n` + : ''; + return { mode: 'delta', sql, stats, newManifest }; +} + +/** Serialize a manifest to a compact JSON string (Maps → arrays). */ +export function serializeManifest(m: Manifest): string { + return JSON.stringify({ + version: m.version, + fingerprint: m.fingerprint, + single: [...m.single].map(([t, info]) => [t, info.pkCol, info.pkIndex, [...info.rows]]), + composite: [...m.composite].map(([t, agg]) => [t, agg.count, agg.sum]), + }); +} + +/** Inverse of serializeManifest. */ +export function deserializeManifest(s: string): Manifest { + const o = JSON.parse(s); + const single = new Map }>( + (o.single ?? []).map((e: any) => [e[0], { pkCol: e[1], pkIndex: e[2], rows: new Map(e[3]) }]), + ); + const composite = new Map( + (o.composite ?? []).map((e: any) => [e[0], { count: e[1], sum: e[2] }]), + ); + return { version: o.version ?? 1, fingerprint: o.fingerprint ?? '', single, composite }; +} diff --git a/src/lib/http-ready.ts b/src/lib/http-ready.ts new file mode 100644 index 0000000..2441ecb --- /dev/null +++ b/src/lib/http-ready.ts @@ -0,0 +1,26 @@ +/** + * Poll a URL until it answers (any HTTP status = DNS resolved + server up), or + * the budget runs out. A fresh site's DNS/edge lags task completion by 30–120s, + * and a large DB import can leave the site briefly unreachable (returns 000) + * right after import/flush — so callers use this instead of hand-rolling a + * curl-retry gate. Capped at 180s regardless of `maxMs`. + */ +export async function waitForHttp(url: string, maxMs: number): Promise { + const deadline = Date.now() + Math.min(Math.max(maxMs, 0), 180000); + while (Date.now() < deadline) { + try { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), 8000); + try { + await fetch(url, { signal: ctrl.signal, redirect: 'follow' }); + return true; // any HTTP response means it's reachable + } finally { + clearTimeout(timer); + } + } catch { + // DNS/connection not ready yet — back off and retry. + } + await new Promise((r) => setTimeout(r, 3000)); + } + return false; +}