diff --git a/CHANGELOG.md b/CHANGELOG.md index 54785cb..157a9c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,13 @@ # Changelog -## 0.0.1-beta.25 (2026-06-25) +## Unreleased + +### Added — db push readiness/scoping + backup retention (large-DB feedback #16) + +- **`db push --verify`** polls the site URL until it answers HTTP after the import (a large import can briefly return `000` right after import/flush); reported as `verified` in the summary. +- **`db push --sr-tables `** scopes `--search-replace` to specific (prefixed) tables instead of all tables — faster on big DBs whose bulk rows have no URLs. Default stays `--all-tables`. +- **`db backups list `** and **`db backups prune [--keep ] [--older-than ] [--force]`** manage the `~/db-backup-*.sql.gz` files `db push` leaves behind (they previously accumulated with no way to view/clean them). +- Internal: extracted the HTTP-readiness helper to `lib/http-ready.ts` (now shared by `sites create` and `db push --verify`). ### Improved — db push transport & progress (large-DB path) diff --git a/src/__tests__/db-backups.test.ts b/src/__tests__/db-backups.test.ts new file mode 100644 index 0000000..ad07616 --- /dev/null +++ b/src/__tests__/db-backups.test.ts @@ -0,0 +1,69 @@ +import { describe, it, expect } from 'vitest'; +import { parseBackupList, selectBackupsToPrune } from '../lib/db-backups.js'; + +describe('parseBackupList', () => { + it('parses tab-separated stat output, newest first', () => { + const out = [ + '/home/u/db-backup-2026-06-20.sql.gz\t1048576\t1718870400', + '/home/u/db-backup-2026-06-25.sql.gz\t2097152\t1719273600', + ].join('\n'); + const list = parseBackupList(out); + expect(list).toHaveLength(2); + expect(list[0].file).toBe('/home/u/db-backup-2026-06-25.sql.gz'); // newest first + expect(list[0].sizeBytes).toBe(2097152); + expect(list[1].mtime).toBe(1718870400); + }); + + it('tolerates an SSH banner/MOTD and skips non-matching lines', () => { + const out = [ + 'Welcome to Ubuntu 22.04', + 'Last login: ...', + '/home/u/db-backup-a.sql.gz\t100\t1719273600', + 'some junk line without tabs', + '/home/u/notanumber.sql.gz\tNaN\tNaN', + ].join('\n'); + const list = parseBackupList(out); + expect(list).toHaveLength(1); + expect(list[0].file).toBe('/home/u/db-backup-a.sql.gz'); + }); + + it('returns [] for empty input', () => { + expect(parseBackupList('')).toEqual([]); + }); +}); + +describe('selectBackupsToPrune', () => { + const now = 1_719_273_600; // fixed "now" + const day = 86400; + // newest → oldest + const backups = [ + { file: 'd', sizeBytes: 1, mtime: now - 1 * day }, + { file: 'c', sizeBytes: 1, mtime: now - 3 * day }, + { file: 'b', sizeBytes: 1, mtime: now - 10 * day }, + { file: 'a', sizeBytes: 1, mtime: now - 30 * day }, + ]; + + it('--keep N keeps the newest N, deletes the rest', () => { + const { toDelete, toKeep } = selectBackupsToPrune(backups, { keep: 2 }, now); + expect(toKeep.map((b) => b.file)).toEqual(['d', 'c']); + expect(toDelete.map((b) => b.file)).toEqual(['b', 'a']); + }); + + it('--older-than D deletes backups older than D days', () => { + const { toDelete, toKeep } = selectBackupsToPrune(backups, { olderThanDays: 5 }, now); + expect(toDelete.map((b) => b.file)).toEqual(['b', 'a']); // 10d, 30d old + expect(toKeep.map((b) => b.file)).toEqual(['d', 'c']); // 1d, 3d + }); + + it('combines criteria with OR (deleted if beyond keep OR too old)', () => { + const { toDelete } = selectBackupsToPrune(backups, { keep: 3, olderThanDays: 5 }, now); + // beyond keep:3 → 'a'; older than 5d → 'b','a' → union {b,a} + expect(toDelete.map((b) => b.file).sort()).toEqual(['a', 'b']); + }); + + it('deletes nothing when no criteria are given', () => { + const { toDelete, toKeep } = selectBackupsToPrune(backups, {}, now); + expect(toDelete).toHaveLength(0); + expect(toKeep).toHaveLength(4); + }); +}); diff --git a/src/__tests__/http-ready.test.ts b/src/__tests__/http-ready.test.ts new file mode 100644 index 0000000..1e22601 --- /dev/null +++ b/src/__tests__/http-ready.test.ts @@ -0,0 +1,22 @@ +import { describe, it, expect, vi, afterEach } from 'vitest'; +import { waitForHttp } from '../lib/http-ready.js'; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +describe('waitForHttp', () => { + it('returns true as soon as the URL answers (any HTTP response)', async () => { + const fetchMock = vi.fn().mockResolvedValue({ status: 200 }); + vi.stubGlobal('fetch', fetchMock); + await expect(waitForHttp('https://x.test', 5000)).resolves.toBe(true); + expect(fetchMock).toHaveBeenCalledOnce(); + }); + + it('returns false without calling fetch when the budget is already exhausted', async () => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + await expect(waitForHttp('https://x.test', 0)).resolves.toBe(false); + expect(fetchMock).not.toHaveBeenCalled(); + }); +}); diff --git a/src/commands/db.ts b/src/commands/db.ts index 5c794ae..32dd808 100644 --- a/src/commands/db.ts +++ b/src/commands/db.ts @@ -11,7 +11,10 @@ import { ensureSshAccess } from '../lib/ssh-keys.js'; import { execViaSsh, execViaSshToFile, scpUpload } from '../lib/ssh-connection.js'; import { parseTablePrefix } from '../lib/local-instance.js'; import { detectDumpPrefix, readSqlHead, rewriteDumpPrefix } from '../lib/sql-dump.js'; -import { success, error, spinner, info, isJsonMode } from '../lib/output.js'; +import { waitForHttp } from '../lib/http-ready.js'; +import { parseBackupList, selectBackupsToPrune, type RemoteBackup } from '../lib/db-backups.js'; +import { success, error, spinner, info, table, isJsonMode } from '../lib/output.js'; +import type { SshConnection } from '../types.js'; /** Timestamp like `2026-05-23T12-34-56` (filename-safe — `:` is illegal on Windows). */ function isoTimestamp(): string { @@ -50,6 +53,31 @@ function isShellSafeUrl(u: string): boolean { return /^https?:\/\/[^\s'"\\$`]+$/.test(u); } +/** Resolve a site and open an SSH connection (shared by the backups subcommands). */ +async function resolveAndConnect(siteIdentifier: string): Promise<{ site: any; conn: SshConnection }> { + const rspin = spinner('Resolving site...'); + rspin.start(); + let site: any; + try { + site = await resolveSite(siteIdentifier); + rspin.succeed(`Site: ${site.name || site.sub_domain} (ID: ${site.id})`); + } catch { + rspin.fail('Site resolution failed'); + process.exit(1); + } + const conn = await ensureSshAccess(site.id); + return { site, conn }; +} + +/** List `~/db-backup-*.sql.gz` on the remote (the files `db push` writes), newest first. */ +function listRemoteBackups(conn: SshConnection): RemoteBackup[] { + // GNU stat interprets the \t escapes; the for-loop guard avoids a literal glob + // when nothing matches. parseBackupList tolerates any MOTD/banner lines. + const cmd = `for f in ~/db-backup-*.sql.gz; do [ -e "$f" ] && stat -c '%n\\t%s\\t%Y' "$f"; done`; + const res = execViaSsh(conn, cmd); + return parseBackupList(res.stdout || ''); +} + export function registerDbCommand(program: Command): void { const db = program .command('db') @@ -136,6 +164,8 @@ export function registerDbCommand(program: Command): void { .option('--no-backup', 'Skip taking a remote backup before overwrite (DANGEROUS)') .option('--rewrite-prefix', "Rewrite the dump's table prefix to match the remote site's prefix (e.g. wp_ → iwpa4c7_)") .option('--search-replace ', 'After import, run wp search-replace across all tables (serialization-safe, skips guid). Pass exactly two URLs.') + .option('--sr-tables ', 'Scope --search-replace to these (prefixed) tables instead of all tables — faster on big DBs whose bulk has no URLs') + .option('--verify', 'After import, poll the site URL until it responds (large imports can briefly return 000)') .addHelpText('after', ` Notes: - Always takes a remote backup first unless --no-backup is passed. @@ -146,10 +176,13 @@ Notes: - URLs: after a cross-domain push, remap URLs with: --search-replace (or run 'instawp wp search-replace ' yourself). + --search-replace scans all tables by default; pass --sr-tables to limit it. + - --verify confirms the site answers HTTP after the import (reported in the summary). Examples: - $ instawp db push my-site dump.sql --rewrite-prefix + $ instawp db push my-site dump.sql --rewrite-prefix --verify $ instawp db push my-site dump.sql --search-replace http://localhost:10115 https://my-site.instawp.site + $ instawp db push my-site dump.sql --search-replace http://old https://new --sr-tables iwpa4c7_options iwpa4c7_posts iwpa4c7_postmeta `) .action(async (siteIdentifier: string, file: string, opts: any) => { requireAuth(); @@ -181,6 +214,20 @@ Examples: } } + // Parse/validate --sr-tables (table names go into a shell command — keep identifier-safe) + const srTables: string[] = Array.isArray(opts.srTables) ? opts.srTables : []; + if (srTables.length) { + const bad = srTables.filter((t) => !/^[A-Za-z0-9_]+$/.test(t)); + if (bad.length) { + error(`--sr-tables: invalid table name(s): ${bad.join(', ')} (use plain prefixed table names)`); + process.exit(1); + } + if (!srFrom) { + error('--sr-tables only applies with --search-replace'); + process.exit(1); + } + } + // In JSON mode, can't prompt — require --force if (isJsonMode() && !opts.force) { error('--force is required when using --json (cannot prompt for confirmation)'); @@ -399,9 +446,11 @@ Examples: // Skip `guid` — post GUIDs are permanent identifiers, not links, and WP // best practice is to never rewrite them on a domain change. if (srFrom && srTo) { + // Scope: explicit (prefixed) tables if --sr-tables given, else all tables. + const scope = srTables.length ? srTables.join(' ') : '--all-tables'; const srSpin = spinner(`Rewriting URLs (${srFrom} -> ${srTo})...`); srSpin.start(); - const srRes = execViaSsh(conn, `cd ${wpPath} && wp search-replace '${srFrom}' '${srTo}' --all-tables --skip-columns=guid --report-changed-only`); + const srRes = execViaSsh(conn, `cd ${wpPath} && wp search-replace '${srFrom}' '${srTo}' ${scope} --skip-columns=guid --report-changed-only`); if (srRes.exitCode === 0) { srSpin.succeed('URLs rewritten'); if (!isJsonMode() && srRes.stdout.trim()) console.log(srRes.stdout.trim()); @@ -421,6 +470,18 @@ Examples: cleanupSpin.succeed('Cleanup complete'); } + // Step 6: Optional readiness check. A large import can briefly leave the + // site returning 000 right after import/flush; poll until it answers. + let verified: string | null = null; + if (opts.verify) { + const siteUrl = String(site.url || `https://${conn.domain}`).replace(/\/+$/, ''); + const vSpin = spinner(`Verifying ${siteUrl} responds...`); + vSpin.start(); + const ok = await waitForHttp(siteUrl, 90000); + if (ok) { vSpin.succeed('Site is responding'); verified = 'ok'; } + else { vSpin.fail('Site did not respond within 90s (large imports can need a moment to warm up)'); verified = 'timeout'; } + } + const elapsedSec = (Date.now() - startedAt) / 1000; const rate = elapsedSec > 0 ? `${formatBytes(localSize / elapsedSec)}/s` : 'n/a'; const elapsedStr = `${elapsedSec < 10 ? elapsedSec.toFixed(1) : Math.round(elapsedSec)}s (${rate})`; @@ -433,10 +494,97 @@ Examples: rewrote_prefix: remapFromPrefix ? `${remapFromPrefix} -> ${remotePrefix}` : null, search_replaced: srFrom && srTo ? `${srFrom} -> ${srTo}` : null, elapsed: elapsedStr, + ...(verified ? { verified } : {}), }); if (!isJsonMode() && takeBackup) { console.log(`\n ${chalk.dim('Backup:')} ~/${backupFilename} ${chalk.dim('(on remote)')}`); } }); + + // db backups list/prune — manage the ~/db-backup-*.sql.gz files db push leaves behind + const backups = db + .command('backups') + .description('List or prune db push backups (~/db-backup-*.sql.gz) on a site'); + + backups + .command('list ') + .description('List the db-backup-*.sql.gz files on the remote site (newest first)') + .action(async (siteIdentifier: string) => { + requireAuth(); + const { conn } = await resolveAndConnect(siteIdentifier); + const list = listRemoteBackups(conn); + + if (isJsonMode()) { + console.log(JSON.stringify({ + success: true, + data: { backups: list.map((b) => ({ file: b.file, size_bytes: b.sizeBytes, modified: new Date(b.mtime * 1000).toISOString() })) }, + })); + return; + } + if (!list.length) { info('No backups found (~/db-backup-*.sql.gz).'); return; } + table(['File', 'Size', 'Modified'], list.map((b) => ({ + file: b.file.replace(/^.*\//, ''), + size: formatBytes(b.sizeBytes), + modified: new Date(b.mtime * 1000).toISOString().replace('T', ' ').slice(0, 16), + }))); + const total = list.reduce((n, b) => n + b.sizeBytes, 0); + info(`${list.length} backup(s), ${formatBytes(total)} total`); + }); + + backups + .command('prune ') + .description('Delete old db-backup-*.sql.gz files (keep newest N and/or drop older than D days)') + .option('--keep ', 'Keep the newest N backups, delete the rest', (v) => parseInt(v, 10)) + .option('--older-than ', 'Delete backups older than this many days', (v) => parseInt(v, 10)) + .option('--force', 'Skip confirmation prompt') + .action(async (siteIdentifier: string, opts: any) => { + requireAuth(); + if (opts.keep === undefined && opts.olderThan === undefined) { + error('Specify --keep and/or --older-than (refusing to prune without a selector)'); + process.exit(1); + } + if ((opts.keep !== undefined && (!Number.isInteger(opts.keep) || opts.keep < 0)) || + (opts.olderThan !== undefined && (!Number.isInteger(opts.olderThan) || opts.olderThan < 0))) { + error('--keep / --older-than must be non-negative integers'); + process.exit(1); + } + if (isJsonMode() && !opts.force) { + error('--force is required when using --json (cannot prompt for confirmation)'); + process.exit(1); + } + + const { conn } = await resolveAndConnect(siteIdentifier); + const list = listRemoteBackups(conn); + const { toDelete, toKeep } = selectBackupsToPrune( + list, + { keep: opts.keep, olderThanDays: opts.olderThan }, + Math.floor(Date.now() / 1000), + ); + + if (!toDelete.length) { + if (isJsonMode()) { console.log(JSON.stringify({ success: true, data: { deleted: [], kept: toKeep.length } })); } + else { info(`Nothing to prune (${toKeep.length} backup(s) kept).`); } + return; + } + + if (!isJsonMode()) { + const freed = formatBytes(toDelete.reduce((n, b) => n + b.sizeBytes, 0)); + console.log(`Will delete ${chalk.bold.red(String(toDelete.length))} backup(s) (${freed}), keeping ${toKeep.length}:`); + for (const b of toDelete) console.log(` ${chalk.dim('-')} ${b.file.replace(/^.*\//, '')}`); + if (!opts.force) { + const ok = await promptYesNo('Delete these backups? (y/N) '); + if (!ok) { info('Cancelled.'); return; } + } + } + + const quoted = toDelete.map((b) => `'${b.file.replace(/'/g, "'\\''")}'`).join(' '); + const rm = execViaSsh(conn, `rm -f ${quoted}`); + if (rm.exitCode !== 0) { + error('Failed to delete some backups'); + if (rm.stderr) error(rm.stderr.trim()); + process.exit(1); + } + success('Backups pruned', { deleted: toDelete.length, kept: toKeep.length }); + }); } diff --git a/src/commands/sites.ts b/src/commands/sites.ts index 139fd7e..d5428ae 100644 --- a/src/commands/sites.ts +++ b/src/commands/sites.ts @@ -3,6 +3,7 @@ import chalk from 'chalk'; import { requireAuth, getClient } from '../lib/api.js'; import { getApiUrl } from '../lib/config.js'; import { resolveSite } from '../lib/site-resolver.js'; +import { waitForHttp } from '../lib/http-ready.js'; import { success, error, table, spinner, info, isJsonMode } from '../lib/output.js'; export function registerSitesCommand(program: Command): void { @@ -390,30 +391,6 @@ export function registerSitesCommand(program: Command): void { }); } -/** - * Poll a URL until it answers (any HTTP status = DNS resolved + server up), or - * the budget runs out. A fresh site's DNS/edge lags task completion by 30–120s, - * so without this "Ready" lies and callers hand-roll curl-retry gates. - */ -async function waitForHttp(url: string, maxMs: number): Promise { - const deadline = Date.now() + Math.min(Math.max(maxMs, 0), 180000); - while (Date.now() < deadline) { - try { - const ctrl = new AbortController(); - const timer = setTimeout(() => ctrl.abort(), 8000); - try { - await fetch(url, { signal: ctrl.signal, redirect: 'follow' }); - return true; // any HTTP response means it's reachable - } finally { - clearTimeout(timer); - } - } catch { - // DNS/connection not ready yet — back off and retry. - } - await new Promise((r) => setTimeout(r, 3000)); - } - return false; -} // Shared create action used by both `sites create` and top-level `create` async function createSiteAction(opts: any): Promise { diff --git a/src/lib/db-backups.ts b/src/lib/db-backups.ts new file mode 100644 index 0000000..ad03ea4 --- /dev/null +++ b/src/lib/db-backups.ts @@ -0,0 +1,53 @@ +// Helpers for `db backups list/prune` — parsing the remote backup listing and +// deciding which `~/db-backup-*.sql.gz` files to prune. Kept pure (no I/O) so +// the listing command and the prune selection are unit-testable. + +export interface RemoteBackup { + file: string; + sizeBytes: number; + mtime: number; // epoch seconds +} + +/** + * Parse the output of `stat -c '%n\t%s\t%Y' ` — tab-separated + * path / size-bytes / mtime-epoch, one per line. Tolerant of an SSH login + * banner/MOTD and any non-matching lines (silently skipped). Returns + * newest-first. + */ +export function parseBackupList(stdout: string): RemoteBackup[] { + const out: RemoteBackup[] = []; + for (const line of stdout.split('\n')) { + const parts = line.split('\t'); + if (parts.length !== 3) continue; + const file = parts[0].trim(); + const sizeBytes = Number(parts[1]); + const mtime = Number(parts[2]); + if (!file || !Number.isFinite(sizeBytes) || !Number.isFinite(mtime)) continue; + out.push({ file, sizeBytes, mtime }); + } + return out.sort((a, b) => b.mtime - a.mtime); +} + +/** + * Decide which backups to delete. A backup is selected if it matches ANY given + * criterion: it falls beyond the newest `keep`, OR it is older than + * `olderThanDays`. The caller must supply at least one criterion (selecting all + * is never implicit). Returns newest-first partitions. + */ +export function selectBackupsToPrune( + backups: RemoteBackup[], + opts: { keep?: number; olderThanDays?: number }, + nowEpoch: number, +): { toDelete: RemoteBackup[]; toKeep: RemoteBackup[] } { + const sorted = [...backups].sort((a, b) => b.mtime - a.mtime); + const cutoff = opts.olderThanDays != null ? nowEpoch - opts.olderThanDays * 86400 : null; + const toDelete: RemoteBackup[] = []; + const toKeep: RemoteBackup[] = []; + sorted.forEach((b, i) => { + const beyondKeep = opts.keep != null && i >= opts.keep; + const tooOld = cutoff != null && b.mtime < cutoff; + if (beyondKeep || tooOld) toDelete.push(b); + else toKeep.push(b); + }); + return { toDelete, toKeep }; +} diff --git a/src/lib/http-ready.ts b/src/lib/http-ready.ts new file mode 100644 index 0000000..2441ecb --- /dev/null +++ b/src/lib/http-ready.ts @@ -0,0 +1,26 @@ +/** + * Poll a URL until it answers (any HTTP status = DNS resolved + server up), or + * the budget runs out. A fresh site's DNS/edge lags task completion by 30–120s, + * and a large DB import can leave the site briefly unreachable (returns 000) + * right after import/flush — so callers use this instead of hand-rolling a + * curl-retry gate. Capped at 180s regardless of `maxMs`. + */ +export async function waitForHttp(url: string, maxMs: number): Promise { + const deadline = Date.now() + Math.min(Math.max(maxMs, 0), 180000); + while (Date.now() < deadline) { + try { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), 8000); + try { + await fetch(url, { signal: ctrl.signal, redirect: 'follow' }); + return true; // any HTTP response means it's reachable + } finally { + clearTimeout(timer); + } + } catch { + // DNS/connection not ready yet — back off and retry. + } + await new Promise((r) => setTimeout(r, 3000)); + } + return false; +}