From 73f154c50366a5a86fb5d8ab37bd04fe42cfe1fb Mon Sep 17 00:00:00 2001 From: MigraDiff Agent Date: Sat, 8 Aug 2026 13:08:39 -0700 Subject: [PATCH 01/11] feat: add migration state tracking (--status, --history, --promote, --record-rollback) Adds a migradiff_history table recorded in target databases via migra/history.py, plus CLI flags to view history, record generated migrations, chain migrations across environments with conflict detection, and close the audit loop on executed rollbacks. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 33 +++ README.md | 138 ++++++++++ migra/command.py | 324 ++++++++++++++++++++++ migra/history.py | 191 +++++++++++++ tests/test_command_promote.py | 260 ++++++++++++++++++ tests/test_command_rollback_tracking.py | 193 ++++++++++++++ tests/test_history.py | 340 ++++++++++++++++++++++++ 7 files changed, 1479 insertions(+) create mode 100644 migra/history.py create mode 100644 tests/test_command_promote.py create mode 100644 tests/test_command_rollback_tracking.py create mode 100644 tests/test_history.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c94c0e7..4ba9cd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,39 @@ ## [Unreleased] +### Added + +- **Migration state tracking** (`migradiff_history` table): + - `--status` / `--history` flags to view migration history from a database + - `--record-history` flag to record generated migrations in the target DB + - `--env-label` flag to tag history entries with an environment name + - New module `migra/history.py` with `ensure_history_table()`, + `record_applied()`, `record_rollback()`, `get_applied_hashes()`, + `get_history()`, and `compute_migration_hash()` + - SQLAlchemy Core implementation via `sqlbag`, no new dependencies +- **Multi-environment promotion** (`--promote`): + - Colon-separated chain of 2+ database URLs or environment aliases + - Conflict detection at each hop (target must not have hashes source lacks) + - Composes with `--explain`, `--rollback`, `--advise`, `--force-destructive` + - Environment aliases resolved from `~/.config/migradiff/environments.json` + - `--record-history` integration per hop +- **Rollback execution tracking** (`--record-rollback`): + - Records rollback status (`rolled_back` / `rollback_failed`) in the history table + - Accepts a migration hash or rollback SQL file path + - `--rollback-status` flag to mark failed rollback attempts + - `--status` / `--history` output shows rollback annotations +- **New tests**: `test_history.py`, `test_command_promote.py`, + `test_command_rollback_tracking.py` — all using real Postgres databases + (no mocks), covering idempotent table creation, hash normalization, + conflict detection, empty-diff hops, safe mode, and rollback tracking. + +### Notes + +- This is the foundational layer for the upcoming "Control Plane" feature set. +- Known limitation: `migradiff_history` records migrations as *generated*, + not *applied* — see the README "Migration State Tracking" section for details. +- All new features work without AI extras (`anthropic` optional dependency). + ## [1.7.2] - 2026-06-08 - Fix: Use PEP 621 [project] table for PyPI metadata (homepage, repository, bug tracker) diff --git a/README.md b/README.md index 03faa5d..e02b320 100644 --- a/README.md +++ b/README.md @@ -269,6 +269,144 @@ Requires pip install migradiff[ai] and an Anthropic API key. --- +## Migration State Tracking + +MigraDiff now includes a migration state tracking system that records +generated migrations in a `migradiff_history` table in your target +databases. This is the foundation for multi-environment promotion and +rollback tracking. + +### ⚠ Known Limitation + +`migradiff_history` records that a migration was **generated/reviewed** +for a target database, not that the SQL was necessarily *executed* by +MigraDiff itself (MigraDiff does not apply migrations to live databases +today — it only diffs and generates SQL). Recording true "applied" state +requires your deployment pipeline to also call +`migra --record-history` (or the future `--apply` flag, not yet +implemented) after running the generated SQL. + +Be explicit about this in your pipeline so you don't assume false +guarantees about whether a migration has actually been applied. + +### View Migration History + +```bash +# Show recent history (last 10 entries) +migra --status postgres://db_target + +# Show full history +migra --history postgres://db_target +``` + +Output includes: +- **Hash** (short, 8 chars) — identifies the migration +- **Name** — optional human label +- **Applied At** — when the migration was generated/recorded +- **Applied By** — database user who recorded it +- **Rollback** — rollback status (ROLLED BACK, ROLLBACK FAILED, or -) + +JSON output is supported: + +```bash +migra --status postgres://db_target --output json +``` + +### Record Migration History + +After generating a migration diff, record it in the target database: + +```bash +migra --record-history postgres://db_target_a postgres://db_target_b +``` + +This calls `ensure_history_table()` and inserts a row with the +generated SQL hash, the forward SQL, and an optional environment label. + +Use `--env-label` to tag the entry: + +```bash +migra --record-history --env-label staging postgres://db_a postgres://db_b +``` + +### Multi-Environment Promotion (`--promote`) + +`--promote` generates migrations along a chain of environments, with +conflict detection at each hop: + +```bash +migra --unsafe --promote dev:staging:prod +``` + +This: +1. Connects to dev, staging, and prod +2. Verifies staging's `migradiff_history` is a subset of dev's + (no unknown migrations in staging) +3. Generates the `dev → staging` diff and prints it +4. Generates the `staging → prod` diff and prints it +5. If `--record-history` is also given, records each hop against + the target environment's history table + +Environment names are resolved from +`~/.config/migradiff/environments.json`: + +```json +{ + "dev": "postgresql://localhost/dev", + "staging": "postgresql://staging.example.com/db", + "prod": "postgresql://prod.example.com/db" +} +``` + +You can also mix aliases with literal URLs: + +```bash +migra --promote dev:postgresql://custom-host/db:prod +``` + +Conflict example — if staging has a migration that dev doesn't: + +``` +MigraDiff: CONFLICT in promotion chain postgres://dev → postgres://staging + postgres://staging has migrations not found in postgres://dev: + abc1234deadbeef + Resolve the conflict before continuing the chain. +``` + +If two adjacent environments already match: + +``` +No changes needed: postgres://dev → postgres://staging +``` + +`--promote` composes with `--explain`, `--rollback`, `--advise`, +`--force-destructive`, and `--output json` the same way a normal +two-argument diff does. + +### Rollback Tracking (`--record-rollback`) + +After executing a rollback, record it to close the audit loop: + +```bash +migra --record-rollback postgres://db_target +``` + +You can also mark a rollback as failed: + +```bash +migra --record-rollback abc1234 --rollback-status rollback_failed postgres://db_target +``` + +The `--status` and `--history` views then show the rollback state: + +``` +Hash Name Applied At Applied By Rollback +----------- -------------------- -------------------------- ---------------- -------------------- +abc1234 v001_create_users 2026-06-18T12:00:00Z deploy_bot ROLLED BACK 2026-06-18T13:00:00Z +``` + +--- + ## Development Setup The test suite requires a running PostgreSQL instance. The easiest diff --git a/migra/command.py b/migra/command.py index 44a9692..7bee21a 100644 --- a/migra/command.py +++ b/migra/command.py @@ -553,6 +553,56 @@ def parse_args(args): help="Generate a PostgreSQL migration from a plain English description." " Optionally provide the description as an argument.", ) + parser.add_argument( + "--record-history", + dest="record_history", + action="store_true", + default=False, + help="Record the generated migration in the target database's migradiff_history table", + ) + parser.add_argument( + "--status", + dest="status", + nargs="?", + const=True, + default=None, + help="Show recent migration history from a database (default limit: 10)", + ) + parser.add_argument( + "--history", + dest="history", + nargs="?", + const=True, + default=None, + help="Show full migration history from a database (no limit)", + ) + parser.add_argument( + "--promote", + dest="promote", + default=None, + help="Generate migrations along a colon-separated chain of environments" + " (e.g. dev:staging:prod). Supports database URLs or environment aliases.", + ) + parser.add_argument( + "--record-rollback", + dest="record_rollback", + default=None, + help="Record a rollback in the migradiff_history table." + " Accepts a migration hash or a rollback SQL file path.", + ) + parser.add_argument( + "--env-label", + dest="env_label", + default=None, + help='Environment label for --record-history (e.g. "dev", "staging", "prod")', + ) + parser.add_argument( + "--rollback-status", + dest="rollback_status", + default="rolled_back", + choices=["rolled_back", "rollback_failed"], + help="Status to record when using --record-rollback (default: rolled_back)", + ) parser.add_argument( "dburl_from", nargs="?", help="The database you want to migrate." ) @@ -562,12 +612,252 @@ def parse_args(args): return parser.parse_args(args) +def _resolve_env_aliases_part(part): + """Resolve a single promote chain segment to a database URL. + Returns the part as-is if it looks like a URL (contains ://), + otherwise resolves it as an environment alias. + """ + if "://" in part: + return part + return _resolve_env_alias(part) + + +def _resolve_env_alias(key): + """Resolve a single environment alias to a database URL.""" + env_file = os.environ.get( + "MIGRADIFF_ENV_FILE", + os.path.join( + os.path.expanduser("~"), ".config", "migradiff", "environments.json" + ), + ) + if os.path.exists(env_file): + with open(env_file) as f: + config = json.load(f) + if key in config: + return config[key] + raise ValueError( + "MigraDiff: Unknown environment alias '{}'.\n" + "Define it in {}".format(key, env_file) + ) + + def run(args, out=None, err=None): if not out: out = sys.stdout # pragma: no cover if not err: err = sys.stderr # pragma: no cover + # --status / --history: display migration history + if args.status is not None or args.history is not None: + from .history import ensure_history_table, get_history + + dburl = args.status if isinstance(args.status, str) else None + if dburl is None: + dburl = args.history if isinstance(args.history, str) else None + if dburl is None: + dburl = args.dburl_from + if not dburl: + print( + "MigraDiff: --status/--history requires a database URL.", + file=err, + ) + return 1 + + limit = None if args.history is not None else 10 + + from sqlbag import S + + with S(dburl) as s: + ensure_history_table(s) + entries = get_history(s, limit=limit) + + if args.output == "json": + import json as json_mod + + print(json_mod.dumps(entries, indent=2), file=out) + else: + if not entries: + print("No migration history found.", file=out) + return 0 + header = "{:<12} {:<20} {:<26} {:<16} {:<20}".format( + "Hash", "Name", "Applied At", "Applied By", "Rollback" + ) + print(header, file=out) + print("-" * len(header), file=out) + for e in entries: + short_hash = e["migration_hash"][:8] if e["migration_hash"] else "" + name = e["migration_name"] or "" + applied_at = e["applied_at"] or "" + applied_by = e["applied_by"] or "" + rb_status = e["rollback_status"] + rb_annotation = "" + if rb_status == "rolled_back": + rb_annotation = "ROLLED BACK" + if e.get("rolled_back_at"): + rb_annotation += " " + e["rolled_back_at"] + elif rb_status == "rollback_failed": + rb_annotation = "ROLLBACK FAILED" + else: + rb_annotation = "-" + print( + "{:<12} {:<20} {:<26} {:<16} {:<20}".format( + short_hash, + name[:18], + applied_at[:24], + applied_by[:14], + rb_annotation[:18], + ), + file=out, + ) + return 0 + + # --record-rollback: record a rollback in the history table + if args.record_rollback: + from .history import ( + compute_migration_hash, + ensure_history_table, + record_rollback, + ) + + value = args.record_rollback + if os.path.exists(value) and value.endswith(".sql"): + with open(value) as f: + migration_hash = compute_migration_hash(f.read()) + else: + migration_hash = value.strip() + + dburl = args.dburl_from + if not dburl: + print( + "MigraDiff: --record-rollback requires a database URL.", + file=err, + ) + return 1 + + from sqlbag import S + + try: + with S(dburl) as s: + ensure_history_table(s) + record_rollback( + s, + migration_hash, + status=args.rollback_status, + ) + print( + "MigraDiff: Rollback recorded for hash {}".format(migration_hash[:12]), + file=out, + ) + return 0 + except ValueError as e: + print(str(e), file=err) + return 1 + + # --promote: multi-environment promotion chain + if args.promote: + from .history import ensure_history_table, get_applied_hashes, record_applied + + promote_parts = re.split(r":(?!//)", args.promote) + if len(promote_parts) < 2: + print( + "MigraDiff: --promote requires at least 2 environments" + " (colon-separated).", + file=err, + ) + return 1 + + try: + chain = [_resolve_env_aliases_part(p) for p in promote_parts] + except ValueError as e: + print(str(e), file=err) + return 1 + + from sqlbag import S + + for i in range(len(chain) - 1): + from_url = chain[i] + to_url = chain[i + 1] + from_label = redact_credentials(from_url) + to_label = redact_credentials(to_url) + + # Conflict check: ensure to_env has no hashes from_env doesn't have + try: + with S(from_url) as s_from: + ensure_history_table(s_from) + from_hashes = get_applied_hashes(s_from) + with S(to_url) as s_to: + ensure_history_table(s_to) + to_hashes = get_applied_hashes(s_to) + except Exception as e: + print( + "MigraDiff: Could not check migration state for" + " {} -> {}: {}".format(from_label, to_label, e), + file=err, + ) + return 1 + + stray_hashes = to_hashes - from_hashes + if stray_hashes: + print( + "MigraDiff: CONFLICT in promotion chain" + " {} -> {}".format(from_label, to_label), + file=err, + ) + print( + " {} has migrations not found in {}:".format(to_label, from_label), + file=err, + ) + for h in sorted(stray_hashes): + print(" {}".format(h[:12]), file=err) + print( + " Resolve the conflict before continuing the chain.", + file=err, + ) + return 1 + + # Run the diff for this hop + print("=== {} -> {} ===".format(from_label, to_label), file=out) + + sub_args = argparse.Namespace(**vars(args)) + sub_args.dburl_from = from_url + sub_args.dburl_target = to_url + sub_args._original_from = from_url + sub_args._original_target = to_url + sub_args.from_migrations_dir = None + sub_args.from_file = False + + hop_status = _run_inner(sub_args, out, err) + + if hop_status == 0 and not sub_args._has_statements: + print( + "No changes needed: {} -> {}".format(from_label, to_label), + file=out, + ) + continue + + if hop_status == 1 or hop_status == 3: + return hop_status + + # Record history if --record-history + if args.record_history and hasattr(sub_args, "_generated_sql"): + from .history import compute_migration_hash + + sql_text = sub_args._generated_sql + if sql_text and sql_text.strip(): + fwd_hash = compute_migration_hash(sql_text) + rollback_sql_text = getattr(sub_args, "_generated_rollback", None) + with S(to_url) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash=fwd_hash, + forward_sql=sql_text, + rollback_sql=rollback_sql_text, + environment_label=args.env_label, + ) + + return 0 + if args.setup_ai: from .ai_explain import setup_ai_interactive @@ -1154,8 +1444,10 @@ def _run_inner(args, out=None, err=None): file=err, ) + sql_output = None try: if statements: + sql_output = modified_statements.sql if args.output == "json": json_out = format_json_output( statements, @@ -1265,6 +1557,38 @@ def _run_inner(args, out=None, err=None): ) return 3 + # Capture metadata for --promote and --record-history + args._has_statements = bool(statements) + args._generated_sql = sql_output if statements else None + args._generated_rollback = rollback_result["text"] if rollback_result else None + + # --record-history: record the generated migration + if args.record_history and statements and args.dburl_target: + from .history import ( + compute_migration_hash, + ensure_history_table, + record_applied, + ) + from sqlbag import S as _S + + fwd_hash = compute_migration_hash(sql_output) + rollback_sql_text = args._generated_rollback + try: + with _S(args.dburl_target) as _s: + ensure_history_table(_s) + record_applied( + _s, + migration_hash=fwd_hash, + forward_sql=sql_output, + rollback_sql=rollback_sql_text, + environment_label=args.env_label, + ) + except Exception as e: + print( + "MigraDiff: Failed to record migration history: {}".format(e), + file=err, + ) + if not statements: return 0 diff --git a/migra/history.py b/migra/history.py new file mode 100644 index 0000000..f8618e8 --- /dev/null +++ b/migra/history.py @@ -0,0 +1,191 @@ +from __future__ import unicode_literals + +import hashlib +import re + +from sqlalchemy import text + +HISTORY_TABLE = "migradiff_history" + + +def compute_migration_hash(sql_text): + normalized = _normalize_sql(sql_text) + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +def _normalize_sql(sql_text): + lines = sql_text.split("\n") + stripped_lines = [line.rstrip() for line in lines] + result = "\n".join(stripped_lines) + result = re.sub(r"\n{3,}", "\n\n", result) + result = result.strip() + return result + + +CREATE_TABLE_SQL = """ +CREATE TABLE IF NOT EXISTS {table} ( + id BIGSERIAL PRIMARY KEY, + migration_hash TEXT NOT NULL, + migration_name TEXT, + checksum TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now(), + applied_by TEXT, + forward_sql TEXT NOT NULL, + rollback_sql TEXT, + rollback_status TEXT NOT NULL DEFAULT 'not_attempted', + rolled_back_at TIMESTAMPTZ, + rolled_back_by TEXT, + environment_label TEXT, + source TEXT NOT NULL DEFAULT 'cli' +); + +CREATE INDEX IF NOT EXISTS idx_{table}_migration_hash ON {table} (migration_hash); +""".format(table=HISTORY_TABLE) + + +def ensure_history_table(session): + session.execute(text(CREATE_TABLE_SQL)) + session.commit() + + +def _current_user(session): + row = session.execute(text("SELECT current_user")).scalar() + return row + + +def record_applied( + session, + *, + migration_hash, + migration_name=None, + forward_sql, + rollback_sql=None, + environment_label=None, + applied_by=None, +): + if applied_by is None: + applied_by = _current_user(session) + session.execute( + text(""" + INSERT INTO {table} + (migration_hash, migration_name, checksum, forward_sql, + rollback_sql, environment_label, applied_by) + VALUES + (:migration_hash, :migration_name, :checksum, :forward_sql, + :rollback_sql, :environment_label, :applied_by) + """.format(table=HISTORY_TABLE)), + { + "migration_hash": migration_hash, + "migration_name": migration_name, + "checksum": migration_hash, + "forward_sql": forward_sql, + "rollback_sql": rollback_sql, + "environment_label": environment_label, + "applied_by": applied_by, + }, + ) + session.commit() + row = session.execute(text("SELECT lastval()")).scalar() + return row + + +def record_rollback( + session, + migration_hash, + *, + status="rolled_back", + rolled_back_by=None, +): + if rolled_back_by is None: + rolled_back_by = _current_user(session) + + row = session.execute( + text(""" + SELECT id FROM {table} + WHERE migration_hash = :migration_hash + ORDER BY applied_at DESC + LIMIT 1 + FOR UPDATE + """.format(table=HISTORY_TABLE)), + {"migration_hash": migration_hash}, + ).fetchone() + + if not row: + raise ValueError( + "MigraDiff: No history entry found for hash {}".format(migration_hash) + ) + + session.execute( + text(""" + UPDATE {table} + SET rollback_status = :status, + rolled_back_at = now(), + rolled_back_by = :rolled_back_by + WHERE id = :id + """.format(table=HISTORY_TABLE)), + { + "status": status, + "rolled_back_by": rolled_back_by, + "id": row[0], + }, + ) + session.commit() + + +def get_applied_hashes(session): + rows = session.execute(text(""" + SELECT DISTINCT migration_hash FROM {table} + """.format(table=HISTORY_TABLE))).fetchall() + return {row[0] for row in rows} + + +def get_history(session, environment_label=None, limit=None): + conditions = [] + params = {} + if environment_label is not None: + conditions.append("environment_label = :environment_label") + params["environment_label"] = environment_label + + where_clause = "" + if conditions: + where_clause = "WHERE " + " AND ".join(conditions) + + limit_clause = "" + if limit is not None: + limit_clause = "LIMIT :limit" + params["limit"] = limit + + rows = session.execute( + text(""" + SELECT id, migration_hash, migration_name, checksum, + applied_at, applied_by, forward_sql, rollback_sql, + rollback_status, rolled_back_at, rolled_back_by, + environment_label, source + FROM {table} + {where} + ORDER BY applied_at DESC + {limit} + """.format(table=HISTORY_TABLE, where=where_clause, limit=limit_clause)), + params, + ).fetchall() + + result = [] + for row in rows: + result.append( + { + "id": row[0], + "migration_hash": row[1], + "migration_name": row[2], + "checksum": row[3], + "applied_at": str(row[4]) if row[4] else None, + "applied_by": row[5], + "forward_sql": row[6], + "rollback_sql": row[7], + "rollback_status": row[8], + "rolled_back_at": str(row[9]) if row[9] else None, + "rolled_back_by": row[10], + "environment_label": row[11], + "source": row[12], + } + ) + return result diff --git a/tests/test_command_promote.py b/tests/test_command_promote.py new file mode 100644 index 0000000..4c99847 --- /dev/null +++ b/tests/test_command_promote.py @@ -0,0 +1,260 @@ +from __future__ import unicode_literals + +import io + +from sqlalchemy import text +from sqlbag import S, temporary_database + + +def outs(): + return io.StringIO(), io.StringIO() + + +def _create_table(db_url, table_name, columns="id int primary key"): + with S(db_url) as s: + s.execute(text("CREATE TABLE {} ({});".format(table_name, columns))) + + +def _create_schema_from_sql(db_url, sql): + with S(db_url) as s: + for stmt in sql.split(";"): + stmt = stmt.strip() + if stmt: + s.execute(text(stmt)) + + +class TestPromoteHappyPath: + def test_two_env_chain(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging: + _create_table(dev, "users") + _create_schema_from_sql( + staging, + "CREATE TABLE users (id int primary key);" + " CREATE TABLE posts (id int primary key);", + ) + + from migra.command import parse_args, run + + args = parse_args( + [ + "--unsafe", + "--promote", + "{}:{}".format(dev, staging), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + output = out.getvalue() + assert "=== " in output + assert "create table" in output or "No changes needed" in output + + def test_three_env_chain(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging, temporary_database(host="localhost") as prod: + _create_table(dev, "users") + _create_table(dev, "posts") + _create_schema_from_sql( + staging, + "CREATE TABLE users (id int primary key);" + " CREATE TABLE posts (id int primary key);", + ) + _create_schema_from_sql( + prod, + "CREATE TABLE users (id int primary key);" + " CREATE TABLE posts (id int primary key);" + " CREATE TABLE comments (id int primary key);", + ) + + from migra.command import parse_args, run + + args = parse_args( + [ + "--unsafe", + "--promote", + "{}:{}:{}".format(dev, staging, prod), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + + +class TestPromoteConflictDetection: + def test_staging_ahead_of_dev_conflict(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging: + _create_table(dev, "users") + _create_table(staging, "users") + _create_table(staging, "posts") + + from migra.command import parse_args, run + from migra.history import ensure_history_table, record_applied + + with S(dev) as s: + ensure_history_table(s) + with S(staging) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash="hash_not_in_dev", + forward_sql="CREATE TABLE posts (id int);", + environment_label="staging", + ) + + args = parse_args( + [ + "--unsafe", + "--promote", + "{}:{}".format(dev, staging), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "CONFLICT" in err.getvalue() + assert "hash_not_in" in err.getvalue() + + +class TestPromoteEmptyDiff: + def test_identical_envs_no_output(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging: + _create_table(dev, "users") + _create_table(staging, "users") + + from migra.command import parse_args, run + + args = parse_args( + [ + "--unsafe", + "--promote", + "{}:{}".format(dev, staging), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + output = out.getvalue() + assert "No changes needed" in output + + +class TestPromoteSafeMode: + def test_destructive_halts_chain(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging: + _create_schema_from_sql( + dev, + "CREATE TABLE users (id int primary key);" + " CREATE TABLE posts (id int primary key);", + ) + _create_table(staging, "users") + + from migra.command import parse_args, run + + args = parse_args( + [ + "--promote", + "{}:{}".format(dev, staging), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "Destructive" in err.getvalue() + + def test_force_destructive_allows_chain(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging: + _create_schema_from_sql( + dev, + "CREATE TABLE users (id int primary key);" + " CREATE TABLE posts (id int primary key);", + ) + _create_table(staging, "users") + + from migra.command import parse_args, run + + args = parse_args( + [ + "--force-destructive", + "--promote", + "{}:{}".format(dev, staging), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + + +class TestPromoteRecordHistory: + def test_record_history_per_hop(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging: + _create_table(dev, "users") + _create_schema_from_sql( + staging, + "CREATE TABLE users (id int primary key);" + " CREATE TABLE posts (id int primary key);", + ) + + from migra.command import parse_args, run + + args = parse_args( + [ + "--unsafe", + "--record-history", + "--env-label", + "staging", + "--promote", + "{}:{}".format(dev, staging), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + + from migra.history import get_history + + with S(staging) as s: + entries = get_history(s) + assert len(entries) >= 1 + assert entries[0]["environment_label"] == "staging" + + +class TestPromoteInvalidArgs: + def test_single_env_error(self): + from migra.command import parse_args, run + + args = parse_args( + [ + "--promote", + "dev", + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "requires at least 2" in err.getvalue() + + def test_unknown_alias_error(self): + from migra.command import parse_args, run + + args = parse_args( + [ + "--promote", + "dev:nonexistent_alias_xyz", + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "Unknown environment alias" in err.getvalue() diff --git a/tests/test_command_rollback_tracking.py b/tests/test_command_rollback_tracking.py new file mode 100644 index 0000000..edffa3f --- /dev/null +++ b/tests/test_command_rollback_tracking.py @@ -0,0 +1,193 @@ +from __future__ import unicode_literals + +import io + +from sqlalchemy import text +from sqlbag import S, temporary_database + + +def outs(): + return io.StringIO(), io.StringIO() + + +def _create_table(db_url, table_name, columns="id int primary key"): + with S(db_url) as s: + s.execute(text("CREATE TABLE {} ({});".format(table_name, columns))) + + +class TestRecordRollbackCli: + def test_record_rollback_after_applied(self): + with temporary_database(host="localhost") as db_url: + from migra.history import ensure_history_table, record_applied + + with S(db_url) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash="abc123hash", + forward_sql="SELECT 1;", + environment_label="dev", + ) + + from migra.command import parse_args, run + + args = parse_args( + [ + "--record-rollback", + "abc123hash", + db_url, + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + assert "Rollback recorded" in out.getvalue() + + from migra.history import get_history + + with S(db_url) as s: + entries = get_history(s) + assert len(entries) == 1 + assert entries[0]["rollback_status"] == "rolled_back" + assert entries[0]["rolled_back_at"] is not None + + def test_record_rollback_failed_status(self): + with temporary_database(host="localhost") as db_url: + from migra.history import ensure_history_table, record_applied + + with S(db_url) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash="abc123hash", + forward_sql="SELECT 1;", + ) + + from migra.command import parse_args, run + + args = parse_args( + [ + "--record-rollback", + "abc123hash", + db_url, + "--rollback-status", + "rollback_failed", + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + + from migra.history import get_history + + with S(db_url) as s: + entries = get_history(s) + assert entries[0]["rollback_status"] == "rollback_failed" + + def test_record_rollback_from_file(self): + import os + import tempfile + + with temporary_database(host="localhost") as db_url: + from migra.history import ( + compute_migration_hash, + ensure_history_table, + record_applied, + ) + + sql_content = "ALTER TABLE users DROP COLUMN email;" + actual_hash = compute_migration_hash(sql_content) + with S(db_url) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash=actual_hash, + forward_sql=sql_content, + ) + + with tempfile.NamedTemporaryFile( + mode="w", suffix=".sql", delete=False + ) as f: + f.write(sql_content) + f.flush() + rollback_file = f.name + + try: + from migra.command import parse_args, run + + args = parse_args( + [ + "--record-rollback", + rollback_file, + db_url, + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + assert "Rollback recorded" in out.getvalue() + finally: + os.unlink(rollback_file) + + def test_missing_history_entry(self): + with temporary_database(host="localhost") as db_url: + from migra.command import parse_args, run + + args = parse_args( + [ + "--record-rollback", + "nonexistent", + db_url, + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "No history entry found" in err.getvalue() + + def test_missing_db_url(self): + from migra.command import parse_args, run + + args = parse_args( + [ + "--record-rollback", + "abc123", + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "requires a database URL" in err.getvalue() + + +class TestStatusAfterRollback: + def test_status_shows_rolled_back_annotation(self): + with temporary_database(host="localhost") as db_url: + from migra.history import ( + ensure_history_table, + record_applied, + record_rollback, + ) + + with S(db_url) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash="abc123", + forward_sql="SELECT 1;", + ) + record_rollback(s, "abc123", status="rolled_back") + + from migra.command import parse_args, run + + args = parse_args( + [ + "--status", + db_url, + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + output = out.getvalue() + assert "ROLLED BACK" in output diff --git a/tests/test_history.py b/tests/test_history.py new file mode 100644 index 0000000..e18c5af --- /dev/null +++ b/tests/test_history.py @@ -0,0 +1,340 @@ +from __future__ import unicode_literals + +import io + +from sqlbag import S, temporary_database + + +def outs(): + return io.StringIO(), io.StringIO() + + +class TestComputeMigrationHash: + def test_basic_hash(self): + from migra.history import compute_migration_hash + + h1 = compute_migration_hash("ALTER TABLE users ADD COLUMN email text;") + h2 = compute_migration_hash("ALTER TABLE users ADD COLUMN email text;") + assert h1 == h2 + assert len(h1) == 64 + + def test_whitespace_normalization(self): + from migra.history import compute_migration_hash + + sql_a = "ALTER TABLE users ADD COLUMN email text;" + sql_b = " ALTER TABLE users ADD COLUMN email text;\n\n" + assert compute_migration_hash(sql_a) == compute_migration_hash(sql_b) + + def test_trailing_whitespace_per_line(self): + from migra.history import compute_migration_hash + + sql_a = "ALTER TABLE users ADD COLUMN email text;\nALTER TABLE posts ADD COLUMN title text;" + sql_b = "ALTER TABLE users ADD COLUMN email text; \nALTER TABLE posts ADD COLUMN title text; " + assert compute_migration_hash(sql_a) == compute_migration_hash(sql_b) + + def test_collapsed_blank_lines(self): + from migra.history import compute_migration_hash + + sql_a = "LINE1\n\nLINE2" + sql_b = "LINE1\n\n\n\n\nLINE2" + assert compute_migration_hash(sql_a) == compute_migration_hash(sql_b) + + def test_different_sql_different_hash(self): + from migra.history import compute_migration_hash + + h1 = compute_migration_hash("ALTER TABLE users ADD COLUMN email text;") + h2 = compute_migration_hash("ALTER TABLE users DROP COLUMN email;") + assert h1 != h2 + + +class TestEnsureHistoryTable: + def test_table_creation_idempotent(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ensure_history_table + + ensure_history_table(s) + ensure_history_table(s) + + # Verify table exists by querying it + rows = s.execute( + "SELECT table_name FROM information_schema.tables " + "WHERE table_name = 'migradiff_history'" + ).fetchall() + assert len(rows) == 1 + + # Verify columns + cols = s.execute( + "SELECT column_name, data_type FROM information_schema.columns " + "WHERE table_name = 'migradiff_history' " + "ORDER BY ordinal_position" + ).fetchall() + col_names = [c[0] for c in cols] + assert "id" in col_names + assert "migration_hash" in col_names + assert "checksum" in col_names + assert "applied_at" in col_names + assert "applied_by" in col_names + assert "forward_sql" in col_names + assert "rollback_sql" in col_names + assert "rollback_status" in col_names + assert "rolled_back_at" in col_names + assert "rolled_back_by" in col_names + assert "environment_label" in col_names + assert "source" in col_names + + def test_default_rollback_status(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ensure_history_table, record_applied + + ensure_history_table(s) + record_applied( + s, + migration_hash="abc123", + forward_sql="SELECT 1;", + ) + row = s.execute( + "SELECT rollback_status FROM migradiff_history" + ).fetchone() + assert row[0] == "not_attempted" + + +class TestRecordAppliedAndGetAppliedHashes: + def test_round_trip(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_applied_hashes, + record_applied, + ) + + ensure_history_table(s) + + record_applied( + s, + migration_hash="hash001", + forward_sql="CREATE TABLE t1 (id int);", + ) + record_applied( + s, + migration_hash="hash002", + forward_sql="ALTER TABLE t1 ADD COLUMN name text;", + ) + + hashes = get_applied_hashes(s) + assert hashes == {"hash001", "hash002"} + + def test_duplicate_hash(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_applied_hashes, + record_applied, + ) + + ensure_history_table(s) + record_applied( + s, + migration_hash="hash001", + forward_sql="CREATE TABLE t1 (id int);", + ) + record_applied( + s, + migration_hash="hash001", + forward_sql="CREATE TABLE t1 (id int);", + ) + hashes = get_applied_hashes(s) + assert hashes == {"hash001"} + + def test_record_with_metadata(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_history, + record_applied, + ) + + ensure_history_table(s) + record_applied( + s, + migration_hash="hash001", + migration_name="v001_create_users", + forward_sql="CREATE TABLE users (id int);", + rollback_sql="DROP TABLE users;", + environment_label="dev", + applied_by="test_user", + ) + + entries = get_history(s) + assert len(entries) == 1 + e = entries[0] + assert e["migration_hash"] == "hash001" + assert e["migration_name"] == "v001_create_users" + assert e["forward_sql"] == "CREATE TABLE users (id int);" + assert e["rollback_sql"] == "DROP TABLE users;" + assert e["environment_label"] == "dev" + assert e["applied_by"] == "test_user" + assert e["checksum"] == "hash001" + assert e["source"] == "cli" + assert e["rollback_status"] == "not_attempted" + + def test_record_applied_returns_id(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ensure_history_table, record_applied + + ensure_history_table(s) + new_id = record_applied( + s, + migration_hash="hash001", + forward_sql="SELECT 1;", + ) + assert new_id is not None + assert isinstance(new_id, int) + assert new_id > 0 + + +class TestRecordRollback: + def test_update_most_recent_matching_row(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_history, + record_applied, + record_rollback, + ) + + ensure_history_table(s) + + record_applied( + s, + migration_hash="abc123", + forward_sql="V1", + ) + record_applied( + s, + migration_hash="abc123", + forward_sql="V2", + ) + + record_rollback(s, "abc123", status="rolled_back") + entries = get_history(s) + assert len(entries) == 2 + most_recent = entries[0] + assert most_recent["forward_sql"] == "V2" + assert most_recent["rollback_status"] == "rolled_back" + assert most_recent["rolled_back_at"] is not None + assert most_recent["rolled_back_by"] is not None + + older = entries[1] + assert older["forward_sql"] == "V1" + assert older["rollback_status"] == "not_attempted" + assert older["rolled_back_at"] is None + + def test_rollback_failed_status(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_history, + record_applied, + record_rollback, + ) + + ensure_history_table(s) + record_applied( + s, + migration_hash="abc123", + forward_sql="SELECT 1;", + ) + record_rollback(s, "abc123", status="rollback_failed") + + entries = get_history(s) + assert entries[0]["rollback_status"] == "rollback_failed" + + def test_no_matching_row_raises(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ensure_history_table, record_rollback + + ensure_history_table(s) + import pytest + + with pytest.raises(ValueError, match="No history entry found"): + record_rollback(s, "nonexistent_hash") + + +class TestGetHistory: + def test_empty_history(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ensure_history_table, get_history + + ensure_history_table(s) + entries = get_history(s) + assert entries == [] + + def test_limit(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_history, + record_applied, + ) + + ensure_history_table(s) + for i in range(5): + record_applied( + s, + migration_hash="hash{:03d}".format(i), + forward_sql="SELECT {};".format(i), + ) + + limited = get_history(s, limit=3) + assert len(limited) == 3 + # Most recent first + assert limited[0]["migration_hash"] == "hash004" + + unlimited = get_history(s) + assert len(unlimited) == 5 + + def test_filter_by_environment(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_history, + record_applied, + ) + + ensure_history_table(s) + + record_applied( + s, + migration_hash="hash001", + forward_sql="SELECT 1;", + environment_label="dev", + ) + record_applied( + s, + migration_hash="hash002", + forward_sql="SELECT 2;", + environment_label="prod", + ) + + dev_entries = get_history(s, environment_label="dev") + assert len(dev_entries) == 1 + assert dev_entries[0]["migration_hash"] == "hash001" + + prod_entries = get_history(s, environment_label="prod") + assert len(prod_entries) == 1 + assert prod_entries[0]["migration_hash"] == "hash002" + + all_entries = get_history(s) + assert len(all_entries) == 2 From 74137c1160b0b3d0a9c851c254ef72473c27becf Mon Sep 17 00:00:00 2001 From: MigraDiff Agent Date: Sat, 8 Aug 2026 13:08:42 -0700 Subject: [PATCH 02/11] docs: add Business License Agreement licensing model Co-Authored-By: Claude Sonnet 5 --- LICENSING.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 LICENSING.md diff --git a/LICENSING.md b/LICENSING.md new file mode 100644 index 0000000..ad98f19 --- /dev/null +++ b/LICENSING.md @@ -0,0 +1,40 @@ +# LICENSING.md + +## Model: MIT + Business License Agreement (BLA) + +All features are available to everyone. No code gatekeeping. +No feature gates. No license key required to unlock functionality. + +### MIT License (Free) + +**Who:** Individuals, open-source projects, and teams evaluating +for production — no time limit. + +**Covers:** All features, unlimited use, full CLI, all schema diffing, +RLS policies, AI flags (--explain, --rollback, --advise, --generate), +JSON output, GitHub Action, Docker images. + +**Support:** Community (GitHub issues) + +### Business License Agreement (Paid) + +**Who:** Required when your team ships to production commercially +(any company with employees using the tool as part of their work). + +**Covers:** Everything in MIT license, plus email support and +legal documentation for compliance auditing. + +**Tiers:** +- Small Team (1–10 developers): $599/year +- Mid-Market (10–50 developers): $1,999/year +- Enterprise (50+ developers): $5,000/year + +**Simple rule:** Evaluate freely, no time limit. Purchase when +your team ships to production. + +Questions? [licensing@lateos.ai](mailto:licensing@lateos.ai) +Full terms: [lateos.ai/migradiff/licensing](https://lateos.ai/migradiff/licensing) + +--- + +Full MIT license text is in the LICENSE file. \ No newline at end of file From 38743ba748b598d0af3bf8c3d12b718c6c50479c Mon Sep 17 00:00:00 2001 From: MigraDiff Agent Date: Sat, 8 Aug 2026 13:08:47 -0700 Subject: [PATCH 03/11] docs: reconcile PROJECT_PLAN docs with actual shipped state Both files still described v1.5.1 as current and --explain-drift/ multi-language README as future work, even though v1.7.2 is on PyPI and both features shipped long ago. Updates completed-features, current version, roadmap, version history, and known limitations (including that docker-compose.yml is referenced but missing from the repo) to match the real code and CHANGELOG. Co-Authored-By: Claude Sonnet 5 --- PROJECT_PLAN.md | 208 ++++++++++++++++++++++++----------------------- PROJECT_PLAN2.md | 199 +++++++++++++++++++++++++-------------------- 2 files changed, 216 insertions(+), 191 deletions(-) diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md index dc8ed1b..eb695f6 100644 --- a/PROJECT_PLAN.md +++ b/PROJECT_PLAN.md @@ -25,17 +25,50 @@ ## Completed Features -### v1.3.0 (Sessions 009–010) +### v1.3.0 (Sessions 009–010) — 2026-05-30 - `--explain`: plain-English explanation of migrations - `--from-migrations-dir`: load migrations from directory -### v1.4.0 (Sessions 011–013) +### v1.4.0 (Sessions 011–013) — 2026-06-01 - `--rollback`: generates reversal migration SQL - `--advise`: deterministic + AI performance/risk assessment - `--generate`: writes migration SQL from plain-English description -### v1.5.0 (Session 017) -- `--comment-on`: diffs COMMENT ON annotations on objects +### v1.5.0 (Session 017) — 2026-06-01 +- COMMENT ON diffing: detects added/changed/removed comments across all + object types; automatic in `--from-file` mode; feeds `--explain`/`--generate` + as schema context. (There is no separate `--comment-on` flag — this runs + by default; the plan previously described it as a flag, which was wrong.) + +### v1.6.0 — 2026-06-04 +- `--explain-drift`: AI-powered schema drift analysis between two live + databases (`--from-db`/`--to-db`), BREAKING/WARNING/INFO categorization, + live table sizes for risk assessment +- `migra/db_inspector.py`: standalone `get_remote_schema()`/`compare_schemas()` + +### v1.7.x — 2026-06-08 +- PyPI metadata fixes (PEP 621 `[project]` table, homepage/repo URLs) +- Repo moved from `migradiff/migra` to `postgresql-tools/migra` + +### Multi-language docs & licensing (merged pre-1.7.2, docs-only, no version bump) +- README translated into 6 languages (fr, de, ja, zh, hi, he) alongside English +- Licensing section (personal story + Business License ask) added to all READMEs + +### Migration state tracking — code-complete, not yet released +- `--status` / `--history`: view migration history from a `migradiff_history` table +- `--record-history` / `--env-label`: record generated migrations against a target DB +- `--promote`: chain migrations across environments (`dev:staging:prod` or + `~/.config/migradiff/environments.json` aliases) with conflict detection per hop +- `--record-rollback` / `--rollback-status`: close the audit loop on executed rollbacks +- New module `migra/history.py`; 32 new tests (`test_history.py`, + `test_command_promote.py`, `test_command_rollback_tracking.py`) +- Verified 2026-08-08: all 32 new tests pass against a real Postgres instance, + full suite (342 tests) passes, flake8/black clean, and the CLI was smoke-tested + end-to-end (`--status`, `--record-history`, `--promote`, `--record-rollback` + round-tripped correctly against live databases) +- Documented in CHANGELOG `[Unreleased]` and README's "Migration State Tracking" + section; sitting uncommitted on branch `new-feature-6-18-2026` — next release + will be **v1.8.0** ### Infrastructure (Session 018) - ✅ Production CI/CD pipeline @@ -47,90 +80,44 @@ ## Current Version -**v1.5.1** (released June 4, 2026) +**v1.7.2** (released 2026-06-08 — matches `pyproject.toml` and PyPI) -Test counts: -- Baseline: 109 tests -- v1.3.0: 142 tests (after --explain) -- v1.4.0: 175 tests (after AI suite) -- v1.5.0: 190 tests (after COMMENT ON) +Migration state tracking (above) is feature-complete and fully tested on +`new-feature-6-18-2026` but not yet merged or released. + +Test counts (actual, `pytest tests -q`, excluding the separate +characterization suite): **342 passed, 2 skipped**, verified 2026-08-08. --- ## Free Tier Roadmap -### Session 019: `--explain-drift` (7–10 days) - -**Feature:** Compare two **live PostgreSQL databases** and explain differences. - -```bash -migra --from-db "postgresql://user:pass@old.example.com/db" \ - --to-db "postgresql://user:pass@prod.example.com/db" \ - --explain-drift -``` - -**Output:** Human-readable explanation of schema differences with risk categorization (BREAKING/WARNING/INFO). - -**Use case:** "What changed in production? Is it safe?" - -**Tests:** 80+ new tests (target: 270+ total) +### Shipped: `--explain-drift` (v1.6.0) and multi-language README -**Release:** v1.6.0 +Both were originally planned as "Session 019" and "Session 020" — both are +done. See "Completed Features" above for what actually shipped: +`--explain-drift` in v1.6.0 (2026-06-04), and the README translations +(fr, de, ja, zh, hi, he — 6 languages, merged as a docs-only PR, no +dedicated version bump). -**Why first:** -- Highest differentiation (competitors don't have this) -- Reuses existing schema inspection logic -- Fills gap: explain migrations vs. reality +### Next: release migration state tracking as v1.8.0 ---- - -### Session 020: Multi-Language README (3–5 days) - -**Feature:** README in multiple languages (i18n). - -Supported languages: -- 🇬🇧 English (default) -- 🇫🇷 French (README.fr.md) -- 🇩🇪 German (README.de.md) -- 🇯🇵 Japanese (README.ja.md) -- 🇨🇳 Chinese (README.zh.md) -- 🇮🇳 Hindi (README.hi.md) -- 🇮🇱 Hebrew (README.he.md) - -**How:** -- Maintain main README.md (English) -- Create translated versions in repo root -- Link from main README -- Use GitHub language detection for auto-redirect - -**Why:** -- Global audience (PostgreSQL is international) -- **Strategic focus:** India has second-largest PostgreSQL user base globally (after US) -- Drives adoption in high-growth markets (India, China, Japan) -- Differentiator (most tools English-only) -- SEO boost (ranked in multiple language searches) -- Hindi captures ~600M+ speakers in India tech ecosystem - -**Market sizing:** -- US: ~40% of users (English) -- India: ~20% of users (Hindi + English) -- Europe: ~15% of users (French, German) -- Asia-Pacific: ~15% of users (Chinese, Japanese) -- Israel/other: ~10% of users (Hebrew, English) - -**Release:** v1.6.1 - ---- +`--status`/`--history`/`--promote`/`--record-rollback` (see "Completed +Features") is code-complete and fully tested but still sitting on branch +`new-feature-6-18-2026`. Landing this — merge, version bump, tag, PyPI +release — is the immediate next step, ahead of any new feature work below. -### Planned Sessions (Backlog) +### Planned (Backlog) -| Session | Feature | Effort | Value | Notes | -|---------|---------|--------|-------|-------| -| 021 | `--document` | Medium-High | High | Schema documentation generation | -| 022 | pgvector support | Low | Medium | Modern Postgres vector types | -| 023 | `--suggest-indexes` | Medium | Medium | AI recommends useful indexes | -| 024 | `--dry-run --explain` | Low | Medium | Preview changes without applying | -| 025 | Multi-schema refactor | Medium | High | Better support for multiple schemas | +| Feature | Effort | Value | Notes | +|---------|--------|-------|-------| +| `--apply` | Medium | High | Execute generated SQL directly instead of requiring the user to pipe to `psql`. Also needed so `migradiff_history` can record true "applied" state instead of just "generated/reviewed" — see the known limitation in README's Migration State Tracking section. | +| Native `--fail-on-destructive` flag | Low | High | Currently this behavior only exists inside the GitHub Action's `action-entrypoint.sh`. CircleCI/GitLab/pre-commit users have no equivalent without wrapping the CLI themselves. | +| `--document` | Medium-High | High | Schema documentation generation | +| Multi-schema hardening | Medium | High | `Migration.__init__` (`migra/migra.py`) still hard-rejects `schema` + `exclude_schema` together; cross-schema FK/dependency ordering in `add_all_changes()` hasn't had a dedicated multi-tenant test pass | +| pgvector support | Low | Medium | Modern Postgres vector types — unconfirmed whether `schemainspect` already round-trips them | +| `--suggest-indexes` | Medium | Medium | AI recommends useful indexes; can reuse `AIAdvisor`'s existing table-stats extraction | +| `--dry-run` | Low | Medium | Only meaningful once `--apply` exists | --- @@ -222,18 +209,31 @@ The `schemainspect` package (upstream: djrobstep, unmaintained) uses deprecated **Migration path (future):** Replace with maintained schema inspection library or migrate to Rust. +### docker-compose.yml referenced but missing from repo +`README.md`'s "Development Setup" section and the CHANGELOG (v1.1.0 entry) +both reference a `docker-compose.yml` for one-command local Postgres, but +no such file currently exists in the repo root. Verified 2026-08-08 by +spinning up Postgres manually (`docker run postgres:16 ...`) to run the +test suite instead. + --- ## Version History | Version | Release Date | Key Features | |---------|--------------|--------------| -| v1.3.0 | May 2026 | --explain, --from-migrations-dir | -| v1.4.0 | June 1, 2026 | --rollback, --advise, --generate | -| v1.5.0 | June 4, 2026 | COMMENT ON diffing | -| v1.5.1 | June 4, 2026 | Version bump (no feature changes) | -| v1.6.0 | TBD (Session 019) | --explain-drift | -| v1.6.1 | TBD (Session 020) | Multi-language README | +| v1.1.0 | 2026-05-29 | --from-file, --output json, pre-commit hook | +| v1.2.0 | 2026-05-29 | Column rename detection, --safe/--force-destructive, enum/composite/domain diffing | +| v1.3.0 | 2026-05-30 | --explain, --setup-ai, --from-migrations-dir | +| v1.4.0 | 2026-06-01 | --rollback, --advise, --generate | +| v1.5.0 | 2026-06-01 | COMMENT ON diffing | +| v1.6.0 | 2026-06-04 | --explain-drift | +| v1.7.1 | 2026-06-08 | PyPI homepage/repo URL fix | +| v1.7.2 | 2026-06-08 | PEP 621 metadata fix (current release) | +| v1.8.0 | TBD | Migration state tracking (--status/--history/--promote/--record-rollback) — code-complete, unreleased | + +Multi-language READMEs and the Licensing section merged as docs-only PRs +without a dedicated version bump — not tied to a CHANGELOG entry. --- @@ -277,16 +277,17 @@ The `schemainspect` package (upstream: djrobstep, unmaintained) uses deprecated ### Q2 2026 - ✅ v1.5.0 shipped - ✅ CI/CD pipeline live -- ⏳ v1.6.0 shipped (`--explain-drift`) -- ⏳ 50+ GitHub stars -- ⏳ 1k+ monthly PyPI downloads +- ✅ v1.6.0 shipped (`--explain-drift`) +- ⏳ 50+ GitHub stars (unverified — outside what a code/repo analysis can confirm) +- ⏳ 1k+ monthly PyPI downloads (unverified — outside what a code/repo analysis can confirm) ### Q3 2026 -- ⏳ v1.6.1 shipped (multi-language README) -- ⏳ v1.7.0 shipped (`--document`, pgvector support) -- ⏳ 10+ enterprise customers ($30k–$100k MRR) -- ⏳ 100+ GitHub stars -- ⏳ 5k+ monthly PyPI downloads +- ✅ Multi-language README shipped (docs-only PR, no dedicated version bump) +- ⏳ v1.8.0 shipped (migration state tracking — code-complete, awaiting release) +- ⏳ `--document`, pgvector support (not yet started) +- ⏳ 10+ enterprise customers ($30k–$100k MRR) (unverified) +- ⏳ 100+ GitHub stars (unverified) +- ⏳ 5k+ monthly PyPI downloads (unverified) ### Q4 2026 - ⏳ Enterprise tier revenue: $50k/month ARR @@ -316,23 +317,28 @@ The `schemainspect` package (upstream: djrobstep, unmaintained) uses deprecated ## Next Steps -1. **Session 019:** Implement `--explain-drift` (7–10 days) - - New feature: `migra --from-db X --to-db Y --explain-drift` - - 80+ tests (target 270+ total) - - Release v1.6.0 +1. **Land migration state tracking as v1.8.0** + - Merge `new-feature-6-18-2026` — code-complete, 342/342 tests passing, + flake8/black clean, and CLI-smoke-tested end-to-end as of 2026-08-08 + - Bump version, tag release, publish to PyPI + +2. **`--apply` flag** + - Closes the "generated vs. applied" gap called out in README's + Migration State Tracking known limitation + - Should auto-call the equivalent of `--record-history` on success -2. **Session 020:** Multi-language README (3–5 days) - - README in 6 languages - - i18n links from main README - - Release v1.6.1 +3. **Native `--fail-on-destructive` CLI flag** + - Promote the GitHub Action's destructive-detection behavior into + `command.py` itself so non-Action CI users (CircleCI, GitLab, plain + scripts) get it too -3. **Post v1.6.0:** Enterprise tier planning +4. **Post v1.8.0:** Enterprise tier planning - Design licensing system - Plan hosted features - Build enterprise marketing narrative --- -**Document version:** Updated June 4, 2026 (post-Session 018) +**Document version:** Updated 2026-08-08 (reconciled with actual repo/code state) **Last updated by:** Claude (with Leo) **Repository:** https://github.com/postgresql-tools/migra diff --git a/PROJECT_PLAN2.md b/PROJECT_PLAN2.md index c08da2a..3fccbcc 100644 --- a/PROJECT_PLAN2.md +++ b/PROJECT_PLAN2.md @@ -25,17 +25,50 @@ ## Completed Features -### v1.3.0 (Sessions 009–010) +### v1.3.0 (Sessions 009–010) — 2026-05-30 - `--explain`: plain-English explanation of migrations - `--from-migrations-dir`: load migrations from directory -### v1.4.0 (Sessions 011–013) +### v1.4.0 (Sessions 011–013) — 2026-06-01 - `--rollback`: generates reversal migration SQL - `--advise`: deterministic + AI performance/risk assessment - `--generate`: writes migration SQL from plain-English description -### v1.5.0 (Session 017) -- `--comment-on`: diffs COMMENT ON annotations on objects +### v1.5.0 (Session 017) — 2026-06-01 +- COMMENT ON diffing: detects added/changed/removed comments across all + object types; automatic in `--from-file` mode; feeds `--explain`/`--generate` + as schema context. (There is no separate `--comment-on` flag — this runs + by default; the plan previously described it as a flag, which was wrong.) + +### v1.6.0 — 2026-06-04 +- `--explain-drift`: AI-powered schema drift analysis between two live + databases (`--from-db`/`--to-db`), BREAKING/WARNING/INFO categorization, + live table sizes for risk assessment +- `migra/db_inspector.py`: standalone `get_remote_schema()`/`compare_schemas()` + +### v1.7.x — 2026-06-08 +- PyPI metadata fixes (PEP 621 `[project]` table, homepage/repo URLs) +- Repo moved from `migradiff/migra` to `postgresql-tools/migra` + +### Multi-language docs & licensing (merged pre-1.7.2, docs-only, no version bump) +- README translated into 6 languages (fr, de, ja, zh, hi, he) alongside English +- Licensing section (personal story + Business License ask) added to all READMEs + +### Migration state tracking — code-complete, not yet released +- `--status` / `--history`: view migration history from a `migradiff_history` table +- `--record-history` / `--env-label`: record generated migrations against a target DB +- `--promote`: chain migrations across environments (`dev:staging:prod` or + `~/.config/migradiff/environments.json` aliases) with conflict detection per hop +- `--record-rollback` / `--rollback-status`: close the audit loop on executed rollbacks +- New module `migra/history.py`; 32 new tests (`test_history.py`, + `test_command_promote.py`, `test_command_rollback_tracking.py`) +- Verified 2026-08-08: all 32 new tests pass against a real Postgres instance, + full suite (342 tests) passes, flake8/black clean, and the CLI was smoke-tested + end-to-end (`--status`, `--record-history`, `--promote`, `--record-rollback` + round-tripped correctly against live databases) +- Documented in CHANGELOG `[Unreleased]` and README's "Migration State Tracking" + section; sitting uncommitted on branch `new-feature-6-18-2026` — next release + will be **v1.8.0** ### Infrastructure (Session 018) - ✅ Production CI/CD pipeline @@ -47,80 +80,47 @@ ## Current Version -**v1.5.1** (released June 4, 2026) +**v1.7.2** (released 2026-06-08 — matches `pyproject.toml` and PyPI) -Test counts: -- Baseline: 109 tests -- v1.3.0: 142 tests (after --explain) -- v1.4.0: 175 tests (after AI suite) -- v1.5.0: 190 tests (after COMMENT ON) +Migration state tracking (above) is feature-complete and fully tested on +`new-feature-6-18-2026` but not yet merged or released. ---- - -## Free Tier Roadmap - -### Session 019: `--explain-drift` (7–10 days) - -**Feature:** Compare two **live PostgreSQL databases** and explain differences. - -```bash -migra --from-db "postgresql://user:pass@old.example.com/db" \ - --to-db "postgresql://user:pass@prod.example.com/db" \ - --explain-drift -``` - -**Output:** Human-readable explanation of schema differences with risk categorization (BREAKING/WARNING/INFO). - -**Use case:** "What changed in production? Is it safe?" - -**Tests:** 80+ new tests (target: 270+ total) - -**Release:** v1.6.0 - -**Why first:** -- Highest differentiation (competitors don't have this) -- Reuses existing schema inspection logic -- Fills gap: explain migrations vs. reality +Test counts (actual, `pytest tests -q`, excluding the separate +characterization suite): **342 passed, 2 skipped**, verified 2026-08-08. --- -### Session 020: Multi-Language README (3–5 days) - -**Feature:** README in multiple languages (i18n). +## Free Tier Roadmap -Supported languages: -- 🇬🇧 English (default) -- 🇪🇸 Spanish (README.es.md) -- 🇫🇷 French (README.fr.md) -- 🇩🇪 German (README.de.md) -- 🇯🇵 Japanese (README.ja.md) -- 🇨🇳 Chinese (README.zh.md) +### Shipped: `--explain-drift` (v1.6.0) and multi-language README -**How:** -- Maintain main README.md (English) -- Create translated versions in repo root -- Link from main README -- Use GitHub language detection for auto-redirect +Both were originally planned as "Session 019" and "Session 020" — both are +done. See "Completed Features" above for what actually shipped: +`--explain-drift` in v1.6.0 (2026-06-04), and the README translations. -**Why:** -- Global audience (PostgreSQL is international) -- Drives adoption outside English-speaking markets -- Differentiator (most tools English-only) -- SEO boost (ranked in multiple language searches) +Note: this document originally planned Spanish (README.es.md) instead of +Hindi/Hebrew for the language set — that changed during implementation. +The 6 languages that actually shipped are fr, de, ja, zh, hi, he (no +Spanish); see `PROJECT_PLAN.md` for the corrected rationale. -**Release:** v1.6.1 +### Next: release migration state tracking as v1.8.0 ---- +`--status`/`--history`/`--promote`/`--record-rollback` (see "Completed +Features") is code-complete and fully tested but still sitting on branch +`new-feature-6-18-2026`. Landing this — merge, version bump, tag, PyPI +release — is the immediate next step, ahead of any new feature work below. -### Planned Sessions (Backlog) +### Planned (Backlog) -| Session | Feature | Effort | Value | Notes | -|---------|---------|--------|-------|-------| -| 021 | `--document` | Medium-High | High | Schema documentation generation | -| 022 | pgvector support | Low | Medium | Modern Postgres vector types | -| 023 | `--suggest-indexes` | Medium | Medium | AI recommends useful indexes | -| 024 | `--dry-run --explain` | Low | Medium | Preview changes without applying | -| 025 | Multi-schema refactor | Medium | High | Better support for multiple schemas | +| Feature | Effort | Value | Notes | +|---------|--------|-------|-------| +| `--apply` | Medium | High | Execute generated SQL directly instead of requiring the user to pipe to `psql`. Also needed so `migradiff_history` can record true "applied" state instead of just "generated/reviewed" — see the known limitation in README's Migration State Tracking section. | +| Native `--fail-on-destructive` flag | Low | High | Currently this behavior only exists inside the GitHub Action's `action-entrypoint.sh`. CircleCI/GitLab/pre-commit users have no equivalent without wrapping the CLI themselves. | +| `--document` | Medium-High | High | Schema documentation generation | +| Multi-schema hardening | Medium | High | `Migration.__init__` (`migra/migra.py`) still hard-rejects `schema` + `exclude_schema` together; cross-schema FK/dependency ordering in `add_all_changes()` hasn't had a dedicated multi-tenant test pass | +| pgvector support | Low | Medium | Modern Postgres vector types — unconfirmed whether `schemainspect` already round-trips them | +| `--suggest-indexes` | Medium | Medium | AI recommends useful indexes; can reuse `AIAdvisor`'s existing table-stats extraction | +| `--dry-run` | Low | Medium | Only meaningful once `--apply` exists | --- @@ -212,18 +212,31 @@ The `schemainspect` package (upstream: djrobstep, unmaintained) uses deprecated **Migration path (future):** Replace with maintained schema inspection library or migrate to Rust. +### docker-compose.yml referenced but missing from repo +`README.md`'s "Development Setup" section and the CHANGELOG (v1.1.0 entry) +both reference a `docker-compose.yml` for one-command local Postgres, but +no such file currently exists in the repo root. Verified 2026-08-08 by +spinning up Postgres manually (`docker run postgres:16 ...`) to run the +test suite instead. + --- ## Version History | Version | Release Date | Key Features | |---------|--------------|--------------| -| v1.3.0 | May 2026 | --explain, --from-migrations-dir | -| v1.4.0 | June 1, 2026 | --rollback, --advise, --generate | -| v1.5.0 | June 4, 2026 | COMMENT ON diffing | -| v1.5.1 | June 4, 2026 | Version bump (no feature changes) | -| v1.6.0 | TBD (Session 019) | --explain-drift | -| v1.6.1 | TBD (Session 020) | Multi-language README | +| v1.1.0 | 2026-05-29 | --from-file, --output json, pre-commit hook | +| v1.2.0 | 2026-05-29 | Column rename detection, --safe/--force-destructive, enum/composite/domain diffing | +| v1.3.0 | 2026-05-30 | --explain, --setup-ai, --from-migrations-dir | +| v1.4.0 | 2026-06-01 | --rollback, --advise, --generate | +| v1.5.0 | 2026-06-01 | COMMENT ON diffing | +| v1.6.0 | 2026-06-04 | --explain-drift | +| v1.7.1 | 2026-06-08 | PyPI homepage/repo URL fix | +| v1.7.2 | 2026-06-08 | PEP 621 metadata fix (current release) | +| v1.8.0 | TBD | Migration state tracking (--status/--history/--promote/--record-rollback) — code-complete, unreleased | + +Multi-language READMEs and the Licensing section merged as docs-only PRs +without a dedicated version bump — not tied to a CHANGELOG entry. --- @@ -267,16 +280,17 @@ The `schemainspect` package (upstream: djrobstep, unmaintained) uses deprecated ### Q2 2026 - ✅ v1.5.0 shipped - ✅ CI/CD pipeline live -- ⏳ v1.6.0 shipped (`--explain-drift`) -- ⏳ 50+ GitHub stars -- ⏳ 1k+ monthly PyPI downloads +- ✅ v1.6.0 shipped (`--explain-drift`) +- ⏳ 50+ GitHub stars (unverified — outside what a code/repo analysis can confirm) +- ⏳ 1k+ monthly PyPI downloads (unverified — outside what a code/repo analysis can confirm) ### Q3 2026 -- ⏳ v1.6.1 shipped (multi-language README) -- ⏳ v1.7.0 shipped (`--document`, pgvector support) -- ⏳ 10+ enterprise customers ($30k–$100k MRR) -- ⏳ 100+ GitHub stars -- ⏳ 5k+ monthly PyPI downloads +- ✅ Multi-language README shipped (docs-only PR, no dedicated version bump) +- ⏳ v1.8.0 shipped (migration state tracking — code-complete, awaiting release) +- ⏳ `--document`, pgvector support (not yet started) +- ⏳ 10+ enterprise customers ($30k–$100k MRR) (unverified) +- ⏳ 100+ GitHub stars (unverified) +- ⏳ 5k+ monthly PyPI downloads (unverified) ### Q4 2026 - ⏳ Enterprise tier revenue: $50k/month ARR @@ -306,23 +320,28 @@ The `schemainspect` package (upstream: djrobstep, unmaintained) uses deprecated ## Next Steps -1. **Session 019:** Implement `--explain-drift` (7–10 days) - - New feature: `migra --from-db X --to-db Y --explain-drift` - - 80+ tests (target 270+ total) - - Release v1.6.0 +1. **Land migration state tracking as v1.8.0** + - Merge `new-feature-6-18-2026` — code-complete, 342/342 tests passing, + flake8/black clean, and CLI-smoke-tested end-to-end as of 2026-08-08 + - Bump version, tag release, publish to PyPI + +2. **`--apply` flag** + - Closes the "generated vs. applied" gap called out in README's + Migration State Tracking known limitation + - Should auto-call the equivalent of `--record-history` on success -2. **Session 020:** Multi-language README (3–5 days) - - README in 6 languages - - i18n links from main README - - Release v1.6.1 +3. **Native `--fail-on-destructive` CLI flag** + - Promote the GitHub Action's destructive-detection behavior into + `command.py` itself so non-Action CI users (CircleCI, GitLab, plain + scripts) get it too -3. **Post v1.6.0:** Enterprise tier planning +4. **Post v1.8.0:** Enterprise tier planning - Design licensing system - Plan hosted features - Build enterprise marketing narrative --- -**Document version:** Updated June 4, 2026 (post-Session 018) +**Document version:** Updated 2026-08-08 (reconciled with actual repo/code state) **Last updated by:** Claude (with Leo) **Repository:** https://github.com/postgresql-tools/migra From 4923cd4676fa28073c09d6c549a67350057f337d Mon Sep 17 00:00:00 2001 From: MigraDiff Agent Date: Sat, 8 Aug 2026 13:08:50 -0700 Subject: [PATCH 04/11] docs: add CLAUDE.md, decisions.md, and status.md CLAUDE.md documents build/test/lint commands and the core diff-engine, CLI, history-tracking, and AI-feature architecture for future agent sessions. decisions.md and status.md are empty/lightweight tracking docs to log decisions and branch status going forward. Co-Authored-By: Claude Sonnet 5 --- CLAUDE.md | 99 ++++++++++++++++++++++++++++++++++++++++++++++++++++ decisions.md | 14 ++++++++ status.md | 23 ++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 CLAUDE.md create mode 100644 decisions.md create mode 100644 status.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..fe650af --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,99 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project + +MigraDiff (package `migradiff`, CLI command `migra`) — an actively maintained fork of the deprecated +`djrobstep/migra`. It compares two PostgreSQL schemas (live connections, `pg_dump -s` files, or a +migrations directory applied to an ephemeral DB) and generates the SQL needed to transform one into +the other. + +**Contributing notice:** per `CONTRIBUTION_GUIDE.md` and the README, this repo is not currently +accepting external PRs — keep that in mind if asked to open one. + +## Commands + +Tests require a live PostgreSQL instance (12+, container uses 16) on `localhost:5432` with trust auth: + +```bash +docker compose up -d # start Postgres for the test suite +docker compose down # stop it (add -v to wipe the data volume) +``` + +```bash +make test # py.test -x -svv --cov-report term-missing --cov migra tests +make lint # flake8 . (config in .flake8: ignores E501, W503) +make fmt # isort . && black . +make clean # remove .pyc, .cache, build artifacts +``` + +Run a single test: `pytest tests/test_command_promote.py::test_name -svv` + +CI (CircleCI, `.circleci/config.yml`) runs `make lint` then `make test` against Postgres 14 on every +branch, and publishes to PyPI + tags a release from `master`. + +## Architecture + +### Core diff engine (upstream-derived, keep changes surgical) + +- `migra/migra.py` — `Migration` class: wraps two `DBInspector`/connection-string/schema-dump sources + (`schemainspect.get_inspector`), and `add_all_changes()` sequences every category of change + (schemas, extensions, enums, sequences, triggers, RLS, constraints, indexes, tables, domains, + privileges, comments, ...) in an order that respects PostgreSQL dependency rules — drops before + creates, dependents before dependencies. **This ordering is load-bearing**; don't reshuffle it + without understanding why each step is where it is. +- `migra/changes.py` — `Changes` class plus the `get_*_changes`/`statements_from_differences` + functions that walk `schemainspect` inspector objects and emit SQL for each object type. + `util.differences()` is the generic added/removed/modified/unmodified diff primitive everything + else is built on. +- `migra/statements.py` — `Statements` (a `list` subclass) accumulates generated SQL and refuses to + render (`.sql`) if any statement contains DROP while `safe=True` (`UnsafeMigrationException`) — + this is the `--unsafe` flag's underlying gate. +- `migra/db_inspector.py` — lower-level live-DB comparison helpers (table sizes, per-object-type + compare functions) used mainly by the AI drift/advise features rather than core diffing. + +### CLI (`migra/command.py`, the biggest file — ~57KB) + +- `parse_args()` defines every flag; `run()`/`_run_inner()` is the dispatch body; `do_command()` is + the `pyproject.toml` entry point (`migra = 'migra:do_command'`). +- Handles multiple input modes: two DB URLs, `--from-file` (pg_dump output), `--from-migrations-dir` + (applies migration files to an ephemeral DB — supports Supabase/Flyway/numeric naming, see + `discover_migration_files`/`apply_migrations`). +- Post-processes raw diff output for safety/UX: destructive-statement detection + (`_check_for_destructive`), column-rename detection (`detect_column_renames`, vs. a naive + drop+add), risk classification for `--output json` (`classify_sql_statement`), credential + redaction in error output (`redact_credentials`). +- `--status`/`--history`/`--promote`/`--record-rollback` etc. talk to `migra/history.py`. + +### Migration state tracking (`migra/history.py`) + +A `migradiff_history` table (`HISTORY_TABLE`) recorded into the *target* database, keyed by a +SHA-256 hash of normalized SQL (`compute_migration_hash`). This is the basis for `--promote` +(multi-environment promotion) and `--record-rollback`. **Important semantic**: a history row means +a migration was generated/reviewed for that target, not that migra itself executed it — migra never +applies SQL to a live database. Preserve this distinction in any related code or docs. + +### AI features (optional extra: `pip install migradiff[ai]`, needs `ANTHROPIC_API_KEY`) + +- `migra/ai_explain.py` — one file holding four related-but-distinct features, each with its own + prompt builder + client class: `AIExplainer` (`--explain`), `AIRollback` (`--rollback`, includes a + fully deterministic non-AI rollback path in `generate_deterministic_rollback` plus AI-assisted + reconstruction using `extract_schema_context`), `AIAdvisor` (`--advise`, performance/lock risk), + `AIGenerator` (`--generate`, plain-English → grounded SQL via `extract_relevant_schema`). Also + owns API key config (`~/.migradiff` style config dir, `--setup-ai`, `resolve_api_key`, + `redact_api_key`). +- `migra/ai_drift.py` — `DriftExplainer` (`--explain-drift`): compares two *live* databases (not a + generated migration) and explains the difference, categorizing BREAKING/WARNING/INFO. +- All AI calls are opt-in per the README's privacy framing ("bring your own API key — no data sent + to MigraDiff servers"); don't add code paths that phone home elsewhere. + +### Tests + +- `tests/` mirrors the module layout (`test_ai_explain.py`, `test_command_promote.py`, etc.) and + needs the live Postgres from `docker compose up -d`. +- `tests/conftest.py` monkey-patches `sqlbag.createdrop` create/drop/exists functions to work + around a Postgres-specific issue in the upstream `sqlbag` dependency — don't remove this shim + without checking `sqlbag`'s current behavior. +- `tests/characterization/` is a separate CLI characterization suite (`capture.py`, `scenarios.py`) + that snapshots CLI behavior for the AI-flag surface, independent of the DB-driven unit tests. diff --git a/decisions.md b/decisions.md new file mode 100644 index 0000000..1730457 --- /dev/null +++ b/decisions.md @@ -0,0 +1,14 @@ +# Decisions + +Log of notable technical/product decisions for MigraDiff, newest first. One entry per decision: +what was decided, why, and any alternatives considered. + + diff --git a/status.md b/status.md new file mode 100644 index 0000000..e9091c3 --- /dev/null +++ b/status.md @@ -0,0 +1,23 @@ +# Status + +Snapshot of current work-in-progress on this branch. Update as work progresses; this is not a +changelog (see CHANGELOG.md for that). + +## Current branch: `new-feature-6-18-2026` + +**In progress / uncommitted:** +- Modified: `CHANGELOG.md`, `README.md`, `migra/command.py` — migration state + tracking (`--status`/`--history`/`--record-history`/`--promote`/`--record-rollback`) +- New: `LICENSING.md` (unrelated), `migra/history.py`, `tests/test_command_promote.py`, + `tests/test_command_rollback_tracking.py`, `tests/test_history.py` +- Also uncommitted (not part of this task): `PROJECT_PLAN.md`, `PROJECT_PLAN2.md` + updated 2026-08-08 to reflect actual shipped state (they were stale, describing + v1.5.1 as current when v1.7.2 was already released) + +**Verified 2026-08-08:** all 32 new tests pass against a real Postgres instance; +full suite is 342 passed / 2 skipped; flake8 and black are clean; CLI smoke-tested +end-to-end (`--status`, `--record-history`, `--promote`, `--record-rollback` +round-tripped correctly). Feature is code-complete. + +**Next steps:** merge this branch and release as v1.8.0 (see PROJECT_PLAN.md +"Next Steps" for the follow-on backlog: `--apply`, native `--fail-on-destructive`). From 704c3d4a43249b29194337943d41e6cf10780eb8 Mon Sep 17 00:00:00 2001 From: MigraDiff Agent Date: Sat, 8 Aug 2026 13:26:39 -0700 Subject: [PATCH 05/11] fix: run CI on main in addition to master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The GitHub Actions CI workflow only triggered on push/PR to `master`, but `main` is the repo's actual default branch (confirmed via `gh repo view --json defaultBranchRef`). PRs targeting `main` — like the one that just merged the migration state tracking feature — got no CI run at all. Adds `main` to the trigger branches alongside `master` and documents the finding in CI_NOTES.md, including that `main` still lacks the branch protection rule that `master` has (a separate repo-settings change, not covered here). Co-Authored-By: Claude Sonnet 5 --- .github/CI_NOTES.md | 14 +++++++++++++- .github/workflows/ci.yml | 4 ++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/CI_NOTES.md b/.github/CI_NOTES.md index c3b5ed2..d28a8fa 100644 --- a/.github/CI_NOTES.md +++ b/.github/CI_NOTES.md @@ -30,7 +30,7 @@ view definitions. ## Workflow -- **CI** runs on: push to `master`, PRs to `master` +- **CI** runs on: push to `main` or `master`, PRs targeting `main` or `master` - Lint (flake8, black) - Test matrix (4 Python versions × 3 Postgres versions = 12 jobs) - Coverage report @@ -39,3 +39,15 @@ view definitions. - Build package - Publish to PyPI - Create GitHub release + +### `main` vs `master` branch protection (2026-08-08) + +`main` is the actual GitHub default branch (`gh repo view --json defaultBranchRef` +confirms this), but only `master` has branch protection configured — 13 required +status checks (`Lint` + the 12-job test matrix), `strict` status checks, no force +pushes/deletions. `main` has **no protection at all** +(`gh api repos/.../branches/main/protection` returns 404). This means PRs merged +into `main` today get no required CI gate. The workflow trigger was fixed here +to run on both branches, but the branch protection *rule* itself still needs to +be applied to `main` — that's a repo-settings change, not a workflow file change, +and hasn't been done yet. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 206e2d3..b27c656 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [master] + branches: [main, master] pull_request: - branches: [master] + branches: [main, master] env: PGUSER: postgres From 27d16b1c2d18327b615bb5b5a12043a95c7d3493 Mon Sep 17 00:00:00 2001 From: MigraDiff Agent Date: Sat, 8 Aug 2026 13:50:34 -0700 Subject: [PATCH 06/11] fix: wrap raw SQL in text() in test_history.py for SQLAlchemy 2.x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three s.execute() calls passed raw strings directly instead of wrapping them in sqlalchemy.text(). SQLAlchemy 1.4 (installed locally) only warns about this; SQLAlchemy 2.x — which CI resolves to since pyproject.toml pins sqlalchemy to "*" — raises ArgumentError and fails the test. Verified against SQLAlchemy 2.0.51 in an isolated venv: these 3 tests plus the other 2 new test files (32 tests total) and the full 342-test suite all pass. Co-Authored-By: Claude Sonnet 5 --- tests/test_history.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tests/test_history.py b/tests/test_history.py index e18c5af..1942082 100644 --- a/tests/test_history.py +++ b/tests/test_history.py @@ -2,6 +2,7 @@ import io +from sqlalchemy import text from sqlbag import S, temporary_database @@ -58,16 +59,20 @@ def test_table_creation_idempotent(self): # Verify table exists by querying it rows = s.execute( - "SELECT table_name FROM information_schema.tables " - "WHERE table_name = 'migradiff_history'" + text( + "SELECT table_name FROM information_schema.tables " + "WHERE table_name = 'migradiff_history'" + ) ).fetchall() assert len(rows) == 1 # Verify columns cols = s.execute( - "SELECT column_name, data_type FROM information_schema.columns " - "WHERE table_name = 'migradiff_history' " - "ORDER BY ordinal_position" + text( + "SELECT column_name, data_type FROM information_schema.columns " + "WHERE table_name = 'migradiff_history' " + "ORDER BY ordinal_position" + ) ).fetchall() col_names = [c[0] for c in cols] assert "id" in col_names @@ -95,7 +100,7 @@ def test_default_rollback_status(self): forward_sql="SELECT 1;", ) row = s.execute( - "SELECT rollback_status FROM migradiff_history" + text("SELECT rollback_status FROM migradiff_history") ).fetchone() assert row[0] == "not_attempted" From a6dc2ef1c0429657817c605f922619a0e4d867af Mon Sep 17 00:00:00 2001 From: MigraDiff Agent Date: Sat, 8 Aug 2026 14:31:11 -0700 Subject: [PATCH 07/11] feat: add --apply flag to execute migrations directly Closes the gap where MigraDiff only ever generated SQL and left users to pipe it to psql themselves, with no reliable way to know whether migradiff_history reflected a migration that actually ran. --apply executes the generated migration against dburl_from (the database being migrated, matching the CLI's own help text and the README's psql dburl_from < migration.sql convention) inside a single transaction -- any failing statement rolls back everything, and nothing is recorded, so migradiff_history only ever reflects migrations that were confirmed applied. On success it auto-records history against dburl_from without needing a separate --record-history call. Rejected up front with --from-file (dburl_from would be an ephemeral throwaway database) and --promote (the chain's from/to direction isn't yet reconciled with which database --apply should execute against -- tracked as a follow-up in PROJECT_PLAN.md rather than guessed at here). Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 22 +++- CLAUDE.md | 19 ++- PROJECT_PLAN.md | 36 +++--- PROJECT_PLAN2.md | 36 +++--- README.md | 44 ++++++- migra/command.py | 126 +++++++++++++++++- status.md | 35 +++-- tests/test_command_apply.py | 246 ++++++++++++++++++++++++++++++++++++ 8 files changed, 497 insertions(+), 67 deletions(-) create mode 100644 tests/test_command_apply.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ba9cd9..c3f6e3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,12 +27,30 @@ `test_command_rollback_tracking.py` — all using real Postgres databases (no mocks), covering idempotent table creation, hash normalization, conflict detection, empty-diff hops, safe mode, and rollback tracking. +- **Migration execution** (`--apply`): + - Executes the generated migration against `dburl_from` in a single + transaction (all-or-nothing — a failed statement rolls back everything + that ran before it) instead of only printing the SQL + - On success, automatically records the migration in `dburl_from`'s + `migradiff_history` table — no need to also pass `--record-history` + - On failure, nothing is recorded (since nothing was actually applied) + and the command exits with code 4 + - Not supported with `--from-file` (no live database to apply to) or + `--promote` (not implemented yet) — both are rejected with a clear + error before anything runs + - New helper `_apply_migration()` in `migra/command.py`; JSON output + (`--output json`) gets a new `"apply"` object with the outcome + - New tests: `test_command_apply.py` (11 tests, real Postgres, including + a direct atomicity test that forces a mid-migration failure) ### Notes - This is the foundational layer for the upcoming "Control Plane" feature set. -- Known limitation: `migradiff_history` records migrations as *generated*, - not *applied* — see the README "Migration State Tracking" section for details. +- Resolved: `migradiff_history` used to only record migrations as + *generated*, not *applied* — `--apply` closes that gap by executing the + migration itself and only recording history on confirmed success. Plain + `--record-history` (without `--apply`) still only means "generated", not + "applied" — see the README "Migration State Tracking" section. - All new features work without AI extras (`anthropic` optional dependency). ## [1.7.2] - 2026-06-08 diff --git a/CLAUDE.md b/CLAUDE.md index fe650af..a5c9697 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -64,15 +64,22 @@ branch, and publishes to PyPI + tags a release from `master`. (`_check_for_destructive`), column-rename detection (`detect_column_renames`, vs. a naive drop+add), risk classification for `--output json` (`classify_sql_statement`), credential redaction in error output (`redact_credentials`). -- `--status`/`--history`/`--promote`/`--record-rollback` etc. talk to `migra/history.py`. +- `--status`/`--history`/`--promote`/`--record-rollback`/`--apply` etc. talk to `migra/history.py`. ### Migration state tracking (`migra/history.py`) -A `migradiff_history` table (`HISTORY_TABLE`) recorded into the *target* database, keyed by a -SHA-256 hash of normalized SQL (`compute_migration_hash`). This is the basis for `--promote` -(multi-environment promotion) and `--record-rollback`. **Important semantic**: a history row means -a migration was generated/reviewed for that target, not that migra itself executed it — migra never -applies SQL to a live database. Preserve this distinction in any related code or docs. +A `migradiff_history` table (`HISTORY_TABLE`), keyed by a SHA-256 hash of normalized SQL +(`compute_migration_hash`). This is the basis for `--promote` (multi-environment promotion), +`--record-rollback`, and `--apply`. **Important semantic, easy to get backwards**: a plain +`--record-history` (without `--apply`) writes into `dburl_target`'s history table and only means +"this migration was generated/reviewed for that target" — not that it was executed anywhere. +`--apply` (`_apply_migration()` in `command.py`) is the one path that actually executes SQL: it runs +the migration against `dburl_from` (the database being migrated — see the CLI's own help text and +the README's `psql dburl_from < migration.sql` convention) inside a single transaction, and only on +confirmed success does it record history — into `dburl_from`'s table, not `dburl_target`'s. A failed +`--apply` rolls back everything and records nothing. `--apply` is rejected up front (before any +dispatch) when combined with `--from-file` (dburl_from would be an ephemeral throwaway database) or +`--promote` (the chain's from/to direction is not yet reconciled with `--apply`'s execution target). ### AI features (optional extra: `pip install migradiff[ai]`, needs `ANTHROPIC_API_KEY`) diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md index eb695f6..8ef249d 100644 --- a/PROJECT_PLAN.md +++ b/PROJECT_PLAN.md @@ -100,24 +100,27 @@ done. See "Completed Features" above for what actually shipped: (fr, de, ja, zh, hi, he — 6 languages, merged as a docs-only PR, no dedicated version bump). -### Next: release migration state tracking as v1.8.0 +### Shipped: migration state tracking and `--apply` -`--status`/`--history`/`--promote`/`--record-rollback` (see "Completed -Features") is code-complete and fully tested but still sitting on branch -`new-feature-6-18-2026`. Landing this — merge, version bump, tag, PyPI -release — is the immediate next step, ahead of any new feature work below. +`--status`/`--history`/`--promote`/`--record-rollback` merged into `main` +2026-08-08 (PRs #8–#10; no version bump/tag yet — still on v1.7.2). +`--apply` (execute the migration against `dburl_from` instead of only +printing it, with automatic history recording on confirmed success — see +README's "Applying Migrations" section) shipped shortly after on top of +that. Neither has been cut into a tagged release yet; `pyproject.toml` +still reads 1.7.2. Bumping to v1.8.0 and publishing is still outstanding. ### Planned (Backlog) | Feature | Effort | Value | Notes | |---------|--------|-------|-------| -| `--apply` | Medium | High | Execute generated SQL directly instead of requiring the user to pipe to `psql`. Also needed so `migradiff_history` can record true "applied" state instead of just "generated/reviewed" — see the known limitation in README's Migration State Tracking section. | | Native `--fail-on-destructive` flag | Low | High | Currently this behavior only exists inside the GitHub Action's `action-entrypoint.sh`. CircleCI/GitLab/pre-commit users have no equivalent without wrapping the CLI themselves. | +| `--apply` + `--promote` | Medium | Medium | `--apply` currently refuses to run when `--promote` is also given — `--promote`'s from/to direction needs to be reconciled with which database `--apply` should actually execute against before this is safe to wire up. | | `--document` | Medium-High | High | Schema documentation generation | | Multi-schema hardening | Medium | High | `Migration.__init__` (`migra/migra.py`) still hard-rejects `schema` + `exclude_schema` together; cross-schema FK/dependency ordering in `add_all_changes()` hasn't had a dedicated multi-tenant test pass | | pgvector support | Low | Medium | Modern Postgres vector types — unconfirmed whether `schemainspect` already round-trips them | | `--suggest-indexes` | Medium | Medium | AI recommends useful indexes; can reuse `AIAdvisor`'s existing table-stats extraction | -| `--dry-run` | Low | Medium | Only meaningful once `--apply` exists | +| `--dry-run` | Low | Medium | Preview what `--apply` would do without executing it | --- @@ -317,21 +320,18 @@ without a dedicated version bump — not tied to a CHANGELOG entry. ## Next Steps -1. **Land migration state tracking as v1.8.0** - - Merge `new-feature-6-18-2026` — code-complete, 342/342 tests passing, - flake8/black clean, and CLI-smoke-tested end-to-end as of 2026-08-08 - - Bump version, tag release, publish to PyPI +1. **Cut v1.8.0** — migration state tracking and `--apply` are both merged + to `main` and fully tested, but `pyproject.toml` is still at 1.7.2 and + nothing has been tagged/published to PyPI yet. -2. **`--apply` flag** - - Closes the "generated vs. applied" gap called out in README's - Migration State Tracking known limitation - - Should auto-call the equivalent of `--record-history` on success - -3. **Native `--fail-on-destructive` CLI flag** +2. **Native `--fail-on-destructive` CLI flag** - Promote the GitHub Action's destructive-detection behavior into `command.py` itself so non-Action CI users (CircleCI, GitLab, plain scripts) get it too +3. **Reconcile `--promote`'s direction with `--apply`** before wiring the + two together — see the Planned/Backlog table above. + 4. **Post v1.8.0:** Enterprise tier planning - Design licensing system - Plan hosted features @@ -339,6 +339,6 @@ without a dedicated version bump — not tied to a CHANGELOG entry. --- -**Document version:** Updated 2026-08-08 (reconciled with actual repo/code state) +**Document version:** Updated 2026-08-08 (reconciled with actual repo/code state, incl. `--apply`) **Last updated by:** Claude (with Leo) **Repository:** https://github.com/postgresql-tools/migra diff --git a/PROJECT_PLAN2.md b/PROJECT_PLAN2.md index 3fccbcc..e37a31a 100644 --- a/PROJECT_PLAN2.md +++ b/PROJECT_PLAN2.md @@ -103,24 +103,27 @@ Hindi/Hebrew for the language set — that changed during implementation. The 6 languages that actually shipped are fr, de, ja, zh, hi, he (no Spanish); see `PROJECT_PLAN.md` for the corrected rationale. -### Next: release migration state tracking as v1.8.0 +### Shipped: migration state tracking and `--apply` -`--status`/`--history`/`--promote`/`--record-rollback` (see "Completed -Features") is code-complete and fully tested but still sitting on branch -`new-feature-6-18-2026`. Landing this — merge, version bump, tag, PyPI -release — is the immediate next step, ahead of any new feature work below. +`--status`/`--history`/`--promote`/`--record-rollback` merged into `main` +2026-08-08 (PRs #8–#10; no version bump/tag yet — still on v1.7.2). +`--apply` (execute the migration against `dburl_from` instead of only +printing it, with automatic history recording on confirmed success — see +README's "Applying Migrations" section) shipped shortly after on top of +that. Neither has been cut into a tagged release yet; `pyproject.toml` +still reads 1.7.2. Bumping to v1.8.0 and publishing is still outstanding. ### Planned (Backlog) | Feature | Effort | Value | Notes | |---------|--------|-------|-------| -| `--apply` | Medium | High | Execute generated SQL directly instead of requiring the user to pipe to `psql`. Also needed so `migradiff_history` can record true "applied" state instead of just "generated/reviewed" — see the known limitation in README's Migration State Tracking section. | | Native `--fail-on-destructive` flag | Low | High | Currently this behavior only exists inside the GitHub Action's `action-entrypoint.sh`. CircleCI/GitLab/pre-commit users have no equivalent without wrapping the CLI themselves. | +| `--apply` + `--promote` | Medium | Medium | `--apply` currently refuses to run when `--promote` is also given — `--promote`'s from/to direction needs to be reconciled with which database `--apply` should actually execute against before this is safe to wire up. | | `--document` | Medium-High | High | Schema documentation generation | | Multi-schema hardening | Medium | High | `Migration.__init__` (`migra/migra.py`) still hard-rejects `schema` + `exclude_schema` together; cross-schema FK/dependency ordering in `add_all_changes()` hasn't had a dedicated multi-tenant test pass | | pgvector support | Low | Medium | Modern Postgres vector types — unconfirmed whether `schemainspect` already round-trips them | | `--suggest-indexes` | Medium | Medium | AI recommends useful indexes; can reuse `AIAdvisor`'s existing table-stats extraction | -| `--dry-run` | Low | Medium | Only meaningful once `--apply` exists | +| `--dry-run` | Low | Medium | Preview what `--apply` would do without executing it | --- @@ -320,21 +323,18 @@ without a dedicated version bump — not tied to a CHANGELOG entry. ## Next Steps -1. **Land migration state tracking as v1.8.0** - - Merge `new-feature-6-18-2026` — code-complete, 342/342 tests passing, - flake8/black clean, and CLI-smoke-tested end-to-end as of 2026-08-08 - - Bump version, tag release, publish to PyPI +1. **Cut v1.8.0** — migration state tracking and `--apply` are both merged + to `main` and fully tested, but `pyproject.toml` is still at 1.7.2 and + nothing has been tagged/published to PyPI yet. -2. **`--apply` flag** - - Closes the "generated vs. applied" gap called out in README's - Migration State Tracking known limitation - - Should auto-call the equivalent of `--record-history` on success - -3. **Native `--fail-on-destructive` CLI flag** +2. **Native `--fail-on-destructive` CLI flag** - Promote the GitHub Action's destructive-detection behavior into `command.py` itself so non-Action CI users (CircleCI, GitLab, plain scripts) get it too +3. **Reconcile `--promote`'s direction with `--apply`** before wiring the + two together — see the Planned/Backlog table above. + 4. **Post v1.8.0:** Enterprise tier planning - Design licensing system - Plan hosted features @@ -342,6 +342,6 @@ without a dedicated version bump — not tied to a CHANGELOG entry. --- -**Document version:** Updated 2026-08-08 (reconciled with actual repo/code state) +**Document version:** Updated 2026-08-08 (reconciled with actual repo/code state, incl. `--apply`) **Last updated by:** Claude (with Leo) **Repository:** https://github.com/postgresql-tools/migra diff --git a/README.md b/README.md index e02b320..f867688 100644 --- a/README.md +++ b/README.md @@ -278,13 +278,13 @@ rollback tracking. ### ⚠ Known Limitation -`migradiff_history` records that a migration was **generated/reviewed** -for a target database, not that the SQL was necessarily *executed* by -MigraDiff itself (MigraDiff does not apply migrations to live databases -today — it only diffs and generates SQL). Recording true "applied" state -requires your deployment pipeline to also call -`migra --record-history` (or the future `--apply` flag, not yet -implemented) after running the generated SQL. +Without `--apply` (below), `migradiff_history` only records that a +migration was **generated/reviewed** for a target database, not that the +SQL was necessarily *executed*. If you generate SQL and pipe it to `psql` +yourself, calling `migra --record-history` only tells MigraDiff "this +migration was proposed" — it has no way to know whether your `psql` step +actually succeeded. Use `--apply` when you want MigraDiff itself to run +the migration and only record history on confirmed success. Be explicit about this in your pipeline so you don't assume false guarantees about whether a migration has actually been applied. @@ -329,6 +329,36 @@ Use `--env-label` to tag the entry: migra --record-history --env-label staging postgres://db_a postgres://db_b ``` +### Applying Migrations (`--apply`) + +`--apply` executes the generated migration directly against `dburl_from` +(the first positional argument — "the database you want to migrate", same +database the README's basic usage example pipes to `psql`) instead of only +printing it: + +```bash +migra --apply postgres://db_production postgres://db_branch +``` + +On success, MigraDiff automatically records the migration in +`dburl_from`'s `migradiff_history` table — you don't need to also pass +`--record-history`. If any statement fails, the whole migration is rolled +back as a single transaction (nothing is partially applied) and **nothing +is recorded**, since it wasn't actually applied. The command exits non-zero +(exit code 4) so pipelines can detect the failure. + +```bash +migra --apply --env-label prod postgres://db_production postgres://db_branch +``` + +`--apply` respects the same safety gates as everything else: destructive +statements are blocked unless `--force-destructive` (or `--unsafe`) is +given, and the block happens *before* anything is executed. + +`--apply` is not supported with `--from-file` (there's no live database to +apply to — the schema files get loaded into temporary throwaway databases) +or with `--promote` (not implemented yet). + ### Multi-Environment Promotion (`--promote`) `--promote` generates migrations along a chain of environments, with diff --git a/migra/command.py b/migra/command.py index 7bee21a..0751d0c 100644 --- a/migra/command.py +++ b/migra/command.py @@ -553,6 +553,16 @@ def parse_args(args): help="Generate a PostgreSQL migration from a plain English description." " Optionally provide the description as an argument.", ) + parser.add_argument( + "--apply", + dest="apply", + action="store_true", + default=False, + help="Execute the generated migration against dburl_from (the database" + " being migrated), instead of only printing it. On success, automatically" + " records the migration in dburl_from's migradiff_history table — no need" + " to also pass --record-history. Not supported with --from-file or --promote.", + ) parser.add_argument( "--record-history", dest="record_history", @@ -641,12 +651,77 @@ def _resolve_env_alias(key): ) +def _apply_migration(dburl_from, statements, sql_output, rollback_sql, env_label): + """Execute `statements` against dburl_from in a single transaction, then + record the migration in dburl_from's migradiff_history table. + + Returns a dict describing the outcome; never raises. Execution and + history-recording are separate try/excepts, since a failure to record + history after a successful apply is a (loud) warning, not an apply + failure — the schema change already happened. + """ + from sqlbag import S as _S + + result = { + "applied": False, + "target": redact_credentials(dburl_from), + "statement_count": len(statements), + "error": None, + "history_recorded": False, + "history_error": None, + } + + try: + with _S(dburl_from) as s: + for stmt in statements: + s.execute(text(stmt)) + except Exception as e: + result["error"] = str(e) + return result + + result["applied"] = True + + from .history import compute_migration_hash, ensure_history_table, record_applied + + try: + with _S(dburl_from) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash=compute_migration_hash(sql_output), + forward_sql=sql_output, + rollback_sql=rollback_sql, + environment_label=env_label, + ) + result["history_recorded"] = True + except Exception as e: + result["history_error"] = str(e) + + return result + + def run(args, out=None, err=None): if not out: out = sys.stdout # pragma: no cover if not err: err = sys.stderr # pragma: no cover + if args.apply and args.from_file: + print( + "MigraDiff: --apply is not supported with --from-file" + " (there is no live database to apply to — dburl_from is a" + " temporary database loaded from the dump file).", + file=err, + ) + return 1 + + if args.apply and args.promote: + print( + "MigraDiff: --apply is not supported with --promote yet.", + file=err, + ) + return 1 + # --status / --history: display migration history if args.status is not None or args.history is not None: from .history import ensure_history_table, get_history @@ -1445,9 +1520,18 @@ def _run_inner(args, out=None, err=None): ) sql_output = None + apply_result = None try: if statements: sql_output = modified_statements.sql + if args.apply: + apply_result = _apply_migration( + args.dburl_from, + modified_statements, + sql_output, + rollback_result["text"] if rollback_result else None, + args.env_label, + ) if args.output == "json": json_out = format_json_output( statements, @@ -1485,6 +1569,12 @@ def _run_inner(args, out=None, err=None): "generated_at": advisory_result.get("generated_at", ""), } json_out = json_mod.dumps(data, indent=2) + if apply_result is not None: + import json as json_mod + + data = json_mod.loads(json_out) + data["apply"] = apply_result + json_out = json_mod.dumps(data, indent=2) print(json_out, file=out) elif args.force_utf8: print(modified_statements.sql.encode("utf8"), file=out) @@ -1501,6 +1591,32 @@ def _run_inner(args, out=None, err=None): print(file=out) print("--- Performance Advisory ---", file=out) print(advisory_result["text"], file=out) + if apply_result is not None and args.output != "json": + print(file=out) + if apply_result["applied"]: + print( + "-- Applied {} statement(s) to {}".format( + apply_result["statement_count"], + apply_result["target"], + ), + file=out, + ) + if not apply_result["history_recorded"]: + print( + "-- WARNING: applied successfully, but failed to" + " record migration history: {}".format( + apply_result["history_error"] + ), + file=err, + ) + else: + print( + "-- ERROR: --apply failed, no changes were committed" + " to {}: {}".format( + apply_result["target"], apply_result["error"] + ), + file=err, + ) elif args.output == "json": json_out = format_json_output( statements, @@ -1562,8 +1678,11 @@ def _run_inner(args, out=None, err=None): args._generated_sql = sql_output if statements else None args._generated_rollback = rollback_result["text"] if rollback_result else None - # --record-history: record the generated migration - if args.record_history and statements and args.dburl_target: + # --record-history: record the generated migration. Skipped when --apply + # already ran, since _apply_migration() already recorded it against + # dburl_from (the database that was actually changed) — recording it + # again here against dburl_target would be a second, misleading entry. + if args.record_history and not args.apply and statements and args.dburl_target: from .history import ( compute_migration_hash, ensure_history_table, @@ -1589,6 +1708,9 @@ def _run_inner(args, out=None, err=None): file=err, ) + if apply_result is not None and not apply_result["applied"]: + return 4 + if not statements: return 0 diff --git a/status.md b/status.md index e9091c3..506f1ff 100644 --- a/status.md +++ b/status.md @@ -3,21 +3,28 @@ Snapshot of current work-in-progress on this branch. Update as work progresses; this is not a changelog (see CHANGELOG.md for that). -## Current branch: `new-feature-6-18-2026` +## Current branch: `feature/apply-flag` (off `main` at `46c2447`) + +Migration state tracking (`--status`/`--history`/`--record-history`/`--promote`/ +`--record-rollback`) and the CI-trigger/SQLAlchemy-2.x fixes are already merged to `main` +(PRs #8, #9, #10) — see git log, not this file, for that history. **In progress / uncommitted:** -- Modified: `CHANGELOG.md`, `README.md`, `migra/command.py` — migration state - tracking (`--status`/`--history`/`--record-history`/`--promote`/`--record-rollback`) -- New: `LICENSING.md` (unrelated), `migra/history.py`, `tests/test_command_promote.py`, - `tests/test_command_rollback_tracking.py`, `tests/test_history.py` -- Also uncommitted (not part of this task): `PROJECT_PLAN.md`, `PROJECT_PLAN2.md` - updated 2026-08-08 to reflect actual shipped state (they were stale, describing - v1.5.1 as current when v1.7.2 was already released) +- Modified: `CHANGELOG.md`, `README.md`, `migra/command.py`, `CLAUDE.md`, + `PROJECT_PLAN.md`, `PROJECT_PLAN2.md` — new `--apply` flag +- New: `tests/test_command_apply.py` + +**What `--apply` does:** executes the generated migration against `dburl_from` in a single +transaction instead of only printing it; on success, automatically records it in `dburl_from`'s +`migradiff_history` table (no need to also pass `--record-history`); on failure, rolls back +everything and records nothing, exits 4. Rejected up front when combined with `--from-file` or +`--promote` — see CLAUDE.md's "Migration state tracking" section for the from/target direction +reasoning. -**Verified 2026-08-08:** all 32 new tests pass against a real Postgres instance; -full suite is 342 passed / 2 skipped; flake8 and black are clean; CLI smoke-tested -end-to-end (`--status`, `--record-history`, `--promote`, `--record-rollback` -round-tripped correctly). Feature is code-complete. +**Verified 2026-08-08:** 11 new tests pass (including a direct atomicity test against +`_apply_migration()` that forces a mid-migration failure); full suite is 353 passed / 2 skipped, +no regressions; flake8 and black clean; also verified against SQLAlchemy 2.0.51 in an isolated +venv (no other raw-SQL issues found). -**Next steps:** merge this branch and release as v1.8.0 (see PROJECT_PLAN.md -"Next Steps" for the follow-on backlog: `--apply`, native `--fail-on-destructive`). +**Next steps:** commit, push, open PR. Known follow-up (not done here): reconcile `--promote`'s +from/to direction with `--apply` before wiring the two together (see PROJECT_PLAN.md backlog). diff --git a/tests/test_command_apply.py b/tests/test_command_apply.py new file mode 100644 index 0000000..fb07802 --- /dev/null +++ b/tests/test_command_apply.py @@ -0,0 +1,246 @@ +from __future__ import unicode_literals + +import io + +from sqlalchemy import text +from sqlbag import S, temporary_database + + +def outs(): + return io.StringIO(), io.StringIO() + + +def _create_table(db_url, table_name, columns="id int primary key"): + with S(db_url) as s: + s.execute(text("CREATE TABLE {} ({});".format(table_name, columns))) + + +def _table_exists(db_url, table_name): + with S(db_url) as s: + row = s.execute( + text( + "SELECT 1 FROM information_schema.tables" + " WHERE table_name = :table_name" + ), + {"table_name": table_name}, + ).fetchone() + return row is not None + + +class TestApplyHappyPath: + def test_apply_creates_table_in_dburl_from(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + _create_table(target_db, "widgets") + assert not _table_exists(from_db, "widgets") + + from migra.command import parse_args, run + + args = parse_args(["--apply", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + + assert status == 2 + assert "-- Applied " in out.getvalue() + assert "statement(s)" in out.getvalue() + assert _table_exists(from_db, "widgets") + + def test_apply_records_history_against_dburl_from(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + _create_table(target_db, "widgets") + + from migra.command import parse_args, run + + args = parse_args(["--apply", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 2 + + from migra.history import ensure_history_table, get_history + + with S(from_db) as s: + entries = get_history(s) + assert len(entries) == 1 + assert entries[0]["rollback_status"] == "not_attempted" + assert "CREATE TABLE" in entries[0]["forward_sql"].upper() + + # A plain --record-history (without --apply) should NOT also have + # written a second entry into dburl_target's history table -- in + # fact its history table should never even have been created. + with S(target_db) as s: + ensure_history_table(s) + target_entries = get_history(s) + assert target_entries == [] + + def test_apply_with_env_label(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + _create_table(target_db, "widgets") + + from migra.command import parse_args, run + + args = parse_args(["--apply", "--env-label", "prod", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 2 + + from migra.history import get_history + + with S(from_db) as s: + entries = get_history(s) + assert entries[0]["environment_label"] == "prod" + + def test_apply_json_output(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + _create_table(target_db, "widgets") + + from migra.command import parse_args, run + import json + + args = parse_args(["--apply", "--output", "json", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 2 + + data = json.loads(out.getvalue()) + assert data["apply"]["applied"] is True + assert data["apply"]["statement_count"] == len(data["statements"]) + assert data["apply"]["history_recorded"] is True + assert _table_exists(from_db, "widgets") + + +class TestApplyNoChanges: + def test_no_diff_is_a_noop(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + from migra.command import parse_args, run + + args = parse_args(["--apply", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + + assert status == 0 + assert "Applied" not in out.getvalue() + # Nothing to apply, so the history table should never even be + # created. + assert not _table_exists(from_db, "migradiff_history") + + +class TestApplySafety: + def test_destructive_change_blocked_without_force(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + _create_table(from_db, "widgets") + # target_db has no "widgets" table -> dropping it would be + # destructive. + + from migra.command import parse_args, run + + args = parse_args(["--apply", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + + assert status == 1 + assert "Destructive operations detected" in err.getvalue() + # The table must still exist -- --apply must never execute + # anything once safe-mode has already rejected the migration. + assert _table_exists(from_db, "widgets") + + def test_destructive_change_applied_with_force_destructive(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + _create_table(from_db, "widgets") + + from migra.command import parse_args, run + + args = parse_args(["--apply", "--force-destructive", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + + assert status == 2 + assert not _table_exists(from_db, "widgets") + + +class TestApplyIncompatibleFlags: + def test_rejected_with_from_file(self): + from migra.command import parse_args, run + + args = parse_args(["--apply", "--from-file", "a.sql", "b.sql"]) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "--apply is not supported with --from-file" in err.getvalue() + + def test_rejected_with_promote(self): + from migra.command import parse_args, run + + args = parse_args(["--apply", "--promote", "dev:staging"]) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "--apply is not supported with --promote" in err.getvalue() + + +class TestApplyMigrationHelper: + """Unit tests against _apply_migration() directly, so we can deterministically + force a mid-migration failure and assert atomicity -- something that's hard + to trigger reliably by going through the full diff-generation pipeline.""" + + def test_failed_statement_rolls_back_and_records_nothing(self): + with temporary_database(host="localhost") as db_url: + from migra.command import _apply_migration + + result = _apply_migration( + db_url, + [ + "CREATE TABLE ok_table (id int primary key);", + "THIS IS NOT VALID SQL;", + ], + sql_output="CREATE TABLE ok_table (id int primary key);\n\n" + "THIS IS NOT VALID SQL;\n\n", + rollback_sql=None, + env_label=None, + ) + + assert result["applied"] is False + assert result["error"] is not None + assert result["history_recorded"] is False + + # The whole transaction must have rolled back -- ok_table must + # NOT exist despite being a valid statement that ran first. + assert not _table_exists(db_url, "ok_table") + assert not _table_exists(db_url, "migradiff_history") + + def test_successful_apply_records_history(self): + with temporary_database(host="localhost") as db_url: + from migra.command import _apply_migration + + sql = "CREATE TABLE widgets (id int primary key);" + result = _apply_migration( + db_url, + [sql], + sql_output=sql, + rollback_sql="DROP TABLE widgets;", + env_label="staging", + ) + + assert result["applied"] is True + assert result["history_recorded"] is True + assert _table_exists(db_url, "widgets") + + from migra.history import get_history + + with S(db_url) as s: + entries = get_history(s) + assert len(entries) == 1 + assert entries[0]["rollback_sql"] == "DROP TABLE widgets;" + assert entries[0]["environment_label"] == "staging" From 46b63729341e48319c0ec82d0b4d3cfea80370c6 Mon Sep 17 00:00:00 2001 From: MigraDiff Agent Date: Sun, 9 Aug 2026 07:41:05 -0700 Subject: [PATCH 08/11] fix: comma-separated --schema/--exclude_schema silently matched nothing --schema public,reporting was documented as supported but the raw comma-joined string was passed straight through to schemainspect's filter_schema(), which compares it for exact equality against each object's schema name -- "public,reporting" never equals "public" or "reporting", so the diff was always empty with no error. Single-schema and no-schema calls were never affected. Adds parse_schema_arg()/filter_inspector_schemas() in migra/util.py (the latter reuses schemainspect's own PROPS list rather than a hardcoded copy, so it can't drift out of sync) and _get_inspector() in migra/migra.py, wired into all 5 of Migration's inspector-construction call sites. Single/no-schema calls pass straight through to the original get_inspector() unchanged; only 2+ comma-separated names take the new post-filter path. New tests each include a third, unlisted schema in the fixture data to prove real filtering rather than "happens to work with one schema", and exercise both the CLI path and Migration.apply() directly. Co-Authored-By: Claude Sonnet 5 --- CHANGELOG.md | 25 ++++++++++ migra/migra.py | 31 ++++++++++-- migra/util.py | 49 +++++++++++++++++++ status.md | 46 ++++++++--------- tests/FIXTURES/exclude_multischema/a.sql | 13 +++++ .../exclude_multischema/additions.sql | 0 tests/FIXTURES/exclude_multischema/b.sql | 17 +++++++ .../FIXTURES/exclude_multischema/expected.sql | 7 +++ .../exclude_multischema/expected2.sql | 0 tests/FIXTURES/multischema/a.sql | 13 +++++ tests/FIXTURES/multischema/additions.sql | 0 tests/FIXTURES/multischema/b.sql | 17 +++++++ tests/FIXTURES/multischema/expected.sql | 7 +++ tests/FIXTURES/multischema/expected2.sql | 0 tests/test_migra.py | 16 ++++++ 15 files changed, 214 insertions(+), 27 deletions(-) create mode 100644 tests/FIXTURES/exclude_multischema/a.sql create mode 100644 tests/FIXTURES/exclude_multischema/additions.sql create mode 100644 tests/FIXTURES/exclude_multischema/b.sql create mode 100644 tests/FIXTURES/exclude_multischema/expected.sql create mode 100644 tests/FIXTURES/exclude_multischema/expected2.sql create mode 100644 tests/FIXTURES/multischema/a.sql create mode 100644 tests/FIXTURES/multischema/additions.sql create mode 100644 tests/FIXTURES/multischema/b.sql create mode 100644 tests/FIXTURES/multischema/expected.sql create mode 100644 tests/FIXTURES/multischema/expected2.sql diff --git a/CHANGELOG.md b/CHANGELOG.md index c3f6e3d..266101b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,31 @@ - New tests: `test_command_apply.py` (11 tests, real Postgres, including a direct atomicity test that forces a mid-migration failure) +### Fixed + +- **Comma-separated `--schema`/`--exclude_schema` silently matched nothing**: + `--schema public,reporting` was documented as supported but the raw + string was passed straight to `schemainspect`, which compares it for + *exact* equality against each object's schema name — `"public,reporting"` + never equals `"public"` or `"reporting"`, so the diff was always empty + and nothing was reported to the user. Single-schema (`--schema public`) + and no-schema calls were never affected — this only broke 2+ names. + - Fix: `migra/util.py` gains `parse_schema_arg()` (splits and trims the + comma-separated value) and `filter_inspector_schemas()` (post-filters + an inspector's tracked object collections using schemainspect's own + `PROPS` list, so it stays in sync with whatever object types + schemainspect adds in the future). + - `migra/migra.py` gains `_get_inspector()`, used at all 5 of + `Migration`'s inspector-construction call sites (`__init__` x2, + `inspect_from()`, `inspect_target()`, `apply()`). Single-schema/no-schema + calls are passed straight through unchanged (zero behavior change); + only 2+ comma-separated names take the new post-filter path. + - New tests: `test_multischema`, `test_multischema_whitespace_tolerant`, + `test_exclude_multischema` in `test_migra.py`, plus new fixtures under + `tests/FIXTURES/multischema/` and `tests/FIXTURES/exclude_multischema/` + — each includes a third, unlisted schema to prove filtering actually + excludes it, not just that it happens to work with one schema. + ### Notes - This is the foundational layer for the upcoming "Control Plane" feature set. diff --git a/migra/migra.py b/migra/migra.py index f7a5932..1150692 100644 --- a/migra/migra.py +++ b/migra/migra.py @@ -4,6 +4,27 @@ from .changes import Changes from .statements import Statements +from .util import filter_inspector_schemas, parse_schema_arg + + +def _get_inspector(x, schema=None, exclude_schema=None): + """Like schemainspect's get_inspector(), but also supports a + comma-separated `schema`/`exclude_schema` (2+ names). schemainspect's + own one_schema()/filter_schema() only ever compare against a single + schema name, so a raw multi-name string would silently match nothing. + Single-schema (or no-schema) calls are passed straight through + unchanged. + """ + schema_list = parse_schema_arg(schema) + exclude_list = parse_schema_arg(exclude_schema) + + if schema_list and len(schema_list) > 1: + return filter_inspector_schemas(get_inspector(x), schema_list) + + if exclude_list and len(exclude_list) > 1: + return filter_inspector_schemas(get_inspector(x), None, exclude=exclude_list) + + return get_inspector(x, schema=schema, exclude_schema=exclude_schema) class Migration(object): @@ -28,7 +49,7 @@ def __init__( if isinstance(x_from, DBInspector): self.changes.i_from = x_from else: - self.changes.i_from = get_inspector( + self.changes.i_from = _get_inspector( x_from, schema=schema, exclude_schema=exclude_schema ) if x_from: @@ -36,7 +57,7 @@ def __init__( if isinstance(x_target, DBInspector): self.changes.i_target = x_target else: - self.changes.i_target = get_inspector( + self.changes.i_target = _get_inspector( x_target, schema=schema, exclude_schema=exclude_schema ) if x_target: @@ -45,12 +66,12 @@ def __init__( self.changes.ignore_extension_versions = ignore_extension_versions def inspect_from(self): - self.changes.i_from = get_inspector( + self.changes.i_from = _get_inspector( self.s_from, schema=self.schema, exclude_schema=self.exclude_schema ) def inspect_target(self): - self.changes.i_target = get_inspector( + self.changes.i_target = _get_inspector( self.s_target, schema=self.schema, exclude_schema=self.exclude_schema ) @@ -62,7 +83,7 @@ def apply(self): for stmt in self.statements: raw_execute(self.s_from, stmt) - self.changes.i_from = get_inspector( + self.changes.i_from = _get_inspector( self.s_from, schema=self.schema, exclude_schema=self.exclude_schema ) safety_on = self.statements.safe diff --git a/migra/util.py b/migra/util.py index 4d0c788..952199c 100644 --- a/migra/util.py +++ b/migra/util.py @@ -3,6 +3,55 @@ from collections import OrderedDict as od +def parse_schema_arg(schema): + """Split a comma-separated --schema/--exclude_schema value into a list + of trimmed, non-empty schema names. Returns None for a falsy input.""" + if not schema: + return None + parts = [s.strip() for s in schema.split(",")] + return [p for p in parts if p] + + +def filter_inspector_schemas(inspector, schemas=None, exclude=None): + """Filter an already-built (unfiltered) inspector down to only the given + schemas, or to exclude the given schemas. Needed for the multi-schema + case (2+ comma-separated names), since schemainspect's own + `filter_schema`/`one_schema` only accepts a single schema name and + compares it for exact equality -- passing it a raw "a,b" string silently + matches nothing. + + Mutates and returns `inspector`. Reuses schemainspect's own PROPS list + (the same one `DBInspector.filter_schema` iterates over) so this stays + in sync with whatever object types schemainspect tracks, rather than + maintaining a second, driftable copy of that list here. + """ + if not schemas and not exclude: + return inspector + if not hasattr(inspector, "filter_schema"): + return inspector + + from schemainspect.pg.obj import PROPS + + schema_set = set(schemas) if schemas else None + exclude_set = set(exclude) if exclude else None + for prop in PROPS.split(): + att = getattr(inspector, prop, {}) + if schema_set: + filtered = { + k: v + for k, v in att.items() + if hasattr(v, "schema") and v.schema in schema_set + } + else: + filtered = { + k: v + for k, v in att.items() + if hasattr(v, "schema") and v.schema not in exclude_set + } + setattr(inspector, prop, filtered) + return inspector + + def differences(a, b, add_dependencies_for_modifications=True): a_keys = set(a.keys()) b_keys = set(b.keys()) diff --git a/status.md b/status.md index 506f1ff..664da83 100644 --- a/status.md +++ b/status.md @@ -3,28 +3,30 @@ Snapshot of current work-in-progress on this branch. Update as work progresses; this is not a changelog (see CHANGELOG.md for that). -## Current branch: `feature/apply-flag` (off `main` at `46c2447`) +## Current branch: `fix/multi-schema-filtering` (off `main` at `a762dec`) -Migration state tracking (`--status`/`--history`/`--record-history`/`--promote`/ -`--record-rollback`) and the CI-trigger/SQLAlchemy-2.x fixes are already merged to `main` -(PRs #8, #9, #10) — see git log, not this file, for that history. +Migration state tracking, the CI-trigger/SQLAlchemy-2.x fixes, and `--apply` are already merged +to `main` (PRs #8–#11) — see git log, not this file, for that history. **In progress / uncommitted:** -- Modified: `CHANGELOG.md`, `README.md`, `migra/command.py`, `CLAUDE.md`, - `PROJECT_PLAN.md`, `PROJECT_PLAN2.md` — new `--apply` flag -- New: `tests/test_command_apply.py` - -**What `--apply` does:** executes the generated migration against `dburl_from` in a single -transaction instead of only printing it; on success, automatically records it in `dburl_from`'s -`migradiff_history` table (no need to also pass `--record-history`); on failure, rolls back -everything and records nothing, exits 4. Rejected up front when combined with `--from-file` or -`--promote` — see CLAUDE.md's "Migration state tracking" section for the from/target direction -reasoning. - -**Verified 2026-08-08:** 11 new tests pass (including a direct atomicity test against -`_apply_migration()` that forces a mid-migration failure); full suite is 353 passed / 2 skipped, -no regressions; flake8 and black clean; also verified against SQLAlchemy 2.0.51 in an isolated -venv (no other raw-SQL issues found). - -**Next steps:** commit, push, open PR. Known follow-up (not done here): reconcile `--promote`'s -from/to direction with `--apply` before wiring the two together (see PROJECT_PLAN.md backlog). +- Modified: `CHANGELOG.md`, `migra/migra.py`, `migra/util.py`, `tests/test_migra.py` +- New: `tests/FIXTURES/multischema/`, `tests/FIXTURES/exclude_multischema/` + +**The bug:** `--schema public,reporting` was documented as supported but silently produced an +empty diff — the raw comma-joined string was passed straight to `schemainspect`, which compares +it for *exact* equality against each object's schema name. Single-schema and no-schema calls were +never affected. + +**The fix:** `migra/util.py` gains `parse_schema_arg()`/`filter_inspector_schemas()` (the latter +reuses schemainspect's own `PROPS` list rather than duplicating it, so it stays in sync with +whatever object types schemainspect tracks). `migra/migra.py` gains `_get_inspector()`, wired into +all 5 of `Migration`'s inspector-construction call sites; single/no-schema calls pass straight +through unchanged, only 2+ comma-separated names take the new post-filter path. + +**Verified 2026-08-08:** new tests (`test_multischema`, `test_multischema_whitespace_tolerant`, +`test_exclude_multischema`) each include a third, unlisted schema in the fixture to prove +filtering actually excludes it, not just that it happens to work with one schema; also exercises +`Migration.apply()` directly (via `do_fixture_test`'s second half), not just the CLI path. Full +suite: 356 passed / 2 skipped, no regressions. flake8/black clean. + +**Next steps:** commit, push, open PR. diff --git a/tests/FIXTURES/exclude_multischema/a.sql b/tests/FIXTURES/exclude_multischema/a.sql new file mode 100644 index 0000000..072e678 --- /dev/null +++ b/tests/FIXTURES/exclude_multischema/a.sql @@ -0,0 +1,13 @@ +create schema schema1; + +create table schema1.t(id uuid, value text); + +create schema schema2; + +create table schema2.x(id uuid, value text); + +create schema schema3; + +create table schema3.untouched(id uuid); + +create table public.other(id uuid, value text); diff --git a/tests/FIXTURES/exclude_multischema/additions.sql b/tests/FIXTURES/exclude_multischema/additions.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/FIXTURES/exclude_multischema/b.sql b/tests/FIXTURES/exclude_multischema/b.sql new file mode 100644 index 0000000..fd7f2bb --- /dev/null +++ b/tests/FIXTURES/exclude_multischema/b.sql @@ -0,0 +1,17 @@ +create schema schema1; + +create table schema1.t(id uuid, value text, name text); + +create schema schema2; + +create table schema2.x(id uuid, value text); + +create table schema2.y(id uuid, value text); + +create schema schema3; + +create table schema3.untouched(id uuid); + +create table schema3.should_be_ignored(id uuid); + +create table public.other(id uuid); diff --git a/tests/FIXTURES/exclude_multischema/expected.sql b/tests/FIXTURES/exclude_multischema/expected.sql new file mode 100644 index 0000000..492ab16 --- /dev/null +++ b/tests/FIXTURES/exclude_multischema/expected.sql @@ -0,0 +1,7 @@ +create table "schema2"."y" ( + "id" uuid, + "value" text +); + + +alter table "schema1"."t" add column "name" text; diff --git a/tests/FIXTURES/exclude_multischema/expected2.sql b/tests/FIXTURES/exclude_multischema/expected2.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/FIXTURES/multischema/a.sql b/tests/FIXTURES/multischema/a.sql new file mode 100644 index 0000000..072e678 --- /dev/null +++ b/tests/FIXTURES/multischema/a.sql @@ -0,0 +1,13 @@ +create schema schema1; + +create table schema1.t(id uuid, value text); + +create schema schema2; + +create table schema2.x(id uuid, value text); + +create schema schema3; + +create table schema3.untouched(id uuid); + +create table public.other(id uuid, value text); diff --git a/tests/FIXTURES/multischema/additions.sql b/tests/FIXTURES/multischema/additions.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/FIXTURES/multischema/b.sql b/tests/FIXTURES/multischema/b.sql new file mode 100644 index 0000000..fd7f2bb --- /dev/null +++ b/tests/FIXTURES/multischema/b.sql @@ -0,0 +1,17 @@ +create schema schema1; + +create table schema1.t(id uuid, value text, name text); + +create schema schema2; + +create table schema2.x(id uuid, value text); + +create table schema2.y(id uuid, value text); + +create schema schema3; + +create table schema3.untouched(id uuid); + +create table schema3.should_be_ignored(id uuid); + +create table public.other(id uuid); diff --git a/tests/FIXTURES/multischema/expected.sql b/tests/FIXTURES/multischema/expected.sql new file mode 100644 index 0000000..492ab16 --- /dev/null +++ b/tests/FIXTURES/multischema/expected.sql @@ -0,0 +1,7 @@ +create table "schema2"."y" ( + "id" uuid, + "value" text +); + + +alter table "schema1"."t" add column "name" text; diff --git a/tests/FIXTURES/multischema/expected2.sql b/tests/FIXTURES/multischema/expected2.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_migra.py b/tests/test_migra.py index c61cd84..ebf502e 100644 --- a/tests/test_migra.py +++ b/tests/test_migra.py @@ -60,11 +60,27 @@ def test_singleschema(): do_fixture_test(FIXTURE_NAME, schema="goodschema") +def test_multischema(): + for FIXTURE_NAME in ["multischema"]: + do_fixture_test(FIXTURE_NAME, schema="schema1,schema2") + + +def test_multischema_whitespace_tolerant(): + # comma-separated schema names should tolerate surrounding whitespace + for FIXTURE_NAME in ["multischema"]: + do_fixture_test(FIXTURE_NAME, schema=" schema1 , schema2 ") + + def test_excludeschema(): for FIXTURE_NAME in ["excludeschema"]: do_fixture_test(FIXTURE_NAME, exclude_schema="excludedschema") +def test_exclude_multischema(): + for FIXTURE_NAME in ["exclude_multischema"]: + do_fixture_test(FIXTURE_NAME, exclude_schema="schema3,public") + + def test_singleschema_ext(): for FIXTURE_NAME in ["singleschema_ext"]: do_fixture_test(FIXTURE_NAME, create_extensions_only=True) From 659b37a9954704f051fb3d131d45388f29032858 Mon Sep 17 00:00:00 2001 From: MigraDiff Agent Date: Sun, 30 Aug 2026 10:27:56 -0700 Subject: [PATCH 09/11] refactor: extract _require_ai helper to deduplicate API key validation --- migra/command.py | 197 ++++++++++++----------------------------------- 1 file changed, 48 insertions(+), 149 deletions(-) diff --git a/migra/command.py b/migra/command.py index 0751d0c..b2d502e 100644 --- a/migra/command.py +++ b/migra/command.py @@ -276,6 +276,45 @@ def _format_destructive_summary(statements): return "\n".join(lines) +def _require_ai(out, err, cli_key=None): + """Resolve an Anthropic API key, printing standard help text on failure. + + Returns the resolved key string or ``None`` on error. + """ + try: + import anthropic # noqa: F401 + except ImportError: + print( + "MigraDiff: AI features require the extras.", + file=err, + ) + print("Install with: pip install migradiff[ai]", file=err) + return None + + from .ai_explain import resolve_api_key + + api_key = resolve_api_key(cli_key=cli_key) + if not api_key: + print( + "MigraDiff: An Anthropic API key is required.", + file=err, + ) + print(file=err) + print("Set it up once with:", file=err) + print(" migra --setup-ai", file=err) + print(file=err) + print("Or set the environment variable:", file=err) + print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) + print(file=err) + print( + "Get an API key at: https://console.anthropic.com", + file=err, + ) + return None + + return api_key + + def _migration_sort_key(filename): """Extract sort key from a migration filename.""" stem = filename.lower().replace(".sql", "") @@ -947,35 +986,8 @@ def run(args, out=None, err=None): ) return 1 - try: - import anthropic # noqa: F401, F811 - except ImportError: - print( - "MigraDiff: --explain-drift requires the AI extras.", - file=err, - ) - print("Install with: pip install migradiff[ai]", file=err) - return 1 - - from .ai_explain import resolve_api_key - - api_key = resolve_api_key(cli_key=args.api_key) + api_key = _require_ai(out, err, args.api_key) if not api_key: - print( - "MigraDiff: --explain-drift requires an Anthropic API key.", - file=err, - ) - print(file=err) - print("Set it up once with:", file=err) - print(" migra --setup-ai", file=err) - print(file=err) - print("Or set the environment variable:", file=err) - print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) - print(file=err) - print( - "Get an API key at: https://console.anthropic.com", - file=err, - ) return 1 try: @@ -1037,33 +1049,8 @@ def run(args, out=None, err=None): ) return 1 - try: - import anthropic # noqa: F401, F811 - except ImportError: - print( - "MigraDiff: --generate requires the AI extras.", - file=err, - ) - print("Install with: pip install migradiff[ai]", file=err) - return 1 - - api_key = resolve_api_key(cli_key=args.api_key) + api_key = _require_ai(out, err, args.api_key) if not api_key: - print( - "MigraDiff: --generate requires an Anthropic API key.", - file=err, - ) - print(file=err) - print("Set it up once with:", file=err) - print(" migra --setup-ai", file=err) - print(file=err) - print("Or set the environment variable:", file=err) - print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) - print(file=err) - print( - "Get an API key at: https://console.anthropic.com", - file=err, - ) return 1 # Safety check first @@ -1165,21 +1152,8 @@ def run(args, out=None, err=None): ): filepath = args.dburl_from if os.path.exists(filepath) and filepath.endswith(".sql"): - from .ai_explain import resolve_api_key - - api_key = resolve_api_key(cli_key=args.api_key) + api_key = _require_ai(out, err, args.api_key) if not api_key: - print( - "MigraDiff: --advise requires an Anthropic API key.", - file=err, - ) - print(file=err) - print("Set it up once with:", file=err) - print(" migra --setup-ai", file=err) - print(file=err) - print("Or set the environment variable:", file=err) - print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) - print(file=err) return 1 from .ai_explain import generate_file_advisory @@ -1337,35 +1311,10 @@ def _run_inner(args, out=None, err=None): rollback_result = None advisory_result = None if args.explain: - from .ai_explain import AIExplainer, resolve_api_key, redact_api_key - - try: - import anthropic # noqa: F401, F811, F811 - except ImportError: - print( - "MigraDiff: --explain requires the AI extras.", - file=err, - ) - print("Install with: pip install migradiff[ai]", file=err) - return 1 + from .ai_explain import AIExplainer, redact_api_key - api_key = resolve_api_key(cli_key=args.api_key) + api_key = _require_ai(out, err, args.api_key) if not api_key: - print( - "MigraDiff: --explain requires an Anthropic API key.", - file=err, - ) - print(file=err) - print("Set it up once with:", file=err) - print(" migra --setup-ai", file=err) - print(file=err) - print("Or set the environment variable:", file=err) - print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) - print(file=err) - print( - "Get an API key at: https://console.anthropic.com", - file=err, - ) return 1 if statements: @@ -1385,35 +1334,10 @@ def _run_inner(args, out=None, err=None): # AI rollback generation if args.rollback: - from .ai_explain import AIRollback, resolve_api_key, redact_api_key - - try: - import anthropic # noqa: F401, F811, F811 - except ImportError: - print( - "MigraDiff: --rollback requires the AI extras.", - file=err, - ) - print("Install with: pip install migradiff[ai]", file=err) - return 1 + from .ai_explain import AIRollback, redact_api_key - api_key = resolve_api_key(cli_key=args.api_key) + api_key = _require_ai(out, err, args.api_key) if not api_key: - print( - "MigraDiff: --rollback requires an Anthropic API key.", - file=err, - ) - print(file=err) - print("Set it up once with:", file=err) - print(" migra --setup-ai", file=err) - print(file=err) - print("Or set the environment variable:", file=err) - print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) - print(file=err) - print( - "Get an API key at: https://console.anthropic.com", - file=err, - ) return 1 if statements: @@ -1446,35 +1370,10 @@ def _run_inner(args, out=None, err=None): # AI advisory generation if args.advise: - from .ai_explain import AIAdvisor, resolve_api_key, redact_api_key - - try: - import anthropic # noqa: F401, F811, F811 - except ImportError: - print( - "MigraDiff: --advise requires the AI extras.", - file=err, - ) - print("Install with: pip install migradiff[ai]", file=err) - return 1 + from .ai_explain import AIAdvisor, redact_api_key - api_key = resolve_api_key(cli_key=args.api_key) + api_key = _require_ai(out, err, args.api_key) if not api_key: - print( - "MigraDiff: --advise requires an Anthropic API key.", - file=err, - ) - print(file=err) - print("Set it up once with:", file=err) - print(" migra --setup-ai", file=err) - print(file=err) - print("Or set the environment variable:", file=err) - print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) - print(file=err) - print( - "Get an API key at: https://console.anthropic.com", - file=err, - ) return 1 if statements: From cf4f309cfa6c2c17822207f381efee76cc577b37 Mon Sep 17 00:00:00 2001 From: MigraDiff Agent Date: Sun, 30 Aug 2026 13:08:13 -0700 Subject: [PATCH 10/11] fix: remove unused resolve_api_key import in --generate block --- migra/command.py | 1 - 1 file changed, 1 deletion(-) diff --git a/migra/command.py b/migra/command.py index b2d502e..1d4a9f8 100644 --- a/migra/command.py +++ b/migra/command.py @@ -1034,7 +1034,6 @@ def run(args, out=None, err=None): check_safety_rules, extract_relevant_schema, parse_schema_file_for_tables, - resolve_api_key, redact_api_key, classify_statement_risk, ) From d56130abd807f4b7db91d545f257ef60a92a2512 Mon Sep 17 00:00:00 2001 From: MigraDiff Agent Date: Sun, 30 Aug 2026 13:21:43 -0700 Subject: [PATCH 11/11] fix: adjust error message to match test assertion --- migra/command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migra/command.py b/migra/command.py index 1d4a9f8..b24e7ac 100644 --- a/migra/command.py +++ b/migra/command.py @@ -285,7 +285,7 @@ def _require_ai(out, err, cli_key=None): import anthropic # noqa: F401 except ImportError: print( - "MigraDiff: AI features require the extras.", + "MigraDiff: AI features require the AI extras.", file=err, ) print("Install with: pip install migradiff[ai]", file=err)