diff --git a/.github/CI_NOTES.md b/.github/CI_NOTES.md index c3b5ed2..d28a8fa 100644 --- a/.github/CI_NOTES.md +++ b/.github/CI_NOTES.md @@ -30,7 +30,7 @@ view definitions. ## Workflow -- **CI** runs on: push to `master`, PRs to `master` +- **CI** runs on: push to `main` or `master`, PRs targeting `main` or `master` - Lint (flake8, black) - Test matrix (4 Python versions × 3 Postgres versions = 12 jobs) - Coverage report @@ -39,3 +39,15 @@ view definitions. - Build package - Publish to PyPI - Create GitHub release + +### `main` vs `master` branch protection (2026-08-08) + +`main` is the actual GitHub default branch (`gh repo view --json defaultBranchRef` +confirms this), but only `master` has branch protection configured — 13 required +status checks (`Lint` + the 12-job test matrix), `strict` status checks, no force +pushes/deletions. `main` has **no protection at all** +(`gh api repos/.../branches/main/protection` returns 404). This means PRs merged +into `main` today get no required CI gate. The workflow trigger was fixed here +to run on both branches, but the branch protection *rule* itself still needs to +be applied to `main` — that's a repo-settings change, not a workflow file change, +and hasn't been done yet. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 206e2d3..b27c656 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [master] + branches: [main, master] pull_request: - branches: [master] + branches: [main, master] env: PGUSER: postgres diff --git a/CHANGELOG.md b/CHANGELOG.md index c94c0e7..266101b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,82 @@ ## [Unreleased] +### Added + +- **Migration state tracking** (`migradiff_history` table): + - `--status` / `--history` flags to view migration history from a database + - `--record-history` flag to record generated migrations in the target DB + - `--env-label` flag to tag history entries with an environment name + - New module `migra/history.py` with `ensure_history_table()`, + `record_applied()`, `record_rollback()`, `get_applied_hashes()`, + `get_history()`, and `compute_migration_hash()` + - SQLAlchemy Core implementation via `sqlbag`, no new dependencies +- **Multi-environment promotion** (`--promote`): + - Colon-separated chain of 2+ database URLs or environment aliases + - Conflict detection at each hop (target must not have hashes source lacks) + - Composes with `--explain`, `--rollback`, `--advise`, `--force-destructive` + - Environment aliases resolved from `~/.config/migradiff/environments.json` + - `--record-history` integration per hop +- **Rollback execution tracking** (`--record-rollback`): + - Records rollback status (`rolled_back` / `rollback_failed`) in the history table + - Accepts a migration hash or rollback SQL file path + - `--rollback-status` flag to mark failed rollback attempts + - `--status` / `--history` output shows rollback annotations +- **New tests**: `test_history.py`, `test_command_promote.py`, + `test_command_rollback_tracking.py` — all using real Postgres databases + (no mocks), covering idempotent table creation, hash normalization, + conflict detection, empty-diff hops, safe mode, and rollback tracking. +- **Migration execution** (`--apply`): + - Executes the generated migration against `dburl_from` in a single + transaction (all-or-nothing — a failed statement rolls back everything + that ran before it) instead of only printing the SQL + - On success, automatically records the migration in `dburl_from`'s + `migradiff_history` table — no need to also pass `--record-history` + - On failure, nothing is recorded (since nothing was actually applied) + and the command exits with code 4 + - Not supported with `--from-file` (no live database to apply to) or + `--promote` (not implemented yet) — both are rejected with a clear + error before anything runs + - New helper `_apply_migration()` in `migra/command.py`; JSON output + (`--output json`) gets a new `"apply"` object with the outcome + - New tests: `test_command_apply.py` (11 tests, real Postgres, including + a direct atomicity test that forces a mid-migration failure) + +### Fixed + +- **Comma-separated `--schema`/`--exclude_schema` silently matched nothing**: + `--schema public,reporting` was documented as supported but the raw + string was passed straight to `schemainspect`, which compares it for + *exact* equality against each object's schema name — `"public,reporting"` + never equals `"public"` or `"reporting"`, so the diff was always empty + and nothing was reported to the user. Single-schema (`--schema public`) + and no-schema calls were never affected — this only broke 2+ names. + - Fix: `migra/util.py` gains `parse_schema_arg()` (splits and trims the + comma-separated value) and `filter_inspector_schemas()` (post-filters + an inspector's tracked object collections using schemainspect's own + `PROPS` list, so it stays in sync with whatever object types + schemainspect adds in the future). + - `migra/migra.py` gains `_get_inspector()`, used at all 5 of + `Migration`'s inspector-construction call sites (`__init__` x2, + `inspect_from()`, `inspect_target()`, `apply()`). Single-schema/no-schema + calls are passed straight through unchanged (zero behavior change); + only 2+ comma-separated names take the new post-filter path. + - New tests: `test_multischema`, `test_multischema_whitespace_tolerant`, + `test_exclude_multischema` in `test_migra.py`, plus new fixtures under + `tests/FIXTURES/multischema/` and `tests/FIXTURES/exclude_multischema/` + — each includes a third, unlisted schema to prove filtering actually + excludes it, not just that it happens to work with one schema. + +### Notes + +- This is the foundational layer for the upcoming "Control Plane" feature set. +- Resolved: `migradiff_history` used to only record migrations as + *generated*, not *applied* — `--apply` closes that gap by executing the + migration itself and only recording history on confirmed success. Plain + `--record-history` (without `--apply`) still only means "generated", not + "applied" — see the README "Migration State Tracking" section. +- All new features work without AI extras (`anthropic` optional dependency). + ## [1.7.2] - 2026-06-08 - Fix: Use PEP 621 [project] table for PyPI metadata (homepage, repository, bug tracker) diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..a5c9697 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,106 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project + +MigraDiff (package `migradiff`, CLI command `migra`) — an actively maintained fork of the deprecated +`djrobstep/migra`. It compares two PostgreSQL schemas (live connections, `pg_dump -s` files, or a +migrations directory applied to an ephemeral DB) and generates the SQL needed to transform one into +the other. + +**Contributing notice:** per `CONTRIBUTION_GUIDE.md` and the README, this repo is not currently +accepting external PRs — keep that in mind if asked to open one. + +## Commands + +Tests require a live PostgreSQL instance (12+, container uses 16) on `localhost:5432` with trust auth: + +```bash +docker compose up -d # start Postgres for the test suite +docker compose down # stop it (add -v to wipe the data volume) +``` + +```bash +make test # py.test -x -svv --cov-report term-missing --cov migra tests +make lint # flake8 . (config in .flake8: ignores E501, W503) +make fmt # isort . && black . +make clean # remove .pyc, .cache, build artifacts +``` + +Run a single test: `pytest tests/test_command_promote.py::test_name -svv` + +CI (CircleCI, `.circleci/config.yml`) runs `make lint` then `make test` against Postgres 14 on every +branch, and publishes to PyPI + tags a release from `master`. + +## Architecture + +### Core diff engine (upstream-derived, keep changes surgical) + +- `migra/migra.py` — `Migration` class: wraps two `DBInspector`/connection-string/schema-dump sources + (`schemainspect.get_inspector`), and `add_all_changes()` sequences every category of change + (schemas, extensions, enums, sequences, triggers, RLS, constraints, indexes, tables, domains, + privileges, comments, ...) in an order that respects PostgreSQL dependency rules — drops before + creates, dependents before dependencies. **This ordering is load-bearing**; don't reshuffle it + without understanding why each step is where it is. +- `migra/changes.py` — `Changes` class plus the `get_*_changes`/`statements_from_differences` + functions that walk `schemainspect` inspector objects and emit SQL for each object type. + `util.differences()` is the generic added/removed/modified/unmodified diff primitive everything + else is built on. +- `migra/statements.py` — `Statements` (a `list` subclass) accumulates generated SQL and refuses to + render (`.sql`) if any statement contains DROP while `safe=True` (`UnsafeMigrationException`) — + this is the `--unsafe` flag's underlying gate. +- `migra/db_inspector.py` — lower-level live-DB comparison helpers (table sizes, per-object-type + compare functions) used mainly by the AI drift/advise features rather than core diffing. + +### CLI (`migra/command.py`, the biggest file — ~57KB) + +- `parse_args()` defines every flag; `run()`/`_run_inner()` is the dispatch body; `do_command()` is + the `pyproject.toml` entry point (`migra = 'migra:do_command'`). +- Handles multiple input modes: two DB URLs, `--from-file` (pg_dump output), `--from-migrations-dir` + (applies migration files to an ephemeral DB — supports Supabase/Flyway/numeric naming, see + `discover_migration_files`/`apply_migrations`). +- Post-processes raw diff output for safety/UX: destructive-statement detection + (`_check_for_destructive`), column-rename detection (`detect_column_renames`, vs. a naive + drop+add), risk classification for `--output json` (`classify_sql_statement`), credential + redaction in error output (`redact_credentials`). +- `--status`/`--history`/`--promote`/`--record-rollback`/`--apply` etc. talk to `migra/history.py`. + +### Migration state tracking (`migra/history.py`) + +A `migradiff_history` table (`HISTORY_TABLE`), keyed by a SHA-256 hash of normalized SQL +(`compute_migration_hash`). This is the basis for `--promote` (multi-environment promotion), +`--record-rollback`, and `--apply`. **Important semantic, easy to get backwards**: a plain +`--record-history` (without `--apply`) writes into `dburl_target`'s history table and only means +"this migration was generated/reviewed for that target" — not that it was executed anywhere. +`--apply` (`_apply_migration()` in `command.py`) is the one path that actually executes SQL: it runs +the migration against `dburl_from` (the database being migrated — see the CLI's own help text and +the README's `psql dburl_from < migration.sql` convention) inside a single transaction, and only on +confirmed success does it record history — into `dburl_from`'s table, not `dburl_target`'s. A failed +`--apply` rolls back everything and records nothing. `--apply` is rejected up front (before any +dispatch) when combined with `--from-file` (dburl_from would be an ephemeral throwaway database) or +`--promote` (the chain's from/to direction is not yet reconciled with `--apply`'s execution target). + +### AI features (optional extra: `pip install migradiff[ai]`, needs `ANTHROPIC_API_KEY`) + +- `migra/ai_explain.py` — one file holding four related-but-distinct features, each with its own + prompt builder + client class: `AIExplainer` (`--explain`), `AIRollback` (`--rollback`, includes a + fully deterministic non-AI rollback path in `generate_deterministic_rollback` plus AI-assisted + reconstruction using `extract_schema_context`), `AIAdvisor` (`--advise`, performance/lock risk), + `AIGenerator` (`--generate`, plain-English → grounded SQL via `extract_relevant_schema`). Also + owns API key config (`~/.migradiff` style config dir, `--setup-ai`, `resolve_api_key`, + `redact_api_key`). +- `migra/ai_drift.py` — `DriftExplainer` (`--explain-drift`): compares two *live* databases (not a + generated migration) and explains the difference, categorizing BREAKING/WARNING/INFO. +- All AI calls are opt-in per the README's privacy framing ("bring your own API key — no data sent + to MigraDiff servers"); don't add code paths that phone home elsewhere. + +### Tests + +- `tests/` mirrors the module layout (`test_ai_explain.py`, `test_command_promote.py`, etc.) and + needs the live Postgres from `docker compose up -d`. +- `tests/conftest.py` monkey-patches `sqlbag.createdrop` create/drop/exists functions to work + around a Postgres-specific issue in the upstream `sqlbag` dependency — don't remove this shim + without checking `sqlbag`'s current behavior. +- `tests/characterization/` is a separate CLI characterization suite (`capture.py`, `scenarios.py`) + that snapshots CLI behavior for the AI-flag surface, independent of the DB-driven unit tests. diff --git a/LICENSING.md b/LICENSING.md new file mode 100644 index 0000000..ad98f19 --- /dev/null +++ b/LICENSING.md @@ -0,0 +1,40 @@ +# LICENSING.md + +## Model: MIT + Business License Agreement (BLA) + +All features are available to everyone. No code gatekeeping. +No feature gates. No license key required to unlock functionality. + +### MIT License (Free) + +**Who:** Individuals, open-source projects, and teams evaluating +for production — no time limit. + +**Covers:** All features, unlimited use, full CLI, all schema diffing, +RLS policies, AI flags (--explain, --rollback, --advise, --generate), +JSON output, GitHub Action, Docker images. + +**Support:** Community (GitHub issues) + +### Business License Agreement (Paid) + +**Who:** Required when your team ships to production commercially +(any company with employees using the tool as part of their work). + +**Covers:** Everything in MIT license, plus email support and +legal documentation for compliance auditing. + +**Tiers:** +- Small Team (1–10 developers): $599/year +- Mid-Market (10–50 developers): $1,999/year +- Enterprise (50+ developers): $5,000/year + +**Simple rule:** Evaluate freely, no time limit. Purchase when +your team ships to production. + +Questions? [licensing@lateos.ai](mailto:licensing@lateos.ai) +Full terms: [lateos.ai/migradiff/licensing](https://lateos.ai/migradiff/licensing) + +--- + +Full MIT license text is in the LICENSE file. \ No newline at end of file diff --git a/PROJECT_PLAN.md b/PROJECT_PLAN.md index dc8ed1b..8ef249d 100644 --- a/PROJECT_PLAN.md +++ b/PROJECT_PLAN.md @@ -25,17 +25,50 @@ ## Completed Features -### v1.3.0 (Sessions 009–010) +### v1.3.0 (Sessions 009–010) — 2026-05-30 - `--explain`: plain-English explanation of migrations - `--from-migrations-dir`: load migrations from directory -### v1.4.0 (Sessions 011–013) +### v1.4.0 (Sessions 011–013) — 2026-06-01 - `--rollback`: generates reversal migration SQL - `--advise`: deterministic + AI performance/risk assessment - `--generate`: writes migration SQL from plain-English description -### v1.5.0 (Session 017) -- `--comment-on`: diffs COMMENT ON annotations on objects +### v1.5.0 (Session 017) — 2026-06-01 +- COMMENT ON diffing: detects added/changed/removed comments across all + object types; automatic in `--from-file` mode; feeds `--explain`/`--generate` + as schema context. (There is no separate `--comment-on` flag — this runs + by default; the plan previously described it as a flag, which was wrong.) + +### v1.6.0 — 2026-06-04 +- `--explain-drift`: AI-powered schema drift analysis between two live + databases (`--from-db`/`--to-db`), BREAKING/WARNING/INFO categorization, + live table sizes for risk assessment +- `migra/db_inspector.py`: standalone `get_remote_schema()`/`compare_schemas()` + +### v1.7.x — 2026-06-08 +- PyPI metadata fixes (PEP 621 `[project]` table, homepage/repo URLs) +- Repo moved from `migradiff/migra` to `postgresql-tools/migra` + +### Multi-language docs & licensing (merged pre-1.7.2, docs-only, no version bump) +- README translated into 6 languages (fr, de, ja, zh, hi, he) alongside English +- Licensing section (personal story + Business License ask) added to all READMEs + +### Migration state tracking — code-complete, not yet released +- `--status` / `--history`: view migration history from a `migradiff_history` table +- `--record-history` / `--env-label`: record generated migrations against a target DB +- `--promote`: chain migrations across environments (`dev:staging:prod` or + `~/.config/migradiff/environments.json` aliases) with conflict detection per hop +- `--record-rollback` / `--rollback-status`: close the audit loop on executed rollbacks +- New module `migra/history.py`; 32 new tests (`test_history.py`, + `test_command_promote.py`, `test_command_rollback_tracking.py`) +- Verified 2026-08-08: all 32 new tests pass against a real Postgres instance, + full suite (342 tests) passes, flake8/black clean, and the CLI was smoke-tested + end-to-end (`--status`, `--record-history`, `--promote`, `--record-rollback` + round-tripped correctly against live databases) +- Documented in CHANGELOG `[Unreleased]` and README's "Migration State Tracking" + section; sitting uncommitted on branch `new-feature-6-18-2026` — next release + will be **v1.8.0** ### Infrastructure (Session 018) - ✅ Production CI/CD pipeline @@ -47,90 +80,47 @@ ## Current Version -**v1.5.1** (released June 4, 2026) +**v1.7.2** (released 2026-06-08 — matches `pyproject.toml` and PyPI) -Test counts: -- Baseline: 109 tests -- v1.3.0: 142 tests (after --explain) -- v1.4.0: 175 tests (after AI suite) -- v1.5.0: 190 tests (after COMMENT ON) +Migration state tracking (above) is feature-complete and fully tested on +`new-feature-6-18-2026` but not yet merged or released. + +Test counts (actual, `pytest tests -q`, excluding the separate +characterization suite): **342 passed, 2 skipped**, verified 2026-08-08. --- ## Free Tier Roadmap -### Session 019: `--explain-drift` (7–10 days) - -**Feature:** Compare two **live PostgreSQL databases** and explain differences. - -```bash -migra --from-db "postgresql://user:pass@old.example.com/db" \ - --to-db "postgresql://user:pass@prod.example.com/db" \ - --explain-drift -``` - -**Output:** Human-readable explanation of schema differences with risk categorization (BREAKING/WARNING/INFO). - -**Use case:** "What changed in production? Is it safe?" - -**Tests:** 80+ new tests (target: 270+ total) +### Shipped: `--explain-drift` (v1.6.0) and multi-language README -**Release:** v1.6.0 +Both were originally planned as "Session 019" and "Session 020" — both are +done. See "Completed Features" above for what actually shipped: +`--explain-drift` in v1.6.0 (2026-06-04), and the README translations +(fr, de, ja, zh, hi, he — 6 languages, merged as a docs-only PR, no +dedicated version bump). -**Why first:** -- Highest differentiation (competitors don't have this) -- Reuses existing schema inspection logic -- Fills gap: explain migrations vs. reality +### Shipped: migration state tracking and `--apply` ---- - -### Session 020: Multi-Language README (3–5 days) - -**Feature:** README in multiple languages (i18n). - -Supported languages: -- 🇬🇧 English (default) -- 🇫🇷 French (README.fr.md) -- 🇩🇪 German (README.de.md) -- 🇯🇵 Japanese (README.ja.md) -- 🇨🇳 Chinese (README.zh.md) -- 🇮🇳 Hindi (README.hi.md) -- 🇮🇱 Hebrew (README.he.md) - -**How:** -- Maintain main README.md (English) -- Create translated versions in repo root -- Link from main README -- Use GitHub language detection for auto-redirect - -**Why:** -- Global audience (PostgreSQL is international) -- **Strategic focus:** India has second-largest PostgreSQL user base globally (after US) -- Drives adoption in high-growth markets (India, China, Japan) -- Differentiator (most tools English-only) -- SEO boost (ranked in multiple language searches) -- Hindi captures ~600M+ speakers in India tech ecosystem - -**Market sizing:** -- US: ~40% of users (English) -- India: ~20% of users (Hindi + English) -- Europe: ~15% of users (French, German) -- Asia-Pacific: ~15% of users (Chinese, Japanese) -- Israel/other: ~10% of users (Hebrew, English) - -**Release:** v1.6.1 - ---- +`--status`/`--history`/`--promote`/`--record-rollback` merged into `main` +2026-08-08 (PRs #8–#10; no version bump/tag yet — still on v1.7.2). +`--apply` (execute the migration against `dburl_from` instead of only +printing it, with automatic history recording on confirmed success — see +README's "Applying Migrations" section) shipped shortly after on top of +that. Neither has been cut into a tagged release yet; `pyproject.toml` +still reads 1.7.2. Bumping to v1.8.0 and publishing is still outstanding. -### Planned Sessions (Backlog) +### Planned (Backlog) -| Session | Feature | Effort | Value | Notes | -|---------|---------|--------|-------|-------| -| 021 | `--document` | Medium-High | High | Schema documentation generation | -| 022 | pgvector support | Low | Medium | Modern Postgres vector types | -| 023 | `--suggest-indexes` | Medium | Medium | AI recommends useful indexes | -| 024 | `--dry-run --explain` | Low | Medium | Preview changes without applying | -| 025 | Multi-schema refactor | Medium | High | Better support for multiple schemas | +| Feature | Effort | Value | Notes | +|---------|--------|-------|-------| +| Native `--fail-on-destructive` flag | Low | High | Currently this behavior only exists inside the GitHub Action's `action-entrypoint.sh`. CircleCI/GitLab/pre-commit users have no equivalent without wrapping the CLI themselves. | +| `--apply` + `--promote` | Medium | Medium | `--apply` currently refuses to run when `--promote` is also given — `--promote`'s from/to direction needs to be reconciled with which database `--apply` should actually execute against before this is safe to wire up. | +| `--document` | Medium-High | High | Schema documentation generation | +| Multi-schema hardening | Medium | High | `Migration.__init__` (`migra/migra.py`) still hard-rejects `schema` + `exclude_schema` together; cross-schema FK/dependency ordering in `add_all_changes()` hasn't had a dedicated multi-tenant test pass | +| pgvector support | Low | Medium | Modern Postgres vector types — unconfirmed whether `schemainspect` already round-trips them | +| `--suggest-indexes` | Medium | Medium | AI recommends useful indexes; can reuse `AIAdvisor`'s existing table-stats extraction | +| `--dry-run` | Low | Medium | Preview what `--apply` would do without executing it | --- @@ -222,18 +212,31 @@ The `schemainspect` package (upstream: djrobstep, unmaintained) uses deprecated **Migration path (future):** Replace with maintained schema inspection library or migrate to Rust. +### docker-compose.yml referenced but missing from repo +`README.md`'s "Development Setup" section and the CHANGELOG (v1.1.0 entry) +both reference a `docker-compose.yml` for one-command local Postgres, but +no such file currently exists in the repo root. Verified 2026-08-08 by +spinning up Postgres manually (`docker run postgres:16 ...`) to run the +test suite instead. + --- ## Version History | Version | Release Date | Key Features | |---------|--------------|--------------| -| v1.3.0 | May 2026 | --explain, --from-migrations-dir | -| v1.4.0 | June 1, 2026 | --rollback, --advise, --generate | -| v1.5.0 | June 4, 2026 | COMMENT ON diffing | -| v1.5.1 | June 4, 2026 | Version bump (no feature changes) | -| v1.6.0 | TBD (Session 019) | --explain-drift | -| v1.6.1 | TBD (Session 020) | Multi-language README | +| v1.1.0 | 2026-05-29 | --from-file, --output json, pre-commit hook | +| v1.2.0 | 2026-05-29 | Column rename detection, --safe/--force-destructive, enum/composite/domain diffing | +| v1.3.0 | 2026-05-30 | --explain, --setup-ai, --from-migrations-dir | +| v1.4.0 | 2026-06-01 | --rollback, --advise, --generate | +| v1.5.0 | 2026-06-01 | COMMENT ON diffing | +| v1.6.0 | 2026-06-04 | --explain-drift | +| v1.7.1 | 2026-06-08 | PyPI homepage/repo URL fix | +| v1.7.2 | 2026-06-08 | PEP 621 metadata fix (current release) | +| v1.8.0 | TBD | Migration state tracking (--status/--history/--promote/--record-rollback) — code-complete, unreleased | + +Multi-language READMEs and the Licensing section merged as docs-only PRs +without a dedicated version bump — not tied to a CHANGELOG entry. --- @@ -277,16 +280,17 @@ The `schemainspect` package (upstream: djrobstep, unmaintained) uses deprecated ### Q2 2026 - ✅ v1.5.0 shipped - ✅ CI/CD pipeline live -- ⏳ v1.6.0 shipped (`--explain-drift`) -- ⏳ 50+ GitHub stars -- ⏳ 1k+ monthly PyPI downloads +- ✅ v1.6.0 shipped (`--explain-drift`) +- ⏳ 50+ GitHub stars (unverified — outside what a code/repo analysis can confirm) +- ⏳ 1k+ monthly PyPI downloads (unverified — outside what a code/repo analysis can confirm) ### Q3 2026 -- ⏳ v1.6.1 shipped (multi-language README) -- ⏳ v1.7.0 shipped (`--document`, pgvector support) -- ⏳ 10+ enterprise customers ($30k–$100k MRR) -- ⏳ 100+ GitHub stars -- ⏳ 5k+ monthly PyPI downloads +- ✅ Multi-language README shipped (docs-only PR, no dedicated version bump) +- ⏳ v1.8.0 shipped (migration state tracking — code-complete, awaiting release) +- ⏳ `--document`, pgvector support (not yet started) +- ⏳ 10+ enterprise customers ($30k–$100k MRR) (unverified) +- ⏳ 100+ GitHub stars (unverified) +- ⏳ 5k+ monthly PyPI downloads (unverified) ### Q4 2026 - ⏳ Enterprise tier revenue: $50k/month ARR @@ -316,23 +320,25 @@ The `schemainspect` package (upstream: djrobstep, unmaintained) uses deprecated ## Next Steps -1. **Session 019:** Implement `--explain-drift` (7–10 days) - - New feature: `migra --from-db X --to-db Y --explain-drift` - - 80+ tests (target 270+ total) - - Release v1.6.0 +1. **Cut v1.8.0** — migration state tracking and `--apply` are both merged + to `main` and fully tested, but `pyproject.toml` is still at 1.7.2 and + nothing has been tagged/published to PyPI yet. + +2. **Native `--fail-on-destructive` CLI flag** + - Promote the GitHub Action's destructive-detection behavior into + `command.py` itself so non-Action CI users (CircleCI, GitLab, plain + scripts) get it too -2. **Session 020:** Multi-language README (3–5 days) - - README in 6 languages - - i18n links from main README - - Release v1.6.1 +3. **Reconcile `--promote`'s direction with `--apply`** before wiring the + two together — see the Planned/Backlog table above. -3. **Post v1.6.0:** Enterprise tier planning +4. **Post v1.8.0:** Enterprise tier planning - Design licensing system - Plan hosted features - Build enterprise marketing narrative --- -**Document version:** Updated June 4, 2026 (post-Session 018) +**Document version:** Updated 2026-08-08 (reconciled with actual repo/code state, incl. `--apply`) **Last updated by:** Claude (with Leo) **Repository:** https://github.com/postgresql-tools/migra diff --git a/PROJECT_PLAN2.md b/PROJECT_PLAN2.md index c08da2a..e37a31a 100644 --- a/PROJECT_PLAN2.md +++ b/PROJECT_PLAN2.md @@ -25,17 +25,50 @@ ## Completed Features -### v1.3.0 (Sessions 009–010) +### v1.3.0 (Sessions 009–010) — 2026-05-30 - `--explain`: plain-English explanation of migrations - `--from-migrations-dir`: load migrations from directory -### v1.4.0 (Sessions 011–013) +### v1.4.0 (Sessions 011–013) — 2026-06-01 - `--rollback`: generates reversal migration SQL - `--advise`: deterministic + AI performance/risk assessment - `--generate`: writes migration SQL from plain-English description -### v1.5.0 (Session 017) -- `--comment-on`: diffs COMMENT ON annotations on objects +### v1.5.0 (Session 017) — 2026-06-01 +- COMMENT ON diffing: detects added/changed/removed comments across all + object types; automatic in `--from-file` mode; feeds `--explain`/`--generate` + as schema context. (There is no separate `--comment-on` flag — this runs + by default; the plan previously described it as a flag, which was wrong.) + +### v1.6.0 — 2026-06-04 +- `--explain-drift`: AI-powered schema drift analysis between two live + databases (`--from-db`/`--to-db`), BREAKING/WARNING/INFO categorization, + live table sizes for risk assessment +- `migra/db_inspector.py`: standalone `get_remote_schema()`/`compare_schemas()` + +### v1.7.x — 2026-06-08 +- PyPI metadata fixes (PEP 621 `[project]` table, homepage/repo URLs) +- Repo moved from `migradiff/migra` to `postgresql-tools/migra` + +### Multi-language docs & licensing (merged pre-1.7.2, docs-only, no version bump) +- README translated into 6 languages (fr, de, ja, zh, hi, he) alongside English +- Licensing section (personal story + Business License ask) added to all READMEs + +### Migration state tracking — code-complete, not yet released +- `--status` / `--history`: view migration history from a `migradiff_history` table +- `--record-history` / `--env-label`: record generated migrations against a target DB +- `--promote`: chain migrations across environments (`dev:staging:prod` or + `~/.config/migradiff/environments.json` aliases) with conflict detection per hop +- `--record-rollback` / `--rollback-status`: close the audit loop on executed rollbacks +- New module `migra/history.py`; 32 new tests (`test_history.py`, + `test_command_promote.py`, `test_command_rollback_tracking.py`) +- Verified 2026-08-08: all 32 new tests pass against a real Postgres instance, + full suite (342 tests) passes, flake8/black clean, and the CLI was smoke-tested + end-to-end (`--status`, `--record-history`, `--promote`, `--record-rollback` + round-tripped correctly against live databases) +- Documented in CHANGELOG `[Unreleased]` and README's "Migration State Tracking" + section; sitting uncommitted on branch `new-feature-6-18-2026` — next release + will be **v1.8.0** ### Infrastructure (Session 018) - ✅ Production CI/CD pipeline @@ -47,80 +80,50 @@ ## Current Version -**v1.5.1** (released June 4, 2026) +**v1.7.2** (released 2026-06-08 — matches `pyproject.toml` and PyPI) -Test counts: -- Baseline: 109 tests -- v1.3.0: 142 tests (after --explain) -- v1.4.0: 175 tests (after AI suite) -- v1.5.0: 190 tests (after COMMENT ON) +Migration state tracking (above) is feature-complete and fully tested on +`new-feature-6-18-2026` but not yet merged or released. ---- - -## Free Tier Roadmap - -### Session 019: `--explain-drift` (7–10 days) - -**Feature:** Compare two **live PostgreSQL databases** and explain differences. - -```bash -migra --from-db "postgresql://user:pass@old.example.com/db" \ - --to-db "postgresql://user:pass@prod.example.com/db" \ - --explain-drift -``` - -**Output:** Human-readable explanation of schema differences with risk categorization (BREAKING/WARNING/INFO). - -**Use case:** "What changed in production? Is it safe?" - -**Tests:** 80+ new tests (target: 270+ total) - -**Release:** v1.6.0 - -**Why first:** -- Highest differentiation (competitors don't have this) -- Reuses existing schema inspection logic -- Fills gap: explain migrations vs. reality +Test counts (actual, `pytest tests -q`, excluding the separate +characterization suite): **342 passed, 2 skipped**, verified 2026-08-08. --- -### Session 020: Multi-Language README (3–5 days) - -**Feature:** README in multiple languages (i18n). +## Free Tier Roadmap -Supported languages: -- 🇬🇧 English (default) -- 🇪🇸 Spanish (README.es.md) -- 🇫🇷 French (README.fr.md) -- 🇩🇪 German (README.de.md) -- 🇯🇵 Japanese (README.ja.md) -- 🇨🇳 Chinese (README.zh.md) +### Shipped: `--explain-drift` (v1.6.0) and multi-language README -**How:** -- Maintain main README.md (English) -- Create translated versions in repo root -- Link from main README -- Use GitHub language detection for auto-redirect +Both were originally planned as "Session 019" and "Session 020" — both are +done. See "Completed Features" above for what actually shipped: +`--explain-drift` in v1.6.0 (2026-06-04), and the README translations. -**Why:** -- Global audience (PostgreSQL is international) -- Drives adoption outside English-speaking markets -- Differentiator (most tools English-only) -- SEO boost (ranked in multiple language searches) +Note: this document originally planned Spanish (README.es.md) instead of +Hindi/Hebrew for the language set — that changed during implementation. +The 6 languages that actually shipped are fr, de, ja, zh, hi, he (no +Spanish); see `PROJECT_PLAN.md` for the corrected rationale. -**Release:** v1.6.1 +### Shipped: migration state tracking and `--apply` ---- +`--status`/`--history`/`--promote`/`--record-rollback` merged into `main` +2026-08-08 (PRs #8–#10; no version bump/tag yet — still on v1.7.2). +`--apply` (execute the migration against `dburl_from` instead of only +printing it, with automatic history recording on confirmed success — see +README's "Applying Migrations" section) shipped shortly after on top of +that. Neither has been cut into a tagged release yet; `pyproject.toml` +still reads 1.7.2. Bumping to v1.8.0 and publishing is still outstanding. -### Planned Sessions (Backlog) +### Planned (Backlog) -| Session | Feature | Effort | Value | Notes | -|---------|---------|--------|-------|-------| -| 021 | `--document` | Medium-High | High | Schema documentation generation | -| 022 | pgvector support | Low | Medium | Modern Postgres vector types | -| 023 | `--suggest-indexes` | Medium | Medium | AI recommends useful indexes | -| 024 | `--dry-run --explain` | Low | Medium | Preview changes without applying | -| 025 | Multi-schema refactor | Medium | High | Better support for multiple schemas | +| Feature | Effort | Value | Notes | +|---------|--------|-------|-------| +| Native `--fail-on-destructive` flag | Low | High | Currently this behavior only exists inside the GitHub Action's `action-entrypoint.sh`. CircleCI/GitLab/pre-commit users have no equivalent without wrapping the CLI themselves. | +| `--apply` + `--promote` | Medium | Medium | `--apply` currently refuses to run when `--promote` is also given — `--promote`'s from/to direction needs to be reconciled with which database `--apply` should actually execute against before this is safe to wire up. | +| `--document` | Medium-High | High | Schema documentation generation | +| Multi-schema hardening | Medium | High | `Migration.__init__` (`migra/migra.py`) still hard-rejects `schema` + `exclude_schema` together; cross-schema FK/dependency ordering in `add_all_changes()` hasn't had a dedicated multi-tenant test pass | +| pgvector support | Low | Medium | Modern Postgres vector types — unconfirmed whether `schemainspect` already round-trips them | +| `--suggest-indexes` | Medium | Medium | AI recommends useful indexes; can reuse `AIAdvisor`'s existing table-stats extraction | +| `--dry-run` | Low | Medium | Preview what `--apply` would do without executing it | --- @@ -212,18 +215,31 @@ The `schemainspect` package (upstream: djrobstep, unmaintained) uses deprecated **Migration path (future):** Replace with maintained schema inspection library or migrate to Rust. +### docker-compose.yml referenced but missing from repo +`README.md`'s "Development Setup" section and the CHANGELOG (v1.1.0 entry) +both reference a `docker-compose.yml` for one-command local Postgres, but +no such file currently exists in the repo root. Verified 2026-08-08 by +spinning up Postgres manually (`docker run postgres:16 ...`) to run the +test suite instead. + --- ## Version History | Version | Release Date | Key Features | |---------|--------------|--------------| -| v1.3.0 | May 2026 | --explain, --from-migrations-dir | -| v1.4.0 | June 1, 2026 | --rollback, --advise, --generate | -| v1.5.0 | June 4, 2026 | COMMENT ON diffing | -| v1.5.1 | June 4, 2026 | Version bump (no feature changes) | -| v1.6.0 | TBD (Session 019) | --explain-drift | -| v1.6.1 | TBD (Session 020) | Multi-language README | +| v1.1.0 | 2026-05-29 | --from-file, --output json, pre-commit hook | +| v1.2.0 | 2026-05-29 | Column rename detection, --safe/--force-destructive, enum/composite/domain diffing | +| v1.3.0 | 2026-05-30 | --explain, --setup-ai, --from-migrations-dir | +| v1.4.0 | 2026-06-01 | --rollback, --advise, --generate | +| v1.5.0 | 2026-06-01 | COMMENT ON diffing | +| v1.6.0 | 2026-06-04 | --explain-drift | +| v1.7.1 | 2026-06-08 | PyPI homepage/repo URL fix | +| v1.7.2 | 2026-06-08 | PEP 621 metadata fix (current release) | +| v1.8.0 | TBD | Migration state tracking (--status/--history/--promote/--record-rollback) — code-complete, unreleased | + +Multi-language READMEs and the Licensing section merged as docs-only PRs +without a dedicated version bump — not tied to a CHANGELOG entry. --- @@ -267,16 +283,17 @@ The `schemainspect` package (upstream: djrobstep, unmaintained) uses deprecated ### Q2 2026 - ✅ v1.5.0 shipped - ✅ CI/CD pipeline live -- ⏳ v1.6.0 shipped (`--explain-drift`) -- ⏳ 50+ GitHub stars -- ⏳ 1k+ monthly PyPI downloads +- ✅ v1.6.0 shipped (`--explain-drift`) +- ⏳ 50+ GitHub stars (unverified — outside what a code/repo analysis can confirm) +- ⏳ 1k+ monthly PyPI downloads (unverified — outside what a code/repo analysis can confirm) ### Q3 2026 -- ⏳ v1.6.1 shipped (multi-language README) -- ⏳ v1.7.0 shipped (`--document`, pgvector support) -- ⏳ 10+ enterprise customers ($30k–$100k MRR) -- ⏳ 100+ GitHub stars -- ⏳ 5k+ monthly PyPI downloads +- ✅ Multi-language README shipped (docs-only PR, no dedicated version bump) +- ⏳ v1.8.0 shipped (migration state tracking — code-complete, awaiting release) +- ⏳ `--document`, pgvector support (not yet started) +- ⏳ 10+ enterprise customers ($30k–$100k MRR) (unverified) +- ⏳ 100+ GitHub stars (unverified) +- ⏳ 5k+ monthly PyPI downloads (unverified) ### Q4 2026 - ⏳ Enterprise tier revenue: $50k/month ARR @@ -306,23 +323,25 @@ The `schemainspect` package (upstream: djrobstep, unmaintained) uses deprecated ## Next Steps -1. **Session 019:** Implement `--explain-drift` (7–10 days) - - New feature: `migra --from-db X --to-db Y --explain-drift` - - 80+ tests (target 270+ total) - - Release v1.6.0 +1. **Cut v1.8.0** — migration state tracking and `--apply` are both merged + to `main` and fully tested, but `pyproject.toml` is still at 1.7.2 and + nothing has been tagged/published to PyPI yet. + +2. **Native `--fail-on-destructive` CLI flag** + - Promote the GitHub Action's destructive-detection behavior into + `command.py` itself so non-Action CI users (CircleCI, GitLab, plain + scripts) get it too -2. **Session 020:** Multi-language README (3–5 days) - - README in 6 languages - - i18n links from main README - - Release v1.6.1 +3. **Reconcile `--promote`'s direction with `--apply`** before wiring the + two together — see the Planned/Backlog table above. -3. **Post v1.6.0:** Enterprise tier planning +4. **Post v1.8.0:** Enterprise tier planning - Design licensing system - Plan hosted features - Build enterprise marketing narrative --- -**Document version:** Updated June 4, 2026 (post-Session 018) +**Document version:** Updated 2026-08-08 (reconciled with actual repo/code state, incl. `--apply`) **Last updated by:** Claude (with Leo) **Repository:** https://github.com/postgresql-tools/migra diff --git a/README.md b/README.md index 03faa5d..f867688 100644 --- a/README.md +++ b/README.md @@ -269,6 +269,174 @@ Requires pip install migradiff[ai] and an Anthropic API key. --- +## Migration State Tracking + +MigraDiff now includes a migration state tracking system that records +generated migrations in a `migradiff_history` table in your target +databases. This is the foundation for multi-environment promotion and +rollback tracking. + +### ⚠ Known Limitation + +Without `--apply` (below), `migradiff_history` only records that a +migration was **generated/reviewed** for a target database, not that the +SQL was necessarily *executed*. If you generate SQL and pipe it to `psql` +yourself, calling `migra --record-history` only tells MigraDiff "this +migration was proposed" — it has no way to know whether your `psql` step +actually succeeded. Use `--apply` when you want MigraDiff itself to run +the migration and only record history on confirmed success. + +Be explicit about this in your pipeline so you don't assume false +guarantees about whether a migration has actually been applied. + +### View Migration History + +```bash +# Show recent history (last 10 entries) +migra --status postgres://db_target + +# Show full history +migra --history postgres://db_target +``` + +Output includes: +- **Hash** (short, 8 chars) — identifies the migration +- **Name** — optional human label +- **Applied At** — when the migration was generated/recorded +- **Applied By** — database user who recorded it +- **Rollback** — rollback status (ROLLED BACK, ROLLBACK FAILED, or -) + +JSON output is supported: + +```bash +migra --status postgres://db_target --output json +``` + +### Record Migration History + +After generating a migration diff, record it in the target database: + +```bash +migra --record-history postgres://db_target_a postgres://db_target_b +``` + +This calls `ensure_history_table()` and inserts a row with the +generated SQL hash, the forward SQL, and an optional environment label. + +Use `--env-label` to tag the entry: + +```bash +migra --record-history --env-label staging postgres://db_a postgres://db_b +``` + +### Applying Migrations (`--apply`) + +`--apply` executes the generated migration directly against `dburl_from` +(the first positional argument — "the database you want to migrate", same +database the README's basic usage example pipes to `psql`) instead of only +printing it: + +```bash +migra --apply postgres://db_production postgres://db_branch +``` + +On success, MigraDiff automatically records the migration in +`dburl_from`'s `migradiff_history` table — you don't need to also pass +`--record-history`. If any statement fails, the whole migration is rolled +back as a single transaction (nothing is partially applied) and **nothing +is recorded**, since it wasn't actually applied. The command exits non-zero +(exit code 4) so pipelines can detect the failure. + +```bash +migra --apply --env-label prod postgres://db_production postgres://db_branch +``` + +`--apply` respects the same safety gates as everything else: destructive +statements are blocked unless `--force-destructive` (or `--unsafe`) is +given, and the block happens *before* anything is executed. + +`--apply` is not supported with `--from-file` (there's no live database to +apply to — the schema files get loaded into temporary throwaway databases) +or with `--promote` (not implemented yet). + +### Multi-Environment Promotion (`--promote`) + +`--promote` generates migrations along a chain of environments, with +conflict detection at each hop: + +```bash +migra --unsafe --promote dev:staging:prod +``` + +This: +1. Connects to dev, staging, and prod +2. Verifies staging's `migradiff_history` is a subset of dev's + (no unknown migrations in staging) +3. Generates the `dev → staging` diff and prints it +4. Generates the `staging → prod` diff and prints it +5. If `--record-history` is also given, records each hop against + the target environment's history table + +Environment names are resolved from +`~/.config/migradiff/environments.json`: + +```json +{ + "dev": "postgresql://localhost/dev", + "staging": "postgresql://staging.example.com/db", + "prod": "postgresql://prod.example.com/db" +} +``` + +You can also mix aliases with literal URLs: + +```bash +migra --promote dev:postgresql://custom-host/db:prod +``` + +Conflict example — if staging has a migration that dev doesn't: + +``` +MigraDiff: CONFLICT in promotion chain postgres://dev → postgres://staging + postgres://staging has migrations not found in postgres://dev: + abc1234deadbeef + Resolve the conflict before continuing the chain. +``` + +If two adjacent environments already match: + +``` +No changes needed: postgres://dev → postgres://staging +``` + +`--promote` composes with `--explain`, `--rollback`, `--advise`, +`--force-destructive`, and `--output json` the same way a normal +two-argument diff does. + +### Rollback Tracking (`--record-rollback`) + +After executing a rollback, record it to close the audit loop: + +```bash +migra --record-rollback postgres://db_target +``` + +You can also mark a rollback as failed: + +```bash +migra --record-rollback abc1234 --rollback-status rollback_failed postgres://db_target +``` + +The `--status` and `--history` views then show the rollback state: + +``` +Hash Name Applied At Applied By Rollback +----------- -------------------- -------------------------- ---------------- -------------------- +abc1234 v001_create_users 2026-06-18T12:00:00Z deploy_bot ROLLED BACK 2026-06-18T13:00:00Z +``` + +--- + ## Development Setup The test suite requires a running PostgreSQL instance. The easiest diff --git a/decisions.md b/decisions.md new file mode 100644 index 0000000..1730457 --- /dev/null +++ b/decisions.md @@ -0,0 +1,14 @@ +# Decisions + +Log of notable technical/product decisions for MigraDiff, newest first. One entry per decision: +what was decided, why, and any alternatives considered. + + diff --git a/migra/command.py b/migra/command.py index 44a9692..b24e7ac 100644 --- a/migra/command.py +++ b/migra/command.py @@ -276,6 +276,45 @@ def _format_destructive_summary(statements): return "\n".join(lines) +def _require_ai(out, err, cli_key=None): + """Resolve an Anthropic API key, printing standard help text on failure. + + Returns the resolved key string or ``None`` on error. + """ + try: + import anthropic # noqa: F401 + except ImportError: + print( + "MigraDiff: AI features require the AI extras.", + file=err, + ) + print("Install with: pip install migradiff[ai]", file=err) + return None + + from .ai_explain import resolve_api_key + + api_key = resolve_api_key(cli_key=cli_key) + if not api_key: + print( + "MigraDiff: An Anthropic API key is required.", + file=err, + ) + print(file=err) + print("Set it up once with:", file=err) + print(" migra --setup-ai", file=err) + print(file=err) + print("Or set the environment variable:", file=err) + print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) + print(file=err) + print( + "Get an API key at: https://console.anthropic.com", + file=err, + ) + return None + + return api_key + + def _migration_sort_key(filename): """Extract sort key from a migration filename.""" stem = filename.lower().replace(".sql", "") @@ -553,6 +592,66 @@ def parse_args(args): help="Generate a PostgreSQL migration from a plain English description." " Optionally provide the description as an argument.", ) + parser.add_argument( + "--apply", + dest="apply", + action="store_true", + default=False, + help="Execute the generated migration against dburl_from (the database" + " being migrated), instead of only printing it. On success, automatically" + " records the migration in dburl_from's migradiff_history table — no need" + " to also pass --record-history. Not supported with --from-file or --promote.", + ) + parser.add_argument( + "--record-history", + dest="record_history", + action="store_true", + default=False, + help="Record the generated migration in the target database's migradiff_history table", + ) + parser.add_argument( + "--status", + dest="status", + nargs="?", + const=True, + default=None, + help="Show recent migration history from a database (default limit: 10)", + ) + parser.add_argument( + "--history", + dest="history", + nargs="?", + const=True, + default=None, + help="Show full migration history from a database (no limit)", + ) + parser.add_argument( + "--promote", + dest="promote", + default=None, + help="Generate migrations along a colon-separated chain of environments" + " (e.g. dev:staging:prod). Supports database URLs or environment aliases.", + ) + parser.add_argument( + "--record-rollback", + dest="record_rollback", + default=None, + help="Record a rollback in the migradiff_history table." + " Accepts a migration hash or a rollback SQL file path.", + ) + parser.add_argument( + "--env-label", + dest="env_label", + default=None, + help='Environment label for --record-history (e.g. "dev", "staging", "prod")', + ) + parser.add_argument( + "--rollback-status", + dest="rollback_status", + default="rolled_back", + choices=["rolled_back", "rollback_failed"], + help="Status to record when using --record-rollback (default: rolled_back)", + ) parser.add_argument( "dburl_from", nargs="?", help="The database you want to migrate." ) @@ -562,57 +661,335 @@ def parse_args(args): return parser.parse_args(args) +def _resolve_env_aliases_part(part): + """Resolve a single promote chain segment to a database URL. + Returns the part as-is if it looks like a URL (contains ://), + otherwise resolves it as an environment alias. + """ + if "://" in part: + return part + return _resolve_env_alias(part) + + +def _resolve_env_alias(key): + """Resolve a single environment alias to a database URL.""" + env_file = os.environ.get( + "MIGRADIFF_ENV_FILE", + os.path.join( + os.path.expanduser("~"), ".config", "migradiff", "environments.json" + ), + ) + if os.path.exists(env_file): + with open(env_file) as f: + config = json.load(f) + if key in config: + return config[key] + raise ValueError( + "MigraDiff: Unknown environment alias '{}'.\n" + "Define it in {}".format(key, env_file) + ) + + +def _apply_migration(dburl_from, statements, sql_output, rollback_sql, env_label): + """Execute `statements` against dburl_from in a single transaction, then + record the migration in dburl_from's migradiff_history table. + + Returns a dict describing the outcome; never raises. Execution and + history-recording are separate try/excepts, since a failure to record + history after a successful apply is a (loud) warning, not an apply + failure — the schema change already happened. + """ + from sqlbag import S as _S + + result = { + "applied": False, + "target": redact_credentials(dburl_from), + "statement_count": len(statements), + "error": None, + "history_recorded": False, + "history_error": None, + } + + try: + with _S(dburl_from) as s: + for stmt in statements: + s.execute(text(stmt)) + except Exception as e: + result["error"] = str(e) + return result + + result["applied"] = True + + from .history import compute_migration_hash, ensure_history_table, record_applied + + try: + with _S(dburl_from) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash=compute_migration_hash(sql_output), + forward_sql=sql_output, + rollback_sql=rollback_sql, + environment_label=env_label, + ) + result["history_recorded"] = True + except Exception as e: + result["history_error"] = str(e) + + return result + + def run(args, out=None, err=None): if not out: out = sys.stdout # pragma: no cover if not err: err = sys.stderr # pragma: no cover - if args.setup_ai: - from .ai_explain import setup_ai_interactive + if args.apply and args.from_file: + print( + "MigraDiff: --apply is not supported with --from-file" + " (there is no live database to apply to — dburl_from is a" + " temporary database loaded from the dump file).", + file=err, + ) + return 1 - return setup_ai_interactive(out, err) + if args.apply and args.promote: + print( + "MigraDiff: --apply is not supported with --promote yet.", + file=err, + ) + return 1 - # Explain drift mode - if args.explain_drift: - if not args.from_db or not args.to_db: + # --status / --history: display migration history + if args.status is not None or args.history is not None: + from .history import ensure_history_table, get_history + + dburl = args.status if isinstance(args.status, str) else None + if dburl is None: + dburl = args.history if isinstance(args.history, str) else None + if dburl is None: + dburl = args.dburl_from + if not dburl: print( - "ERROR: --explain-drift requires both --from-db and --to-db.", + "MigraDiff: --status/--history requires a database URL.", file=err, ) return 1 - try: - import anthropic # noqa: F401, F811 - except ImportError: + limit = None if args.history is not None else 10 + + from sqlbag import S + + with S(dburl) as s: + ensure_history_table(s) + entries = get_history(s, limit=limit) + + if args.output == "json": + import json as json_mod + + print(json_mod.dumps(entries, indent=2), file=out) + else: + if not entries: + print("No migration history found.", file=out) + return 0 + header = "{:<12} {:<20} {:<26} {:<16} {:<20}".format( + "Hash", "Name", "Applied At", "Applied By", "Rollback" + ) + print(header, file=out) + print("-" * len(header), file=out) + for e in entries: + short_hash = e["migration_hash"][:8] if e["migration_hash"] else "" + name = e["migration_name"] or "" + applied_at = e["applied_at"] or "" + applied_by = e["applied_by"] or "" + rb_status = e["rollback_status"] + rb_annotation = "" + if rb_status == "rolled_back": + rb_annotation = "ROLLED BACK" + if e.get("rolled_back_at"): + rb_annotation += " " + e["rolled_back_at"] + elif rb_status == "rollback_failed": + rb_annotation = "ROLLBACK FAILED" + else: + rb_annotation = "-" + print( + "{:<12} {:<20} {:<26} {:<16} {:<20}".format( + short_hash, + name[:18], + applied_at[:24], + applied_by[:14], + rb_annotation[:18], + ), + file=out, + ) + return 0 + + # --record-rollback: record a rollback in the history table + if args.record_rollback: + from .history import ( + compute_migration_hash, + ensure_history_table, + record_rollback, + ) + + value = args.record_rollback + if os.path.exists(value) and value.endswith(".sql"): + with open(value) as f: + migration_hash = compute_migration_hash(f.read()) + else: + migration_hash = value.strip() + + dburl = args.dburl_from + if not dburl: print( - "MigraDiff: --explain-drift requires the AI extras.", + "MigraDiff: --record-rollback requires a database URL.", file=err, ) - print("Install with: pip install migradiff[ai]", file=err) return 1 - from .ai_explain import resolve_api_key + from sqlbag import S - api_key = resolve_api_key(cli_key=args.api_key) - if not api_key: + try: + with S(dburl) as s: + ensure_history_table(s) + record_rollback( + s, + migration_hash, + status=args.rollback_status, + ) print( - "MigraDiff: --explain-drift requires an Anthropic API key.", + "MigraDiff: Rollback recorded for hash {}".format(migration_hash[:12]), + file=out, + ) + return 0 + except ValueError as e: + print(str(e), file=err) + return 1 + + # --promote: multi-environment promotion chain + if args.promote: + from .history import ensure_history_table, get_applied_hashes, record_applied + + promote_parts = re.split(r":(?!//)", args.promote) + if len(promote_parts) < 2: + print( + "MigraDiff: --promote requires at least 2 environments" + " (colon-separated).", file=err, ) - print(file=err) - print("Set it up once with:", file=err) - print(" migra --setup-ai", file=err) - print(file=err) - print("Or set the environment variable:", file=err) - print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) - print(file=err) + return 1 + + try: + chain = [_resolve_env_aliases_part(p) for p in promote_parts] + except ValueError as e: + print(str(e), file=err) + return 1 + + from sqlbag import S + + for i in range(len(chain) - 1): + from_url = chain[i] + to_url = chain[i + 1] + from_label = redact_credentials(from_url) + to_label = redact_credentials(to_url) + + # Conflict check: ensure to_env has no hashes from_env doesn't have + try: + with S(from_url) as s_from: + ensure_history_table(s_from) + from_hashes = get_applied_hashes(s_from) + with S(to_url) as s_to: + ensure_history_table(s_to) + to_hashes = get_applied_hashes(s_to) + except Exception as e: + print( + "MigraDiff: Could not check migration state for" + " {} -> {}: {}".format(from_label, to_label, e), + file=err, + ) + return 1 + + stray_hashes = to_hashes - from_hashes + if stray_hashes: + print( + "MigraDiff: CONFLICT in promotion chain" + " {} -> {}".format(from_label, to_label), + file=err, + ) + print( + " {} has migrations not found in {}:".format(to_label, from_label), + file=err, + ) + for h in sorted(stray_hashes): + print(" {}".format(h[:12]), file=err) + print( + " Resolve the conflict before continuing the chain.", + file=err, + ) + return 1 + + # Run the diff for this hop + print("=== {} -> {} ===".format(from_label, to_label), file=out) + + sub_args = argparse.Namespace(**vars(args)) + sub_args.dburl_from = from_url + sub_args.dburl_target = to_url + sub_args._original_from = from_url + sub_args._original_target = to_url + sub_args.from_migrations_dir = None + sub_args.from_file = False + + hop_status = _run_inner(sub_args, out, err) + + if hop_status == 0 and not sub_args._has_statements: + print( + "No changes needed: {} -> {}".format(from_label, to_label), + file=out, + ) + continue + + if hop_status == 1 or hop_status == 3: + return hop_status + + # Record history if --record-history + if args.record_history and hasattr(sub_args, "_generated_sql"): + from .history import compute_migration_hash + + sql_text = sub_args._generated_sql + if sql_text and sql_text.strip(): + fwd_hash = compute_migration_hash(sql_text) + rollback_sql_text = getattr(sub_args, "_generated_rollback", None) + with S(to_url) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash=fwd_hash, + forward_sql=sql_text, + rollback_sql=rollback_sql_text, + environment_label=args.env_label, + ) + + return 0 + + if args.setup_ai: + from .ai_explain import setup_ai_interactive + + return setup_ai_interactive(out, err) + + # Explain drift mode + if args.explain_drift: + if not args.from_db or not args.to_db: print( - "Get an API key at: https://console.anthropic.com", + "ERROR: --explain-drift requires both --from-db and --to-db.", file=err, ) return 1 + api_key = _require_ai(out, err, args.api_key) + if not api_key: + return 1 + try: from .db_inspector import compare_schemas, get_remote_schema @@ -657,7 +1034,6 @@ def run(args, out=None, err=None): check_safety_rules, extract_relevant_schema, parse_schema_file_for_tables, - resolve_api_key, redact_api_key, classify_statement_risk, ) @@ -672,33 +1048,8 @@ def run(args, out=None, err=None): ) return 1 - try: - import anthropic # noqa: F401, F811 - except ImportError: - print( - "MigraDiff: --generate requires the AI extras.", - file=err, - ) - print("Install with: pip install migradiff[ai]", file=err) - return 1 - - api_key = resolve_api_key(cli_key=args.api_key) + api_key = _require_ai(out, err, args.api_key) if not api_key: - print( - "MigraDiff: --generate requires an Anthropic API key.", - file=err, - ) - print(file=err) - print("Set it up once with:", file=err) - print(" migra --setup-ai", file=err) - print(file=err) - print("Or set the environment variable:", file=err) - print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) - print(file=err) - print( - "Get an API key at: https://console.anthropic.com", - file=err, - ) return 1 # Safety check first @@ -800,21 +1151,8 @@ def run(args, out=None, err=None): ): filepath = args.dburl_from if os.path.exists(filepath) and filepath.endswith(".sql"): - from .ai_explain import resolve_api_key - - api_key = resolve_api_key(cli_key=args.api_key) + api_key = _require_ai(out, err, args.api_key) if not api_key: - print( - "MigraDiff: --advise requires an Anthropic API key.", - file=err, - ) - print(file=err) - print("Set it up once with:", file=err) - print(" migra --setup-ai", file=err) - print(file=err) - print("Or set the environment variable:", file=err) - print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) - print(file=err) return 1 from .ai_explain import generate_file_advisory @@ -972,35 +1310,10 @@ def _run_inner(args, out=None, err=None): rollback_result = None advisory_result = None if args.explain: - from .ai_explain import AIExplainer, resolve_api_key, redact_api_key + from .ai_explain import AIExplainer, redact_api_key - try: - import anthropic # noqa: F401, F811, F811 - except ImportError: - print( - "MigraDiff: --explain requires the AI extras.", - file=err, - ) - print("Install with: pip install migradiff[ai]", file=err) - return 1 - - api_key = resolve_api_key(cli_key=args.api_key) + api_key = _require_ai(out, err, args.api_key) if not api_key: - print( - "MigraDiff: --explain requires an Anthropic API key.", - file=err, - ) - print(file=err) - print("Set it up once with:", file=err) - print(" migra --setup-ai", file=err) - print(file=err) - print("Or set the environment variable:", file=err) - print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) - print(file=err) - print( - "Get an API key at: https://console.anthropic.com", - file=err, - ) return 1 if statements: @@ -1020,35 +1333,10 @@ def _run_inner(args, out=None, err=None): # AI rollback generation if args.rollback: - from .ai_explain import AIRollback, resolve_api_key, redact_api_key + from .ai_explain import AIRollback, redact_api_key - try: - import anthropic # noqa: F401, F811, F811 - except ImportError: - print( - "MigraDiff: --rollback requires the AI extras.", - file=err, - ) - print("Install with: pip install migradiff[ai]", file=err) - return 1 - - api_key = resolve_api_key(cli_key=args.api_key) + api_key = _require_ai(out, err, args.api_key) if not api_key: - print( - "MigraDiff: --rollback requires an Anthropic API key.", - file=err, - ) - print(file=err) - print("Set it up once with:", file=err) - print(" migra --setup-ai", file=err) - print(file=err) - print("Or set the environment variable:", file=err) - print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) - print(file=err) - print( - "Get an API key at: https://console.anthropic.com", - file=err, - ) return 1 if statements: @@ -1081,35 +1369,10 @@ def _run_inner(args, out=None, err=None): # AI advisory generation if args.advise: - from .ai_explain import AIAdvisor, resolve_api_key, redact_api_key + from .ai_explain import AIAdvisor, redact_api_key - try: - import anthropic # noqa: F401, F811, F811 - except ImportError: - print( - "MigraDiff: --advise requires the AI extras.", - file=err, - ) - print("Install with: pip install migradiff[ai]", file=err) - return 1 - - api_key = resolve_api_key(cli_key=args.api_key) + api_key = _require_ai(out, err, args.api_key) if not api_key: - print( - "MigraDiff: --advise requires an Anthropic API key.", - file=err, - ) - print(file=err) - print("Set it up once with:", file=err) - print(" migra --setup-ai", file=err) - print(file=err) - print("Or set the environment variable:", file=err) - print(" export ANTHROPIC_API_KEY=sk-ant-...", file=err) - print(file=err) - print( - "Get an API key at: https://console.anthropic.com", - file=err, - ) return 1 if statements: @@ -1154,8 +1417,19 @@ def _run_inner(args, out=None, err=None): file=err, ) + sql_output = None + apply_result = None try: if statements: + sql_output = modified_statements.sql + if args.apply: + apply_result = _apply_migration( + args.dburl_from, + modified_statements, + sql_output, + rollback_result["text"] if rollback_result else None, + args.env_label, + ) if args.output == "json": json_out = format_json_output( statements, @@ -1193,6 +1467,12 @@ def _run_inner(args, out=None, err=None): "generated_at": advisory_result.get("generated_at", ""), } json_out = json_mod.dumps(data, indent=2) + if apply_result is not None: + import json as json_mod + + data = json_mod.loads(json_out) + data["apply"] = apply_result + json_out = json_mod.dumps(data, indent=2) print(json_out, file=out) elif args.force_utf8: print(modified_statements.sql.encode("utf8"), file=out) @@ -1209,6 +1489,32 @@ def _run_inner(args, out=None, err=None): print(file=out) print("--- Performance Advisory ---", file=out) print(advisory_result["text"], file=out) + if apply_result is not None and args.output != "json": + print(file=out) + if apply_result["applied"]: + print( + "-- Applied {} statement(s) to {}".format( + apply_result["statement_count"], + apply_result["target"], + ), + file=out, + ) + if not apply_result["history_recorded"]: + print( + "-- WARNING: applied successfully, but failed to" + " record migration history: {}".format( + apply_result["history_error"] + ), + file=err, + ) + else: + print( + "-- ERROR: --apply failed, no changes were committed" + " to {}: {}".format( + apply_result["target"], apply_result["error"] + ), + file=err, + ) elif args.output == "json": json_out = format_json_output( statements, @@ -1265,6 +1571,44 @@ def _run_inner(args, out=None, err=None): ) return 3 + # Capture metadata for --promote and --record-history + args._has_statements = bool(statements) + args._generated_sql = sql_output if statements else None + args._generated_rollback = rollback_result["text"] if rollback_result else None + + # --record-history: record the generated migration. Skipped when --apply + # already ran, since _apply_migration() already recorded it against + # dburl_from (the database that was actually changed) — recording it + # again here against dburl_target would be a second, misleading entry. + if args.record_history and not args.apply and statements and args.dburl_target: + from .history import ( + compute_migration_hash, + ensure_history_table, + record_applied, + ) + from sqlbag import S as _S + + fwd_hash = compute_migration_hash(sql_output) + rollback_sql_text = args._generated_rollback + try: + with _S(args.dburl_target) as _s: + ensure_history_table(_s) + record_applied( + _s, + migration_hash=fwd_hash, + forward_sql=sql_output, + rollback_sql=rollback_sql_text, + environment_label=args.env_label, + ) + except Exception as e: + print( + "MigraDiff: Failed to record migration history: {}".format(e), + file=err, + ) + + if apply_result is not None and not apply_result["applied"]: + return 4 + if not statements: return 0 diff --git a/migra/history.py b/migra/history.py new file mode 100644 index 0000000..f8618e8 --- /dev/null +++ b/migra/history.py @@ -0,0 +1,191 @@ +from __future__ import unicode_literals + +import hashlib +import re + +from sqlalchemy import text + +HISTORY_TABLE = "migradiff_history" + + +def compute_migration_hash(sql_text): + normalized = _normalize_sql(sql_text) + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +def _normalize_sql(sql_text): + lines = sql_text.split("\n") + stripped_lines = [line.rstrip() for line in lines] + result = "\n".join(stripped_lines) + result = re.sub(r"\n{3,}", "\n\n", result) + result = result.strip() + return result + + +CREATE_TABLE_SQL = """ +CREATE TABLE IF NOT EXISTS {table} ( + id BIGSERIAL PRIMARY KEY, + migration_hash TEXT NOT NULL, + migration_name TEXT, + checksum TEXT NOT NULL, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now(), + applied_by TEXT, + forward_sql TEXT NOT NULL, + rollback_sql TEXT, + rollback_status TEXT NOT NULL DEFAULT 'not_attempted', + rolled_back_at TIMESTAMPTZ, + rolled_back_by TEXT, + environment_label TEXT, + source TEXT NOT NULL DEFAULT 'cli' +); + +CREATE INDEX IF NOT EXISTS idx_{table}_migration_hash ON {table} (migration_hash); +""".format(table=HISTORY_TABLE) + + +def ensure_history_table(session): + session.execute(text(CREATE_TABLE_SQL)) + session.commit() + + +def _current_user(session): + row = session.execute(text("SELECT current_user")).scalar() + return row + + +def record_applied( + session, + *, + migration_hash, + migration_name=None, + forward_sql, + rollback_sql=None, + environment_label=None, + applied_by=None, +): + if applied_by is None: + applied_by = _current_user(session) + session.execute( + text(""" + INSERT INTO {table} + (migration_hash, migration_name, checksum, forward_sql, + rollback_sql, environment_label, applied_by) + VALUES + (:migration_hash, :migration_name, :checksum, :forward_sql, + :rollback_sql, :environment_label, :applied_by) + """.format(table=HISTORY_TABLE)), + { + "migration_hash": migration_hash, + "migration_name": migration_name, + "checksum": migration_hash, + "forward_sql": forward_sql, + "rollback_sql": rollback_sql, + "environment_label": environment_label, + "applied_by": applied_by, + }, + ) + session.commit() + row = session.execute(text("SELECT lastval()")).scalar() + return row + + +def record_rollback( + session, + migration_hash, + *, + status="rolled_back", + rolled_back_by=None, +): + if rolled_back_by is None: + rolled_back_by = _current_user(session) + + row = session.execute( + text(""" + SELECT id FROM {table} + WHERE migration_hash = :migration_hash + ORDER BY applied_at DESC + LIMIT 1 + FOR UPDATE + """.format(table=HISTORY_TABLE)), + {"migration_hash": migration_hash}, + ).fetchone() + + if not row: + raise ValueError( + "MigraDiff: No history entry found for hash {}".format(migration_hash) + ) + + session.execute( + text(""" + UPDATE {table} + SET rollback_status = :status, + rolled_back_at = now(), + rolled_back_by = :rolled_back_by + WHERE id = :id + """.format(table=HISTORY_TABLE)), + { + "status": status, + "rolled_back_by": rolled_back_by, + "id": row[0], + }, + ) + session.commit() + + +def get_applied_hashes(session): + rows = session.execute(text(""" + SELECT DISTINCT migration_hash FROM {table} + """.format(table=HISTORY_TABLE))).fetchall() + return {row[0] for row in rows} + + +def get_history(session, environment_label=None, limit=None): + conditions = [] + params = {} + if environment_label is not None: + conditions.append("environment_label = :environment_label") + params["environment_label"] = environment_label + + where_clause = "" + if conditions: + where_clause = "WHERE " + " AND ".join(conditions) + + limit_clause = "" + if limit is not None: + limit_clause = "LIMIT :limit" + params["limit"] = limit + + rows = session.execute( + text(""" + SELECT id, migration_hash, migration_name, checksum, + applied_at, applied_by, forward_sql, rollback_sql, + rollback_status, rolled_back_at, rolled_back_by, + environment_label, source + FROM {table} + {where} + ORDER BY applied_at DESC + {limit} + """.format(table=HISTORY_TABLE, where=where_clause, limit=limit_clause)), + params, + ).fetchall() + + result = [] + for row in rows: + result.append( + { + "id": row[0], + "migration_hash": row[1], + "migration_name": row[2], + "checksum": row[3], + "applied_at": str(row[4]) if row[4] else None, + "applied_by": row[5], + "forward_sql": row[6], + "rollback_sql": row[7], + "rollback_status": row[8], + "rolled_back_at": str(row[9]) if row[9] else None, + "rolled_back_by": row[10], + "environment_label": row[11], + "source": row[12], + } + ) + return result diff --git a/migra/migra.py b/migra/migra.py index f7a5932..1150692 100644 --- a/migra/migra.py +++ b/migra/migra.py @@ -4,6 +4,27 @@ from .changes import Changes from .statements import Statements +from .util import filter_inspector_schemas, parse_schema_arg + + +def _get_inspector(x, schema=None, exclude_schema=None): + """Like schemainspect's get_inspector(), but also supports a + comma-separated `schema`/`exclude_schema` (2+ names). schemainspect's + own one_schema()/filter_schema() only ever compare against a single + schema name, so a raw multi-name string would silently match nothing. + Single-schema (or no-schema) calls are passed straight through + unchanged. + """ + schema_list = parse_schema_arg(schema) + exclude_list = parse_schema_arg(exclude_schema) + + if schema_list and len(schema_list) > 1: + return filter_inspector_schemas(get_inspector(x), schema_list) + + if exclude_list and len(exclude_list) > 1: + return filter_inspector_schemas(get_inspector(x), None, exclude=exclude_list) + + return get_inspector(x, schema=schema, exclude_schema=exclude_schema) class Migration(object): @@ -28,7 +49,7 @@ def __init__( if isinstance(x_from, DBInspector): self.changes.i_from = x_from else: - self.changes.i_from = get_inspector( + self.changes.i_from = _get_inspector( x_from, schema=schema, exclude_schema=exclude_schema ) if x_from: @@ -36,7 +57,7 @@ def __init__( if isinstance(x_target, DBInspector): self.changes.i_target = x_target else: - self.changes.i_target = get_inspector( + self.changes.i_target = _get_inspector( x_target, schema=schema, exclude_schema=exclude_schema ) if x_target: @@ -45,12 +66,12 @@ def __init__( self.changes.ignore_extension_versions = ignore_extension_versions def inspect_from(self): - self.changes.i_from = get_inspector( + self.changes.i_from = _get_inspector( self.s_from, schema=self.schema, exclude_schema=self.exclude_schema ) def inspect_target(self): - self.changes.i_target = get_inspector( + self.changes.i_target = _get_inspector( self.s_target, schema=self.schema, exclude_schema=self.exclude_schema ) @@ -62,7 +83,7 @@ def apply(self): for stmt in self.statements: raw_execute(self.s_from, stmt) - self.changes.i_from = get_inspector( + self.changes.i_from = _get_inspector( self.s_from, schema=self.schema, exclude_schema=self.exclude_schema ) safety_on = self.statements.safe diff --git a/migra/util.py b/migra/util.py index 4d0c788..952199c 100644 --- a/migra/util.py +++ b/migra/util.py @@ -3,6 +3,55 @@ from collections import OrderedDict as od +def parse_schema_arg(schema): + """Split a comma-separated --schema/--exclude_schema value into a list + of trimmed, non-empty schema names. Returns None for a falsy input.""" + if not schema: + return None + parts = [s.strip() for s in schema.split(",")] + return [p for p in parts if p] + + +def filter_inspector_schemas(inspector, schemas=None, exclude=None): + """Filter an already-built (unfiltered) inspector down to only the given + schemas, or to exclude the given schemas. Needed for the multi-schema + case (2+ comma-separated names), since schemainspect's own + `filter_schema`/`one_schema` only accepts a single schema name and + compares it for exact equality -- passing it a raw "a,b" string silently + matches nothing. + + Mutates and returns `inspector`. Reuses schemainspect's own PROPS list + (the same one `DBInspector.filter_schema` iterates over) so this stays + in sync with whatever object types schemainspect tracks, rather than + maintaining a second, driftable copy of that list here. + """ + if not schemas and not exclude: + return inspector + if not hasattr(inspector, "filter_schema"): + return inspector + + from schemainspect.pg.obj import PROPS + + schema_set = set(schemas) if schemas else None + exclude_set = set(exclude) if exclude else None + for prop in PROPS.split(): + att = getattr(inspector, prop, {}) + if schema_set: + filtered = { + k: v + for k, v in att.items() + if hasattr(v, "schema") and v.schema in schema_set + } + else: + filtered = { + k: v + for k, v in att.items() + if hasattr(v, "schema") and v.schema not in exclude_set + } + setattr(inspector, prop, filtered) + return inspector + + def differences(a, b, add_dependencies_for_modifications=True): a_keys = set(a.keys()) b_keys = set(b.keys()) diff --git a/status.md b/status.md new file mode 100644 index 0000000..664da83 --- /dev/null +++ b/status.md @@ -0,0 +1,32 @@ +# Status + +Snapshot of current work-in-progress on this branch. Update as work progresses; this is not a +changelog (see CHANGELOG.md for that). + +## Current branch: `fix/multi-schema-filtering` (off `main` at `a762dec`) + +Migration state tracking, the CI-trigger/SQLAlchemy-2.x fixes, and `--apply` are already merged +to `main` (PRs #8–#11) — see git log, not this file, for that history. + +**In progress / uncommitted:** +- Modified: `CHANGELOG.md`, `migra/migra.py`, `migra/util.py`, `tests/test_migra.py` +- New: `tests/FIXTURES/multischema/`, `tests/FIXTURES/exclude_multischema/` + +**The bug:** `--schema public,reporting` was documented as supported but silently produced an +empty diff — the raw comma-joined string was passed straight to `schemainspect`, which compares +it for *exact* equality against each object's schema name. Single-schema and no-schema calls were +never affected. + +**The fix:** `migra/util.py` gains `parse_schema_arg()`/`filter_inspector_schemas()` (the latter +reuses schemainspect's own `PROPS` list rather than duplicating it, so it stays in sync with +whatever object types schemainspect tracks). `migra/migra.py` gains `_get_inspector()`, wired into +all 5 of `Migration`'s inspector-construction call sites; single/no-schema calls pass straight +through unchanged, only 2+ comma-separated names take the new post-filter path. + +**Verified 2026-08-08:** new tests (`test_multischema`, `test_multischema_whitespace_tolerant`, +`test_exclude_multischema`) each include a third, unlisted schema in the fixture to prove +filtering actually excludes it, not just that it happens to work with one schema; also exercises +`Migration.apply()` directly (via `do_fixture_test`'s second half), not just the CLI path. Full +suite: 356 passed / 2 skipped, no regressions. flake8/black clean. + +**Next steps:** commit, push, open PR. diff --git a/tests/FIXTURES/exclude_multischema/a.sql b/tests/FIXTURES/exclude_multischema/a.sql new file mode 100644 index 0000000..072e678 --- /dev/null +++ b/tests/FIXTURES/exclude_multischema/a.sql @@ -0,0 +1,13 @@ +create schema schema1; + +create table schema1.t(id uuid, value text); + +create schema schema2; + +create table schema2.x(id uuid, value text); + +create schema schema3; + +create table schema3.untouched(id uuid); + +create table public.other(id uuid, value text); diff --git a/tests/FIXTURES/exclude_multischema/additions.sql b/tests/FIXTURES/exclude_multischema/additions.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/FIXTURES/exclude_multischema/b.sql b/tests/FIXTURES/exclude_multischema/b.sql new file mode 100644 index 0000000..fd7f2bb --- /dev/null +++ b/tests/FIXTURES/exclude_multischema/b.sql @@ -0,0 +1,17 @@ +create schema schema1; + +create table schema1.t(id uuid, value text, name text); + +create schema schema2; + +create table schema2.x(id uuid, value text); + +create table schema2.y(id uuid, value text); + +create schema schema3; + +create table schema3.untouched(id uuid); + +create table schema3.should_be_ignored(id uuid); + +create table public.other(id uuid); diff --git a/tests/FIXTURES/exclude_multischema/expected.sql b/tests/FIXTURES/exclude_multischema/expected.sql new file mode 100644 index 0000000..492ab16 --- /dev/null +++ b/tests/FIXTURES/exclude_multischema/expected.sql @@ -0,0 +1,7 @@ +create table "schema2"."y" ( + "id" uuid, + "value" text +); + + +alter table "schema1"."t" add column "name" text; diff --git a/tests/FIXTURES/exclude_multischema/expected2.sql b/tests/FIXTURES/exclude_multischema/expected2.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/FIXTURES/multischema/a.sql b/tests/FIXTURES/multischema/a.sql new file mode 100644 index 0000000..072e678 --- /dev/null +++ b/tests/FIXTURES/multischema/a.sql @@ -0,0 +1,13 @@ +create schema schema1; + +create table schema1.t(id uuid, value text); + +create schema schema2; + +create table schema2.x(id uuid, value text); + +create schema schema3; + +create table schema3.untouched(id uuid); + +create table public.other(id uuid, value text); diff --git a/tests/FIXTURES/multischema/additions.sql b/tests/FIXTURES/multischema/additions.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/FIXTURES/multischema/b.sql b/tests/FIXTURES/multischema/b.sql new file mode 100644 index 0000000..fd7f2bb --- /dev/null +++ b/tests/FIXTURES/multischema/b.sql @@ -0,0 +1,17 @@ +create schema schema1; + +create table schema1.t(id uuid, value text, name text); + +create schema schema2; + +create table schema2.x(id uuid, value text); + +create table schema2.y(id uuid, value text); + +create schema schema3; + +create table schema3.untouched(id uuid); + +create table schema3.should_be_ignored(id uuid); + +create table public.other(id uuid); diff --git a/tests/FIXTURES/multischema/expected.sql b/tests/FIXTURES/multischema/expected.sql new file mode 100644 index 0000000..492ab16 --- /dev/null +++ b/tests/FIXTURES/multischema/expected.sql @@ -0,0 +1,7 @@ +create table "schema2"."y" ( + "id" uuid, + "value" text +); + + +alter table "schema1"."t" add column "name" text; diff --git a/tests/FIXTURES/multischema/expected2.sql b/tests/FIXTURES/multischema/expected2.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_command_apply.py b/tests/test_command_apply.py new file mode 100644 index 0000000..fb07802 --- /dev/null +++ b/tests/test_command_apply.py @@ -0,0 +1,246 @@ +from __future__ import unicode_literals + +import io + +from sqlalchemy import text +from sqlbag import S, temporary_database + + +def outs(): + return io.StringIO(), io.StringIO() + + +def _create_table(db_url, table_name, columns="id int primary key"): + with S(db_url) as s: + s.execute(text("CREATE TABLE {} ({});".format(table_name, columns))) + + +def _table_exists(db_url, table_name): + with S(db_url) as s: + row = s.execute( + text( + "SELECT 1 FROM information_schema.tables" + " WHERE table_name = :table_name" + ), + {"table_name": table_name}, + ).fetchone() + return row is not None + + +class TestApplyHappyPath: + def test_apply_creates_table_in_dburl_from(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + _create_table(target_db, "widgets") + assert not _table_exists(from_db, "widgets") + + from migra.command import parse_args, run + + args = parse_args(["--apply", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + + assert status == 2 + assert "-- Applied " in out.getvalue() + assert "statement(s)" in out.getvalue() + assert _table_exists(from_db, "widgets") + + def test_apply_records_history_against_dburl_from(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + _create_table(target_db, "widgets") + + from migra.command import parse_args, run + + args = parse_args(["--apply", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 2 + + from migra.history import ensure_history_table, get_history + + with S(from_db) as s: + entries = get_history(s) + assert len(entries) == 1 + assert entries[0]["rollback_status"] == "not_attempted" + assert "CREATE TABLE" in entries[0]["forward_sql"].upper() + + # A plain --record-history (without --apply) should NOT also have + # written a second entry into dburl_target's history table -- in + # fact its history table should never even have been created. + with S(target_db) as s: + ensure_history_table(s) + target_entries = get_history(s) + assert target_entries == [] + + def test_apply_with_env_label(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + _create_table(target_db, "widgets") + + from migra.command import parse_args, run + + args = parse_args(["--apply", "--env-label", "prod", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 2 + + from migra.history import get_history + + with S(from_db) as s: + entries = get_history(s) + assert entries[0]["environment_label"] == "prod" + + def test_apply_json_output(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + _create_table(target_db, "widgets") + + from migra.command import parse_args, run + import json + + args = parse_args(["--apply", "--output", "json", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 2 + + data = json.loads(out.getvalue()) + assert data["apply"]["applied"] is True + assert data["apply"]["statement_count"] == len(data["statements"]) + assert data["apply"]["history_recorded"] is True + assert _table_exists(from_db, "widgets") + + +class TestApplyNoChanges: + def test_no_diff_is_a_noop(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + from migra.command import parse_args, run + + args = parse_args(["--apply", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + + assert status == 0 + assert "Applied" not in out.getvalue() + # Nothing to apply, so the history table should never even be + # created. + assert not _table_exists(from_db, "migradiff_history") + + +class TestApplySafety: + def test_destructive_change_blocked_without_force(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + _create_table(from_db, "widgets") + # target_db has no "widgets" table -> dropping it would be + # destructive. + + from migra.command import parse_args, run + + args = parse_args(["--apply", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + + assert status == 1 + assert "Destructive operations detected" in err.getvalue() + # The table must still exist -- --apply must never execute + # anything once safe-mode has already rejected the migration. + assert _table_exists(from_db, "widgets") + + def test_destructive_change_applied_with_force_destructive(self): + with temporary_database(host="localhost") as from_db, temporary_database( + host="localhost" + ) as target_db: + _create_table(from_db, "widgets") + + from migra.command import parse_args, run + + args = parse_args(["--apply", "--force-destructive", from_db, target_db]) + out, err = outs() + status = run(args, out=out, err=err) + + assert status == 2 + assert not _table_exists(from_db, "widgets") + + +class TestApplyIncompatibleFlags: + def test_rejected_with_from_file(self): + from migra.command import parse_args, run + + args = parse_args(["--apply", "--from-file", "a.sql", "b.sql"]) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "--apply is not supported with --from-file" in err.getvalue() + + def test_rejected_with_promote(self): + from migra.command import parse_args, run + + args = parse_args(["--apply", "--promote", "dev:staging"]) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "--apply is not supported with --promote" in err.getvalue() + + +class TestApplyMigrationHelper: + """Unit tests against _apply_migration() directly, so we can deterministically + force a mid-migration failure and assert atomicity -- something that's hard + to trigger reliably by going through the full diff-generation pipeline.""" + + def test_failed_statement_rolls_back_and_records_nothing(self): + with temporary_database(host="localhost") as db_url: + from migra.command import _apply_migration + + result = _apply_migration( + db_url, + [ + "CREATE TABLE ok_table (id int primary key);", + "THIS IS NOT VALID SQL;", + ], + sql_output="CREATE TABLE ok_table (id int primary key);\n\n" + "THIS IS NOT VALID SQL;\n\n", + rollback_sql=None, + env_label=None, + ) + + assert result["applied"] is False + assert result["error"] is not None + assert result["history_recorded"] is False + + # The whole transaction must have rolled back -- ok_table must + # NOT exist despite being a valid statement that ran first. + assert not _table_exists(db_url, "ok_table") + assert not _table_exists(db_url, "migradiff_history") + + def test_successful_apply_records_history(self): + with temporary_database(host="localhost") as db_url: + from migra.command import _apply_migration + + sql = "CREATE TABLE widgets (id int primary key);" + result = _apply_migration( + db_url, + [sql], + sql_output=sql, + rollback_sql="DROP TABLE widgets;", + env_label="staging", + ) + + assert result["applied"] is True + assert result["history_recorded"] is True + assert _table_exists(db_url, "widgets") + + from migra.history import get_history + + with S(db_url) as s: + entries = get_history(s) + assert len(entries) == 1 + assert entries[0]["rollback_sql"] == "DROP TABLE widgets;" + assert entries[0]["environment_label"] == "staging" diff --git a/tests/test_command_promote.py b/tests/test_command_promote.py new file mode 100644 index 0000000..4c99847 --- /dev/null +++ b/tests/test_command_promote.py @@ -0,0 +1,260 @@ +from __future__ import unicode_literals + +import io + +from sqlalchemy import text +from sqlbag import S, temporary_database + + +def outs(): + return io.StringIO(), io.StringIO() + + +def _create_table(db_url, table_name, columns="id int primary key"): + with S(db_url) as s: + s.execute(text("CREATE TABLE {} ({});".format(table_name, columns))) + + +def _create_schema_from_sql(db_url, sql): + with S(db_url) as s: + for stmt in sql.split(";"): + stmt = stmt.strip() + if stmt: + s.execute(text(stmt)) + + +class TestPromoteHappyPath: + def test_two_env_chain(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging: + _create_table(dev, "users") + _create_schema_from_sql( + staging, + "CREATE TABLE users (id int primary key);" + " CREATE TABLE posts (id int primary key);", + ) + + from migra.command import parse_args, run + + args = parse_args( + [ + "--unsafe", + "--promote", + "{}:{}".format(dev, staging), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + output = out.getvalue() + assert "=== " in output + assert "create table" in output or "No changes needed" in output + + def test_three_env_chain(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging, temporary_database(host="localhost") as prod: + _create_table(dev, "users") + _create_table(dev, "posts") + _create_schema_from_sql( + staging, + "CREATE TABLE users (id int primary key);" + " CREATE TABLE posts (id int primary key);", + ) + _create_schema_from_sql( + prod, + "CREATE TABLE users (id int primary key);" + " CREATE TABLE posts (id int primary key);" + " CREATE TABLE comments (id int primary key);", + ) + + from migra.command import parse_args, run + + args = parse_args( + [ + "--unsafe", + "--promote", + "{}:{}:{}".format(dev, staging, prod), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + + +class TestPromoteConflictDetection: + def test_staging_ahead_of_dev_conflict(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging: + _create_table(dev, "users") + _create_table(staging, "users") + _create_table(staging, "posts") + + from migra.command import parse_args, run + from migra.history import ensure_history_table, record_applied + + with S(dev) as s: + ensure_history_table(s) + with S(staging) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash="hash_not_in_dev", + forward_sql="CREATE TABLE posts (id int);", + environment_label="staging", + ) + + args = parse_args( + [ + "--unsafe", + "--promote", + "{}:{}".format(dev, staging), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "CONFLICT" in err.getvalue() + assert "hash_not_in" in err.getvalue() + + +class TestPromoteEmptyDiff: + def test_identical_envs_no_output(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging: + _create_table(dev, "users") + _create_table(staging, "users") + + from migra.command import parse_args, run + + args = parse_args( + [ + "--unsafe", + "--promote", + "{}:{}".format(dev, staging), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + output = out.getvalue() + assert "No changes needed" in output + + +class TestPromoteSafeMode: + def test_destructive_halts_chain(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging: + _create_schema_from_sql( + dev, + "CREATE TABLE users (id int primary key);" + " CREATE TABLE posts (id int primary key);", + ) + _create_table(staging, "users") + + from migra.command import parse_args, run + + args = parse_args( + [ + "--promote", + "{}:{}".format(dev, staging), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "Destructive" in err.getvalue() + + def test_force_destructive_allows_chain(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging: + _create_schema_from_sql( + dev, + "CREATE TABLE users (id int primary key);" + " CREATE TABLE posts (id int primary key);", + ) + _create_table(staging, "users") + + from migra.command import parse_args, run + + args = parse_args( + [ + "--force-destructive", + "--promote", + "{}:{}".format(dev, staging), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + + +class TestPromoteRecordHistory: + def test_record_history_per_hop(self): + with temporary_database(host="localhost") as dev, temporary_database( + host="localhost" + ) as staging: + _create_table(dev, "users") + _create_schema_from_sql( + staging, + "CREATE TABLE users (id int primary key);" + " CREATE TABLE posts (id int primary key);", + ) + + from migra.command import parse_args, run + + args = parse_args( + [ + "--unsafe", + "--record-history", + "--env-label", + "staging", + "--promote", + "{}:{}".format(dev, staging), + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + + from migra.history import get_history + + with S(staging) as s: + entries = get_history(s) + assert len(entries) >= 1 + assert entries[0]["environment_label"] == "staging" + + +class TestPromoteInvalidArgs: + def test_single_env_error(self): + from migra.command import parse_args, run + + args = parse_args( + [ + "--promote", + "dev", + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "requires at least 2" in err.getvalue() + + def test_unknown_alias_error(self): + from migra.command import parse_args, run + + args = parse_args( + [ + "--promote", + "dev:nonexistent_alias_xyz", + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "Unknown environment alias" in err.getvalue() diff --git a/tests/test_command_rollback_tracking.py b/tests/test_command_rollback_tracking.py new file mode 100644 index 0000000..edffa3f --- /dev/null +++ b/tests/test_command_rollback_tracking.py @@ -0,0 +1,193 @@ +from __future__ import unicode_literals + +import io + +from sqlalchemy import text +from sqlbag import S, temporary_database + + +def outs(): + return io.StringIO(), io.StringIO() + + +def _create_table(db_url, table_name, columns="id int primary key"): + with S(db_url) as s: + s.execute(text("CREATE TABLE {} ({});".format(table_name, columns))) + + +class TestRecordRollbackCli: + def test_record_rollback_after_applied(self): + with temporary_database(host="localhost") as db_url: + from migra.history import ensure_history_table, record_applied + + with S(db_url) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash="abc123hash", + forward_sql="SELECT 1;", + environment_label="dev", + ) + + from migra.command import parse_args, run + + args = parse_args( + [ + "--record-rollback", + "abc123hash", + db_url, + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + assert "Rollback recorded" in out.getvalue() + + from migra.history import get_history + + with S(db_url) as s: + entries = get_history(s) + assert len(entries) == 1 + assert entries[0]["rollback_status"] == "rolled_back" + assert entries[0]["rolled_back_at"] is not None + + def test_record_rollback_failed_status(self): + with temporary_database(host="localhost") as db_url: + from migra.history import ensure_history_table, record_applied + + with S(db_url) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash="abc123hash", + forward_sql="SELECT 1;", + ) + + from migra.command import parse_args, run + + args = parse_args( + [ + "--record-rollback", + "abc123hash", + db_url, + "--rollback-status", + "rollback_failed", + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + + from migra.history import get_history + + with S(db_url) as s: + entries = get_history(s) + assert entries[0]["rollback_status"] == "rollback_failed" + + def test_record_rollback_from_file(self): + import os + import tempfile + + with temporary_database(host="localhost") as db_url: + from migra.history import ( + compute_migration_hash, + ensure_history_table, + record_applied, + ) + + sql_content = "ALTER TABLE users DROP COLUMN email;" + actual_hash = compute_migration_hash(sql_content) + with S(db_url) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash=actual_hash, + forward_sql=sql_content, + ) + + with tempfile.NamedTemporaryFile( + mode="w", suffix=".sql", delete=False + ) as f: + f.write(sql_content) + f.flush() + rollback_file = f.name + + try: + from migra.command import parse_args, run + + args = parse_args( + [ + "--record-rollback", + rollback_file, + db_url, + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + assert "Rollback recorded" in out.getvalue() + finally: + os.unlink(rollback_file) + + def test_missing_history_entry(self): + with temporary_database(host="localhost") as db_url: + from migra.command import parse_args, run + + args = parse_args( + [ + "--record-rollback", + "nonexistent", + db_url, + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "No history entry found" in err.getvalue() + + def test_missing_db_url(self): + from migra.command import parse_args, run + + args = parse_args( + [ + "--record-rollback", + "abc123", + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 1 + assert "requires a database URL" in err.getvalue() + + +class TestStatusAfterRollback: + def test_status_shows_rolled_back_annotation(self): + with temporary_database(host="localhost") as db_url: + from migra.history import ( + ensure_history_table, + record_applied, + record_rollback, + ) + + with S(db_url) as s: + ensure_history_table(s) + record_applied( + s, + migration_hash="abc123", + forward_sql="SELECT 1;", + ) + record_rollback(s, "abc123", status="rolled_back") + + from migra.command import parse_args, run + + args = parse_args( + [ + "--status", + db_url, + ] + ) + out, err = outs() + status = run(args, out=out, err=err) + assert status == 0 + output = out.getvalue() + assert "ROLLED BACK" in output diff --git a/tests/test_history.py b/tests/test_history.py new file mode 100644 index 0000000..1942082 --- /dev/null +++ b/tests/test_history.py @@ -0,0 +1,345 @@ +from __future__ import unicode_literals + +import io + +from sqlalchemy import text +from sqlbag import S, temporary_database + + +def outs(): + return io.StringIO(), io.StringIO() + + +class TestComputeMigrationHash: + def test_basic_hash(self): + from migra.history import compute_migration_hash + + h1 = compute_migration_hash("ALTER TABLE users ADD COLUMN email text;") + h2 = compute_migration_hash("ALTER TABLE users ADD COLUMN email text;") + assert h1 == h2 + assert len(h1) == 64 + + def test_whitespace_normalization(self): + from migra.history import compute_migration_hash + + sql_a = "ALTER TABLE users ADD COLUMN email text;" + sql_b = " ALTER TABLE users ADD COLUMN email text;\n\n" + assert compute_migration_hash(sql_a) == compute_migration_hash(sql_b) + + def test_trailing_whitespace_per_line(self): + from migra.history import compute_migration_hash + + sql_a = "ALTER TABLE users ADD COLUMN email text;\nALTER TABLE posts ADD COLUMN title text;" + sql_b = "ALTER TABLE users ADD COLUMN email text; \nALTER TABLE posts ADD COLUMN title text; " + assert compute_migration_hash(sql_a) == compute_migration_hash(sql_b) + + def test_collapsed_blank_lines(self): + from migra.history import compute_migration_hash + + sql_a = "LINE1\n\nLINE2" + sql_b = "LINE1\n\n\n\n\nLINE2" + assert compute_migration_hash(sql_a) == compute_migration_hash(sql_b) + + def test_different_sql_different_hash(self): + from migra.history import compute_migration_hash + + h1 = compute_migration_hash("ALTER TABLE users ADD COLUMN email text;") + h2 = compute_migration_hash("ALTER TABLE users DROP COLUMN email;") + assert h1 != h2 + + +class TestEnsureHistoryTable: + def test_table_creation_idempotent(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ensure_history_table + + ensure_history_table(s) + ensure_history_table(s) + + # Verify table exists by querying it + rows = s.execute( + text( + "SELECT table_name FROM information_schema.tables " + "WHERE table_name = 'migradiff_history'" + ) + ).fetchall() + assert len(rows) == 1 + + # Verify columns + cols = s.execute( + text( + "SELECT column_name, data_type FROM information_schema.columns " + "WHERE table_name = 'migradiff_history' " + "ORDER BY ordinal_position" + ) + ).fetchall() + col_names = [c[0] for c in cols] + assert "id" in col_names + assert "migration_hash" in col_names + assert "checksum" in col_names + assert "applied_at" in col_names + assert "applied_by" in col_names + assert "forward_sql" in col_names + assert "rollback_sql" in col_names + assert "rollback_status" in col_names + assert "rolled_back_at" in col_names + assert "rolled_back_by" in col_names + assert "environment_label" in col_names + assert "source" in col_names + + def test_default_rollback_status(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ensure_history_table, record_applied + + ensure_history_table(s) + record_applied( + s, + migration_hash="abc123", + forward_sql="SELECT 1;", + ) + row = s.execute( + text("SELECT rollback_status FROM migradiff_history") + ).fetchone() + assert row[0] == "not_attempted" + + +class TestRecordAppliedAndGetAppliedHashes: + def test_round_trip(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_applied_hashes, + record_applied, + ) + + ensure_history_table(s) + + record_applied( + s, + migration_hash="hash001", + forward_sql="CREATE TABLE t1 (id int);", + ) + record_applied( + s, + migration_hash="hash002", + forward_sql="ALTER TABLE t1 ADD COLUMN name text;", + ) + + hashes = get_applied_hashes(s) + assert hashes == {"hash001", "hash002"} + + def test_duplicate_hash(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_applied_hashes, + record_applied, + ) + + ensure_history_table(s) + record_applied( + s, + migration_hash="hash001", + forward_sql="CREATE TABLE t1 (id int);", + ) + record_applied( + s, + migration_hash="hash001", + forward_sql="CREATE TABLE t1 (id int);", + ) + hashes = get_applied_hashes(s) + assert hashes == {"hash001"} + + def test_record_with_metadata(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_history, + record_applied, + ) + + ensure_history_table(s) + record_applied( + s, + migration_hash="hash001", + migration_name="v001_create_users", + forward_sql="CREATE TABLE users (id int);", + rollback_sql="DROP TABLE users;", + environment_label="dev", + applied_by="test_user", + ) + + entries = get_history(s) + assert len(entries) == 1 + e = entries[0] + assert e["migration_hash"] == "hash001" + assert e["migration_name"] == "v001_create_users" + assert e["forward_sql"] == "CREATE TABLE users (id int);" + assert e["rollback_sql"] == "DROP TABLE users;" + assert e["environment_label"] == "dev" + assert e["applied_by"] == "test_user" + assert e["checksum"] == "hash001" + assert e["source"] == "cli" + assert e["rollback_status"] == "not_attempted" + + def test_record_applied_returns_id(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ensure_history_table, record_applied + + ensure_history_table(s) + new_id = record_applied( + s, + migration_hash="hash001", + forward_sql="SELECT 1;", + ) + assert new_id is not None + assert isinstance(new_id, int) + assert new_id > 0 + + +class TestRecordRollback: + def test_update_most_recent_matching_row(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_history, + record_applied, + record_rollback, + ) + + ensure_history_table(s) + + record_applied( + s, + migration_hash="abc123", + forward_sql="V1", + ) + record_applied( + s, + migration_hash="abc123", + forward_sql="V2", + ) + + record_rollback(s, "abc123", status="rolled_back") + entries = get_history(s) + assert len(entries) == 2 + most_recent = entries[0] + assert most_recent["forward_sql"] == "V2" + assert most_recent["rollback_status"] == "rolled_back" + assert most_recent["rolled_back_at"] is not None + assert most_recent["rolled_back_by"] is not None + + older = entries[1] + assert older["forward_sql"] == "V1" + assert older["rollback_status"] == "not_attempted" + assert older["rolled_back_at"] is None + + def test_rollback_failed_status(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_history, + record_applied, + record_rollback, + ) + + ensure_history_table(s) + record_applied( + s, + migration_hash="abc123", + forward_sql="SELECT 1;", + ) + record_rollback(s, "abc123", status="rollback_failed") + + entries = get_history(s) + assert entries[0]["rollback_status"] == "rollback_failed" + + def test_no_matching_row_raises(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ensure_history_table, record_rollback + + ensure_history_table(s) + import pytest + + with pytest.raises(ValueError, match="No history entry found"): + record_rollback(s, "nonexistent_hash") + + +class TestGetHistory: + def test_empty_history(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ensure_history_table, get_history + + ensure_history_table(s) + entries = get_history(s) + assert entries == [] + + def test_limit(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_history, + record_applied, + ) + + ensure_history_table(s) + for i in range(5): + record_applied( + s, + migration_hash="hash{:03d}".format(i), + forward_sql="SELECT {};".format(i), + ) + + limited = get_history(s, limit=3) + assert len(limited) == 3 + # Most recent first + assert limited[0]["migration_hash"] == "hash004" + + unlimited = get_history(s) + assert len(unlimited) == 5 + + def test_filter_by_environment(self): + with temporary_database(host="localhost") as db_url: + with S(db_url) as s: + from migra.history import ( + ensure_history_table, + get_history, + record_applied, + ) + + ensure_history_table(s) + + record_applied( + s, + migration_hash="hash001", + forward_sql="SELECT 1;", + environment_label="dev", + ) + record_applied( + s, + migration_hash="hash002", + forward_sql="SELECT 2;", + environment_label="prod", + ) + + dev_entries = get_history(s, environment_label="dev") + assert len(dev_entries) == 1 + assert dev_entries[0]["migration_hash"] == "hash001" + + prod_entries = get_history(s, environment_label="prod") + assert len(prod_entries) == 1 + assert prod_entries[0]["migration_hash"] == "hash002" + + all_entries = get_history(s) + assert len(all_entries) == 2 diff --git a/tests/test_migra.py b/tests/test_migra.py index c61cd84..ebf502e 100644 --- a/tests/test_migra.py +++ b/tests/test_migra.py @@ -60,11 +60,27 @@ def test_singleschema(): do_fixture_test(FIXTURE_NAME, schema="goodschema") +def test_multischema(): + for FIXTURE_NAME in ["multischema"]: + do_fixture_test(FIXTURE_NAME, schema="schema1,schema2") + + +def test_multischema_whitespace_tolerant(): + # comma-separated schema names should tolerate surrounding whitespace + for FIXTURE_NAME in ["multischema"]: + do_fixture_test(FIXTURE_NAME, schema=" schema1 , schema2 ") + + def test_excludeschema(): for FIXTURE_NAME in ["excludeschema"]: do_fixture_test(FIXTURE_NAME, exclude_schema="excludedschema") +def test_exclude_multischema(): + for FIXTURE_NAME in ["exclude_multischema"]: + do_fixture_test(FIXTURE_NAME, exclude_schema="schema3,public") + + def test_singleschema_ext(): for FIXTURE_NAME in ["singleschema_ext"]: do_fixture_test(FIXTURE_NAME, create_extensions_only=True)