From 0051c850059ae2dd428ded1f871cf5cb6230a5de Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Tue, 4 Aug 2026 16:14:32 +1000 Subject: [PATCH 01/24] test/ci: PG 14-18 dev database, TLS tests, docs-only gating, hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compose-based local database mirrors the CI matrix (PG_DSN path in the harness); docs-only PRs skip the heavy jobs behind a fail-open filter with a single ci-ok required status; pre-commit/pre-push hooks give fast local feedback and skip cleanly on Markdown-only changes. A TLS-only server harness proves the verify-full CACertPath path live (trusted CA encrypts, plaintext and untrusted CA fail closed) — the one peer-CI pattern (pg-delta) applicable to Phase 0. docs/testing.md records Phase 0 coverage, Phases 1-7 test obligations including the topology patterns surveyed from peer-tool CIs, and the vanilla PostgreSQL vs real-Aurora validation boundary. The lint config adopts SchemaBot's proven linter set plus gofmt/goimports formatters, with a staged-files lint-fix pre-commit flow ported from the same repo. --- .githooks/pre-commit | 48 ++++++++ .githooks/pre-push | 115 +++++++++++++++++++ .github/workflows/ci.yml | 74 +++++++++++- .golangci.yml | 33 +++++- AGENTS.md | 10 +- Makefile | 39 ++++++- README.md | 15 ++- compose/compose.yml | 21 ++++ docs/README.md | 1 + docs/testing.md | 84 ++++++++++++++ internal/testutil/postgres.go | 25 +++-- internal/testutil/postgres_test.go | 59 ++++++++++ internal/testutil/postgres_tls.go | 174 +++++++++++++++++++++++++++++ pkg/dbconn/tls_integration_test.go | 64 +++++++++++ scripts/lint-fix.sh | 86 ++++++++++++++ 15 files changed, 829 insertions(+), 19 deletions(-) create mode 100755 .githooks/pre-commit create mode 100755 .githooks/pre-push create mode 100644 compose/compose.yml create mode 100644 docs/testing.md create mode 100644 internal/testutil/postgres_test.go create mode 100644 internal/testutil/postgres_tls.go create mode 100644 pkg/dbconn/tls_integration_test.go create mode 100755 scripts/lint-fix.sh diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 0000000..22b1492 --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,48 @@ +#!/bin/sh +set -e + +# Pre-commit: fast local feedback on staged Go files. Docs-only commits +# (no staged .go files) skip the Go checks entirely so Markdown edits stay +# instant. CI remains the authoritative gate. +# +# Enable with: make setup (sets core.hooksPath .githooks) + +# Ensure we're at the worktree root (not the git common dir). Unset GIT_DIR +# first — git sets it during hooks and it makes subprocess git commands +# resolve the wrong repo root. +unset GIT_DIR +cd "$(git rev-parse --show-toplevel)" + +staged_go=$(git diff --cached --name-only --diff-filter=d -- '*.go') +if [ -z "$staged_go" ]; then + echo "pre-commit: no staged Go files; skipping Go checks." + exit 0 +fi + +# Ensure GOROOT matches the Go version in go.mod. External tools (hermit, +# asdf) may set GOROOT to a different version, causing "compile: version +# does not match go tool version" errors in golangci-lint and go build. +GO_VERSION=$(sed -n 's/^go //p' go.mod | head -1) +if [ -n "$GO_VERSION" ]; then + for candidate in \ + "/opt/homebrew/opt/go/libexec" \ + "/usr/local/go" \ + "$HOME/sdk/go${GO_VERSION}" \ + "$HOME/Library/Caches/hermit/pkg/go-${GO_VERSION}"; do + if [ -x "$candidate/bin/go" ]; then + CANDIDATE_VERSION=$("$candidate/bin/go" version 2>/dev/null | grep -o "go${GO_VERSION}" || true) + if [ -n "$CANDIDATE_VERSION" ]; then + export GOROOT="$candidate" + export PATH="$GOROOT/bin:$PATH" + break + fi + fi + done +fi +export PATH="$PATH:$HOME/go/bin:/usr/local/go/bin:/opt/homebrew/bin" + +# Everything staged must still build. +go build ./... + +# gofmt + golangci-lint on staged files: auto-fix, re-stage, verify. +scripts/lint-fix.sh diff --git a/.githooks/pre-push b/.githooks/pre-push new file mode 100755 index 0000000..3e7c8bd --- /dev/null +++ b/.githooks/pre-push @@ -0,0 +1,115 @@ +#!/bin/sh +set -e + +# Pre-push: run unit tests (go test -race, integration skipped) for the +# packages touched by the commits being pushed. Docs-only pushes skip +# entirely. CI remains the authoritative gate and runs the full suite +# including the PostgreSQL matrix — this hook does not replace it. +# +# Reads the standard pre-push stdin records: +# + +unset GIT_DIR +cd "$(git rev-parse --show-toplevel)" + +remote_name="$1" + +# Ensure GOROOT matches the Go version in go.mod (see pre-commit). +GO_VERSION=$(sed -n 's/^go //p' go.mod | head -1) +if [ -n "$GO_VERSION" ]; then + for candidate in \ + "/opt/homebrew/opt/go/libexec" \ + "/usr/local/go" \ + "$HOME/sdk/go${GO_VERSION}" \ + "$HOME/Library/Caches/hermit/pkg/go-${GO_VERSION}"; do + if [ -x "$candidate/bin/go" ]; then + CANDIDATE_VERSION=$("$candidate/bin/go" version 2>/dev/null | grep -o "go${GO_VERSION}" || true) + if [ -n "$CANDIDATE_VERSION" ]; then + export GOROOT="$candidate" + export PATH="$GOROOT/bin:$PATH" + break + fi + fi + done +fi +export PATH="$PATH:$HOME/go/bin:/usr/local/go/bin:/opt/homebrew/bin" + +# Resolve a base ref for new-branch pushes (no remote sha yet): prefer the +# push remote's default branch, then origin/main, then main. +base_ref="" +for cand in "refs/remotes/$remote_name/HEAD" "refs/remotes/$remote_name/main" "refs/remotes/origin/main" "main"; do + if git rev-parse --verify --quiet "$cand" >/dev/null 2>&1; then + base_ref="$cand" + break + fi +done + +# is_zero_sha returns success when the sha is all zeros (branch create/delete +# sentinel), independent of SHA-1 vs SHA-256 length. +is_zero_sha() { + case "$1" in + *[!0]*) return 1 ;; + *) return 0 ;; + esac +} + +changed_go_files="" +while read -r local_ref local_sha remote_ref remote_sha; do + # Skip branch deletions — nothing to test. + if is_zero_sha "$local_sha"; then + continue + fi + + if is_zero_sha "$remote_sha"; then + # New branch on the remote: test what this branch introduces relative + # to the base. Fall back to the single tip commit if no base exists. + if [ -n "$base_ref" ]; then + range_base=$(git merge-base "$base_ref" "$local_sha" 2>/dev/null || true) + else + range_base="" + fi + if [ -z "$range_base" ]; then + range_base="${local_sha}~1" + fi + else + range_base="$remote_sha" + fi + + files=$(git diff --name-only --diff-filter=d "$range_base" "$local_sha" -- '*.go' 2>/dev/null || true) + if [ -n "$files" ]; then + if [ -z "$changed_go_files" ]; then + changed_go_files="$files" + else + changed_go_files="$changed_go_files +$files" + fi + fi +done + +changed_go_files=$(printf '%s\n' "$changed_go_files" | sed '/^$/d' | sort -u) +if [ -z "$changed_go_files" ]; then + echo "pre-push: no Go changes in pushed commits; skipping unit tests." + exit 0 +fi + +# Map changed files to package directories that still exist and contain Go +# source for the default build. +pkgs="" +for dir in $(printf '%s\n' "$changed_go_files" | xargs -n1 dirname | sort -u); do + testable_pkg=$(go list -e -f '{{if or (or .GoFiles .TestGoFiles) .XTestGoFiles}}{{.ImportPath}}{{end}}' "./$dir" 2>/dev/null || true) + if [ -d "$dir" ] && [ -n "$testable_pkg" ]; then + pkgs="$pkgs ./$dir" + fi +done + +pkgs=$(printf '%s' "$pkgs" | sed 's/^ *//') +if [ -z "$pkgs" ]; then + echo "pre-push: changed Go files map to no testable packages; skipping unit tests." + exit 0 +fi + +echo "pre-push: running unit tests (race, integration skipped) for changed packages:" +for p in $pkgs; do echo " $p"; done + +# shellcheck disable=SC2086 +SKIP_INTEGRATION=1 go test -race -count=1 $pkgs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7546aec..4fc3def 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,8 +5,55 @@ on: branches: [main] pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: + # Detect whether the change touches anything other than *.md / docs/**. + # PRs that touch only docs skip the heavy jobs below; pushes to main + # always run. Fails open: if detection fails, treat as a code change. + changes: + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + code: ${{ steps.out.outputs.code }} + steps: + - name: Detect non-docs changes + if: github.event_name == 'pull_request' + id: filter + continue-on-error: true + uses: dorny/paths-filter@d1c1ffe0248fe513906c8e24db8ea791d46f8590 # v3.0.3 + with: + filters: | + code: + - '**' + - '!**/*.md' + - '!docs/**' + - name: Resolve code-change flag + id: out + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + FILTER_OUTCOME: ${{ steps.filter.outcome }} + FILTER_CODE: ${{ steps.filter.outputs.code }} + run: | + set -euo pipefail + if [ "$EVENT_NAME" != "pull_request" ]; then + code=true + elif [ "$FILTER_OUTCOME" != "success" ]; then + echo "::warning::paths-filter failed; running CI instead of skipping" + code=true + else + code="$FILTER_CODE" + fi + echo "code=$code" >> "$GITHUB_OUTPUT" + echo "event=$EVENT_NAME code=$code" + lint: + needs: changes + if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -16,6 +63,8 @@ jobs: - uses: golangci/golangci-lint-action@v6 build: + needs: changes + if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -25,9 +74,13 @@ jobs: - run: make build # The integration suite runs against every Aurora-supported PostgreSQL - # major (see the version-support research doc): the version floor is a - # promise CI enforces, not documentation. + # major (see docs/postgresql-version-support.md): the version floor is a + # promise CI enforces, not documentation. These are vanilla PostgreSQL + # images — real Aurora engine-version validation is a separate gate that + # cannot run in public CI. test: + needs: changes + if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest strategy: fail-fast: false @@ -41,3 +94,20 @@ jobs: with: go-version-file: go.mod - run: make test + + # Single required status for branch protection. Succeeds when nothing + # failed — including docs-only PRs where the heavy jobs were skipped. + ci-ok: + if: always() + needs: [changes, lint, build, test] + runs-on: ubuntu-latest + steps: + - name: Check job results + env: + RESULTS: ${{ toJSON(needs) }} + run: | + echo "$RESULTS" + if echo "$RESULTS" | grep -Eq '"result": *"(failure|cancelled)"'; then + echo "a required job failed or was cancelled" + exit 1 + fi diff --git a/.golangci.yml b/.golangci.yml index 31c51ae..c4d6ddc 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -2,6 +2,33 @@ version: "2" linters: enable: - - bodyclose - - misspell - - nolintlint + - errcheck # unchecked errors + - govet # suspicious constructs + - staticcheck # comprehensive static analysis + - ineffassign # useless assignments + - unused # unused code + - gocritic # opinionated but catches real bugs + - errorlint # proper error wrapping/comparison + - modernize # suggest modern Go idioms (wg.Go, etc.) + - noctx # requests without context + - bodyclose # unclosed HTTP response bodies + - usetesting # enforce t.Context() over context.Background() in tests + - unparam # unused/constant function parameters and results + - misspell # commonly misspelled words + - nolintlint # ill-formed or unexplained nolint directives + settings: + usetesting: + context-background: true + context-todo: true + exclusions: + rules: + # Test helpers intentionally keep uniform signatures and fixed arguments + # for readability, so unused/constant params there are not worth churn. + - path: _test\.go + linters: + - unparam + +formatters: + enable: + - gofmt + - goimports diff --git a/AGENTS.md b/AGENTS.md index efcd234..2f730d0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,9 +14,12 @@ SAFETY.md — the review bar and the AI-assistance posture differ by side. ## Build and test ```sh +make setup # one-time: configure git hooks (core.hooksPath .githooks) make build # build ./... and bin/pg-sprite make test # full suite; integration tests need Docker make test-unit # SKIP_INTEGRATION=1, no Docker +make test-db # suite against the compose DB (make db-up first); PG_DSN +make test-supported-postgres # full suite on every major 14 -> 18 make lint # golangci-lint ``` @@ -33,9 +36,10 @@ make lint # golangci-lint Every session runs under bounded `lock_timeout` / `statement_timeout`. - All SQL parsing goes through `pg_query_go` (once `pkg/statement` exists). No `strings.Split(";")`, no hand-parsing; a parse failure is an error surfaced to the caller. -- Tests use testify (`require` for setup, `assert` for verification), `t.Context()` (except in - cleanups, which run after the context is cancelled), and named polling deadlines — no bare - `time.Sleep` readiness waits. +- Tests use testify (`require` for setup, `assert` for verification), `t.Context()` (in + cleanups, which run after the context is cancelled, use + `context.WithoutCancel(t.Context())`), and named polling deadlines — no bare `time.Sleep` + readiness waits. - Errors: wrap with context and identifiers (`fmt.Errorf("create slot %s: %w", name, err)`); never log-and-continue; no silent branch cases; no `nolint`; no `--no-verify`. diff --git a/Makefile b/Makefile index 6fe553f..f8a23fc 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,17 @@ GO ?= go +COMPOSE ?= docker compose -.PHONY: build test test-unit lint clean +# PostgreSQL major under test (see docs/postgresql-version-support.md). +PG_VERSION ?= 16 +PG_PORT ?= 5432 +# Local dev database credentials (compose/compose.yml); test-only defaults. +PG_USER ?= pgsprite +PG_PASSWORD ?= pgsprite +PG_DATABASE ?= pgsprite +# Localhost-only test credentials, parameterized above — not a real secret. +PG_DSN_LOCAL = postgres://$(PG_USER):$(PG_PASSWORD)@localhost:$(PG_PORT)/$(PG_DATABASE)?sslmode=disable# sadscan:disable np.postgres.1 + +.PHONY: build test test-unit test-db test-supported-postgres lint setup db-up db-down clean build: $(GO) build -o bin/pg-sprite ./cmd/pg-sprite @@ -13,8 +24,34 @@ test: test-unit: SKIP_INTEGRATION=1 $(GO) test -race ./... +# Integration suite against the long-lived compose database (no per-test +# containers). Start it first: make db-up [PG_VERSION=14] +test-db: + PG_DSN="$(PG_DSN_LOCAL)" $(GO) test -race -count=1 ./... + +# Full suite against every supported PostgreSQL major (14 -> 18) via +# testcontainers — the local mirror of the CI matrix. +test-supported-postgres: + @for v in 14 15 16 17 18; do \ + echo "=== PostgreSQL $$v ==="; \ + PG_VERSION=$$v $(GO) test -race -count=1 ./... || exit 1; \ + done + lint: golangci-lint run +# Configure git hooks (relative path so worktrees work too). +setup: + git config core.hooksPath .githooks + +# Start / stop the local development database (compose/compose.yml). +COMPOSE_ENV = PG_VERSION=$(PG_VERSION) PG_PORT=$(PG_PORT) PG_USER=$(PG_USER) PG_PASSWORD=$(PG_PASSWORD) PG_DATABASE=$(PG_DATABASE) + +db-up: + $(COMPOSE_ENV) $(COMPOSE) -f compose/compose.yml up --wait -d + +db-down: + $(COMPOSE_ENV) $(COMPOSE) -f compose/compose.yml down -v + clean: rm -rf bin diff --git a/README.md b/README.md index 8dc2a51..e1c9562 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ rules that apply inside the core. Read it before changing anything under ## Development ```sh +make setup # one-time: configure git hooks (.githooks) make build # build ./... and the bin/pg-sprite binary make test # full suite; integration tests need Docker make test-unit # unit tests only (SKIP_INTEGRATION=1) @@ -36,7 +37,19 @@ make lint # golangci-lint ``` Integration tests run against a real PostgreSQL via testcontainers. `PG_VERSION` -selects the major (default 16); CI runs the matrix 14 → 18. +selects the major (default 16); CI runs the matrix 14 → 18. To iterate against a +long-lived local database instead of per-test containers: + +```sh +make db-up PG_VERSION=14 # start PostgreSQL 14 on localhost via compose +make test-db # run the suite against it (PG_DSN) +make db-down # stop and discard it +``` + +`make test-supported-postgres` runs the full suite against every supported +major (14 → 18) — the local mirror of the CI matrix. See +[docs/testing.md](docs/testing.md) for the test-suite layout, what each build +phase owes, and the vanilla-PostgreSQL-vs-real-Aurora validation boundary. ## Contributing diff --git a/compose/compose.yml b/compose/compose.yml new file mode 100644 index 0000000..3a9046f --- /dev/null +++ b/compose/compose.yml @@ -0,0 +1,21 @@ +# Local development database — the long-lived analogue of the testcontainers +# harness. `make db-up PG_VERSION=14` starts one PostgreSQL major on +# localhost; `make test-db` points the integration suite at it via PG_DSN so +# no per-test containers are started. This is a test database: durability is +# deliberately relaxed and all state is discarded on `make db-down`. +services: + postgres: + image: postgres:${PG_VERSION:-16} + environment: + POSTGRES_USER: ${PG_USER:-pgsprite} + POSTGRES_PASSWORD: ${PG_PASSWORD:-pgsprite} + POSTGRES_DB: ${PG_DATABASE:-pgsprite} + ports: + - "${PG_PORT:-5432}:5432" + # Test-only speed settings; never carry these to a real deployment. + command: ["postgres", "-c", "fsync=off", "-c", "full_page_writes=off"] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 1s + timeout: 3s + retries: 60 diff --git a/docs/README.md b/docs/README.md index 5a44e96..fa46b97 100644 --- a/docs/README.md +++ b/docs/README.md @@ -44,6 +44,7 @@ checkpoint/resume, tuned for Aurora. That is the gap this engine targets. | [change-capture-tradeoff.md](change-capture-tradeoff.md) | The canonical **triggers vs logical-decoding** trade-off for copy-and-swap — overhead, failover survival, WAL risk, and whether either lets us drop the checksum/checkpoint (answer: keep the checksum; triggers simplify but don't remove the checkpoint). Any doc proposing logical decoding as the default points here. | | [invariants.md](invariants.md) | The canonical **invariant registry** — testable runtime MUST-statements (correctness, locking, state/resume, refusals, orchestration), each with its enforcement point and source. Mined from this doc set plus [Spirit](https://github.com/block/spirit)'s stated safety invariants and [SchemaBot](https://github.com/block/schemabot)'s control-plane discipline; the build plan's phases carry per-invariant test obligations. | | [tcb-model.md](tcb-model.md) | The **TCB model** — the trusted-computing-base partition of the engine: which components are the small trusted core that enforces the invariant registry vs the untrusted periphery, the never-trust-callers rule, domain types that make illegal states unrepresentable, the in-TCB engineering rules (from TigerBeetle TIGER_STYLE, s2n-tls, qmail, bitcoin-core), the verification ladder, and the per-side AI-assisted development policy. | +| [testing.md](testing.md) | The **test-suite guide** — how to run the suite (unit, per-major, all supported majors, compose database), what Phase 0 covers today, the per-phase deferred test obligations, and the vanilla-PostgreSQL-matrix vs real-Aurora validation boundary. | | [schemabot-integration.md](schemabot-integration.md) | The **single home for orchestrator integration** — how SchemaBot (the reference orchestrator) drives the engine: the pluggable-engine overview, the verb mappings, the concrete adapter contract, and the design constraints (OC-* invariants) the integration imposes on the core. | ## TL;DR recommendation diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..8fdb56b --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,84 @@ +# Testing + +How the test suite is organized, what it covers today, and what each build +phase is obligated to add. Core logic is validated against a **real +PostgreSQL** — no mocked-DB tests for core logic (see +[design-principles.md](design-principles.md)). + +## How to run + +| Command | What it does | +| --- | --- | +| `make test-unit` | Race-enabled unit tests, no Docker (`SKIP_INTEGRATION=1`). | +| `make test` | Full suite; integration tests start disposable PostgreSQL containers (testcontainers). `PG_VERSION` selects the major (default 16). | +| `make test-supported-postgres` | Full suite against every supported major, 14 → 18 — the local mirror of the CI matrix. | +| `make db-up` / `make test-db` / `make db-down` | Long-lived compose database on localhost; the suite connects to it via `PG_DSN` instead of starting per-test containers. Fastest loop for repeated integration runs. | + +The harness is [internal/testutil](../internal/testutil/postgres.go): +`StartPostgres` returns a connection URL (container, or `PG_DSN` when set) +and `NewSchema` gives each test a throwaway schema so parallel tests never +collide — which also means every integration test runs against a +**non-`public` schema**, an axis some peer tools (pgroll) treat as a separate +matrix dimension. `StartPostgresTLS` starts a TLS-only server with a +generated CA for verify-full tests. The harness has its own tests proving +the version selected by `PG_VERSION` is the version actually running, and +that throwaway schemas are isolated. + +## Version matrix vs real Aurora + +CI runs the matrix against **vanilla PostgreSQL 14 → 18 images** — the floor +promised in [postgresql-version-support.md](postgresql-version-support.md) is +enforced by CI, not just documented. Vanilla PostgreSQL is *not* Aurora: +storage internals, replication, and failover behavior differ, and some +Aurora-specific behavior (e.g. `rds.logical_replication`, failover slot +loss) cannot be exercised in public CI. Validation against real Aurora +engine versions is a separate, environment-specific gate that lives outside +this repository's CI. + +## Current coverage (Phase 0) + +| Area | Tests | +| --- | --- | +| CLI grammar / config | [internal/cli](../internal/cli/cli_test.go) | +| Pool config, bounded session timeouts | [pkg/dbconn](../pkg/dbconn/pool_config_test.go), [integration](../pkg/dbconn/dbconn_integration_test.go) | +| Retry classification and behavior | [pkg/dbconn/retry_test.go](../pkg/dbconn/retry_test.go) | +| RDS/Aurora TLS (unit) | [pkg/dbconn/rds_test.go](../pkg/dbconn/rds_test.go) | +| Verify-full TLS against a live TLS-only server | [pkg/dbconn/tls_integration_test.go](../pkg/dbconn/tls_integration_test.go) | +| Targeted blocker termination | [pkg/dbconn/dbconn_integration_test.go](../pkg/dbconn/dbconn_integration_test.go) | +| Test harness self-checks | [internal/testutil](../internal/testutil/postgres_test.go) | + +## Deferred test obligations (Phases 1–3) + +These are owed when the corresponding implementation lands — they are not +written speculatively against unimplemented behavior. The authoritative +per-phase test lists live in the build plan; the invariant registry +([invariants.md](invariants.md)) carries the per-invariant enforcement +points. + +| Phase | Test obligations (summary) | +| --- | --- | +| 1 — statement/classifier | Parse-based classification per DDL form; refusal (`not-native-safe`) contract; every classification decision tested against the reference table in [postgres-online-ddl-reference.md](postgres-online-ddl-reference.md). | +| 2 — native executor | Each native idiom (`CONCURRENTLY`, `NOT VALID` + `VALIDATE`, fast default, `USING INDEX`) exercised against all supported majors; bounded lock behavior under contention; invalid-index cleanup. | +| 3 — declarative diff | Desired-state → `ALTER` derivation correctness; diff idempotency (no-op on converged schema); refusal propagation through the diff path. | + +Copy-and-swap (shadow table, CDC, checksum gate, cutover, resume) is +Phases 4–7 and carries its own obligations, including checksum-gate and +checkpoint/resume fault-injection tests. + +## Topology obligations from peer-tool CIs + +A survey of the CI setups of pgroll, Reshape, pg-osc, pg_repack, +pg-schema-diff, pg-delta, migra, Atlas, SchemaHero, and Bytebase found no +peer testing physical replicas, poolers as live intermediaries, failover, or +cloud-managed PostgreSQL — those remain environment-gate territory (see +above). The patterns worth carrying, tied to the phase whose implementation +makes them meaningful: + +| Pattern (peer precedent) | Where it lands here | +| --- | --- | +| TLS-required server, verify-full + untrusted-CA rejection (pg-delta) | **Done** — `StartPostgresTLS` + [tls_integration_test.go](../pkg/dbconn/tls_integration_test.go). | +| Non-`public` schema placement (pgroll matrix dimension) | Structural — every test already runs in a throwaway non-`public` schema; Phase 1 classifier tests must keep qualifying objects. | +| Partitioned tables (pg_repack regression, pg-schema-diff acceptance) | Phases 1–2 — classifier and native-executor cases for partitioned parents/partitions (`DETACH PARTITION CONCURRENTLY` is PG 14+). | +| Tablespaces, including quoted names (pg_repack) | Phases 4–7 — shadow-table placement must preserve tablespace. | +| `wal_level=logical` server + publication interaction (pg_repack, pg-delta) | Phases 4–7 — CDC tests run against logical-decoding-enabled servers; add `wal_level=logical` to the harness/compose when Phase 4 starts. | +| Pinned minor versions in the matrix (pgroll, SchemaHero) vs floating major tags | Deliberate choice: we track floating major tags (`postgres:14` … `postgres:18`) so CI follows each major's latest minor automatically. Revisit if a minor-specific regression ever matters. | diff --git a/internal/testutil/postgres.go b/internal/testutil/postgres.go index 8742f39..1535f61 100644 --- a/internal/testutil/postgres.go +++ b/internal/testutil/postgres.go @@ -29,17 +29,24 @@ func PGVersion() string { return DefaultPGVersion } -// StartPostgres starts a disposable PostgreSQL container for the test and -// returns its connection URL. The container is terminated when the test ends. -// Set SKIP_INTEGRATION=1 to skip tests that need Docker. +// StartPostgres returns a PostgreSQL connection URL for the test. +// +// By default it starts a disposable container (terminated when the test +// ends). When PG_DSN is set, that external server is used instead and no +// container is started — the compose/ workflow and CI variants that run a +// long-lived server use this. Set SKIP_INTEGRATION=1 to skip tests that need +// a database entirely. func StartPostgres(t *testing.T) string { t.Helper() if os.Getenv("SKIP_INTEGRATION") != "" { - t.Skip("SKIP_INTEGRATION set; skipping test that needs Docker") + t.Skip("SKIP_INTEGRATION set; skipping test that needs a database") } - // The container must outlive t.Context (which is cancelled before - // cleanups run), so use Background and terminate via t.Cleanup. - ctx := context.Background() + if dsn := os.Getenv("PG_DSN"); dsn != "" { + return dsn + } + // t.Context only governs the start request; the running container is + // not tied to it and is terminated via t.Cleanup below. + ctx := t.Context() ctr, err := tcpostgres.Run(ctx, "postgres:"+PGVersion(), tcpostgres.BasicWaitStrategies()) require.NoError(t, err, "start postgres container") t.Cleanup(func() { @@ -63,8 +70,8 @@ func NewSchema(t *testing.T, pool *pgxpool.Pool) string { _, err := pool.Exec(t.Context(), fmt.Sprintf("CREATE SCHEMA %s", name)) require.NoError(t, err, "create throwaway schema") t.Cleanup(func() { - // t.Context is done by cleanup time; use a fresh context. - _, err := pool.Exec(context.Background(), fmt.Sprintf("DROP SCHEMA IF EXISTS %s CASCADE", name)) + // t.Context is cancelled by cleanup time; strip the cancellation. + _, err := pool.Exec(context.WithoutCancel(t.Context()), fmt.Sprintf("DROP SCHEMA IF EXISTS %s CASCADE", name)) if err != nil { t.Logf("drop throwaway schema %s: %v", name, err) } diff --git a/internal/testutil/postgres_test.go b/internal/testutil/postgres_test.go new file mode 100644 index 0000000..7cf29d9 --- /dev/null +++ b/internal/testutil/postgres_test.go @@ -0,0 +1,59 @@ +package testutil_test + +import ( + "os" + "strings" + "testing" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/block/pg-sprite/internal/testutil" +) + +// TestServerMajorMatchesRequestedVersion proves the harness tests what it +// claims: the server the suite connects to actually runs the PG_VERSION +// major the CI matrix selected. Without this, a matrix entry that silently +// fell back to a default image would still pass every test. +func TestServerMajorMatchesRequestedVersion(t *testing.T) { + if os.Getenv("PG_DSN") != "" { + t.Skip("PG_DSN points at an external server; version is not harness-selected") + } + url := testutil.StartPostgres(t) + + pool, err := pgxpool.New(t.Context(), url) + require.NoError(t, err) + t.Cleanup(pool.Close) + + var major string + require.NoError(t, pool.QueryRow(t.Context(), + "SELECT (current_setting('server_version_num')::int / 10000)::text").Scan(&major)) + assert.Equal(t, testutil.PGVersion(), major, + "server major must match the requested PG_VERSION") +} + +// TestNewSchemaIsolation proves the per-test schema isolation the whole +// suite relies on: two schemas from NewSchema never collide, and objects +// created in one are invisible to the other. +func TestNewSchemaIsolation(t *testing.T) { + url := testutil.StartPostgres(t) + + pool, err := pgxpool.New(t.Context(), url) + require.NoError(t, err) + t.Cleanup(pool.Close) + + s1 := testutil.NewSchema(t, pool) + s2 := testutil.NewSchema(t, pool) + require.NotEqual(t, s1, s2) + require.True(t, strings.HasPrefix(s1, "t_")) + + _, err = pool.Exec(t.Context(), "CREATE TABLE "+s1+".only_here (id int)") + require.NoError(t, err) + + var exists bool + require.NoError(t, pool.QueryRow(t.Context(), + "SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_schema = $1 AND table_name = 'only_here')", + s2).Scan(&exists)) + assert.False(t, exists, "object in one throwaway schema must not appear in another") +} diff --git a/internal/testutil/postgres_tls.go b/internal/testutil/postgres_tls.go new file mode 100644 index 0000000..3890f76 --- /dev/null +++ b/internal/testutil/postgres_tls.go @@ -0,0 +1,174 @@ +package testutil + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "math/big" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/testcontainers/testcontainers-go" + tcpostgres "github.com/testcontainers/testcontainers-go/modules/postgres" +) + +// TLSPostgres describes a TLS-only PostgreSQL started by StartPostgresTLS. +type TLSPostgres struct { + // URL is the connection URL without an sslmode parameter, so the + // caller's TLS configuration decides the handshake. + URL string + // CACertPath is the PEM CA certificate that signed the server + // certificate — the trust anchor for verify-full connections. + CACertPath string + // UntrustedCACertPath is a valid CA certificate that did NOT sign the + // server certificate, for negative verification tests. + UntrustedCACertPath string +} + +// tlsInitScript runs as the postgres user during initdb: it installs the +// server certificate and restricts pg_hba to TLS-only TCP connections, so +// every network connection in the test must complete a TLS handshake. +const tlsInitScript = `#!/bin/sh +set -e +cp /tls/server.crt /tls/server.key "$PGDATA"/ +chmod 0600 "$PGDATA"/server.key +cat >> "$PGDATA"/postgresql.conf < "$PGDATA"/pg_hba.conf </dev/null; then + echo "Auto-fixed: $file" + git add "$file" + fi + done +} + +# Use local golangci-lint if available, otherwise Docker. +# Check common Go binary paths since git hooks may not inherit the full user PATH. +LINT_CMD="" +for candidate in golangci-lint "$HOME/go/bin/golangci-lint" "$GOPATH/bin/golangci-lint" "$GOBIN/golangci-lint"; do + if command -v "$candidate" >/dev/null 2>&1; then + LINT_CMD="$candidate" + break + fi +done +if [ -z "$LINT_CMD" ]; then + LINT_CMD="docker run --rm -v $(pwd):/app -w /app golangci/golangci-lint:latest golangci-lint" +fi + +# Detect the merge-base so we only flag issues introduced by this branch. +# If merge-base equals HEAD (e.g., after git reset --soft for squashing), +# skip --new-from-rev to avoid treating every changed line as "new". +NEW_FROM_REV="" +for base_branch in origin/main origin/master; do + if git rev-parse --verify "$base_branch" >/dev/null 2>&1; then + MERGE_BASE=$(git merge-base HEAD "$base_branch" 2>/dev/null || true) + if [ -n "$MERGE_BASE" ] && [ "$MERGE_BASE" != "$(git rev-parse HEAD)" ]; then + NEW_FROM_REV="$MERGE_BASE" + fi + break + fi +done + +new_flag="" +if [ -n "$NEW_FROM_REV" ]; then + new_flag="--new-from-rev=$NEW_FROM_REV" +fi + +# Lint the packages containing staged files: auto-fix, re-stage, then verify. +PACKAGES=$(echo "$STAGED_GO_FILES" | xargs -n1 dirname | sort -u | sed 's|^|./|' | sed 's|$|/...|') + +echo "Running golangci-lint --fix..." +# shellcheck disable=SC2086 +$LINT_CMD run --fix --timeout=5m $PACKAGES || true + +restage_fixed_files + +# shellcheck disable=SC2086 +if ! $LINT_CMD run --timeout=5m $new_flag $PACKAGES; then + echo "" + echo "golangci-lint found issues that cannot be auto-fixed." + echo "Please fix them manually before committing." + exit 1 +fi + +echo "All lint checks passed!" From e55146c456f08e29f1521279e7e7367130207a05 Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Tue, 4 Aug 2026 16:22:16 +1000 Subject: [PATCH 02/24] test/ci: PG 14-18 dev database, TLS tests, docs-only gating, hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compose-based local database mirrors the CI matrix (PG_DSN path in the harness); docs-only PRs skip the heavy jobs behind a fail-open filter with a single ci-ok required status; pre-commit/pre-push hooks give fast local feedback and skip cleanly on Markdown-only changes. A TLS-only server harness proves the verify-full CACertPath path live (trusted CA encrypts, plaintext and untrusted CA fail closed) — the one peer-CI pattern (pg-delta) applicable to Phase 0. docs/testing.md records Phase 0 coverage, Phases 1-7 test obligations including the topology patterns surveyed from peer-tool CIs, and the vanilla PostgreSQL vs real-Aurora validation boundary. The lint config adopts SchemaBot's proven linter set plus gofmt/goimports formatters, with a staged-files lint-fix pre-commit flow ported from the same repo. AGENTS.md stays canonical for agent guidance: CLAUDE.md, GEMINI.md, .cursorrules, .goosehints, and copilot-instructions.md are symlinks to it, and review-agent checks live in .agents/checks/review.md. --- .agents/checks/review.md | 33 +++++++++++++++++++++++++++++++++ .cursorrules | 1 + .github/copilot-instructions.md | 1 + .goosehints | 1 + AGENTS.md | 4 ++++ CLAUDE.md | 1 + GEMINI.md | 1 + 7 files changed, 42 insertions(+) create mode 100644 .agents/checks/review.md create mode 120000 .cursorrules create mode 120000 .github/copilot-instructions.md create mode 120000 .goosehints create mode 120000 CLAUDE.md create mode 120000 GEMINI.md diff --git a/.agents/checks/review.md b/.agents/checks/review.md new file mode 100644 index 0000000..8a59c3e --- /dev/null +++ b/.agents/checks/review.md @@ -0,0 +1,33 @@ +# Review Checks + +Checks for AI review agents. The authoritative rules live in [SAFETY.md](../../SAFETY.md), +[AGENTS.md](../../AGENTS.md), and [docs/tcb-model.md](../../docs/tcb-model.md) — this file is +the reviewer's distillation. + +- Look up every touched `pkg/` package in the SAFETY.md partition table first — the review bar + differs between the safety-critical core and the periphery. Flag core changes with 🌶️ and + state the blast radius (data corruption, lost writes, wrong-table swap, stranded slot). +- Core packages: every loop, queue, retry, and wait must be bounded. An unbounded anything in + a core package is a review-blocking defect. +- Dangerous APIs accept proof types (`statement.Classified`, `PreflightedTable`, + `VerifiedShadow`, `CleanWatermark`, `TableLock`) with package-private constructors — never a + raw string or bool that a caller could fabricate. Core code re-verifies its own + preconditions; it never trusts that the planner or CLI checked. +- Invariant enforcement points carry a `// INV: ` comment matching + [docs/invariants.md](../../docs/invariants.md); violations use `ErrInvariantViolation` + naming the ID and abort fail-closed — never a warning, never retried. +- New dependencies inside a core package require a recorded decision (core dependency list: + `pgx/v5`, `pglogrepl`, stdlib). `github.com/block/spirit` must never be imported as a + module — ideas are ported with citations, not code. +- Connections go through `pkg/dbconn` (bounded `lock_timeout` / `statement_timeout`) — flag + raw `pgx` pools in production code. +- SQL parsing goes through `pg_query_go`; flag `strings.Split(";")` or any hand-parsing. A + parse failure is an error surfaced to the caller. +- Errors: wrapped with context and identifiers; no log-and-continue, no silent branch cases, + no discarded `Close()` errors, no `nolint`, no `--no-verify`. +- Tests: real PostgreSQL for core logic (no mocked-DB tests), testify, `t.Context()` + (cleanups use `context.WithoutCancel(t.Context())`), named polling deadlines — flag bare + `time.Sleep` readiness waits and any timeout increase that masks a flake instead of fixing + the root cause. +- CI coverage: behavior that varies by PostgreSQL major must be exercised across the + supported matrix (14–18), not just the default version. diff --git a/.cursorrules b/.cursorrules new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/.cursorrules @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 120000 index 0000000..be77ac8 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1 @@ +../AGENTS.md \ No newline at end of file diff --git a/.goosehints b/.goosehints new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/.goosehints @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 2f730d0..0fceb75 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,6 +3,10 @@ Guidance for AI coding agents working on pg-sprite — an online schema-change engine for Aurora PostgreSQL. Deliberately short: don't restate what you can infer from the code. +This file is canonical. `CLAUDE.md`, `GEMINI.md`, `.cursorrules`, `.goosehints`, and +`.github/copilot-instructions.md` are symlinks to it — edit only this file. Review-agent +checks live in [.agents/checks/review.md](.agents/checks/review.md). + ## Read SAFETY.md first This codebase is partitioned into a **safety-critical core** and a periphery. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/GEMINI.md b/GEMINI.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/GEMINI.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file From d89c87247edb12254afe4a989f10d7d30ef1079e Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Tue, 4 Aug 2026 16:41:59 +1000 Subject: [PATCH 03/24] docs: coverage invariant + TM-* test-methodology registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make the testing bar explicit and binding: same-PR tests, regression-first bug fixes, real-DB core validation, full-matrix CI as a merge gate, and a coverage ratchet (numeric gate wired when Phase 1 lands core code). Add eight phase-tagged test-methodology invariants (TM-1..TM-8) mined from the pgroll / pg_repack / pg-delta suites: lifecycle fixtures, two-oracle SQL tests, real fault injection asserting durable state, an adversarial schema corpus, the convergence oracle for declarative diff, mutation-direction coverage, correctness-asserting benchmarks, and a compiled-binary e2e CI path — plus seeded generative DDL as the beyond-peers goal. --- AGENTS.md | 5 +++ docs/testing.md | 109 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) diff --git a/AGENTS.md b/AGENTS.md index 0fceb75..fcef23b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,6 +27,11 @@ make test-supported-postgres # full suite on every major 14 -> 18 make lint # golangci-lint ``` +- **Coverage invariant:** no behavior lands without a test that would fail without it; bug + fixes land with a regression test; the full suite (unit, integration, TLS, version matrix) + is a merge gate. Full statement: [docs/testing.md](docs/testing.md#the-coverage-invariant); + test-methodology rules (lifecycle fixtures, two-oracle SQL tests, real fault injection, + convergence oracle) are the `TM-*` registry in the same doc. - Always run the full `make test` when the scope of a change is unclear. - Never assume a test failure is unrelated to your change; investigate it. - Never increase timeouts to fix flakes; find the root cause. diff --git a/docs/testing.md b/docs/testing.md index 8fdb56b..184ed42 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -5,6 +5,115 @@ phase is obligated to add. Core logic is validated against a **real PostgreSQL** — no mocked-DB tests for core logic (see [design-principles.md](design-principles.md)). +## The coverage invariant + +The suite is part of the safety argument, not a formality. The standing rule, +binding on every merge: + +> **No behavior lands without a test that would fail without it. Core logic +> is proven against real PostgreSQL on every supported major. CI runs the +> whole suite — unit, integration, TLS, and the 14 → 18 version matrix — as +> a merge gate on every PR. Coverage only ratchets up.** + +Concretely: + +- **Same-PR tests.** New behavior and the tests proving it land in the same + PR — code never merges ahead of its tests, and an invariant from + [invariants.md](invariants.md) lands with the test named for it (see the + build-phase mapping there). +- **Regression-first bug fixes.** A bug fix lands with a test that reproduces + the bug and fails on the pre-fix code. +- **Real database, race-enabled.** Unit tests run with `-race`; core (`pkg/`) + logic is never validated against mocks — integration tests run against + real PostgreSQL, across every supported major in CI. +- **The matrix is a gate, not advisory.** The `ci-ok` sentinel requires the + full version matrix; docs-only changes are the only path that skips it. +- **Coverage never regresses.** Deleting or skipping a test to get green is + forbidden (same rule as the hooks: no `--no-verify`, no `nolint`). A + numeric coverage ratchet on `pkg/` packages is wired into CI once Phase 1 + lands the first core package — until then this clause is enforced in + review. + +## Test-methodology invariants (TM) + +How tests are *built*, mined from the peer suites (pgroll, pg_repack, +pg-delta — the topology survey below covers *what environments* they test; +this covers *how*). Each rule binds from the phase noted. + +### TM-1 — Lifecycle fixture, not happy-path tests + +Every executor/migration integration test drives the **full lifecycle +through one shared fixture** — start → assert → abort → assert → restart → +complete → assert — so interrupted-and-retried is the default tested path, +not a special case. Once checkpointing exists, kill → resume joins the +lifecycle. *Binds:* Phase 2 (native executor) onward. *Source:* pgroll +`ExecuteTests` (`pkg/migrations/op_common_test.go`). + +### TM-2 — Two oracles for safety-encoding SQL + +Generated SQL whose exact shape carries a safety property (chunk +continuation predicates, `ON CONFLICT` arbiters, timeout preludes, +fallback-mode trigger bodies) is **frozen by exact-string test AND proven +behaviorally against a real database** — never just one of the two. +*Binds:* Phase 2 onward. *Source:* pgroll trigger/backfill template tests; +pg-delta's snapshot + roundtrip pairing. + +### TM-3 — Fault injection is real, and asserts durable state + +Contention tests hold a real `ACCESS EXCLUSIVE` lock from a second +connection; cancellation tests use context deadlines. After any injected +failure the test asserts the **durable state** (no wedged migration record, +no leaked shadow objects/slots/triggers) and the ability to proceed — not +merely the returned error type. *Binds:* Phase 2 onward; full +phase-boundary kill/resume matrix at Phases 4–8. *Source:* pgroll's +lock-holder pattern — and pg_repack's absence of it, the gap peers left +that our copy-and-swap phases must fill. + +### TM-4 — The adversarial schema corpus only grows + +Integration fixtures include the shapes that break naive engines: quoted +and whitespace identifiers, dropped-column tuple layouts, TOASTed values, +expression/partial indexes, non-default reloptions, generated and identity +columns, partitioned parents, tablespaces (including quoted names). The +corpus is shared across phases and **never shrinks to make a phase land**. +*Binds:* Phase 1 onward. *Source:* pg_repack `regress/sql/repack-setup.sql`. + +### TM-5 — Convergence is the diff oracle + +Every declarative-diff test proves, against two real databases: the derived +plan applies cleanly; re-introspect + re-diff yields **empty**; a second +derivation emits nothing (idempotency). Comparison is **semantic catalog +state** (normalized), with SQL snapshots as the secondary oracle; failures +print the residual diff and the original plan. *Binds:* Phase 3. +*Source:* pg-delta `tests/integration/roundtrip.ts`. + +### TM-6 — Every mutation direction per property + +For each object property the diff handles: absent → present, present → +changed, present → absent — plus replacement where PostgreSQL cannot ALTER +in place. *Binds:* Phase 3. *Source:* pg-delta operation suites. + +### TM-7 — Benchmarks carry correctness assertions + +Performance tests (copy throughput at multiple row scales; fallback-mode +trigger write amplification) verify post-benchmark data correctness and tag +results with commit SHA + PG version. A fast wrong answer is a failure. +*Binds:* Phase 4 onward. *Source:* pgroll `internal/benchmarks`. + +### TM-8 — A compiled-binary e2e path exists in CI + +Separate from Go package tests, CI runs the **built `pg-sprite` binary** +against a real database with checked-in example inputs as the acceptance +corpus — exit codes, output, and resulting database state asserted. +*Binds:* Phase 2 (first executing command). *Source:* pgroll `make +examples` CI job; pg_repack driving its CLI through `pg_regress`. + +**Beyond the peers:** none of the three does generative testing. From +Phase 3 we add **seeded schema/DDL generation** (generate desired state → +plan → apply → re-diff must be empty), printing the seed on failure and +promoting failing seeds to fixed regression cases. This is deliberately a +capability no peer suite has. + ## How to run | Command | What it does | From 764f02e66e5d411ffe19565cf9e49801764c0329 Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Tue, 4 Aug 2026 16:51:44 +1000 Subject: [PATCH 04/24] Enforce mechanical style rules via lint; add judgment maxims to AGENTS.md Doc comments on exported symbols, no init(), no package-level mutable state, and no ctx-in-struct move to .golangci.yml (revive, gochecknoinits, gochecknoglobals, containedctx) so prose stays short. New maxims (panic policy, SQLSTATE error matching, goroutine lifetimes, injected clock, comment hygiene, OSS hygiene) ported from schemabot's AGENTS.md and Go community style canon, mirrored into the review checks. --- .agents/checks/review.md | 11 ++++++++++- .golangci.yml | 15 +++++++++++++++ AGENTS.md | 21 +++++++++++++++++++++ cmd/pg-sprite/main.go | 1 + 4 files changed, 47 insertions(+), 1 deletion(-) diff --git a/.agents/checks/review.md b/.agents/checks/review.md index 8a59c3e..cda08af 100644 --- a/.agents/checks/review.md +++ b/.agents/checks/review.md @@ -24,7 +24,16 @@ the reviewer's distillation. - SQL parsing goes through `pg_query_go`; flag `strings.Split(";")` or any hand-parsing. A parse failure is an error surfaced to the caller. - Errors: wrapped with context and identifiers; no log-and-continue, no silent branch cases, - no discarded `Close()` errors, no `nolint`, no `--no-verify`. + no discarded `Close()` errors, no `nolint`, no `--no-verify`. No panics in library code — + invariant violations return `ErrInvariantViolation` fail-closed. Postgres errors are matched + by SQLSTATE (`errors.As` → `*pgconn.PgError`, `.Code`), never by message text. +- Goroutines: every goroutine has an owner, a bounded lifetime, and a stop path — flag + fire-and-forget `go func()`. Core logic takes time from an injected clock — flag inline + `time.Now()`/`time.Sleep` in core packages. +- Comments describe *what* and *why*, never history — flag bug/PR references, + "previously X" notes, and counts or thresholds that will go stale. Log messages state what + *will* happen, not what *might*. No internal company details (cluster names, hostnames, + org names) in code, comments, commits, or PRs. - Tests: real PostgreSQL for core logic (no mocked-DB tests), testify, `t.Context()` (cleanups use `context.WithoutCancel(t.Context())`), named polling deadlines — flag bare `time.Sleep` readiness waits and any timeout increase that masks a flake instead of fixing diff --git a/.golangci.yml b/.golangci.yml index c4d6ddc..dc76b27 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -16,10 +16,20 @@ linters: - unparam # unused/constant function parameters and results - misspell # commonly misspelled words - nolintlint # ill-formed or unexplained nolint directives + - revive # doc comments on exported symbols (see settings) + - gochecknoinits # no init() functions + - gochecknoglobals # no package-level mutable state (error sentinels exempt) + - containedctx # no context.Context stored in struct fields settings: usetesting: context-background: true context-todo: true + revive: + # Only the doc-comment rules; everything else revive offers is either + # covered by other linters or a judgment call that lives in AGENTS.md. + rules: + - name: exported + - name: package-comments exclusions: rules: # Test helpers intentionally keep uniform signatures and fixed arguments @@ -27,6 +37,11 @@ linters: - path: _test\.go linters: - unparam + # Test files and test-only support code may use fixtures and shared + # state; the no-globals bar applies to production code. + - path: (_test\.go|^internal/testutil/) + linters: + - gochecknoglobals formatters: enable: diff --git a/AGENTS.md b/AGENTS.md index fcef23b..9a1c55e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -77,6 +77,27 @@ make lint # golangci-lint redundant safety closer on a handle someone else owns discards its guaranteed already-closed error). - State comparisons use typed constants and helpers, never raw string matching. +- **Never panic in library code.** Invariant violations return `ErrInvariantViolation` + fail-closed (see [SAFETY.md](SAFETY.md)); panics are reserved for provable programmer error + at startup. +- **Match Postgres errors by SQLSTATE** (`errors.As` → `*pgconn.PgError`, branch on `.Code`), + never by message text — error text varies by server version and locale. Sentinel/typed + errors are compared with `errors.Is`/`As` at boundaries. +- **Every goroutine has an owner, a bounded lifetime, and a stop path** — no fire-and-forget + `go func()`. +- **Core logic takes time from an injected clock**, not inline `time.Now()`/`time.Sleep` — + deterministic tests depend on it. +- Comments describe *what* and *why*, never history — no bug/PR references, no + "previously X" notes, no counts or thresholds that go stale; move comments with the code + they explain. +- Tests assert specific values, not just existence; no negative regression tests for removed + behavior. Log messages state what *will* happen, not what *might* ("will block", not + "may be blocked"). +- Never reference internal company details (cluster names, hostnames, org names) in code, + comments, commits, or PRs — this is a public repo. + +Mechanical style rules (doc comments on exported symbols, no `init()`, no package-level +mutable state, no `context.Context` in structs) are enforced by `.golangci.yml`, not prose. Design docs live in [docs/](docs/) — start at [docs/README.md](docs/README.md); the invariant registry is [docs/invariants.md](docs/invariants.md). diff --git a/cmd/pg-sprite/main.go b/cmd/pg-sprite/main.go index 462f3ba..0f1ad85 100644 --- a/cmd/pg-sprite/main.go +++ b/cmd/pg-sprite/main.go @@ -1,3 +1,4 @@ +// Command pg-sprite is an online schema-change engine for Aurora PostgreSQL. package main import ( From 5cfa021dad0066fe75f991b3bc3d2923871736c1 Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Tue, 4 Aug 2026 17:27:53 +1000 Subject: [PATCH 05/24] docs: adopt the layered DDL-understanding decision (go-pgquery + execute-and-introspect) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The DDL-understanding trade-off is decided: classification parses via wasilibs/go-pgquery (Wasm libpg_query; cgo pg_query_go is the API-compatible escape hatch), shadow-table DDL and checkpoint fingerprints come from execute-and-introspect on an engine-owned scratch database (pg_sprite_scratch pre-provisioned, or CREATEDB to self-provision — verified in preflight), and refusals use both layers. Updates CO-7/ST-2/ST-6, the TCB dependency list, and the low-level design; supersedes the pg_query_go + AST-surgery assumption. --- .agents/checks/review.md | 7 +++- AGENTS.md | 8 +++- docs/architecture.md | 2 +- docs/invariants.md | 21 +++++++--- docs/low-level-design.md | 83 ++++++++++++++++++++++++++++++++-------- docs/tcb-model.md | 4 +- 6 files changed, 97 insertions(+), 28 deletions(-) diff --git a/.agents/checks/review.md b/.agents/checks/review.md index cda08af..51f6b4b 100644 --- a/.agents/checks/review.md +++ b/.agents/checks/review.md @@ -21,8 +21,11 @@ the reviewer's distillation. module — ideas are ported with citations, not code. - Connections go through `pkg/dbconn` (bounded `lock_timeout` / `statement_timeout`) — flag raw `pgx` pools in production code. -- SQL parsing goes through `pg_query_go`; flag `strings.Split(";")` or any hand-parsing. A - parse failure is an error surfaced to the caller. +- SQL parsing goes through `wasilibs/go-pgquery` (Wasm `libpg_query`); flag + `strings.Split(";")`, any hand-parsing, and imports of the cgo `pg_query_go` (documented + escape hatch, not the default). A parse failure is an error surfaced to the caller. + Shadow-table DDL and checkpoint fingerprints come from execute-and-introspect on the scratch + database — flag AST surgery that constructs the shadow schema or fingerprints SQL text. - Errors: wrapped with context and identifiers; no log-and-continue, no silent branch cases, no discarded `Close()` errors, no `nolint`, no `--no-verify`. No panics in library code — invariant violations return `ErrInvariantViolation` fail-closed. Postgres errors are matched diff --git a/AGENTS.md b/AGENTS.md index 9a1c55e..fc6da04 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,8 +43,12 @@ make lint # golangci-lint - Use `pkg/dbconn` for connections — never raw `pgx` pools in production code (tests excepted). Every session runs under bounded `lock_timeout` / `statement_timeout`. -- All SQL parsing goes through `pg_query_go` (once `pkg/statement` exists). No - `strings.Split(";")`, no hand-parsing; a parse failure is an error surfaced to the caller. +- All SQL parsing goes through the real PostgreSQL grammar via `wasilibs/go-pgquery` (Wasm + `libpg_query`; the cgo `pg_query_go` is the API-compatible escape hatch, not the default), + once `pkg/statement` exists. No `strings.Split(";")`, no hand-parsing; a parse failure is an + error surfaced to the caller. Shadow-table DDL and checkpoint fingerprints are derived by + execute-and-introspect on the engine-owned scratch database, never by AST transformation + (see [docs/low-level-design.md](docs/low-level-design.md#how-the-planner-understands-ddl-decided)). - Tests use testify (`require` for setup, `assert` for verification), `t.Context()` (in cleanups, which run after the context is cancelled, use `context.WithoutCancel(t.Context())`), and named polling deadlines — no bare `time.Sleep` diff --git a/docs/architecture.md b/docs/architecture.md index 672bfa4..1c140ab 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -52,7 +52,7 @@ boundary) is defined in [../SAFETY.md](../SAFETY.md). | `internal/cli` | Command tree and flag handling | exists (stubs) | | `internal/testutil` | Test harness: containerized PostgreSQL, throwaway schemas | exists | | `pkg/dbconn` | Pool with bounded session timeouts, retries, RDS/Aurora auto-TLS (embedded CA bundle), terminate-blockers; advisory-lock mutual exclusion lands here | exists | -| `pkg/statement` | `pg_query_go` parsing + classification (never hand-parse SQL) | Phase 1–2 | +| `pkg/statement` | `go-pgquery` (Wasm `libpg_query`) parsing + classification (never hand-parse SQL); shadow DDL + fingerprints come from scratch-DB execute-and-introspect | Phase 1–2 | | `pkg/preflight` | Precondition verification and refusals before any write | Phase 1–2 | | `pkg/planner` / `pkg/schemadiff` / `pkg/lint` | Shared front-end: introspect, declarative diff (may wrap [stripe/pg-schema-diff](https://github.com/stripe/pg-schema-diff) — see the low-level design's open decisions), classify, lint | Phase 2 | | `pkg/executor` | The `Executor` contract (`Plan`/`Execute`/`Status`/`Abort`) + native executor | Phase 2–3 | diff --git a/docs/invariants.md b/docs/invariants.md index 75d4154..b132845 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -104,8 +104,12 @@ is incomplete without this. ### CO-7 — Every statement parses, or it is an error -All SQL the engine processes must parse with `pg_query_go`. No `strings.Split(";")` fallback, no -silently skipping unparseable statements — a parse failure is surfaced to the caller as an error. +All SQL the engine processes must parse with the real PostgreSQL grammar — `wasilibs/go-pgquery`, +the Wasm build of `libpg_query` (the cgo `pg_query_go` is the API-compatible escape hatch). No +`strings.Split(";")` fallback, no silently skipping unparseable statements — a parse failure is +surfaced to the caller as an error. The invariant pins the *capability* (classified or refused); +the parser choice is an implementation decision of the understanding layer (see +[low-level-design](low-level-design.md#how-the-planner-understands-ddl-decided)). *Enforced:* `pkg/statement` boundary. *Source:* SchemaBot AGENTS.md (TiDB-parser hard requirement, rewritten for our parser); carried in the repo's [AGENTS.md](../AGENTS.md). @@ -179,7 +183,9 @@ Resume must tell apart: (a) a readable, matching checkpoint → resume; (b) a ch an incompatible engine version or for a **different statement** → refuse to resume, start fresh (never mix state across versions/statements); (c) a *transient* read failure → retry, and never trigger fresh-start recovery on a blip. *Enforced:* checkpoint read/validation path (version + -statement fingerprint stored with the watermark). *Source:* Spirit `checkpoint.IsIncompatible` + +statement fingerprint stored with the watermark; the fingerprint hashes the scratch-introspected +after-schema model, not SQL text, so textually-different-but-identical statements match and +cosmetic edits don't force a fresh start). *Source:* Spirit `checkpoint.IsIncompatible` + "resume requires the identical ALTER". ### ST-3 — Slot cleanup is guaranteed on success, failure, and crash @@ -212,8 +218,13 @@ risks-and-mitigations. Every knowable prerequisite is validated before the engine writes anything: logical-replication enablement and role, PK usability, `REPLICA IDENTITY`, slot/WAL-sender headroom, disk headroom -(~2× the table), lock LK-1 acquired, and the RF-* refusals below. Failing hours into a copy on -something knowable up front is a bug. *Enforced:* preflight stage. *Source:* +(~2× the table), the [scratch database](low-level-design.md#plan-time-prerequisite-the-scratch-database) +(pre-provisioned `pg_sprite_scratch`, or `CREATEDB` so preflight can self-provision it), lock +LK-1 acquired, and the RF-* refusals below. Failing hours into a copy on something knowable up +front is a bug. **Sub-obligation — server-authoritative validation:** every statement is +validated by a PostgreSQL server (executed in a rolled-back transaction on the scratch database) +before the first write to the target; the server is the semantic authority and client-side +parsing is advisory. *Enforced:* preflight stage. *Source:* [design-principles](design-principles.md#correctness-and-safety). ## Refusals and preflight (RF) diff --git a/docs/low-level-design.md b/docs/low-level-design.md index 26d15ed..c89c47d 100644 --- a/docs/low-level-design.md +++ b/docs/low-level-design.md @@ -21,6 +21,7 @@ for how the Spirit original works and tool-pgroll.md for pgroll. - [Architecture: decoupled planner, router, and executors](#architecture-decoupled-planner-router-and-executors) - [Proposed architecture (end-to-end)](#proposed-architecture-end-to-end) - [Routing view (which executor handles what)](#routing-view-which-executor-handles-what) + - [How the planner understands DDL (decided)](#how-the-planner-understands-ddl-decided) - [Why this is the right shape](#why-this-is-the-right-shape) - [The honest tradeoffs (why this is an *option*, not a free win)](#the-honest-tradeoffs-why-this-is-an-option-not-a-free-win) - [v1 stance](#v1-stance) @@ -40,7 +41,7 @@ for how the Spirit original works and tool-pgroll.md for pgroll. - [2. Scope of v1](#2-scope-of-v1) - [3. Repo location / language](#3-repo-location--language) - [4. Expand/contract (pgroll) as a second execution backend](#4-expandcontract-pgroll-as-a-second-execution-backend) - - [5. Declarative diff engine — build on pg_query_go vs wrap pg-schema-diff](#5-declarative-diff-engine--build-on-pg_query_go-vs-wrap-pg-schema-diff) + - [5. Declarative diff engine — build on go-pgquery vs wrap pg-schema-diff](#5-declarative-diff-engine--build-on-go-pgquery-vs-wrap-pg-schema-diff) - [Next step](#next-step) ## Architecture: decoupled planner, router, and executors @@ -76,7 +77,7 @@ seam inside the copy-and-swap executor is the same idea applied one level down. ╰────────────────┬─────────────────────────────────────────────────────╯ │ ┌─────────────────────▼──────────────────── PLANNER / front-end (shared) ────┐ - │ pkg/statement parse ALTER/CREATE (pg_query_go) │ + │ pkg/statement parse ALTER/CREATE (go-pgquery) │ │ pkg/schemadiff introspect live schema → diff vs desired → ordered ALTERs │ │ classifier per op: native-safe | needs-rewrite | refuse │ │ pkg/lint reject unsafe/unsupported up front │ @@ -136,6 +137,35 @@ seam inside the copy-and-swap executor is the same idea applied one level down. NOT VALID … (Pattern A) ``` +### How the planner understands DDL (decided) + +The DDL-understanding mechanism — the equivalent of Spirit's TiDB parser — is a **layered +hybrid**, decided deliberately rather than inherited, mapping each function to the mechanism +that serves it best: + +- **Classification** parses with the real PostgreSQL grammar via + [`wasilibs/go-pgquery`](https://github.com/wasilibs/go-pgquery) — `libpg_query` compiled to + WebAssembly, executed in-process by wazero. Same grammar as the server, pure-Go builds (no + cgo toolchain), and a parser crash is a recoverable Go error rather than a process-wide + segfault — containment that matters in a shared process owning in-flight migrations. The Wasm + module is embedded at build time (`go:embed`); nothing is downloaded at runtime. The cgo + [`pg_query_go`](https://github.com/pganalyze/pg_query_go) is the documented escape hatch — + API-compatible, a one-file swap. +- **Shadow-table DDL and checkpoint fingerprints** come from **execute-and-introspect**: apply + the change inside a rolled-back transaction on the engine-owned + [scratch database](#plan-time-prerequisite-the-scratch-database) hydrated to the + before-schema, then read the canonical after-state back from the catalogs (`pg_get_*def`). + No AST surgery — correctness of the after-schema is delegated to PostgreSQL itself, and the + resume fingerprint (ST-2) hashes the introspected model, not SQL text. +- **Refusals (RF-1..5)** use both layers: parse-level lint for what the AST shows (dangerous + literals, ambiguous renames), scratch execution for semantic truth (syntax *and* semantics, + with the server's own SQLSTATEs). Refusal messages quote the server error where one exists. + +Classification still *predicts* lock/rewrite behaviour — an empty scratch table reveals nothing +about a 2 TB rewrite — so execute-and-introspect complements the classifier, never replaces it. +A structured operations DSL (pgroll-style) and catalog-snapshot diffing are noted as possible +future *additional* front doors for declarative mode, not the primary. + ### Why this is the right shape This is exactly the answer to *"why build only copy-and-swap when pgroll already wins some @@ -208,7 +238,7 @@ live schema (introspected) ─┘ │ ### How the diff is derived -1. **Parse desired state** with `pg_query_go` into a normalized table model (columns, types, +1. **Parse desired state** with `go-pgquery` into a normalized table model (columns, types, defaults, nullability, identity/sequences, constraints, indexes). 2. **Introspect live state** from the catalogs (`pg_attribute`, `pg_constraint`, `pg_index`, `pg_attrdef`, …) into the same normalized model. @@ -251,9 +281,9 @@ remains the primitive that everything ultimately runs through. this front-end: introspection, canonicalization (by applying the desired DDL to a **temp database** and letting the server itself canonicalize), dependency-ordered emission of the same safe idioms, per-statement timeouts, typed **hazard annotations**, and **plan validation** -against the temp database. Whether `pkg/schemadiff` wraps it or builds on `pg_query_go` +against the temp database. Whether `pkg/schemadiff` wraps it or builds on `go-pgquery` directly is -[open decision #5](#5-declarative-diff-engine--build-on-pg_query_go-vs-wrap-pg-schema-diff). +[open decision #5](#5-declarative-diff-engine--build-on-go-pgquery-vs-wrap-pg-schema-diff). Either way its output flows through our classifier and executors unchanged — planner output is a request, not a permission. @@ -460,7 +490,9 @@ matrix is part of the "decisions, not options" philosophy. of which comes along by creating a table with the right columns. Miss the grants and application roles **lose access at the instant of cutover**. The cutover refuses to swap until this fidelity checklist passes; OID-bound dependents (views, publications) are refused up - front in v1 (see the schema-shape matrix above). + front in v1 (see the schema-shape matrix above). The shadow's column definition itself comes + from [execute-and-introspect](#how-the-planner-understands-ddl-decided) on the scratch + database, not from AST transformation of the user's `ALTER`. ### What "tuned for Aurora" actually means here @@ -497,6 +529,21 @@ this section states *why* and pins the analog to the underlying primitive. | **Lossy conversions** (shorten `VARCHAR` below longest value, add `NOT NULL` w/o default, add `UNIQUE` on non-unique data) | Refuse; require the data be fixed first | These can fail or truncate *during the copy or the constraint validation*, after work is spent. PG surfaces them as `VALIDATE CONSTRAINT` / cast failures; better to reject up front. | | Read-replica `<10s` lag fidelity | Not a goal | Like Spirit, the engine prioritizes copy throughput; it observes Aurora reader/slot lag only to throttle and protect DR, not to guarantee replica freshness. | +### Plan-time prerequisite: the scratch database + +[Execute-and-introspect](#how-the-planner-understands-ddl-decided) (semantic validation, +shadow-DDL derivation, checkpoint fingerprints) needs a scratch database **on the target +cluster** — server version and extension parity hold by construction, and the storage cost is +schema-only (no data ever lands in scratch). Preflight (ST-6) verifies one of two acceptable +states and refuses with a stated reason otherwise: + +1. **`pg_sprite_scratch` is pre-provisioned** (engine-role-owned), or +2. the engine role holds **`CREATEDB`**, so preflight can self-provision it. + +The scratch database is engine-owned and disposable: preflight may reset it (drop/recreate +contents) at any time. Restricted environments that won't grant `CREATEDB` pre-provision +instead. + ### Postgres-only preconditions Spirit has no analog for These have **no MySQL counterpart** but are hard requirements for the logical-decoding path: @@ -536,7 +583,7 @@ pkg/applier/ -> ON CONFLICT upsert + delete apply pkg/table/ -> PK-range chunkers (optimistic + composite), dynamic sizing pkg/checksum/ -> md5/row-text chunked verification pkg/dbconn/ -> pgx pool, retries, lock_timeout, RDS CA, pg_terminate_backend -pkg/statement/ -> pg_query_go parsing + "is this natively safe?" classifier +pkg/statement/ -> go-pgquery parsing + "is this natively safe?" classifier pkg/schemadiff/ -> declarative mode: introspect live schema, diff vs desired CREATE TABLE, derive ordered ALTER/CREATE statements (+ fmt) pkg/lint/ -> unsafe-DDL linters (PG flavored) @@ -549,12 +596,15 @@ pkg/throttler/ -> Aurora PG replica-lag / slot-lag throttle `pgconn`/`pglogrepl` building blocks for logical replication. - **`github.com/jackc/pglogrepl`** — start replication, parse `pgoutput`/`wal2json` messages, send standby status (LSN flush) updates. This is the binlog-syncer analog. -- **`github.com/pganalyze/pg_query_go/v5`** — parse `ALTER`/`CREATE TABLE` (libpg_query, - the actual Postgres grammar). Analog of Spirit's TiDB parser. +- **`github.com/wasilibs/go-pgquery`** — parse `ALTER`/`CREATE TABLE` with the actual Postgres + grammar (`libpg_query` compiled to Wasm, executed by wazero: pure-Go builds, parser crashes + contained). Analog of Spirit's TiDB parser. The cgo `github.com/pganalyze/pg_query_go/v5` is + the API-compatible escape hatch (see + [How the planner understands DDL](#how-the-planner-understands-ddl-decided)). - **`github.com/stripe/pg-schema-diff`** *(candidate — open decision #5)* — declarative diff engine: introspection + dependency-ordered plan emission with hazard annotations and temp-database plan validation; would power `pkg/schemadiff` instead of building the diff on - `pg_query_go` directly. + `go-pgquery` directly. - **`github.com/alecthomas/kong`** — CLI, same as Spirit. ## Design decisions inherited from Spirit (safety over speed) @@ -665,7 +715,7 @@ why building declarative first costs nothing on the execution side. ### 3. Repo location / language -Go (reuse `pgx` + `pglogrepl` + `pg_query_go`; matches Spirit's language and idioms). Fresh +Go (reuse `pgx` + `pglogrepl` + `go-pgquery`; matches Spirit's language and idioms). Fresh standalone repo — this repository. ### 4. Expand/contract (pgroll) as a second execution backend @@ -684,9 +734,9 @@ rewrites use copy-and-swap. one-shot vs start/complete/rollback lifecycles under one `status`; and the default routing policy (auto-route vs explicit `--strategy`) given "decisions, not options". -### 5. Declarative diff engine — build on pg_query_go vs wrap pg-schema-diff +### 5. Declarative diff engine — build on go-pgquery vs wrap pg-schema-diff -Whether `pkg/schemadiff` builds the desired-vs-live diff on `pg_query_go` + our own schema +Whether `pkg/schemadiff` builds the desired-vs-live diff on `go-pgquery` + our own schema model, or wraps [stripe/pg-schema-diff](https://github.com/stripe/pg-schema-diff) (MIT, Go, PG 14–17, actively maintained) as the diff engine. @@ -697,9 +747,10 @@ PG 14–17, actively maintained) as the diff engine. validation against the temp database. Plan generation is cleanly separated from application, so our executors keep our own timeout/lock/retry discipline. It is a **periphery** dependency (plan generation), so the TCB bar does not apply — pinned like any load-bearing dep. -- **Costs of wrapping:** the temp-database factory is an operational precondition - (`CREATE DATABASE` on the target or a scratch instance — needs a deliberate answer for - locked-down production clusters); renames surface as drop+add and **must** sit behind our +- **Costs of wrapping:** the temp-database factory is an operational precondition — answered: + the engine-owned [scratch database](#plan-time-prerequisite-the-scratch-database) is already + a preflight-verified prerequisite for execute-and-introspect, so wrapping adds no new + operational demand; renames surface as drop+add and **must** sit behind our destructive-diff gate and never-guess-renames refusals; type support beyond enums is missing; its embedded timeout policy is replaced by ours at execution. - **Build:** full control and no temp-database precondition — at the cost of the hardest code diff --git a/docs/tcb-model.md b/docs/tcb-model.md index ba12912..69a5f72 100644 --- a/docs/tcb-model.md +++ b/docs/tcb-model.md @@ -161,14 +161,14 @@ short: | --- | --- | --- | | `pgx/v5` / `pgconn` | TCB (unavoidable — the wire) | pin, review upgrades like TCB changes, changelog read before bump | | `pglogrepl` | TCB (decode path) | same | -| `pg_query_go` | boundary (parses untrusted input into `Classified`) | fuzz at our boundary; parse failure is an error (CO-7), never a fallback | +| `go-pgquery` (Wasm `libpg_query`) | boundary (parses untrusted input into `Classified`) | fuzz at our boundary; parse failure is an error (CO-7), never a fallback; a parser crash is a Wasm trap surfaced as a Go error, not a process crash; verify wasilibs' reproducible-build provenance on every bump (cgo `pg_query_go` is the API-compatible escape hatch) | | `kong`, `testcontainers`, testify | periphery / test-only | normal hygiene | Rule: **no new dependency inside TCB packages without an explicit recorded decision.** CI enforces the import boundary (below), so a periphery-only dep physically cannot creep into the core. The decision rubric, in order: -1. **Is it load-bearing expertise?** A real SQL grammar (`pg_query_go`), the wire protocol +1. **Is it load-bearing expertise?** A real SQL grammar (`go-pgquery`), the wire protocol (`pgx`/`pglogrepl`), crypto — take the dependency, pin it, treat it as TCB. Hand-rolling a SQL parser to avoid a dependency would be the *opposite* of safety (CO-7 exists because string-splitting SQL is how tools corrupt data). From 5cb78f1ed371e587aacd654ba5ca9888c0cfe06f Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Tue, 4 Aug 2026 20:14:13 +1000 Subject: [PATCH 06/24] Phase 1: optimistic front door with typed not-native-safe verdicts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit migrate gates statements through go-pgquery (ALTER TABLE only; index maintenance is pointed at CONCURRENTLY), size-guards the table via pg_table_size over the partition tree, attempts the change under SET LOCAL lock/statement budgets, and ends in a structured verdict — executed-natively, or refused with a typed reason and exit code 2. The verdict seam is what Phase 11 maps to ExecutionModeBlocked. --- AGENTS.md | 2 +- README.md | 11 +- SAFETY.md | 8 +- cmd/pg-sprite/main.go | 12 +- docs/architecture.md | 11 +- go.mod | 5 + go.sum | 10 ++ internal/cli/bytesize.go | 52 ++++++ internal/cli/bytesize_test.go | 42 +++++ internal/cli/cli.go | 19 ++- internal/cli/migrate.go | 157 ++++++++++++++++++ internal/cli/migrate_integration_test.go | 172 ++++++++++++++++++++ internal/cli/status.go | 61 +++++++ pkg/executor/optimistic.go | 154 ++++++++++++++++++ pkg/executor/optimistic_integration_test.go | 138 ++++++++++++++++ pkg/preflight/preflight.go | 125 ++++++++++++++ pkg/preflight/preflight_integration_test.go | 122 ++++++++++++++ pkg/statement/statement.go | 97 +++++++++++ pkg/statement/statement_test.go | 114 +++++++++++++ pkg/verdict/verdict.go | 104 ++++++++++++ pkg/verdict/verdict_test.go | 57 +++++++ 21 files changed, 1454 insertions(+), 19 deletions(-) create mode 100644 internal/cli/bytesize.go create mode 100644 internal/cli/bytesize_test.go create mode 100644 internal/cli/migrate.go create mode 100644 internal/cli/migrate_integration_test.go create mode 100644 internal/cli/status.go create mode 100644 pkg/executor/optimistic.go create mode 100644 pkg/executor/optimistic_integration_test.go create mode 100644 pkg/preflight/preflight.go create mode 100644 pkg/preflight/preflight_integration_test.go create mode 100644 pkg/statement/statement.go create mode 100644 pkg/statement/statement_test.go create mode 100644 pkg/verdict/verdict.go create mode 100644 pkg/verdict/verdict_test.go diff --git a/AGENTS.md b/AGENTS.md index fc6da04..6b9503e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,7 +45,7 @@ make lint # golangci-lint Every session runs under bounded `lock_timeout` / `statement_timeout`. - All SQL parsing goes through the real PostgreSQL grammar via `wasilibs/go-pgquery` (Wasm `libpg_query`; the cgo `pg_query_go` is the API-compatible escape hatch, not the default), - once `pkg/statement` exists. No `strings.Split(";")`, no hand-parsing; a parse failure is an + with `pkg/statement` as the parse boundary. No `strings.Split(";")`, no hand-parsing; a parse failure is an error surfaced to the caller. Shadow-table DDL and checkpoint fingerprints are derived by execute-and-introspect on the engine-owned scratch database, never by AST transformation (see [docs/low-level-design.md](docs/low-level-design.md#how-the-planner-understands-ddl-decided)). diff --git a/README.md b/README.md index e1c9562..b17ccf7 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,14 @@ when one exists (`CONCURRENTLY`, `NOT VALID` + `VALIDATE`, fast default, `USING INDEX`), and a log-based, checksum-gated, resumable copy-and-swap when a genuine table rewrite is unavoidable. -**Status: Phase 0 (scaffold + test harness).** All subcommands are stubs. The -design docs and the phased build plan live in [docs/](docs/) — start with -[docs/README.md](docs/README.md). +**Status: Phase 1 (optimistic front door).** `pg-sprite migrate --alter '…'` +runs easy `ALTER TABLE` changes directly under tight lock/statement budgets +and refuses everything else with a structured verdict (exit code 2): index +maintenance gets a pointer to the `CONCURRENTLY` idiom, and changes that need +a table rewrite — caught by the size guard or a cancelled bounded attempt — +get an explicit **not native-safe** verdict. `diff`, `fmt`, and `lint` are +still stubs. The design docs and the phased build plan live in +[docs/](docs/) — start with [docs/README.md](docs/README.md). The codebase is partitioned into a small safety-critical core and a periphery — **[SAFETY.md](SAFETY.md)** says which packages are which and the diff --git a/SAFETY.md b/SAFETY.md index 1bd04f8..cc56d6a 100644 --- a/SAFETY.md +++ b/SAFETY.md @@ -18,7 +18,8 @@ The invariant registry (invariant IDs referenced below) lives in | Package | Core? | Status | Invariants enforced | | --- | --- | --- | --- | | `pkg/dbconn` — pool defaults, advisory lock, terminate-blockers, retries, RDS TLS | ✅ core | exists (Phase 0) | LK-1, LK-2 primitives | -| `pkg/preflight` — precondition verifier, refusals | ✅ core | planned (Phase 1–2) | ST-6, RF-1..RF-5 | +| `pkg/preflight` — precondition verifier, refusals | ✅ core | exists (Phase 1: table-size guard); grows through Phase 2 | ST-6, RF-1..RF-5 | +| `pkg/executor` — bounded optimistic attempt; native executor later | ✅ core | exists (Phase 1: attempt-under-budget); Executor contract at Phase 2–3 | LK-2 (attempt bound) | | `pkg/checksum` — chunk verifier, continuous checker, repair | ✅ core | planned (Phase 5) | CO-1, CO-2, CO-3 | | `pkg/copier` — shadow-table chunked copy | ✅ core | planned (Phase 4) | CO-4, LK-3 | | `pkg/applier` — change apply, buffer, flush scheduling | ✅ core | planned (Phase 6) | CO-4, CO-5, CO-6, LK-3 | @@ -26,8 +27,9 @@ The invariant registry (invariant IDs referenced below) lives in | `pkg/checkpoint` — durable resume state | ✅ core | planned (Phase 8) | ST-1, ST-2 | | slot lifecycle (in `pkg/decode`) — create, reap, lag ceiling | ✅ core | planned (Phase 8) | ST-3 | | `pkg/migration` — orchestrator, **cutover swap + fidelity gate** | ✅ core | planned (Phase 7) | LK-2, LK-4, ST-5 | -| `pkg/statement`, `pkg/planner`, `pkg/schemadiff`, `pkg/lint` — classify/diff/route | ❌ periphery¹ | planned (Phase 1–2) | (CO-7 holds at the parse boundary) | -| `internal/cli` — CLI, flags, help, prompts | ❌ periphery | exists (stubs) | — | +| `pkg/statement`, `pkg/planner`, `pkg/schemadiff`, `pkg/lint` — classify/diff/route | ❌ periphery¹ | `pkg/statement` exists (Phase 1: type gate); rest planned (Phase 2) | (CO-7 holds at the parse boundary) | +| `pkg/verdict` — structured outcome contract, rendering, exit codes | ❌ periphery | exists (Phase 1) | — | +| `internal/cli` — CLI, flags, help, prompts | ❌ periphery | `migrate`/`status` exist (Phase 1); rest stubs | — | | status / progress / advisory rendering, metrics | ❌ periphery | planned | — | | orchestrator adapter | ❌ periphery | planned (Phase 11) | OC-* hold *at* the boundary | | `internal/testutil` | ❌ test-only | exists | — | diff --git a/cmd/pg-sprite/main.go b/cmd/pg-sprite/main.go index 0f1ad85..1c8ed7d 100644 --- a/cmd/pg-sprite/main.go +++ b/cmd/pg-sprite/main.go @@ -2,9 +2,13 @@ package main import ( + "errors" + "os" + "github.com/alecthomas/kong" "github.com/block/pg-sprite/internal/cli" + "github.com/block/pg-sprite/pkg/verdict" ) // version is stamped at release time via -ldflags "-X main.version=…". @@ -17,5 +21,11 @@ func main() { kong.UsageOnError(), kong.Vars{"version": version}, ) - k.FatalIfErrorf(k.Run()) + err := k.Run() + // A refusal verdict was already printed; its exit code is distinct from + // operational errors so automation can branch on the difference. + if errors.Is(err, verdict.ErrRefused) { + os.Exit(verdict.ExitCodeRefused) + } + k.FatalIfErrorf(err) } diff --git a/docs/architecture.md b/docs/architecture.md index 1c140ab..e75ff98 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -48,14 +48,15 @@ boundary) is defined in [../SAFETY.md](../SAFETY.md). | Package | Role | Status | | --- | --- | --- | -| `cmd/pg-sprite` | CLI entry point (Kong): `migrate` · `diff` · `fmt` · `lint` · `status` | exists (stubs) | -| `internal/cli` | Command tree and flag handling | exists (stubs) | +| `cmd/pg-sprite` | CLI entry point (Kong): `migrate` · `diff` · `fmt` · `lint` · `status` | `migrate`/`status` exist; rest stubs | +| `internal/cli` | Command tree and flag handling | `migrate`/`status` exist; rest stubs | | `internal/testutil` | Test harness: containerized PostgreSQL, throwaway schemas | exists | | `pkg/dbconn` | Pool with bounded session timeouts, retries, RDS/Aurora auto-TLS (embedded CA bundle), terminate-blockers; advisory-lock mutual exclusion lands here | exists | -| `pkg/statement` | `go-pgquery` (Wasm `libpg_query`) parsing + classification (never hand-parse SQL); shadow DDL + fingerprints come from scratch-DB execute-and-introspect | Phase 1–2 | -| `pkg/preflight` | Precondition verification and refusals before any write | Phase 1–2 | +| `pkg/statement` | `go-pgquery` (Wasm `libpg_query`) parsing + classification (never hand-parse SQL); shadow DDL + fingerprints come from scratch-DB execute-and-introspect | exists (Phase 1: type gate); classification at Phase 2 | +| `pkg/preflight` | Precondition verification and refusals before any write | exists (Phase 1: table-size guard); grows through Phase 2 | +| `pkg/verdict` | Structured outcome contract (executed / refused + reason + safer idiom), rendering, exit codes | exists (Phase 1) | | `pkg/planner` / `pkg/schemadiff` / `pkg/lint` | Shared front-end: introspect, declarative diff (may wrap [stripe/pg-schema-diff](https://github.com/stripe/pg-schema-diff) — see the low-level design's open decisions), classify, lint | Phase 2 | -| `pkg/executor` | The `Executor` contract (`Plan`/`Execute`/`Status`/`Abort`) + native executor | Phase 2–3 | +| `pkg/executor` | The `Executor` contract (`Plan`/`Execute`/`Status`/`Abort`) + native executor | exists (Phase 1: bounded optimistic attempt); contract at Phase 2–3 | | `pkg/table` | PK-range chunkers (single-column fast path, composite), dynamic time-based sizing | Phase 4 | | `pkg/copier` | Parallel chunked copy into the shadow table (never overwrites) | Phase 4 | | `pkg/checksum` | The mandatory correctness gate; continuous checker; repair primitive | Phase 5 | diff --git a/go.mod b/go.mod index 1f75851..c9120f1 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,11 @@ go 1.26 require ( github.com/alecthomas/kong v1.15.0 github.com/jackc/pgx/v5 v5.10.0 + github.com/pganalyze/pg_query_go/v6 v6.2.2 github.com/stretchr/testify v1.11.1 github.com/testcontainers/testcontainers-go v0.43.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.43.0 + github.com/wasilibs/go-pgquery v0.0.0-20260728010200-155ebad2880e ) require ( @@ -52,8 +54,10 @@ require ( github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect github.com/shirou/gopsutil/v4 v4.26.5 // indirect github.com/sirupsen/logrus v1.9.4 // indirect + github.com/tetratelabs/wazero v1.12.0 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect + github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect @@ -64,5 +68,6 @@ require ( golang.org/x/sync v0.20.0 // indirect golang.org/x/sys v0.45.0 // indirect golang.org/x/text v0.37.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 3f1ddff..34d264f 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,8 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pganalyze/pg_query_go/v6 v6.2.2 h1:O0L6zMC226R82RF3X5n0Ki6HjytDsoAzuzp4ATVAHNo= +github.com/pganalyze/pg_query_go/v6 v6.2.2/go.mod h1:Cn6+j4870kJz3iYNsb0VsNG04vpSWgEvBwc590J4qD0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= @@ -120,10 +122,16 @@ github.com/testcontainers/testcontainers-go v0.43.0 h1:oEQx5MW2DGd9z3AeEQfB2lPM0 github.com/testcontainers/testcontainers-go v0.43.0/go.mod h1:+VxkT2NQnKOZPKi6praMuMKYHYyOGXr0XSBSlSMCzFo= github.com/testcontainers/testcontainers-go/modules/postgres v0.43.0 h1:ShNOFYAF4lKHvdIG258hi69bSxC88uXnxJkJvNs/IVs= github.com/testcontainers/testcontainers-go/modules/postgres v0.43.0/go.mod h1:vdq5/RqmGfWeefzyfcVI/pID1rzmc1TDvqXa15bPJks= +github.com/tetratelabs/wazero v1.12.0 h1:DuWcpNu/FzgEXgGBDp8J1Spc+CWOvvtvVyjKlaZopYU= +github.com/tetratelabs/wazero v1.12.0/go.mod h1:LvKtzl2RqO4gyF27BiXU+nKAjcV8f38U+kP/q2vgxh0= github.com/tklauser/go-sysconf v0.3.16 h1:frioLaCQSsF5Cy1jgRBrzr6t502KIIwQ0MArYICU0nA= github.com/tklauser/go-sysconf v0.3.16/go.mod h1:/qNL9xxDhc7tx3HSRsLWNnuzbVfh3e7gh/BmM179nYI= github.com/tklauser/numcpus v0.11.0 h1:nSTwhKH5e1dMNsCdVBukSZrURJRoHbSEQjdEbY+9RXw= github.com/tklauser/numcpus v0.11.0/go.mod h1:z+LwcLq54uWZTX0u/bGobaV34u6V7KNlTZejzM6/3MQ= +github.com/wasilibs/go-pgquery v0.0.0-20260728010200-155ebad2880e h1:yWIo9Ibxg0qNScjPcdaH99BfetgmYepCxs9a6TFC2LM= +github.com/wasilibs/go-pgquery v0.0.0-20260728010200-155ebad2880e/go.mod h1:ZSyYLCRbk2xPqu7lgfrDSSHm+g/7Rxk6JK4KE2cxJ3s= +github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb h1:gQ+ZV4wJke/EBKYciZ2MshEouEHFuinB85dY3f5s1q8= +github.com/wasilibs/wazero-helpers v0.0.0-20250123031827-cd30c44769bb/go.mod h1:jMeV4Vpbi8osrE/pKUxRZkVaA0EX7NZN0A9/oRzgpgY= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -154,6 +162,8 @@ golang.org/x/term v0.43.0/go.mod h1:lrhlHNdQJHO+1qVYiHfFKVuVioJIheAc3fBSMFYEIsk= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/internal/cli/bytesize.go b/internal/cli/bytesize.go new file mode 100644 index 0000000..dc3c6fc --- /dev/null +++ b/internal/cli/bytesize.go @@ -0,0 +1,52 @@ +package cli + +import ( + "fmt" + "strconv" + "strings" +) + +// byteSize is a flag type for human-readable sizes ("512MiB", "1GiB", plain +// bytes). All suffixes are binary multiples — PostgreSQL's own convention for +// size units. +type byteSize int64 + +// suffixMultiplier maps an accepted (lowercased) size suffix to its +// multiplier; ok is false for anything unrecognized. +func suffixMultiplier(suffix string) (mult int64, ok bool) { + switch suffix { + case "", "b": + return 1, true + case "kb", "kib": + return 1 << 10, true + case "mb", "mib": + return 1 << 20, true + case "gb", "gib": + return 1 << 30, true + case "tb", "tib": + return 1 << 40, true + default: + return 0, false + } +} + +// UnmarshalText implements encoding.TextUnmarshaler so kong can parse the +// flag directly. +func (b *byteSize) UnmarshalText(text []byte) error { + s := strings.TrimSpace(strings.ToLower(string(text))) + digits := strings.TrimRight(s, "bkmgit ") + suffix := strings.TrimSpace(s[len(digits):]) + mult, ok := suffixMultiplier(suffix) + if !ok { + return fmt.Errorf("unknown size suffix %q in %q (use B, KiB, MiB, GiB, or TiB)", suffix, string(text)) + } + n, err := strconv.ParseInt(strings.TrimSpace(digits), 10, 64) + if err != nil { + return fmt.Errorf("parse size %q: %w", string(text), err) + } + if n <= 0 { + return fmt.Errorf("size must be positive, got %q", string(text)) + } + *b = byteSize(n * mult) + return nil +} diff --git a/internal/cli/bytesize_test.go b/internal/cli/bytesize_test.go new file mode 100644 index 0000000..e788d9e --- /dev/null +++ b/internal/cli/bytesize_test.go @@ -0,0 +1,42 @@ +package cli + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestByteSizeUnmarshal(t *testing.T) { + tests := []struct { + in string + want int64 + }{ + {"1024", 1024}, + {"512B", 512}, + {"4KiB", 4 << 10}, + {"4kb", 4 << 10}, + {"100MiB", 100 << 20}, + {"100MB", 100 << 20}, + {"1GiB", 1 << 30}, + {"1gb", 1 << 30}, + {"2TiB", 2 << 40}, + {" 8 MiB ", 8 << 20}, + } + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + var b byteSize + require.NoError(t, b.UnmarshalText([]byte(tt.in))) + assert.Equal(t, byteSize(tt.want), b) + }) + } +} + +func TestByteSizeUnmarshalRejectsInvalid(t *testing.T) { + for _, in := range []string{"", "GiB", "1XB", "-5MiB", "0", "1.5GiB"} { + t.Run(in, func(t *testing.T) { + var b byteSize + assert.Error(t, b.UnmarshalText([]byte(in))) + }) + } +} diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 3b3d805..cc37b14 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -1,9 +1,12 @@ -// Package cli defines the pg-sprite command tree (Kong). Subcommand Run -// methods are stubs; each build-plan phase fills one in. +// Package cli defines the pg-sprite command tree (Kong). migrate and status +// are implemented (the Phase 1 optimistic front door); the remaining +// subcommand Run methods are stubs each build-plan phase fills in. package cli import ( + "context" "fmt" + "os" "time" "github.com/alecthomas/kong" @@ -48,15 +51,19 @@ func (f DBFlags) Config() dbconn.Config { } } -// MigrateCmd runs a schema change (imperative front-end). +// MigrateCmd runs a schema change (imperative front-end): the Phase 1 +// optimistic front door. Easy changes execute directly under tight budgets; +// everything else is refused with an explicit verdict. type MigrateCmd struct { DBFlags `embed:""` - Alter string `help:"Imperative ALTER statement to run." name:"alter"` + Alter string `help:"Imperative ALTER statement to run." name:"alter" required:""` + MaxTableSize byteSize `help:"Size threshold above which the optimistic attempt is skipped (binary units: B, KiB, MiB, GiB, TiB)." default:"1GiB"` + JSON bool `help:"Emit the verdict as JSON."` } // Run implements the migrate subcommand. -func (c *MigrateCmd) Run() error { return notImplemented("migrate") } +func (c *MigrateCmd) Run() error { return c.run(context.Background(), os.Stdout) } // DiffCmd derives statements from a desired-state schema (declarative front-end). type DiffCmd struct { @@ -88,4 +95,4 @@ type StatusCmd struct { } // Run implements the status subcommand. -func (c *StatusCmd) Run() error { return notImplemented("status") } +func (c *StatusCmd) Run() error { return c.run(context.Background(), os.Stdout) } diff --git a/internal/cli/migrate.go b/internal/cli/migrate.go new file mode 100644 index 0000000..4dcaa75 --- /dev/null +++ b/internal/cli/migrate.go @@ -0,0 +1,157 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/block/pg-sprite/pkg/dbconn" + "github.com/block/pg-sprite/pkg/executor" + "github.com/block/pg-sprite/pkg/preflight" + "github.com/block/pg-sprite/pkg/statement" + "github.com/block/pg-sprite/pkg/verdict" +) + +// run is the migrate flow: gate the statement type, size-guard the table, +// attempt the change under budget, and end in exactly one verdict. Refusal +// verdicts are printed to out and returned as verdict.ErrRefused so the entry +// point maps them to the refusal exit code. +func (c *MigrateCmd) run(ctx context.Context, out io.Writer) error { + st, err := statement.ParseOne(c.Alter) + if err != nil { + return err + } + if v, refused := gateVerdict(st); refused { + return c.emit(out, v) + } + + pool, err := dbconn.NewPool(ctx, c.Config()) + if err != nil { + return err + } + defer pool.Close() + + pt, err := preflight.CheckTable(ctx, pool, st.Schema, st.Table, int64(c.MaxTableSize)) + var sizeErr *preflight.SizeError + if errors.As(err, &sizeErr) { + return c.emit(out, sizeGuardVerdict(st, sizeErr)) + } + if err != nil { + return err + } + + budget := executor.Budget{LockTimeout: c.LockTimeout, StatementTimeout: c.StatementTimeout} + err = executor.AttemptNative(ctx, pool, pt, st.SQL, budget) + var budgetErr *executor.BudgetError + if errors.As(err, &budgetErr) { + return c.emit(out, budgetVerdict(st, budgetErr)) + } + if err != nil { + return err + } + return c.emit(out, verdict.Verdict{ + Outcome: verdict.OutcomeExecuted, + Statement: st.SQL, + Table: qualified(st), + Detail: fmt.Sprintf("committed within budgets (lock %s, statement %s): the change was effectively instant", + budget.LockTimeout, budget.StatementTimeout), + }) +} + +// emit prints the verdict in the selected format and returns ErrRefused for +// refusals so the exit code distinguishes them from operational errors. +func (c *MigrateCmd) emit(out io.Writer, v verdict.Verdict) error { + text := v.String() + if c.JSON { + var err error + if text, err = v.JSON(); err != nil { + return err + } + } + if _, err := fmt.Fprintln(out, text); err != nil { + return fmt.Errorf("write verdict: %w", err) + } + if v.Outcome == verdict.OutcomeRefused { + return verdict.ErrRefused + } + return nil +} + +// gateVerdict is the Phase 1 statement-type gate: only ALTER TABLE proceeds; +// index maintenance is pointed at its concurrent idiom, everything else is +// unsupported. Refused statements are never executed. +func gateVerdict(st statement.Statement) (verdict.Verdict, bool) { + v := verdict.Verdict{Outcome: verdict.OutcomeRefused, Statement: st.SQL} + switch st.Kind { + case statement.KindAlterTable: + return verdict.Verdict{}, false + case statement.KindCreateIndex: + v.Reason = verdict.ReasonIndexStatement + v.Detail = "a plain CREATE INDEX blocks writes for the whole build; the concurrent build does not" + v.SaferIdiom = "CREATE INDEX CONCURRENTLY" + case statement.KindDropIndex: + v.Reason = verdict.ReasonIndexStatement + v.Detail = "a plain DROP INDEX takes ACCESS EXCLUSIVE on the table; the concurrent drop does not" + v.SaferIdiom = "DROP INDEX CONCURRENTLY" + case statement.KindReindex: + v.Reason = verdict.ReasonIndexStatement + v.Detail = "a plain REINDEX blocks writes; the concurrent rebuild does not" + v.SaferIdiom = "REINDEX ... CONCURRENTLY" + case statement.KindOther: + v.Reason = verdict.ReasonUnsupportedStatement + v.Detail = "only ALTER TABLE statements are supported by the optimistic front door" + } + return v, true +} + +// sizeGuardVerdict is the refusal for tables above the size threshold, where +// even a budget-bounded attempt would visibly stall the table. +func sizeGuardVerdict(st statement.Statement, sizeErr *preflight.SizeError) verdict.Verdict { + return verdict.Verdict{ + Outcome: verdict.OutcomeRefused, + Reason: verdict.ReasonTableTooLarge, + Statement: st.SQL, + Table: qualified(st), + Detail: fmt.Sprintf("table is %d bytes on disk, above the %d-byte --max-table-size threshold: "+ + "a cancelled attempt is not a free probe — it would hold ACCESS EXCLUSIVE doing rewrite work "+ + "for the whole budget. This change needs a copy-and-swap rewrite, which pg-sprite does not perform yet", + sizeErr.TotalBytes, sizeErr.LimitBytes), + } +} + +// budgetVerdict is the refusal for an attempt that exceeded its lock or +// statement budget and was cancelled without executing. +func budgetVerdict(st statement.Statement, budgetErr *executor.BudgetError) verdict.Verdict { + v := verdict.Verdict{ + Outcome: verdict.OutcomeRefused, + Reason: verdict.ReasonBudgetExceeded, + Statement: st.SQL, + Table: qualified(st), + } + switch budgetErr.Cause { + case executor.CauseLock: + v.Detail = fmt.Sprintf("the lock was not granted within the %s lock budget: the table is too "+ + "contended for a blind attempt; nothing was executed", budgetErr.Budget) + case executor.CauseStatement: + v.Detail = fmt.Sprintf("cancelled after the %s statement budget: the change does real rewrite work, "+ + "not an in-place catalog change, and needs a copy-and-swap rewrite that pg-sprite does not perform yet. "+ + "If it adds a constraint, ADD CONSTRAINT ... NOT VALID followed by VALIDATE CONSTRAINT avoids the long lock", + budgetErr.Budget) + default: + v.Detail = budgetErr.Error() + } + return v +} + +// qualified renders the statement's target table for the verdict, empty when +// the statement has none. +func qualified(st statement.Statement) string { + if st.Table == "" { + return "" + } + if st.Schema == "" { + return st.Table + } + return st.Schema + "." + st.Table +} diff --git a/internal/cli/migrate_integration_test.go b/internal/cli/migrate_integration_test.go new file mode 100644 index 0000000..21b1aa7 --- /dev/null +++ b/internal/cli/migrate_integration_test.go @@ -0,0 +1,172 @@ +package cli + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/block/pg-sprite/internal/testutil" + "github.com/block/pg-sprite/pkg/dbconn" + "github.com/block/pg-sprite/pkg/verdict" +) + +// newMigrateCmd builds a MigrateCmd with the flag defaults kong would apply. +func newMigrateCmd(url, alter string) *MigrateCmd { + return &MigrateCmd{ + DBFlags: DBFlags{ + URL: url, + LockTimeout: 3 * time.Second, + StatementTimeout: 30 * time.Second, + }, + Alter: alter, + MaxTableSize: 1 << 30, + } +} + +// Acceptance (i): an instant-eligible change runs and commits within budget. +func TestMigrateExecutesInstantChange(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY)", schema)) + require.NoError(t, err) + + cmd := newMigrateCmd(url, fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int", schema)) + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + assert.Contains(t, out.String(), "executed natively") + + var typ string + require.NoError(t, pool.QueryRow(t.Context(), + `SELECT data_type FROM information_schema.columns + WHERE table_schema = $1 AND table_name = 't' AND column_name = 'age'`, schema).Scan(&typ)) + assert.Equal(t, "integer", typ) +} + +// Acceptance (ii): a rewrite-requiring change is cancelled, leaves schema and +// data unchanged, and returns the not-native-safe verdict with its reason. +func TestMigrateRefusesRewriteWithBudgetVerdict(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY, v text)", schema)) + require.NoError(t, err) + _, err = pool.Exec(t.Context(), fmt.Sprintf( + "INSERT INTO %s.t SELECT g, repeat('x', 100) FROM generate_series(1, 300000) g", schema)) + require.NoError(t, err) + + cmd := newMigrateCmd(url, fmt.Sprintf("ALTER TABLE %s.t ALTER COLUMN id TYPE bigint", schema)) + cmd.StatementTimeout = 50 * time.Millisecond + cmd.JSON = true + var out strings.Builder + err = cmd.run(t.Context(), &out) + require.ErrorIs(t, err, verdict.ErrRefused) + + var v verdict.Verdict + require.NoError(t, json.Unmarshal([]byte(out.String()), &v)) + assert.Equal(t, verdict.OutcomeRefused, v.Outcome) + assert.Equal(t, verdict.ReasonBudgetExceeded, v.Reason) + assert.Equal(t, schema+".t", v.Table) + + var typ string + require.NoError(t, pool.QueryRow(t.Context(), + `SELECT data_type FROM information_schema.columns + WHERE table_schema = $1 AND table_name = 't' AND column_name = 'id'`, schema).Scan(&typ)) + assert.Equal(t, "integer", typ, "the cancelled attempt must not change the schema") + var count int + require.NoError(t, pool.QueryRow(t.Context(), + fmt.Sprintf("SELECT count(*) FROM %s.t", schema)).Scan(&count)) + assert.Equal(t, 300000, count, "the cancelled attempt must not change the data") +} + +// Acceptance (iii): a table above the size threshold skips the attempt and +// returns the same verdict class. +func TestMigrateSizeGuardSkipsAttempt(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY, v text)", schema)) + require.NoError(t, err) + _, err = pool.Exec(t.Context(), fmt.Sprintf( + "INSERT INTO %s.t SELECT g, 'v' FROM generate_series(1, 1000) g", schema)) + require.NoError(t, err) + + cmd := newMigrateCmd(url, fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int", schema)) + cmd.MaxTableSize = 1 // guarantees the guard fires without a big fixture + cmd.JSON = true + var out strings.Builder + err = cmd.run(t.Context(), &out) + require.ErrorIs(t, err, verdict.ErrRefused) + + var v verdict.Verdict + require.NoError(t, json.Unmarshal([]byte(out.String()), &v)) + assert.Equal(t, verdict.ReasonTableTooLarge, v.Reason) + + // The attempt was skipped, so the (instant-eligible) change must not + // have been applied. + var n int + require.NoError(t, pool.QueryRow(t.Context(), + `SELECT count(*) FROM information_schema.columns + WHERE table_schema = $1 AND table_name = 't' AND column_name = 'age'`, schema).Scan(&n)) + assert.Zero(t, n, "the size guard must skip the attempt entirely") +} + +// Acceptance (iv): non-ALTER TABLE statements are refused with the safe-idiom +// pointer and never executed. The gate needs no database at all. +func TestMigrateGateRefusesWithoutDatabase(t *testing.T) { + tests := []struct { + name string + alter string + reason verdict.Reason + saferIdiom string + }{ + {"create index", "CREATE INDEX i ON t (c)", verdict.ReasonIndexStatement, "CREATE INDEX CONCURRENTLY"}, + {"drop index", "DROP INDEX i", verdict.ReasonIndexStatement, "DROP INDEX CONCURRENTLY"}, + {"reindex", "REINDEX TABLE t", verdict.ReasonIndexStatement, "REINDEX ... CONCURRENTLY"}, + {"alter index", "ALTER INDEX i SET (fillfactor = 90)", verdict.ReasonUnsupportedStatement, ""}, + {"create table", "CREATE TABLE t (id int)", verdict.ReasonUnsupportedStatement, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // An unroutable URL proves the gate refuses before connecting. + cmd := newMigrateCmd("postgres://nobody@localhost:1/nope", tt.alter) + cmd.JSON = true + var out strings.Builder + err := cmd.run(t.Context(), &out) + require.ErrorIs(t, err, verdict.ErrRefused) + + var v verdict.Verdict + require.NoError(t, json.Unmarshal([]byte(out.String()), &v)) + assert.Equal(t, verdict.OutcomeRefused, v.Outcome) + assert.Equal(t, tt.reason, v.Reason) + assert.Equal(t, tt.saferIdiom, v.SaferIdiom) + }) + } +} + +func TestMigrateSurfacesParseErrors(t *testing.T) { + cmd := newMigrateCmd("postgres://nobody@localhost:1/nope", "ALTER TABEL t ADD COLUMN x int") + var out strings.Builder + err := cmd.run(t.Context(), &out) + require.Error(t, err) + assert.NotErrorIs(t, err, verdict.ErrRefused, "a parse failure is an operational error, not a refusal") +} + +func TestStatusReportsNoSessions(t *testing.T) { + url := testutil.StartPostgres(t) + cmd := &StatusCmd{DBFlags: DBFlags{URL: url}} + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + assert.Contains(t, out.String(), "no active pg-sprite sessions") +} diff --git a/internal/cli/status.go b/internal/cli/status.go new file mode 100644 index 0000000..70f7e6d --- /dev/null +++ b/internal/cli/status.go @@ -0,0 +1,61 @@ +package cli + +import ( + "context" + "fmt" + "io" + + "github.com/block/pg-sprite/pkg/dbconn" +) + +// run reports the engine's live database sessions. Phase 1 has no durable +// migration state — a change either committed within its budgets or was +// refused — so status is a view over pg_stat_activity for pg-sprite sessions. +func (c *StatusCmd) run(ctx context.Context, out io.Writer) error { + pool, err := dbconn.NewPool(ctx, c.Config()) + if err != nil { + return err + } + defer pool.Close() + + rows, err := pool.Query(ctx, ` + SELECT pid, state, + COALESCE(wait_event_type || '/' || wait_event, '-'), + COALESCE(now() - query_start, '0'::interval)::text, + left(query, 80) + FROM pg_stat_activity + WHERE application_name = 'pg-sprite' AND pid <> pg_backend_pid() + ORDER BY query_start`) + if err != nil { + return fmt.Errorf("query pg_stat_activity: %w", err) + } + defer rows.Close() + + sessions := 0 + for rows.Next() { + var pid int + var state, waitEvent, runningFor, query string + if err := rows.Scan(&pid, &state, &waitEvent, &runningFor, &query); err != nil { + return fmt.Errorf("scan pg_stat_activity row: %w", err) + } + sessions++ + if sessions == 1 { + if _, err := fmt.Fprintln(out, "active pg-sprite sessions:"); err != nil { + return fmt.Errorf("write status: %w", err) + } + } + if _, err := fmt.Fprintf(out, " pid=%d state=%s wait=%s running_for=%s query=%q\n", + pid, state, waitEvent, runningFor, query); err != nil { + return fmt.Errorf("write status: %w", err) + } + } + if err := rows.Err(); err != nil { + return fmt.Errorf("read pg_stat_activity: %w", err) + } + if sessions == 0 { + if _, err := fmt.Fprintln(out, "no active pg-sprite sessions (Phase 1 keeps no durable migration state: a change either committed within its budgets or was refused)"); err != nil { + return fmt.Errorf("write status: %w", err) + } + } + return nil +} diff --git a/pkg/executor/optimistic.go b/pkg/executor/optimistic.go new file mode 100644 index 0000000..16bccae --- /dev/null +++ b/pkg/executor/optimistic.go @@ -0,0 +1,154 @@ +// Package executor runs schema changes against the database. In Phase 1 it +// holds the optimistic front door: attempt the change directly under a tight +// lock_timeout and statement_timeout so it can only succeed if it is +// effectively an instant / in-place change. A budget overrun cancels the +// statement cleanly — nothing is executed — and surfaces as a typed +// BudgetError the caller turns into a not-native-safe verdict. +// +// This is a safety-critical core package: see SAFETY.md. It never trusts the +// caller's classification — its own protection is the budget, applied with +// SET LOCAL inside the attempt's transaction regardless of session defaults. +package executor + +import ( + "context" + "errors" + "fmt" + "strconv" + "time" + + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/block/pg-sprite/pkg/preflight" +) + +// SQLSTATE codes the attempt maps to budget outcomes. Postgres errors are +// matched by SQLSTATE, never by message text. +const ( + sqlstateLockNotAvailable = "55P03" // lock_timeout expired + sqlstateQueryCanceled = "57014" // statement_timeout expired +) + +// BudgetCause says which budget the optimistic attempt exceeded. +type BudgetCause int + +// The two budgets an attempt runs under. +const ( + // CauseLock means the lock was not granted within lock_timeout: the + // table is too contended for a blind attempt right now. + CauseLock BudgetCause = iota + 1 + // CauseStatement means the statement ran past statement_timeout: the + // change is doing real work (a rewrite), not an in-place catalog change. + CauseStatement +) + +// String returns the human-readable budget name. +func (c BudgetCause) String() string { + switch c { + case CauseLock: + return "lock budget" + case CauseStatement: + return "statement budget" + default: + return "unknown budget" + } +} + +// BudgetError reports that the optimistic attempt exceeded one of its budgets +// and was cancelled cleanly: the statement did not execute and the +// transaction rolled back. It is a refusal input, not an operational failure. +type BudgetError struct { + // Cause is the budget that was exceeded. + Cause BudgetCause + // Budget is the configured limit for that cause. + Budget time.Duration +} + +// Error implements the error interface. +func (e *BudgetError) Error() string { + return fmt.Sprintf("optimistic attempt exceeded its %s (%s) and was cancelled", e.Cause, e.Budget) +} + +// Budget bounds one optimistic attempt. Both limits must be positive: an +// unbounded attempt is exactly the stall the front door exists to prevent. +type Budget struct { + // LockTimeout bounds how long the attempt may wait in the lock queue. + LockTimeout time.Duration + // StatementTimeout bounds how long the statement may run once started. + StatementTimeout time.Duration +} + +// validate rejects budgets that would leave the attempt unbounded. +func (b Budget) validate() error { + // INV: LK-2 — the attempt is bounded by construction; a zero or negative + // timeout would disable the corresponding PostgreSQL limit. + if b.LockTimeout <= 0 { + return fmt.Errorf("lock budget must be positive, got %s", b.LockTimeout) + } + if b.StatementTimeout <= 0 { + return fmt.Errorf("statement budget must be positive, got %s", b.StatementTimeout) + } + return nil +} + +// AttemptNative runs sql once, directly, inside a transaction bounded by b. +// The table must have passed preflight — the proof parameter makes skipping +// the size guard unrepresentable. On success the change is committed: it was +// effectively instant. If a budget is exceeded the statement is cancelled by +// the server, the transaction rolls back, and a *BudgetError is returned. +// Any other failure is surfaced as an operational error. +func AttemptNative(ctx context.Context, pool *pgxpool.Pool, _ preflight.PreflightedTable, sql string, b Budget) error { + if err := b.validate(); err != nil { + return err + } + tx, err := pool.Begin(ctx) + if err != nil { + return fmt.Errorf("begin optimistic attempt: %w", err) + } + defer func() { + // Redundant safety closer: after a successful Commit this returns + // the guaranteed ErrTxClosed; on a failure path a rollback error + // only means the connection died, and the server aborts the + // transaction with its session either way. + _ = tx.Rollback(context.WithoutCancel(ctx)) + }() + + // INV: LK-2 — budgets are applied inside this transaction regardless of + // the session defaults, so the attempt cannot outlive them even on a + // misconfigured pool. A bare integer is milliseconds to PostgreSQL; + // SET LOCAL cannot use bind parameters. + setBudgets := "SET LOCAL lock_timeout = " + strconv.FormatInt(b.LockTimeout.Milliseconds(), 10) + + "; SET LOCAL statement_timeout = " + strconv.FormatInt(b.StatementTimeout.Milliseconds(), 10) + if _, err := tx.Exec(ctx, setBudgets); err != nil { + return fmt.Errorf("set attempt budgets: %w", err) + } + + if _, err := tx.Exec(ctx, sql); err != nil { + if budgetErr := asBudgetError(err, b); budgetErr != nil { + return budgetErr + } + return fmt.Errorf("optimistic attempt: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit optimistic attempt: %w", err) + } + return nil +} + +// asBudgetError maps a PostgreSQL error to the budget it exceeded, or nil +// when the error is not a budget overrun. +func asBudgetError(err error, b Budget) *BudgetError { + var pgErr *pgconn.PgError + if !errors.As(err, &pgErr) { + return nil + } + switch pgErr.Code { + case sqlstateLockNotAvailable: + return &BudgetError{Cause: CauseLock, Budget: b.LockTimeout} + case sqlstateQueryCanceled: + return &BudgetError{Cause: CauseStatement, Budget: b.StatementTimeout} + default: + return nil + } +} diff --git a/pkg/executor/optimistic_integration_test.go b/pkg/executor/optimistic_integration_test.go new file mode 100644 index 0000000..1f9ccd9 --- /dev/null +++ b/pkg/executor/optimistic_integration_test.go @@ -0,0 +1,138 @@ +package executor_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/block/pg-sprite/internal/testutil" + "github.com/block/pg-sprite/pkg/dbconn" + "github.com/block/pg-sprite/pkg/executor" + "github.com/block/pg-sprite/pkg/preflight" +) + +// budget is generous enough that an instant catalog change always fits and +// tight enough that a blocked or rewriting attempt is cancelled quickly. +var budget = executor.Budget{LockTimeout: 500 * time.Millisecond, StatementTimeout: 2 * time.Second} + +func newPool(t *testing.T) (*pgxpool.Pool, string) { + t.Helper() + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + t.Cleanup(pool.Close) + return pool, testutil.NewSchema(t, pool) +} + +func mustPreflight(t *testing.T, pool *pgxpool.Pool, schema, table string) preflight.PreflightedTable { + t.Helper() + pt, err := preflight.CheckTable(t.Context(), pool, schema, table, 1<<30) + require.NoError(t, err) + return pt +} + +func columnType(t *testing.T, pool *pgxpool.Pool, schema, table, column string) string { + t.Helper() + var typ string + err := pool.QueryRow(t.Context(), + `SELECT data_type FROM information_schema.columns + WHERE table_schema = $1 AND table_name = $2 AND column_name = $3`, + schema, table, column).Scan(&typ) + require.NoError(t, err) + return typ +} + +func TestAttemptNativeCommitsInstantChange(t *testing.T) { + pool, schema := newPool(t) + _, err := pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY)", schema)) + require.NoError(t, err) + pt := mustPreflight(t, pool, schema, "t") + + sql := fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int NOT NULL DEFAULT 0", schema) + require.NoError(t, executor.AttemptNative(t.Context(), pool, pt, sql, budget)) + + assert.Equal(t, "integer", columnType(t, pool, schema, "t", "age"), "the committed change must be visible") +} + +func TestAttemptNativeCancelsWhenLockBlocked(t *testing.T) { + pool, schema := newPool(t) + _, err := pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY)", schema)) + require.NoError(t, err) + pt := mustPreflight(t, pool, schema, "t") + + // A second session holds ACCESS EXCLUSIVE for the whole test, so the + // attempt can never be granted its lock. + blocker, err := pool.Begin(t.Context()) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, blocker.Rollback(context.WithoutCancel(t.Context()))) + }) + _, err = blocker.Exec(t.Context(), fmt.Sprintf("LOCK TABLE %s.t IN ACCESS EXCLUSIVE MODE", schema)) + require.NoError(t, err) + + sql := fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int", schema) + err = executor.AttemptNative(t.Context(), pool, pt, sql, budget) + + var budgetErr *executor.BudgetError + require.ErrorAs(t, err, &budgetErr) + assert.Equal(t, executor.CauseLock, budgetErr.Cause) + assert.Equal(t, budget.LockTimeout, budgetErr.Budget) +} + +func TestAttemptNativeCancelsRewriteAndLeavesTableUnchanged(t *testing.T) { + pool, schema := newPool(t) + _, err := pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY, v text)", schema)) + require.NoError(t, err) + // Enough rows that a full table rewrite cannot finish inside a + // millisecond-scale statement budget. + _, err = pool.Exec(t.Context(), fmt.Sprintf( + "INSERT INTO %s.t SELECT g, repeat('x', 100) FROM generate_series(1, 300000) g", schema)) + require.NoError(t, err) + pt := mustPreflight(t, pool, schema, "t") + + // int -> bigint forces a full table rewrite under ACCESS EXCLUSIVE. + sql := fmt.Sprintf("ALTER TABLE %s.t ALTER COLUMN id TYPE bigint", schema) + tight := executor.Budget{LockTimeout: budget.LockTimeout, StatementTimeout: 50 * time.Millisecond} + err = executor.AttemptNative(t.Context(), pool, pt, sql, tight) + + var budgetErr *executor.BudgetError + require.ErrorAs(t, err, &budgetErr) + assert.Equal(t, executor.CauseStatement, budgetErr.Cause) + + // The cancelled attempt must leave schema and data untouched. + assert.Equal(t, "integer", columnType(t, pool, schema, "t", "id")) + var count int + require.NoError(t, pool.QueryRow(t.Context(), + fmt.Sprintf("SELECT count(*) FROM %s.t", schema)).Scan(&count)) + assert.Equal(t, 300000, count) +} + +func TestAttemptNativeSurfacesOperationalErrors(t *testing.T) { + pool, schema := newPool(t) + _, err := pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY)", schema)) + require.NoError(t, err) + pt := mustPreflight(t, pool, schema, "t") + + // Dropping a column that does not exist is a plain SQL error, not a + // budget overrun. + sql := fmt.Sprintf("ALTER TABLE %s.t DROP COLUMN nope", schema) + err = executor.AttemptNative(t.Context(), pool, pt, sql, budget) + require.Error(t, err) + var budgetErr *executor.BudgetError + assert.NotErrorAs(t, err, &budgetErr) +} + +func TestAttemptNativeRejectsUnboundedBudgets(t *testing.T) { + pool, schema := newPool(t) + _, err := pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY)", schema)) + require.NoError(t, err) + pt := mustPreflight(t, pool, schema, "t") + + sql := fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int", schema) + require.Error(t, executor.AttemptNative(t.Context(), pool, pt, sql, executor.Budget{LockTimeout: 0, StatementTimeout: time.Second})) + require.Error(t, executor.AttemptNative(t.Context(), pool, pt, sql, executor.Budget{LockTimeout: time.Second, StatementTimeout: 0})) +} diff --git a/pkg/preflight/preflight.go b/pkg/preflight/preflight.go new file mode 100644 index 0000000..e8b64dd --- /dev/null +++ b/pkg/preflight/preflight.go @@ -0,0 +1,125 @@ +// Package preflight verifies preconditions before the engine writes anything +// (invariant ST-6). In Phase 1 that is the table-size guard in front of the +// optimistic attempt: a cancelled rewrite attempt is not a free probe — it +// holds ACCESS EXCLUSIVE and does real rewrite work for the full statement +// budget — so above a size threshold the attempt is skipped entirely. +// +// This is a safety-critical core package: see SAFETY.md. It returns proof +// types with package-private constructors; dangerous downstream APIs accept +// only the proof. +package preflight + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +// ErrTableNotFound is returned when the target table does not exist (or is +// not visible with the session's search_path). +var ErrTableNotFound = errors.New("table not found") + +// ErrNotTable is returned when the target exists but is not an ordinary or +// partitioned table (e.g. a view or foreign table). +var ErrNotTable = errors.New("not an ordinary or partitioned table") + +// SizeError reports that the table exceeds the configured size threshold, so +// the optimistic attempt must be skipped. It is a refusal input, not an +// operational failure. +type SizeError struct { + // TotalBytes is the table's measured on-disk size (all partitions, + // including TOAST). + TotalBytes int64 + // LimitBytes is the threshold that was exceeded. + LimitBytes int64 +} + +// Error implements the error interface. +func (e *SizeError) Error() string { + return fmt.Sprintf("table size %d bytes exceeds the %d-byte threshold for an optimistic attempt", e.TotalBytes, e.LimitBytes) +} + +// PreflightedTable proves the target table exists, is a table, and is under +// the size threshold for an optimistic attempt. It can only be constructed by +// CheckTable in this package. +type PreflightedTable struct { + schema string + table string + totalBytes int64 + relTuples float64 +} + +// Schema returns the schema qualification the check ran with (empty when the +// lookup used the session search_path). +func (t PreflightedTable) Schema() string { return t.schema } + +// Table returns the verified table name. +func (t PreflightedTable) Table() string { return t.table } + +// TotalBytes returns the measured on-disk size across all partitions. +func (t PreflightedTable) TotalBytes() int64 { return t.totalBytes } + +// RelTuples returns the planner's row estimate (-1 when the table has never +// been vacuumed or analyzed). Reporting only — the size guard's authority is +// bytes on disk. +func (t PreflightedTable) RelTuples() float64 { return t.relTuples } + +// CheckTable verifies that schema.table (search_path when schema is empty) +// exists, is an ordinary or partitioned table, and is at most limitBytes on +// disk. Above the limit it returns a *SizeError; on success it returns the +// PreflightedTable proof. +func CheckTable(ctx context.Context, pool *pgxpool.Pool, schema, table string, limitBytes int64) (PreflightedTable, error) { + if limitBytes <= 0 { + return PreflightedTable{}, fmt.Errorf("size limit must be positive, got %d", limitBytes) + } + // INV: ST-6 — size facts are measured on-disk bytes (pg_table_size, + // which includes TOAST), summed over pg_partition_tree so a partitioned + // parent (whose own relation is 0 bytes) cannot fail open. Stale planner + // statistics (relpages) are never the guard's authority. + // The table's own size plus every descendant in its partition tree: + // pg_partition_tree returns no rows for a plain table (its own + // pg_table_size carries the total) and the parent's own relation is 0 + // bytes for a partitioned table (the descendants carry the total). + const q = ` + SELECT c.relkind::text, + pg_table_size(c.oid) + + (SELECT COALESCE(sum(pg_table_size(p.relid)), 0) + FROM pg_partition_tree(c.oid) p + WHERE p.relid <> c.oid), + c.reltuples + FROM pg_class c + WHERE c.oid = to_regclass( + CASE WHEN $1 = '' THEN quote_ident($2) + ELSE quote_ident($1) || '.' || quote_ident($2) END)` + var relkind string + var totalBytes int64 + var relTuples float64 + err := pool.QueryRow(ctx, q, schema, table).Scan(&relkind, &totalBytes, &relTuples) + if errors.Is(err, pgx.ErrNoRows) { + return PreflightedTable{}, fmt.Errorf("%w: %s", ErrTableNotFound, qualifiedName(schema, table)) + } + if err != nil { + return PreflightedTable{}, fmt.Errorf("look up table %s: %w", qualifiedName(schema, table), err) + } + // relkind 'r' is an ordinary table, 'p' a partitioned parent; anything + // else (view, matview, foreign table, sequence) is refused fail-closed. + if relkind != "r" && relkind != "p" { + return PreflightedTable{}, fmt.Errorf("%w: %s has relkind %q", ErrNotTable, qualifiedName(schema, table), relkind) + } + if totalBytes > limitBytes { + return PreflightedTable{}, &SizeError{TotalBytes: totalBytes, LimitBytes: limitBytes} + } + return PreflightedTable{schema: schema, table: table, totalBytes: totalBytes, relTuples: relTuples}, nil +} + +// qualifiedName renders schema.table for error messages, omitting the dot +// when the name is unqualified. +func qualifiedName(schema, table string) string { + if schema == "" { + return table + } + return schema + "." + table +} diff --git a/pkg/preflight/preflight_integration_test.go b/pkg/preflight/preflight_integration_test.go new file mode 100644 index 0000000..c91ad9a --- /dev/null +++ b/pkg/preflight/preflight_integration_test.go @@ -0,0 +1,122 @@ +package preflight_test + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/block/pg-sprite/internal/testutil" + "github.com/block/pg-sprite/pkg/dbconn" + "github.com/block/pg-sprite/pkg/preflight" +) + +func TestCheckTableUnderLimit(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.small (id int PRIMARY KEY, v text)", schema)) + require.NoError(t, err) + _, err = pool.Exec(t.Context(), fmt.Sprintf("INSERT INTO %s.small SELECT g, 'v' FROM generate_series(1, 100) g", schema)) + require.NoError(t, err) + + pt, err := preflight.CheckTable(t.Context(), pool, schema, "small", 1<<30) + require.NoError(t, err) + assert.Equal(t, schema, pt.Schema()) + assert.Equal(t, "small", pt.Table()) + assert.Positive(t, pt.TotalBytes(), "a populated table must report a nonzero on-disk size") +} + +func TestCheckTableOverLimit(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.big (id int PRIMARY KEY, v text)", schema)) + require.NoError(t, err) + _, err = pool.Exec(t.Context(), fmt.Sprintf("INSERT INTO %s.big SELECT g, repeat('x', 100) FROM generate_series(1, 10000) g", schema)) + require.NoError(t, err) + + _, err = preflight.CheckTable(t.Context(), pool, schema, "big", 1) + var sizeErr *preflight.SizeError + require.ErrorAs(t, err, &sizeErr) + assert.Positive(t, sizeErr.TotalBytes) + assert.Equal(t, int64(1), sizeErr.LimitBytes) +} + +// A partitioned parent's own relation is 0 bytes on disk; the guard must sum +// the partitions so a huge partitioned table cannot slip under the limit. +func TestCheckTableSumsPartitions(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + for _, ddl := range []string{ + fmt.Sprintf("CREATE TABLE %s.parted (id int, v text) PARTITION BY RANGE (id)", schema), + fmt.Sprintf("CREATE TABLE %s.parted_lo PARTITION OF %s.parted FOR VALUES FROM (0) TO (5000)", schema, schema), + fmt.Sprintf("CREATE TABLE %s.parted_hi PARTITION OF %s.parted FOR VALUES FROM (5000) TO (10001)", schema, schema), + fmt.Sprintf("INSERT INTO %s.parted SELECT g, repeat('x', 100) FROM generate_series(0, 10000) g", schema), + } { + _, err = pool.Exec(t.Context(), ddl) + require.NoError(t, err) + } + + pt, err := preflight.CheckTable(t.Context(), pool, schema, "parted", 1<<30) + require.NoError(t, err) + assert.Positive(t, pt.TotalBytes(), "the guard must see the partitions' bytes, not the parent's zero") + + _, err = preflight.CheckTable(t.Context(), pool, schema, "parted", 1) + var sizeErr *preflight.SizeError + require.ErrorAs(t, err, &sizeErr, "a populated partitioned table must exceed a 1-byte limit") +} + +func TestCheckTableMissingTable(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + _, err = preflight.CheckTable(t.Context(), pool, schema, "nope", 1<<30) + require.ErrorIs(t, err, preflight.ErrTableNotFound) +} + +func TestCheckTableRefusesNonTable(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE VIEW %s.v AS SELECT 1 AS one", schema)) + require.NoError(t, err) + + _, err = preflight.CheckTable(t.Context(), pool, schema, "v", 1<<30) + require.ErrorIs(t, err, preflight.ErrNotTable) +} + +func TestCheckTableQuotedIdentifiers(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + _, err = pool.Exec(t.Context(), fmt.Sprintf(`CREATE TABLE %s."Order Items" (id int)`, schema)) + require.NoError(t, err) + + pt, err := preflight.CheckTable(t.Context(), pool, schema, "Order Items", 1<<30) + require.NoError(t, err) + assert.Equal(t, "Order Items", pt.Table()) +} + +func TestCheckTableRejectsNonPositiveLimit(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + + _, err = preflight.CheckTable(t.Context(), pool, "", "whatever", 0) + require.Error(t, err) +} diff --git a/pkg/statement/statement.go b/pkg/statement/statement.go new file mode 100644 index 0000000..45b2697 --- /dev/null +++ b/pkg/statement/statement.go @@ -0,0 +1,97 @@ +// Package statement parses SQL through the real PostgreSQL grammar +// (wasilibs/go-pgquery, Wasm libpg_query) and reports the facts the engine's +// front door needs. In Phase 1 that is a statement-type gate only: which kind +// of statement this is and, for ALTER TABLE, which table it targets. No +// schema model, no classification. +package statement + +import ( + "errors" + "fmt" + + pganalyze "github.com/pganalyze/pg_query_go/v6" + pgquery "github.com/wasilibs/go-pgquery" +) + +// Kind is the statement-type bucket the Phase 1 gate branches on. +type Kind int + +// The kinds the gate distinguishes. Everything the engine does not recognize +// as one of the named kinds is KindOther and is refused by the front door. +const ( + KindOther Kind = iota + KindAlterTable + KindCreateIndex + KindDropIndex + KindReindex +) + +// String returns the human-readable name of the kind. +func (k Kind) String() string { + switch k { + case KindAlterTable: + return "ALTER TABLE" + case KindCreateIndex: + return "CREATE INDEX" + case KindDropIndex: + return "DROP INDEX" + case KindReindex: + return "REINDEX" + default: + return "other" + } +} + +// Statement is one parsed SQL statement plus the facts the gate needs. +type Statement struct { + // SQL is the original statement text as submitted. + SQL string + // Kind is the statement-type bucket. + Kind Kind + // Schema is the target table's schema qualification for ALTER TABLE + // statements; empty when the statement was unqualified (search_path + // resolves it) or when the kind has no single table target. + Schema string + // Table is the target table name for ALTER TABLE statements; empty for + // other kinds. + Table string +} + +// ErrNotOneStatement is returned by ParseOne when the input does not contain +// exactly one SQL statement. +var ErrNotOneStatement = errors.New("input must contain exactly one SQL statement") + +// ParseOne parses sql with the PostgreSQL grammar and requires exactly one +// statement. A parse failure is surfaced to the caller, never guessed around. +func ParseOne(sql string) (Statement, error) { + tree, err := pgquery.Parse(sql) + if err != nil { + return Statement{}, fmt.Errorf("parse statement: %w", err) + } + if n := len(tree.GetStmts()); n != 1 { + return Statement{}, fmt.Errorf("%w: got %d", ErrNotOneStatement, n) + } + st := Statement{SQL: sql} + node := tree.GetStmts()[0].GetStmt() + switch { + case node.GetAlterTableStmt() != nil: + alter := node.GetAlterTableStmt() + // ALTER INDEX (and ALTER VIEW etc.) also parse as AlterTableStmt; + // only a true table target is KindAlterTable. + if alter.GetObjtype() != pganalyze.ObjectType_OBJECT_TABLE { + return st, nil + } + st.Kind = KindAlterTable + st.Schema = alter.GetRelation().GetSchemaname() + st.Table = alter.GetRelation().GetRelname() + case node.GetIndexStmt() != nil: + st.Kind = KindCreateIndex + case node.GetDropStmt() != nil: + if node.GetDropStmt().GetRemoveType() == pganalyze.ObjectType_OBJECT_INDEX { + st.Kind = KindDropIndex + } + case node.GetReindexStmt() != nil: + st.Kind = KindReindex + } + return st, nil +} diff --git a/pkg/statement/statement_test.go b/pkg/statement/statement_test.go new file mode 100644 index 0000000..b61c264 --- /dev/null +++ b/pkg/statement/statement_test.go @@ -0,0 +1,114 @@ +package statement + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseOneKinds(t *testing.T) { + tests := []struct { + name string + sql string + want Statement + }{ + { + name: "alter table unqualified", + sql: "ALTER TABLE users ADD COLUMN age int", + want: Statement{Kind: KindAlterTable, Table: "users"}, + }, + { + name: "alter table schema-qualified", + sql: "ALTER TABLE billing.invoices DROP COLUMN note", + want: Statement{Kind: KindAlterTable, Schema: "billing", Table: "invoices"}, + }, + { + name: "alter table quoted mixed-case identifier", + sql: `ALTER TABLE "Order Items" ADD COLUMN qty int`, + want: Statement{Kind: KindAlterTable, Table: "Order Items"}, + }, + { + name: "alter table if exists", + sql: "ALTER TABLE IF EXISTS users ADD COLUMN age int", + want: Statement{Kind: KindAlterTable, Table: "users"}, + }, + { + name: "create index", + sql: "CREATE INDEX idx_users_email ON users (email)", + want: Statement{Kind: KindCreateIndex}, + }, + { + name: "create unique index concurrently is still an index statement", + sql: "CREATE UNIQUE INDEX CONCURRENTLY idx ON users (email)", + want: Statement{Kind: KindCreateIndex}, + }, + { + name: "drop index", + sql: "DROP INDEX idx_users_email", + want: Statement{Kind: KindDropIndex}, + }, + { + name: "reindex table", + sql: "REINDEX TABLE users", + want: Statement{Kind: KindReindex}, + }, + { + name: "reindex index", + sql: "REINDEX INDEX idx_users_email", + want: Statement{Kind: KindReindex}, + }, + { + name: "alter index parses as AlterTableStmt but is not a table target", + sql: "ALTER INDEX idx_users_email SET (fillfactor = 90)", + want: Statement{Kind: KindOther}, + }, + { + name: "drop table is not a drop-index", + sql: "DROP TABLE users", + want: Statement{Kind: KindOther}, + }, + { + name: "create table", + sql: "CREATE TABLE t (id int)", + want: Statement{Kind: KindOther}, + }, + { + name: "dml", + sql: "UPDATE users SET age = 1", + want: Statement{Kind: KindOther}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseOne(tt.sql) + require.NoError(t, err) + tt.want.SQL = tt.sql + assert.Equal(t, tt.want, got) + }) + } +} + +func TestParseOneRejectsInvalidSQL(t *testing.T) { + _, err := ParseOne("ALTER TABEL users ADD COLUMN age int") + require.Error(t, err) + assert.Contains(t, err.Error(), "parse statement") +} + +func TestParseOneRejectsMultipleStatements(t *testing.T) { + _, err := ParseOne("ALTER TABLE a ADD COLUMN x int; ALTER TABLE b ADD COLUMN y int") + require.ErrorIs(t, err, ErrNotOneStatement) +} + +func TestParseOneRejectsEmptyInput(t *testing.T) { + _, err := ParseOne("") + require.ErrorIs(t, err, ErrNotOneStatement) +} + +func TestKindString(t *testing.T) { + assert.Equal(t, "ALTER TABLE", KindAlterTable.String()) + assert.Equal(t, "CREATE INDEX", KindCreateIndex.String()) + assert.Equal(t, "DROP INDEX", KindDropIndex.String()) + assert.Equal(t, "REINDEX", KindReindex.String()) + assert.Equal(t, "other", KindOther.String()) +} diff --git a/pkg/verdict/verdict.go b/pkg/verdict/verdict.go new file mode 100644 index 0000000..a769d21 --- /dev/null +++ b/pkg/verdict/verdict.go @@ -0,0 +1,104 @@ +// Package verdict is the engine's structured outcome contract: every migrate +// invocation ends in exactly one verdict — executed natively, or refused with +// a typed reason and, where one exists, a safer native idiom. Refusals use a +// distinct exit code from operational errors. This type is the seam a future +// orchestrator adapter maps onto SchemaBot's ExecutionModeBlocked. +package verdict + +import ( + "encoding/json" + "errors" + "fmt" + "strings" +) + +// ExitCodeRefused is the process exit code for a refusal verdict — distinct +// from 1, which means an operational error (could not connect, bad flag, SQL +// error). Automation branches on the difference. +const ExitCodeRefused = 2 + +// ErrRefused is the sentinel the CLI returns after printing a refusal +// verdict, so the entry point can map it to ExitCodeRefused. +var ErrRefused = errors.New("refused") + +// Outcome is what happened to the submitted change. +type Outcome string + +// The two outcomes a migrate run can end in. +const ( + // OutcomeExecuted means the change ran and committed natively within + // its budgets. + OutcomeExecuted Outcome = "executed-natively" + // OutcomeRefused means the change was not executed; Reason says why. + OutcomeRefused Outcome = "refused" +) + +// Reason is the typed cause of a refusal. +type Reason string + +// The refusal reasons Phase 1 can emit. +const ( + // ReasonNone is the zero reason carried by an executed verdict. + ReasonNone Reason = "" + // ReasonUnsupportedStatement: only ALTER TABLE is supported. + ReasonUnsupportedStatement Reason = "unsupported-statement" + // ReasonIndexStatement: index maintenance has a native safe idiom + // (CONCURRENTLY) and is never attempted here. + ReasonIndexStatement Reason = "index-statement" + // ReasonTableTooLarge: the size guard skipped the optimistic attempt. + ReasonTableTooLarge Reason = "not-native-safe: table too large" + // ReasonBudgetExceeded: the optimistic attempt exceeded its lock or + // statement budget and was cancelled. + ReasonBudgetExceeded Reason = "not-native-safe: budget exceeded" +) + +// Verdict is the structured outcome of one migrate invocation. +type Verdict struct { + // Outcome is what happened. + Outcome Outcome `json:"outcome"` + // Reason is the typed refusal cause; empty when executed. + Reason Reason `json:"reason,omitempty"` + // Statement is the submitted SQL. + Statement string `json:"statement"` + // Table is the target table (schema-qualified when the statement was), + // when the statement has one. + Table string `json:"table,omitempty"` + // Detail is the human explanation: why refused, or what committed. + Detail string `json:"detail,omitempty"` + // SaferIdiom is a native alternative to the refused statement, when one + // exists (e.g. CREATE INDEX CONCURRENTLY, ADD CONSTRAINT ... NOT VALID). + SaferIdiom string `json:"safer_idiom,omitempty"` +} + +// JSON renders the verdict as a single JSON object. +func (v Verdict) JSON() (string, error) { + b, err := json.MarshalIndent(v, "", " ") + if err != nil { + return "", fmt.Errorf("encode verdict: %w", err) + } + return string(b), nil +} + +// String renders the verdict for humans. +func (v Verdict) String() string { + var b strings.Builder + switch v.Outcome { + case OutcomeExecuted: + b.WriteString("executed natively") + case OutcomeRefused: + fmt.Fprintf(&b, "refused (%s)", v.Reason) + default: + fmt.Fprintf(&b, "unknown outcome %q", string(v.Outcome)) + } + if v.Table != "" { + fmt.Fprintf(&b, "\n table: %s", v.Table) + } + fmt.Fprintf(&b, "\n statement: %s", v.Statement) + if v.Detail != "" { + fmt.Fprintf(&b, "\n detail: %s", v.Detail) + } + if v.SaferIdiom != "" { + fmt.Fprintf(&b, "\n safer: %s", v.SaferIdiom) + } + return b.String() +} diff --git a/pkg/verdict/verdict_test.go b/pkg/verdict/verdict_test.go new file mode 100644 index 0000000..44ba1cc --- /dev/null +++ b/pkg/verdict/verdict_test.go @@ -0,0 +1,57 @@ +package verdict + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestJSONRoundTrip(t *testing.T) { + v := Verdict{ + Outcome: OutcomeRefused, + Reason: ReasonBudgetExceeded, + Statement: "ALTER TABLE t ALTER COLUMN id TYPE bigint", + Table: "t", + Detail: "the optimistic attempt exceeded its statement budget", + SaferIdiom: "ADD CONSTRAINT ... NOT VALID; VALIDATE CONSTRAINT", + } + s, err := v.JSON() + require.NoError(t, err) + + var got Verdict + require.NoError(t, json.Unmarshal([]byte(s), &got)) + assert.Equal(t, v, got) +} + +func TestJSONOmitsEmptyOptionalFields(t *testing.T) { + s, err := Verdict{Outcome: OutcomeExecuted, Statement: "ALTER TABLE t ADD COLUMN x int"}.JSON() + require.NoError(t, err) + assert.NotContains(t, s, "reason") + assert.NotContains(t, s, "table") + assert.NotContains(t, s, "safer_idiom") +} + +func TestStringExecuted(t *testing.T) { + s := Verdict{ + Outcome: OutcomeExecuted, + Statement: "ALTER TABLE t ADD COLUMN x int", + Table: "t", + Detail: "committed within budget", + }.String() + assert.Contains(t, s, "executed natively") + assert.Contains(t, s, "table: t") + assert.Contains(t, s, "ALTER TABLE t ADD COLUMN x int") +} + +func TestStringRefusedIncludesReasonAndIdiom(t *testing.T) { + s := Verdict{ + Outcome: OutcomeRefused, + Reason: ReasonIndexStatement, + Statement: "CREATE INDEX i ON t (c)", + SaferIdiom: "CREATE INDEX CONCURRENTLY i ON t (c)", + }.String() + assert.Contains(t, s, "refused (index-statement)") + assert.Contains(t, s, "CREATE INDEX CONCURRENTLY") +} From 6fee8932edc3b9d22cbc2b1fa4cb525438662fd4 Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Tue, 4 Aug 2026 20:35:45 +1000 Subject: [PATCH 07/24] Cover status active-session rendering and budgetVerdict lock branch Closes the two behavioral coverage gaps from the Phase 1 coverage review: the status command's live-session output (pid/state/query fields against a held pg-sprite session) and the lock-budget refusal wording, which were previously untested paths. --- internal/cli/migrate_integration_test.go | 31 +++++++++++++++++ internal/cli/migrate_test.go | 44 ++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 internal/cli/migrate_test.go diff --git a/internal/cli/migrate_integration_test.go b/internal/cli/migrate_integration_test.go index 21b1aa7..fafbcf5 100644 --- a/internal/cli/migrate_integration_test.go +++ b/internal/cli/migrate_integration_test.go @@ -170,3 +170,34 @@ func TestStatusReportsNoSessions(t *testing.T) { require.NoError(t, cmd.run(t.Context(), &out)) assert.Contains(t, out.String(), "no active pg-sprite sessions") } + +// A live pg-sprite session (any connection made through pkg/dbconn) is +// rendered with its pid and per-session fields. The session is held open on a +// pinned connection so pg_stat_activity is guaranteed to contain it while +// status runs. +func TestStatusReportsActiveSession(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + + conn, err := pool.Acquire(t.Context()) + require.NoError(t, err) + defer conn.Release() + var pid int + // The marker aliases make the session's pid and last-query text + // deterministic for the assertions below. + require.NoError(t, conn.QueryRow(t.Context(), + "SELECT pg_backend_pid() AS pgsprite_status_marker").Scan(&pid)) + + cmd := &StatusCmd{DBFlags: DBFlags{URL: url}} + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + + got := out.String() + assert.Contains(t, got, "active pg-sprite sessions:") + assert.Contains(t, got, fmt.Sprintf("pid=%d state=idle", pid), + "the held session must be listed with its pid and state") + assert.Contains(t, got, "pgsprite_status_marker", + "the session's last query must be rendered") +} diff --git a/internal/cli/migrate_test.go b/internal/cli/migrate_test.go new file mode 100644 index 0000000..0bd9566 --- /dev/null +++ b/internal/cli/migrate_test.go @@ -0,0 +1,44 @@ +package cli + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/block/pg-sprite/pkg/executor" + "github.com/block/pg-sprite/pkg/statement" + "github.com/block/pg-sprite/pkg/verdict" +) + +func TestBudgetVerdict(t *testing.T) { + st := statement.Statement{ + SQL: "ALTER TABLE billing.invoices ALTER COLUMN id TYPE bigint", + Kind: statement.KindAlterTable, + Schema: "billing", + Table: "invoices", + } + + t.Run("lock budget", func(t *testing.T) { + v := budgetVerdict(st, &executor.BudgetError{Cause: executor.CauseLock, Budget: 3 * time.Second}) + assert.Equal(t, verdict.OutcomeRefused, v.Outcome) + assert.Equal(t, verdict.ReasonBudgetExceeded, v.Reason) + assert.Equal(t, "billing.invoices", v.Table) + assert.Contains(t, v.Detail, "lock was not granted within the 3s lock budget") + assert.Contains(t, v.Detail, "nothing was executed") + }) + + t.Run("statement budget", func(t *testing.T) { + v := budgetVerdict(st, &executor.BudgetError{Cause: executor.CauseStatement, Budget: 30 * time.Second}) + assert.Equal(t, verdict.ReasonBudgetExceeded, v.Reason) + assert.Contains(t, v.Detail, "cancelled after the 30s statement budget") + assert.Contains(t, v.Detail, "ADD CONSTRAINT ... NOT VALID") + }) + + t.Run("unknown cause falls back to the error text", func(t *testing.T) { + budgetErr := &executor.BudgetError{Budget: time.Second} + v := budgetVerdict(st, budgetErr) + assert.Equal(t, verdict.ReasonBudgetExceeded, v.Reason) + assert.Equal(t, budgetErr.Error(), v.Detail) + }) +} From 4ea0e6c9820b50ba31b789b3faa56fedd65b673c Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Tue, 4 Aug 2026 20:42:36 +1000 Subject: [PATCH 08/24] Add logging principles, slog lint hygiene, and a --debug diagnostics flag Establishes the observability posture early: stdout carries only command output, diagnostics go through log/slog to stderr, errors are logged once at the entry point, and credentials are never logged. sloglint/forbidigo enforce the mechanical half; --debug wires the previously unreachable pgx statement tracing in pkg/dbconn plus migrate lifecycle events. --- .golangci.yml | 16 ++++++ AGENTS.md | 21 +++++++- internal/cli/cli.go | 29 ++++++++++- internal/cli/migrate.go | 10 ++++ internal/cli/migrate_integration_test.go | 64 ++++++++++++++++++++++++ 5 files changed, 138 insertions(+), 2 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index dc76b27..026d2ff 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -20,7 +20,16 @@ linters: - gochecknoinits # no init() functions - gochecknoglobals # no package-level mutable state (error sentinels exempt) - containedctx # no context.Context stored in struct fields + - sloglint # structured-logging hygiene: static messages, snake_case keys + - forbidigo # no printing to process stdout; output goes to the injected writer settings: + sloglint: + static-msg: true + key-naming-case: snake + forbidigo: + forbid: + - pattern: ^(fmt\.Print(f|ln)?|print|println)$ + msg: command output goes to the injected io.Writer and diagnostics to slog, never process stdout usetesting: context-background: true context-todo: true @@ -32,6 +41,13 @@ linters: - name: package-comments exclusions: rules: + # The tracelog adapter forwards pgx's own message strings; the + # static-message rule applies to messages we author, not to a bridge + # for a foreign logger. + - path: pkg/dbconn/dbconn\.go + linters: + - sloglint + text: message should be a string literal # Test helpers intentionally keep uniform signatures and fixed arguments # for readability, so unused/constant params there are not worth churn. - path: _test\.go diff --git a/AGENTS.md b/AGENTS.md index 6b9503e..5259034 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -100,8 +100,27 @@ make lint # golangci-lint - Never reference internal company details (cluster names, hostnames, org names) in code, comments, commits, or PRs — this is a public repo. +## Logging and observability + +- **stdout is the product's output; diagnostics go to stderr.** Command results (verdicts, + status) are written to the injected writer only; everything diagnostic goes through + `log/slog`. `--debug` on DB commands enables statement-level tracing (pgx tracelog via + `pkg/dbconn`) plus lifecycle events; without it, diagnostics are discarded. +- **Log decisions and state transitions, not progress noise.** Static messages; the + variability goes into attrs with stable snake_case keys, and the same key means the same + thing everywhere (`schema`, `table`, `total_bytes`, `elapsed`). +- **One error, one log.** Errors are wrapped and returned; only the entry point logs or + prints them. `pkg/` packages never log an error they also return. +- **Never log credentials or connection strings** — a DSN/URL carries a password; log host, + database, and user as separate attrs when needed. Never log row data. +- **Operational quantities ride on logs until there is a metrics runtime.** Durations, + sizes, and retry counts are logged as attrs. When the long-running phases need real + metrics, core packages emit through one narrow engine-owned interface — no direct + prometheus/otel imports in core (the dependency rule in SAFETY.md applies). + Mechanical style rules (doc comments on exported symbols, no `init()`, no package-level -mutable state, no `context.Context` in structs) are enforced by `.golangci.yml`, not prose. +mutable state, no `context.Context` in structs, static slog messages with snake_case keys, +no printing to process stdout) are enforced by `.golangci.yml`, not prose. Design docs live in [docs/](docs/) — start at [docs/README.md](docs/README.md); the invariant registry is [docs/invariants.md](docs/invariants.md). diff --git a/internal/cli/cli.go b/internal/cli/cli.go index cc37b14..5134a21 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -6,6 +6,8 @@ package cli import ( "context" "fmt" + "io" + "log/slog" "os" "time" @@ -39,16 +41,41 @@ type DBFlags struct { CACert string `help:"CA bundle path for verify-full TLS. RDS/Aurora endpoints verify with the embedded bundle automatically." env:"PGSPRITE_CA_CERT" type:"existingfile"` LockTimeout time.Duration `help:"Session lock_timeout applied to every statement." default:"3s"` StatementTimeout time.Duration `help:"Session statement_timeout applied to every statement." default:"30s"` + Debug bool `help:"Log statement-level tracing and lifecycle diagnostics to stderr."` + + // diagOut overrides the diagnostics destination (stderr) in tests. Kong + // ignores unexported fields. + diagOut io.Writer } // Config translates the flags into the connectivity layer's configuration. +// The tracer is wired only under --debug: dbconn skips statement tracing +// entirely for a nil logger. func (f DBFlags) Config() dbconn.Config { - return dbconn.Config{ + cfg := dbconn.Config{ URL: f.URL, CACertPath: f.CACert, LockTimeout: f.LockTimeout, StatementTimeout: f.StatementTimeout, } + if f.Debug { + cfg.Logger = f.diag() + } + return cfg +} + +// diag returns the diagnostics logger: debug-level text on stderr (or the +// test override) under --debug, a discarding logger otherwise. Diagnostics +// never share stdout with command output. +func (f DBFlags) diag() *slog.Logger { + if !f.Debug { + return slog.New(slog.DiscardHandler) + } + out := f.diagOut + if out == nil { + out = os.Stderr + } + return slog.New(slog.NewTextHandler(out, &slog.HandlerOptions{Level: slog.LevelDebug})) } // MigrateCmd runs a schema change (imperative front-end): the Phase 1 diff --git a/internal/cli/migrate.go b/internal/cli/migrate.go index 4dcaa75..ac10db4 100644 --- a/internal/cli/migrate.go +++ b/internal/cli/migrate.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "time" "github.com/block/pg-sprite/pkg/dbconn" "github.com/block/pg-sprite/pkg/executor" @@ -18,10 +19,12 @@ import ( // verdicts are printed to out and returned as verdict.ErrRefused so the entry // point maps them to the refusal exit code. func (c *MigrateCmd) run(ctx context.Context, out io.Writer) error { + logger := c.diag() st, err := statement.ParseOne(c.Alter) if err != nil { return err } + logger.Debug("statement parsed", "kind", st.Kind, "schema", st.Schema, "table", st.Table) if v, refused := gateVerdict(st); refused { return c.emit(out, v) } @@ -40,16 +43,23 @@ func (c *MigrateCmd) run(ctx context.Context, out io.Writer) error { if err != nil { return err } + logger.Debug("preflight passed", + "table", qualified(st), "total_bytes", pt.TotalBytes(), "limit_bytes", int64(c.MaxTableSize)) budget := executor.Budget{LockTimeout: c.LockTimeout, StatementTimeout: c.StatementTimeout} + start := time.Now() err = executor.AttemptNative(ctx, pool, pt, st.SQL, budget) + elapsed := time.Since(start) var budgetErr *executor.BudgetError if errors.As(err, &budgetErr) { + logger.Debug("optimistic attempt cancelled", + "cause", budgetErr.Cause, "budget", budgetErr.Budget, "elapsed", elapsed) return c.emit(out, budgetVerdict(st, budgetErr)) } if err != nil { return err } + logger.Debug("optimistic attempt committed", "table", qualified(st), "elapsed", elapsed) return c.emit(out, verdict.Verdict{ Outcome: verdict.OutcomeExecuted, Statement: st.SQL, diff --git a/internal/cli/migrate_integration_test.go b/internal/cli/migrate_integration_test.go index fafbcf5..ada0119 100644 --- a/internal/cli/migrate_integration_test.go +++ b/internal/cli/migrate_integration_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" "strings" + "sync" "testing" "time" @@ -163,6 +164,69 @@ func TestMigrateSurfacesParseErrors(t *testing.T) { assert.NotErrorIs(t, err, verdict.ErrRefused, "a parse failure is an operational error, not a refusal") } +// syncWriter guards the diagnostics buffer: pgx tracelog can write from pool +// housekeeping goroutines concurrently with the command's own lifecycle logs. +type syncWriter struct { + mu sync.Mutex + b strings.Builder +} + +func (w *syncWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.b.Write(p) +} + +func (w *syncWriter) String() string { + w.mu.Lock() + defer w.mu.Unlock() + return w.b.String() +} + +// --debug emits lifecycle events and pgx statement tracing on the diagnostics +// stream, and never leaks them into the command's stdout output; without the +// flag diagnostics are discarded entirely. +func TestMigrateDebugDiagnostics(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY)", schema)) + require.NoError(t, err) + + t.Run("debug on", func(t *testing.T) { + cmd := newMigrateCmd(url, fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int", schema)) + cmd.Debug = true + var diag syncWriter + cmd.diagOut = &diag + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + + assert.Contains(t, out.String(), "executed natively") + assert.NotContains(t, out.String(), "level=DEBUG", + "diagnostics must not leak into command output") + + got := diag.String() + assert.Contains(t, got, "statement parsed") + assert.Contains(t, got, `kind="ALTER TABLE"`) + assert.Contains(t, got, "preflight passed") + assert.Contains(t, got, "total_bytes=") + assert.Contains(t, got, "optimistic attempt committed") + assert.Contains(t, got, "elapsed=") + assert.Contains(t, got, "msg=Query", "pgx statement tracing must be wired under --debug") + }) + + t.Run("debug off discards diagnostics", func(t *testing.T) { + cmd := newMigrateCmd(url, fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age2 int", schema)) + var diag syncWriter + cmd.diagOut = &diag + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + assert.Empty(t, diag.String()) + }) +} + func TestStatusReportsNoSessions(t *testing.T) { url := testutil.StartPostgres(t) cmd := &StatusCmd{DBFlags: DBFlags{URL: url}} From 4adba6f1b89492d6586a179f3ba8de33602e16d6 Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Tue, 4 Aug 2026 21:05:18 +1000 Subject: [PATCH 09/24] Make CLI tests assert typed outcomes, not wording; add verdict cause Log text and human-facing prose are no longer a test surface (rule added to AGENTS.md, aligned with SchemaBot's observability guidance). The lock-vs-statement budget distinction was only visible in prose, so the verdict gains a typed machine-readable cause field, and status gains --json so its tests can assert structured session fields. --- AGENTS.md | 13 ++++- internal/cli/cli.go | 2 + internal/cli/migrate.go | 2 + internal/cli/migrate_integration_test.go | 73 ++++++++++++++++-------- internal/cli/migrate_test.go | 9 +-- internal/cli/status.go | 73 ++++++++++++++++++------ pkg/verdict/verdict.go | 19 ++++++ 7 files changed, 142 insertions(+), 49 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5259034..5682218 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -109,14 +109,23 @@ make lint # golangci-lint - **Log decisions and state transitions, not progress noise.** Static messages; the variability goes into attrs with stable snake_case keys, and the same key means the same thing everywhere (`schema`, `table`, `total_bytes`, `elapsed`). +- **Logs answer the triage question.** Error- and warn-path logs carry the identifiers an + operator needs to act — schema, table, database, the operation being attempted — as + attrs, not buried in prose. - **One error, one log.** Errors are wrapped and returned; only the entry point logs or prints them. `pkg/` packages never log an error they also return. - **Never log credentials or connection strings** — a DSN/URL carries a password; log host, database, and user as separate attrs when needed. Never log row data. +- **Log output is never a test surface.** Tests assert typed outcomes — `errors.Is`/`As`, + verdict fields, exit codes, JSON output — never log text or human-facing wording. If a + behavioral difference is visible only in prose, make it machine-readable first (a typed + field or reason), then test that. The only exception is a renderer's own unit test. - **Operational quantities ride on logs until there is a metrics runtime.** Durations, sizes, and retry counts are logged as attrs. When the long-running phases need real - metrics, core packages emit through one narrow engine-owned interface — no direct - prometheus/otel imports in core (the dependency rule in SAFETY.md applies). + metrics, they arrive as OpenTelemetry instruments behind one engine-owned `pkg/metrics` + with `Record*` helpers — dotted `pgsprite.` names with explicit units, low-cardinality + snake_case attributes, counters for rare or dangerous branches operators can act on — + never direct exporter imports in core (the dependency rule in SAFETY.md applies). Mechanical style rules (doc comments on exported symbols, no `init()`, no package-level mutable state, no `context.Context` in structs, static slog messages with snake_case keys, diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 5134a21..157a343 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -119,6 +119,8 @@ func (c *LintCmd) Run() error { return notImplemented("lint") } // StatusCmd reports migration progress. type StatusCmd struct { DBFlags `embed:""` + + JSON bool `help:"Emit the session listing as JSON."` } // Run implements the status subcommand. diff --git a/internal/cli/migrate.go b/internal/cli/migrate.go index ac10db4..e2eff93 100644 --- a/internal/cli/migrate.go +++ b/internal/cli/migrate.go @@ -141,9 +141,11 @@ func budgetVerdict(st statement.Statement, budgetErr *executor.BudgetError) verd } switch budgetErr.Cause { case executor.CauseLock: + v.Cause = verdict.CauseLockBudget v.Detail = fmt.Sprintf("the lock was not granted within the %s lock budget: the table is too "+ "contended for a blind attempt; nothing was executed", budgetErr.Budget) case executor.CauseStatement: + v.Cause = verdict.CauseStatementBudget v.Detail = fmt.Sprintf("cancelled after the %s statement budget: the change does real rewrite work, "+ "not an in-place catalog change, and needs a copy-and-swap rewrite that pg-sprite does not perform yet. "+ "If it adds a constraint, ADD CONSTRAINT ... NOT VALID followed by VALIDATE CONSTRAINT avoids the long lock", diff --git a/internal/cli/migrate_integration_test.go b/internal/cli/migrate_integration_test.go index ada0119..20b2db2 100644 --- a/internal/cli/migrate_integration_test.go +++ b/internal/cli/migrate_integration_test.go @@ -3,6 +3,8 @@ package cli import ( "encoding/json" "fmt" + "slices" + "strconv" "strings" "sync" "testing" @@ -40,9 +42,14 @@ func TestMigrateExecutesInstantChange(t *testing.T) { require.NoError(t, err) cmd := newMigrateCmd(url, fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int", schema)) + cmd.JSON = true var out strings.Builder require.NoError(t, cmd.run(t.Context(), &out)) - assert.Contains(t, out.String(), "executed natively") + + var v verdict.Verdict + require.NoError(t, json.Unmarshal([]byte(out.String()), &v)) + assert.Equal(t, verdict.OutcomeExecuted, v.Outcome) + assert.Equal(t, schema+".t", v.Table) var typ string require.NoError(t, pool.QueryRow(t.Context(), @@ -76,6 +83,7 @@ func TestMigrateRefusesRewriteWithBudgetVerdict(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(out.String()), &v)) assert.Equal(t, verdict.OutcomeRefused, v.Outcome) assert.Equal(t, verdict.ReasonBudgetExceeded, v.Reason) + assert.Equal(t, verdict.CauseStatementBudget, v.Cause) assert.Equal(t, schema+".t", v.Table) var typ string @@ -198,23 +206,21 @@ func TestMigrateDebugDiagnostics(t *testing.T) { t.Run("debug on", func(t *testing.T) { cmd := newMigrateCmd(url, fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int", schema)) cmd.Debug = true + cmd.JSON = true var diag syncWriter cmd.diagOut = &diag var out strings.Builder require.NoError(t, cmd.run(t.Context(), &out)) - assert.Contains(t, out.String(), "executed natively") - assert.NotContains(t, out.String(), "level=DEBUG", - "diagnostics must not leak into command output") - - got := diag.String() - assert.Contains(t, got, "statement parsed") - assert.Contains(t, got, `kind="ALTER TABLE"`) - assert.Contains(t, got, "preflight passed") - assert.Contains(t, got, "total_bytes=") - assert.Contains(t, got, "optimistic attempt committed") - assert.Contains(t, got, "elapsed=") - assert.Contains(t, got, "msg=Query", "pgx statement tracing must be wired under --debug") + // A strict decode of stdout proves diagnostics did not leak into the + // command's output stream: any interleaved log line would break it. + var v verdict.Verdict + require.NoError(t, json.Unmarshal([]byte(out.String()), &v), + "stdout must carry exactly the verdict, nothing else") + assert.Equal(t, verdict.OutcomeExecuted, v.Outcome) + + assert.NotEmpty(t, diag.String(), + "--debug must emit diagnostics on the diagnostics stream") }) t.Run("debug off discards diagnostics", func(t *testing.T) { @@ -227,16 +233,26 @@ func TestMigrateDebugDiagnostics(t *testing.T) { }) } +// Both output modes report the empty case: an empty JSON list, and a +// non-empty human explanation. func TestStatusReportsNoSessions(t *testing.T) { url := testutil.StartPostgres(t) - cmd := &StatusCmd{DBFlags: DBFlags{URL: url}} + + cmd := &StatusCmd{DBFlags: DBFlags{URL: url}, JSON: true} var out strings.Builder require.NoError(t, cmd.run(t.Context(), &out)) - assert.Contains(t, out.String(), "no active pg-sprite sessions") + var sessions []session + require.NoError(t, json.Unmarshal([]byte(out.String()), &sessions)) + assert.Empty(t, sessions) + + cmd = &StatusCmd{DBFlags: DBFlags{URL: url}} + var text strings.Builder + require.NoError(t, cmd.run(t.Context(), &text)) + assert.NotEmpty(t, text.String()) } // A live pg-sprite session (any connection made through pkg/dbconn) is -// rendered with its pid and per-session fields. The session is held open on a +// reported with its pid and per-session fields. The session is held open on a // pinned connection so pg_stat_activity is guaranteed to contain it while // status runs. func TestStatusReportsActiveSession(t *testing.T) { @@ -249,19 +265,26 @@ func TestStatusReportsActiveSession(t *testing.T) { require.NoError(t, err) defer conn.Release() var pid int - // The marker aliases make the session's pid and last-query text - // deterministic for the assertions below. + // The marker alias makes the session's last-query text deterministic + // for the field assertion below. require.NoError(t, conn.QueryRow(t.Context(), "SELECT pg_backend_pid() AS pgsprite_status_marker").Scan(&pid)) - cmd := &StatusCmd{DBFlags: DBFlags{URL: url}} + cmd := &StatusCmd{DBFlags: DBFlags{URL: url}, JSON: true} var out strings.Builder require.NoError(t, cmd.run(t.Context(), &out)) - got := out.String() - assert.Contains(t, got, "active pg-sprite sessions:") - assert.Contains(t, got, fmt.Sprintf("pid=%d state=idle", pid), - "the held session must be listed with its pid and state") - assert.Contains(t, got, "pgsprite_status_marker", - "the session's last query must be rendered") + var sessions []session + require.NoError(t, json.Unmarshal([]byte(out.String()), &sessions)) + i := slices.IndexFunc(sessions, func(s session) bool { return s.PID == pid }) + require.GreaterOrEqual(t, i, 0, "the held session must be listed") + assert.Equal(t, "idle", sessions[i].State) + assert.Contains(t, sessions[i].Query, "pgsprite_status_marker", + "the session's last query must be reported") + + // The human rendering carries the same session. + cmd = &StatusCmd{DBFlags: DBFlags{URL: url}} + var text strings.Builder + require.NoError(t, cmd.run(t.Context(), &text)) + assert.Contains(t, text.String(), strconv.Itoa(pid)) } diff --git a/internal/cli/migrate_test.go b/internal/cli/migrate_test.go index 0bd9566..6016687 100644 --- a/internal/cli/migrate_test.go +++ b/internal/cli/migrate_test.go @@ -23,22 +23,23 @@ func TestBudgetVerdict(t *testing.T) { v := budgetVerdict(st, &executor.BudgetError{Cause: executor.CauseLock, Budget: 3 * time.Second}) assert.Equal(t, verdict.OutcomeRefused, v.Outcome) assert.Equal(t, verdict.ReasonBudgetExceeded, v.Reason) + assert.Equal(t, verdict.CauseLockBudget, v.Cause) assert.Equal(t, "billing.invoices", v.Table) - assert.Contains(t, v.Detail, "lock was not granted within the 3s lock budget") - assert.Contains(t, v.Detail, "nothing was executed") + assert.NotEmpty(t, v.Detail) }) t.Run("statement budget", func(t *testing.T) { v := budgetVerdict(st, &executor.BudgetError{Cause: executor.CauseStatement, Budget: 30 * time.Second}) assert.Equal(t, verdict.ReasonBudgetExceeded, v.Reason) - assert.Contains(t, v.Detail, "cancelled after the 30s statement budget") - assert.Contains(t, v.Detail, "ADD CONSTRAINT ... NOT VALID") + assert.Equal(t, verdict.CauseStatementBudget, v.Cause) + assert.NotEmpty(t, v.Detail) }) t.Run("unknown cause falls back to the error text", func(t *testing.T) { budgetErr := &executor.BudgetError{Budget: time.Second} v := budgetVerdict(st, budgetErr) assert.Equal(t, verdict.ReasonBudgetExceeded, v.Reason) + assert.Equal(t, verdict.CauseNone, v.Cause) assert.Equal(t, budgetErr.Error(), v.Detail) }) } diff --git a/internal/cli/status.go b/internal/cli/status.go index 70f7e6d..27bddff 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -2,12 +2,24 @@ package cli import ( "context" + "encoding/json" "fmt" "io" + "github.com/jackc/pgx/v5/pgxpool" + "github.com/block/pg-sprite/pkg/dbconn" ) +// session is one live pg-sprite backend from pg_stat_activity. +type session struct { + PID int `json:"pid"` + State string `json:"state"` + WaitEvent string `json:"wait_event"` + RunningFor string `json:"running_for"` + Query string `json:"query"` +} + // run reports the engine's live database sessions. Phase 1 has no durable // migration state — a change either committed within its budgets or was // refused — so status is a view over pg_stat_activity for pg-sprite sessions. @@ -18,6 +30,26 @@ func (c *StatusCmd) run(ctx context.Context, out io.Writer) error { } defer pool.Close() + sessions, err := querySessions(ctx, pool) + if err != nil { + return err + } + if c.JSON { + b, err := json.MarshalIndent(sessions, "", " ") + if err != nil { + return fmt.Errorf("encode status: %w", err) + } + if _, err := fmt.Fprintln(out, string(b)); err != nil { + return fmt.Errorf("write status: %w", err) + } + return nil + } + return renderSessions(out, sessions) +} + +// querySessions lists the live pg-sprite backends other than the one running +// the status query itself. +func querySessions(ctx context.Context, pool *pgxpool.Pool) ([]session, error) { rows, err := pool.Query(ctx, ` SELECT pid, state, COALESCE(wait_event_type || '/' || wait_event, '-'), @@ -27,35 +59,40 @@ func (c *StatusCmd) run(ctx context.Context, out io.Writer) error { WHERE application_name = 'pg-sprite' AND pid <> pg_backend_pid() ORDER BY query_start`) if err != nil { - return fmt.Errorf("query pg_stat_activity: %w", err) + return nil, fmt.Errorf("query pg_stat_activity: %w", err) } defer rows.Close() - sessions := 0 + sessions := []session{} for rows.Next() { - var pid int - var state, waitEvent, runningFor, query string - if err := rows.Scan(&pid, &state, &waitEvent, &runningFor, &query); err != nil { - return fmt.Errorf("scan pg_stat_activity row: %w", err) - } - sessions++ - if sessions == 1 { - if _, err := fmt.Fprintln(out, "active pg-sprite sessions:"); err != nil { - return fmt.Errorf("write status: %w", err) - } - } - if _, err := fmt.Fprintf(out, " pid=%d state=%s wait=%s running_for=%s query=%q\n", - pid, state, waitEvent, runningFor, query); err != nil { - return fmt.Errorf("write status: %w", err) + var s session + if err := rows.Scan(&s.PID, &s.State, &s.WaitEvent, &s.RunningFor, &s.Query); err != nil { + return nil, fmt.Errorf("scan pg_stat_activity row: %w", err) } + sessions = append(sessions, s) } if err := rows.Err(); err != nil { - return fmt.Errorf("read pg_stat_activity: %w", err) + return nil, fmt.Errorf("read pg_stat_activity: %w", err) } - if sessions == 0 { + return sessions, nil +} + +// renderSessions writes the human-readable session listing. +func renderSessions(out io.Writer, sessions []session) error { + if len(sessions) == 0 { if _, err := fmt.Fprintln(out, "no active pg-sprite sessions (Phase 1 keeps no durable migration state: a change either committed within its budgets or was refused)"); err != nil { return fmt.Errorf("write status: %w", err) } + return nil + } + if _, err := fmt.Fprintln(out, "active pg-sprite sessions:"); err != nil { + return fmt.Errorf("write status: %w", err) + } + for _, s := range sessions { + if _, err := fmt.Fprintf(out, " pid=%d state=%s wait=%s running_for=%s query=%q\n", + s.PID, s.State, s.WaitEvent, s.RunningFor, s.Query); err != nil { + return fmt.Errorf("write status: %w", err) + } } return nil } diff --git a/pkg/verdict/verdict.go b/pkg/verdict/verdict.go index a769d21..928d6f4 100644 --- a/pkg/verdict/verdict.go +++ b/pkg/verdict/verdict.go @@ -52,12 +52,31 @@ const ( ReasonBudgetExceeded Reason = "not-native-safe: budget exceeded" ) +// Cause narrows ReasonBudgetExceeded to the budget that was exceeded, so +// automation can branch on which limit fired without parsing prose. +type Cause string + +// The budget causes a refusal can carry. +const ( + // CauseNone is the zero cause for verdicts that are not budget refusals. + CauseNone Cause = "" + // CauseLockBudget: the lock was not granted within lock_timeout; nothing + // was executed. + CauseLockBudget Cause = "lock-budget" + // CauseStatementBudget: the statement ran past statement_timeout and was + // cancelled; the change needs a rewrite. + CauseStatementBudget Cause = "statement-budget" +) + // Verdict is the structured outcome of one migrate invocation. type Verdict struct { // Outcome is what happened. Outcome Outcome `json:"outcome"` // Reason is the typed refusal cause; empty when executed. Reason Reason `json:"reason,omitempty"` + // Cause narrows a budget refusal to the budget that fired; empty + // otherwise. + Cause Cause `json:"cause,omitempty"` // Statement is the submitted SQL. Statement string `json:"statement"` // Table is the target table (schema-qualified when the statement was), From d626da7494e8434763591225b34b7d56e1d94478 Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Tue, 4 Aug 2026 22:29:50 +1000 Subject: [PATCH 10/24] Phase 2.1: desired-state parse boundary and execute-and-introspect MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Admit a declarative desired-state file (one CREATE TABLE plus its CREATE INDEXes) through the real grammar, materialize it in an always-rolled-back transaction-scoped scratch schema on the target, and introspect both live and desired state into one canonical model from server catalogs — semantic truth comes from PostgreSQL, never from AST transformation, per the layered DDL-understanding decision. --- SAFETY.md | 2 +- docs/low-level-design.md | 9 ++ internal/cli/migrate.go | 4 + internal/cli/migrate_integration_test.go | 2 +- pkg/schemadiff/desired.go | 74 +++++++++ pkg/schemadiff/introspect.go | 185 +++++++++++++++++++++++ pkg/schemadiff/schemadiff.go | 79 ++++++++++ pkg/statement/desired.go | 155 +++++++++++++++++++ pkg/statement/desired_test.go | 87 +++++++++++ pkg/statement/statement.go | 8 + pkg/statement/statement_test.go | 2 +- 11 files changed, 604 insertions(+), 3 deletions(-) create mode 100644 pkg/schemadiff/desired.go create mode 100644 pkg/schemadiff/introspect.go create mode 100644 pkg/schemadiff/schemadiff.go create mode 100644 pkg/statement/desired.go create mode 100644 pkg/statement/desired_test.go diff --git a/SAFETY.md b/SAFETY.md index cc56d6a..418341c 100644 --- a/SAFETY.md +++ b/SAFETY.md @@ -27,7 +27,7 @@ The invariant registry (invariant IDs referenced below) lives in | `pkg/checkpoint` — durable resume state | ✅ core | planned (Phase 8) | ST-1, ST-2 | | slot lifecycle (in `pkg/decode`) — create, reap, lag ceiling | ✅ core | planned (Phase 8) | ST-3 | | `pkg/migration` — orchestrator, **cutover swap + fidelity gate** | ✅ core | planned (Phase 7) | LK-2, LK-4, ST-5 | -| `pkg/statement`, `pkg/planner`, `pkg/schemadiff`, `pkg/lint` — classify/diff/route | ❌ periphery¹ | `pkg/statement` exists (Phase 1: type gate); rest planned (Phase 2) | (CO-7 holds at the parse boundary) | +| `pkg/statement`, `pkg/planner`, `pkg/schemadiff`, `pkg/lint` — classify/diff/route | ❌ periphery¹ | `pkg/statement` (parse boundary) and `pkg/schemadiff` (introspect/diff via scratch execute-and-introspect) exist (Phase 2); `pkg/planner`, `pkg/lint` planned (Phase 2) | (CO-7 holds at the parse boundary) | | `pkg/verdict` — structured outcome contract, rendering, exit codes | ❌ periphery | exists (Phase 1) | — | | `internal/cli` — CLI, flags, help, prompts | ❌ periphery | `migrate`/`status` exist (Phase 1); rest stubs | — | | status / progress / advisory rendering, metrics | ❌ periphery | planned | — | diff --git a/docs/low-level-design.md b/docs/low-level-design.md index c89c47d..e27ed12 100644 --- a/docs/low-level-design.md +++ b/docs/low-level-design.md @@ -544,6 +544,15 @@ The scratch database is engine-owned and disposable: preflight may reset it (dro contents) at any time. Restricted environments that won't grant `CREATEDB` pre-provision instead. +**Plan-time diffing uses a lighter mechanism.** `pkg/schemadiff` materializes the desired +state inside a single always-rolled-back transaction in the *target* database, in a +randomly named transaction-scoped schema (`pgsprite_scratch_`). This keeps the +same-server semantic-truth property (same version, extensions, and defaults as the live +table) while requiring no `CREATEDB`, no pre-provisioning, and leaving zero footprint — +appropriate because diffing is read-only planning. The durable `pg_sprite_scratch` +database above is required only by the migration path proper (shadow-DDL derivation and +checkpoint fingerprints), where objects must outlive a transaction. + ### Postgres-only preconditions Spirit has no analog for These have **no MySQL counterpart** but are hard requirements for the logical-decoding path: diff --git a/internal/cli/migrate.go b/internal/cli/migrate.go index e2eff93..e43227c 100644 --- a/internal/cli/migrate.go +++ b/internal/cli/migrate.go @@ -108,6 +108,10 @@ func gateVerdict(st statement.Statement) (verdict.Verdict, bool) { v.Reason = verdict.ReasonIndexStatement v.Detail = "a plain REINDEX blocks writes; the concurrent rebuild does not" v.SaferIdiom = "REINDEX ... CONCURRENTLY" + case statement.KindCreateTable: + v.Reason = verdict.ReasonUnsupportedStatement + v.Detail = "migrate changes an existing table; to converge a table onto a desired-state CREATE TABLE, use the declarative front-end" + v.SaferIdiom = "pg-sprite diff --desired schema.sql" case statement.KindOther: v.Reason = verdict.ReasonUnsupportedStatement v.Detail = "only ALTER TABLE statements are supported by the optimistic front door" diff --git a/internal/cli/migrate_integration_test.go b/internal/cli/migrate_integration_test.go index 20b2db2..fe2c634 100644 --- a/internal/cli/migrate_integration_test.go +++ b/internal/cli/migrate_integration_test.go @@ -144,7 +144,7 @@ func TestMigrateGateRefusesWithoutDatabase(t *testing.T) { {"drop index", "DROP INDEX i", verdict.ReasonIndexStatement, "DROP INDEX CONCURRENTLY"}, {"reindex", "REINDEX TABLE t", verdict.ReasonIndexStatement, "REINDEX ... CONCURRENTLY"}, {"alter index", "ALTER INDEX i SET (fillfactor = 90)", verdict.ReasonUnsupportedStatement, ""}, - {"create table", "CREATE TABLE t (id int)", verdict.ReasonUnsupportedStatement, ""}, + {"create table", "CREATE TABLE t (id int)", verdict.ReasonUnsupportedStatement, "pg-sprite diff --desired schema.sql"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/schemadiff/desired.go b/pkg/schemadiff/desired.go new file mode 100644 index 0000000..646d80e --- /dev/null +++ b/pkg/schemadiff/desired.go @@ -0,0 +1,74 @@ +package schemadiff + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/block/pg-sprite/pkg/statement" +) + +// IntrospectDesired materializes a desired-state schema on a scratch schema +// and introspects it into the canonical model: execute-and-introspect, the +// decided way the engine understands DDL semantics. The scratch schema is +// created inside a single transaction that is always rolled back — nothing +// the desired file defines ever persists, no CREATEDB privilege is needed, +// and server-version and extension parity with the live table hold by +// construction because it runs on the same database. +func IntrospectDesired(ctx context.Context, db *pgxpool.Pool, desired statement.DesiredSchema) (Model, error) { + scratch, err := scratchSchemaName() + if err != nil { + return Model{}, err + } + tx, err := db.Begin(ctx) + if err != nil { + return Model{}, fmt.Errorf("begin scratch transaction: %w", err) + } + // The scratch transaction is never committed: rollback is the cleanup + // path for success and failure alike, so the redundant-closer exception + // does not apply — this rollback is load-bearing and its error is + // surfaced on the success path below. + defer func() { + _ = tx.Rollback(context.WithoutCancel(ctx)) + }() + + if _, err := tx.Exec(ctx, "CREATE SCHEMA "+pgx.Identifier{scratch}.Sanitize()); err != nil { + return Model{}, fmt.Errorf("create scratch schema: %w", err) + } + // Unqualified desired statements must land on the scratch schema, while + // extension types installed in public stay resolvable. search_path + // cannot use bind parameters; the identifier is sanitized. + setPath := "SET LOCAL search_path = " + pgx.Identifier{scratch}.Sanitize() + ", public" + if _, err := tx.Exec(ctx, setPath); err != nil { + return Model{}, fmt.Errorf("set scratch search_path: %w", err) + } + for _, st := range desired.Statements { + if _, err := tx.Exec(ctx, st.SQL); err != nil { + return Model{}, fmt.Errorf("execute desired statement on scratch schema: %w", err) + } + } + m, err := introspectInTx(ctx, tx, scratch, desired.Table) + if err != nil { + return Model{}, fmt.Errorf("introspect desired state: %w", err) + } + if err := tx.Rollback(ctx); err != nil { + return Model{}, fmt.Errorf("roll back scratch schema: %w", err) + } + return m, nil +} + +// scratchSchemaName returns a collision-resistant scratch schema name. The +// name only has to be unique among concurrent scratch transactions on the +// same database; the schema itself never outlives its transaction. The +// prefix avoids "pg_", which PostgreSQL reserves for system schemas. +func scratchSchemaName() (string, error) { + var b [8]byte + if _, err := rand.Read(b[:]); err != nil { + return "", fmt.Errorf("generate scratch schema name: %w", err) + } + return "pgsprite_scratch_" + hex.EncodeToString(b[:]), nil +} diff --git a/pkg/schemadiff/introspect.go b/pkg/schemadiff/introspect.go new file mode 100644 index 0000000..34d212c --- /dev/null +++ b/pkg/schemadiff/introspect.go @@ -0,0 +1,185 @@ +package schemadiff + +import ( + "context" + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/block/pg-sprite/pkg/statement" +) + +// ErrTableNotFound is returned when the target table does not exist in the +// requested schema. +var ErrTableNotFound = errors.New("table not found") + +// ErrNotTable is returned when the target exists but is not an ordinary or +// partitioned table (e.g. a view or foreign table). +var ErrNotTable = errors.New("not an ordinary or partitioned table") + +// Introspect reads the live table schema.table into the canonical model. It +// runs inside a read-only transaction whose search_path is set to the target +// schema (then public), so the server's decompilers print definitions +// unqualified — directly comparable with a desired-state model introspected +// the same way. +func Introspect(ctx context.Context, db *pgxpool.Pool, schema, table string) (Model, error) { + tx, err := db.Begin(ctx) + if err != nil { + return Model{}, fmt.Errorf("begin introspection: %w", err) + } + defer func() { + // Redundant safety closer: the transaction is read-only and always + // rolled back below; this only covers early error returns. + _ = tx.Rollback(context.WithoutCancel(ctx)) + }() + m, err := introspectInTx(ctx, tx, schema, table) + if err != nil { + return Model{}, err + } + if err := tx.Rollback(ctx); err != nil { + return Model{}, fmt.Errorf("end introspection: %w", err) + } + return m, nil +} + +// introspectInTx introspects schema.table inside an open transaction. It +// sets the transaction-local search_path so decompiled definitions print +// unqualified, resolves the relation by explicit qualification (never via +// search_path), and reads columns, constraints, and indexes. +func introspectInTx(ctx context.Context, tx pgx.Tx, schema, table string) (Model, error) { + // search_path cannot use bind parameters; identifiers are sanitized. + setPath := "SET LOCAL search_path = " + pgx.Identifier{schema}.Sanitize() + ", public" + if _, err := tx.Exec(ctx, setPath); err != nil { + return Model{}, fmt.Errorf("set introspection search_path: %w", err) + } + + var oid uint32 + var relkind string + err := tx.QueryRow(ctx, ` + SELECT c.oid, c.relkind::text + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = $1 AND c.relname = $2`, schema, table).Scan(&oid, &relkind) + if errors.Is(err, pgx.ErrNoRows) { + return Model{}, fmt.Errorf("%s.%s: %w", schema, table, ErrTableNotFound) + } + if err != nil { + return Model{}, fmt.Errorf("resolve table %s.%s: %w", schema, table, err) + } + if relkind != "r" && relkind != "p" { + return Model{}, fmt.Errorf("%s.%s has relkind %q: %w", schema, table, relkind, ErrNotTable) + } + + m := Model{Table: table} + if m.Columns, err = introspectColumns(ctx, tx, oid); err != nil { + return Model{}, fmt.Errorf("introspect columns of %s.%s: %w", schema, table, err) + } + if m.Constraints, err = introspectConstraints(ctx, tx, oid); err != nil { + return Model{}, fmt.Errorf("introspect constraints of %s.%s: %w", schema, table, err) + } + if m.Indexes, err = introspectIndexes(ctx, tx, oid); err != nil { + return Model{}, fmt.Errorf("introspect indexes of %s.%s: %w", schema, table, err) + } + return m, nil +} + +// introspectColumns reads the canonical column list: server-formatted types +// and server-decompiled default/generation expressions, in attribute order. +func introspectColumns(ctx context.Context, tx pgx.Tx, oid uint32) ([]Column, error) { + rows, err := tx.Query(ctx, ` + SELECT a.attname, + format_type(a.atttypid, a.atttypmod), + a.attnotnull, + COALESCE(pg_get_expr(d.adbin, d.adrelid), ''), + a.attidentity::text, + a.attgenerated::text + FROM pg_attribute a + LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum + WHERE a.attrelid = $1 AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum`, oid) + if err != nil { + return nil, fmt.Errorf("query columns: %w", err) + } + defer rows.Close() + var cols []Column + for rows.Next() { + var c Column + var identity, generated string + if err := rows.Scan(&c.Name, &c.Type, &c.NotNull, &c.Default, &identity, &generated); err != nil { + return nil, fmt.Errorf("scan column: %w", err) + } + c.Identity = Identity(identity) + c.Generated = generated == "s" + cols = append(cols, c) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read columns: %w", err) + } + return cols, nil +} + +// introspectConstraints reads the table's own constraints (primary key, +// unique, check, foreign key, exclusion) as server-decompiled definitions. +// NOT NULL is modeled on the column (pg_attribute.attnotnull), so the PG 18 +// pg_constraint rows for NOT NULL are deliberately excluded to keep the +// model identical across supported majors. +func introspectConstraints(ctx context.Context, tx pgx.Tx, oid uint32) ([]Constraint, error) { + rows, err := tx.Query(ctx, ` + SELECT conname, pg_get_constraintdef(oid) + FROM pg_constraint + WHERE conrelid = $1 AND contype IN ('p','u','c','f','x') AND conislocal + ORDER BY conname`, oid) + if err != nil { + return nil, fmt.Errorf("query constraints: %w", err) + } + defer rows.Close() + var cons []Constraint + for rows.Next() { + var c Constraint + if err := rows.Scan(&c.Name, &c.Def); err != nil { + return nil, fmt.Errorf("scan constraint: %w", err) + } + cons = append(cons, c) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read constraints: %w", err) + } + return cons, nil +} + +// introspectIndexes reads the non-constraint indexes as server-decompiled +// CREATE INDEX statements. Constraint-backed indexes (primary key, unique +// constraint, exclusion) are represented by their constraint instead. +// pg_get_indexdef always schema-qualifies the ON clause, so the +// qualification is stripped to keep the model schema-relative and +// comparable between the live and scratch sides. +func introspectIndexes(ctx context.Context, tx pgx.Tx, oid uint32) ([]Index, error) { + rows, err := tx.Query(ctx, ` + SELECT c.relname, pg_get_indexdef(i.indexrelid) + FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + WHERE i.indrelid = $1 + AND NOT EXISTS (SELECT 1 FROM pg_constraint con WHERE con.conindid = i.indexrelid) + ORDER BY c.relname`, oid) + if err != nil { + return nil, fmt.Errorf("query indexes: %w", err) + } + defer rows.Close() + var idxs []Index + for rows.Next() { + var ix Index + if err := rows.Scan(&ix.Name, &ix.Def); err != nil { + return nil, fmt.Errorf("scan index: %w", err) + } + if ix.Def, err = statement.Qualify(ix.Def, ""); err != nil { + return nil, fmt.Errorf("unqualify index %s: %w", ix.Name, err) + } + idxs = append(idxs, ix) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("read indexes: %w", err) + } + return idxs, nil +} diff --git a/pkg/schemadiff/schemadiff.go b/pkg/schemadiff/schemadiff.go new file mode 100644 index 0000000..1cdfe9e --- /dev/null +++ b/pkg/schemadiff/schemadiff.go @@ -0,0 +1,79 @@ +// Package schemadiff builds the canonical table model both front-ends share +// and diffs two models into an ordered statement list. The model always +// comes from a real PostgreSQL catalog — the live table is introspected +// directly, and a desired-state file is executed on a transaction-scoped +// scratch schema and introspected the same way, then the transaction is +// rolled back (execute-and-introspect; semantics are never derived from the +// AST). Canonical text (types, defaults, constraint and index definitions) +// is whatever the server's own decompilers print, so cosmetic differences +// (type aliases, default formatting, implicit names) never show up as diffs. +// +// This is a periphery package (see SAFETY.md): its output is a plan request, +// and the core executors re-verify their own preconditions. +package schemadiff + +// Identity is a column's identity kind, as pg_attribute.attidentity spells +// it. +type Identity string + +// The identity kinds. +const ( + // IdentityNone means the column is not an identity column. + IdentityNone Identity = "" + // IdentityAlways is GENERATED ALWAYS AS IDENTITY. + IdentityAlways Identity = "a" + // IdentityByDefault is GENERATED BY DEFAULT AS IDENTITY. + IdentityByDefault Identity = "d" +) + +// Column is one column of the canonical model. +type Column struct { + // Name is the column name. + Name string + // Type is the canonical type text (format_type), e.g. "character + // varying(50)" — never an alias like varchar(50). + Type string + // NotNull reports the NOT NULL attribute. + NotNull bool + // Default is the canonical default expression (pg_get_expr), empty when + // none. For a generated column it is the generation expression. + Default string + // Identity is the identity kind, IdentityNone for plain columns. + Identity Identity + // Generated reports GENERATED ALWAYS AS (...) STORED. + Generated bool +} + +// Constraint is one table constraint: its name plus the server-decompiled +// definition (pg_get_constraintdef), e.g. "PRIMARY KEY (id)". +type Constraint struct { + // Name is the constraint name. + Name string + // Def is the canonical definition text. + Def string +} + +// Index is one non-constraint index: its name plus the server-decompiled +// CREATE INDEX statement (pg_get_indexdef), unqualified under the +// introspection search_path. +type Index struct { + // Name is the index name. + Name string + // Def is the canonical CREATE INDEX statement. + Def string +} + +// Model is the canonical, comparison-ready description of one table. It +// carries no schema qualification: the live and desired sides are +// introspected under matching search_path settings so their definitions +// compare textually. +type Model struct { + // Table is the unqualified table name. + Table string + // Columns are the table's columns in attribute order. + Columns []Column + // Constraints are the table constraints, name-sorted. + Constraints []Constraint + // Indexes are the non-constraint indexes, name-sorted. + Indexes []Index +} diff --git a/pkg/statement/desired.go b/pkg/statement/desired.go new file mode 100644 index 0000000..3548dc6 --- /dev/null +++ b/pkg/statement/desired.go @@ -0,0 +1,155 @@ +package statement + +import ( + "errors" + "fmt" + + pganalyze "github.com/pganalyze/pg_query_go/v6" + pgquery "github.com/wasilibs/go-pgquery" +) + +// Typed refusals for desired-state schema files. Each names one rule of the +// declarative front door; the caller branches with errors.Is, never on text. +var ( + // ErrEmptyDesired is returned when the input contains no statements. + ErrEmptyDesired = errors.New("desired schema contains no statements") + // ErrNoCreateTable is returned when the input has no CREATE TABLE. + ErrNoCreateTable = errors.New("desired schema must contain a CREATE TABLE") + // ErrMultipleCreateTables is returned for more than one CREATE TABLE: + // the engine is single-table scoped. + ErrMultipleCreateTables = errors.New("desired schema must contain exactly one CREATE TABLE") + // ErrDisallowedStatement is returned for any statement kind other than + // CREATE TABLE / CREATE INDEX. The desired file is executed verbatim on + // a scratch schema, so only pure schema definition is admitted. + ErrDisallowedStatement = errors.New("statement kind not allowed in a desired schema") + // ErrQualifiedName is returned when a statement schema-qualifies its + // target. Desired files are schema-relative; the live schema comes from + // the caller, and qualification could escape the scratch schema. + ErrQualifiedName = errors.New("desired schema statements must use unqualified names") + // ErrConcurrentIndex is returned for CREATE INDEX CONCURRENTLY, which + // cannot run inside the scratch transaction. + ErrConcurrentIndex = errors.New("CONCURRENTLY cannot be used in a desired schema") + // ErrWrongIndexTarget is returned when an index targets a table other + // than the desired CREATE TABLE. + ErrWrongIndexTarget = errors.New("index must target the desired table") +) + +// DesiredSchema is a validated desired-state schema file: exactly one +// CREATE TABLE plus any number of CREATE INDEX statements on that table. +// Statement SQL is canonical (parsed and deparsed through the PostgreSQL +// grammar), in input order, one statement per entry. +type DesiredSchema struct { + // Table is the unqualified name of the single CREATE TABLE target. + Table string + // Statements are the admitted statements, the CREATE TABLE among them. + Statements []Statement +} + +// ParseDesired parses a desired-state schema file and admits only what the +// declarative front door can execute on a scratch schema: one unqualified +// CREATE TABLE and unqualified, non-concurrent CREATE INDEX statements on +// it. Anything else is refused with a typed error. +func ParseDesired(sql string) (DesiredSchema, error) { + tree, err := pgquery.Parse(sql) + if err != nil { + return DesiredSchema{}, fmt.Errorf("parse desired schema: %w", err) + } + if len(tree.GetStmts()) == 0 { + return DesiredSchema{}, ErrEmptyDesired + } + var ds DesiredSchema + for i, raw := range tree.GetStmts() { + st, err := admitDesiredStatement(raw.GetStmt(), ds.Table) + if err != nil { + return DesiredSchema{}, fmt.Errorf("statement %d: %w", i+1, err) + } + if st.Kind == KindCreateTable { + ds.Table = st.Table + } + if st.SQL, err = deparseOne(raw.GetStmt()); err != nil { + return DesiredSchema{}, fmt.Errorf("statement %d: %w", i+1, err) + } + ds.Statements = append(ds.Statements, st) + } + if ds.Table == "" { + return DesiredSchema{}, ErrNoCreateTable + } + for _, st := range ds.Statements { + if st.Kind == KindCreateIndex && st.Table != ds.Table { + return DesiredSchema{}, fmt.Errorf("%w: index on %q, desired table is %q", + ErrWrongIndexTarget, st.Table, ds.Table) + } + } + return ds, nil +} + +// admitDesiredStatement applies the per-statement admission rules and +// returns the statement's kind and target. seenTable is the CREATE TABLE +// target admitted so far, empty when none. +func admitDesiredStatement(node *pganalyze.Node, seenTable string) (Statement, error) { + switch { + case node.GetCreateStmt() != nil: + rel := node.GetCreateStmt().GetRelation() + if rel.GetSchemaname() != "" { + return Statement{}, fmt.Errorf("%w: %s.%s", ErrQualifiedName, rel.GetSchemaname(), rel.GetRelname()) + } + if seenTable != "" { + return Statement{}, ErrMultipleCreateTables + } + return Statement{Kind: KindCreateTable, Table: rel.GetRelname()}, nil + case node.GetIndexStmt() != nil: + idx := node.GetIndexStmt() + if idx.GetConcurrent() { + return Statement{}, ErrConcurrentIndex + } + rel := idx.GetRelation() + if rel.GetSchemaname() != "" { + return Statement{}, fmt.Errorf("%w: %s.%s", ErrQualifiedName, rel.GetSchemaname(), rel.GetRelname()) + } + return Statement{Kind: KindCreateIndex, Table: rel.GetRelname()}, nil + default: + return Statement{}, ErrDisallowedStatement + } +} + +// Qualify returns sql with its target relation qualified by schema; an +// empty schema strips an existing qualification instead. It supports exactly +// one CREATE TABLE or CREATE INDEX statement. This touches qualification +// only — no semantics are ever derived or transformed at the AST level +// (that is the scratch database's job). +func Qualify(sql, schema string) (string, error) { + tree, err := pgquery.Parse(sql) + if err != nil { + return "", fmt.Errorf("parse statement to qualify: %w", err) + } + if n := len(tree.GetStmts()); n != 1 { + return "", fmt.Errorf("%w: got %d", ErrNotOneStatement, n) + } + node := tree.GetStmts()[0].GetStmt() + var rel *pganalyze.RangeVar + switch { + case node.GetCreateStmt() != nil: + rel = node.GetCreateStmt().GetRelation() + case node.GetIndexStmt() != nil: + rel = node.GetIndexStmt().GetRelation() + default: + return "", ErrDisallowedStatement + } + if rel == nil { + return "", ErrDisallowedStatement + } + rel.Schemaname = schema + return deparseOne(node) +} + +// deparseOne renders a single parsed statement back to canonical SQL through +// the PostgreSQL deparser. +func deparseOne(node *pganalyze.Node) (string, error) { + out, err := pgquery.Deparse(&pganalyze.ParseResult{ + Stmts: []*pganalyze.RawStmt{{Stmt: node}}, + }) + if err != nil { + return "", fmt.Errorf("deparse statement: %w", err) + } + return out, nil +} diff --git a/pkg/statement/desired_test.go b/pkg/statement/desired_test.go new file mode 100644 index 0000000..c5bb7b3 --- /dev/null +++ b/pkg/statement/desired_test.go @@ -0,0 +1,87 @@ +package statement + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseDesiredAdmitsTableAndIndexes(t *testing.T) { + ds, err := ParseDesired(`create table events ( + id bigint primary key, + name varchar(50) not null +); +create index events_name_idx on events (name);`) + require.NoError(t, err) + + assert.Equal(t, "events", ds.Table) + require.Len(t, ds.Statements, 2) + assert.Equal(t, KindCreateTable, ds.Statements[0].Kind) + assert.Equal(t, "CREATE TABLE events (id bigint PRIMARY KEY, name varchar(50) NOT NULL)", ds.Statements[0].SQL) + assert.Equal(t, KindCreateIndex, ds.Statements[1].Kind) + assert.Equal(t, "CREATE INDEX events_name_idx ON events USING btree (name)", ds.Statements[1].SQL) +} + +func TestParseDesiredRefusals(t *testing.T) { + tests := []struct { + name string + sql string + wantErr error + }{ + {"empty input", "", ErrEmptyDesired}, + {"comment only", "-- nothing here", ErrEmptyDesired}, + {"no create table", "CREATE INDEX i ON t (c)", ErrNoCreateTable}, + {"two create tables", "CREATE TABLE a (id int); CREATE TABLE b (id int)", ErrMultipleCreateTables}, + {"dml", "CREATE TABLE t (id int); DELETE FROM t", ErrDisallowedStatement}, + {"alter table", "CREATE TABLE t (id int); ALTER TABLE t ADD COLUMN c int", ErrDisallowedStatement}, + {"drop", "CREATE TABLE t (id int); DROP TABLE other", ErrDisallowedStatement}, + {"qualified table", "CREATE TABLE prod.t (id int)", ErrQualifiedName}, + {"qualified index", "CREATE TABLE t (id int); CREATE INDEX i ON prod.t (id)", ErrQualifiedName}, + {"concurrent index", "CREATE TABLE t (id int); CREATE INDEX CONCURRENTLY i ON t (id)", ErrConcurrentIndex}, + {"index on another table", "CREATE TABLE t (id int); CREATE INDEX i ON other (id)", ErrWrongIndexTarget}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := ParseDesired(tt.sql) + require.ErrorIs(t, err, tt.wantErr) + }) + } +} + +func TestParseDesiredRefusesInvalidSQL(t *testing.T) { + _, err := ParseDesired("CREATE TABEL t (id int)") + require.Error(t, err) +} + +func TestQualify(t *testing.T) { + got, err := Qualify("CREATE INDEX i ON t USING btree (c)", "s1") + require.NoError(t, err) + assert.Equal(t, "CREATE INDEX i ON s1.t USING btree (c)", got) + + got, err = Qualify("CREATE TABLE t (id int)", "s1") + require.NoError(t, err) + assert.Equal(t, "CREATE TABLE s1.t (id int)", got) +} + +func TestQualifyEmptySchemaStripsQualification(t *testing.T) { + got, err := Qualify("CREATE INDEX i ON s1.t USING btree (c)", "") + require.NoError(t, err) + assert.Equal(t, "CREATE INDEX i ON t USING btree (c)", got) +} + +func TestQualifyRefusesOtherStatements(t *testing.T) { + _, err := Qualify("ALTER TABLE t ADD COLUMN c int", "s1") + require.ErrorIs(t, err, ErrDisallowedStatement) + + _, err = Qualify("CREATE TABLE a (id int); CREATE TABLE b (id int)", "s1") + require.ErrorIs(t, err, ErrNotOneStatement) +} + +func TestParseOneRecognizesCreateTable(t *testing.T) { + st, err := ParseOne("CREATE TABLE prod.events (id int)") + require.NoError(t, err) + assert.Equal(t, KindCreateTable, st.Kind) + assert.Equal(t, "prod", st.Schema) + assert.Equal(t, "events", st.Table) +} diff --git a/pkg/statement/statement.go b/pkg/statement/statement.go index 45b2697..7036649 100644 --- a/pkg/statement/statement.go +++ b/pkg/statement/statement.go @@ -24,6 +24,7 @@ const ( KindCreateIndex KindDropIndex KindReindex + KindCreateTable ) // String returns the human-readable name of the kind. @@ -37,6 +38,8 @@ func (k Kind) String() string { return "DROP INDEX" case KindReindex: return "REINDEX" + case KindCreateTable: + return "CREATE TABLE" default: return "other" } @@ -84,6 +87,11 @@ func ParseOne(sql string) (Statement, error) { st.Kind = KindAlterTable st.Schema = alter.GetRelation().GetSchemaname() st.Table = alter.GetRelation().GetRelname() + case node.GetCreateStmt() != nil: + rel := node.GetCreateStmt().GetRelation() + st.Kind = KindCreateTable + st.Schema = rel.GetSchemaname() + st.Table = rel.GetRelname() case node.GetIndexStmt() != nil: st.Kind = KindCreateIndex case node.GetDropStmt() != nil: diff --git a/pkg/statement/statement_test.go b/pkg/statement/statement_test.go index b61c264..f48ae2c 100644 --- a/pkg/statement/statement_test.go +++ b/pkg/statement/statement_test.go @@ -71,7 +71,7 @@ func TestParseOneKinds(t *testing.T) { { name: "create table", sql: "CREATE TABLE t (id int)", - want: Statement{Kind: KindOther}, + want: Statement{Kind: KindCreateTable, Table: "t"}, }, { name: "dml", From 4fcdefc60e03c48a6d72c1b357ee1c166e5b8423 Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Tue, 4 Aug 2026 22:30:05 +1000 Subject: [PATCH 11/24] Phase 2.2: declarative diff plus diff and fmt commands Diff live vs desired canonical models into dependency-ordered SQL changes with a destructive marker and typed refusals for unsupported identity/generation changes. `diff` emits an executable text plan or a JSON report; `fmt` canonicalizes offline via the parser/deparser. --- internal/cli/cli.go | 13 +- internal/cli/diff.go | 162 +++++++++++++ internal/cli/diff_integration_test.go | 175 ++++++++++++++ internal/cli/diff_test.go | 42 ++++ pkg/schemadiff/diff.go | 227 ++++++++++++++++++ pkg/schemadiff/diff_test.go | 194 +++++++++++++++ pkg/schemadiff/schemadiff_integration_test.go | 198 +++++++++++++++ 7 files changed, 1007 insertions(+), 4 deletions(-) create mode 100644 internal/cli/diff.go create mode 100644 internal/cli/diff_integration_test.go create mode 100644 internal/cli/diff_test.go create mode 100644 pkg/schemadiff/diff.go create mode 100644 pkg/schemadiff/diff_test.go create mode 100644 pkg/schemadiff/schemadiff_integration_test.go diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 157a343..321de51 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -92,23 +92,28 @@ type MigrateCmd struct { // Run implements the migrate subcommand. func (c *MigrateCmd) Run() error { return c.run(context.Background(), os.Stdout) } -// DiffCmd derives statements from a desired-state schema (declarative front-end). +// DiffCmd derives statements from a desired-state schema (declarative +// front-end): introspect the live table, materialize the desired state on a +// rolled-back scratch schema, and print the ordered plan without executing +// anything. type DiffCmd struct { DBFlags `embed:""` Desired string `help:"Path to the desired-state CREATE TABLE .sql file." name:"desired" type:"existingfile" required:""` + Schema string `help:"Schema containing the live table." default:"public"` + JSON bool `help:"Emit the plan as JSON."` } // Run implements the diff subcommand. -func (c *DiffCmd) Run() error { return notImplemented("diff") } +func (c *DiffCmd) Run() error { return c.run(context.Background(), os.Stdout) } // FmtCmd canonicalizes a schema file. It is offline — no database flags. type FmtCmd struct { - Path string `arg:"" optional:"" help:"Schema file to format." type:"existingfile"` + Path string `arg:"" optional:"" help:"Schema file to format; stdin when omitted." type:"existingfile"` } // Run implements the fmt subcommand. -func (c *FmtCmd) Run() error { return notImplemented("fmt") } +func (c *FmtCmd) Run() error { return c.runFmt(os.Stdin, os.Stdout) } // LintCmd checks DDL for unsafe or unsupported patterns. type LintCmd struct{} diff --git a/internal/cli/diff.go b/internal/cli/diff.go new file mode 100644 index 0000000..39f170a --- /dev/null +++ b/internal/cli/diff.go @@ -0,0 +1,162 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + + "github.com/block/pg-sprite/pkg/dbconn" + "github.com/block/pg-sprite/pkg/schemadiff" + "github.com/block/pg-sprite/pkg/statement" +) + +// diffReport is the diff command's JSON output contract. +type diffReport struct { + // Schema is the live schema the diff targeted. + Schema string `json:"schema"` + // Table is the desired (and live) table name. + Table string `json:"table"` + // TableExists reports whether the live table was found; when false the + // changes are the full desired schema. + TableExists bool `json:"table_exists"` + // Changes is the ordered statement plan; empty means the live table + // already matches the desired state. + Changes []schemadiff.Change `json:"changes"` +} + +// run is the diff flow: parse and admit the desired file, introspect the +// live table and the desired state (execute-and-introspect on a rolled-back +// scratch schema), and print the ordered plan. Nothing is ever executed +// against the live table. +func (c *DiffCmd) run(ctx context.Context, out io.Writer) error { + logger := c.diag() + raw, err := os.ReadFile(c.Desired) + if err != nil { + return fmt.Errorf("read desired schema: %w", err) + } + ds, err := statement.ParseDesired(string(raw)) + if err != nil { + return err + } + logger.Debug("desired schema parsed", "table", ds.Table, "statements", len(ds.Statements)) + + pool, err := dbconn.NewPool(ctx, c.Config()) + if err != nil { + return err + } + defer pool.Close() + + report := diffReport{Schema: c.Schema, Table: ds.Table, TableExists: true} + live, err := schemadiff.Introspect(ctx, pool, c.Schema, ds.Table) + switch { + case errors.Is(err, schemadiff.ErrTableNotFound): + // No live table: the plan is the desired schema itself, qualified + // onto the target schema. + report.TableExists = false + if report.Changes, err = qualifiedDesired(ds, c.Schema); err != nil { + return err + } + case err != nil: + return err + default: + desired, err := schemadiff.IntrospectDesired(ctx, pool, ds) + if err != nil { + return err + } + if report.Changes, err = schemadiff.Diff(c.Schema, live, desired); err != nil { + return err + } + } + logger.Debug("diff derived", + "schema", c.Schema, "table", ds.Table, "changes", len(report.Changes), "table_exists", report.TableExists) + + if c.JSON { + return writeJSON(out, report) + } + return writePlanText(out, report) +} + +// qualifiedDesired renders the desired statements as the plan for a table +// that does not exist yet, qualified onto the target schema. +func qualifiedDesired(ds statement.DesiredSchema, schema string) ([]schemadiff.Change, error) { + changes := make([]schemadiff.Change, 0, len(ds.Statements)) + for _, st := range ds.Statements { + qualified, err := statement.Qualify(st.SQL, schema) + if err != nil { + return nil, fmt.Errorf("qualify desired statement: %w", err) + } + changes = append(changes, schemadiff.Change{SQL: qualified}) + } + return changes, nil +} + +// writeJSON emits the report as JSON. +func writeJSON(out io.Writer, report diffReport) error { + if report.Changes == nil { + report.Changes = []schemadiff.Change{} + } + enc := json.NewEncoder(out) + enc.SetIndent("", " ") + if err := enc.Encode(report); err != nil { + return fmt.Errorf("write diff report: %w", err) + } + return nil +} + +// writePlanText emits the plan as an executable SQL script: one statement +// per line, destructive statements flagged with a leading comment line, and +// SQL comments for the no-change and missing-table cases so the output stays +// valid SQL. +func writePlanText(out io.Writer, report diffReport) error { + if len(report.Changes) == 0 { + if _, err := fmt.Fprintln(out, "-- no changes: live table matches the desired schema"); err != nil { + return fmt.Errorf("write plan: %w", err) + } + return nil + } + if !report.TableExists { + if _, err := fmt.Fprintf(out, "-- table %s.%s does not exist; the plan is the full desired schema\n", + report.Schema, report.Table); err != nil { + return fmt.Errorf("write plan: %w", err) + } + } + for _, ch := range report.Changes { + if ch.Destructive { + if _, err := fmt.Fprintln(out, "-- destructive"); err != nil { + return fmt.Errorf("write plan: %w", err) + } + } + if _, err := fmt.Fprintf(out, "%s;\n", ch.SQL); err != nil { + return fmt.Errorf("write plan: %w", err) + } + } + return nil +} + +// runFmt canonicalizes a desired-state schema file: every statement is +// parsed through the PostgreSQL grammar, admitted by the same rules as diff, +// and printed back in the deparser's canonical form. Offline — no database. +func (c *FmtCmd) runFmt(in io.Reader, out io.Writer) error { + var src []byte + var err error + if c.Path == "" { + if src, err = io.ReadAll(in); err != nil { + return fmt.Errorf("read schema from stdin: %w", err) + } + } else if src, err = os.ReadFile(c.Path); err != nil { + return fmt.Errorf("read schema file: %w", err) + } + ds, err := statement.ParseDesired(string(src)) + if err != nil { + return err + } + for _, st := range ds.Statements { + if _, err := fmt.Fprintf(out, "%s;\n", st.SQL); err != nil { + return fmt.Errorf("write formatted schema: %w", err) + } + } + return nil +} diff --git a/internal/cli/diff_integration_test.go b/internal/cli/diff_integration_test.go new file mode 100644 index 0000000..da8c88a --- /dev/null +++ b/internal/cli/diff_integration_test.go @@ -0,0 +1,175 @@ +package cli + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/block/pg-sprite/internal/testutil" + "github.com/block/pg-sprite/pkg/dbconn" +) + +// newDiffCmd builds a DiffCmd with the flag defaults kong would apply, +// pointing at a desired-state file written for the test. +func newDiffCmd(t *testing.T, url, schema, desiredSQL string) *DiffCmd { + t.Helper() + path := filepath.Join(t.TempDir(), "schema.sql") + require.NoError(t, os.WriteFile(path, []byte(desiredSQL), 0o600)) + return &DiffCmd{ + DBFlags: DBFlags{ + URL: url, + LockTimeout: 3 * time.Second, + StatementTimeout: 30 * time.Second, + }, + Desired: path, + Schema: schema, + } +} + +func TestDiffPrintsOrderedPlanJSON(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf( + "CREATE TABLE %s.events (id bigint PRIMARY KEY, name varchar(20), legacy int)", schema)) + require.NoError(t, err) + + cmd := newDiffCmd(t, url, schema, + "CREATE TABLE events (id bigint PRIMARY KEY, name varchar(50) NOT NULL);\n"+ + "CREATE INDEX events_name_idx ON events (name);") + cmd.JSON = true + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + + var report diffReport + require.NoError(t, json.Unmarshal([]byte(out.String()), &report)) + assert.Equal(t, schema, report.Schema) + assert.Equal(t, "events", report.Table) + assert.True(t, report.TableExists) + + var sqls []string + var destructive []bool + for _, ch := range report.Changes { + sqls = append(sqls, ch.SQL) + destructive = append(destructive, ch.Destructive) + } + assert.Equal(t, []string{ + fmt.Sprintf(`ALTER TABLE "%s"."events" DROP COLUMN "legacy"`, schema), + fmt.Sprintf(`ALTER TABLE "%s"."events" ALTER COLUMN "name" TYPE character varying(50)`, schema), + fmt.Sprintf(`ALTER TABLE "%s"."events" ALTER COLUMN "name" SET NOT NULL`, schema), + fmt.Sprintf("CREATE INDEX events_name_idx ON %s.events USING btree (name)", schema), + }, sqls) + assert.Equal(t, []bool{true, false, false, false}, destructive) +} + +// diff must never write: the live table is bit-identical before and after. +func TestDiffNeverWrites(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf( + "CREATE TABLE %s.events (id bigint PRIMARY KEY, legacy int)", schema)) + require.NoError(t, err) + _, err = pool.Exec(t.Context(), fmt.Sprintf( + "INSERT INTO %s.events SELECT g, g FROM generate_series(1, 10) g", schema)) + require.NoError(t, err) + + cmd := newDiffCmd(t, url, schema, "CREATE TABLE events (id bigint PRIMARY KEY, name text NOT NULL)") + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + + var cols int + require.NoError(t, pool.QueryRow(t.Context(), + `SELECT count(*) FROM information_schema.columns WHERE table_schema = $1 AND table_name = 'events'`, + schema).Scan(&cols)) + assert.Equal(t, 2, cols, "diff must not change the live table") + var rows int + require.NoError(t, pool.QueryRow(t.Context(), + fmt.Sprintf("SELECT count(*) FROM %s.events", schema)).Scan(&rows)) + assert.Equal(t, 10, rows, "diff must not touch data") +} + +func TestDiffNoChangesEmptyPlan(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf( + "CREATE TABLE %s.events (id bigint PRIMARY KEY, name text NOT NULL)", schema)) + require.NoError(t, err) + + cmd := newDiffCmd(t, url, schema, "CREATE TABLE events (id bigint PRIMARY KEY, name text NOT NULL)") + cmd.JSON = true + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + + var report diffReport + require.NoError(t, json.Unmarshal([]byte(out.String()), &report)) + assert.True(t, report.TableExists) + assert.Empty(t, report.Changes) +} + +func TestDiffMissingTableEmitsFullDesiredSchema(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + cmd := newDiffCmd(t, url, schema, + "CREATE TABLE events (id bigint PRIMARY KEY);\nCREATE INDEX events_id_idx ON events (id);") + cmd.JSON = true + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + + var report diffReport + require.NoError(t, json.Unmarshal([]byte(out.String()), &report)) + assert.False(t, report.TableExists) + var sqls []string + for _, ch := range report.Changes { + sqls = append(sqls, ch.SQL) + } + assert.Equal(t, []string{ + fmt.Sprintf("CREATE TABLE %s.events (id bigint PRIMARY KEY)", schema), + fmt.Sprintf("CREATE INDEX events_id_idx ON %s.events USING btree (id)", schema), + }, sqls) +} + +func TestDiffTextPlanIsExecutableSQL(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf( + "CREATE TABLE %s.events (id bigint PRIMARY KEY, legacy int)", schema)) + require.NoError(t, err) + + cmd := newDiffCmd(t, url, schema, "CREATE TABLE events (id bigint PRIMARY KEY, name text NOT NULL)") + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + + // The text plan is an executable script: running it converges the table. + _, err = pool.Exec(t.Context(), out.String()) + require.NoError(t, err, "text plan must be executable SQL: %s", out.String()) + + cmd2 := newDiffCmd(t, url, schema, "CREATE TABLE events (id bigint PRIMARY KEY, name text NOT NULL)") + cmd2.JSON = true + var out2 strings.Builder + require.NoError(t, cmd2.run(t.Context(), &out2)) + var report diffReport + require.NoError(t, json.Unmarshal([]byte(out2.String()), &report)) + assert.Empty(t, report.Changes, "executing the text plan must converge the table") +} diff --git a/internal/cli/diff_test.go b/internal/cli/diff_test.go new file mode 100644 index 0000000..bcbcf4f --- /dev/null +++ b/internal/cli/diff_test.go @@ -0,0 +1,42 @@ +package cli + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/block/pg-sprite/pkg/statement" +) + +func TestFmtCanonicalizesFromStdin(t *testing.T) { + cmd := &FmtCmd{} + in := strings.NewReader(`create table events ( + id bigint primary key, + name varchar(50) not null +); +create index events_name_idx on events (name);`) + var out strings.Builder + require.NoError(t, cmd.runFmt(in, &out)) + assert.Equal(t, + "CREATE TABLE events (id bigint PRIMARY KEY, name varchar(50) NOT NULL);\n"+ + "CREATE INDEX events_name_idx ON events USING btree (name);\n", + out.String()) +} + +func TestFmtRefusesDisallowedStatements(t *testing.T) { + cmd := &FmtCmd{} + var out strings.Builder + err := cmd.runFmt(strings.NewReader("CREATE TABLE t (id int); DELETE FROM t"), &out) + require.ErrorIs(t, err, statement.ErrDisallowedStatement) + assert.Empty(t, out.String(), "nothing is written when the input is refused") +} + +func TestFmtRefusesInvalidSQL(t *testing.T) { + cmd := &FmtCmd{} + var out strings.Builder + err := cmd.runFmt(strings.NewReader("CREATE TABEL t (id int)"), &out) + require.Error(t, err) + assert.Empty(t, out.String()) +} diff --git a/pkg/schemadiff/diff.go b/pkg/schemadiff/diff.go new file mode 100644 index 0000000..4d60366 --- /dev/null +++ b/pkg/schemadiff/diff.go @@ -0,0 +1,227 @@ +package schemadiff + +import ( + "errors" + "fmt" + + "github.com/jackc/pgx/v5" + + "github.com/block/pg-sprite/pkg/statement" +) + +// ErrUnsupportedChange is returned when converging live onto desired would +// need a change the engine does not derive (identity or generation changes +// on an existing column). The caller surfaces it; nothing is guessed. +var ErrUnsupportedChange = errors.New("unsupported schema change") + +// ErrDifferentTables is returned when the two models describe different +// tables — a caller bug, refused rather than diffed. +var ErrDifferentTables = errors.New("models describe different tables") + +// Change is one derived statement of the ordered plan. +type Change struct { + // SQL is the literal statement, without a trailing semicolon. + SQL string `json:"sql"` + // Destructive marks statements that discard data or constraints + // (column and constraint drops). Destructive changes are gated by the + // caller, never executed silently. + Destructive bool `json:"destructive,omitempty"` +} + +// Diff derives the ordered statement list that converges live onto desired. +// Order is dependency-correct: drops first (indexes, then constraints, then +// columns), then column adds and alters, then constraint adds, then index +// creates — so an added column exists before an index or constraint that +// references it. Within each bucket the order is deterministic: attribute +// order for columns, name order for constraints and indexes. schema +// qualifies the emitted statements' table references. +func Diff(schema string, live, desired Model) ([]Change, error) { + if live.Table != desired.Table { + return nil, fmt.Errorf("%w: %q vs %q", ErrDifferentTables, live.Table, desired.Table) + } + table := pgx.Identifier{schema, live.Table}.Sanitize() + + liveCols := columnsByName(live.Columns) + desiredCols := columnsByName(desired.Columns) + liveCons := constraintsByName(live.Constraints) + desiredCons := constraintsByName(desired.Constraints) + liveIdx := indexesByName(live.Indexes) + desiredIdx := indexesByName(desired.Indexes) + + var changes []Change + + // Indexes to drop: gone from desired, or changed (dropped here, + // recreated in the create bucket below). + for _, ix := range live.Indexes { + want, ok := desiredIdx[ix.Name] + if !ok || want.Def != ix.Def { + changes = append(changes, Change{ + SQL: "DROP INDEX " + pgx.Identifier{schema, ix.Name}.Sanitize(), + }) + } + } + + // Constraints to drop: gone from desired, or changed (re-added below). + for _, con := range live.Constraints { + want, ok := desiredCons[con.Name] + if !ok || want.Def != con.Def { + changes = append(changes, Change{ + SQL: "ALTER TABLE " + table + " DROP CONSTRAINT " + pgx.Identifier{con.Name}.Sanitize(), + Destructive: true, + }) + } + } + + // Columns to drop. A rename is indistinguishable from drop+add at the + // catalog level, so it surfaces as exactly that — and the drop is + // flagged destructive for the caller to gate. + for _, col := range live.Columns { + if _, ok := desiredCols[col.Name]; !ok { + changes = append(changes, Change{ + SQL: "ALTER TABLE " + table + " DROP COLUMN " + pgx.Identifier{col.Name}.Sanitize(), + Destructive: true, + }) + } + } + + // Columns to add. + for _, col := range desired.Columns { + if _, ok := liveCols[col.Name]; !ok { + changes = append(changes, Change{ + SQL: "ALTER TABLE " + table + " ADD COLUMN " + columnDef(col), + }) + } + } + + // Columns present on both sides: type, default, and nullability deltas. + for _, col := range desired.Columns { + liveCol, ok := liveCols[col.Name] + if !ok { + continue + } + alter, err := alterColumnChanges(table, liveCol, col) + if err != nil { + return nil, err + } + changes = append(changes, alter...) + } + + // Constraints to add: new, or re-added after a definition change. + for _, con := range desired.Constraints { + had, ok := liveCons[con.Name] + if !ok || had.Def != con.Def { + changes = append(changes, Change{ + SQL: "ALTER TABLE " + table + " ADD CONSTRAINT " + pgx.Identifier{con.Name}.Sanitize() + " " + con.Def, + }) + } + } + + // Indexes to create: new, or recreated after a definition change. The + // desired definition is server-decompiled and unqualified; only the + // schema qualification is injected. + for _, ix := range desired.Indexes { + had, ok := liveIdx[ix.Name] + if !ok || had.Def != ix.Def { + qualified, err := statement.Qualify(ix.Def, schema) + if err != nil { + return nil, fmt.Errorf("qualify index %s: %w", ix.Name, err) + } + changes = append(changes, Change{SQL: qualified}) + } + } + + return changes, nil +} + +// alterColumnChanges derives the in-place column alterations between two +// versions of the same column. Identity and generation cannot be altered in +// place, so a delta there is refused as unsupported. +func alterColumnChanges(table string, live, desired Column) ([]Change, error) { + if live.Identity != desired.Identity { + return nil, fmt.Errorf("%w: column %q identity change", ErrUnsupportedChange, live.Name) + } + if live.Generated != desired.Generated { + return nil, fmt.Errorf("%w: column %q generated change", ErrUnsupportedChange, live.Name) + } + if desired.Generated && (live.Type != desired.Type || live.Default != desired.Default) { + return nil, fmt.Errorf("%w: column %q generation expression or type change", ErrUnsupportedChange, live.Name) + } + col := pgx.Identifier{live.Name}.Sanitize() + var changes []Change + if live.Type != desired.Type { + changes = append(changes, Change{ + SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " TYPE " + desired.Type, + }) + } + if !desired.Generated && desired.Identity == IdentityNone && live.Default != desired.Default { + if desired.Default == "" { + changes = append(changes, Change{ + SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " DROP DEFAULT", + }) + } else { + changes = append(changes, Change{ + SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " SET DEFAULT " + desired.Default, + }) + } + } + if live.NotNull != desired.NotNull { + if desired.NotNull { + changes = append(changes, Change{ + SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " SET NOT NULL", + }) + } else { + changes = append(changes, Change{ + SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " DROP NOT NULL", + }) + } + } + return changes, nil +} + +// columnDef renders a canonical column definition for ADD COLUMN. All parts +// are server-canonical: the type from format_type, expressions from +// pg_get_expr. +func columnDef(c Column) string { + def := pgx.Identifier{c.Name}.Sanitize() + " " + c.Type + switch { + case c.Generated: + def += " GENERATED ALWAYS AS (" + c.Default + ") STORED" + case c.Identity == IdentityAlways: + def += " GENERATED ALWAYS AS IDENTITY" + case c.Identity == IdentityByDefault: + def += " GENERATED BY DEFAULT AS IDENTITY" + case c.Default != "": + def += " DEFAULT " + c.Default + } + if c.NotNull { + def += " NOT NULL" + } + return def +} + +// columnsByName indexes columns for lookup during the diff. +func columnsByName(cols []Column) map[string]Column { + m := make(map[string]Column, len(cols)) + for _, c := range cols { + m[c.Name] = c + } + return m +} + +// constraintsByName indexes constraints for lookup during the diff. +func constraintsByName(cons []Constraint) map[string]Constraint { + m := make(map[string]Constraint, len(cons)) + for _, c := range cons { + m[c.Name] = c + } + return m +} + +// indexesByName indexes indexes for lookup during the diff. +func indexesByName(idxs []Index) map[string]Index { + m := make(map[string]Index, len(idxs)) + for _, ix := range idxs { + m[ix.Name] = ix + } + return m +} diff --git a/pkg/schemadiff/diff_test.go b/pkg/schemadiff/diff_test.go new file mode 100644 index 0000000..06d137f --- /dev/null +++ b/pkg/schemadiff/diff_test.go @@ -0,0 +1,194 @@ +package schemadiff + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// base returns a small canonical model to mutate per test. +func base() Model { + return Model{ + Table: "events", + Columns: []Column{ + {Name: "id", Type: "bigint", NotNull: true}, + {Name: "name", Type: "character varying(50)", NotNull: true}, + }, + Constraints: []Constraint{ + {Name: "events_pkey", Def: "PRIMARY KEY (id)"}, + }, + Indexes: []Index{ + {Name: "events_name_idx", Def: "CREATE INDEX events_name_idx ON events USING btree (name)"}, + }, + } +} + +func sqls(changes []Change) []string { + out := make([]string, len(changes)) + for i, c := range changes { + out[i] = c.SQL + } + return out +} + +func TestDiffNoChanges(t *testing.T) { + changes, err := Diff("public", base(), base()) + require.NoError(t, err) + assert.Empty(t, changes) +} + +func TestDiffRefusesDifferentTables(t *testing.T) { + other := base() + other.Table = "users" + _, err := Diff("public", base(), other) + require.ErrorIs(t, err, ErrDifferentTables) +} + +func TestDiffAddColumn(t *testing.T) { + desired := base() + desired.Columns = append(desired.Columns, Column{ + Name: "created_at", Type: "timestamp with time zone", NotNull: true, Default: "now()", + }) + changes, err := Diff("public", base(), desired) + require.NoError(t, err) + require.Equal(t, []string{ + `ALTER TABLE "public"."events" ADD COLUMN "created_at" timestamp with time zone DEFAULT now() NOT NULL`, + }, sqls(changes)) + assert.False(t, changes[0].Destructive) +} + +func TestDiffDropColumnIsDestructive(t *testing.T) { + desired := base() + desired.Columns = desired.Columns[:1] // drop "name" + changes, err := Diff("public", base(), desired) + require.NoError(t, err) + require.Equal(t, []string{ + `ALTER TABLE "public"."events" DROP COLUMN "name"`, + }, sqls(changes)) + assert.True(t, changes[0].Destructive) +} + +func TestDiffColumnAlterations(t *testing.T) { + desired := base() + desired.Columns[1] = Column{Name: "name", Type: "text", NotNull: false, Default: "'unnamed'::text"} + changes, err := Diff("public", base(), desired) + require.NoError(t, err) + require.Equal(t, []string{ + `ALTER TABLE "public"."events" ALTER COLUMN "name" TYPE text`, + `ALTER TABLE "public"."events" ALTER COLUMN "name" SET DEFAULT 'unnamed'::text`, + `ALTER TABLE "public"."events" ALTER COLUMN "name" DROP NOT NULL`, + }, sqls(changes)) +} + +func TestDiffDropDefaultAndSetNotNull(t *testing.T) { + live := base() + live.Columns[1].Default = "'x'::character varying" + live.Columns[1].NotNull = false + changes, err := Diff("public", live, base()) + require.NoError(t, err) + require.Equal(t, []string{ + `ALTER TABLE "public"."events" ALTER COLUMN "name" DROP DEFAULT`, + `ALTER TABLE "public"."events" ALTER COLUMN "name" SET NOT NULL`, + }, sqls(changes)) +} + +func TestDiffConstraintChangeDropsAndReadds(t *testing.T) { + desired := base() + desired.Constraints = []Constraint{ + {Name: "events_pkey", Def: "PRIMARY KEY (id, name)"}, + } + changes, err := Diff("public", base(), desired) + require.NoError(t, err) + require.Equal(t, []string{ + `ALTER TABLE "public"."events" DROP CONSTRAINT "events_pkey"`, + `ALTER TABLE "public"."events" ADD CONSTRAINT "events_pkey" PRIMARY KEY (id, name)`, + }, sqls(changes)) + assert.True(t, changes[0].Destructive) + assert.False(t, changes[1].Destructive) +} + +func TestDiffIndexChangeDropsAndRecreatesQualified(t *testing.T) { + desired := base() + desired.Indexes = []Index{ + {Name: "events_name_idx", Def: "CREATE UNIQUE INDEX events_name_idx ON events USING btree (name)"}, + } + changes, err := Diff("public", base(), desired) + require.NoError(t, err) + require.Equal(t, []string{ + `DROP INDEX "public"."events_name_idx"`, + `CREATE UNIQUE INDEX events_name_idx ON public.events USING btree (name)`, + }, sqls(changes)) +} + +func TestDiffOrderingDropsBeforeAddsBeforeIndexes(t *testing.T) { + live := base() + live.Columns = append(live.Columns, Column{Name: "legacy", Type: "integer"}) + + desired := base() + desired.Columns = append(desired.Columns, Column{Name: "email", Type: "text", NotNull: true}) + desired.Indexes = append(desired.Indexes, Index{ + Name: "events_email_idx", Def: "CREATE INDEX events_email_idx ON events USING btree (email)", + }) + + changes, err := Diff("public", live, desired) + require.NoError(t, err) + require.Equal(t, []string{ + `ALTER TABLE "public"."events" DROP COLUMN "legacy"`, + `ALTER TABLE "public"."events" ADD COLUMN "email" text NOT NULL`, + `CREATE INDEX events_email_idx ON public.events USING btree (email)`, + }, sqls(changes)) +} + +func TestDiffRefusesIdentityAndGeneratedChanges(t *testing.T) { + tests := []struct { + name string + mutate func(*Column) + }{ + {"identity change", func(c *Column) { c.Identity = IdentityAlways }}, + {"generated change", func(c *Column) { c.Generated = true; c.Default = "(id + 1)" }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + desired := base() + tt.mutate(&desired.Columns[0]) + _, err := Diff("public", base(), desired) + require.ErrorIs(t, err, ErrUnsupportedChange) + }) + } +} + +func TestDiffRefusesGenerationExpressionChange(t *testing.T) { + live := base() + live.Columns[0] = Column{Name: "id", Type: "bigint", Generated: true, Default: "(1)"} + desired := base() + desired.Columns[0] = Column{Name: "id", Type: "bigint", Generated: true, Default: "(2)"} + _, err := Diff("public", live, desired) + require.ErrorIs(t, err, ErrUnsupportedChange) +} + +func TestDiffIdentityColumnAddRendersIdentity(t *testing.T) { + desired := base() + desired.Columns = append(desired.Columns, + Column{Name: "seq_a", Type: "bigint", NotNull: true, Identity: IdentityAlways}, + Column{Name: "seq_d", Type: "bigint", NotNull: true, Identity: IdentityByDefault}, + ) + changes, err := Diff("public", base(), desired) + require.NoError(t, err) + require.Equal(t, []string{ + `ALTER TABLE "public"."events" ADD COLUMN "seq_a" bigint GENERATED ALWAYS AS IDENTITY NOT NULL`, + `ALTER TABLE "public"."events" ADD COLUMN "seq_d" bigint GENERATED BY DEFAULT AS IDENTITY NOT NULL`, + }, sqls(changes)) +} + +func TestDiffGeneratedColumnAddRendersGenerationExpression(t *testing.T) { + desired := base() + desired.Columns = append(desired.Columns, Column{ + Name: "name_upper", Type: "text", Generated: true, Default: "upper((name)::text)", + }) + changes, err := Diff("public", base(), desired) + require.NoError(t, err) + require.Equal(t, []string{ + `ALTER TABLE "public"."events" ADD COLUMN "name_upper" text GENERATED ALWAYS AS (upper((name)::text)) STORED`, + }, sqls(changes)) +} diff --git a/pkg/schemadiff/schemadiff_integration_test.go b/pkg/schemadiff/schemadiff_integration_test.go new file mode 100644 index 0000000..cf14829 --- /dev/null +++ b/pkg/schemadiff/schemadiff_integration_test.go @@ -0,0 +1,198 @@ +package schemadiff_test + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/block/pg-sprite/internal/testutil" + "github.com/block/pg-sprite/pkg/dbconn" + "github.com/block/pg-sprite/pkg/schemadiff" + "github.com/block/pg-sprite/pkg/statement" +) + +const desiredSQL = ` +CREATE TABLE events ( + id bigint PRIMARY KEY, + name varchar(50) NOT NULL, + payload jsonb DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT name_not_empty CHECK (length(name) > 0) +); +CREATE INDEX events_created_at_idx ON events (created_at); +CREATE UNIQUE INDEX events_name_key ON events (name); +` + +// The two-oracle test of execute-and-introspect: creating the table live and +// materializing the same file on the scratch schema must introspect to the +// identical canonical model. +func TestIntrospectDesiredMatchesLiveIntrospection(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + ds, err := statement.ParseDesired(desiredSQL) + require.NoError(t, err) + for _, st := range ds.Statements { + qualified, err := statement.Qualify(st.SQL, schema) + require.NoError(t, err) + _, err = pool.Exec(t.Context(), qualified) + require.NoError(t, err) + } + + live, err := schemadiff.Introspect(t.Context(), pool, schema, "events") + require.NoError(t, err) + desired, err := schemadiff.IntrospectDesired(t.Context(), pool, ds) + require.NoError(t, err) + + assert.Equal(t, live, desired, + "live introspection and scratch execute-and-introspect must agree on the canonical model") +} + +// Cosmetically different spellings of the same schema must introspect to the +// same canonical model: the server's decompilers are the canonicalizer. +func TestIntrospectCanonicalizesTypeAliases(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + _, err = pool.Exec(t.Context(), fmt.Sprintf( + "CREATE TABLE %s.aliased (a int4, b varchar(50), c timestamptz, d bool DEFAULT TRUE)", schema)) + require.NoError(t, err) + + m, err := schemadiff.Introspect(t.Context(), pool, schema, "aliased") + require.NoError(t, err) + require.Len(t, m.Columns, 4) + assert.Equal(t, "integer", m.Columns[0].Type) + assert.Equal(t, "character varying(50)", m.Columns[1].Type) + assert.Equal(t, "timestamp with time zone", m.Columns[2].Type) + assert.Equal(t, "boolean", m.Columns[3].Type) + assert.Equal(t, "true", m.Columns[3].Default) +} + +func TestIntrospectDesiredLeavesNoFootprint(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + + ds, err := statement.ParseDesired(desiredSQL) + require.NoError(t, err) + _, err = schemadiff.IntrospectDesired(t.Context(), pool, ds) + require.NoError(t, err) + + var leftover int + err = pool.QueryRow(t.Context(), + "SELECT count(*) FROM pg_namespace WHERE nspname LIKE 'pgsprite\\_scratch\\_%'").Scan(&leftover) + require.NoError(t, err) + assert.Zero(t, leftover, "the scratch schema must never survive its transaction") +} + +func TestIntrospectTableNotFound(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + _, err = schemadiff.Introspect(t.Context(), pool, schema, "missing") + require.ErrorIs(t, err, schemadiff.ErrTableNotFound) +} + +func TestIntrospectRefusesViews(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE VIEW %s.v AS SELECT 1 AS one", schema)) + require.NoError(t, err) + + _, err = schemadiff.Introspect(t.Context(), pool, schema, "v") + require.ErrorIs(t, err, schemadiff.ErrNotTable) +} + +// A desired statement that is valid grammar but invalid semantics (a type +// that does not exist) must fail at scratch execution — semantic truth comes +// from the server, not the parser. +func TestIntrospectDesiredSurfacesSemanticErrors(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + + ds, err := statement.ParseDesired("CREATE TABLE t (id no_such_type)") + require.NoError(t, err, "the grammar accepts unknown type names; only the server can refuse them") + _, err = schemadiff.IntrospectDesired(t.Context(), pool, ds) + require.Error(t, err) +} + +// The convergence oracle: diff live against desired, execute the plan, and +// the re-diff must be empty. This closes the loop between the diff engine +// and the real server semantics. +func TestDiffConverges(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + // The live table starts from an older shape: a column to drop, a column + // to retype, a default to add, an index to replace. + for _, ddl := range []string{ + fmt.Sprintf(`CREATE TABLE %s.events ( + id bigint PRIMARY KEY, + name varchar(20) NOT NULL, + legacy int, + created_at timestamptz NOT NULL + )`, schema), + fmt.Sprintf("CREATE INDEX events_created_at_idx ON %s.events (created_at DESC)", schema), + } { + _, err := pool.Exec(t.Context(), ddl) + require.NoError(t, err) + } + + ds, err := statement.ParseDesired(desiredSQL) + require.NoError(t, err) + + live, err := schemadiff.Introspect(t.Context(), pool, schema, "events") + require.NoError(t, err) + desired, err := schemadiff.IntrospectDesired(t.Context(), pool, ds) + require.NoError(t, err) + changes, err := schemadiff.Diff(schema, live, desired) + require.NoError(t, err) + require.NotEmpty(t, changes) + + for _, ch := range changes { + _, err := pool.Exec(t.Context(), ch.SQL) + require.NoError(t, err, "derived statement must execute: %s", ch.SQL) + } + + after, err := schemadiff.Introspect(t.Context(), pool, schema, "events") + require.NoError(t, err) + rediff, err := schemadiff.Diff(schema, after, desired) + require.NoError(t, err) + assert.Empty(t, rediff, "after executing the plan the live table must match the desired state") +} + +// Identity and generated columns round-trip through both introspection paths. +func TestIntrospectIdentityAndGeneratedColumns(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + _, err = pool.Exec(t.Context(), fmt.Sprintf(`CREATE TABLE %s.gen ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + n int NOT NULL, + doubled int GENERATED ALWAYS AS (n * 2) STORED + )`, schema)) + require.NoError(t, err) + + m, err := schemadiff.Introspect(t.Context(), pool, schema, "gen") + require.NoError(t, err) + require.Len(t, m.Columns, 3) + assert.Equal(t, schemadiff.IdentityAlways, m.Columns[0].Identity) + assert.True(t, m.Columns[2].Generated) + assert.Equal(t, "(n * 2)", m.Columns[2].Default) +} From 68814979d84249772b48fbff1752e18cc770e74d Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Tue, 4 Aug 2026 22:32:36 +1000 Subject: [PATCH 12/24] ci: pin golangci-lint v2 and name test matrix jobs The unpinned lint action installed a v1 binary built with an older Go than the module targets, so it could not load the v2 config. Matrix jobs now render as "test (PostgreSQL NN)" in the checks UI. --- .github/workflows/ci.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fc3def..16f01d9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,7 +60,11 @@ jobs: - uses: actions/setup-go@v5 with: go-version-file: go.mod - - uses: golangci/golangci-lint-action@v6 + # Pin the same golangci-lint major used locally (v2 config format); + # the action's default binary lags and cannot load a v2 config. + - uses: golangci/golangci-lint-action@v9 + with: + version: v2.12.2 build: needs: changes @@ -79,6 +83,7 @@ jobs: # images — real Aurora engine-version validation is a separate gate that # cannot run in public CI. test: + name: test (PostgreSQL ${{ matrix.pg }}) needs: changes if: needs.changes.outputs.code == 'true' runs-on: ubuntu-latest From f61d263fdbf2569435b2044e0c88b140ca4ae0dd Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Wed, 5 Aug 2026 16:06:52 +1000 Subject: [PATCH 13/24] ci: rename workflow to CI and the ci-ok sentinel to all-green Checks now render as CI / lint, CI / test (PostgreSQL NN), and CI / all-green instead of the ci / ci-ok stutter. "all-green" is the context to require once branch protection is wired. --- .github/workflows/ci.yml | 9 +++++---- docs/testing.md | 5 +++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16f01d9..16fc3bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: ci +name: CI on: push: @@ -100,9 +100,10 @@ jobs: go-version-file: go.mod - run: make test - # Single required status for branch protection. Succeeds when nothing - # failed — including docs-only PRs where the heavy jobs were skipped. - ci-ok: + # Single required status for branch protection ("all-green" is the + # context to require). Succeeds when nothing failed — including + # docs-only PRs where the heavy jobs were skipped. + all-green: if: always() needs: [changes, lint, build, test] runs-on: ubuntu-latest diff --git a/docs/testing.md b/docs/testing.md index 184ed42..c57861d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -26,8 +26,9 @@ Concretely: - **Real database, race-enabled.** Unit tests run with `-race`; core (`pkg/`) logic is never validated against mocks — integration tests run against real PostgreSQL, across every supported major in CI. -- **The matrix is a gate, not advisory.** The `ci-ok` sentinel requires the - full version matrix; docs-only changes are the only path that skips it. +- **The matrix is a gate, not advisory.** The `all-green` sentinel job + requires the full version matrix; docs-only changes are the only path + that skips it. - **Coverage never regresses.** Deleting or skipping a test to get green is forbidden (same rule as the hooks: no `--no-verify`, no `nolint`). A numeric coverage ratchet on `pkg/` packages is wired into CI once Phase 1 From 14c9dee89ab115023e972a05a2537fc3619d3d4a Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Wed, 5 Aug 2026 16:11:46 +1000 Subject: [PATCH 14/24] ci: pin golangci-lint-action to commit SHA Semgrep and zizmor code-scanning gates require third-party actions pinned to a full commit SHA, matching the existing paths-filter pin. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 16fc3bd..d21e296 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,7 @@ jobs: go-version-file: go.mod # Pin the same golangci-lint major used locally (v2 config format); # the action's default binary lags and cannot load a v2 config. - - uses: golangci/golangci-lint-action@v9 + - uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: version: v2.12.2 From e06ed27d4bda71e02de3a2d83b57b93973e84049 Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Wed, 5 Aug 2026 16:48:12 +1000 Subject: [PATCH 15/24] Phase 2.3a: typed operation descriptors and advisory rewrites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the parse boundary with ParseOps — one typed shape descriptor per operation (default constancy, generated/identity, NOT VALID, USING INDEX, CONCURRENTLY, renames) — plus two syntactic advisory rewriters: Concurrently and AddNotValid. These are the classifier's inputs; no semantics are derived from the AST. --- pkg/statement/ops.go | 406 ++++++++++++++++++++++++++++++++++++++ pkg/statement/ops_test.go | 294 +++++++++++++++++++++++++++ pkg/statement/rewrite.go | 109 ++++++++++ 3 files changed, 809 insertions(+) create mode 100644 pkg/statement/ops.go create mode 100644 pkg/statement/ops_test.go create mode 100644 pkg/statement/rewrite.go diff --git a/pkg/statement/ops.go b/pkg/statement/ops.go new file mode 100644 index 0000000..682451b --- /dev/null +++ b/pkg/statement/ops.go @@ -0,0 +1,406 @@ +package statement + +import ( + "fmt" + "strings" + + pganalyze "github.com/pganalyze/pg_query_go/v6" + pgquery "github.com/wasilibs/go-pgquery" +) + +// OpKind names one operation shape the classifier distinguishes. A single +// ALTER TABLE statement yields one Op per subcommand; index, rename, and +// schema statements yield exactly one. +type OpKind int + +// The operation shapes ParseOps reports. OpUnrecognized is everything the engine +// does not recognize; the classifier refuses it. +const ( + OpUnrecognized OpKind = iota + OpCreateTable + OpAddColumn + OpDropColumn + OpAlterColumnType + OpSetDefault + OpDropDefault + OpSetNotNull + OpDropNotNull + OpSetColumnOptions + OpRenameColumn + OpRenameTable + OpRenameIndex + OpSetSchema + OpSetTablespace + OpSetRelOptions + OpAddConstraint + OpValidateConstraint + OpDropConstraint + OpAttachPartition + OpDetachPartition + OpCreateIndex + OpDropIndex + OpReindex +) + +// DefaultKind classifies the DEFAULT expression shape of an added column. +// Only a provable constant qualifies for PostgreSQL's fast default; any +// other expression is treated as volatile, conservatively. +type DefaultKind int + +// The default shapes an added column can carry. +const ( + // DefaultNone: no DEFAULT clause. + DefaultNone DefaultKind = iota + // DefaultConstant: a literal (possibly type-cast) — fast-default safe. + DefaultConstant + // DefaultExpression: anything else — function calls, identity, serial. + // The engine does not evaluate volatility offline; it assumes the worst. + DefaultExpression +) + +// ConstraintKind names the constraint families the classifier routes +// differently. +type ConstraintKind int + +// The constraint families ParseOps distinguishes. ConstraintUnrecognized +// (e.g. EXCLUDE) has no known safe pattern and is refused. +const ( + ConstraintUnrecognized ConstraintKind = iota + ConstraintPrimaryKey + ConstraintUnique + ConstraintCheck + ConstraintForeignKey + ConstraintNotNull +) + +// Op is one parsed operation: the shape facts the classifier needs, nothing +// executable. Fields beyond Kind are populated only where meaningful for +// that kind; see each field's comment. +type Op struct { + // Kind is the operation shape. + Kind OpKind + // Column is the target column for column operations. + Column string + // Name is the constraint or index name where the operation has one, + // or the new name for renames. + Name string + // Columns are the plain key columns of an ADD PRIMARY KEY / UNIQUE; + // empty when the keys are expressions. + Columns []string + // Constraint is the constraint family for OpAddConstraint. + Constraint ConstraintKind + // NotValid is true for ADD CONSTRAINT ... NOT VALID. + NotValid bool + // UsingIndex is true for ADD CONSTRAINT ... USING INDEX. + UsingIndex bool + // Concurrent is true when the statement carries CONCURRENTLY. + Concurrent bool + // Unique is true for CREATE UNIQUE INDEX. + Unique bool + // GeneratedStored is true for ADD COLUMN ... GENERATED ... STORED. + GeneratedStored bool + // Default is the DEFAULT shape for OpAddColumn. + Default DefaultKind + // NewType is the target type for OpAlterColumnType and the column type + // for OpAddColumn, as the bare grammar type name (e.g. "varchar", + // "numeric") without the pg_catalog qualification. + NewType string + // NewTypeMods are the target type's modifiers (e.g. 50 in varchar(50), + // 12 and 2 in numeric(12,2)); empty when unconstrained. + NewTypeMods []int32 + // HasUsing is true for ALTER COLUMN TYPE ... USING , which always + // means a conversion, never a binary-coercible relabel. + HasUsing bool +} + +// Describe returns a short operator-facing label for the operation, e.g. +// "ADD COLUMN age" — for plan rendering, never for branching. +func (o Op) Describe() string { + switch o.Kind { + case OpCreateTable: + return "CREATE TABLE" + case OpAddColumn: + return "ADD COLUMN " + o.Column + case OpDropColumn: + return "DROP COLUMN " + o.Column + case OpAlterColumnType: + return "ALTER COLUMN " + o.Column + " TYPE " + o.NewType + case OpSetDefault: + return "ALTER COLUMN " + o.Column + " SET DEFAULT" + case OpDropDefault: + return "ALTER COLUMN " + o.Column + " DROP DEFAULT" + case OpSetNotNull: + return "ALTER COLUMN " + o.Column + " SET NOT NULL" + case OpDropNotNull: + return "ALTER COLUMN " + o.Column + " DROP NOT NULL" + case OpSetColumnOptions: + return "ALTER COLUMN " + o.Column + " SET options" + case OpRenameColumn: + return "RENAME COLUMN " + o.Column + " TO " + o.Name + case OpRenameTable: + return "RENAME TO " + o.Name + case OpRenameIndex: + return "RENAME INDEX TO " + o.Name + case OpSetSchema: + return "SET SCHEMA " + o.Name + case OpSetTablespace: + return "SET TABLESPACE " + o.Name + case OpSetRelOptions: + return "SET storage parameters" + case OpAddConstraint: + return "ADD CONSTRAINT " + o.Name + case OpValidateConstraint: + return "VALIDATE CONSTRAINT " + o.Name + case OpDropConstraint: + return "DROP CONSTRAINT " + o.Name + case OpAttachPartition: + return "ATTACH PARTITION" + case OpDetachPartition: + return "DETACH PARTITION" + case OpCreateIndex: + return "CREATE INDEX " + o.Name + case OpDropIndex: + return "DROP INDEX " + o.Name + case OpReindex: + return "REINDEX " + o.Name + default: + return "unrecognized operation" + } +} + +// ParseOps parses one SQL statement and returns its typed operations. An +// ALTER TABLE yields one Op per subcommand; every other supported statement +// yields exactly one. Statements and subcommands the engine does not +// recognize come back as OpUnrecognized — never an error — so the classifier can +// refuse them with context. A parse failure is surfaced to the caller. +func ParseOps(sql string) ([]Op, error) { + tree, err := pgquery.Parse(sql) + if err != nil { + return nil, fmt.Errorf("parse statement: %w", err) + } + if n := len(tree.GetStmts()); n != 1 { + return nil, fmt.Errorf("%w: got %d", ErrNotOneStatement, n) + } + node := tree.GetStmts()[0].GetStmt() + switch { + case node.GetAlterTableStmt() != nil: + alter := node.GetAlterTableStmt() + if alter.GetObjtype() != pganalyze.ObjectType_OBJECT_TABLE { + return []Op{{Kind: OpUnrecognized}}, nil + } + ops := make([]Op, 0, len(alter.GetCmds())) + for _, cmd := range alter.GetCmds() { + ops = append(ops, alterTableOp(cmd.GetAlterTableCmd())) + } + return ops, nil + case node.GetCreateStmt() != nil: + return []Op{{Kind: OpCreateTable}}, nil + case node.GetIndexStmt() != nil: + idx := node.GetIndexStmt() + return []Op{{ + Kind: OpCreateIndex, + Name: idx.GetIdxname(), + Concurrent: idx.GetConcurrent(), + Unique: idx.GetUnique(), + }}, nil + case node.GetDropStmt() != nil: + drop := node.GetDropStmt() + if drop.GetRemoveType() != pganalyze.ObjectType_OBJECT_INDEX { + return []Op{{Kind: OpUnrecognized}}, nil + } + return []Op{{Kind: OpDropIndex, Concurrent: drop.GetConcurrent()}}, nil + case node.GetReindexStmt() != nil: + re := node.GetReindexStmt() + return []Op{{ + Kind: OpReindex, + Name: re.GetRelation().GetRelname(), + Concurrent: reindexConcurrent(re), + }}, nil + case node.GetRenameStmt() != nil: + return []Op{renameOp(node.GetRenameStmt())}, nil + case node.GetAlterObjectSchemaStmt() != nil: + alter := node.GetAlterObjectSchemaStmt() + if alter.GetObjectType() != pganalyze.ObjectType_OBJECT_TABLE { + return []Op{{Kind: OpUnrecognized}}, nil + } + return []Op{{Kind: OpSetSchema, Name: alter.GetNewschema()}}, nil + default: + return []Op{{Kind: OpUnrecognized}}, nil + } +} + +// alterTableOp maps one ALTER TABLE subcommand to its Op. +func alterTableOp(cmd *pganalyze.AlterTableCmd) Op { + switch cmd.GetSubtype() { + case pganalyze.AlterTableType_AT_AddColumn: + return addColumnOp(cmd.GetDef().GetColumnDef()) + case pganalyze.AlterTableType_AT_DropColumn: + return Op{Kind: OpDropColumn, Column: cmd.GetName()} + case pganalyze.AlterTableType_AT_AlterColumnType: + def := cmd.GetDef().GetColumnDef() + name, mods := typeRef(def.GetTypeName()) + return Op{ + Kind: OpAlterColumnType, + Column: cmd.GetName(), + NewType: name, + NewTypeMods: mods, + // For ALTER COLUMN TYPE the grammar carries the USING + // expression in the column definition's raw default slot. + HasUsing: def.GetRawDefault() != nil, + } + case pganalyze.AlterTableType_AT_ColumnDefault: + if cmd.GetDef() == nil { + return Op{Kind: OpDropDefault, Column: cmd.GetName()} + } + return Op{Kind: OpSetDefault, Column: cmd.GetName()} + case pganalyze.AlterTableType_AT_SetNotNull: + return Op{Kind: OpSetNotNull, Column: cmd.GetName()} + case pganalyze.AlterTableType_AT_DropNotNull: + return Op{Kind: OpDropNotNull, Column: cmd.GetName()} + case pganalyze.AlterTableType_AT_SetStatistics, + pganalyze.AlterTableType_AT_SetStorage, + pganalyze.AlterTableType_AT_SetOptions, + pganalyze.AlterTableType_AT_ResetOptions: + return Op{Kind: OpSetColumnOptions, Column: cmd.GetName()} + case pganalyze.AlterTableType_AT_SetRelOptions, + pganalyze.AlterTableType_AT_ResetRelOptions: + return Op{Kind: OpSetRelOptions} + case pganalyze.AlterTableType_AT_SetTableSpace: + return Op{Kind: OpSetTablespace, Name: cmd.GetName()} + case pganalyze.AlterTableType_AT_AddConstraint: + return addConstraintOp(cmd.GetDef().GetConstraint()) + case pganalyze.AlterTableType_AT_ValidateConstraint: + return Op{Kind: OpValidateConstraint, Name: cmd.GetName()} + case pganalyze.AlterTableType_AT_DropConstraint: + return Op{Kind: OpDropConstraint, Name: cmd.GetName()} + case pganalyze.AlterTableType_AT_AttachPartition: + return Op{Kind: OpAttachPartition} + case pganalyze.AlterTableType_AT_DetachPartition: + return Op{Kind: OpDetachPartition, Concurrent: cmd.GetDef().GetPartitionCmd().GetConcurrent()} + default: + return Op{Kind: OpUnrecognized} + } +} + +// addColumnOp extracts the shape facts of an added column: its DEFAULT +// shape and whether it is a stored generated column. Identity and serial +// columns are reported as expression defaults — their values come from a +// sequence, which fast default cannot cover. +func addColumnOp(def *pganalyze.ColumnDef) Op { + op := Op{Kind: OpAddColumn, Column: def.GetColname()} + op.NewType, op.NewTypeMods = typeRef(def.GetTypeName()) + if isSerialType(op.NewType) { + op.Default = DefaultExpression + } + for _, c := range def.GetConstraints() { + con := c.GetConstraint() + switch con.GetContype() { + case pganalyze.ConstrType_CONSTR_DEFAULT: + if isConstantExpr(con.GetRawExpr()) { + op.Default = DefaultConstant + } else { + op.Default = DefaultExpression + } + case pganalyze.ConstrType_CONSTR_IDENTITY: + op.Default = DefaultExpression + case pganalyze.ConstrType_CONSTR_GENERATED: + op.GeneratedStored = true + } + } + return op +} + +// addConstraintOp extracts the shape facts of an ADD CONSTRAINT. +func addConstraintOp(con *pganalyze.Constraint) Op { + op := Op{ + Kind: OpAddConstraint, + Name: con.GetConname(), + NotValid: con.GetSkipValidation(), + UsingIndex: con.GetIndexname() != "", + } + switch con.GetContype() { + case pganalyze.ConstrType_CONSTR_PRIMARY: + op.Constraint = ConstraintPrimaryKey + case pganalyze.ConstrType_CONSTR_UNIQUE: + op.Constraint = ConstraintUnique + case pganalyze.ConstrType_CONSTR_CHECK: + op.Constraint = ConstraintCheck + case pganalyze.ConstrType_CONSTR_FOREIGN: + op.Constraint = ConstraintForeignKey + case pganalyze.ConstrType_CONSTR_NOTNULL: + op.Constraint = ConstraintNotNull + default: + op.Constraint = ConstraintUnrecognized + } + for _, k := range con.GetKeys() { + op.Columns = append(op.Columns, k.GetString_().GetSval()) + } + return op +} + +// renameOp maps a RENAME statement (column, table, or index — the grammar +// parses all three as RenameStmt) to its Op. +func renameOp(ren *pganalyze.RenameStmt) Op { + switch ren.GetRenameType() { + case pganalyze.ObjectType_OBJECT_COLUMN: + return Op{Kind: OpRenameColumn, Column: ren.GetSubname(), Name: ren.GetNewname()} + case pganalyze.ObjectType_OBJECT_TABLE: + return Op{Kind: OpRenameTable, Name: ren.GetNewname()} + case pganalyze.ObjectType_OBJECT_INDEX: + return Op{Kind: OpRenameIndex, Name: ren.GetNewname()} + default: + return Op{Kind: OpUnrecognized} + } +} + +// reindexConcurrent reports whether a REINDEX statement carries the +// CONCURRENTLY option (a DefElem in the statement's parameter list). +func reindexConcurrent(re *pganalyze.ReindexStmt) bool { + for _, p := range re.GetParams() { + if p.GetDefElem().GetDefname() == "concurrently" { + return true + } + } + return false +} + +// typeRef returns the bare grammar type name (last path element, without +// the pg_catalog qualification) and its integer modifiers. +func typeRef(tn *pganalyze.TypeName) (string, []int32) { + names := tn.GetNames() + if len(names) == 0 { + return "", nil + } + name := names[len(names)-1].GetString_().GetSval() + var mods []int32 + for _, m := range tn.GetTypmods() { + mods = append(mods, int32(m.GetAConst().GetIval().GetIval())) + } + return name, mods +} + +// isSerialType reports whether the grammar type name is one of the serial +// pseudo-types, which expand to a sequence-backed default. +func isSerialType(name string) bool { + switch strings.ToLower(name) { + case "serial", "serial2", "serial4", "serial8", "smallserial", "bigserial": + return true + default: + return false + } +} + +// isConstantExpr reports whether a DEFAULT expression is a provable +// constant: a literal, possibly wrapped in type casts. Anything else — +// function calls, value functions like CURRENT_TIMESTAMP, expressions — +// is not, and the caller treats it as volatile. +func isConstantExpr(node *pganalyze.Node) bool { + switch { + case node.GetAConst() != nil: + return true + case node.GetTypeCast() != nil: + return isConstantExpr(node.GetTypeCast().GetArg()) + default: + return false + } +} diff --git a/pkg/statement/ops_test.go b/pkg/statement/ops_test.go new file mode 100644 index 0000000..93b4ebd --- /dev/null +++ b/pkg/statement/ops_test.go @@ -0,0 +1,294 @@ +package statement_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/block/pg-sprite/pkg/statement" +) + +func parseOneOp(t *testing.T, sql string) statement.Op { + t.Helper() + ops, err := statement.ParseOps(sql) + require.NoError(t, err) + require.Len(t, ops, 1) + return ops[0] +} + +func TestParseOpsShapes(t *testing.T) { + cases := []struct { + name string + sql string + want statement.Op + }{ + { + name: "add column plain", + sql: "ALTER TABLE t ADD COLUMN age int", + want: statement.Op{Kind: statement.OpAddColumn, Column: "age", NewType: "int4"}, + }, + { + name: "add column constant default", + sql: "ALTER TABLE t ADD COLUMN age int DEFAULT 0", + want: statement.Op{Kind: statement.OpAddColumn, Column: "age", NewType: "int4", Default: statement.DefaultConstant}, + }, + { + name: "add column cast constant default", + sql: "ALTER TABLE t ADD COLUMN created timestamptz DEFAULT '2020-01-01'::timestamptz", + want: statement.Op{Kind: statement.OpAddColumn, Column: "created", NewType: "timestamptz", Default: statement.DefaultConstant}, + }, + { + name: "add column function default", + sql: "ALTER TABLE t ADD COLUMN id uuid DEFAULT uuid_generate_v4()", + want: statement.Op{Kind: statement.OpAddColumn, Column: "id", NewType: "uuid", Default: statement.DefaultExpression}, + }, + { + name: "add column value function default", + sql: "ALTER TABLE t ADD COLUMN created timestamptz DEFAULT CURRENT_TIMESTAMP", + want: statement.Op{Kind: statement.OpAddColumn, Column: "created", NewType: "timestamptz", Default: statement.DefaultExpression}, + }, + { + name: "add column serial", + sql: "ALTER TABLE t ADD COLUMN n serial", + want: statement.Op{Kind: statement.OpAddColumn, Column: "n", NewType: "serial", Default: statement.DefaultExpression}, + }, + { + name: "add column identity", + sql: "ALTER TABLE t ADD COLUMN n bigint GENERATED ALWAYS AS IDENTITY", + want: statement.Op{Kind: statement.OpAddColumn, Column: "n", NewType: "int8", Default: statement.DefaultExpression}, + }, + { + name: "add column generated stored", + sql: "ALTER TABLE t ADD COLUMN total numeric GENERATED ALWAYS AS (price * qty) STORED", + want: statement.Op{Kind: statement.OpAddColumn, Column: "total", NewType: "numeric", GeneratedStored: true}, + }, + { + name: "drop column", + sql: "ALTER TABLE t DROP COLUMN age", + want: statement.Op{Kind: statement.OpDropColumn, Column: "age"}, + }, + { + name: "alter type with mods", + sql: "ALTER TABLE t ALTER COLUMN name TYPE varchar(100)", + want: statement.Op{Kind: statement.OpAlterColumnType, Column: "name", NewType: "varchar", NewTypeMods: []int32{100}}, + }, + { + name: "alter type with using", + sql: "ALTER TABLE t ALTER COLUMN doc TYPE jsonb USING doc::jsonb", + want: statement.Op{Kind: statement.OpAlterColumnType, Column: "doc", NewType: "jsonb", HasUsing: true}, + }, + { + name: "set default", + sql: "ALTER TABLE t ALTER COLUMN age SET DEFAULT 1", + want: statement.Op{Kind: statement.OpSetDefault, Column: "age"}, + }, + { + name: "drop default", + sql: "ALTER TABLE t ALTER COLUMN age DROP DEFAULT", + want: statement.Op{Kind: statement.OpDropDefault, Column: "age"}, + }, + { + name: "set not null", + sql: "ALTER TABLE t ALTER COLUMN age SET NOT NULL", + want: statement.Op{Kind: statement.OpSetNotNull, Column: "age"}, + }, + { + name: "drop not null", + sql: "ALTER TABLE t ALTER COLUMN age DROP NOT NULL", + want: statement.Op{Kind: statement.OpDropNotNull, Column: "age"}, + }, + { + name: "set statistics", + sql: "ALTER TABLE t ALTER COLUMN age SET STATISTICS 500", + want: statement.Op{Kind: statement.OpSetColumnOptions, Column: "age"}, + }, + { + name: "rename column", + sql: "ALTER TABLE t RENAME COLUMN a TO b", + want: statement.Op{Kind: statement.OpRenameColumn, Column: "a", Name: "b"}, + }, + { + name: "rename table", + sql: "ALTER TABLE t RENAME TO t2", + want: statement.Op{Kind: statement.OpRenameTable, Name: "t2"}, + }, + { + name: "rename index", + sql: "ALTER INDEX i RENAME TO i2", + want: statement.Op{Kind: statement.OpRenameIndex, Name: "i2"}, + }, + { + name: "set schema", + sql: "ALTER TABLE t SET SCHEMA s2", + want: statement.Op{Kind: statement.OpSetSchema, Name: "s2"}, + }, + { + name: "set tablespace", + sql: "ALTER TABLE t SET TABLESPACE fast", + want: statement.Op{Kind: statement.OpSetTablespace, Name: "fast"}, + }, + { + name: "set rel options", + sql: "ALTER TABLE t SET (fillfactor = 70)", + want: statement.Op{Kind: statement.OpSetRelOptions}, + }, + { + name: "add primary key", + sql: "ALTER TABLE t ADD CONSTRAINT t_pkey PRIMARY KEY (id)", + want: statement.Op{Kind: statement.OpAddConstraint, Name: "t_pkey", Constraint: statement.ConstraintPrimaryKey, Columns: []string{"id"}}, + }, + { + name: "add unique using index", + sql: "ALTER TABLE t ADD CONSTRAINT u UNIQUE USING INDEX u_idx", + want: statement.Op{Kind: statement.OpAddConstraint, Name: "u", Constraint: statement.ConstraintUnique, UsingIndex: true}, + }, + { + name: "add check not valid", + sql: "ALTER TABLE t ADD CONSTRAINT c CHECK (age > 0) NOT VALID", + want: statement.Op{Kind: statement.OpAddConstraint, Name: "c", Constraint: statement.ConstraintCheck, NotValid: true}, + }, + { + name: "add foreign key", + sql: "ALTER TABLE t ADD CONSTRAINT fk FOREIGN KEY (pid) REFERENCES p (id)", + want: statement.Op{Kind: statement.OpAddConstraint, Name: "fk", Constraint: statement.ConstraintForeignKey}, + }, + { + name: "validate constraint", + sql: "ALTER TABLE t VALIDATE CONSTRAINT c", + want: statement.Op{Kind: statement.OpValidateConstraint, Name: "c"}, + }, + { + name: "drop constraint", + sql: "ALTER TABLE t DROP CONSTRAINT c", + want: statement.Op{Kind: statement.OpDropConstraint, Name: "c"}, + }, + { + name: "attach partition", + sql: "ALTER TABLE t ATTACH PARTITION p FOR VALUES FROM (1) TO (10)", + want: statement.Op{Kind: statement.OpAttachPartition}, + }, + { + name: "detach partition", + sql: "ALTER TABLE t DETACH PARTITION p", + want: statement.Op{Kind: statement.OpDetachPartition}, + }, + { + name: "detach partition concurrently", + sql: "ALTER TABLE t DETACH PARTITION p CONCURRENTLY", + want: statement.Op{Kind: statement.OpDetachPartition, Concurrent: true}, + }, + { + name: "create index", + sql: "CREATE INDEX i ON t (a)", + want: statement.Op{Kind: statement.OpCreateIndex, Name: "i"}, + }, + { + name: "create unique index concurrently", + sql: "CREATE UNIQUE INDEX CONCURRENTLY i ON t (a)", + want: statement.Op{Kind: statement.OpCreateIndex, Name: "i", Concurrent: true, Unique: true}, + }, + { + name: "drop index", + sql: "DROP INDEX i", + want: statement.Op{Kind: statement.OpDropIndex}, + }, + { + name: "drop index concurrently", + sql: "DROP INDEX CONCURRENTLY i", + want: statement.Op{Kind: statement.OpDropIndex, Concurrent: true}, + }, + { + name: "reindex", + sql: "REINDEX INDEX i", + want: statement.Op{Kind: statement.OpReindex, Name: "i"}, + }, + { + name: "reindex concurrently", + sql: "REINDEX INDEX CONCURRENTLY i", + want: statement.Op{Kind: statement.OpReindex, Name: "i", Concurrent: true}, + }, + { + name: "create table", + sql: "CREATE TABLE t (id int PRIMARY KEY)", + want: statement.Op{Kind: statement.OpCreateTable}, + }, + { + name: "unrecognized statement", + sql: "VACUUM FULL t", + want: statement.Op{Kind: statement.OpUnrecognized}, + }, + { + name: "unrecognized subcommand", + sql: "ALTER TABLE t ENABLE ROW LEVEL SECURITY", + want: statement.Op{Kind: statement.OpUnrecognized}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.want, parseOneOp(t, tc.sql)) + }) + } +} + +func TestParseOpsMultipleSubcommands(t *testing.T) { + ops, err := statement.ParseOps("ALTER TABLE t ADD COLUMN a int, DROP COLUMN b") + require.NoError(t, err) + require.Len(t, ops, 2) + assert.Equal(t, statement.OpAddColumn, ops[0].Kind) + assert.Equal(t, statement.OpDropColumn, ops[1].Kind) +} + +func TestParseOpsRejectsMultipleStatements(t *testing.T) { + _, err := statement.ParseOps("SELECT 1; SELECT 2") + assert.ErrorIs(t, err, statement.ErrNotOneStatement) +} + +func TestConcurrentlyRewrites(t *testing.T) { + // The rewrite contract is syntactic: the result must parse back with + // the concurrency flag set. Exact deparser wording is not a contract. + for _, sql := range []string{ + "CREATE INDEX i ON t (a)", + "DROP INDEX i", + "REINDEX INDEX i", + "ALTER TABLE t DETACH PARTITION p", + } { + t.Run(sql, func(t *testing.T) { + safer, err := statement.Concurrently(sql) + require.NoError(t, err) + op := parseOneOp(t, safer) + assert.True(t, op.Concurrent, "rewritten statement must be concurrent: %s", safer) + }) + } +} + +func TestConcurrentlyRefusesOtherStatements(t *testing.T) { + _, err := statement.Concurrently("ALTER TABLE t ADD COLUMN a int") + assert.ErrorIs(t, err, statement.ErrNotRewritable) +} + +func TestAddNotValidRewritesNamedCheck(t *testing.T) { + safer, name, err := statement.AddNotValid("ALTER TABLE t ADD CONSTRAINT c CHECK (age > 0)") + require.NoError(t, err) + assert.Equal(t, "c", name) + op := parseOneOp(t, safer) + assert.True(t, op.NotValid, "rewritten constraint must be NOT VALID: %s", safer) +} + +func TestAddNotValidRefusals(t *testing.T) { + cases := []struct { + name string + sql string + }{ + {"unnamed constraint", "ALTER TABLE t ADD CHECK (age > 0)"}, + {"primary key", "ALTER TABLE t ADD CONSTRAINT p PRIMARY KEY (id)"}, + {"not an add constraint", "ALTER TABLE t DROP COLUMN a"}, + {"multiple subcommands", "ALTER TABLE t ADD CONSTRAINT c CHECK (a > 0), DROP COLUMN b"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, _, err := statement.AddNotValid(tc.sql) + assert.ErrorIs(t, err, statement.ErrNotValidNotApplicable) + }) + } +} diff --git a/pkg/statement/rewrite.go b/pkg/statement/rewrite.go new file mode 100644 index 0000000..2adaa5b --- /dev/null +++ b/pkg/statement/rewrite.go @@ -0,0 +1,109 @@ +package statement + +import ( + "errors" + "fmt" + + pganalyze "github.com/pganalyze/pg_query_go/v6" + pgquery "github.com/wasilibs/go-pgquery" +) + +// Typed refusals for advisory rewrites. The rewriters flip one syntactic +// flag and deparse — no semantics are derived; a statement that cannot be +// rewritten that way is refused with one of these. +var ( + // ErrNotRewritable is returned when the statement kind has no + // CONCURRENTLY form to rewrite to. + ErrNotRewritable = errors.New("statement has no concurrent form") + // ErrNotValidNotApplicable is returned when the statement is not a + // single named ADD CHECK / ADD FOREIGN KEY that could take NOT VALID. + ErrNotValidNotApplicable = errors.New("statement cannot take NOT VALID") +) + +// Concurrently returns sql rewritten to its CONCURRENTLY form: CREATE +// INDEX, DROP INDEX, REINDEX, or ALTER TABLE ... DETACH PARTITION. The +// rewrite flips the grammar's concurrency flag and deparses — nothing else +// changes. A statement already concurrent comes back canonicalized. +func Concurrently(sql string) (string, error) { + node, err := parseSingle(sql) + if err != nil { + return "", err + } + switch { + case node.GetIndexStmt() != nil: + node.GetIndexStmt().Concurrent = true + case node.GetDropStmt() != nil && node.GetDropStmt().GetRemoveType() == pganalyze.ObjectType_OBJECT_INDEX: + node.GetDropStmt().Concurrent = true + case node.GetReindexStmt() != nil: + re := node.GetReindexStmt() + if !reindexConcurrent(re) { + re.Params = append(re.Params, &pganalyze.Node{ + Node: &pganalyze.Node_DefElem{DefElem: &pganalyze.DefElem{Defname: "concurrently"}}, + }) + } + case detachPartitionCmd(node) != nil: + detachPartitionCmd(node).Concurrent = true + default: + return "", ErrNotRewritable + } + return deparseOne(node) +} + +// AddNotValid rewrites a single-command ALTER TABLE ... ADD CONSTRAINT +// (named CHECK or FOREIGN KEY) to its NOT VALID form and returns the +// rewritten statement plus the constraint name for the follow-up +// VALIDATE CONSTRAINT step. +func AddNotValid(sql string) (rewritten, constraint string, err error) { + node, err := parseSingle(sql) + if err != nil { + return "", "", err + } + alter := node.GetAlterTableStmt() + if alter == nil || alter.GetObjtype() != pganalyze.ObjectType_OBJECT_TABLE || len(alter.GetCmds()) != 1 { + return "", "", ErrNotValidNotApplicable + } + cmd := alter.GetCmds()[0].GetAlterTableCmd() + if cmd.GetSubtype() != pganalyze.AlterTableType_AT_AddConstraint { + return "", "", ErrNotValidNotApplicable + } + con := cmd.GetDef().GetConstraint() + validatable := con.GetContype() == pganalyze.ConstrType_CONSTR_CHECK || + con.GetContype() == pganalyze.ConstrType_CONSTR_FOREIGN + if !validatable || con.GetConname() == "" { + return "", "", ErrNotValidNotApplicable + } + con.SkipValidation = true + con.InitiallyValid = false + if rewritten, err = deparseOne(node); err != nil { + return "", "", err + } + return rewritten, con.GetConname(), nil +} + +// parseSingle parses sql and requires exactly one statement, returning its +// root node for in-place rewriting. +func parseSingle(sql string) (*pganalyze.Node, error) { + tree, err := pgquery.Parse(sql) + if err != nil { + return nil, fmt.Errorf("parse statement: %w", err) + } + if n := len(tree.GetStmts()); n != 1 { + return nil, fmt.Errorf("%w: got %d", ErrNotOneStatement, n) + } + return tree.GetStmts()[0].GetStmt(), nil +} + +// detachPartitionCmd returns the PartitionCmd of a single-command +// ALTER TABLE ... DETACH PARTITION, or nil when the statement is anything +// else. +func detachPartitionCmd(node *pganalyze.Node) *pganalyze.PartitionCmd { + alter := node.GetAlterTableStmt() + if alter == nil || len(alter.GetCmds()) != 1 { + return nil + } + cmd := alter.GetCmds()[0].GetAlterTableCmd() + if cmd.GetSubtype() != pganalyze.AlterTableType_AT_DetachPartition { + return nil + } + return cmd.GetDef().GetPartitionCmd() +} From b08f7b8c61b4145edd4708064a16f00d93d14430 Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Wed, 5 Aug 2026 16:48:13 +1000 Subject: [PATCH 16/24] Phase 2.3b: classifier routing native / copy-and-swap / refuse pkg/planner maps each operation to a route with a typed reason, golden-tested against every row of the online-DDL reference. Risky literals get the safer native sequence (CONCURRENTLY, NOT VALID + VALIDATE, USING INDEX attach, the four-step SET NOT NULL pattern). Conservative by construction: unproven defaults are volatile, type changes without live column facts are rewrites, unknown operations are refused. --- SAFETY.md | 2 +- pkg/planner/planner.go | 426 ++++++++++++++++++++++++++++++++++++ pkg/planner/planner_test.go | 179 +++++++++++++++ 3 files changed, 606 insertions(+), 1 deletion(-) create mode 100644 pkg/planner/planner.go create mode 100644 pkg/planner/planner_test.go diff --git a/SAFETY.md b/SAFETY.md index 418341c..9fc1b66 100644 --- a/SAFETY.md +++ b/SAFETY.md @@ -27,7 +27,7 @@ The invariant registry (invariant IDs referenced below) lives in | `pkg/checkpoint` — durable resume state | ✅ core | planned (Phase 8) | ST-1, ST-2 | | slot lifecycle (in `pkg/decode`) — create, reap, lag ceiling | ✅ core | planned (Phase 8) | ST-3 | | `pkg/migration` — orchestrator, **cutover swap + fidelity gate** | ✅ core | planned (Phase 7) | LK-2, LK-4, ST-5 | -| `pkg/statement`, `pkg/planner`, `pkg/schemadiff`, `pkg/lint` — classify/diff/route | ❌ periphery¹ | `pkg/statement` (parse boundary) and `pkg/schemadiff` (introspect/diff via scratch execute-and-introspect) exist (Phase 2); `pkg/planner`, `pkg/lint` planned (Phase 2) | (CO-7 holds at the parse boundary) | +| `pkg/statement`, `pkg/planner`, `pkg/schemadiff`, `pkg/lint` — classify/diff/route | ❌ periphery¹ | `pkg/statement` (parse boundary), `pkg/schemadiff` (introspect/diff via scratch execute-and-introspect), and `pkg/planner` (classifier) exist (Phase 2); `pkg/lint` planned (Phase 2) | (CO-7 holds at the parse boundary) | | `pkg/verdict` — structured outcome contract, rendering, exit codes | ❌ periphery | exists (Phase 1) | — | | `internal/cli` — CLI, flags, help, prompts | ❌ periphery | `migrate`/`status` exist (Phase 1); rest stubs | — | | status / progress / advisory rendering, metrics | ❌ periphery | planned | — | diff --git a/pkg/planner/planner.go b/pkg/planner/planner.go new file mode 100644 index 0000000..8a07dec --- /dev/null +++ b/pkg/planner/planner.go @@ -0,0 +1,426 @@ +// Package planner classifies schema-change statements: for each operation +// it decides whether PostgreSQL can run it online natively (possibly via a +// safer idiom it suggests), whether it needs the engine's copy-and-swap +// path, or whether it is refused. The mapping is the "Needs copy-and-swap?" +// column of docs/postgres-online-ddl-reference.md, applied conservatively: +// anything the planner cannot prove safe routes to copy-and-swap or refuse. +// Classification predicts; executors keep their own protections regardless. +package planner + +import ( + "fmt" + "strconv" + "strings" + + "github.com/jackc/pgx/v5" + + "github.com/block/pg-sprite/pkg/statement" +) + +// Route is where an operation is sent. +type Route string + +// The three routes. +const ( + // RouteNative: PostgreSQL runs it online natively — directly or via + // the safer idiom in Decision.SaferSQL. + RouteNative Route = "native" + // RouteCopyAndSwap: needs a table rewrite; only the engine's shadow + // copy + cutover can do it online. + RouteCopyAndSwap Route = "copy-and-swap" + // RouteRefuse: no known safe path; not executed. + RouteRefuse Route = "refuse" +) + +// worse orders routes for aggregation: refuse > copy-and-swap > native. +func worse(a, b Route) Route { + rank := map[Route]int{RouteNative: 0, RouteCopyAndSwap: 1, RouteRefuse: 2} + if rank[b] > rank[a] { + return b + } + return a +} + +// Reason is the typed cause of a routing decision; automation branches on +// it, never on prose. +type Reason string + +// The reasons a decision can carry. +const ( + // ReasonMetadataOnly: a brief ACCESS EXCLUSIVE catalog change, no scan + // and no rewrite. + ReasonMetadataOnly Reason = "metadata-only" + // ReasonOnlineIdiom: already the safe native form (CONCURRENTLY, + // NOT VALID, VALIDATE, USING INDEX). + ReasonOnlineIdiom Reason = "online-idiom" + // ReasonFastDefault: ADD COLUMN with a constant default — the catalog + // stores the default, no rewrite (PG 11+). + ReasonFastDefault Reason = "fast-default" + // ReasonBinaryCoercible: a type change PostgreSQL relabels without a + // rewrite (widen varchar, varchar to text, widen numeric precision). + ReasonBinaryCoercible Reason = "binary-coercible" + // ReasonSaferIdiom: native, but the submitted form blocks; SaferSQL + // carries the online rewrite when one can be constructed. + ReasonSaferIdiom Reason = "safer-idiom" + // ReasonVolatileDefault: ADD COLUMN whose default the planner cannot + // prove constant — PostgreSQL rewrites the table. + ReasonVolatileDefault Reason = "volatile-default" + // ReasonGeneratedStored: adding a stored generated column computes + // every row — a full rewrite. + ReasonGeneratedStored Reason = "generated-stored" + // ReasonTypeRewrite: a type conversion PostgreSQL cannot relabel — + // rewrite plus reindex. + ReasonTypeRewrite Reason = "type-rewrite" + // ReasonRelocation: SET TABLESPACE moves the heap — a rewrite-scale + // copy. + ReasonRelocation Reason = "relocation" + // ReasonUnsupportedOperation: the planner does not recognize the + // operation or knows no safe path for it. + ReasonUnsupportedOperation Reason = "unsupported-operation" +) + +// Decision is the classification of one operation. +type Decision struct { + // Operation is the operator-facing label (display only). + Operation string `json:"operation"` + // Route is where the operation goes. + Route Route `json:"route"` + // Reason is why. + Reason Reason `json:"reason"` + // SaferSQL is the ordered native sequence to run instead of the + // submitted form, present only for safer-idiom decisions where the + // planner could construct it. + SaferSQL []string `json:"safer_sql,omitempty"` +} + +// Plan is the classification of one statement: one decision per operation +// and the aggregate route (the worst of its decisions — one rewrite makes +// the whole statement a copy, one refusal refuses it). +type Plan struct { + // Statement is the submitted SQL. + Statement string `json:"statement"` + // Route is the aggregate route. + Route Route `json:"route"` + // Decisions are the per-operation classifications, in statement order. + Decisions []Decision `json:"decisions"` +} + +// Facts are introspected properties of the live table that sharpen +// classification. The zero value is valid: with no facts the planner is +// strictly more conservative (every type change becomes copy-and-swap). +type Facts struct { + // ColumnTypes maps a column name to its live type as rendered by + // PostgreSQL's format_type (e.g. "character varying(50)"). + ColumnTypes map[string]string +} + +// Classify parses one statement and routes each of its operations. A parse +// failure is an error; an unrecognized operation is not — it comes back as +// a refuse decision so the caller can render the whole plan. +func Classify(sql string, facts Facts) (Plan, error) { + st, err := statement.ParseOne(sql) + if err != nil { + return Plan{}, err + } + ops, err := statement.ParseOps(sql) + if err != nil { + return Plan{}, err + } + plan := Plan{Statement: sql, Route: RouteNative} + // Safer rewrites are only constructed for single-operation statements: + // a partial rewrite of a multi-operation ALTER would be misleading. + single := len(ops) == 1 + for _, op := range ops { + d := classifyOp(op, st, facts, sql, single) + plan.Route = worse(plan.Route, d.Route) + plan.Decisions = append(plan.Decisions, d) + } + return plan, nil +} + +// classifyOp routes one operation per the reference table. +func classifyOp(op statement.Op, st statement.Statement, facts Facts, sql string, single bool) Decision { + d := Decision{Operation: op.Describe()} + switch op.Kind { + case statement.OpCreateTable: + // A new table has no readers to lock out. + d.Route, d.Reason = RouteNative, ReasonMetadataOnly + + case statement.OpAddColumn: + switch { + case op.GeneratedStored: + d.Route, d.Reason = RouteCopyAndSwap, ReasonGeneratedStored + case op.Default == statement.DefaultExpression: + d.Route, d.Reason = RouteCopyAndSwap, ReasonVolatileDefault + case op.Default == statement.DefaultConstant: + d.Route, d.Reason = RouteNative, ReasonFastDefault + default: + d.Route, d.Reason = RouteNative, ReasonMetadataOnly + } + + case statement.OpDropColumn, statement.OpSetDefault, statement.OpDropDefault, + statement.OpDropNotNull, statement.OpRenameColumn, statement.OpRenameTable, + statement.OpRenameIndex, statement.OpSetColumnOptions, statement.OpSetRelOptions, + statement.OpSetSchema, statement.OpDropConstraint: + d.Route, d.Reason = RouteNative, ReasonMetadataOnly + + case statement.OpAlterColumnType: + d.Route, d.Reason = classifyTypeChange(op, facts) + + case statement.OpSetNotNull: + // Native pattern: prove the invariant with a NOT VALID CHECK plus + // an online VALIDATE, then SET NOT NULL is a catalog flip (PG 12+). + d.Route, d.Reason = RouteNative, ReasonSaferIdiom + if single { + d.SaferSQL = setNotNullSequence(st, op.Column) + } + + case statement.OpSetTablespace: + d.Route, d.Reason = RouteCopyAndSwap, ReasonRelocation + + case statement.OpAddConstraint: + d = classifyAddConstraint(op, st, sql, single) + + case statement.OpValidateConstraint: + d.Route, d.Reason = RouteNative, ReasonOnlineIdiom + + case statement.OpAttachPartition: + // Native pattern: pre-add a validated CHECK matching the bound on + // the child to skip the attach-time scan. The planner cannot + // construct that CHECK, so it routes native without a rewrite. + d.Route, d.Reason = RouteNative, ReasonSaferIdiom + + case statement.OpDetachPartition: + d = concurrentlyDecision(d, op.Concurrent, sql, single) + + case statement.OpCreateIndex, statement.OpDropIndex, statement.OpReindex: + d = concurrentlyDecision(d, op.Concurrent, sql, single) + + default: + d.Route, d.Reason = RouteRefuse, ReasonUnsupportedOperation + } + return d +} + +// concurrentlyDecision routes an operation that is online in its +// CONCURRENTLY form: already concurrent is the idiom; otherwise native with +// the concurrent rewrite as the safer sequence. +func concurrentlyDecision(d Decision, concurrent bool, sql string, single bool) Decision { + if concurrent { + d.Route, d.Reason = RouteNative, ReasonOnlineIdiom + return d + } + d.Route, d.Reason = RouteNative, ReasonSaferIdiom + if single { + if safer, err := statement.Concurrently(sql); err == nil { + d.SaferSQL = []string{safer} + } + } + return d +} + +// classifyAddConstraint routes ADD CONSTRAINT per constraint family. +func classifyAddConstraint(op statement.Op, st statement.Statement, sql string, single bool) Decision { + d := Decision{Operation: op.Describe()} + switch { + case op.UsingIndex, op.NotValid: + // Already the safe pattern's cheap step. + d.Route, d.Reason = RouteNative, ReasonOnlineIdiom + + case op.Constraint == statement.ConstraintPrimaryKey, + op.Constraint == statement.ConstraintUnique: + // Direct ADD PK/UNIQUE builds its index under ACCESS EXCLUSIVE; + // the safer sequence builds it concurrently and attaches it. + d.Route, d.Reason = RouteNative, ReasonSaferIdiom + if single && len(op.Columns) > 0 { + d.SaferSQL = usingIndexSequence(st, op) + } + + case op.Constraint == statement.ConstraintCheck, + op.Constraint == statement.ConstraintForeignKey: + // Direct ADD CHECK/FK validates under ACCESS EXCLUSIVE; the safer + // sequence is NOT VALID plus an online VALIDATE. + d.Route, d.Reason = RouteNative, ReasonSaferIdiom + if single { + if notValid, name, err := statement.AddNotValid(sql); err == nil { + d.SaferSQL = []string{ + notValid, + "ALTER TABLE " + tableIdent(st) + " VALIDATE CONSTRAINT " + pgx.Identifier{name}.Sanitize(), + } + } + } + + case op.Constraint == statement.ConstraintNotNull: + d.Route, d.Reason = RouteNative, ReasonSaferIdiom + + default: + // EXCLUDE and anything unrecognized: no online pattern exists. + d.Route, d.Reason = RouteRefuse, ReasonUnsupportedOperation + } + return d +} + +// classifyTypeChange routes ALTER COLUMN TYPE: binary-coercible changes are +// a brief catalog relabel; everything else — or anything the planner cannot +// verify against live column facts — is a rewrite. +func classifyTypeChange(op statement.Op, facts Facts) (Route, Reason) { + if op.HasUsing { + return RouteCopyAndSwap, ReasonTypeRewrite + } + oldType, ok := facts.ColumnTypes[op.Column] + if !ok { + return RouteCopyAndSwap, ReasonTypeRewrite + } + if binaryCoercible(parseTypeText(oldType), typeShape{name: normalizeTypeName(op.NewType), mods: op.NewTypeMods}) { + return RouteNative, ReasonBinaryCoercible + } + return RouteCopyAndSwap, ReasonTypeRewrite +} + +// typeShape is a normalized type family plus its modifiers, comparable +// across the grammar's spelling and format_type's rendering. +type typeShape struct { + name string + mods []int32 +} + +// binaryCoercible reports whether changing old to new is a relabel +// PostgreSQL performs without a rewrite or scan. The rules are the +// reference table's rows, deliberately narrow: widening varchar, varchar to +// text, and widening numeric precision at the same scale. Anything not +// provably on this list is not coercible. +func binaryCoercible(old, next typeShape) bool { + if old.name == "" || next.name == "" { + return false + } + if old.name == next.name && int32sEqual(old.mods, next.mods) { + return true // no-op relabel + } + switch old.name { + case "varchar": + if next.name == "text" { + return true + } + if next.name != "varchar" { + return false + } + if len(next.mods) == 0 { + return true // dropping the length bound + } + return len(old.mods) == 1 && len(next.mods) == 1 && next.mods[0] >= old.mods[0] + case "numeric": + if next.name != "numeric" { + return false + } + if len(next.mods) == 0 { + return true // dropping the precision bound + } + return len(old.mods) == 2 && len(next.mods) == 2 && + next.mods[1] == old.mods[1] && next.mods[0] >= old.mods[0] + default: + return false + } +} + +// int32sEqual reports element-wise equality. +func int32sEqual(a, b []int32) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// normalizeTypeName maps spelling variants of the families binaryCoercible +// knows onto one name. Unknown names pass through unchanged (and will never +// match a rule). +func normalizeTypeName(name string) string { + switch strings.ToLower(strings.TrimSpace(name)) { + case "varchar", "character varying": + return "varchar" + case "numeric", "decimal": + return "numeric" + case "text": + return "text" + default: + return strings.ToLower(strings.TrimSpace(name)) + } +} + +// parseTypeText splits a format_type rendering ("character varying(50)", +// "numeric(10,2)", "text") into its normalized shape. This reads catalog +// output, not SQL — format_type's rendering is stable. +func parseTypeText(s string) typeShape { + name, rest, found := strings.Cut(s, "(") + shape := typeShape{name: normalizeTypeName(name)} + if !found { + return shape + } + rest, _, found = strings.Cut(rest, ")") + if !found { + return typeShape{} + } + for part := range strings.SplitSeq(rest, ",") { + n, err := strconv.ParseInt(strings.TrimSpace(part), 10, 32) + if err != nil { + return typeShape{} + } + shape.mods = append(shape.mods, int32(n)) + } + return shape +} + +// tableIdent renders the statement's target table as a quoted identifier, +// schema-qualified when the statement was. +func tableIdent(st statement.Statement) string { + if st.Schema != "" { + return pgx.Identifier{st.Schema, st.Table}.Sanitize() + } + return pgx.Identifier{st.Table}.Sanitize() +} + +// setNotNullSequence is the native four-step SET NOT NULL pattern: prove +// the invariant online with a NOT VALID CHECK, flip the column, drop the +// scaffold. +func setNotNullSequence(st statement.Statement, column string) []string { + table := tableIdent(st) + conName := fmt.Sprintf("%s_%s_not_null", st.Table, column) + con := pgx.Identifier{conName}.Sanitize() + col := pgx.Identifier{column}.Sanitize() + return []string{ + fmt.Sprintf("ALTER TABLE %s ADD CONSTRAINT %s CHECK (%s IS NOT NULL) NOT VALID", table, con, col), + fmt.Sprintf("ALTER TABLE %s VALIDATE CONSTRAINT %s", table, con), + fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s SET NOT NULL", table, col), + fmt.Sprintf("ALTER TABLE %s DROP CONSTRAINT %s", table, con), + } +} + +// usingIndexSequence is the native two-step ADD PRIMARY KEY / UNIQUE +// pattern: build the unique index concurrently, then attach it as the +// constraint under a brief lock. +func usingIndexSequence(st statement.Statement, op statement.Op) []string { + suffix := "_key" + keyword := "UNIQUE" + if op.Constraint == statement.ConstraintPrimaryKey { + suffix = "_pkey" + keyword = "PRIMARY KEY" + } + name := op.Name + if name == "" { + name = st.Table + "_" + strings.Join(op.Columns, "_") + suffix + } + idx := pgx.Identifier{name}.Sanitize() + cols := make([]string, len(op.Columns)) + for i, c := range op.Columns { + cols[i] = pgx.Identifier{c}.Sanitize() + } + table := tableIdent(st) + return []string{ + fmt.Sprintf("CREATE UNIQUE INDEX CONCURRENTLY %s ON %s (%s)", idx, table, strings.Join(cols, ", ")), + fmt.Sprintf("ALTER TABLE %s ADD CONSTRAINT %s %s USING INDEX %s", table, idx, keyword, idx), + } +} diff --git a/pkg/planner/planner_test.go b/pkg/planner/planner_test.go new file mode 100644 index 0000000..28113ec --- /dev/null +++ b/pkg/planner/planner_test.go @@ -0,0 +1,179 @@ +package planner_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/block/pg-sprite/pkg/planner" +) + +// facts mirrors a live table whose column types exercise both sides of the +// binary-coercible rules. +var facts = planner.Facts{ColumnTypes: map[string]string{ + "v50": "character varying(50)", + "vany": "character varying", + "num": "numeric(10,2)", + "i": "integer", + "txt": "text", +}} + +func classifyOne(t *testing.T, sql string) planner.Decision { + t.Helper() + plan, err := planner.Classify(sql, facts) + require.NoError(t, err) + require.Len(t, plan.Decisions, 1) + assert.Equal(t, plan.Decisions[0].Route, plan.Route, "single-decision plan route must match") + return plan.Decisions[0] +} + +// TestClassifyReferenceRows is the golden mapping: one case per row of +// docs/postgres-online-ddl-reference.md. saferSteps is the length of the +// expected safer sequence (0 when the decision carries none). +func TestClassifyReferenceRows(t *testing.T) { + cases := []struct { + name string + sql string + route planner.Route + reason planner.Reason + saferSteps int + }{ + // Column operations. + {"add column plain", "ALTER TABLE t ADD COLUMN age int", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + {"add column constant default", "ALTER TABLE t ADD COLUMN age int DEFAULT 0", planner.RouteNative, planner.ReasonFastDefault, 0}, + {"add column volatile default now", "ALTER TABLE t ADD COLUMN created timestamptz DEFAULT now()", planner.RouteCopyAndSwap, planner.ReasonVolatileDefault, 0}, + {"add column volatile default random", "ALTER TABLE t ADD COLUMN r float8 DEFAULT random()", planner.RouteCopyAndSwap, planner.ReasonVolatileDefault, 0}, + {"add column volatile default uuid", "ALTER TABLE t ADD COLUMN id uuid DEFAULT uuid_generate_v4()", planner.RouteCopyAndSwap, planner.ReasonVolatileDefault, 0}, + {"add column generated stored", "ALTER TABLE t ADD COLUMN total numeric GENERATED ALWAYS AS (price * qty) STORED", planner.RouteCopyAndSwap, planner.ReasonGeneratedStored, 0}, + {"drop column", "ALTER TABLE t DROP COLUMN age", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + {"alter type widen varchar", "ALTER TABLE t ALTER COLUMN v50 TYPE varchar(100)", planner.RouteNative, planner.ReasonBinaryCoercible, 0}, + {"alter type varchar to text", "ALTER TABLE t ALTER COLUMN v50 TYPE text", planner.RouteNative, planner.ReasonBinaryCoercible, 0}, + {"alter type drop varchar bound", "ALTER TABLE t ALTER COLUMN v50 TYPE varchar", planner.RouteNative, planner.ReasonBinaryCoercible, 0}, + {"alter type widen numeric precision", "ALTER TABLE t ALTER COLUMN num TYPE numeric(12,2)", planner.RouteNative, planner.ReasonBinaryCoercible, 0}, + {"alter type shrink varchar", "ALTER TABLE t ALTER COLUMN v50 TYPE varchar(10)", planner.RouteCopyAndSwap, planner.ReasonTypeRewrite, 0}, + {"alter type bound an unbounded varchar", "ALTER TABLE t ALTER COLUMN vany TYPE varchar(50)", planner.RouteCopyAndSwap, planner.ReasonTypeRewrite, 0}, + {"alter type numeric scale change", "ALTER TABLE t ALTER COLUMN num TYPE numeric(12,4)", planner.RouteCopyAndSwap, planner.ReasonTypeRewrite, 0}, + {"alter type general int to bigint", "ALTER TABLE t ALTER COLUMN i TYPE bigint", planner.RouteCopyAndSwap, planner.ReasonTypeRewrite, 0}, + {"alter type text to jsonb with using", "ALTER TABLE t ALTER COLUMN txt TYPE jsonb USING txt::jsonb", planner.RouteCopyAndSwap, planner.ReasonTypeRewrite, 0}, + {"alter type unknown column", "ALTER TABLE t ALTER COLUMN mystery TYPE text", planner.RouteCopyAndSwap, planner.ReasonTypeRewrite, 0}, + {"set default", "ALTER TABLE t ALTER COLUMN age SET DEFAULT 1", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + {"drop default", "ALTER TABLE t ALTER COLUMN age DROP DEFAULT", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + {"set not null", "ALTER TABLE t ALTER COLUMN age SET NOT NULL", planner.RouteNative, planner.ReasonSaferIdiom, 4}, + {"drop not null", "ALTER TABLE t ALTER COLUMN age DROP NOT NULL", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + {"rename column", "ALTER TABLE t RENAME COLUMN a TO b", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + {"set statistics", "ALTER TABLE t ALTER COLUMN age SET STATISTICS 500", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + {"set storage", "ALTER TABLE t ALTER COLUMN blob SET STORAGE EXTERNAL", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + {"set column options", "ALTER TABLE t ALTER COLUMN age SET (n_distinct = 100)", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + + // Index operations. + {"create index", "CREATE INDEX i ON t (a)", planner.RouteNative, planner.ReasonSaferIdiom, 1}, + {"create index concurrently", "CREATE INDEX CONCURRENTLY i ON t (a)", planner.RouteNative, planner.ReasonOnlineIdiom, 0}, + {"drop index", "DROP INDEX i", planner.RouteNative, planner.ReasonSaferIdiom, 1}, + {"drop index concurrently", "DROP INDEX CONCURRENTLY i", planner.RouteNative, planner.ReasonOnlineIdiom, 0}, + {"reindex", "REINDEX INDEX i", planner.RouteNative, planner.ReasonSaferIdiom, 1}, + {"reindex concurrently", "REINDEX INDEX CONCURRENTLY i", planner.RouteNative, planner.ReasonOnlineIdiom, 0}, + {"rename index", "ALTER INDEX i RENAME TO i2", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + + // Constraint operations. + {"add primary key direct", "ALTER TABLE t ADD CONSTRAINT t_pkey PRIMARY KEY (id)", planner.RouteNative, planner.ReasonSaferIdiom, 2}, + {"add unique direct", "ALTER TABLE t ADD CONSTRAINT u UNIQUE (a, b)", planner.RouteNative, planner.ReasonSaferIdiom, 2}, + {"add primary key using index", "ALTER TABLE t ADD CONSTRAINT t_pkey PRIMARY KEY USING INDEX t_pkey", planner.RouteNative, planner.ReasonOnlineIdiom, 0}, + {"add check", "ALTER TABLE t ADD CONSTRAINT c CHECK (age > 0)", planner.RouteNative, planner.ReasonSaferIdiom, 2}, + {"add foreign key", "ALTER TABLE t ADD CONSTRAINT fk FOREIGN KEY (pid) REFERENCES p (id)", planner.RouteNative, planner.ReasonSaferIdiom, 2}, + {"add check not valid", "ALTER TABLE t ADD CONSTRAINT c CHECK (age > 0) NOT VALID", planner.RouteNative, planner.ReasonOnlineIdiom, 0}, + {"add foreign key not valid", "ALTER TABLE t ADD CONSTRAINT fk FOREIGN KEY (pid) REFERENCES p (id) NOT VALID", planner.RouteNative, planner.ReasonOnlineIdiom, 0}, + {"add unnamed check", "ALTER TABLE t ADD CHECK (age > 0)", planner.RouteNative, planner.ReasonSaferIdiom, 0}, + {"add exclusion", "ALTER TABLE t ADD CONSTRAINT ex EXCLUDE USING gist (room WITH =)", planner.RouteRefuse, planner.ReasonUnsupportedOperation, 0}, + {"validate constraint", "ALTER TABLE t VALIDATE CONSTRAINT c", planner.RouteNative, planner.ReasonOnlineIdiom, 0}, + {"drop constraint", "ALTER TABLE t DROP CONSTRAINT c", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + + // Table and partition operations. + {"rename table", "ALTER TABLE t RENAME TO t2", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + {"set schema", "ALTER TABLE t SET SCHEMA s2", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + {"set tablespace", "ALTER TABLE t SET TABLESPACE fast", planner.RouteCopyAndSwap, planner.ReasonRelocation, 0}, + {"set fillfactor", "ALTER TABLE t SET (fillfactor = 70)", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + {"cluster", "CLUSTER t USING i", planner.RouteRefuse, planner.ReasonUnsupportedOperation, 0}, + {"vacuum full", "VACUUM FULL t", planner.RouteRefuse, planner.ReasonUnsupportedOperation, 0}, + {"attach partition", "ALTER TABLE t ATTACH PARTITION p FOR VALUES FROM (1) TO (10)", planner.RouteNative, planner.ReasonSaferIdiom, 0}, + {"detach partition", "ALTER TABLE t DETACH PARTITION p", planner.RouteNative, planner.ReasonSaferIdiom, 1}, + {"detach partition concurrently", "ALTER TABLE t DETACH PARTITION p CONCURRENTLY", planner.RouteNative, planner.ReasonOnlineIdiom, 0}, + + // Non-DDL and unknown statements. + {"dml", "INSERT INTO t VALUES (1)", planner.RouteRefuse, planner.ReasonUnsupportedOperation, 0}, + {"create table", "CREATE TABLE t (id int PRIMARY KEY)", planner.RouteNative, planner.ReasonMetadataOnly, 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + d := classifyOne(t, tc.sql) + assert.Equal(t, tc.route, d.Route) + assert.Equal(t, tc.reason, d.Reason) + assert.Len(t, d.SaferSQL, tc.saferSteps) + }) + } +} + +func TestClassifySetNotNullSequence(t *testing.T) { + d := classifyOne(t, "ALTER TABLE s.t ALTER COLUMN age SET NOT NULL") + assert.Equal(t, []string{ + `ALTER TABLE "s"."t" ADD CONSTRAINT "t_age_not_null" CHECK ("age" IS NOT NULL) NOT VALID`, + `ALTER TABLE "s"."t" VALIDATE CONSTRAINT "t_age_not_null"`, + `ALTER TABLE "s"."t" ALTER COLUMN "age" SET NOT NULL`, + `ALTER TABLE "s"."t" DROP CONSTRAINT "t_age_not_null"`, + }, d.SaferSQL) +} + +func TestClassifyAddPrimaryKeySequence(t *testing.T) { + d := classifyOne(t, "ALTER TABLE s.t ADD CONSTRAINT t_pkey PRIMARY KEY (id)") + assert.Equal(t, []string{ + `CREATE UNIQUE INDEX CONCURRENTLY "t_pkey" ON "s"."t" ("id")`, + `ALTER TABLE "s"."t" ADD CONSTRAINT "t_pkey" PRIMARY KEY USING INDEX "t_pkey"`, + }, d.SaferSQL) +} + +func TestClassifyAddCheckSequenceValidates(t *testing.T) { + d := classifyOne(t, "ALTER TABLE t ADD CONSTRAINT c CHECK (age > 0)") + require.Len(t, d.SaferSQL, 2) + // The rewritten first step must parse back as NOT VALID; the second + // step is the online validation of the same constraint. + assert.Contains(t, d.SaferSQL[1], "VALIDATE CONSTRAINT") +} + +func TestClassifyAggregatesWorstRoute(t *testing.T) { + plan, err := planner.Classify( + "ALTER TABLE t ADD COLUMN a int, ADD COLUMN created timestamptz DEFAULT now()", facts) + require.NoError(t, err) + require.Len(t, plan.Decisions, 2) + assert.Equal(t, planner.RouteNative, plan.Decisions[0].Route) + assert.Equal(t, planner.RouteCopyAndSwap, plan.Decisions[1].Route) + assert.Equal(t, planner.RouteCopyAndSwap, plan.Route, "one rewrite makes the statement a copy") +} + +func TestClassifyRefusalDominatesAggregate(t *testing.T) { + plan, err := planner.Classify( + "ALTER TABLE t ADD COLUMN created timestamptz DEFAULT now(), ENABLE ROW LEVEL SECURITY", facts) + require.NoError(t, err) + assert.Equal(t, planner.RouteRefuse, plan.Route, "a refused operation refuses the statement") +} + +func TestClassifyMultiOpStatementCarriesNoRewrites(t *testing.T) { + plan, err := planner.Classify( + "ALTER TABLE t ALTER COLUMN age SET NOT NULL, DROP COLUMN b", facts) + require.NoError(t, err) + for _, d := range plan.Decisions { + assert.Empty(t, d.SaferSQL, "multi-operation statements must not carry partial rewrites") + } +} + +func TestClassifyNoFactsIsConservative(t *testing.T) { + plan, err := planner.Classify("ALTER TABLE t ALTER COLUMN v50 TYPE varchar(100)", planner.Facts{}) + require.NoError(t, err) + require.Len(t, plan.Decisions, 1) + assert.Equal(t, planner.RouteCopyAndSwap, plan.Decisions[0].Route, + "without live column facts every type change is a rewrite") +} + +func TestClassifyParseErrorSurfaces(t *testing.T) { + _, err := planner.Classify("ALTER TABLE", planner.Facts{}) + assert.Error(t, err) +} From 16c42c0d6180c2a344575640ee42dace43b9bced Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Wed, 5 Aug 2026 17:36:46 +1000 Subject: [PATCH 17/24] Phase 2.4: router seam assigning classified statements to backends pkg/router is the policy layer between the classifier and the executors: every classified statement gets a backend (native / copy-and-swap) and a typed disposition; copy-and-swap routes come back unavailable until that executor exists, instead of pretending to run. diff and the new migrate --dry-run share the identical classify-and-route pipeline, with live column types feeding the classifier. Refs PLAT-38439. --- SAFETY.md | 2 +- internal/cli/cli.go | 3 +- internal/cli/diff.go | 136 ++++++++++++++++++++--- internal/cli/diff_integration_test.go | 57 ++++++++++ internal/cli/dryrun.go | 98 +++++++++++++++++ internal/cli/dryrun_integration_test.go | 126 ++++++++++++++++++++++ internal/cli/migrate.go | 6 +- pkg/router/router.go | 137 ++++++++++++++++++++++++ pkg/router/router_test.go | 93 ++++++++++++++++ 9 files changed, 643 insertions(+), 15 deletions(-) create mode 100644 internal/cli/dryrun.go create mode 100644 internal/cli/dryrun_integration_test.go create mode 100644 pkg/router/router.go create mode 100644 pkg/router/router_test.go diff --git a/SAFETY.md b/SAFETY.md index 9fc1b66..191f7e9 100644 --- a/SAFETY.md +++ b/SAFETY.md @@ -27,7 +27,7 @@ The invariant registry (invariant IDs referenced below) lives in | `pkg/checkpoint` — durable resume state | ✅ core | planned (Phase 8) | ST-1, ST-2 | | slot lifecycle (in `pkg/decode`) — create, reap, lag ceiling | ✅ core | planned (Phase 8) | ST-3 | | `pkg/migration` — orchestrator, **cutover swap + fidelity gate** | ✅ core | planned (Phase 7) | LK-2, LK-4, ST-5 | -| `pkg/statement`, `pkg/planner`, `pkg/schemadiff`, `pkg/lint` — classify/diff/route | ❌ periphery¹ | `pkg/statement` (parse boundary), `pkg/schemadiff` (introspect/diff via scratch execute-and-introspect), and `pkg/planner` (classifier) exist (Phase 2); `pkg/lint` planned (Phase 2) | (CO-7 holds at the parse boundary) | +| `pkg/statement`, `pkg/planner`, `pkg/schemadiff`, `pkg/router`, `pkg/lint` — classify/diff/route | ❌ periphery¹ | `pkg/statement` (parse boundary), `pkg/schemadiff` (introspect/diff via scratch execute-and-introspect), `pkg/planner` (classifier), and `pkg/router` (backend assignment + availability policy) exist (Phase 2); `pkg/lint` planned (Phase 2) | (CO-7 holds at the parse boundary) | | `pkg/verdict` — structured outcome contract, rendering, exit codes | ❌ periphery | exists (Phase 1) | — | | `internal/cli` — CLI, flags, help, prompts | ❌ periphery | `migrate`/`status` exist (Phase 1); rest stubs | — | | status / progress / advisory rendering, metrics | ❌ periphery | planned | — | diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 321de51..665d904 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -86,7 +86,8 @@ type MigrateCmd struct { Alter string `help:"Imperative ALTER statement to run." name:"alter" required:""` MaxTableSize byteSize `help:"Size threshold above which the optimistic attempt is skipped (binary units: B, KiB, MiB, GiB, TiB)." default:"1GiB"` - JSON bool `help:"Emit the verdict as JSON."` + DryRun bool `help:"Classify and route the statement, print the plan, and execute nothing."` + JSON bool `help:"Emit the verdict (or dry-run plan) as JSON."` } // Run implements the migrate subcommand. diff --git a/internal/cli/diff.go b/internal/cli/diff.go index 39f170a..e5af19a 100644 --- a/internal/cli/diff.go +++ b/internal/cli/diff.go @@ -7,8 +7,11 @@ import ( "fmt" "io" "os" + "strings" "github.com/block/pg-sprite/pkg/dbconn" + "github.com/block/pg-sprite/pkg/planner" + "github.com/block/pg-sprite/pkg/router" "github.com/block/pg-sprite/pkg/schemadiff" "github.com/block/pg-sprite/pkg/statement" ) @@ -22,9 +25,68 @@ type diffReport struct { // TableExists reports whether the live table was found; when false the // changes are the full desired schema. TableExists bool `json:"table_exists"` + // Disposition is the routed plan's aggregate disposition: what would + // happen if the engine executed this plan now. + Disposition router.Disposition `json:"disposition"` // Changes is the ordered statement plan; empty means the live table // already matches the desired state. - Changes []schemadiff.Change `json:"changes"` + Changes []plannedChange `json:"changes"` +} + +// plannedChange is one diff statement with its classification and routing: +// the derived SQL plus where the engine would send it and what would run. +type plannedChange struct { + schemadiff.Change + // Route is the planner's aggregate route for the statement. + Route planner.Route `json:"route"` + // Backend is the assigned execution strategy; empty for refusals. + Backend router.Backend `json:"backend,omitempty"` + // Disposition is what execution would do with the statement now. + Disposition router.Disposition `json:"disposition"` + // Decisions are the planner's per-operation classifications. + Decisions []planner.Decision `json:"decisions"` + // ExecSQL is the ordered SQL the native backend would run — the safer + // sequence when the planner constructed one. Empty for non-native + // routes. + ExecSQL []string `json:"exec_sql,omitempty"` +} + +// classifyChanges routes every derived change through the shared +// classify-and-route pipeline. facts sharpen type-change classification; +// the zero value is valid and strictly more conservative. +func classifyChanges(changes []schemadiff.Change, facts planner.Facts) ([]plannedChange, router.Disposition, error) { + plans := make([]planner.Plan, 0, len(changes)) + for _, ch := range changes { + plan, err := planner.Classify(ch.SQL, facts) + if err != nil { + return nil, "", fmt.Errorf("classify derived statement %q: %w", ch.SQL, err) + } + plans = append(plans, plan) + } + routed := router.Route(plans) + planned := make([]plannedChange, 0, len(changes)) + for i, ch := range changes { + st := routed.Statements[i] + planned = append(planned, plannedChange{ + Change: ch, + Route: st.Route, + Backend: st.Backend, + Disposition: st.Disposition, + Decisions: st.Decisions, + ExecSQL: st.ExecSQL, + }) + } + return planned, routed.Disposition, nil +} + +// liveFacts extracts the planner facts the live model provides: the +// canonical type of every live column. +func liveFacts(live schemadiff.Model) planner.Facts { + types := make(map[string]string, len(live.Columns)) + for _, col := range live.Columns { + types[col.Name] = col.Type + } + return planner.Facts{ColumnTypes: types} } // run is the diff flow: parse and admit the desired file, introspect the @@ -50,28 +112,36 @@ func (c *DiffCmd) run(ctx context.Context, out io.Writer) error { defer pool.Close() report := diffReport{Schema: c.Schema, Table: ds.Table, TableExists: true} + var changes []schemadiff.Change + var facts planner.Facts live, err := schemadiff.Introspect(ctx, pool, c.Schema, ds.Table) switch { case errors.Is(err, schemadiff.ErrTableNotFound): // No live table: the plan is the desired schema itself, qualified - // onto the target schema. + // onto the target schema, classified with zero facts (there are no + // live columns to sharpen type-change decisions). report.TableExists = false - if report.Changes, err = qualifiedDesired(ds, c.Schema); err != nil { + if changes, err = qualifiedDesired(ds, c.Schema); err != nil { return err } case err != nil: return err default: + facts = liveFacts(live) desired, err := schemadiff.IntrospectDesired(ctx, pool, ds) if err != nil { return err } - if report.Changes, err = schemadiff.Diff(c.Schema, live, desired); err != nil { + if changes, err = schemadiff.Diff(c.Schema, live, desired); err != nil { return err } } + if report.Changes, report.Disposition, err = classifyChanges(changes, facts); err != nil { + return err + } logger.Debug("diff derived", - "schema", c.Schema, "table", ds.Table, "changes", len(report.Changes), "table_exists", report.TableExists) + "schema", c.Schema, "table", ds.Table, "changes", len(report.Changes), + "table_exists", report.TableExists, "disposition", string(report.Disposition)) if c.JSON { return writeJSON(out, report) @@ -96,7 +166,7 @@ func qualifiedDesired(ds statement.DesiredSchema, schema string) ([]schemadiff.C // writeJSON emits the report as JSON. func writeJSON(out io.Writer, report diffReport) error { if report.Changes == nil { - report.Changes = []schemadiff.Change{} + report.Changes = []plannedChange{} } enc := json.NewEncoder(out) enc.SetIndent("", " ") @@ -107,9 +177,11 @@ func writeJSON(out io.Writer, report diffReport) error { } // writePlanText emits the plan as an executable SQL script: one statement -// per line, destructive statements flagged with a leading comment line, and -// SQL comments for the no-change and missing-table cases so the output stays -// valid SQL. +// per line, each annotated with its route, destructive statements flagged, +// and SQL comments for the no-change and missing-table cases so the output +// stays valid SQL. Safer sequences appear as comment lines — never +// substituted into the script body, which stays the literal convergence +// plan (a CONCURRENTLY rewrite could not run inside a transaction block). func writePlanText(out io.Writer, report diffReport) error { if len(report.Changes) == 0 { if _, err := fmt.Fprintln(out, "-- no changes: live table matches the desired schema"); err != nil { @@ -124,18 +196,58 @@ func writePlanText(out io.Writer, report diffReport) error { } } for _, ch := range report.Changes { - if ch.Destructive { - if _, err := fmt.Fprintln(out, "-- destructive"); err != nil { + if err := writeChangeText(out, ch); err != nil { + return err + } + } + return nil +} + +// writeChangeText emits one annotated statement of the text plan. +func writeChangeText(out io.Writer, ch plannedChange) error { + if _, err := fmt.Fprintf(out, "-- %s\n", annotate(ch)); err != nil { + return fmt.Errorf("write plan: %w", err) + } + if len(ch.ExecSQL) > 0 && ch.ExecSQL[0] != ch.SQL { + if _, err := fmt.Fprintln(out, "-- the engine would run instead:"); err != nil { + return fmt.Errorf("write plan: %w", err) + } + for _, safer := range ch.ExecSQL { + if _, err := fmt.Fprintf(out, "-- %s;\n", safer); err != nil { return fmt.Errorf("write plan: %w", err) } } - if _, err := fmt.Fprintf(out, "%s;\n", ch.SQL); err != nil { + } + if ch.Destructive { + if _, err := fmt.Fprintln(out, "-- destructive"); err != nil { return fmt.Errorf("write plan: %w", err) } } + if _, err := fmt.Fprintf(out, "%s;\n", ch.SQL); err != nil { + return fmt.Errorf("write plan: %w", err) + } return nil } +// annotate renders one statement's route annotation: the route, the +// distinct decision reasons, and the availability note for backends this +// build does not implement. +func annotate(ch plannedChange) string { + var reasons []string + seen := map[planner.Reason]bool{} + for _, d := range ch.Decisions { + if !seen[d.Reason] { + seen[d.Reason] = true + reasons = append(reasons, string(d.Reason)) + } + } + s := fmt.Sprintf("%s (%s)", ch.Route, strings.Join(reasons, ", ")) + if ch.Disposition == router.DispositionUnavailable { + s += ": needs the " + string(ch.Backend) + " backend, which is not implemented yet" + } + return s +} + // runFmt canonicalizes a desired-state schema file: every statement is // parsed through the PostgreSQL grammar, admitted by the same rules as diff, // and printed back in the deparser's canonical form. Offline — no database. diff --git a/internal/cli/diff_integration_test.go b/internal/cli/diff_integration_test.go index da8c88a..a415c3c 100644 --- a/internal/cli/diff_integration_test.go +++ b/internal/cli/diff_integration_test.go @@ -14,6 +14,8 @@ import ( "github.com/block/pg-sprite/internal/testutil" "github.com/block/pg-sprite/pkg/dbconn" + "github.com/block/pg-sprite/pkg/planner" + "github.com/block/pg-sprite/pkg/router" ) // newDiffCmd builds a DiffCmd with the flag defaults kong would apply, @@ -69,6 +71,61 @@ func TestDiffPrintsOrderedPlanJSON(t *testing.T) { fmt.Sprintf("CREATE INDEX events_name_idx ON %s.events USING btree (name)", schema), }, sqls) assert.Equal(t, []bool{true, false, false, false}, destructive) + + // Every derived statement is classified and routed: the widen is proven + // binary-coercible by the live facts, SET NOT NULL and CREATE INDEX + // carry their safer native sequences, and the whole plan would execute. + assert.Equal(t, router.DispositionExecute, report.Disposition) + routes := make([]planner.Route, 0, len(report.Changes)) + for _, ch := range report.Changes { + routes = append(routes, ch.Route) + assert.Equal(t, router.BackendNative, ch.Backend, ch.SQL) + assert.Equal(t, router.DispositionExecute, ch.Disposition, ch.SQL) + require.NotEmpty(t, ch.Decisions, ch.SQL) + } + assert.Equal(t, []planner.Route{ + planner.RouteNative, planner.RouteNative, planner.RouteNative, planner.RouteNative, + }, routes) + assert.Equal(t, planner.ReasonBinaryCoercible, report.Changes[1].Decisions[0].Reason, + "live column types must feed the classifier") + assert.Equal(t, planner.ReasonSaferIdiom, report.Changes[2].Decisions[0].Reason) + assert.NotEqual(t, []string{report.Changes[2].SQL}, report.Changes[2].ExecSQL, + "SET NOT NULL carries its safer native sequence") + assert.Equal(t, planner.ReasonSaferIdiom, report.Changes[3].Decisions[0].Reason) + require.Len(t, report.Changes[3].ExecSQL, 1) + assert.NotEqual(t, report.Changes[3].SQL, report.Changes[3].ExecSQL[0], + "CREATE INDEX carries its concurrent rewrite") +} + +// A desired state that needs a table rewrite routes to the copy-and-swap +// backend, and the routed plan says that backend is unavailable in this +// build — the plan is honest about what execution would do. +func TestDiffRoutesRewriteToCopyAndSwap(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf( + "CREATE TABLE %s.events (id int PRIMARY KEY)", schema)) + require.NoError(t, err) + + cmd := newDiffCmd(t, url, schema, "CREATE TABLE events (id bigint PRIMARY KEY)") + cmd.JSON = true + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + + var report diffReport + require.NoError(t, json.Unmarshal([]byte(out.String()), &report)) + assert.Equal(t, router.DispositionUnavailable, report.Disposition) + require.Len(t, report.Changes, 1) + ch := report.Changes[0] + assert.Equal(t, planner.RouteCopyAndSwap, ch.Route) + assert.Equal(t, router.BackendCopyAndSwap, ch.Backend) + assert.Equal(t, router.DispositionUnavailable, ch.Disposition) + assert.Empty(t, ch.ExecSQL) + require.Len(t, ch.Decisions, 1) + assert.Equal(t, planner.ReasonTypeRewrite, ch.Decisions[0].Reason) } // diff must never write: the live table is bit-identical before and after. diff --git a/internal/cli/dryrun.go b/internal/cli/dryrun.go new file mode 100644 index 0000000..3ebd57a --- /dev/null +++ b/internal/cli/dryrun.go @@ -0,0 +1,98 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/block/pg-sprite/pkg/dbconn" + "github.com/block/pg-sprite/pkg/planner" + "github.com/block/pg-sprite/pkg/router" + "github.com/block/pg-sprite/pkg/schemadiff" + "github.com/block/pg-sprite/pkg/statement" +) + +// runDryRun is the imperative dry-run flow: the identical classify-and-route +// pipeline the declarative front-end uses, with the diff step skipped — the +// submitted statement feeds the classifier directly. It prints the routed +// plan and never executes anything. Introspecting the target table sharpens +// type-change classification; a missing table means zero facts and a +// strictly more conservative plan. +func (c *MigrateCmd) runDryRun(ctx context.Context, out io.Writer) error { + logger := c.diag() + st, err := statement.ParseOne(c.Alter) + if err != nil { + return err + } + logger.Debug("statement parsed", "kind", st.Kind, "schema", st.Schema, "table", st.Table) + + pool, err := dbconn.NewPool(ctx, c.Config()) + if err != nil { + return err + } + defer pool.Close() + + facts, err := dryRunFacts(ctx, pool, st) + if err != nil { + return err + } + plan, err := planner.Classify(st.SQL, facts) + if err != nil { + return err + } + routed := router.Route([]planner.Plan{plan}) + logger.Debug("statement routed", + "route", string(plan.Route), "disposition", string(routed.Disposition)) + + if c.JSON { + enc := json.NewEncoder(out) + enc.SetIndent("", " ") + if err := enc.Encode(routed); err != nil { + return fmt.Errorf("write dry-run plan: %w", err) + } + return nil + } + for _, rs := range routed.Statements { + if err := writeChangeText(out, plannedFromRouted(rs)); err != nil { + return err + } + } + return nil +} + +// dryRunFacts introspects the statement's target table for classifier +// facts. Statements without a single table target (index maintenance) and +// missing tables classify with zero facts. +func dryRunFacts(ctx context.Context, pool *pgxpool.Pool, st statement.Statement) (planner.Facts, error) { + if st.Table == "" { + return planner.Facts{}, nil + } + schema := st.Schema + if schema == "" { + schema = "public" + } + live, err := schemadiff.Introspect(ctx, pool, schema, st.Table) + switch { + case errors.Is(err, schemadiff.ErrTableNotFound): + return planner.Facts{}, nil + case err != nil: + return planner.Facts{}, err + } + return liveFacts(live), nil +} + +// plannedFromRouted adapts a routed statement to the shared text renderer. +func plannedFromRouted(rs router.Statement) plannedChange { + return plannedChange{ + Change: schemadiff.Change{SQL: rs.Statement}, + Route: rs.Route, + Backend: rs.Backend, + Disposition: rs.Disposition, + Decisions: rs.Decisions, + ExecSQL: rs.ExecSQL, + } +} diff --git a/internal/cli/dryrun_integration_test.go b/internal/cli/dryrun_integration_test.go new file mode 100644 index 0000000..2ab3a6e --- /dev/null +++ b/internal/cli/dryrun_integration_test.go @@ -0,0 +1,126 @@ +package cli + +import ( + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/block/pg-sprite/internal/testutil" + "github.com/block/pg-sprite/pkg/dbconn" + "github.com/block/pg-sprite/pkg/planner" + "github.com/block/pg-sprite/pkg/router" +) + +// dryRunPlan runs migrate --dry-run --json and decodes the routed plan. +func dryRunPlan(t *testing.T, url, alter string) router.Plan { + t.Helper() + cmd := newMigrateCmd(url, alter) + cmd.DryRun = true + cmd.JSON = true + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + var plan router.Plan + require.NoError(t, json.Unmarshal([]byte(out.String()), &plan)) + return plan +} + +// A rewrite-requiring change dry-runs to the copy-and-swap backend as +// unavailable, and nothing executes: the live column type is untouched. +func TestMigrateDryRunRoutesRewriteWithoutExecuting(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY)", schema)) + require.NoError(t, err) + + plan := dryRunPlan(t, url, fmt.Sprintf("ALTER TABLE %s.t ALTER COLUMN id TYPE bigint", schema)) + assert.Equal(t, router.DispositionUnavailable, plan.Disposition) + require.Len(t, plan.Statements, 1) + st := plan.Statements[0] + assert.Equal(t, planner.RouteCopyAndSwap, st.Route) + assert.Equal(t, router.BackendCopyAndSwap, st.Backend) + assert.Empty(t, st.ExecSQL) + + var typ string + require.NoError(t, pool.QueryRow(t.Context(), + `SELECT data_type FROM information_schema.columns + WHERE table_schema = $1 AND table_name = 't' AND column_name = 'id'`, schema).Scan(&typ)) + assert.Equal(t, "integer", typ, "dry-run must not execute the change") +} + +// Live facts feed the imperative dry-run: a widen the classifier can only +// prove with the live column type routes native, and still executes nothing. +func TestMigrateDryRunUsesLiveFacts(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf( + "CREATE TABLE %s.t (id int PRIMARY KEY, name varchar(20))", schema)) + require.NoError(t, err) + + alter := fmt.Sprintf("ALTER TABLE %s.t ALTER COLUMN name TYPE varchar(50)", schema) + plan := dryRunPlan(t, url, alter) + assert.Equal(t, router.DispositionExecute, plan.Disposition) + require.Len(t, plan.Statements, 1) + st := plan.Statements[0] + assert.Equal(t, planner.RouteNative, st.Route) + assert.Equal(t, router.BackendNative, st.Backend) + require.Len(t, st.Decisions, 1) + assert.Equal(t, planner.ReasonBinaryCoercible, st.Decisions[0].Reason, + "the live varchar(20) must be introspected to prove the widen") + assert.Equal(t, []string{alter}, st.ExecSQL) + + var maxLen int + require.NoError(t, pool.QueryRow(t.Context(), + `SELECT character_maximum_length FROM information_schema.columns + WHERE table_schema = $1 AND table_name = 't' AND column_name = 'name'`, schema).Scan(&maxLen)) + assert.Equal(t, 20, maxLen, "dry-run must not execute the change") +} + +// The dry-run advisory covers statements the execute gate refuses: a plain +// CREATE INDEX comes back native with its concurrent rewrite, not created. +func TestMigrateDryRunSuggestsConcurrentIndex(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY)", schema)) + require.NoError(t, err) + + submitted := fmt.Sprintf("CREATE INDEX t_id_idx ON %s.t (id)", schema) + plan := dryRunPlan(t, url, submitted) + assert.Equal(t, router.DispositionExecute, plan.Disposition) + require.Len(t, plan.Statements, 1) + st := plan.Statements[0] + assert.Equal(t, planner.RouteNative, st.Route) + require.Len(t, st.Decisions, 1) + assert.Equal(t, planner.ReasonSaferIdiom, st.Decisions[0].Reason) + require.Len(t, st.ExecSQL, 1) + assert.NotEqual(t, submitted, st.ExecSQL[0], "the plan carries the concurrent rewrite") + + var indexes int + require.NoError(t, pool.QueryRow(t.Context(), + `SELECT count(*) FROM pg_indexes WHERE schemaname = $1 AND indexname = 't_id_idx'`, + schema).Scan(&indexes)) + assert.Equal(t, 0, indexes, "dry-run must not create the index") +} + +// A dry-run against a table that does not exist classifies with zero facts: +// the unprovable type change routes conservatively instead of failing. +func TestMigrateDryRunMissingTableIsConservative(t *testing.T) { + url := testutil.StartPostgres(t) + + plan := dryRunPlan(t, url, "ALTER TABLE missing ALTER COLUMN v TYPE varchar(50)") + assert.Equal(t, router.DispositionUnavailable, plan.Disposition) + require.Len(t, plan.Statements, 1) + assert.Equal(t, planner.RouteCopyAndSwap, plan.Statements[0].Route) +} diff --git a/internal/cli/migrate.go b/internal/cli/migrate.go index e43227c..1bdbf48 100644 --- a/internal/cli/migrate.go +++ b/internal/cli/migrate.go @@ -17,8 +17,12 @@ import ( // run is the migrate flow: gate the statement type, size-guard the table, // attempt the change under budget, and end in exactly one verdict. Refusal // verdicts are printed to out and returned as verdict.ErrRefused so the entry -// point maps them to the refusal exit code. +// point maps them to the refusal exit code. --dry-run diverts to the +// classify-and-route plan instead. func (c *MigrateCmd) run(ctx context.Context, out io.Writer) error { + if c.DryRun { + return c.runDryRun(ctx, out) + } logger := c.diag() st, err := statement.ParseOne(c.Alter) if err != nil { diff --git a/pkg/router/router.go b/pkg/router/router.go new file mode 100644 index 0000000..642086a --- /dev/null +++ b/pkg/router/router.go @@ -0,0 +1,137 @@ +// Package router assigns every classified statement to an execution +// backend. It sits between the planner (which decides what must change and +// how PostgreSQL would treat it) and the executors (which decide how a +// change actually runs), and is the single place migration policy lives: +// which backends exist, which are available, and what happens to a +// statement whose backend is not built yet. Callers branch on the typed +// Backend and Disposition, never on prose. +// +// This is a periphery package (see SAFETY.md): a routed plan is a request. +// Executors enforce their own protections regardless of the route. +package router + +import "github.com/block/pg-sprite/pkg/planner" + +// Backend identifies an execution strategy. +type Backend string + +// The backends the router can assign. A refused statement has no backend. +const ( + // BackendNative runs the change as direct PostgreSQL DDL (the safer + // online idiom when the planner constructed one) under bounded + // lock_timeout / statement_timeout. + BackendNative Backend = "native" + // BackendCopyAndSwap performs the change as a shadow-table copy with a + // logical-replication catch-up and a locked cutover swap. + BackendCopyAndSwap Backend = "copy-and-swap" +) + +// available reports whether the backend is implemented in this build. This +// is the routing policy for backends: copy-and-swap is a known strategy the +// planner routes to, but until its executor exists the router marks the +// statement unavailable rather than pretending it could run. +func available(b Backend) bool { + return b == BackendNative +} + +// Disposition is what would happen to one statement if the routed plan were +// executed now. +type Disposition string + +// The dispositions a routed statement can carry. +const ( + // DispositionExecute: the assigned backend is available; the statement + // would run. + DispositionExecute Disposition = "execute" + // DispositionUnavailable: the change needs a backend this build does + // not implement; the statement would be refused at execution. + DispositionUnavailable Disposition = "unavailable" + // DispositionRefuse: the planner refused the statement; no backend is + // assigned. + DispositionRefuse Disposition = "refuse" +) + +// worse orders dispositions for aggregation: refuse > unavailable > execute. +func worse(a, b Disposition) Disposition { + rank := map[Disposition]int{DispositionExecute: 0, DispositionUnavailable: 1, DispositionRefuse: 2} + if rank[b] > rank[a] { + return b + } + return a +} + +// Statement is one routed statement: the planner's classification plus the +// backend assignment and the literal SQL the native backend would execute. +type Statement struct { + planner.Plan + // Backend is the assigned execution strategy; empty for refusals. + Backend Backend `json:"backend,omitempty"` + // Disposition is what execution would do with the statement now. + Disposition Disposition `json:"disposition"` + // ExecSQL is the ordered SQL the native backend would run: the + // planner's safer sequence when it constructed one, otherwise the + // submitted statement. Empty for non-native routes. + ExecSQL []string `json:"exec_sql,omitempty"` +} + +// Plan is the routed plan for an ordered statement list: one routed +// statement per input plan plus the aggregate disposition (the worst of its +// statements — one unavailable backend makes the whole plan unavailable, +// one refusal refuses it). +type Plan struct { + // Statements are the routed statements, in input order. + Statements []Statement `json:"statements"` + // Disposition is the aggregate disposition. + Disposition Disposition `json:"disposition"` +} + +// Route assigns a backend to every classified statement. It is pure policy: +// no parsing, no database access — classification happens before, execution +// after. +func Route(plans []planner.Plan) Plan { + routed := Plan{Statements: make([]Statement, 0, len(plans)), Disposition: DispositionExecute} + for _, p := range plans { + st := routeStatement(p) + routed.Disposition = worse(routed.Disposition, st.Disposition) + routed.Statements = append(routed.Statements, st) + } + return routed +} + +// routeStatement maps one classified statement to its backend and +// disposition. +func routeStatement(p planner.Plan) Statement { + st := Statement{Plan: p} + switch p.Route { + case planner.RouteNative: + st.Backend = BackendNative + st.ExecSQL = nativeExecSQL(p) + case planner.RouteCopyAndSwap: + st.Backend = BackendCopyAndSwap + case planner.RouteRefuse: + st.Disposition = DispositionRefuse + return st + default: + // An unknown route is a planner/router version skew; refuse it + // rather than guess a backend. + st.Disposition = DispositionRefuse + return st + } + if !available(st.Backend) { + st.Disposition = DispositionUnavailable + return st + } + st.Disposition = DispositionExecute + return st +} + +// nativeExecSQL is the literal SQL the native backend would run for a +// native-routed statement: the planner's safer sequence when it constructed +// one (only single-operation statements carry one), otherwise the submitted +// form. +func nativeExecSQL(p planner.Plan) []string { + if len(p.Decisions) == 1 && len(p.Decisions[0].SaferSQL) > 0 { + return p.Decisions[0].SaferSQL + } + return []string{p.Statement} +} diff --git a/pkg/router/router_test.go b/pkg/router/router_test.go new file mode 100644 index 0000000..47d23d6 --- /dev/null +++ b/pkg/router/router_test.go @@ -0,0 +1,93 @@ +package router_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/block/pg-sprite/pkg/planner" + "github.com/block/pg-sprite/pkg/router" +) + +// classify runs the real classifier so routing tests exercise the same +// plans the CLI produces, not hand-built ones. +func classify(t *testing.T, sql string) planner.Plan { + t.Helper() + plan, err := planner.Classify(sql, planner.Facts{}) + require.NoError(t, err) + return plan +} + +func TestRouteNativeExecutesSubmittedForm(t *testing.T) { + plan := classify(t, "ALTER TABLE t ADD COLUMN age int DEFAULT 0") + routed := router.Route([]planner.Plan{plan}) + + require.Len(t, routed.Statements, 1) + st := routed.Statements[0] + assert.Equal(t, router.BackendNative, st.Backend) + assert.Equal(t, router.DispositionExecute, st.Disposition) + assert.Equal(t, []string{plan.Statement}, st.ExecSQL, + "a native statement with no safer sequence executes as submitted") + assert.Equal(t, router.DispositionExecute, routed.Disposition) +} + +func TestRouteNativeExecutesSaferSequence(t *testing.T) { + plan := classify(t, "CREATE INDEX events_name_idx ON events (name)") + require.Len(t, plan.Decisions, 1) + require.NotEmpty(t, plan.Decisions[0].SaferSQL, "classifier must construct the concurrent rewrite") + + routed := router.Route([]planner.Plan{plan}) + st := routed.Statements[0] + assert.Equal(t, router.BackendNative, st.Backend) + assert.Equal(t, router.DispositionExecute, st.Disposition) + assert.Equal(t, plan.Decisions[0].SaferSQL, st.ExecSQL, + "the native backend runs the safer sequence, not the submitted form") +} + +func TestRouteCopyAndSwapIsUnavailable(t *testing.T) { + plan := classify(t, "ALTER TABLE t ALTER COLUMN id TYPE bigint") + require.Equal(t, planner.RouteCopyAndSwap, plan.Route) + + routed := router.Route([]planner.Plan{plan}) + st := routed.Statements[0] + assert.Equal(t, router.BackendCopyAndSwap, st.Backend) + assert.Equal(t, router.DispositionUnavailable, st.Disposition) + assert.Empty(t, st.ExecSQL, "no literal SQL for a backend that is not implemented") + assert.Equal(t, router.DispositionUnavailable, routed.Disposition) +} + +func TestRouteRefusedStatementHasNoBackend(t *testing.T) { + plan := classify(t, "ALTER TABLE t ADD CONSTRAINT ex EXCLUDE USING gist (room WITH =)") + require.Equal(t, planner.RouteRefuse, plan.Route) + + routed := router.Route([]planner.Plan{plan}) + st := routed.Statements[0] + assert.Empty(t, st.Backend) + assert.Equal(t, router.DispositionRefuse, st.Disposition) + assert.Empty(t, st.ExecSQL) + assert.Equal(t, router.DispositionRefuse, routed.Disposition) +} + +func TestRouteAggregateIsWorstDisposition(t *testing.T) { + plans := []planner.Plan{ + classify(t, "ALTER TABLE t ADD COLUMN age int"), + classify(t, "ALTER TABLE t ALTER COLUMN id TYPE bigint"), + classify(t, "ALTER TABLE t ADD CONSTRAINT ex EXCLUDE USING gist (room WITH =)"), + } + routed := router.Route(plans) + + require.Len(t, routed.Statements, 3) + assert.Equal(t, router.DispositionExecute, routed.Statements[0].Disposition) + assert.Equal(t, router.DispositionUnavailable, routed.Statements[1].Disposition) + assert.Equal(t, router.DispositionRefuse, routed.Statements[2].Disposition) + assert.Equal(t, router.DispositionRefuse, routed.Disposition, + "one refusal refuses the whole plan") +} + +func TestRouteEmptyPlanExecutes(t *testing.T) { + routed := router.Route(nil) + assert.Empty(t, routed.Statements) + assert.Equal(t, router.DispositionExecute, routed.Disposition, + "a plan with nothing to do has nothing blocking execution") +} From d4e45d439131ea51dffa7c0986ae1c9e6f435a5e Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Wed, 5 Aug 2026 19:20:41 +1000 Subject: [PATCH 18/24] ci: pin golangci-lint-action and lint binary version The action's default binary is built with an older Go than the module targets and cannot load the v2 config; SHA-pinning also satisfies the semgrep and zizmor unpinned-action checks. --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fc3def..0bbc1c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,7 +60,11 @@ jobs: - uses: actions/setup-go@v5 with: go-version-file: go.mod - - uses: golangci/golangci-lint-action@v6 + # Pin the same golangci-lint major used locally (v2 config format); + # the action's default binary lags and cannot load a v2 config. + - uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 + with: + version: v2.12.2 build: needs: changes From 70d274e86d0b2daec47a01ebb78cdd066ac215bc Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Wed, 5 Aug 2026 19:20:48 +1000 Subject: [PATCH 19/24] ci: pin golangci-lint-action and lint binary version The action's default binary is built with an older Go than the module targets and cannot load the v2 config; SHA-pinning also satisfies the semgrep and zizmor unpinned-action checks. --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4fc3def..0bbc1c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,7 +60,11 @@ jobs: - uses: actions/setup-go@v5 with: go-version-file: go.mod - - uses: golangci/golangci-lint-action@v6 + # Pin the same golangci-lint major used locally (v2 config format); + # the action's default binary lags and cannot load a v2 config. + - uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 + with: + version: v2.12.2 build: needs: changes From 8ed1454ce09d6401df2365947f197edc5103c2ab Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Fri, 7 Aug 2026 07:00:39 +1000 Subject: [PATCH 20/24] docs: port reviewed SchemaBot AGENTS.md conventions Addresses the scoped agent review on PR #4: git/PR workflow section, identifier-quoting and DSN-handling rules, schema-change terminology, flake-verification script, and TM-9 (operation must outlive the observer). --- .agents/checks/review.md | 6 ++++++ AGENTS.md | 28 +++++++++++++++++++++++++++- docs/README.md | 2 +- docs/testing.md | 17 +++++++++++++++-- scripts/test-flaky.sh | 27 +++++++++++++++++++++++++++ 5 files changed, 76 insertions(+), 4 deletions(-) create mode 100755 scripts/test-flaky.sh diff --git a/.agents/checks/review.md b/.agents/checks/review.md index cda08af..4635d39 100644 --- a/.agents/checks/review.md +++ b/.agents/checks/review.md @@ -23,6 +23,12 @@ the reviewer's distillation. raw `pgx` pools in production code. - SQL parsing goes through `pg_query_go`; flag `strings.Split(";")` or any hand-parsing. A parse failure is an error surfaced to the caller. +- Generated SQL quotes every user-supplied or introspected identifier + (`pgx.Identifier{...}.Sanitize()` / `quote_ident()`) — flag raw interpolation of names into + SQL. Connection strings are parsed and re-serialized (`pgx.ParseConfig`), never + string-manipulated. +- Terminology: "schema change", not "migration", in code, CLI output, error messages, and new + docs — flag new occurrences except citations of external sources. - Errors: wrapped with context and identifiers; no log-and-continue, no silent branch cases, no discarded `Close()` errors, no `nolint`, no `--no-verify`. No panics in library code — invariant violations return `ErrInvariantViolation` fail-closed. Postgres errors are matched diff --git a/AGENTS.md b/AGENTS.md index 9a1c55e..9bb974d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,15 +34,23 @@ make lint # golangci-lint convergence oracle) are the `TM-*` registry in the same doc. - Always run the full `make test` when the scope of a change is unclear. - Never assume a test failure is unrelated to your change; investigate it. -- Never increase timeouts to fix flakes; find the root cause. +- Never increase timeouts to fix flakes; find the root cause — then prove the fix holds with + `scripts/test-flaky.sh [iterations] [package]` before declaring it fixed. - Integration tests run against real PostgreSQL (testcontainers); `PG_VERSION` selects the major (default 16), CI runs the matrix 14 → 18. Core logic is validated against a real database — no mocked-DB tests for core logic. ## Conventions +- Say **"schema change"**, not "migration", in code, CLI output, error messages, and new docs — + pg-sprite strings surface through orchestrators that ban "migration". Use "migration" only + when citing external sources (Spirit's `pkg/migration`, peer tools, PostgreSQL docs). - Use `pkg/dbconn` for connections — never raw `pgx` pools in production code (tests excepted). Every session runs under bounded `lock_timeout` / `statement_timeout`. +- Never build SQL by interpolating raw identifiers: any user-supplied or introspected name in + generated SQL goes through `pgx.Identifier{...}.Sanitize()` (or `quote_ident()` server-side). +- Never string-manipulate connection strings/DSNs — parse (`pgx.ParseConfig`), modify fields, + re-serialize; string ops break on passwords containing `/`, `@`, or `%`. - All SQL parsing goes through `pg_query_go` (once `pkg/statement` exists). No `strings.Split(";")`, no hand-parsing; a parse failure is an error surfaced to the caller. - Tests use testify (`require` for setup, `assert` for verification), `t.Context()` (in @@ -99,6 +107,24 @@ make lint # golangci-lint Mechanical style rules (doc comments on exported symbols, no `init()`, no package-level mutable state, no `context.Context` in structs) are enforced by `.golangci.yml`, not prose. +## Git and PRs + +- Do not create PRs automatically — pushing a branch is fine; opening the PR is the author's + decision. When asked, create PRs as drafts (`gh pr create --draft`); the author marks ready. +- Never squash or rewrite history after a human has reviewed (comments or approval) — add + commits so reviewers can see increments. Squash freely before review. +- Agent disclosure lines (agent name + model) go at the *bottom* of PR bodies and issue + bodies, after the content. +- Never reply to, post on, or resolve *human* review threads without the author's explicit + approval — agents do not speak for the author. Automated reviewer (e.g. Copilot) comments + may be replied to and resolved without separate approval, provided each reply describes the + fix with a commit link (or a reasoned rejection), is prefixed 🤖, and carries the agent + disclosure — resolve only after the reply is posted. +- After pushing new commits, refresh the PR title/summary to match — unless a human has + edited it. +- Upstream large branches with the leaf approach: map the dependency graph, peel off leaf + changes as small independent PRs first, in topological order. + Design docs live in [docs/](docs/) — start at [docs/README.md](docs/README.md); the invariant registry is [docs/invariants.md](docs/invariants.md). diff --git a/docs/README.md b/docs/README.md index fa46b97..17dbd2e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,6 @@ # Online schema change engine for Aurora PostgreSQL -Research and design notes for building an online schema migration engine targeting +Research and design notes for building an online schema-change engine targeting **Aurora PostgreSQL**, by deriving and combining the best practices from established tools — [Spirit](https://github.com/block/spirit) (Aurora MySQL), [pg_osc](https://github.com/shayonj/pg-osc), [pg_repack](https://github.com/reorg/pg_repack), diff --git a/docs/testing.md b/docs/testing.md index 184ed42..91fb224 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -42,7 +42,7 @@ this covers *how*). Each rule binds from the phase noted. ### TM-1 — Lifecycle fixture, not happy-path tests -Every executor/migration integration test drives the **full lifecycle +Every executor/schema-change integration test drives the **full lifecycle through one shared fixture** — start → assert → abort → assert → restart → complete → assert — so interrupted-and-retried is the default tested path, not a special case. Once checkpointing exists, kill → resume joins the @@ -62,7 +62,7 @@ pg-delta's snapshot + roundtrip pairing. Contention tests hold a real `ACCESS EXCLUSIVE` lock from a second connection; cancellation tests use context deadlines. After any injected -failure the test asserts the **durable state** (no wedged migration record, +failure the test asserts the **durable state** (no wedged schema-change record, no leaked shadow objects/slots/triggers) and the ability to proceed — not merely the returned error type. *Binds:* Phase 2 onward; full phase-boundary kill/resume matrix at Phases 4–8. *Source:* pgroll's @@ -108,6 +108,19 @@ corpus — exit codes, output, and resulting database state asserted. *Binds:* Phase 2 (first executing command). *Source:* pgroll `make examples` CI job; pg_repack driving its CLI through `pg_regress`. +### TM-9 — The operation must outlive the observer + +Any test that observes or interrupts an operation **in flight** (progress +polling, kill/resume mid-copy, injected faults between phases) seeds enough +rows that the operation demonstrably spans the observation or injection +point — otherwise the operation can finish before the fault lands and the +test passes without testing anything. Vacuous runs are a failure: the test +asserts the interruption actually hit mid-operation (e.g. the checkpoint +shows partial progress), not just the final state. *Binds:* Phase 4 +(copy-and-swap kill/resume) onward. *Source:* SchemaBot's in-flight +progress tests, which seed large row counts so an operation spans a poll +interval. + **Beyond the peers:** none of the three does generative testing. From Phase 3 we add **seeded schema/DDL generation** (generate desired state → plan → apply → re-diff must be empty), printing the seed on failure and diff --git a/scripts/test-flaky.sh b/scripts/test-flaky.sh new file mode 100755 index 0000000..7443440 --- /dev/null +++ b/scripts/test-flaky.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Proves a flake fix holds by running one test repeatedly. +# +# Usage: scripts/test-flaky.sh [iterations] [package] +# +# scripts/test-flaky.sh TestBuildIndexConcurrently +# scripts/test-flaky.sh TestBuildIndexConcurrently 20 ./pkg/executor/... +# +# Fails fast on the first failing iteration. Environment variables +# (PG_VERSION, PG_DSN, SKIP_INTEGRATION) pass through to `go test`. + +set -euo pipefail + +TEST_NAME="${1:?usage: scripts/test-flaky.sh [iterations] [package]}" +ITERATIONS="${2:-10}" +PACKAGE="${3:-./...}" + +for i in $(seq 1 "$ITERATIONS"); do + echo "=== iteration $i/$ITERATIONS: $TEST_NAME ===" + if ! go test -race -count=1 -run "^${TEST_NAME}$" "$PACKAGE"; then + echo "FAILED on iteration $i/$ITERATIONS" + exit 1 + fi +done + +echo "PASSED all $ITERATIONS iterations" From f44207e840b44be57c0848a8e666ae12f66f50df Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Fri, 7 Aug 2026 08:08:40 +1000 Subject: [PATCH 21/24] Add the two project lenses to AGENTS.md and review checks Every PR, design, and review is judged as (a) an OSS-first standalone Postgres online-DDL tool and (b) a clean SchemaBot engine integration; trading one off against the other needs an explicit decision. --- .agents/checks/review.md | 6 ++++++ AGENTS.md | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/.agents/checks/review.md b/.agents/checks/review.md index 2cb9a49..3968ce3 100644 --- a/.agents/checks/review.md +++ b/.agents/checks/review.md @@ -4,6 +4,12 @@ Checks for AI review agents. The authoritative rules live in [SAFETY.md](../../S [AGENTS.md](../../AGENTS.md), and [docs/tcb-model.md](../../docs/tcb-model.md) — this file is the reviewer's distillation. +- Judge the change through both project lenses: (a) OSS-first — pg-sprite as the preferred + standalone PostgreSQL online-DDL tool (CLI usable without an orchestrator, external-user + docs and errors, no Block-internal assumptions); (b) clean SchemaBot integration — a stable + adapter-friendly seam (library API, verdict/plan JSON, error taxonomy) with the core never + depending on SchemaBot. Flag changes that serve one lens at the other's expense without a + recorded decision. - Look up every touched `pkg/` package in the SAFETY.md partition table first — the review bar differs between the safety-critical core and the periphery. Flag core changes with 🌶️ and state the blast radius (data corruption, lost writes, wrong-table swap, stranded slot). diff --git a/AGENTS.md b/AGENTS.md index 4fa46c4..7aab437 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,6 +7,18 @@ This file is canonical. `CLAUDE.md`, `GEMINI.md`, `.cursorrules`, `.goosehints`, `.github/copilot-instructions.md` are symlinks to it — edit only this file. Review-agent checks live in [.agents/checks/review.md](.agents/checks/review.md). +## Two lenses on every change + +Judge every PR, design, and review through both lenses — a change that serves one at the +expense of the other needs an explicit decision, not a silent trade: + +1. **OSS-first.** pg-sprite aims to be the preferred PostgreSQL online-DDL tool in its own + right: the CLI works standalone with no orchestrator setup, docs and error messages are + written for external users, and nothing assumes a Block-internal environment. +2. **Clean SchemaBot integration.** pg-sprite must slot into SchemaBot as an engine behind a + stable seam: keep the library API, verdict/plan JSON contracts, and error taxonomy + adapter-friendly, and never let the core depend on SchemaBot (or any orchestrator). + ## Read SAFETY.md first This codebase is partitioned into a **safety-critical core** and a periphery. From 11e4a71081f4a91165f202dccbf1c8c62e511391 Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Fri, 7 Aug 2026 08:08:42 +1000 Subject: [PATCH 22/24] Harden the front door per PR #5 reviews MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the two-lens and adversarial review findings: - ST-7: the executor accepts only a ParseOne-constructed Statement whose target matches the preflight proof, so multi-statement SQL and cross-table smuggling are unrepresentable - size guard measures pg_total_relation_size (heap, indexes, TOAST) — the rewrite it fears rebuilds every index under the same lock - sub-millisecond budgets are rejected (they truncate to 0ms, which disables the PostgreSQL limit) - table-targeted RENAME / SET SCHEMA route through the front door instead of being refused as non-ALTER TABLE - status coalesces pg_stat_activity fields nulled for other roles - already-concurrent index forms no longer get circular safer-idiom advice; verdict reasons are flat kebab-case tokens; byte-size parsing rejects int64 overflow; CI builds with CGO_ENABLED=0 --- .github/workflows/ci.yml | 8 ++ SAFETY.md | 4 +- docs/architecture.md | 2 +- docs/invariants.md | 11 +++ docs/testing.md | 5 +- internal/cli/bytesize.go | 4 + internal/cli/bytesize_test.go | 3 +- internal/cli/cli.go | 6 +- internal/cli/migrate.go | 63 +++++++------ internal/cli/migrate_integration_test.go | 69 +++++++++++++++ internal/cli/migrate_test.go | 9 +- internal/cli/status.go | 14 +-- pkg/executor/optimistic.go | 74 ++++++++++++---- pkg/executor/optimistic_integration_test.go | 96 +++++++++++++++++--- pkg/preflight/preflight.go | 25 +++--- pkg/preflight/preflight_integration_test.go | 36 ++++++++ pkg/statement/statement.go | 97 +++++++++++++++++---- pkg/statement/statement_test.go | 93 ++++++++++++++++---- pkg/verdict/verdict.go | 7 +- pkg/verdict/verdict_test.go | 15 ++++ 20 files changed, 518 insertions(+), 123 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bbc1c7..fd6c64a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,6 +76,14 @@ jobs: with: go-version-file: go.mod - run: make build + # The no-cgo promise is a contract: the Wasm parser keeps `go install` + # toolchain-free. One accidental import of the cgo escape hatch + # (pg_query_go's parser) would silently start requiring a C toolchain + # on every contributor's machine. + - name: Build with CGO disabled + run: go build ./... + env: + CGO_ENABLED: "0" # The integration suite runs against every Aurora-supported PostgreSQL # major (see docs/postgresql-version-support.md): the version floor is a diff --git a/SAFETY.md b/SAFETY.md index cc56d6a..b78a1fb 100644 --- a/SAFETY.md +++ b/SAFETY.md @@ -26,7 +26,7 @@ The invariant registry (invariant IDs referenced below) lives in | `pkg/decode` — logical decoding, LSN/position accounting | ✅ core | planned (Phase 6) | ST-4, CO-4 | | `pkg/checkpoint` — durable resume state | ✅ core | planned (Phase 8) | ST-1, ST-2 | | slot lifecycle (in `pkg/decode`) — create, reap, lag ceiling | ✅ core | planned (Phase 8) | ST-3 | -| `pkg/migration` — orchestrator, **cutover swap + fidelity gate** | ✅ core | planned (Phase 7) | LK-2, LK-4, ST-5 | +| `pkg/schemachange` — orchestrator, **cutover swap + fidelity gate** | ✅ core | planned (Phase 7) | LK-2, LK-4, ST-5 | | `pkg/statement`, `pkg/planner`, `pkg/schemadiff`, `pkg/lint` — classify/diff/route | ❌ periphery¹ | `pkg/statement` exists (Phase 1: type gate); rest planned (Phase 2) | (CO-7 holds at the parse boundary) | | `pkg/verdict` — structured outcome contract, rendering, exit codes | ❌ periphery | exists (Phase 1) | — | | `internal/cli` — CLI, flags, help, prompts | ❌ periphery | `migrate`/`status` exist (Phase 1); rest stubs | — | @@ -36,7 +36,7 @@ The invariant registry (invariant IDs referenced below) lives in ¹ **The planner is deliberately outside the core.** Its verdicts are *requests*, not permissions: a wrong "native-safe" verdict is capped by the executor's own `lock_timeout` bound; -a wrong "copy" verdict produces a wasteful but *correct* migration (the checksum still gates). +a wrong "copy" verdict produces a wasteful but *correct* schema change (the checksum still gates). The core executors re-verify their own preconditions and never trust that the planner checked. ## Rules inside the core diff --git a/docs/architecture.md b/docs/architecture.md index e75ff98..ff3b784 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -62,7 +62,7 @@ boundary) is defined in [../SAFETY.md](../SAFETY.md). | `pkg/checksum` | The mandatory correctness gate; continuous checker; repair primitive | Phase 5 | | `pkg/decode` | Logical-decoding change capture, LSN accounting, slot lifecycle | Phase 6, 8 | | `pkg/applier` | Change apply onto the shadow (always wins), buffer/dedup, flush scheduling | Phase 6 | -| `pkg/migration` | Orchestrator: lifecycle, cutover swap + fidelity gate, checkpoint/resume | Phase 7–8 | +| `pkg/schemachange` | Orchestrator: lifecycle, cutover swap + fidelity gate, checkpoint/resume | Phase 7–8 | | `pkg/checkpoint` | Durable single-row resume state | Phase 8 | | `pkg/throttler` | Aurora reader-lag / slot-lag / WAL throttling | Phase 8 | diff --git a/docs/invariants.md b/docs/invariants.md index b132845..955a912 100644 --- a/docs/invariants.md +++ b/docs/invariants.md @@ -227,6 +227,16 @@ before the first write to the target; the server is the semantic authority and c parsing is advisory. *Enforced:* preflight stage. *Source:* [design-principles](design-principles.md#correctness-and-safety). +### ST-7 — The executor runs exactly the statement that was gated + +The executor accepts only a parsed `statement.Statement` — constructible solely by `ParseOne`, +which enforces exactly one statement through the real grammar — and refuses, before anything +executes, any statement whose target table does not match the preflight proof it was handed. +A proof for one table can never smuggle SQL against another, and a multi-statement string can +never reach the database through the executor (pgx's simple protocol would happily run all of +it). *Enforced:* `pkg/executor` (`AttemptNative`), `pkg/statement` (proof construction). +*Source:* adversarial review of the optimistic front door. + ## Refusals and preflight (RF) Each refusal is a preflight **error with a stated reason** — never a warning, never attempted. @@ -318,4 +328,5 @@ about **how we write and review the code**. | LK-4, ST-5 | 7 | dropped-connection cutover, fidelity checklist | | ST-1, ST-2, ST-3, ST-4 | 8 | kill/resume, cross-version refuse, orphan-slot reap, failover reconcile | | ST-6 | 1 onward, complete by 8 | preflight matrix | +| ST-7 | 1 | target-mismatch refusal + single-statement-by-construction tests | | OC-1..OC-6 | shape APIs from 2; bind at 11 | engine-contract tests | diff --git a/docs/testing.md b/docs/testing.md index 91fb224..721d08d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -116,8 +116,9 @@ rows that the operation demonstrably spans the observation or injection point — otherwise the operation can finish before the fault lands and the test passes without testing anything. Vacuous runs are a failure: the test asserts the interruption actually hit mid-operation (e.g. the checkpoint -shows partial progress), not just the final state. *Binds:* Phase 4 -(copy-and-swap kill/resume) onward. *Source:* SchemaBot's in-flight +shows partial progress), not just the final state. *Binds:* Phase 1 +(budget-cancellation fixtures, which seed enough rows that a rewrite cannot +finish inside its statement budget) onward. *Source:* SchemaBot's in-flight progress tests, which seed large row counts so an operation spans a poll interval. diff --git a/internal/cli/bytesize.go b/internal/cli/bytesize.go index dc3c6fc..0e3a501 100644 --- a/internal/cli/bytesize.go +++ b/internal/cli/bytesize.go @@ -2,6 +2,7 @@ package cli import ( "fmt" + "math" "strconv" "strings" ) @@ -47,6 +48,9 @@ func (b *byteSize) UnmarshalText(text []byte) error { if n <= 0 { return fmt.Errorf("size must be positive, got %q", string(text)) } + if n > math.MaxInt64/mult { + return fmt.Errorf("size %q overflows int64 bytes", string(text)) + } *b = byteSize(n * mult) return nil } diff --git a/internal/cli/bytesize_test.go b/internal/cli/bytesize_test.go index e788d9e..0b39eda 100644 --- a/internal/cli/bytesize_test.go +++ b/internal/cli/bytesize_test.go @@ -33,7 +33,8 @@ func TestByteSizeUnmarshal(t *testing.T) { } func TestByteSizeUnmarshalRejectsInvalid(t *testing.T) { - for _, in := range []string{"", "GiB", "1XB", "-5MiB", "0", "1.5GiB"} { + // The last two would overflow int64 bytes after unit multiplication. + for _, in := range []string{"", "GiB", "1XB", "-5MiB", "0", "1.5GiB", "9999999999GiB", "9223372036854775807KiB"} { t.Run(in, func(t *testing.T) { var b byteSize assert.Error(t, b.UnmarshalText([]byte(in))) diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 157a343..b718b7c 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -24,7 +24,7 @@ type CLI struct { Diff DiffCmd `cmd:"" help:"Diff a desired-state schema file against the live schema."` Fmt FmtCmd `cmd:"" help:"Canonicalize a schema file."` Lint LintCmd `cmd:"" help:"Lint DDL for unsafe patterns."` - Status StatusCmd `cmd:"" help:"Report the status of a running migration."` + Status StatusCmd `cmd:"" help:"Report the status of a running schema change."` } // New returns an empty command tree for kong.Parse. @@ -85,7 +85,7 @@ type MigrateCmd struct { DBFlags `embed:""` Alter string `help:"Imperative ALTER statement to run." name:"alter" required:""` - MaxTableSize byteSize `help:"Size threshold above which the optimistic attempt is skipped (binary units: B, KiB, MiB, GiB, TiB)." default:"1GiB"` + MaxTableSize byteSize `help:"Size threshold above which the optimistic attempt is skipped, measured as the table's full on-disk footprint: heap, indexes, and TOAST, all partitions (binary units: B, KiB, MiB, GiB, TiB)." default:"1GiB"` JSON bool `help:"Emit the verdict as JSON."` } @@ -116,7 +116,7 @@ type LintCmd struct{} // Run implements the lint subcommand. func (c *LintCmd) Run() error { return notImplemented("lint") } -// StatusCmd reports migration progress. +// StatusCmd reports schema-change progress. type StatusCmd struct { DBFlags `embed:""` diff --git a/internal/cli/migrate.go b/internal/cli/migrate.go index e2eff93..6566ec7 100644 --- a/internal/cli/migrate.go +++ b/internal/cli/migrate.go @@ -24,7 +24,7 @@ func (c *MigrateCmd) run(ctx context.Context, out io.Writer) error { if err != nil { return err } - logger.Debug("statement parsed", "kind", st.Kind, "schema", st.Schema, "table", st.Table) + logger.Debug("statement parsed", "kind", st.Kind(), "schema", st.Schema(), "table", st.Table()) if v, refused := gateVerdict(st); refused { return c.emit(out, v) } @@ -35,7 +35,7 @@ func (c *MigrateCmd) run(ctx context.Context, out io.Writer) error { } defer pool.Close() - pt, err := preflight.CheckTable(ctx, pool, st.Schema, st.Table, int64(c.MaxTableSize)) + pt, err := preflight.CheckTable(ctx, pool, st.Schema(), st.Table(), int64(c.MaxTableSize)) var sizeErr *preflight.SizeError if errors.As(err, &sizeErr) { return c.emit(out, sizeGuardVerdict(st, sizeErr)) @@ -48,7 +48,7 @@ func (c *MigrateCmd) run(ctx context.Context, out io.Writer) error { budget := executor.Budget{LockTimeout: c.LockTimeout, StatementTimeout: c.StatementTimeout} start := time.Now() - err = executor.AttemptNative(ctx, pool, pt, st.SQL, budget) + err = executor.AttemptNative(ctx, pool, pt, st, budget) elapsed := time.Since(start) var budgetErr *executor.BudgetError if errors.As(err, &budgetErr) { @@ -62,7 +62,7 @@ func (c *MigrateCmd) run(ctx context.Context, out io.Writer) error { logger.Debug("optimistic attempt committed", "table", qualified(st), "elapsed", elapsed) return c.emit(out, verdict.Verdict{ Outcome: verdict.OutcomeExecuted, - Statement: st.SQL, + Statement: st.SQL(), Table: qualified(st), Detail: fmt.Sprintf("committed within budgets (lock %s, statement %s): the change was effectively instant", budget.LockTimeout, budget.StatementTimeout), @@ -92,22 +92,13 @@ func (c *MigrateCmd) emit(out io.Writer, v verdict.Verdict) error { // index maintenance is pointed at its concurrent idiom, everything else is // unsupported. Refused statements are never executed. func gateVerdict(st statement.Statement) (verdict.Verdict, bool) { - v := verdict.Verdict{Outcome: verdict.OutcomeRefused, Statement: st.SQL} - switch st.Kind { + v := verdict.Verdict{Outcome: verdict.OutcomeRefused, Statement: st.SQL()} + switch st.Kind() { case statement.KindAlterTable: return verdict.Verdict{}, false - case statement.KindCreateIndex: - v.Reason = verdict.ReasonIndexStatement - v.Detail = "a plain CREATE INDEX blocks writes for the whole build; the concurrent build does not" - v.SaferIdiom = "CREATE INDEX CONCURRENTLY" - case statement.KindDropIndex: + case statement.KindCreateIndex, statement.KindDropIndex, statement.KindReindex: v.Reason = verdict.ReasonIndexStatement - v.Detail = "a plain DROP INDEX takes ACCESS EXCLUSIVE on the table; the concurrent drop does not" - v.SaferIdiom = "DROP INDEX CONCURRENTLY" - case statement.KindReindex: - v.Reason = verdict.ReasonIndexStatement - v.Detail = "a plain REINDEX blocks writes; the concurrent rebuild does not" - v.SaferIdiom = "REINDEX ... CONCURRENTLY" + v.Detail, v.SaferIdiom = indexAdvice(st) case statement.KindOther: v.Reason = verdict.ReasonUnsupportedStatement v.Detail = "only ALTER TABLE statements are supported by the optimistic front door" @@ -115,17 +106,37 @@ func gateVerdict(st statement.Statement) (verdict.Verdict, bool) { return v, true } +// indexAdvice explains an index-statement refusal. The already-concurrent +// forms carry no safer idiom: suggesting the statement the user submitted +// would confuse a human once and send a resubmitting automation into a loop. +func indexAdvice(st statement.Statement) (detail, saferIdiom string) { + if st.Concurrent() { + return "this is already the safe concurrent idiom; pg-sprite does not drive index maintenance yet — run it directly against the database", "" + } + switch st.Kind() { + case statement.KindCreateIndex: + return "a plain CREATE INDEX blocks writes for the whole build; the concurrent build does not", "CREATE INDEX CONCURRENTLY" + case statement.KindDropIndex: + return "a plain DROP INDEX takes ACCESS EXCLUSIVE on the table; the concurrent drop does not", "DROP INDEX CONCURRENTLY" + case statement.KindReindex: + return "a plain REINDEX blocks writes; the concurrent rebuild does not", "REINDEX ... CONCURRENTLY" + default: + return "", "" + } +} + // sizeGuardVerdict is the refusal for tables above the size threshold, where // even a budget-bounded attempt would visibly stall the table. func sizeGuardVerdict(st statement.Statement, sizeErr *preflight.SizeError) verdict.Verdict { return verdict.Verdict{ Outcome: verdict.OutcomeRefused, Reason: verdict.ReasonTableTooLarge, - Statement: st.SQL, + Statement: st.SQL(), Table: qualified(st), - Detail: fmt.Sprintf("table is %d bytes on disk, above the %d-byte --max-table-size threshold: "+ - "a cancelled attempt is not a free probe — it would hold ACCESS EXCLUSIVE doing rewrite work "+ - "for the whole budget. This change needs a copy-and-swap rewrite, which pg-sprite does not perform yet", + Detail: fmt.Sprintf("table is %d bytes on disk (heap, indexes, and TOAST), above the %d-byte "+ + "--max-table-size threshold. pg-sprite cannot yet prove this change is instant on a table this "+ + "size; if it requires a rewrite, a cancelled attempt is not a free probe — it would hold "+ + "ACCESS EXCLUSIVE doing rewrite work for the whole budget", sizeErr.TotalBytes, sizeErr.LimitBytes), } } @@ -136,7 +147,7 @@ func budgetVerdict(st statement.Statement, budgetErr *executor.BudgetError) verd v := verdict.Verdict{ Outcome: verdict.OutcomeRefused, Reason: verdict.ReasonBudgetExceeded, - Statement: st.SQL, + Statement: st.SQL(), Table: qualified(st), } switch budgetErr.Cause { @@ -159,11 +170,11 @@ func budgetVerdict(st statement.Statement, budgetErr *executor.BudgetError) verd // qualified renders the statement's target table for the verdict, empty when // the statement has none. func qualified(st statement.Statement) string { - if st.Table == "" { + if st.Table() == "" { return "" } - if st.Schema == "" { - return st.Table + if st.Schema() == "" { + return st.Table() } - return st.Schema + "." + st.Table + return st.Schema() + "." + st.Table() } diff --git a/internal/cli/migrate_integration_test.go b/internal/cli/migrate_integration_test.go index 20b2db2..edde1b6 100644 --- a/internal/cli/migrate_integration_test.go +++ b/internal/cli/migrate_integration_test.go @@ -3,6 +3,7 @@ package cli import ( "encoding/json" "fmt" + neturl "net/url" "slices" "strconv" "strings" @@ -58,6 +59,34 @@ func TestMigrateExecutesInstantChange(t *testing.T) { assert.Equal(t, "integer", typ) } +// ALTER TABLE ... RENAME COLUMN parses as a RenameStmt, not an +// AlterTableStmt, but is a table-targeted instant catalog change: the front +// door must route it through, not refuse it as unsupported. +func TestMigrateExecutesRenameColumn(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY, a int)", schema)) + require.NoError(t, err) + + cmd := newMigrateCmd(url, fmt.Sprintf("ALTER TABLE %s.t RENAME COLUMN a TO b", schema)) + cmd.JSON = true + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + + var v verdict.Verdict + require.NoError(t, json.Unmarshal([]byte(out.String()), &v)) + assert.Equal(t, verdict.OutcomeExecuted, v.Outcome) + + var n int + require.NoError(t, pool.QueryRow(t.Context(), + `SELECT count(*) FROM information_schema.columns + WHERE table_schema = $1 AND table_name = 't' AND column_name = 'b'`, schema).Scan(&n)) + assert.Equal(t, 1, n, "the rename must have committed") +} + // Acceptance (ii): a rewrite-requiring change is cancelled, leaves schema and // data unchanged, and returns the not-native-safe verdict with its reason. func TestMigrateRefusesRewriteWithBudgetVerdict(t *testing.T) { @@ -143,6 +172,11 @@ func TestMigrateGateRefusesWithoutDatabase(t *testing.T) { {"create index", "CREATE INDEX i ON t (c)", verdict.ReasonIndexStatement, "CREATE INDEX CONCURRENTLY"}, {"drop index", "DROP INDEX i", verdict.ReasonIndexStatement, "DROP INDEX CONCURRENTLY"}, {"reindex", "REINDEX TABLE t", verdict.ReasonIndexStatement, "REINDEX ... CONCURRENTLY"}, + // The already-concurrent forms carry no safer idiom: suggesting the + // statement the user submitted would loop a resubmitting automation. + {"create index concurrently", "CREATE INDEX CONCURRENTLY i ON t (c)", verdict.ReasonIndexStatement, ""}, + {"drop index concurrently", "DROP INDEX CONCURRENTLY i", verdict.ReasonIndexStatement, ""}, + {"reindex concurrently", "REINDEX TABLE CONCURRENTLY t", verdict.ReasonIndexStatement, ""}, {"alter index", "ALTER INDEX i SET (fillfactor = 90)", verdict.ReasonUnsupportedStatement, ""}, {"create table", "CREATE TABLE t (id int)", verdict.ReasonUnsupportedStatement, ""}, } @@ -251,6 +285,41 @@ func TestStatusReportsNoSessions(t *testing.T) { assert.NotEmpty(t, text.String()) } +// pg_stat_activity nulls out state and query for other roles' backends when +// the viewer lacks pg_read_all_stats — the read-only-operator shape. status +// must render those sessions, not crash the scan. +func TestStatusHandlesOtherRolesSessions(t *testing.T) { + superURL := testutil.StartPostgres(t) + superPool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: superURL}) + require.NoError(t, err) + defer superPool.Close() + + _, err = superPool.Exec(t.Context(), "CREATE ROLE limited LOGIN PASSWORD 'limited-test-only'") + require.NoError(t, err) + + // Hold a superuser pg-sprite session open on a pinned connection so + // pg_stat_activity is guaranteed to contain a foreign-role row while + // status runs. + conn, err := superPool.Acquire(t.Context()) + require.NoError(t, err) + defer conn.Release() + var pid int + require.NoError(t, conn.QueryRow(t.Context(), "SELECT pg_backend_pid()").Scan(&pid)) + + u, err := neturl.Parse(superURL) + require.NoError(t, err) + u.User = neturl.UserPassword("limited", "limited-test-only") + + cmd := &StatusCmd{DBFlags: DBFlags{URL: u.String()}, JSON: true} + var out strings.Builder + require.NoError(t, cmd.run(t.Context(), &out)) + + var sessions []session + require.NoError(t, json.Unmarshal([]byte(out.String()), &sessions)) + i := slices.IndexFunc(sessions, func(s session) bool { return s.PID == pid }) + require.GreaterOrEqual(t, i, 0, "the other role's session must be listed, not crash the scan") +} + // A live pg-sprite session (any connection made through pkg/dbconn) is // reported with its pid and per-session fields. The session is held open on a // pinned connection so pg_stat_activity is guaranteed to contain it while diff --git a/internal/cli/migrate_test.go b/internal/cli/migrate_test.go index 6016687..bd0df77 100644 --- a/internal/cli/migrate_test.go +++ b/internal/cli/migrate_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/block/pg-sprite/pkg/executor" "github.com/block/pg-sprite/pkg/statement" @@ -12,12 +13,8 @@ import ( ) func TestBudgetVerdict(t *testing.T) { - st := statement.Statement{ - SQL: "ALTER TABLE billing.invoices ALTER COLUMN id TYPE bigint", - Kind: statement.KindAlterTable, - Schema: "billing", - Table: "invoices", - } + st, err := statement.ParseOne("ALTER TABLE billing.invoices ALTER COLUMN id TYPE bigint") + require.NoError(t, err) t.Run("lock budget", func(t *testing.T) { v := budgetVerdict(st, &executor.BudgetError{Cause: executor.CauseLock, Budget: 3 * time.Second}) diff --git a/internal/cli/status.go b/internal/cli/status.go index 27bddff..21b8106 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -21,7 +21,7 @@ type session struct { } // run reports the engine's live database sessions. Phase 1 has no durable -// migration state — a change either committed within its budgets or was +// schema-change state — a change either committed within its budgets or was // refused — so status is a view over pg_stat_activity for pg-sprite sessions. func (c *StatusCmd) run(ctx context.Context, out io.Writer) error { pool, err := dbconn.NewPool(ctx, c.Config()) @@ -48,13 +48,17 @@ func (c *StatusCmd) run(ctx context.Context, out io.Writer) error { } // querySessions lists the live pg-sprite backends other than the one running -// the status query itself. +// the status query itself. pg_stat_activity nulls out state and query for +// other roles' backends unless the viewer has pg_read_all_stats — exactly +// the read-only-operator-checking-on-the-engine-role shape — so those +// columns are coalesced instead of crashing the scan. func querySessions(ctx context.Context, pool *pgxpool.Pool) ([]session, error) { rows, err := pool.Query(ctx, ` - SELECT pid, state, + SELECT pid, + COALESCE(state, '-'), COALESCE(wait_event_type || '/' || wait_event, '-'), COALESCE(now() - query_start, '0'::interval)::text, - left(query, 80) + COALESCE(left(query, 80), '') FROM pg_stat_activity WHERE application_name = 'pg-sprite' AND pid <> pg_backend_pid() ORDER BY query_start`) @@ -80,7 +84,7 @@ func querySessions(ctx context.Context, pool *pgxpool.Pool) ([]session, error) { // renderSessions writes the human-readable session listing. func renderSessions(out io.Writer, sessions []session) error { if len(sessions) == 0 { - if _, err := fmt.Fprintln(out, "no active pg-sprite sessions (Phase 1 keeps no durable migration state: a change either committed within its budgets or was refused)"); err != nil { + if _, err := fmt.Fprintln(out, "no active pg-sprite sessions (Phase 1 keeps no durable schema-change state: a change either committed within its budgets or was refused)"); err != nil { return fmt.Errorf("write status: %w", err) } return nil diff --git a/pkg/executor/optimistic.go b/pkg/executor/optimistic.go index 16bccae..911b350 100644 --- a/pkg/executor/optimistic.go +++ b/pkg/executor/optimistic.go @@ -6,8 +6,11 @@ // BudgetError the caller turns into a not-native-safe verdict. // // This is a safety-critical core package: see SAFETY.md. It never trusts the -// caller's classification — its own protection is the budget, applied with -// SET LOCAL inside the attempt's transaction regardless of session defaults. +// caller's classification — its own protections are the budget, applied with +// SET LOCAL inside the attempt's transaction regardless of session defaults, +// and the statement binding: it accepts only a parsed statement.Statement +// (exactly one statement by construction) whose target matches the +// preflighted table (invariant ST-7). package executor import ( @@ -21,8 +24,14 @@ import ( "github.com/jackc/pgx/v5/pgxpool" "github.com/block/pg-sprite/pkg/preflight" + "github.com/block/pg-sprite/pkg/statement" ) +// ErrInvariantViolation is the fail-closed error class for breaches of the +// registry in docs/invariants.md (see SAFETY.md); the message names the +// invariant ID. It is never a warning and never retried. +var ErrInvariantViolation = errors.New("invariant violation") + // SQLSTATE codes the attempt maps to budget outcomes. Postgres errors are // matched by SQLSTATE, never by message text. const ( @@ -70,8 +79,9 @@ func (e *BudgetError) Error() string { return fmt.Sprintf("optimistic attempt exceeded its %s (%s) and was cancelled", e.Cause, e.Budget) } -// Budget bounds one optimistic attempt. Both limits must be positive: an -// unbounded attempt is exactly the stall the front door exists to prevent. +// Budget bounds one optimistic attempt. Both limits must be at least +// minBudget: an unbounded attempt is exactly the stall the front door exists +// to prevent. type Budget struct { // LockTimeout bounds how long the attempt may wait in the lock queue. LockTimeout time.Duration @@ -79,29 +89,46 @@ type Budget struct { StatementTimeout time.Duration } -// validate rejects budgets that would leave the attempt unbounded. +// minBudget is the smallest expressible budget: budgets are applied to +// PostgreSQL in whole milliseconds, and a sub-millisecond value would +// truncate to 0 — which disables the corresponding limit entirely. +const minBudget = time.Millisecond + +// validate rejects budgets that would leave the attempt unbounded. Rejecting +// a sub-millisecond budget is more fail-closed than silently rounding it up. func (b Budget) validate() error { - // INV: LK-2 — the attempt is bounded by construction; a zero or negative - // timeout would disable the corresponding PostgreSQL limit. - if b.LockTimeout <= 0 { - return fmt.Errorf("lock budget must be positive, got %s", b.LockTimeout) + // INV: LK-2 — the attempt is bounded by construction; a zero, negative, + // or sub-millisecond timeout would disable the corresponding PostgreSQL + // limit after millisecond truncation. + if b.LockTimeout < minBudget { + return fmt.Errorf("lock budget must be at least %s, got %s", minBudget, b.LockTimeout) } - if b.StatementTimeout <= 0 { - return fmt.Errorf("statement budget must be positive, got %s", b.StatementTimeout) + if b.StatementTimeout < minBudget { + return fmt.Errorf("statement budget must be at least %s, got %s", minBudget, b.StatementTimeout) } return nil } -// AttemptNative runs sql once, directly, inside a transaction bounded by b. -// The table must have passed preflight — the proof parameter makes skipping -// the size guard unrepresentable. On success the change is committed: it was -// effectively instant. If a budget is exceeded the statement is cancelled by -// the server, the transaction rolls back, and a *BudgetError is returned. -// Any other failure is surfaced as an operational error. -func AttemptNative(ctx context.Context, pool *pgxpool.Pool, _ preflight.PreflightedTable, sql string, b Budget) error { +// AttemptNative runs st once, directly, inside a transaction bounded by b. +// The table must have passed preflight and the statement must target it — +// both proofs make the unsafe call unrepresentable: a statement.Statement +// can only come from ParseOne (exactly one statement, parsed by the real +// grammar), and a target mismatch is refused before anything executes, so a +// proof for one table cannot smuggle SQL against another. On success the +// change is committed: it was effectively instant. If a budget is exceeded +// the statement is cancelled by the server, the transaction rolls back, and +// a *BudgetError is returned. Any other failure is surfaced as an +// operational error. +func AttemptNative(ctx context.Context, pool *pgxpool.Pool, pt preflight.PreflightedTable, st statement.Statement, b Budget) error { if err := b.validate(); err != nil { return err } + // INV: ST-7 — the executor runs exactly the statement that was gated, + // and only against the table the preflight proof verified. + if st.Table() == "" || st.Schema() != pt.Schema() || st.Table() != pt.Table() { + return fmt.Errorf("%w: ST-7: statement targets %q but preflight verified %q", + ErrInvariantViolation, qualifiedName(st.Schema(), st.Table()), qualifiedName(pt.Schema(), pt.Table())) + } tx, err := pool.Begin(ctx) if err != nil { return fmt.Errorf("begin optimistic attempt: %w", err) @@ -124,7 +151,7 @@ func AttemptNative(ctx context.Context, pool *pgxpool.Pool, _ preflight.Prefligh return fmt.Errorf("set attempt budgets: %w", err) } - if _, err := tx.Exec(ctx, sql); err != nil { + if _, err := tx.Exec(ctx, st.SQL()); err != nil { if budgetErr := asBudgetError(err, b); budgetErr != nil { return budgetErr } @@ -136,6 +163,15 @@ func AttemptNative(ctx context.Context, pool *pgxpool.Pool, _ preflight.Prefligh return nil } +// qualifiedName renders schema.table for error messages, omitting the dot +// when the name is unqualified. +func qualifiedName(schema, table string) string { + if schema == "" { + return table + } + return schema + "." + table +} + // asBudgetError maps a PostgreSQL error to the budget it exceeded, or nil // when the error is not a budget overrun. func asBudgetError(err error, b Budget) *BudgetError { diff --git a/pkg/executor/optimistic_integration_test.go b/pkg/executor/optimistic_integration_test.go index 1f9ccd9..e277b80 100644 --- a/pkg/executor/optimistic_integration_test.go +++ b/pkg/executor/optimistic_integration_test.go @@ -14,6 +14,7 @@ import ( "github.com/block/pg-sprite/pkg/dbconn" "github.com/block/pg-sprite/pkg/executor" "github.com/block/pg-sprite/pkg/preflight" + "github.com/block/pg-sprite/pkg/statement" ) // budget is generous enough that an instant catalog change always fits and @@ -35,6 +36,13 @@ func mustPreflight(t *testing.T, pool *pgxpool.Pool, schema, table string) prefl return pt } +func mustParse(t *testing.T, sql string) statement.Statement { + t.Helper() + st, err := statement.ParseOne(sql) + require.NoError(t, err) + return st +} + func columnType(t *testing.T, pool *pgxpool.Pool, schema, table, column string) string { t.Helper() var typ string @@ -52,8 +60,8 @@ func TestAttemptNativeCommitsInstantChange(t *testing.T) { require.NoError(t, err) pt := mustPreflight(t, pool, schema, "t") - sql := fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int NOT NULL DEFAULT 0", schema) - require.NoError(t, executor.AttemptNative(t.Context(), pool, pt, sql, budget)) + st := mustParse(t, fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int NOT NULL DEFAULT 0", schema)) + require.NoError(t, executor.AttemptNative(t.Context(), pool, pt, st, budget)) assert.Equal(t, "integer", columnType(t, pool, schema, "t", "age"), "the committed change must be visible") } @@ -74,8 +82,8 @@ func TestAttemptNativeCancelsWhenLockBlocked(t *testing.T) { _, err = blocker.Exec(t.Context(), fmt.Sprintf("LOCK TABLE %s.t IN ACCESS EXCLUSIVE MODE", schema)) require.NoError(t, err) - sql := fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int", schema) - err = executor.AttemptNative(t.Context(), pool, pt, sql, budget) + st := mustParse(t, fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int", schema)) + err = executor.AttemptNative(t.Context(), pool, pt, st, budget) var budgetErr *executor.BudgetError require.ErrorAs(t, err, &budgetErr) @@ -95,9 +103,9 @@ func TestAttemptNativeCancelsRewriteAndLeavesTableUnchanged(t *testing.T) { pt := mustPreflight(t, pool, schema, "t") // int -> bigint forces a full table rewrite under ACCESS EXCLUSIVE. - sql := fmt.Sprintf("ALTER TABLE %s.t ALTER COLUMN id TYPE bigint", schema) + st := mustParse(t, fmt.Sprintf("ALTER TABLE %s.t ALTER COLUMN id TYPE bigint", schema)) tight := executor.Budget{LockTimeout: budget.LockTimeout, StatementTimeout: 50 * time.Millisecond} - err = executor.AttemptNative(t.Context(), pool, pt, sql, tight) + err = executor.AttemptNative(t.Context(), pool, pt, st, tight) var budgetErr *executor.BudgetError require.ErrorAs(t, err, &budgetErr) @@ -119,20 +127,86 @@ func TestAttemptNativeSurfacesOperationalErrors(t *testing.T) { // Dropping a column that does not exist is a plain SQL error, not a // budget overrun. - sql := fmt.Sprintf("ALTER TABLE %s.t DROP COLUMN nope", schema) - err = executor.AttemptNative(t.Context(), pool, pt, sql, budget) + st := mustParse(t, fmt.Sprintf("ALTER TABLE %s.t DROP COLUMN nope", schema)) + err = executor.AttemptNative(t.Context(), pool, pt, st, budget) require.Error(t, err) var budgetErr *executor.BudgetError assert.NotErrorAs(t, err, &budgetErr) } +// Sub-millisecond budgets are as unbounded as zero ones: they truncate to +// PostgreSQL's 0ms, which disables the corresponding limit entirely. func TestAttemptNativeRejectsUnboundedBudgets(t *testing.T) { pool, schema := newPool(t) _, err := pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY)", schema)) require.NoError(t, err) pt := mustPreflight(t, pool, schema, "t") - sql := fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int", schema) - require.Error(t, executor.AttemptNative(t.Context(), pool, pt, sql, executor.Budget{LockTimeout: 0, StatementTimeout: time.Second})) - require.Error(t, executor.AttemptNative(t.Context(), pool, pt, sql, executor.Budget{LockTimeout: time.Second, StatementTimeout: 0})) + st := mustParse(t, fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN age int", schema)) + unbounded := map[string]executor.Budget{ + "zero lock": {LockTimeout: 0, StatementTimeout: time.Second}, + "zero statement": {LockTimeout: time.Second, StatementTimeout: 0}, + "sub-millisecond lock": {LockTimeout: 500 * time.Microsecond, StatementTimeout: time.Second}, + "sub-millisecond statement": {LockTimeout: time.Second, StatementTimeout: 999 * time.Microsecond}, + } + for name, b := range unbounded { + t.Run(name, func(t *testing.T) { + require.Error(t, executor.AttemptNative(t.Context(), pool, pt, st, b)) + }) + } + + // The smallest representable budget is valid and does not serialize to 0: + // a 1ms lock timeout must still cancel a blocked attempt rather than + // disabling the limit. + blocker, err := pool.Begin(t.Context()) + require.NoError(t, err) + t.Cleanup(func() { + require.NoError(t, blocker.Rollback(context.WithoutCancel(t.Context()))) + }) + _, err = blocker.Exec(t.Context(), fmt.Sprintf("LOCK TABLE %s.t IN ACCESS EXCLUSIVE MODE", schema)) + require.NoError(t, err) + err = executor.AttemptNative(t.Context(), pool, pt, st, executor.Budget{LockTimeout: time.Millisecond, StatementTimeout: time.Second}) + var budgetErr *executor.BudgetError + require.ErrorAs(t, err, &budgetErr) + assert.Equal(t, executor.CauseLock, budgetErr.Cause) +} + +// INV: ST-7 — a preflight proof for one table can never execute a statement +// against another, and a statement without a table target never executes. +func TestAttemptNativeRefusesTargetMismatch(t *testing.T) { + pool, schema := newPool(t) + for _, ddl := range []string{ + fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY)", schema), + fmt.Sprintf("CREATE TABLE %s.victim (id int PRIMARY KEY)", schema), + } { + _, err := pool.Exec(t.Context(), ddl) + require.NoError(t, err) + } + pt := mustPreflight(t, pool, schema, "t") + + t.Run("statement targets a different table", func(t *testing.T) { + st := mustParse(t, fmt.Sprintf("ALTER TABLE %s.victim ADD COLUMN a int", schema)) + err := executor.AttemptNative(t.Context(), pool, pt, st, budget) + require.ErrorIs(t, err, executor.ErrInvariantViolation) + + var n int + require.NoError(t, pool.QueryRow(t.Context(), + `SELECT count(*) FROM information_schema.columns + WHERE table_schema = $1 AND table_name = 'victim' AND column_name = 'a'`, schema).Scan(&n)) + assert.Zero(t, n, "the refused statement must never reach the database") + }) + + t.Run("unqualified statement does not match a qualified proof", func(t *testing.T) { + // Fail-closed: the proof verified schema.t, the statement names a + // bare t that search_path could resolve elsewhere. + st := mustParse(t, "ALTER TABLE t ADD COLUMN a int") + err := executor.AttemptNative(t.Context(), pool, pt, st, budget) + require.ErrorIs(t, err, executor.ErrInvariantViolation) + }) + + t.Run("statement without a table target", func(t *testing.T) { + st := mustParse(t, "CREATE TABLE elsewhere (id int)") + err := executor.AttemptNative(t.Context(), pool, pt, st, budget) + require.ErrorIs(t, err, executor.ErrInvariantViolation) + }) } diff --git a/pkg/preflight/preflight.go b/pkg/preflight/preflight.go index e8b64dd..59bed7c 100644 --- a/pkg/preflight/preflight.go +++ b/pkg/preflight/preflight.go @@ -31,7 +31,7 @@ var ErrNotTable = errors.New("not an ordinary or partitioned table") // operational failure. type SizeError struct { // TotalBytes is the table's measured on-disk size (all partitions, - // including TOAST). + // including indexes and TOAST). TotalBytes int64 // LimitBytes is the threshold that was exceeded. LimitBytes int64 @@ -59,7 +59,8 @@ func (t PreflightedTable) Schema() string { return t.schema } // Table returns the verified table name. func (t PreflightedTable) Table() string { return t.table } -// TotalBytes returns the measured on-disk size across all partitions. +// TotalBytes returns the measured on-disk size across all partitions, +// including indexes and TOAST. func (t PreflightedTable) TotalBytes() int64 { return t.totalBytes } // RelTuples returns the planner's row estimate (-1 when the table has never @@ -75,18 +76,22 @@ func CheckTable(ctx context.Context, pool *pgxpool.Pool, schema, table string, l if limitBytes <= 0 { return PreflightedTable{}, fmt.Errorf("size limit must be positive, got %d", limitBytes) } - // INV: ST-6 — size facts are measured on-disk bytes (pg_table_size, - // which includes TOAST), summed over pg_partition_tree so a partitioned - // parent (whose own relation is 0 bytes) cannot fail open. Stale planner - // statistics (relpages) are never the guard's authority. + // INV: ST-6 — size facts are measured on-disk bytes + // (pg_total_relation_size: heap, indexes, and TOAST — the rewrite the + // guard fears rebuilds every index under the same ACCESS EXCLUSIVE + // lock, so an index-heavy table must not sail under the threshold), + // summed over pg_partition_tree so a partitioned parent (whose own + // relation is 0 bytes) cannot fail open. Stale planner statistics + // (relpages) are never the guard's authority. // The table's own size plus every descendant in its partition tree: // pg_partition_tree returns no rows for a plain table (its own - // pg_table_size carries the total) and the parent's own relation is 0 - // bytes for a partitioned table (the descendants carry the total). + // pg_total_relation_size carries the total) and the parent's own + // relation is 0 bytes for a partitioned table (the descendants carry + // the total). const q = ` SELECT c.relkind::text, - pg_table_size(c.oid) + - (SELECT COALESCE(sum(pg_table_size(p.relid)), 0) + pg_total_relation_size(c.oid) + + (SELECT COALESCE(sum(pg_total_relation_size(p.relid)), 0) FROM pg_partition_tree(c.oid) p WHERE p.relid <> c.oid), c.reltuples diff --git a/pkg/preflight/preflight_integration_test.go b/pkg/preflight/preflight_integration_test.go index c91ad9a..8e444e2 100644 --- a/pkg/preflight/preflight_integration_test.go +++ b/pkg/preflight/preflight_integration_test.go @@ -75,6 +75,42 @@ func TestCheckTableSumsPartitions(t *testing.T) { require.ErrorAs(t, err, &sizeErr, "a populated partitioned table must exceed a 1-byte limit") } +// The rewrite the guard fears rebuilds every index under the same ACCESS +// EXCLUSIVE lock, so the measured footprint must include index bytes: a +// heavily indexed table exceeds a threshold its heap alone would fit under. +func TestCheckTableCountsIndexBytes(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY, a text, b text)", schema)) + require.NoError(t, err) + _, err = pool.Exec(t.Context(), fmt.Sprintf( + "INSERT INTO %s.t SELECT g, md5(g::text), md5((g+1)::text) FROM generate_series(1, 20000) g", schema)) + require.NoError(t, err) + for _, idx := range []string{ + fmt.Sprintf("CREATE INDEX ON %s.t (a, b)", schema), + fmt.Sprintf("CREATE INDEX ON %s.t (b, a)", schema), + } { + _, err = pool.Exec(t.Context(), idx) + require.NoError(t, err) + } + + var heap, total int64 + require.NoError(t, pool.QueryRow(t.Context(), + "SELECT pg_table_size($1::regclass), pg_total_relation_size($1::regclass)", + schema+".t").Scan(&heap, &total)) + require.Greater(t, total, heap, "the fixture's indexes must add measurable bytes") + + // A limit the heap alone would fit under must still refuse, and the + // reported size must be the full footprint. + _, err = preflight.CheckTable(t.Context(), pool, schema, "t", heap) + var sizeErr *preflight.SizeError + require.ErrorAs(t, err, &sizeErr) + assert.Equal(t, total, sizeErr.TotalBytes) +} + func TestCheckTableMissingTable(t *testing.T) { pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) require.NoError(t, err) diff --git a/pkg/statement/statement.go b/pkg/statement/statement.go index 45b2697..6e1fed4 100644 --- a/pkg/statement/statement.go +++ b/pkg/statement/statement.go @@ -8,6 +8,7 @@ package statement import ( "errors" "fmt" + "strings" pganalyze "github.com/pganalyze/pg_query_go/v6" pgquery "github.com/wasilibs/go-pgquery" @@ -42,21 +43,37 @@ func (k Kind) String() string { } } -// Statement is one parsed SQL statement plus the facts the gate needs. +// Statement is one parsed SQL statement plus the facts the gate needs. It can +// only be constructed by ParseOne, so holding one proves the SQL parsed as +// exactly one statement through the PostgreSQL grammar — the proof the +// executor requires before running anything (invariant ST-7). type Statement struct { - // SQL is the original statement text as submitted. - SQL string - // Kind is the statement-type bucket. - Kind Kind - // Schema is the target table's schema qualification for ALTER TABLE - // statements; empty when the statement was unqualified (search_path - // resolves it) or when the kind has no single table target. - Schema string - // Table is the target table name for ALTER TABLE statements; empty for - // other kinds. - Table string + sql string + kind Kind + schema string + table string + concurrent bool } +// SQL returns the original statement text as submitted. +func (s Statement) SQL() string { return s.sql } + +// Kind returns the statement-type bucket. +func (s Statement) Kind() Kind { return s.kind } + +// Schema returns the target table's schema qualification for ALTER TABLE +// statements; empty when the statement was unqualified (search_path resolves +// it) or when the kind has no single table target. +func (s Statement) Schema() string { return s.schema } + +// Table returns the target table name for ALTER TABLE statements; empty for +// other kinds. +func (s Statement) Table() string { return s.table } + +// Concurrent reports whether an index statement used its CONCURRENTLY form. +// It is always false for non-index kinds. +func (s Statement) Concurrent() bool { return s.concurrent } + // ErrNotOneStatement is returned by ParseOne when the input does not contain // exactly one SQL statement. var ErrNotOneStatement = errors.New("input must contain exactly one SQL statement") @@ -71,7 +88,7 @@ func ParseOne(sql string) (Statement, error) { if n := len(tree.GetStmts()); n != 1 { return Statement{}, fmt.Errorf("%w: got %d", ErrNotOneStatement, n) } - st := Statement{SQL: sql} + st := Statement{sql: sql} node := tree.GetStmts()[0].GetStmt() switch { case node.GetAlterTableStmt() != nil: @@ -81,17 +98,59 @@ func ParseOne(sql string) (Statement, error) { if alter.GetObjtype() != pganalyze.ObjectType_OBJECT_TABLE { return st, nil } - st.Kind = KindAlterTable - st.Schema = alter.GetRelation().GetSchemaname() - st.Table = alter.GetRelation().GetRelname() + st.kind = KindAlterTable + st.schema = alter.GetRelation().GetSchemaname() + st.table = alter.GetRelation().GetRelname() + case node.GetRenameStmt() != nil: + // ALTER TABLE ... RENAME TO / RENAME COLUMN / RENAME CONSTRAINT + // parse as RenameStmt, not AlterTableStmt. Only table-targeted + // renames are KindAlterTable: RENAME TO carries OBJECT_TABLE as the + // rename type, RENAME COLUMN carries it as the relation type, and + // RENAME CONSTRAINT carries the table-specific OBJECT_TABCONSTRAINT. + // ALTER INDEX/VIEW ... RENAME carry their own object types and stay + // KindOther. + ren := node.GetRenameStmt() + if ren.GetRenameType() != pganalyze.ObjectType_OBJECT_TABLE && + ren.GetRenameType() != pganalyze.ObjectType_OBJECT_TABCONSTRAINT && + ren.GetRelationType() != pganalyze.ObjectType_OBJECT_TABLE { + return st, nil + } + st.kind = KindAlterTable + st.schema = ren.GetRelation().GetSchemaname() + st.table = ren.GetRelation().GetRelname() + case node.GetAlterObjectSchemaStmt() != nil: + // ALTER TABLE ... SET SCHEMA parses as AlterObjectSchemaStmt; only + // the table-targeted form is KindAlterTable. + move := node.GetAlterObjectSchemaStmt() + if move.GetObjectType() != pganalyze.ObjectType_OBJECT_TABLE { + return st, nil + } + st.kind = KindAlterTable + st.schema = move.GetRelation().GetSchemaname() + st.table = move.GetRelation().GetRelname() case node.GetIndexStmt() != nil: - st.Kind = KindCreateIndex + st.kind = KindCreateIndex + st.concurrent = node.GetIndexStmt().GetConcurrent() case node.GetDropStmt() != nil: if node.GetDropStmt().GetRemoveType() == pganalyze.ObjectType_OBJECT_INDEX { - st.Kind = KindDropIndex + st.kind = KindDropIndex + st.concurrent = node.GetDropStmt().GetConcurrent() } case node.GetReindexStmt() != nil: - st.Kind = KindReindex + st.kind = KindReindex + st.concurrent = reindexConcurrently(node.GetReindexStmt()) } return st, nil } + +// reindexConcurrently reports whether a REINDEX statement used its +// CONCURRENTLY form, which the grammar carries as a generic option rather +// than a dedicated field. +func reindexConcurrently(stmt *pganalyze.ReindexStmt) bool { + for _, p := range stmt.GetParams() { + if strings.EqualFold(p.GetDefElem().GetDefname(), "concurrently") { + return true + } + } + return false +} diff --git a/pkg/statement/statement_test.go b/pkg/statement/statement_test.go index b61c264..f55f826 100644 --- a/pkg/statement/statement_test.go +++ b/pkg/statement/statement_test.go @@ -16,74 +16,129 @@ func TestParseOneKinds(t *testing.T) { { name: "alter table unqualified", sql: "ALTER TABLE users ADD COLUMN age int", - want: Statement{Kind: KindAlterTable, Table: "users"}, + want: Statement{kind: KindAlterTable, table: "users"}, }, { name: "alter table schema-qualified", sql: "ALTER TABLE billing.invoices DROP COLUMN note", - want: Statement{Kind: KindAlterTable, Schema: "billing", Table: "invoices"}, + want: Statement{kind: KindAlterTable, schema: "billing", table: "invoices"}, }, { name: "alter table quoted mixed-case identifier", sql: `ALTER TABLE "Order Items" ADD COLUMN qty int`, - want: Statement{Kind: KindAlterTable, Table: "Order Items"}, + want: Statement{kind: KindAlterTable, table: "Order Items"}, }, { name: "alter table if exists", sql: "ALTER TABLE IF EXISTS users ADD COLUMN age int", - want: Statement{Kind: KindAlterTable, Table: "users"}, + want: Statement{kind: KindAlterTable, table: "users"}, + }, + { + name: "alter table rename to parses as RenameStmt but is a table target", + sql: "ALTER TABLE users RENAME TO users_old", + want: Statement{kind: KindAlterTable, table: "users"}, + }, + { + name: "alter table rename column", + sql: "ALTER TABLE users RENAME COLUMN a TO b", + want: Statement{kind: KindAlterTable, table: "users"}, + }, + { + name: "alter table rename constraint", + sql: "ALTER TABLE users RENAME CONSTRAINT users_pk TO users_pkey", + want: Statement{kind: KindAlterTable, table: "users"}, + }, + { + name: "alter table rename schema-qualified", + sql: "ALTER TABLE billing.users RENAME COLUMN a TO b", + want: Statement{kind: KindAlterTable, schema: "billing", table: "users"}, + }, + { + name: "alter table set schema parses as AlterObjectSchemaStmt but is a table target", + sql: "ALTER TABLE users SET SCHEMA archive", + want: Statement{kind: KindAlterTable, table: "users"}, + }, + { + name: "alter table owner to", + sql: "ALTER TABLE users OWNER TO app_owner", + want: Statement{kind: KindAlterTable, table: "users"}, + }, + { + name: "alter view rename is not a table target", + sql: "ALTER VIEW v RENAME TO w", + want: Statement{kind: KindOther}, + }, + { + name: "alter index rename is not a table target", + sql: "ALTER INDEX i RENAME TO j", + want: Statement{kind: KindOther}, + }, + { + name: "alter sequence set schema is not a table target", + sql: "ALTER SEQUENCE s SET SCHEMA archive", + want: Statement{kind: KindOther}, }, { name: "create index", sql: "CREATE INDEX idx_users_email ON users (email)", - want: Statement{Kind: KindCreateIndex}, + want: Statement{kind: KindCreateIndex}, }, { - name: "create unique index concurrently is still an index statement", + name: "create unique index concurrently is a concurrent index statement", sql: "CREATE UNIQUE INDEX CONCURRENTLY idx ON users (email)", - want: Statement{Kind: KindCreateIndex}, + want: Statement{kind: KindCreateIndex, concurrent: true}, }, { name: "drop index", sql: "DROP INDEX idx_users_email", - want: Statement{Kind: KindDropIndex}, + want: Statement{kind: KindDropIndex}, + }, + { + name: "drop index concurrently", + sql: "DROP INDEX CONCURRENTLY idx_users_email", + want: Statement{kind: KindDropIndex, concurrent: true}, }, { name: "reindex table", sql: "REINDEX TABLE users", - want: Statement{Kind: KindReindex}, + want: Statement{kind: KindReindex}, }, { name: "reindex index", sql: "REINDEX INDEX idx_users_email", - want: Statement{Kind: KindReindex}, + want: Statement{kind: KindReindex}, + }, + { + name: "reindex table concurrently", + sql: "REINDEX TABLE CONCURRENTLY users", + want: Statement{kind: KindReindex, concurrent: true}, }, { name: "alter index parses as AlterTableStmt but is not a table target", sql: "ALTER INDEX idx_users_email SET (fillfactor = 90)", - want: Statement{Kind: KindOther}, + want: Statement{kind: KindOther}, }, { name: "drop table is not a drop-index", sql: "DROP TABLE users", - want: Statement{Kind: KindOther}, + want: Statement{kind: KindOther}, }, { name: "create table", sql: "CREATE TABLE t (id int)", - want: Statement{Kind: KindOther}, + want: Statement{kind: KindOther}, }, { name: "dml", sql: "UPDATE users SET age = 1", - want: Statement{Kind: KindOther}, + want: Statement{kind: KindOther}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := ParseOne(tt.sql) require.NoError(t, err) - tt.want.SQL = tt.sql + tt.want.sql = tt.sql assert.Equal(t, tt.want, got) }) } @@ -100,6 +155,14 @@ func TestParseOneRejectsMultipleStatements(t *testing.T) { require.ErrorIs(t, err, ErrNotOneStatement) } +// A second statement smuggled behind a legitimate ALTER never yields a +// Statement at all — the executor only accepts what ParseOne constructs, so +// multi-statement SQL is unrepresentable downstream (invariant ST-7). +func TestParseOneRejectsSmuggledStatement(t *testing.T) { + _, err := ParseOne("ALTER TABLE t ADD COLUMN a int; DROP TABLE victim") + require.ErrorIs(t, err, ErrNotOneStatement) +} + func TestParseOneRejectsEmptyInput(t *testing.T) { _, err := ParseOne("") require.ErrorIs(t, err, ErrNotOneStatement) diff --git a/pkg/verdict/verdict.go b/pkg/verdict/verdict.go index 928d6f4..676093e 100644 --- a/pkg/verdict/verdict.go +++ b/pkg/verdict/verdict.go @@ -33,7 +33,8 @@ const ( OutcomeRefused Outcome = "refused" ) -// Reason is the typed cause of a refusal. +// Reason is the typed cause of a refusal. Reasons are flat kebab-case +// tokens — they are what automation switches on; prose belongs in Detail. type Reason string // The refusal reasons Phase 1 can emit. @@ -46,10 +47,10 @@ const ( // (CONCURRENTLY) and is never attempted here. ReasonIndexStatement Reason = "index-statement" // ReasonTableTooLarge: the size guard skipped the optimistic attempt. - ReasonTableTooLarge Reason = "not-native-safe: table too large" + ReasonTableTooLarge Reason = "not-native-safe-table-too-large" // ReasonBudgetExceeded: the optimistic attempt exceeded its lock or // statement budget and was cancelled. - ReasonBudgetExceeded Reason = "not-native-safe: budget exceeded" + ReasonBudgetExceeded Reason = "not-native-safe-budget-exceeded" ) // Cause narrows ReasonBudgetExceeded to the budget that was exceeded, so diff --git a/pkg/verdict/verdict_test.go b/pkg/verdict/verdict_test.go index 44ba1cc..73dacae 100644 --- a/pkg/verdict/verdict_test.go +++ b/pkg/verdict/verdict_test.go @@ -33,6 +33,21 @@ func TestJSONOmitsEmptyOptionalFields(t *testing.T) { assert.NotContains(t, s, "safer_idiom") } +// Reason and Cause values are the machine contract automation switches on: +// flat kebab-case tokens, no spaces or colons — prose belongs in Detail. +func TestReasonAndCauseTokensAreFlat(t *testing.T) { + for _, tok := range []string{ + string(ReasonUnsupportedStatement), + string(ReasonIndexStatement), + string(ReasonTableTooLarge), + string(ReasonBudgetExceeded), + string(CauseLockBudget), + string(CauseStatementBudget), + } { + assert.Regexp(t, `^[a-z0-9]+(-[a-z0-9]+)*$`, tok) + } +} + func TestStringExecuted(t *testing.T) { s := Verdict{ Outcome: OutcomeExecuted, From 4e24b46f4445a3f9159ef53521a70b9a92baab8e Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Fri, 7 Aug 2026 09:20:44 +1000 Subject: [PATCH 23/24] Address PR #6 review: FK refusal, serial adoption, change kinds, fmt comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes from the two-lens and adversarial reviews at 14c9dee: - refuse REFERENCES at desired-file admission (typed ErrForeignKey) — the scratch transaction cannot faithfully bind an unqualified FK - refuse serial/sequence-backed default adoption (ErrUnsupportedChange) via pg_depend-backed Column.SequenceDefault — the plan would name a sequence that only existed in the rolled-back scratch transaction - classify every Change with a typed Kind; mark DROP INDEX destructive - annotate lock-hazardous statements in the text plan and point the header at migrate as the executing front door - fmt fails closed on commented input (ErrCommentLoss) instead of silently discarding comments - document column-order-by-name convergence and the two-canon boundary --- docs/low-level-design.md | 19 +++- internal/cli/diff.go | 48 +++++++++- internal/cli/diff_integration_test.go | 9 ++ internal/cli/diff_test.go | 14 +++ pkg/schemadiff/diff.go | 92 ++++++++++++++++--- pkg/schemadiff/diff_test.go | 71 ++++++++++++++ pkg/schemadiff/introspect.go | 11 ++- pkg/schemadiff/schemadiff.go | 6 ++ pkg/schemadiff/schemadiff_integration_test.go | 49 ++++++++++ pkg/statement/comments.go | 31 +++++++ pkg/statement/comments_test.go | 31 +++++++ pkg/statement/desired.go | 35 ++++++- pkg/statement/desired_test.go | 3 + pkg/statement/statement.go | 7 ++ 14 files changed, 403 insertions(+), 23 deletions(-) create mode 100644 pkg/statement/comments.go create mode 100644 pkg/statement/comments_test.go diff --git a/docs/low-level-design.md b/docs/low-level-design.md index e27ed12..32bc1c1 100644 --- a/docs/low-level-design.md +++ b/docs/low-level-design.md @@ -249,15 +249,28 @@ live schema (introspected) ─┘ │ 4. **Diff** the two models and emit the minimal set of statements: `ADD/DROP/ALTER COLUMN`, `ADD/DROP CONSTRAINT`, `CREATE/DROP INDEX`, default/nullability changes, etc., in a **dependency-correct order** (e.g. add a column before an index that references it). + Columns are compared **by name**: a live table whose columns are ordered differently from + the desired file converges to "no changes". Attribute order carries no semantics in + PostgreSQL and cannot be changed in place, so — unlike some declarative MySQL tooling — + column order is deliberately out of scope for convergence. 5. **Hand the derived statements to the same classifier**, so a declarative change that turns out to be, say, a binary-coercible type widening still takes the native fast path, and only a genuine rewrite triggers a copy. ### Safety rules (inherited philosophy: surprise-free, decisions-not-options) -- **Destructive diffs are gated.** Dropping a column or constraint, or anything that loses - data, requires an explicit confirmation flag — never inferred silently from "it's missing in - the desired file". +- **Destructive diffs are gated.** Dropping a column, constraint, or index — anything that + loses data or a guarantee (a unique index discards the same uniqueness guarantee as a unique + constraint) — requires an explicit confirmation flag — never inferred silently from "it's + missing in the desired file". +- **Unsupported constructs are refused, never guessed.** The desired file admits one + unqualified `CREATE TABLE` plus `CREATE INDEX` statements on it; each rule is a typed error. + Foreign keys are refused at admission — a `REFERENCES` clause cannot be faithfully executed + in the transaction-scoped scratch schema (an unqualified reference resolves against the + scratch search_path, not the target schema), and FK support needs its own design. Changes + the plan cannot express — identity or generation changes on an existing column, adopting a + sequence-backed (serial) default whose sequence only existed in the rolled-back scratch + transaction — are refused as unsupported rather than emitted as an unexecutable plan. - **Renames are ambiguous and are not guessed.** A column present in live but absent in desired plus a new column in desired is, by default, a *drop + add*, not a rename. Rename intent must be stated explicitly (the engine will not heuristically pair columns), mirroring Spirit's diff --git a/internal/cli/diff.go b/internal/cli/diff.go index 46be182..a7e2556 100644 --- a/internal/cli/diff.go +++ b/internal/cli/diff.go @@ -88,7 +88,11 @@ func qualifiedDesired(ds statement.DesiredSchema, schema string) ([]schemadiff.C if err != nil { return nil, fmt.Errorf("qualify desired statement: %w", err) } - changes = append(changes, schemadiff.Change{SQL: qualified}) + kind := schemadiff.ChangeCreateTable + if st.Kind() == statement.KindCreateIndex { + kind = schemadiff.ChangeCreateIndex + } + changes = append(changes, schemadiff.Change{SQL: qualified, Kind: kind}) } return changes, nil } @@ -107,9 +111,11 @@ func writeJSON(out io.Writer, report diffReport) error { } // writePlanText emits the plan as an executable SQL script: one statement -// per line, destructive statements flagged with a leading comment line, and -// SQL comments for the no-change and missing-table cases so the output stays -// valid SQL. +// per line, destructive and lock-hazardous statements flagged with leading +// comment lines, and SQL comments for the no-change and missing-table cases +// so the output stays valid SQL. The header points at migrate as the +// executing front door: running this script directly bypasses the gate that +// refuses blocking statements. func writePlanText(out io.Writer, report diffReport) error { if len(report.Changes) == 0 { if _, err := fmt.Fprintln(out, "-- no changes: live table matches the desired schema"); err != nil { @@ -117,6 +123,12 @@ func writePlanText(out io.Writer, report diffReport) error { } return nil } + if _, err := fmt.Fprintln(out, "-- plan derived by pg-sprite diff; execute statements via pg-sprite migrate,"); err != nil { + return fmt.Errorf("write plan: %w", err) + } + if _, err := fmt.Fprintln(out, "-- which refuses blocking forms — running this script directly bypasses that gate"); err != nil { + return fmt.Errorf("write plan: %w", err) + } if !report.TableExists { if _, err := fmt.Fprintf(out, "-- table %s.%s does not exist; the plan is the full desired schema\n", report.Schema, report.Table); err != nil { @@ -129,6 +141,11 @@ func writePlanText(out io.Writer, report diffReport) error { return fmt.Errorf("write plan: %w", err) } } + if hazard := lockHazard(ch.Kind); hazard != "" { + if _, err := fmt.Fprintf(out, "-- %s\n", hazard); err != nil { + return fmt.Errorf("write plan: %w", err) + } + } if _, err := fmt.Fprintf(out, "%s;\n", ch.SQL); err != nil { return fmt.Errorf("write plan: %w", err) } @@ -136,9 +153,29 @@ func writePlanText(out io.Writer, report diffReport) error { return nil } +// lockHazard describes the blocking behavior of a change kind, empty when +// the statement is not expected to block writers. This is presentation for +// the text plan; the machine contract is the kind itself. +func lockHazard(kind schemadiff.ChangeKind) string { + switch kind { + case schemadiff.ChangeAlterType: + return "rewrites the table under ACCESS EXCLUSIVE, blocking reads and writes" + case schemadiff.ChangeSetNotNull: + return "full table scan under ACCESS EXCLUSIVE" + case schemadiff.ChangeAddConstraint: + return "validation scan or index build that blocks writes" + case schemadiff.ChangeCreateIndex: + return "blocks writes for the whole index build" + default: + return "" + } +} + // runFmt canonicalizes a desired-state schema file: every statement is // parsed through the PostgreSQL grammar, admitted by the same rules as diff, // and printed back in the deparser's canonical form. Offline — no database. +// Commented input is refused (statement.ErrCommentLoss): the parser drops +// comments, and a formatter must never silently discard content. func (c *FmtCmd) runFmt(in io.Reader, out io.Writer) error { var src []byte var err error @@ -149,6 +186,9 @@ func (c *FmtCmd) runFmt(in io.Reader, out io.Writer) error { } else if src, err = os.ReadFile(c.Path); err != nil { return fmt.Errorf("read schema file: %w", err) } + if err := statement.CheckNoComments(string(src)); err != nil { + return err + } ds, err := statement.ParseDesired(string(src)) if err != nil { return err diff --git a/internal/cli/diff_integration_test.go b/internal/cli/diff_integration_test.go index da8c88a..7d1f85c 100644 --- a/internal/cli/diff_integration_test.go +++ b/internal/cli/diff_integration_test.go @@ -14,6 +14,7 @@ import ( "github.com/block/pg-sprite/internal/testutil" "github.com/block/pg-sprite/pkg/dbconn" + "github.com/block/pg-sprite/pkg/schemadiff" ) // newDiffCmd builds a DiffCmd with the flag defaults kong would apply, @@ -57,9 +58,11 @@ func TestDiffPrintsOrderedPlanJSON(t *testing.T) { assert.True(t, report.TableExists) var sqls []string + var kinds []schemadiff.ChangeKind var destructive []bool for _, ch := range report.Changes { sqls = append(sqls, ch.SQL) + kinds = append(kinds, ch.Kind) destructive = append(destructive, ch.Destructive) } assert.Equal(t, []string{ @@ -68,6 +71,12 @@ func TestDiffPrintsOrderedPlanJSON(t *testing.T) { fmt.Sprintf(`ALTER TABLE "%s"."events" ALTER COLUMN "name" SET NOT NULL`, schema), fmt.Sprintf("CREATE INDEX events_name_idx ON %s.events USING btree (name)", schema), }, sqls) + assert.Equal(t, []schemadiff.ChangeKind{ + schemadiff.ChangeDropColumn, + schemadiff.ChangeAlterType, + schemadiff.ChangeSetNotNull, + schemadiff.ChangeCreateIndex, + }, kinds) assert.Equal(t, []bool{true, false, false, false}, destructive) } diff --git a/internal/cli/diff_test.go b/internal/cli/diff_test.go index bcbcf4f..35a6bab 100644 --- a/internal/cli/diff_test.go +++ b/internal/cli/diff_test.go @@ -33,6 +33,20 @@ func TestFmtRefusesDisallowedStatements(t *testing.T) { assert.Empty(t, out.String(), "nothing is written when the input is refused") } +func TestFmtRefusesCommentedInput(t *testing.T) { + cmd := &FmtCmd{} + var out strings.Builder + err := cmd.runFmt(strings.NewReader(`-- events: one row per business event +CREATE TABLE events ( + id bigint PRIMARY KEY, + name varchar(50) NOT NULL -- display name +); +-- covering index for the dashboard query +CREATE INDEX events_name_idx ON events (name);`), &out) + require.ErrorIs(t, err, statement.ErrCommentLoss) + assert.Empty(t, out.String(), "a formatter must never emit output that lost content") +} + func TestFmtRefusesInvalidSQL(t *testing.T) { cmd := &FmtCmd{} var out strings.Builder diff --git a/pkg/schemadiff/diff.go b/pkg/schemadiff/diff.go index 4d60366..896d636 100644 --- a/pkg/schemadiff/diff.go +++ b/pkg/schemadiff/diff.go @@ -18,13 +18,50 @@ var ErrUnsupportedChange = errors.New("unsupported schema change") // tables — a caller bug, refused rather than diffed. var ErrDifferentTables = errors.New("models describe different tables") +// ChangeKind classifies a derived statement so a consumer can gate whole +// classes of change (destructive, rewriting, index-building) without +// parsing SQL. +type ChangeKind string + +// The change kinds a plan can contain. +const ( + // ChangeCreateTable creates the table (missing-table plans only). + ChangeCreateTable ChangeKind = "create-table" + // ChangeDropIndex drops an index. + ChangeDropIndex ChangeKind = "drop-index" + // ChangeDropConstraint drops a table constraint. + ChangeDropConstraint ChangeKind = "drop-constraint" + // ChangeDropColumn drops a column. + ChangeDropColumn ChangeKind = "drop-column" + // ChangeAddColumn adds a column. + ChangeAddColumn ChangeKind = "add-column" + // ChangeAlterType changes a column's type. + ChangeAlterType ChangeKind = "alter-type" + // ChangeSetDefault sets or replaces a column default. + ChangeSetDefault ChangeKind = "set-default" + // ChangeDropDefault drops a column default. + ChangeDropDefault ChangeKind = "drop-default" + // ChangeSetNotNull adds the NOT NULL attribute. + ChangeSetNotNull ChangeKind = "set-not-null" + // ChangeDropNotNull removes the NOT NULL attribute. + ChangeDropNotNull ChangeKind = "drop-not-null" + // ChangeAddConstraint adds a table constraint. + ChangeAddConstraint ChangeKind = "add-constraint" + // ChangeCreateIndex creates an index. + ChangeCreateIndex ChangeKind = "create-index" +) + // Change is one derived statement of the ordered plan. type Change struct { // SQL is the literal statement, without a trailing semicolon. SQL string `json:"sql"` - // Destructive marks statements that discard data or constraints - // (column and constraint drops). Destructive changes are gated by the - // caller, never executed silently. + // Kind classifies the statement for consumers that gate by class. + Kind ChangeKind `json:"kind"` + // Destructive marks statements that discard data, constraints, or + // indexes (column, constraint, and index drops — dropping a unique + // index discards the same guarantee as dropping a unique constraint). + // Destructive changes are gated by the caller, never executed + // silently. Destructive bool `json:"destructive,omitempty"` } @@ -34,7 +71,9 @@ type Change struct { // creates — so an added column exists before an index or constraint that // references it. Within each bucket the order is deterministic: attribute // order for columns, name order for constraints and indexes. schema -// qualifies the emitted statements' table references. +// qualifies the emitted statements' table references. Columns are compared +// by name only: attribute order carries no semantics in PostgreSQL and is +// deliberately out of scope for convergence. func Diff(schema string, live, desired Model) ([]Change, error) { if live.Table != desired.Table { return nil, fmt.Errorf("%w: %q vs %q", ErrDifferentTables, live.Table, desired.Table) @@ -56,7 +95,9 @@ func Diff(schema string, live, desired Model) ([]Change, error) { want, ok := desiredIdx[ix.Name] if !ok || want.Def != ix.Def { changes = append(changes, Change{ - SQL: "DROP INDEX " + pgx.Identifier{schema, ix.Name}.Sanitize(), + SQL: "DROP INDEX " + pgx.Identifier{schema, ix.Name}.Sanitize(), + Kind: ChangeDropIndex, + Destructive: true, }) } } @@ -67,6 +108,7 @@ func Diff(schema string, live, desired Model) ([]Change, error) { if !ok || want.Def != con.Def { changes = append(changes, Change{ SQL: "ALTER TABLE " + table + " DROP CONSTRAINT " + pgx.Identifier{con.Name}.Sanitize(), + Kind: ChangeDropConstraint, Destructive: true, }) } @@ -79,16 +121,24 @@ func Diff(schema string, live, desired Model) ([]Change, error) { if _, ok := desiredCols[col.Name]; !ok { changes = append(changes, Change{ SQL: "ALTER TABLE " + table + " DROP COLUMN " + pgx.Identifier{col.Name}.Sanitize(), + Kind: ChangeDropColumn, Destructive: true, }) } } - // Columns to add. + // Columns to add. A sequence-backed default (serial) cannot be added: + // the desired-side sequence existed only inside the rolled-back + // scratch transaction, so the emitted default would reference a + // relation the plan never creates. for _, col := range desired.Columns { if _, ok := liveCols[col.Name]; !ok { + if col.SequenceDefault { + return nil, fmt.Errorf("%w: column %q has a sequence-backed default (serial); the plan cannot create its sequence", ErrUnsupportedChange, col.Name) + } changes = append(changes, Change{ - SQL: "ALTER TABLE " + table + " ADD COLUMN " + columnDef(col), + SQL: "ALTER TABLE " + table + " ADD COLUMN " + columnDef(col), + Kind: ChangeAddColumn, }) } } @@ -111,7 +161,8 @@ func Diff(schema string, live, desired Model) ([]Change, error) { had, ok := liveCons[con.Name] if !ok || had.Def != con.Def { changes = append(changes, Change{ - SQL: "ALTER TABLE " + table + " ADD CONSTRAINT " + pgx.Identifier{con.Name}.Sanitize() + " " + con.Def, + SQL: "ALTER TABLE " + table + " ADD CONSTRAINT " + pgx.Identifier{con.Name}.Sanitize() + " " + con.Def, + Kind: ChangeAddConstraint, }) } } @@ -126,7 +177,7 @@ func Diff(schema string, live, desired Model) ([]Change, error) { if err != nil { return nil, fmt.Errorf("qualify index %s: %w", ix.Name, err) } - changes = append(changes, Change{SQL: qualified}) + changes = append(changes, Change{SQL: qualified, Kind: ChangeCreateIndex}) } } @@ -150,28 +201,41 @@ func alterColumnChanges(table string, live, desired Column) ([]Change, error) { var changes []Change if live.Type != desired.Type { changes = append(changes, Change{ - SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " TYPE " + desired.Type, + SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " TYPE " + desired.Type, + Kind: ChangeAlterType, }) } if !desired.Generated && desired.Identity == IdentityNone && live.Default != desired.Default { + // A sequence-backed desired default (serial adoption) is refused: + // the sequence existed only inside the rolled-back scratch + // transaction, so the emitted SET DEFAULT would reference a + // relation the plan never creates — or worse, silently bind to an + // unrelated live sequence of the same name. + if desired.SequenceDefault { + return nil, fmt.Errorf("%w: column %q would adopt a sequence-backed default (serial); the plan cannot create its sequence", ErrUnsupportedChange, live.Name) + } if desired.Default == "" { changes = append(changes, Change{ - SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " DROP DEFAULT", + SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " DROP DEFAULT", + Kind: ChangeDropDefault, }) } else { changes = append(changes, Change{ - SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " SET DEFAULT " + desired.Default, + SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " SET DEFAULT " + desired.Default, + Kind: ChangeSetDefault, }) } } if live.NotNull != desired.NotNull { if desired.NotNull { changes = append(changes, Change{ - SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " SET NOT NULL", + SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " SET NOT NULL", + Kind: ChangeSetNotNull, }) } else { changes = append(changes, Change{ - SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " DROP NOT NULL", + SQL: "ALTER TABLE " + table + " ALTER COLUMN " + col + " DROP NOT NULL", + Kind: ChangeDropNotNull, }) } } diff --git a/pkg/schemadiff/diff_test.go b/pkg/schemadiff/diff_test.go index 06d137f..7617153 100644 --- a/pkg/schemadiff/diff_test.go +++ b/pkg/schemadiff/diff_test.go @@ -69,6 +69,77 @@ func TestDiffDropColumnIsDestructive(t *testing.T) { assert.True(t, changes[0].Destructive) } +func TestDiffDropIndexIsDestructive(t *testing.T) { + desired := base() + desired.Indexes = nil + changes, err := Diff("public", base(), desired) + require.NoError(t, err) + require.Equal(t, []string{ + `DROP INDEX "public"."events_name_idx"`, + }, sqls(changes)) + assert.True(t, changes[0].Destructive) + assert.Equal(t, ChangeDropIndex, changes[0].Kind) +} + +func TestDiffRefusesSequenceDefaultAdoption(t *testing.T) { + desired := base() + desired.Columns[0].Default = "nextval('events_id_seq'::regclass)" + desired.Columns[0].SequenceDefault = true + _, err := Diff("public", base(), desired) + require.ErrorIs(t, err, ErrUnsupportedChange) +} + +func TestDiffRefusesSequenceDefaultOnAddedColumn(t *testing.T) { + desired := base() + desired.Columns = append(desired.Columns, Column{ + Name: "seq_col", Type: "integer", + Default: "nextval('events_seq_col_seq'::regclass)", SequenceDefault: true, + }) + _, err := Diff("public", base(), desired) + require.ErrorIs(t, err, ErrUnsupportedChange) +} + +func TestDiffIdenticalSequenceDefaultsConverge(t *testing.T) { + withSerial := func() Model { + m := base() + m.Columns[0].Default = "nextval('events_id_seq'::regclass)" + m.Columns[0].SequenceDefault = true + return m + } + changes, err := Diff("public", withSerial(), withSerial()) + require.NoError(t, err) + assert.Empty(t, changes) +} + +func TestDiffChangeKinds(t *testing.T) { + live := base() + live.Columns = append(live.Columns, Column{Name: "legacy", Type: "integer"}) + + desired := base() + desired.Columns[1] = Column{Name: "name", Type: "text", NotNull: true} + desired.Columns = append(desired.Columns, Column{Name: "email", Type: "text"}) + desired.Constraints = append(desired.Constraints, Constraint{ + Name: "events_email_key", Def: "UNIQUE (email)", + }) + desired.Indexes = append(desired.Indexes, Index{ + Name: "events_email_idx", Def: "CREATE INDEX events_email_idx ON events USING btree (email)", + }) + + changes, err := Diff("public", live, desired) + require.NoError(t, err) + kinds := make([]ChangeKind, len(changes)) + for i, c := range changes { + kinds[i] = c.Kind + } + assert.Equal(t, []ChangeKind{ + ChangeDropColumn, + ChangeAddColumn, + ChangeAlterType, + ChangeAddConstraint, + ChangeCreateIndex, + }, kinds) +} + func TestDiffColumnAlterations(t *testing.T) { desired := base() desired.Columns[1] = Column{Name: "name", Type: "text", NotNull: false, Default: "'unnamed'::text"} diff --git a/pkg/schemadiff/introspect.go b/pkg/schemadiff/introspect.go index 34d212c..8c6867d 100644 --- a/pkg/schemadiff/introspect.go +++ b/pkg/schemadiff/introspect.go @@ -93,6 +93,15 @@ func introspectColumns(ctx context.Context, tx pgx.Tx, oid uint32) ([]Column, er format_type(a.atttypid, a.atttypmod), a.attnotnull, COALESCE(pg_get_expr(d.adbin, d.adrelid), ''), + COALESCE(( + SELECT true + FROM pg_depend dep + JOIN pg_class s ON s.oid = dep.refobjid AND s.relkind = 'S' + WHERE dep.classid = 'pg_attrdef'::regclass + AND dep.objid = d.oid + AND dep.refclassid = 'pg_class'::regclass + LIMIT 1 + ), false), a.attidentity::text, a.attgenerated::text FROM pg_attribute a @@ -107,7 +116,7 @@ func introspectColumns(ctx context.Context, tx pgx.Tx, oid uint32) ([]Column, er for rows.Next() { var c Column var identity, generated string - if err := rows.Scan(&c.Name, &c.Type, &c.NotNull, &c.Default, &identity, &generated); err != nil { + if err := rows.Scan(&c.Name, &c.Type, &c.NotNull, &c.Default, &c.SequenceDefault, &identity, &generated); err != nil { return nil, fmt.Errorf("scan column: %w", err) } c.Identity = Identity(identity) diff --git a/pkg/schemadiff/schemadiff.go b/pkg/schemadiff/schemadiff.go index 1cdfe9e..6a47e4c 100644 --- a/pkg/schemadiff/schemadiff.go +++ b/pkg/schemadiff/schemadiff.go @@ -38,6 +38,12 @@ type Column struct { // Default is the canonical default expression (pg_get_expr), empty when // none. For a generated column it is the generation expression. Default string + // SequenceDefault reports that the default expression depends on a + // sequence (per pg_depend) — a serial column or a hand-written nextval + // default. In a desired-state model that sequence exists only inside + // the rolled-back scratch transaction, so no derived plan can + // reference it. + SequenceDefault bool // Identity is the identity kind, IdentityNone for plain columns. Identity Identity // Generated reports GENERATED ALWAYS AS (...) STORED. diff --git a/pkg/schemadiff/schemadiff_integration_test.go b/pkg/schemadiff/schemadiff_integration_test.go index bb33095..0746aff 100644 --- a/pkg/schemadiff/schemadiff_integration_test.go +++ b/pkg/schemadiff/schemadiff_integration_test.go @@ -52,6 +52,55 @@ func TestIntrospectDesiredMatchesLiveIntrospection(t *testing.T) { "live introspection and scratch execute-and-introspect must agree on the canonical model") } +// Converging a plain integer column onto serial would emit a SET DEFAULT +// referencing a sequence that only ever existed inside the rolled-back +// scratch transaction — a plan that cannot execute. The diff refuses it as +// an unsupported change instead. +func TestDiffRefusesSerialAdoption(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY, v text)", schema)) + require.NoError(t, err) + + ds, err := statement.ParseDesired("CREATE TABLE t (id serial PRIMARY KEY, v text)") + require.NoError(t, err) + live, err := schemadiff.Introspect(t.Context(), pool, schema, "t") + require.NoError(t, err) + desired, err := schemadiff.IntrospectDesired(t.Context(), pool, ds) + require.NoError(t, err) + + _, err = schemadiff.Diff(schema, live, desired) + require.ErrorIs(t, err, schemadiff.ErrUnsupportedChange) +} + +// A serial table that already matches its desired file must converge to no +// changes: both sides decompile the sequence default identically under +// their introspection search_path. +func TestDiffSerialTableConverges(t *testing.T) { + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: testutil.StartPostgres(t)}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id serial PRIMARY KEY, v text)", schema)) + require.NoError(t, err) + + ds, err := statement.ParseDesired("CREATE TABLE t (id serial PRIMARY KEY, v text)") + require.NoError(t, err) + live, err := schemadiff.Introspect(t.Context(), pool, schema, "t") + require.NoError(t, err) + assert.True(t, live.Columns[0].SequenceDefault, "serial column default must be marked sequence-backed") + desired, err := schemadiff.IntrospectDesired(t.Context(), pool, ds) + require.NoError(t, err) + + changes, err := schemadiff.Diff(schema, live, desired) + require.NoError(t, err) + assert.Empty(t, changes) +} + // Cosmetically different spellings of the same schema must introspect to the // same canonical model: the server's decompilers are the canonicalizer. func TestIntrospectCanonicalizesTypeAliases(t *testing.T) { diff --git a/pkg/statement/comments.go b/pkg/statement/comments.go new file mode 100644 index 0000000..f2620f4 --- /dev/null +++ b/pkg/statement/comments.go @@ -0,0 +1,31 @@ +package statement + +import ( + "errors" + "fmt" + + pganalyze "github.com/pganalyze/pg_query_go/v6" + pgquery "github.com/wasilibs/go-pgquery" +) + +// ErrCommentLoss is returned when an operation that reprints SQL through the +// deparser would silently discard comments. The parser drops comments at +// parse time, so a formatter cannot carry them; refusing is the fail-closed +// alternative to destroying documentation in a source-of-truth file. +var ErrCommentLoss = errors.New("input contains comments, which formatting would discard") + +// CheckNoComments scans sql with the PostgreSQL lexer and returns +// ErrCommentLoss when it contains any SQL (--) or C-style (/* */) comment. +// A scan failure is surfaced to the caller, never guessed around. +func CheckNoComments(sql string) error { + scan, err := pgquery.Scan(sql) + if err != nil { + return fmt.Errorf("scan statement: %w", err) + } + for _, tok := range scan.GetTokens() { + if tok.GetToken() == pganalyze.Token_SQL_COMMENT || tok.GetToken() == pganalyze.Token_C_COMMENT { + return ErrCommentLoss + } + } + return nil +} diff --git a/pkg/statement/comments_test.go b/pkg/statement/comments_test.go new file mode 100644 index 0000000..03009d4 --- /dev/null +++ b/pkg/statement/comments_test.go @@ -0,0 +1,31 @@ +package statement + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestCheckNoComments(t *testing.T) { + tests := []struct { + name string + sql string + wantErr error + }{ + {"no comments", "CREATE TABLE t (id int)", nil}, + {"line comment", "-- events table\nCREATE TABLE t (id int)", ErrCommentLoss}, + {"inline line comment", "CREATE TABLE t (\n id int -- surrogate key\n)", ErrCommentLoss}, + {"block comment", "/* header */ CREATE TABLE t (id int)", ErrCommentLoss}, + {"comment inside a string literal is not a comment", "CREATE TABLE t (id int DEFAULT length('--'))", nil}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := CheckNoComments(tt.sql) + if tt.wantErr == nil { + require.NoError(t, err) + return + } + require.ErrorIs(t, err, tt.wantErr) + }) + } +} diff --git a/pkg/statement/desired.go b/pkg/statement/desired.go index 3753994..2ba8fbf 100644 --- a/pkg/statement/desired.go +++ b/pkg/statement/desired.go @@ -29,6 +29,14 @@ var ( // ErrConcurrentIndex is returned for CREATE INDEX CONCURRENTLY, which // cannot run inside the scratch transaction. ErrConcurrentIndex = errors.New("CONCURRENTLY cannot be used in a desired schema") + // ErrForeignKey is returned when the CREATE TABLE carries a REFERENCES + // clause. The scratch transaction cannot faithfully execute a foreign + // key: an unqualified reference resolves against the scratch + // search_path, not the target schema, so it either fails or silently + // binds to the wrong table. Foreign-key support needs its own design + // (cross-file ordering, lock behavior, qualification policy); until + // then the admission gate refuses it. + ErrForeignKey = errors.New("foreign keys are not supported in a desired schema") // ErrWrongIndexTarget is returned when an index targets a table other // than the desired CREATE TABLE. ErrWrongIndexTarget = errors.New("index must target the desired table") @@ -89,13 +97,17 @@ func ParseDesired(sql string) (DesiredSchema, error) { func admitDesiredStatement(node *pganalyze.Node, seenTable string) (Statement, error) { switch { case node.GetCreateStmt() != nil: - rel := node.GetCreateStmt().GetRelation() + create := node.GetCreateStmt() + rel := create.GetRelation() if rel.GetSchemaname() != "" { return Statement{}, fmt.Errorf("%w: %s.%s", ErrQualifiedName, rel.GetSchemaname(), rel.GetRelname()) } if seenTable != "" { return Statement{}, ErrMultipleCreateTables } + if err := refuseForeignKeys(create); err != nil { + return Statement{}, err + } return Statement{kind: KindCreateTable, table: rel.GetRelname()}, nil case node.GetIndexStmt() != nil: idx := node.GetIndexStmt() @@ -112,6 +124,27 @@ func admitDesiredStatement(node *pganalyze.Node, seenTable string) (Statement, e } } +// refuseForeignKeys returns ErrForeignKey when the CREATE TABLE carries a +// REFERENCES clause, in either its column-constraint or table-constraint +// form. This inspects constraint kinds only — no semantics are derived. +func refuseForeignKeys(create *pganalyze.CreateStmt) error { + for _, elt := range create.GetTableElts() { + if con := elt.GetConstraint(); con != nil && con.GetContype() == pganalyze.ConstrType_CONSTR_FOREIGN { + return fmt.Errorf("%w: table constraint on %q", ErrForeignKey, create.GetRelation().GetRelname()) + } + col := elt.GetColumnDef() + if col == nil { + continue + } + for _, c := range col.GetConstraints() { + if con := c.GetConstraint(); con != nil && con.GetContype() == pganalyze.ConstrType_CONSTR_FOREIGN { + return fmt.Errorf("%w: column %q", ErrForeignKey, col.GetColname()) + } + } + } + return nil +} + // Qualify returns sql with its target relation qualified by schema; an // empty schema strips an existing qualification instead. It supports exactly // one CREATE TABLE or CREATE INDEX statement. This touches qualification diff --git a/pkg/statement/desired_test.go b/pkg/statement/desired_test.go index 0a33520..d0f0c94 100644 --- a/pkg/statement/desired_test.go +++ b/pkg/statement/desired_test.go @@ -40,6 +40,9 @@ func TestParseDesiredRefusals(t *testing.T) { {"qualified index", "CREATE TABLE t (id int); CREATE INDEX i ON prod.t (id)", ErrQualifiedName}, {"concurrent index", "CREATE TABLE t (id int); CREATE INDEX CONCURRENTLY i ON t (id)", ErrConcurrentIndex}, {"index on another table", "CREATE TABLE t (id int); CREATE INDEX i ON other (id)", ErrWrongIndexTarget}, + {"column foreign key", "CREATE TABLE child (id int PRIMARY KEY, pid int REFERENCES parent(id))", ErrForeignKey}, + {"table foreign key", "CREATE TABLE child (id int PRIMARY KEY, pid int, FOREIGN KEY (pid) REFERENCES parent(id))", ErrForeignKey}, + {"self-referencing foreign key", "CREATE TABLE node (id int PRIMARY KEY, parent_id int REFERENCES node(id))", ErrForeignKey}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/statement/statement.go b/pkg/statement/statement.go index a454e9d..14d69df 100644 --- a/pkg/statement/statement.go +++ b/pkg/statement/statement.go @@ -3,6 +3,13 @@ // front door needs. In Phase 1 that is a statement-type gate only: which kind // of statement this is and, for ALTER TABLE, which table it targets. No // schema model, no classification. +// +// Two canonical forms coexist deliberately: this package's deparser prints +// grammar-canonical SQL (e.g. varchar(50)) for formatting, while +// pkg/schemadiff models carry server-decompiled text (character +// varying(50)) for comparison. The two canons never mix: models only ever +// compare server output against server output, and deparser output must not +// feed a model comparison or a schema fingerprint. package statement import ( From f700319a242efa2886578f523f6a7d3ce621f3ad Mon Sep 17 00:00:00 2001 From: Kiran Muddukrishna Date: Fri, 7 Aug 2026 10:04:33 +1000 Subject: [PATCH 24/24] planner, router: fail closed on unconstructed safer rewrites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address PR #7 review findings: inline ADD COLUMN constraints and CREATE TABLE ... PARTITION OF are now classified as the work they do, and routing refuses (rewrite-required) any statement whose submitted form blocks when no complete online rewrite was constructed — partial rewrites of multi-operation statements can no longer execute the original SQL. Generated scaffold names fit PostgreSQL's 63-byte identifier limit deterministically. --- docs/postgres-online-ddl-reference.md | 4 + internal/cli/diff.go | 5 +- internal/cli/dryrun_integration_test.go | 29 +++++++ pkg/planner/planner.go | 110 ++++++++++++++++++++++-- pkg/planner/planner_test.go | 45 ++++++++++ pkg/router/router.go | 46 ++++++++-- pkg/router/router_test.go | 61 ++++++++++++- pkg/statement/ops.go | 44 +++++++++- pkg/statement/ops_test.go | 37 ++++++++ 9 files changed, 357 insertions(+), 24 deletions(-) diff --git a/docs/postgres-online-ddl-reference.md b/docs/postgres-online-ddl-reference.md index 1c07f03..c707722 100644 --- a/docs/postgres-online-ddl-reference.md +++ b/docs/postgres-online-ddl-reference.md @@ -57,6 +57,7 @@ the same dynamic via metadata locks. Read it first if any of that is unfamiliar. | `ADD COLUMN ... DEFAULT ` | ACCESS EXCLUSIVE (brief) | **No** (PG 11+) | Yes | ❌ No | "Fast default" stored in catalog; pre-PG11 this rewrote | | `ADD COLUMN ... DEFAULT ` (e.g. `now()`, `random()`, `uuid_generate_v4()`) | ACCESS EXCLUSIVE | **Yes** (full rewrite) | No | ✅ **Yes** | The expensive case | | `ADD COLUMN ... GENERATED ALWAYS AS (...) STORED` | ACCESS EXCLUSIVE | Yes | No | ✅ **Yes** | Values must be computed | +| `ADD COLUMN ... UNIQUE` / `PRIMARY KEY` / `REFERENCES` / `CHECK` (inline constraint) | ACCESS EXCLUSIVE + index build or validation | No | No | ➖ Native pattern | Same work as the `ADD CONSTRAINT` form, under the `ADD COLUMN` lock — split: add the column first, then build the constraint online (`CONCURRENTLY` + `USING INDEX`, or `NOT VALID` + `VALIDATE`) | | `DROP COLUMN` | ACCESS EXCLUSIVE (brief) | No | Yes | ❌ No | Metadata only; disk space reclaimed lazily by VACUUM | | `ALTER COLUMN TYPE` — binary-coercible (`varchar(50)→varchar(100)`, `varchar→text`, `numeric(10,2)→numeric(12,2)`) | ACCESS EXCLUSIVE (brief) | **No** | No (brief) | ❌ No | No scan when binary-coercible and no length restriction is added | | `ALTER COLUMN TYPE` — general (`int→bigint`, `text→jsonb`, `timestamp→timestamptz` w/ conversion) | ACCESS EXCLUSIVE | **Yes** (rewrite + reindex + revalidate FKs) | No | ✅ **Yes** | The classic "needs a tool" case | @@ -96,6 +97,7 @@ online path, so the heavy **shadow-copy + atomic cutover** path is required · | `ALTER COLUMN SET STATISTICS` / `SET STORAGE` / `SET (n_distinct=...)` | No | varies | ❌ No | SHARE UPDATE EXCLUSIVE / ACCESS EXCLUSIVE | | `ALTER COLUMN TYPE` — binary-coercible | No | No (brief) | ❌ No | ACCESS EXCLUSIVE (brief) | | `ALTER COLUMN SET NOT NULL` | No, but full scan | No | ➖ Native pattern | ACCESS EXCLUSIVE | +| `ADD COLUMN ...` (inline constraint) | No, but index build / validation | No | ➖ Native pattern | ACCESS EXCLUSIVE | | `ADD COLUMN ... DEFAULT ` | Yes | No | ✅ **Yes** | ACCESS EXCLUSIVE | | `ADD COLUMN ... GENERATED ALWAYS AS (...) STORED` | Yes | No | ✅ **Yes** | ACCESS EXCLUSIVE | | `ALTER COLUMN TYPE` — general | Yes | No | ✅ **Yes** | ACCESS EXCLUSIVE | @@ -116,6 +118,7 @@ online path, so the heavy **shadow-copy + atomic cutover** path is required · | `ALTER COLUMN SET STATISTICS` / `SET STORAGE` / `SET (n_distinct=...)` | varies | No | ❌ No | SHARE UPDATE EXCLUSIVE / ACCESS EXCLUSIVE | | `ALTER COLUMN TYPE` — binary-coercible | No (brief) | No | ❌ No | ACCESS EXCLUSIVE (brief) | | `ALTER COLUMN SET NOT NULL` | No | No, but full scan | ➖ Native pattern | ACCESS EXCLUSIVE | +| `ADD COLUMN ...` (inline constraint) | No | No, but index build / validation | ➖ Native pattern | ACCESS EXCLUSIVE | | `ADD COLUMN ... DEFAULT ` | No | Yes | ✅ **Yes** | ACCESS EXCLUSIVE | | `ADD COLUMN ... GENERATED ALWAYS AS (...) STORED` | No | Yes | ✅ **Yes** | ACCESS EXCLUSIVE | | `ALTER COLUMN TYPE` — general | No | Yes | ✅ **Yes** | ACCESS EXCLUSIVE | @@ -188,6 +191,7 @@ ALTER TABLE t ADD CONSTRAINT t_pkey PRIMARY KEY USING INDEX t_pkey; -- brief lo | `SET TABLESPACE` | ACCESS EXCLUSIVE | **Yes** (moves heap) | No | ✅ **Yes** (repack-style) | Rewrite/move; use a repack-style copy instead | | `SET (fillfactor=...)` and most reloptions | SHARE UPDATE EXCLUSIVE | No | Yes | ❌ No | Applies to new rows | | `CLUSTER` / `VACUUM FULL` | ACCESS EXCLUSIVE | **Yes** (full rewrite) | No | ✅ **Yes** (`pg_repack`) | Use `pg_repack` | +| `CREATE TABLE ... PARTITION OF` | ACCESS EXCLUSIVE on **parent** (brief) | No | Blocked on parent while held | ❌ No | Brief and no scan, but it queues behind long-running queries and then blocks every reader of the parent | | `ATTACH PARTITION` | SHARE UPDATE EXCLUSIVE on parent + scan of child | No | Yes | ➖ Native pattern | Add a validated `CHECK` matching the bound on the child first to skip the scan | | `DETACH PARTITION` | ACCESS EXCLUSIVE | No | No | ➖ Use `CONCURRENTLY` | | | `DETACH PARTITION CONCURRENTLY` | SHARE UPDATE EXCLUSIVE | No | Yes | ❌ No | PG 14+ | diff --git a/internal/cli/diff.go b/internal/cli/diff.go index 53ccca6..735520b 100644 --- a/internal/cli/diff.go +++ b/internal/cli/diff.go @@ -254,8 +254,11 @@ func annotate(ch plannedChange) string { } } s := fmt.Sprintf("%s (%s)", ch.Route, strings.Join(reasons, ", ")) - if ch.Disposition == router.DispositionUnavailable { + switch ch.Disposition { + case router.DispositionUnavailable: s += ": needs the " + string(ch.Backend) + " backend, which is not implemented yet" + case router.DispositionRewriteRequired: + s += ": blocks as submitted and no online rewrite was constructed — the engine will not run it" } return s } diff --git a/internal/cli/dryrun_integration_test.go b/internal/cli/dryrun_integration_test.go index 2ab3a6e..dbc72e5 100644 --- a/internal/cli/dryrun_integration_test.go +++ b/internal/cli/dryrun_integration_test.go @@ -114,6 +114,35 @@ func TestMigrateDryRunSuggestsConcurrentIndex(t *testing.T) { assert.Equal(t, 0, indexes, "dry-run must not create the index") } +// A safer-idiom decision without a constructed rewrite dry-runs to +// rewrite-required with no executable SQL, and nothing executes: the +// engine must not fall back to the submitted blocking form. +func TestMigrateDryRunInlineConstraintIsRewriteRequired(t *testing.T) { + url := testutil.StartPostgres(t) + pool, err := dbconn.NewPool(t.Context(), dbconn.Config{URL: url}) + require.NoError(t, err) + defer pool.Close() + schema := testutil.NewSchema(t, pool) + _, err = pool.Exec(t.Context(), fmt.Sprintf("CREATE TABLE %s.t (id int PRIMARY KEY)", schema)) + require.NoError(t, err) + + plan := dryRunPlan(t, url, fmt.Sprintf("ALTER TABLE %s.t ADD COLUMN c int UNIQUE", schema)) + assert.Equal(t, router.DispositionRewriteRequired, plan.Disposition) + require.Len(t, plan.Statements, 1) + st := plan.Statements[0] + assert.Equal(t, planner.RouteNative, st.Route) + assert.Equal(t, router.BackendNative, st.Backend) + require.Len(t, st.Decisions, 1) + assert.Equal(t, planner.ReasonSaferIdiom, st.Decisions[0].Reason) + assert.Empty(t, st.ExecSQL, "no executable SQL for an unconstructed rewrite") + + var columns int + require.NoError(t, pool.QueryRow(t.Context(), + `SELECT count(*) FROM information_schema.columns + WHERE table_schema = $1 AND table_name = 't' AND column_name = 'c'`, schema).Scan(&columns)) + assert.Equal(t, 0, columns, "dry-run must not add the column") +} + // A dry-run against a table that does not exist classifies with zero facts: // the unprovable type change routes conservatively instead of failing. func TestMigrateDryRunMissingTableIsConservative(t *testing.T) { diff --git a/pkg/planner/planner.go b/pkg/planner/planner.go index a6ccfeb..cf9e1a6 100644 --- a/pkg/planner/planner.go +++ b/pkg/planner/planner.go @@ -5,12 +5,23 @@ // column of docs/postgres-online-ddl-reference.md, applied conservatively: // anything the planner cannot prove safe routes to copy-and-swap or refuse. // Classification predicts; executors keep their own protections regardless. +// +// The rules assume PostgreSQL 14, the oldest major the test matrix runs; +// every rule holds unconditionally across the supported range (14–18). +// Rules that were version-dependent below that floor — fast default +// (PG 11+), SET NOT NULL proven by a validated CHECK (PG 12+), DETACH +// PARTITION CONCURRENTLY (PG 14+) — carry no version annotation because +// the floor makes them unconditional. A rule that varies within the +// supported range must carry an explicit version fact before it lands. package planner import ( + "crypto/sha256" "fmt" + "slices" "strconv" "strings" + "unicode/utf8" "github.com/jackc/pgx/v5" @@ -74,6 +85,10 @@ const ( // ReasonRelocation: SET TABLESPACE moves the heap — a rewrite-scale // copy. ReasonRelocation Reason = "relocation" + // ReasonPartitionParentLock: creating a partition takes a brief ACCESS + // EXCLUSIVE on the partitioned parent — no scan, but it queues behind + // and then blocks every query on the parent while held. + ReasonPartitionParentLock Reason = "partition-parent-lock" // ReasonUnsupportedOperation: the planner does not recognize the // operation or knows no safe path for it. ReasonUnsupportedOperation Reason = "unsupported-operation" @@ -89,10 +104,23 @@ type Decision struct { Reason Reason `json:"reason"` // SaferSQL is the ordered native sequence to run instead of the // submitted form, present only for safer-idiom decisions where the - // planner could construct it. + // planner could construct it. Execution contract: the steps run one at + // a time, in order, each in its own implicit transaction — never inside + // an enclosing transaction block, which the CONCURRENTLY forms refuse. + // Each sequence constructor documents what a failed step leaves behind + // and how a retry resumes. SaferSQL []string `json:"safer_sql,omitempty"` } +// ExecutableAsSubmitted reports whether the operation's submitted form is +// itself safe to run. It is false exactly for safer-idiom decisions: their +// submitted form blocks and must be replaced by the safer sequence — +// whether or not one was constructed. Routing fails closed on the +// combination of a false ExecutableAsSubmitted and an empty SaferSQL. +func (d Decision) ExecutableAsSubmitted() bool { + return d.Reason != ReasonSaferIdiom +} + // Plan is the classification of one statement: one decision per operation // and the aggregate route (the worst of its decisions — one rewrite makes // the whole statement a copy, one refusal refuses it). @@ -105,9 +133,13 @@ type Plan struct { Decisions []Decision `json:"decisions"` } -// Facts are introspected properties of the live table that sharpen -// classification. The zero value is valid: with no facts the planner is -// strictly more conservative (every type change becomes copy-and-swap). +// Facts are properties of the live table that sharpen classification, and +// they are trusted as stated: the CLI fills them by introspecting the +// target database, and a library caller may supply facts it already holds — +// but they must describe the database the change will run on, because a +// wrong fact can upgrade a rewrite to native. Missing facts are always +// safe: the zero value is valid and classifies strictly more +// conservatively (every type change becomes copy-and-swap). type Facts struct { // ColumnTypes maps a column name to its live type as rendered by // PostgreSQL's format_type (e.g. "character varying(50)"). @@ -143,15 +175,33 @@ func classifyOp(op statement.Op, st statement.Statement, facts Facts, sql string d := Decision{Operation: op.Describe()} switch op.Kind { case statement.OpCreateTable: - // A new table has no readers to lock out. - d.Route, d.Reason = RouteNative, ReasonMetadataOnly + if op.PartitionOf { + // Creating a partition locks the partitioned parent ACCESS + // EXCLUSIVE — briefly and without a scan, but it queues behind + // any long-running query and then blocks every reader of the + // parent while held. + d.Route, d.Reason = RouteNative, ReasonPartitionParentLock + } else { + // A brand-new standalone table has no readers to lock out. + d.Route, d.Reason = RouteNative, ReasonMetadataOnly + } case statement.OpAddColumn: switch { + case hasUnrecognizedConstraint(op.InlineConstraints): + d.Route, d.Reason = RouteRefuse, ReasonUnsupportedOperation case op.GeneratedStored: d.Route, d.Reason = RouteCopyAndSwap, ReasonGeneratedStored case op.Default == statement.DefaultExpression: d.Route, d.Reason = RouteCopyAndSwap, ReasonVolatileDefault + case len(op.InlineConstraints) > 0: + // An inline UNIQUE / PRIMARY KEY / FOREIGN KEY / CHECK does the + // same index build or validation as its ADD CONSTRAINT form, + // under the ADD COLUMN's ACCESS EXCLUSIVE lock. The safer path + // splits the column addition from an online constraint build; + // the planner does not construct multi-statement splits, so the + // decision carries no rewrite and routing fails closed. + d.Route, d.Reason = RouteNative, ReasonSaferIdiom case op.Default == statement.DefaultConstant: d.Route, d.Reason = RouteNative, ReasonFastDefault default: @@ -374,6 +424,34 @@ func parseTypeText(s string) typeShape { return shape } +// hasUnrecognizedConstraint reports whether an added column carries an +// inline constraint family the engine does not model. +func hasUnrecognizedConstraint(kinds []statement.ConstraintKind) bool { + return slices.Contains(kinds, statement.ConstraintUnrecognized) +} + +// maxIdentifierBytes is PostgreSQL's NAMEDATALEN-1: the server silently +// truncates longer identifiers, which would let a generated name collide +// with the table itself or with a sibling scaffold. +const maxIdentifierBytes = 63 + +// fitIdentifier returns name unchanged when it fits PostgreSQL's identifier +// limit, otherwise a deterministic variant that does: the head of the name +// plus an 8-hex-digit hash of the full name, so distinct inputs stay +// distinct after fitting. +func fitIdentifier(name string) string { + if len(name) <= maxIdentifierBytes { + return name + } + sum := sha256.Sum256([]byte(name)) + suffix := fmt.Sprintf("_%x", sum[:4]) + head := name[:maxIdentifierBytes-len(suffix)] + for !utf8.ValidString(head) { + head = head[:len(head)-1] + } + return head + suffix +} + // tableIdent renders the statement's target table as a quoted identifier, // schema-qualified when the statement was. func tableIdent(st statement.Statement) string { @@ -386,9 +464,16 @@ func tableIdent(st statement.Statement) string { // setNotNullSequence is the native four-step SET NOT NULL pattern: prove // the invariant online with a NOT VALID CHECK, flip the column, drop the // scaffold. +// +// Partial-failure contract: step 1 leaves a NOT VALID CHECK under the +// generated scaffold name, and re-running step 1 then fails with SQLSTATE +// 42710 (duplicate_object) — a retry resumes at step 2. A failed VALIDATE +// (step 2) leaves the same scaffold and is safe to re-run. Steps 3 and 4 +// are metadata-only and safe to re-run; a leftover scaffold is removed by +// running step 4 alone. func setNotNullSequence(st statement.Statement, column string) []string { table := tableIdent(st) - conName := fmt.Sprintf("%s_%s_not_null", st.Table(), column) + conName := fitIdentifier(fmt.Sprintf("%s_%s_not_null", st.Table(), column)) con := pgx.Identifier{conName}.Sanitize() col := pgx.Identifier{column}.Sanitize() return []string{ @@ -402,6 +487,12 @@ func setNotNullSequence(st statement.Statement, column string) []string { // usingIndexSequence is the native two-step ADD PRIMARY KEY / UNIQUE // pattern: build the unique index concurrently, then attach it as the // constraint under a brief lock. +// +// Partial-failure contract: a failed CREATE INDEX CONCURRENTLY (step 1) +// leaves an INVALID index under the generated name, and re-running step 1 +// then fails with SQLSTATE 42P07 (duplicate_table) — the retry path is +// DROP INDEX, then re-run step 1. Step 2 consumes the index into the +// constraint under a brief lock and does not scan. func usingIndexSequence(st statement.Statement, op statement.Op) []string { suffix := "_key" keyword := "UNIQUE" @@ -411,7 +502,10 @@ func usingIndexSequence(st statement.Statement, op statement.Op) []string { } name := op.Name if name == "" { - name = st.Table() + "_" + strings.Join(op.Columns, "_") + suffix + // A user-supplied name is used as-is: the server truncates it the + // same way in every step. A generated name is built to fit so it + // cannot truncate into the table's own name or a sibling's. + name = fitIdentifier(st.Table() + "_" + strings.Join(op.Columns, "_") + suffix) } idx := pgx.Identifier{name}.Sanitize() cols := make([]string, len(op.Columns)) diff --git a/pkg/planner/planner_test.go b/pkg/planner/planner_test.go index 28113ec..7dc2593 100644 --- a/pkg/planner/planner_test.go +++ b/pkg/planner/planner_test.go @@ -1,12 +1,15 @@ package planner_test import ( + "fmt" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/block/pg-sprite/pkg/planner" + "github.com/block/pg-sprite/pkg/statement" ) // facts mirrors a live table whose column types exercise both sides of the @@ -46,6 +49,10 @@ func TestClassifyReferenceRows(t *testing.T) { {"add column volatile default random", "ALTER TABLE t ADD COLUMN r float8 DEFAULT random()", planner.RouteCopyAndSwap, planner.ReasonVolatileDefault, 0}, {"add column volatile default uuid", "ALTER TABLE t ADD COLUMN id uuid DEFAULT uuid_generate_v4()", planner.RouteCopyAndSwap, planner.ReasonVolatileDefault, 0}, {"add column generated stored", "ALTER TABLE t ADD COLUMN total numeric GENERATED ALWAYS AS (price * qty) STORED", planner.RouteCopyAndSwap, planner.ReasonGeneratedStored, 0}, + {"add column inline unique", "ALTER TABLE t ADD COLUMN c int UNIQUE", planner.RouteNative, planner.ReasonSaferIdiom, 0}, + {"add column inline primary key", "ALTER TABLE t ADD COLUMN c int PRIMARY KEY", planner.RouteNative, planner.ReasonSaferIdiom, 0}, + {"add column inline foreign key", "ALTER TABLE t ADD COLUMN c int REFERENCES p (id)", planner.RouteNative, planner.ReasonSaferIdiom, 0}, + {"add column inline check", "ALTER TABLE t ADD COLUMN c int CHECK (c > 0)", planner.RouteNative, planner.ReasonSaferIdiom, 0}, {"drop column", "ALTER TABLE t DROP COLUMN age", planner.RouteNative, planner.ReasonMetadataOnly, 0}, {"alter type widen varchar", "ALTER TABLE t ALTER COLUMN v50 TYPE varchar(100)", planner.RouteNative, planner.ReasonBinaryCoercible, 0}, {"alter type varchar to text", "ALTER TABLE t ALTER COLUMN v50 TYPE text", planner.RouteNative, planner.ReasonBinaryCoercible, 0}, @@ -98,6 +105,7 @@ func TestClassifyReferenceRows(t *testing.T) { {"attach partition", "ALTER TABLE t ATTACH PARTITION p FOR VALUES FROM (1) TO (10)", planner.RouteNative, planner.ReasonSaferIdiom, 0}, {"detach partition", "ALTER TABLE t DETACH PARTITION p", planner.RouteNative, planner.ReasonSaferIdiom, 1}, {"detach partition concurrently", "ALTER TABLE t DETACH PARTITION p CONCURRENTLY", planner.RouteNative, planner.ReasonOnlineIdiom, 0}, + {"create table partition of", "CREATE TABLE p PARTITION OF t FOR VALUES FROM (1) TO (10)", planner.RouteNative, planner.ReasonPartitionParentLock, 0}, // Non-DDL and unknown statements. {"dml", "INSERT INTO t VALUES (1)", planner.RouteRefuse, planner.ReasonUnsupportedOperation, 0}, @@ -177,3 +185,40 @@ func TestClassifyParseErrorSurfaces(t *testing.T) { _, err := planner.Classify("ALTER TABLE", planner.Facts{}) assert.Error(t, err) } + +// generatedConstraintName parses a safer-sequence ADD CONSTRAINT step back +// through the statement parser and returns the typed constraint name, so +// the test asserts identifier facts rather than SQL prose. +func generatedConstraintName(t *testing.T, step string) string { + t.Helper() + ops, err := statement.ParseOps(step) + require.NoError(t, err) + require.Len(t, ops, 1) + require.Equal(t, statement.OpAddConstraint, ops[0].Kind) + require.NotEmpty(t, ops[0].Name) + return ops[0].Name +} + +func TestClassifyGeneratedNamesFitIdentifierLimit(t *testing.T) { + // Long enough that table + column + suffix would exceed PostgreSQL's + // 63-byte identifier limit, where the server would silently truncate. + table := strings.Repeat("t", 40) + colA := strings.Repeat("a", 40) + colB := strings.Repeat("b", 40) + + dA := classifyOne(t, fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s SET NOT NULL", table, colA)) + dB := classifyOne(t, fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s SET NOT NULL", table, colB)) + require.NotEmpty(t, dA.SaferSQL) + require.NotEmpty(t, dB.SaferSQL) + + nameA := generatedConstraintName(t, dA.SaferSQL[0]) + nameB := generatedConstraintName(t, dB.SaferSQL[0]) + assert.LessOrEqual(t, len(nameA), 63, "generated names must fit PostgreSQL's identifier limit") + assert.LessOrEqual(t, len(nameB), 63) + assert.NotEqual(t, nameA, nameB, + "columns differing only past the truncation point must not collide") + + // Deterministic: the same input yields the same fitted name. + dA2 := classifyOne(t, fmt.Sprintf("ALTER TABLE %s ALTER COLUMN %s SET NOT NULL", table, colA)) + assert.Equal(t, dA.SaferSQL, dA2.SaferSQL) +} diff --git a/pkg/router/router.go b/pkg/router/router.go index 642086a..a5a721f 100644 --- a/pkg/router/router.go +++ b/pkg/router/router.go @@ -43,6 +43,12 @@ const ( // DispositionExecute: the assigned backend is available; the statement // would run. DispositionExecute Disposition = "execute" + // DispositionRewriteRequired: the planner says the submitted form + // blocks and must run as a safer idiom, but no executable rewrite was + // constructed (a multi-operation statement, or a pattern the planner + // cannot build). The statement will be refused at execution rather + // than run in its blocking form. + DispositionRewriteRequired Disposition = "rewrite-required" // DispositionUnavailable: the change needs a backend this build does // not implement; the statement would be refused at execution. DispositionUnavailable Disposition = "unavailable" @@ -51,9 +57,15 @@ const ( DispositionRefuse Disposition = "refuse" ) -// worse orders dispositions for aggregation: refuse > unavailable > execute. +// worse orders dispositions for aggregation: +// refuse > unavailable > rewrite-required > execute. func worse(a, b Disposition) Disposition { - rank := map[Disposition]int{DispositionExecute: 0, DispositionUnavailable: 1, DispositionRefuse: 2} + rank := map[Disposition]int{ + DispositionExecute: 0, + DispositionRewriteRequired: 1, + DispositionUnavailable: 2, + DispositionRefuse: 3, + } if rank[b] > rank[a] { return b } @@ -70,7 +82,11 @@ type Statement struct { Disposition Disposition `json:"disposition"` // ExecSQL is the ordered SQL the native backend would run: the // planner's safer sequence when it constructed one, otherwise the - // submitted statement. Empty for non-native routes. + // submitted statement. Execution contract: the steps run one at a + // time, in order, each in its own implicit transaction — never wrapped + // in an enclosing transaction block, which the CONCURRENTLY forms + // refuse. Empty for non-native routes and for statements the engine + // will not run (DispositionRewriteRequired). ExecSQL []string `json:"exec_sql,omitempty"` } @@ -105,7 +121,6 @@ func routeStatement(p planner.Plan) Statement { switch p.Route { case planner.RouteNative: st.Backend = BackendNative - st.ExecSQL = nativeExecSQL(p) case planner.RouteCopyAndSwap: st.Backend = BackendCopyAndSwap case planner.RouteRefuse: @@ -121,6 +136,14 @@ func routeStatement(p planner.Plan) Statement { st.Disposition = DispositionUnavailable return st } + if st.Backend == BackendNative { + sql, ok := nativeExecSQL(p) + if !ok { + st.Disposition = DispositionRewriteRequired + return st + } + st.ExecSQL = sql + } st.Disposition = DispositionExecute return st } @@ -128,10 +151,17 @@ func routeStatement(p planner.Plan) Statement { // nativeExecSQL is the literal SQL the native backend would run for a // native-routed statement: the planner's safer sequence when it constructed // one (only single-operation statements carry one), otherwise the submitted -// form. -func nativeExecSQL(p planner.Plan) []string { +// form — but only when every decision is safe to run as submitted. A +// safer-idiom decision without a constructed rewrite yields no executable +// SQL: running the submitted form would falsify the plan's own reason. +func nativeExecSQL(p planner.Plan) ([]string, bool) { if len(p.Decisions) == 1 && len(p.Decisions[0].SaferSQL) > 0 { - return p.Decisions[0].SaferSQL + return p.Decisions[0].SaferSQL, true + } + for _, d := range p.Decisions { + if !d.ExecutableAsSubmitted() { + return nil, false + } } - return []string{p.Statement} + return []string{p.Statement}, true } diff --git a/pkg/router/router_test.go b/pkg/router/router_test.go index 47d23d6..6531d4a 100644 --- a/pkg/router/router_test.go +++ b/pkg/router/router_test.go @@ -45,6 +45,49 @@ func TestRouteNativeExecutesSaferSequence(t *testing.T) { "the native backend runs the safer sequence, not the submitted form") } +func TestRouteSaferIdiomWithoutRewriteFailsClosed(t *testing.T) { + // ATTACH PARTITION is a safer-idiom decision the planner does not + // construct a rewrite for: routing must not fall back to the + // submitted blocking form. + plan := classify(t, "ALTER TABLE t ATTACH PARTITION p FOR VALUES FROM (1) TO (10)") + require.Equal(t, planner.RouteNative, plan.Route) + require.Len(t, plan.Decisions, 1) + require.Empty(t, plan.Decisions[0].SaferSQL) + require.False(t, plan.Decisions[0].ExecutableAsSubmitted()) + + routed := router.Route([]planner.Plan{plan}) + st := routed.Statements[0] + assert.Equal(t, router.BackendNative, st.Backend) + assert.Equal(t, router.DispositionRewriteRequired, st.Disposition) + assert.Empty(t, st.ExecSQL, "no executable SQL: running the submitted form would falsify the plan") + assert.Equal(t, router.DispositionRewriteRequired, routed.Disposition) +} + +func TestRouteMultiOpPartialRewriteFailsClosed(t *testing.T) { + // SET NOT NULL needs a safer sequence, but multi-operation statements + // carry no rewrites — the submitted form must not run on the strength + // of the harmless sibling operation. + plan := classify(t, "ALTER TABLE t ALTER COLUMN age SET NOT NULL, DROP COLUMN b") + require.Equal(t, planner.RouteNative, plan.Route) + require.Len(t, plan.Decisions, 2) + + routed := router.Route([]planner.Plan{plan}) + st := routed.Statements[0] + assert.Equal(t, router.DispositionRewriteRequired, st.Disposition) + assert.Empty(t, st.ExecSQL) +} + +func TestRouteInlineConstraintAddColumnFailsClosed(t *testing.T) { + plan := classify(t, "ALTER TABLE t ADD COLUMN c int UNIQUE") + require.Equal(t, planner.RouteNative, plan.Route) + + routed := router.Route([]planner.Plan{plan}) + st := routed.Statements[0] + assert.Equal(t, router.DispositionRewriteRequired, st.Disposition) + assert.Empty(t, st.ExecSQL, + "an inline constraint builds its index under ACCESS EXCLUSIVE; the submitted form must not run") +} + func TestRouteCopyAndSwapIsUnavailable(t *testing.T) { plan := classify(t, "ALTER TABLE t ALTER COLUMN id TYPE bigint") require.Equal(t, planner.RouteCopyAndSwap, plan.Route) @@ -72,19 +115,31 @@ func TestRouteRefusedStatementHasNoBackend(t *testing.T) { func TestRouteAggregateIsWorstDisposition(t *testing.T) { plans := []planner.Plan{ classify(t, "ALTER TABLE t ADD COLUMN age int"), + classify(t, "ALTER TABLE t ATTACH PARTITION p FOR VALUES FROM (1) TO (10)"), classify(t, "ALTER TABLE t ALTER COLUMN id TYPE bigint"), classify(t, "ALTER TABLE t ADD CONSTRAINT ex EXCLUDE USING gist (room WITH =)"), } routed := router.Route(plans) - require.Len(t, routed.Statements, 3) + require.Len(t, routed.Statements, 4) assert.Equal(t, router.DispositionExecute, routed.Statements[0].Disposition) - assert.Equal(t, router.DispositionUnavailable, routed.Statements[1].Disposition) - assert.Equal(t, router.DispositionRefuse, routed.Statements[2].Disposition) + assert.Equal(t, router.DispositionRewriteRequired, routed.Statements[1].Disposition) + assert.Equal(t, router.DispositionUnavailable, routed.Statements[2].Disposition) + assert.Equal(t, router.DispositionRefuse, routed.Statements[3].Disposition) assert.Equal(t, router.DispositionRefuse, routed.Disposition, "one refusal refuses the whole plan") } +func TestRouteAggregateRanksRewriteRequiredBelowUnavailable(t *testing.T) { + plans := []planner.Plan{ + classify(t, "ALTER TABLE t ATTACH PARTITION p FOR VALUES FROM (1) TO (10)"), + classify(t, "ALTER TABLE t ADD COLUMN age int"), + } + routed := router.Route(plans) + assert.Equal(t, router.DispositionRewriteRequired, routed.Disposition, + "one rewrite-required statement blocks the whole plan") +} + func TestRouteEmptyPlanExecutes(t *testing.T) { routed := router.Route(nil) assert.Empty(t, routed.Statements) diff --git a/pkg/statement/ops.go b/pkg/statement/ops.go index 682451b..f28a8b2 100644 --- a/pkg/statement/ops.go +++ b/pkg/statement/ops.go @@ -99,6 +99,15 @@ type Op struct { Unique bool // GeneratedStored is true for ADD COLUMN ... GENERATED ... STORED. GeneratedStored bool + // InlineConstraints are the table-constraint families an added column + // carries inline (UNIQUE, PRIMARY KEY, REFERENCES, CHECK) — each does + // the same index build or validation scan as its ADD CONSTRAINT form. + // An inline constraint the engine does not model is reported as + // ConstraintUnrecognized so the classifier can refuse it. + InlineConstraints []ConstraintKind + // PartitionOf is true for CREATE TABLE ... PARTITION OF, which locks + // the partitioned parent, not just the new relation. + PartitionOf bool // Default is the DEFAULT shape for OpAddColumn. Default DefaultKind // NewType is the target type for OpAlterColumnType and the column type @@ -118,6 +127,9 @@ type Op struct { func (o Op) Describe() string { switch o.Kind { case OpCreateTable: + if o.PartitionOf { + return "CREATE TABLE PARTITION OF" + } return "CREATE TABLE" case OpAddColumn: return "ADD COLUMN " + o.Column @@ -194,7 +206,10 @@ func ParseOps(sql string) ([]Op, error) { } return ops, nil case node.GetCreateStmt() != nil: - return []Op{{Kind: OpCreateTable}}, nil + return []Op{{ + Kind: OpCreateTable, + PartitionOf: node.GetCreateStmt().GetPartbound() != nil, + }}, nil case node.GetIndexStmt() != nil: idx := node.GetIndexStmt() return []Op{{ @@ -283,9 +298,13 @@ func alterTableOp(cmd *pganalyze.AlterTableCmd) Op { } // addColumnOp extracts the shape facts of an added column: its DEFAULT -// shape and whether it is a stored generated column. Identity and serial -// columns are reported as expression defaults — their values come from a -// sequence, which fast default cannot cover. +// shape, whether it is a stored generated column, and any inline table +// constraints it carries. Identity and serial columns are reported as +// expression defaults — their values come from a sequence, which fast +// default cannot cover. Nullability clauses and the deferrability +// attributes that modify a preceding FOREIGN KEY add no work of their own +// and are not reported; any constraint family the engine does not model +// is reported as ConstraintUnrecognized so the classifier fails closed. func addColumnOp(def *pganalyze.ColumnDef) Op { op := Op{Kind: OpAddColumn, Column: def.GetColname()} op.NewType, op.NewTypeMods = typeRef(def.GetTypeName()) @@ -305,6 +324,23 @@ func addColumnOp(def *pganalyze.ColumnDef) Op { op.Default = DefaultExpression case pganalyze.ConstrType_CONSTR_GENERATED: op.GeneratedStored = true + case pganalyze.ConstrType_CONSTR_NULL, + pganalyze.ConstrType_CONSTR_NOTNULL, + pganalyze.ConstrType_CONSTR_ATTR_DEFERRABLE, + pganalyze.ConstrType_CONSTR_ATTR_NOT_DEFERRABLE, + pganalyze.ConstrType_CONSTR_ATTR_DEFERRED, + pganalyze.ConstrType_CONSTR_ATTR_IMMEDIATE: + // No scan and no index build of their own. + case pganalyze.ConstrType_CONSTR_UNIQUE: + op.InlineConstraints = append(op.InlineConstraints, ConstraintUnique) + case pganalyze.ConstrType_CONSTR_PRIMARY: + op.InlineConstraints = append(op.InlineConstraints, ConstraintPrimaryKey) + case pganalyze.ConstrType_CONSTR_FOREIGN: + op.InlineConstraints = append(op.InlineConstraints, ConstraintForeignKey) + case pganalyze.ConstrType_CONSTR_CHECK: + op.InlineConstraints = append(op.InlineConstraints, ConstraintCheck) + default: + op.InlineConstraints = append(op.InlineConstraints, ConstraintUnrecognized) } } return op diff --git a/pkg/statement/ops_test.go b/pkg/statement/ops_test.go index 93b4ebd..cadba02 100644 --- a/pkg/statement/ops_test.go +++ b/pkg/statement/ops_test.go @@ -63,6 +63,43 @@ func TestParseOpsShapes(t *testing.T) { sql: "ALTER TABLE t ADD COLUMN total numeric GENERATED ALWAYS AS (price * qty) STORED", want: statement.Op{Kind: statement.OpAddColumn, Column: "total", NewType: "numeric", GeneratedStored: true}, }, + { + name: "add column inline unique", + sql: "ALTER TABLE t ADD COLUMN c int UNIQUE", + want: statement.Op{Kind: statement.OpAddColumn, Column: "c", NewType: "int4", + InlineConstraints: []statement.ConstraintKind{statement.ConstraintUnique}}, + }, + { + name: "add column inline primary key", + sql: "ALTER TABLE t ADD COLUMN c int PRIMARY KEY", + want: statement.Op{Kind: statement.OpAddColumn, Column: "c", NewType: "int4", + InlineConstraints: []statement.ConstraintKind{statement.ConstraintPrimaryKey}}, + }, + { + name: "add column inline foreign key", + sql: "ALTER TABLE t ADD COLUMN c int REFERENCES parent (id)", + want: statement.Op{Kind: statement.OpAddColumn, Column: "c", NewType: "int4", + InlineConstraints: []statement.ConstraintKind{statement.ConstraintForeignKey}}, + }, + { + name: "add column inline check", + sql: "ALTER TABLE t ADD COLUMN c int CHECK (c > 0)", + want: statement.Op{Kind: statement.OpAddColumn, Column: "c", NewType: "int4", + InlineConstraints: []statement.ConstraintKind{statement.ConstraintCheck}}, + }, + { + name: "add column inline check with constant default", + sql: "ALTER TABLE t ADD COLUMN c int DEFAULT 0 CHECK (c > 0)", + want: statement.Op{Kind: statement.OpAddColumn, Column: "c", NewType: "int4", + Default: statement.DefaultConstant, + InlineConstraints: []statement.ConstraintKind{statement.ConstraintCheck}}, + }, + { + name: "add column not null carries no inline constraint", + sql: "ALTER TABLE t ADD COLUMN c int NOT NULL DEFAULT 0", + want: statement.Op{Kind: statement.OpAddColumn, Column: "c", NewType: "int4", + Default: statement.DefaultConstant}, + }, { name: "drop column", sql: "ALTER TABLE t DROP COLUMN age",