From 719d788eb662d9bf0e80b371be6ccab4263d73ca Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 20:43:49 +0000 Subject: [PATCH 1/2] test, docs: pg_partman manages columnar partitions, with one setting (#1291) The owner's question on #1291: does pg_partman work with columnar partitions on every major? If it does, document and test it. Measured with pg_partman 5.4.1 on PostgreSQL 15..19, and now asserted by partman_columnar.sh and its pytest twin, one claim per sentence of the new docs/how-to.md section: no setting every partition heap, even on 17+ with a USING parent default_table_access_method = pgcolumnar every partition columnar, DEFAULT and premade included; rows land there and read back as heap does retention partitions dropped, no storage or row-group row left run_maintenance holds AccessExclusiveLock on the parent The first arm pins the trap, so a pg_partman that starts honouring the parent's access method turns it red instead of leaving the warning stale. Removal proof: a drop hook that stops deleting row_group and storage rows leaves 20 orphans on every major (pytest twin: 18). CI: postgresql-N-partman from PGDG in the suites and pytest-cluster jobs of ci.yml and nightly.yml, in each cache key as well as each install. A box without pg_partman FAILS the suite unless PGC_ALLOW_MISSING_PG_PARTMAN=1. partman_columnar 12 passed, PG 15..19; mutation 11 + 1 red, every major pytest twin 1 passed (12 checks); mutation red; parity 12/12 harness_selftest, docs_style, guard half 404: pass ledger 1866 -> 1878, never 1653 -> 1664 Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 6 +- .github/workflows/nightly.yml | 6 +- CHANGELOG.md | 20 +++++ docs/how-to.md | 54 ++++++++++++ test/check_ledger.tsv | 12 +++ test/partman_columnar.sh | 111 +++++++++++++++++++++++ test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_partman_columnar.py | 127 +++++++++++++++++++++++++++ test/run_all_versions.sh | 1 + 9 files changed, 332 insertions(+), 6 deletions(-) create mode 100755 test/partman_columnar.sh create mode 100644 test/pytest/test_partman_columnar.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d42d4d90..c53a7497 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -184,7 +184,7 @@ jobs: | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null sudo timeout 300 apt-get -o Acquire::Retries=5 update sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ - postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ + postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman \ liblz4-dev libzstd-dev zlib1g-dev # NOT cached, deliberately. The suites job caches because it runs a matrix for # an hour; a second copy of that cache-key derivation is a thing that goes @@ -580,7 +580,7 @@ jobs: run: | set -euo pipefail v=$(apt-cache policy \ - postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ + postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman \ liblz4-dev libzstd-dev zlib1g-dev python3-pip \ | awk '/Candidate:/ {print $2}' | sort) # $ImageOS, from the shell, where it actually has a value. Reported @@ -628,7 +628,7 @@ jobs: mkdir -p "$HOME/apt-archives/partial" sudo timeout 600 apt-get -o Acquire::Retries=5 \ -o Dir::Cache::archives="$HOME/apt-archives" install -y --no-install-recommends \ - postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ + postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman \ liblz4-dev libzstd-dev zlib1g-dev python3-pip 2>&1 | tee /tmp/apt-install.log # The Arrow and Parquet suites use pyarrow as an independent reader and # skip themselves without it, which would be a silent loss of coverage. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 41e5059b..0a3ff68e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -126,7 +126,7 @@ jobs: run: | set -euo pipefail v=$(apt-cache policy \ - postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ + postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman \ liblz4-dev libzstd-dev zlib1g-dev python3-pip \ | awk '/Candidate:/ {print $2}' | sort) # $ImageOS, from the shell, where it actually has a value. Reported @@ -156,7 +156,7 @@ jobs: mkdir -p "$HOME/apt-archives/partial" sudo timeout 600 apt-get -o Acquire::Retries=5 \ -o Dir::Cache::archives="$HOME/apt-archives" install -y --no-install-recommends \ - postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ + postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman \ liblz4-dev libzstd-dev zlib1g-dev python3-pip 2>&1 | tee /tmp/apt-install.log # pyarrow for root (the suites run under sudo); the silent-skip trap the # per-PR workflow documents applies here too. @@ -743,7 +743,7 @@ jobs: | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null sudo timeout 300 apt-get -o Acquire::Retries=5 update sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ - postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ + postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman \ liblz4-dev libzstd-dev zlib1g-dev # NOT cached, deliberately. The suites job caches because it runs a matrix for # an hour; a second copy of that cache-key derivation is a thing that goes diff --git a/CHANGELOG.md b/CHANGELOG.md index 521f328a..85a87d7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,26 @@ true until the next version shipped. showed a vacuous arm: "under a tenth of the table" passed on empty SQL, because a query that reads nothing reads under a tenth of anything. It now requires at least one block read. +- `docs/how-to.md` documents managing time partitions with pg_partman (#1291), and + `test/partman_columnar.sh` with its pytest twin checks every statement the + section makes. Measured with pg_partman 5.4.1 on PostgreSQL 15 to 19: + + without default_table_access_method every partition is heap, even on 17+ + where the parent carries USING pgcolumnar + with it every partition is columnar, including the + DEFAULT one and those made ahead; rows + land there and read back as heap does + retention partitions dropped, no pgcolumnar + storage or row-group row left behind + run_maintenance holds ACCESS EXCLUSIVE on the parent + + The first arm pins the trap rather than the recipe, so a pg_partman that starts + honouring the parent's access method turns it red instead of leaving the warning + stale. Without the drop hook's catalog cleanup, the retention arm reads 20 + orphaned storage rows. pg_partman is an external extension, so a box without it + FAILS the suite unless `PGC_ALLOW_MISSING_PG_PARTMAN=1` is set. The CI suites + and pytest jobs now install `postgresql-N-partman` from PGDG, in the cache key + as well as the install. ### Changed diff --git a/docs/how-to.md b/docs/how-to.md index ad47e5d6..0fecd064 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -323,6 +323,60 @@ declared retention raises an error rather than doing nothing. `pgcolumnar.reset_options` has no argument for the retention. To stop expiring, set `ttl_interval` to a span no row will reach. +## Manage time partitions with pg_partman + +pg_partman can create and drop columnar partitions. It needs one setting to do +so, and without it the partitions it makes are not columnar. + +Before you start: install pg_partman (PGDG ships it as `postgresql-N-partman`), +and create it in its own schema. + +```sql +CREATE SCHEMA partman; +CREATE EXTENSION pg_partman SCHEMA partman; +``` + +1. Make columnar the default access method where pg_partman runs. A database + setting is the simplest place. + +```sql +ALTER DATABASE mydb SET default_table_access_method = 'pgcolumnar'; +``` + +2. Create the parent and hand it to pg_partman. + +```sql +CREATE TABLE metrics (ts timestamptz NOT NULL, v float8) + PARTITION BY RANGE (ts); +SELECT partman.create_parent('public.metrics', 'ts', '1 day', p_premake := 4); +``` + +3. Set the retention, and run maintenance on a schedule. + +```sql +UPDATE partman.part_config + SET retention = '90 days', retention_keep_table = false + WHERE parent_table = 'public.metrics'; +SELECT partman.run_maintenance('public.metrics'); +``` + +**Without step 1, every partition is a heap table.** pg_partman does not take the +access method from the parent. That holds even on PostgreSQL 17 and later, where +the parent itself can carry `USING pgcolumnar`. Changing the access method of +pg_partman's template table does not help either. Measured with pg_partman 5.4.1 +on PostgreSQL 15 to 19. + +**The setting has a cost.** Every table created in that database becomes +columnar unless it names another access method. + +**Maintenance locks the parent.** `run_maintenance` holds `ACCESS EXCLUSIVE` on +the parent table while it runs, so queries on the table wait for it. pg_partman +also creates a `DEFAULT` partition. + +When a partition is dropped by retention, its pgcolumnar catalog rows go with it. + +`test/partman_columnar.sh` checks each of these statements. + ## Keep tables optimized automatically A background daemon can compact and recluster tables on a schedule. The threshold diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 1903f5f7..dbf40d5e 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -1759,6 +1759,18 @@ parallel_scan_cost parallel_scan_cost premise: the serial plan has no Gather 15; parallel_scan_cost parallel_scan_cost premise: the serial plan is a columnar scan 15;16;17;18;19 never - parallel_scan_cost parallel_scan_cost premise: the table holds every inserted row 15;16;17;18;19 never - parallel_scan_cost parallel_scan_cost the leader-participation branch changes the divisor 15;16;17;18;19 never - +partman_columnar partman_columnar and no row-group row behind 15;16;17;18;19 never - +partman_columnar partman_columnar and read back as heap does 15;16;17;18;19 never - +partman_columnar partman_columnar and the rows land in columnar partitions 15;16;17;18;19 never - +partman_columnar partman_columnar pg_partman creates a DEFAULT partition, as docs/how-to.md says 15;16;17;18;19 never - +partman_columnar partman_columnar premise: pg_partman is installed in this database 15;16;17;18;19 never - +partman_columnar partman_columnar premise: retention dropped partitions 15;16;17;18;19 never - +partman_columnar partman_columnar premise: the rows went in 15;16;17;18;19 never - +partman_columnar partman_columnar retention leaves no pgcolumnar storage row behind 15;16;17;18;19 2026-10-12 the drop hook stops deleting a storage's row_group and storage rows, so a partition pg_partman drops leaves them behind +partman_columnar partman_columnar run_maintenance holds an exclusive lock on the parent, as docs/how-to.md says 15;16;17;18;19 never - +partman_columnar partman_columnar the partitions maintenance creates ahead are columnar too 15;16;17;18;19 never - +partman_columnar partman_columnar with default_table_access_method every partition pg_partman makes is columnar 15;16;17;18;19 never - +partman_columnar partman_columnar without default_table_access_method pg_partman makes heap partitions 15;16;17;18;19 never - projection_parallel projection_parallel a covering projection can be a parallel scan 15;16;17;18;19 2026-09-18 - projection_parallel projection_parallel a page cost that makes I/O dominate costs the parallel covering path 15;16;17;18;19 2026-09-23 the partial covering total amortises I/O as well as CPU: (ioRunProj + cpuRunProj) / divisor projection_parallel projection_parallel a parallel covering projection returns the covering rows once 15;16;17;18;19 2026-10-12 a parallel worker never wires the shared row-group counter, so every worker scans every group diff --git a/test/partman_columnar.sh b/test/partman_columnar.sh new file mode 100755 index 00000000..c2b67e45 --- /dev/null +++ b/test/partman_columnar.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# +# pg_partman manages columnar partitions -- with one setting, and not without it (#1291). +# +# The owner's question on #1291 was whether pg_partman works with columnar +# partitions on every major; if it does, we document AND TEST that. Measured with +# pg_partman 5.4.1 on PG 15..19, and asserted here: +# +# OUT OF THE BOX, every partition pg_partman makes is HEAP -- even on 17+, where +# the parent itself carries USING pgcolumnar. pg_partman does not take the access +# method from the parent, and switching its template table does not help either. +# +# WITH default_table_access_method = pgcolumnar where its maintenance runs, every +# partition it makes is columnar, the rows land there, and retention drops +# partitions leaving no pgcolumnar catalog row behind. +# +# run_maintenance holds ACCESS EXCLUSIVE on the parent. That is pg_partman's +# behaviour, not ours; it is asserted because docs/how-to.md tells users so, and a +# pg_partman that changed it should turn this red rather than leave the doc wrong. +# +# The first arm pins the trap rather than the recipe, on purpose: it is the reason +# the documentation exists, and a pg_partman that started honouring the parent's +# access method should be noticed, not silently make the warning stale. +# +# pg_partman is an external extension. A box without it FAILS this suite (lib.sh's +# rule for every capability but pyarrow); set PGC_ALLOW_MISSING_PG_PARTMAN=1 to make +# that a named skip. CI installs postgresql-N-partman from PGDG. +# +# Usage: test/partman_columnar.sh [PG_CONFIG] + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +q "CREATE EXTENSION IF NOT EXISTS pgcolumnar;" >/dev/null +if [ "$(q "SELECT count(*) FROM pg_available_extensions WHERE name = 'pg_partman';")" != 1 ]; then + pgc_skip pg_partman "pg_partman is not installed for this major" +fi +q "CREATE SCHEMA partman; CREATE EXTENSION pg_partman SCHEMA partman;" >/dev/null +check_text "premise: pg_partman is installed in this database" \ + "$(q "SELECT count(*) FROM pg_extension WHERE extname = 'pg_partman';")" "1" +echo "-- pg_partman $(q "SELECT extversion FROM pg_extension WHERE extname = 'pg_partman';")" + +# am_counts PARENT -> "am=n am=n" over the partitions pg_partman made +am_counts() { + q "SELECT coalesce(string_agg(amname || '=' || n, ' ' ORDER BY amname), 'none') + FROM (SELECT a.amname, count(*) n FROM pg_inherits i + JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_am a ON a.oid = c.relam + WHERE i.inhparent = '$1'::regclass GROUP BY 1) x;" +} +# On 17+ a partitioned parent can carry USING; give it to them where it can be +# given, so the first arm shows pg_partman ignores even that. +using=""; [ "$PGC_MAJOR" -ge 17 ] && using="USING pgcolumnar" + +# ---- 1. the trap: no setting, heap partitions ------------------------------- +q "CREATE TABLE plain_m (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts) $using;" >/dev/null +q "SELECT partman.create_parent('public.plain_m', 'ts', '1 day', p_premake := 2, + p_start_partition := (date_trunc('day', now()) - interval '2 days')::text);" >/dev/null +check "without default_table_access_method pg_partman makes heap partitions" \ + "$(am_counts plain_m | sed -E 's/=[0-9]+//g')" "heap" + +# ---- 2. the recipe ----------------------------------------------------------- +q "ALTER DATABASE \"$PGC_DB\" SET default_table_access_method = 'pgcolumnar';" >/dev/null +q "CREATE TABLE ts_m (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts) $using;" >/dev/null +q "SELECT partman.create_parent('public.ts_m', 'ts', '1 day', p_premake := 3, + p_start_partition := (date_trunc('day', now()) - interval '40 days')::text);" >/dev/null +check_num "pg_partman creates a DEFAULT partition, as docs/how-to.md says" \ + "$(q "SELECT count(*) FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = 'ts_m'::regclass + AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT';")" "1" +check "with default_table_access_method every partition pg_partman makes is columnar" \ + "$(am_counts ts_m)" "pgcolumnar=$(q "SELECT count(*) FROM pg_inherits WHERE inhparent = 'ts_m'::regclass;")" + +q "CREATE TABLE ts_h (ts timestamptz, v int) USING heap;" >/dev/null +for t in ts_m ts_h; do + q "INSERT INTO $t SELECT date_trunc('day', now()) - interval '40 days' + g * interval '17 min', g + FROM generate_series(1, 3000) g;" >/dev/null +done +check_num "premise: the rows went in" "$(q "SELECT count(*) FROM ts_m;")" "3000" +check "and the rows land in columnar partitions" \ + "$(q "SELECT string_agg(DISTINCT a.amname, ',') FROM ts_m t + JOIN pg_class c ON c.oid = t.tableoid JOIN pg_am a ON a.oid = c.relam;")" "pgcolumnar" +check "and read back as heap does" \ + "$(q "SELECT count(*), sum(v), min(ts), max(ts) FROM ts_m;")" \ + "$(q "SELECT count(*), sum(v), min(ts), max(ts) FROM ts_h;")" + +# ---- 3. maintenance: ahead, retention, and the parent lock ------------------ +q "UPDATE partman.part_config SET premake = 6, retention = '20 days', + retention_keep_table = false WHERE parent_table = 'public.ts_m';" >/dev/null +st_before="$(q "SELECT count(*) FROM pgcolumnar.storage;")" +kids_before="$(q "SELECT count(*) FROM pg_inherits WHERE inhparent = 'ts_m'::regclass;")" +lock="$(q "BEGIN; SELECT partman.run_maintenance('public.ts_m'); + SELECT coalesce(string_agg(DISTINCT mode, ','), 'none') FROM pg_locks + WHERE relation = 'ts_m'::regclass AND pid = pg_backend_pid(); COMMIT;" | grep -vE '^(BEGIN|COMMIT)$|^$' | tail -1)" +kids_after="$(q "SELECT count(*) FROM pg_inherits WHERE inhparent = 'ts_m'::regclass;")" +echo "-- partitions $kids_before -> $kids_after, storage rows $st_before -> $(q "SELECT count(*) FROM pgcolumnar.storage;")" +dropped=$((kids_before - kids_after)) +check "premise: retention dropped partitions" \ + "$([ "$dropped" -ge 1 ] && echo dropped || echo "dropped $dropped")" "dropped" +check "the partitions maintenance creates ahead are columnar too" \ + "$(am_counts ts_m)" "pgcolumnar=$kids_after" +check_num "retention leaves no pgcolumnar storage row behind" \ + "$(q "SELECT count(*) FROM pgcolumnar.storage s + WHERE NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.oid = s.relation_oid);")" "0" +check_num "and no row-group row behind" \ + "$(q "SELECT count(*) FROM pgcolumnar.row_group r + WHERE NOT EXISTS (SELECT 1 FROM pgcolumnar.storage s WHERE s.storage_id = r.storage_id);")" "0" +check "run_maintenance holds an exclusive lock on the parent, as docs/how-to.md says" \ + "$lock" "AccessExclusiveLock" + +pgc_summary diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 14976e0a..cd584275 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -117,6 +117,7 @@ "native_vacuum_race", "parallel_am_scan", "parallel_scan_cost", + "partman_columnar", "projection_drop_column", "projection_parallel", "projection_privilege", diff --git a/test/pytest/test_partman_columnar.py b/test/pytest/test_partman_columnar.py new file mode 100644 index 00000000..e995d543 --- /dev/null +++ b/test/pytest/test_partman_columnar.py @@ -0,0 +1,127 @@ +"""pg_partman manages columnar partitions, with one setting and not without it (#1291). + +The owner's question on #1291: does pg_partman work with columnar partitions on +every major? If it does, we document and test that. Measured with pg_partman +5.4.1 on PG 15..19: + +- Out of the box every partition it makes is HEAP, even on 17+ where the parent + itself carries USING pgcolumnar. It does not take the parent's access method. +- With `default_table_access_method = pgcolumnar` where its maintenance runs, + every partition it makes is columnar, the rows land there, and retention drops + partitions leaving no pgcolumnar catalog row behind. +- `run_maintenance` holds ACCESS EXCLUSIVE on the parent. That is pg_partman's + behaviour; it is asserted because docs/how-to.md tells users so. + +The first arm pins the TRAP, on purpose: it is why the documentation exists, and +a pg_partman that began honouring the parent's access method must turn it red +rather than leave the warning stale. + +ON A PRIVATE DATABASE (`pgc_own_db`): the recipe is a database-level setting, and +retention is checked against the whole storage catalog. pg_partman missing is +MISSING_DEPENDENCY (unrunnable, which fails the run), never a pass; CI installs +postgresql-N-partman from PGDG. + +Independent of `test/partman_columnar.sh`: its own tables, partition widths, +dates, retention window and row counts. +""" + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + row = cur.fetchone() + return row[0] if row else None + + +def _am_counts(conn, parent): + return _one(conn, + "SELECT coalesce(string_agg(amname || '=' || n, ' ' ORDER BY amname), 'none')" + " FROM (SELECT a.amname, count(*) n FROM pg_inherits i" + " JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_am a ON a.oid = c.relam" + f" WHERE i.inhparent = '{parent}'::regclass GROUP BY 1) x") + + +def _kids(conn, parent): + return _one(conn, f"SELECT count(*) FROM pg_inherits WHERE inhparent = '{parent}'::regclass") + + +def test_pg_partman_makes_columnar_partitions_with_one_setting(pgc_own_db, expect): + conn = pgc_own_db + if _one(conn, "SELECT count(*) FROM pg_available_extensions WHERE name = 'pg_partman'") != 1: + expect.cannot_run("MISSING_DEPENDENCY", "pg_partman is not installed for this major", + name="pg_partman is not installed for this major") + return + with conn.cursor() as cur: + cur.execute("CREATE SCHEMA pm") + cur.execute("CREATE EXTENSION pg_partman SCHEMA pm") + expect.num(_one(conn, "SELECT count(*) FROM pg_extension WHERE extname = 'pg_partman'"), 1, + "premise: pg_partman is installed in this database") + major = int(_one(conn, "SHOW server_version_num")) // 10000 + using = "USING pgcolumnar" if major >= 17 else "" + + # The trap: no setting, heap partitions. + with conn.cursor() as cur: + cur.execute(f"CREATE TABLE raw_events (at timestamptz NOT NULL, n bigint)" + f" PARTITION BY RANGE (at) {using}") + cur.execute("SELECT pm.create_parent('public.raw_events', 'at', '1 week', p_premake := 2," + " p_start_partition := (date_trunc('week', now()) - interval '1 week')::text)") + expect.text(" ".join(x.split("=")[0] for x in _am_counts(conn, "raw_events").split()), "heap", + "without default_table_access_method pg_partman makes heap partitions") + + # The recipe. + dbname = _one(conn, "SELECT current_database()") + with conn.cursor() as cur: + cur.execute(f'ALTER DATABASE "{dbname}" SET default_table_access_method = \'pgcolumnar\'') + cur.execute("SET default_table_access_method = 'pgcolumnar'") + cur.execute(f"CREATE TABLE metrics (at timestamptz NOT NULL, n bigint)" + f" PARTITION BY RANGE (at) {using}") + cur.execute("SELECT pm.create_parent('public.metrics', 'at', '1 day', p_premake := 2," + " p_start_partition := (date_trunc('day', now()) - interval '30 days')::text)") + expect.num(_one(conn, "SELECT count(*) FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid" + " WHERE i.inhparent = 'metrics'::regclass" + " AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'"), 1, + "pg_partman creates a DEFAULT partition, as docs/how-to.md says") + kids = _kids(conn, "metrics") + expect.text(_am_counts(conn, "metrics"), f"pgcolumnar={kids}", + "with default_table_access_method every partition pg_partman makes is columnar") + + with conn.cursor() as cur: + cur.execute("CREATE TABLE metrics_heap (at timestamptz, n bigint) USING heap") + for t in ("metrics", "metrics_heap"): + cur.execute(f"INSERT INTO {t} SELECT date_trunc('day', now()) - interval '30 days'" + f" + g * interval '11 min', g * 3 FROM generate_series(1, 3600) g") + expect.num(_one(conn, "SELECT count(*) FROM metrics"), 3600, "premise: the rows went in") + expect.text(_one(conn, "SELECT string_agg(DISTINCT a.amname, ',') FROM metrics t" + " JOIN pg_class c ON c.oid = t.tableoid JOIN pg_am a ON a.oid = c.relam"), + "pgcolumnar", "and the rows land in columnar partitions") + agg = "SELECT count(*)::text || '|' || sum(n) || '|' || min(at) || '|' || max(at) FROM {}" + expect.text(_one(conn, agg.format("metrics")), _one(conn, agg.format("metrics_heap")), + "and read back as heap does") + + # Maintenance: created ahead, retention, and the parent lock. + with conn.cursor() as cur: + cur.execute("UPDATE pm.part_config SET premake = 5, retention = '12 days'," + " retention_keep_table = false WHERE parent_table = 'public.metrics'") + before = _kids(conn, "metrics") + # One explicit transaction, so the lock is still held when it is read. + with conn.cursor() as cur: + cur.execute("BEGIN") + cur.execute("SELECT pm.run_maintenance('public.metrics')") + cur.execute("SELECT coalesce(string_agg(DISTINCT mode, ','), 'none') FROM pg_locks" + " WHERE relation = 'metrics'::regclass AND pid = pg_backend_pid()") + lock = cur.fetchone()[0] + cur.execute("COMMIT") + after = _kids(conn, "metrics") + print(f"-- partitions {before} -> {after}") + expect.text("dropped" if after < before else f"dropped {before - after}", "dropped", + "premise: retention dropped partitions") + expect.text(_am_counts(conn, "metrics"), f"pgcolumnar={after}", + "the partitions maintenance creates ahead are columnar too") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.storage s WHERE NOT EXISTS" + " (SELECT 1 FROM pg_class c WHERE c.oid = s.relation_oid)"), 0, + "retention leaves no pgcolumnar storage row behind") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.row_group r WHERE NOT EXISTS" + " (SELECT 1 FROM pgcolumnar.storage s WHERE s.storage_id = r.storage_id)"), 0, + "and no row-group row behind") + expect.text(lock, "AccessExclusiveLock", + "run_maintenance holds an exclusive lock on the parent, as docs/how-to.md says") diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 499e3650..a537c3b2 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -248,6 +248,7 @@ SUITES=( parquet_level_width parquet_nested parquet_nested_import + partman_columnar pg19_vacuum_options pg_dump_roundtrip phase2 From 345a18e7b37eb161fea4181b70e67e6a69975275 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 20:54:54 +0000 Subject: [PATCH 2/2] test: partman_columnar per review -- premises for what no change here can redden, one mutation per arm (#1302) The six checks asserting core's and pg_partman's behaviour (the access method a partition gets, the DEFAULT partition, the parent lock) are premises by the owner's #1236 rule. Each remaining arm is dated by its own mutation on PG 15..19: the drop hook keeping storage rows, keeping row_group rows (jdatcmd's catch: the first mutation removed both, so the row-group arm could not fail), and a scan dropping its last group. The read-back arm hashes every row in order; an unfiltered count/sum/min/max is answered from zone maps and would not reach a scan. Ledger 1866 -> 1878, never 1653 -> 1660, every arm dated. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 20 ++++++++++---------- test/check_ledger_budget.txt | 20 +++++++++++++++++++- test/partman_columnar.sh | 26 ++++++++++++++++++-------- test/pytest/TESTS.md | 21 +++++++++++++++++++++ test/pytest/expected_tests.txt | 2 +- test/pytest/test_partman_columnar.py | 14 ++++++++------ 6 files changed, 77 insertions(+), 26 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index dbf40d5e..0e449de9 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -1759,18 +1759,18 @@ parallel_scan_cost parallel_scan_cost premise: the serial plan has no Gather 15; parallel_scan_cost parallel_scan_cost premise: the serial plan is a columnar scan 15;16;17;18;19 never - parallel_scan_cost parallel_scan_cost premise: the table holds every inserted row 15;16;17;18;19 never - parallel_scan_cost parallel_scan_cost the leader-participation branch changes the divisor 15;16;17;18;19 never - -partman_columnar partman_columnar and no row-group row behind 15;16;17;18;19 never - -partman_columnar partman_columnar and read back as heap does 15;16;17;18;19 never - -partman_columnar partman_columnar and the rows land in columnar partitions 15;16;17;18;19 never - -partman_columnar partman_columnar pg_partman creates a DEFAULT partition, as docs/how-to.md says 15;16;17;18;19 never - +partman_columnar partman_columnar and no row-group row behind 15;16;17;18;19 2026-10-12 the drop hook stops deleting row_group rows, so they outlive their deleted storage row +partman_columnar partman_columnar and read back as heap does 15;16;17;18;19 2026-10-12 a sequential scan drops the last row group from its list, so every one-group partition reads empty +partman_columnar partman_columnar and the rows land in columnar partitions 15;16;17;18;19 2026-10-12 a sequential scan drops the last row group from its list, so every one-group partition reads empty +partman_columnar partman_columnar premise: pg_partman creates a DEFAULT partition, as docs/how-to.md says 15;16;17;18;19 never - partman_columnar partman_columnar premise: pg_partman is installed in this database 15;16;17;18;19 never - partman_columnar partman_columnar premise: retention dropped partitions 15;16;17;18;19 never - -partman_columnar partman_columnar premise: the rows went in 15;16;17;18;19 never - -partman_columnar partman_columnar retention leaves no pgcolumnar storage row behind 15;16;17;18;19 2026-10-12 the drop hook stops deleting a storage's row_group and storage rows, so a partition pg_partman drops leaves them behind -partman_columnar partman_columnar run_maintenance holds an exclusive lock on the parent, as docs/how-to.md says 15;16;17;18;19 never - -partman_columnar partman_columnar the partitions maintenance creates ahead are columnar too 15;16;17;18;19 never - -partman_columnar partman_columnar with default_table_access_method every partition pg_partman makes is columnar 15;16;17;18;19 never - -partman_columnar partman_columnar without default_table_access_method pg_partman makes heap partitions 15;16;17;18;19 never - +partman_columnar partman_columnar premise: run_maintenance holds an exclusive lock on the parent, as docs/how-to.md says 15;16;17;18;19 never - +partman_columnar partman_columnar premise: the partitions maintenance creates ahead are columnar too 15;16;17;18;19 never - +partman_columnar partman_columnar premise: the rows went in 15;16;17;18;19 2026-10-12 a sequential scan drops the last row group from its list, so every one-group partition reads empty +partman_columnar partman_columnar premise: with default_table_access_method every partition pg_partman makes is columnar 15;16;17;18;19 never - +partman_columnar partman_columnar premise: without default_table_access_method pg_partman makes heap partitions 15;16;17;18;19 never - +partman_columnar partman_columnar retention leaves no pgcolumnar storage row behind 15;16;17;18;19 2026-10-12 the drop hook stops deleting the storage row, so a partition pg_partman drops leaves it behind projection_parallel projection_parallel a covering projection can be a parallel scan 15;16;17;18;19 2026-09-18 - projection_parallel projection_parallel a page cost that makes I/O dominate costs the parallel covering path 15;16;17;18;19 2026-09-23 the partial covering total amortises I/O as well as CPU: (ioRunProj + cpuRunProj) / divisor projection_parallel projection_parallel a parallel covering projection returns the covering rows once 15;16;17;18;19 2026-10-12 a parallel worker never wires the shared row-group counter, so every worker scans every group diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 96f8a404..b28c4209 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1258,4 +1258,22 @@ suites_not_covered 247 # onto #1303's ledger, not merged by hand. # # never 1660 + 4 = 1664 -checks_never_observed_red 1664 +# #1291 (OffgridwithJD): 1664 -> 1671. partman_columnar enters the ledger with +# 12 checks from five-major runs, replayed from the saved logs onto this base. +# Every arm is dated, each by its own mutation: +# +# drop hook keeps the storage row retention leaves no pgcolumnar storage row behind +# drop hook keeps row_group rows and no row-group row behind +# the scan drops its last group and read back as heap does, +# and the rows land in columnar partitions +# (and the premise "the rows went in") +# +# The 7 still `never` are premises: the access method core gives a partition, +# pg_partman's DEFAULT partition and its parent lock. No change to this +# repository can redden them (the owner's #1236 rule). +# +# ledger rows 1883 + 12 = 1895 +# never 1664 + 7 = 1671 +# dated 219 + 5 = 224 +# 1671 + 224 == 1895 +checks_never_observed_red 1671 diff --git a/test/partman_columnar.sh b/test/partman_columnar.sh index c2b67e45..291da24a 100755 --- a/test/partman_columnar.sh +++ b/test/partman_columnar.sh @@ -18,10 +18,18 @@ # behaviour, not ours; it is asserted because docs/how-to.md tells users so, and a # pg_partman that changed it should turn this red rather than leave the doc wrong. # -# The first arm pins the trap rather than the recipe, on purpose: it is the reason +# The first check pins the trap rather than the recipe, on purpose: it is the reason # the documentation exists, and a pg_partman that started honouring the parent's # access method should be noticed, not silently make the warning stale. # +# CHECKS NO CHANGE TO THIS REPOSITORY CAN REDDEN ARE PREMISES (the owner's rule, +# #1236). Which access method a partition gets, whether pg_partman makes a DEFAULT +# partition, and which lock its maintenance holds are core's and pg_partman's +# behaviour. They are still asserted -- a pg_partman release that changes one +# turns it red, and the document with it -- but as premises, not as arms of this +# repository. The arms are what pgcolumnar decides: what a scan returns, and what +# the drop hook leaves in the catalogs. +# # pg_partman is an external extension. A box without it FAILS this suite (lib.sh's # rule for every capability but pyarrow); set PGC_ALLOW_MISSING_PG_PARTMAN=1 to make # that a named skip. CI installs postgresql-N-partman from PGDG. @@ -56,7 +64,7 @@ using=""; [ "$PGC_MAJOR" -ge 17 ] && using="USING pgcolumnar" q "CREATE TABLE plain_m (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts) $using;" >/dev/null q "SELECT partman.create_parent('public.plain_m', 'ts', '1 day', p_premake := 2, p_start_partition := (date_trunc('day', now()) - interval '2 days')::text);" >/dev/null -check "without default_table_access_method pg_partman makes heap partitions" \ +check "premise: without default_table_access_method pg_partman makes heap partitions" \ "$(am_counts plain_m | sed -E 's/=[0-9]+//g')" "heap" # ---- 2. the recipe ----------------------------------------------------------- @@ -64,11 +72,11 @@ q "ALTER DATABASE \"$PGC_DB\" SET default_table_access_method = 'pgcolumnar';" > q "CREATE TABLE ts_m (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts) $using;" >/dev/null q "SELECT partman.create_parent('public.ts_m', 'ts', '1 day', p_premake := 3, p_start_partition := (date_trunc('day', now()) - interval '40 days')::text);" >/dev/null -check_num "pg_partman creates a DEFAULT partition, as docs/how-to.md says" \ +check_num "premise: pg_partman creates a DEFAULT partition, as docs/how-to.md says" \ "$(q "SELECT count(*) FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'ts_m'::regclass AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT';")" "1" -check "with default_table_access_method every partition pg_partman makes is columnar" \ +check "premise: with default_table_access_method every partition pg_partman makes is columnar" \ "$(am_counts ts_m)" "pgcolumnar=$(q "SELECT count(*) FROM pg_inherits WHERE inhparent = 'ts_m'::regclass;")" q "CREATE TABLE ts_h (ts timestamptz, v int) USING heap;" >/dev/null @@ -80,9 +88,11 @@ check_num "premise: the rows went in" "$(q "SELECT count(*) FROM ts_m;")" "3000" check "and the rows land in columnar partitions" \ "$(q "SELECT string_agg(DISTINCT a.amname, ',') FROM ts_m t JOIN pg_class c ON c.oid = t.tableoid JOIN pg_am a ON a.oid = c.relam;")" "pgcolumnar" +# Every row, in order, hashed: an unfiltered count/sum/min/max is answered from +# zone maps and would not exercise the scan at all. check "and read back as heap does" \ - "$(q "SELECT count(*), sum(v), min(ts), max(ts) FROM ts_m;")" \ - "$(q "SELECT count(*), sum(v), min(ts), max(ts) FROM ts_h;")" + "$(q "SELECT md5(string_agg(ts::text || ':' || v, ',' ORDER BY ts, v)) FROM ts_m;")" \ + "$(q "SELECT md5(string_agg(ts::text || ':' || v, ',' ORDER BY ts, v)) FROM ts_h;")" # ---- 3. maintenance: ahead, retention, and the parent lock ------------------ q "UPDATE partman.part_config SET premake = 6, retention = '20 days', @@ -97,7 +107,7 @@ echo "-- partitions $kids_before -> $kids_after, storage rows $st_before -> $(q dropped=$((kids_before - kids_after)) check "premise: retention dropped partitions" \ "$([ "$dropped" -ge 1 ] && echo dropped || echo "dropped $dropped")" "dropped" -check "the partitions maintenance creates ahead are columnar too" \ +check "premise: the partitions maintenance creates ahead are columnar too" \ "$(am_counts ts_m)" "pgcolumnar=$kids_after" check_num "retention leaves no pgcolumnar storage row behind" \ "$(q "SELECT count(*) FROM pgcolumnar.storage s @@ -105,7 +115,7 @@ check_num "retention leaves no pgcolumnar storage row behind" \ check_num "and no row-group row behind" \ "$(q "SELECT count(*) FROM pgcolumnar.row_group r WHERE NOT EXISTS (SELECT 1 FROM pgcolumnar.storage s WHERE s.storage_id = r.storage_id);")" "0" -check "run_maintenance holds an exclusive lock on the parent, as docs/how-to.md says" \ +check "premise: run_maintenance holds an exclusive lock on the parent, as docs/how-to.md says" \ "$lock" "AccessExclusiveLock" pgc_summary diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index b58705f1..5d9ebc59 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -135,6 +135,7 @@ behaviour, the source of that number is named. - [87. test_latest_reading.py: the latest-reading recipe does what the document says](#87-test_latest_readingpy-the-latest-reading-recipe-does-what-the-document-says) - [88. test_storage_row_repair.py: the alpha5 release notes' repair procedure works as written](#88-test_storage_row_repairpy-the-alpha5-release-notes-repair-procedure-works-as-written) - [89. test_docs_cross_database_query.py: no documented query asks another database for its extensions](#89-test_docs_cross_database_querypy-no-documented-query-asks-another-database-for-its-extensions) +- [90. test_partman_columnar.py: pg_partman makes columnar partitions with one setting](#90-test_partman_columnarpy-pg_partman-makes-columnar-partitions-with-one-setting) ## 1. How to read a test in here @@ -6434,3 +6435,23 @@ Red with the old query restored (got 1, want 0). Its premise is red when the fen | test | what it holds | | --- | --- | | `test_no_documented_query_asks_another_database_for_its_extensions` | at least ten SQL statements were read; none reads `pg_database` and tests `pg_extension` | + +## 90. test_partman_columnar.py: pg_partman makes columnar partitions with one setting + +Port of `partman_columnar.sh` (#1291). The owner's question was whether pg_partman works with columnar partitions on every major; if it does, we document and test that. Measured with pg_partman 5.4.1 on PostgreSQL 15 to 19. Out of the box every partition it makes is heap, even on 17+ where the parent carries `USING pgcolumnar`. With `default_table_access_method = pgcolumnar` every partition is columnar, including the `DEFAULT` one and those made ahead, and retention leaves no pgcolumnar catalog row behind. + +THE TRAP IS PINNED, AS A PREMISE. Which access method core gives a partition, whether pg_partman makes a `DEFAULT` partition, and the `ACCESS EXCLUSIVE` its maintenance holds on the parent are core's and pg_partman's behaviour. No change to this repository can redden them, so by the owner's #1236 rule they are premises (found by @jdatcmd reviewing #1302). They are still asserted, because `docs/how-to.md` states them, and a pg_partman release that changes one must turn the document red with it. The arms are what pgcolumnar decides: what a scan returns and what the drop hook leaves in the catalogs. + +"READ BACK AS HEAP DOES" HASHES EVERY ROW IN ORDER. An unfiltered `count`/`sum`/`min`/`max` is answered from zone maps and would never exercise the scan, so a scan mutation could not have reddened it. + +ON A PRIVATE DATABASE, because the recipe is a database setting and the retention arms count the whole storage catalog. The lock is read inside one explicit `BEGIN`/`COMMIT`. A first draft set `autocommit = False` on the fixture's connection, which did not open a transaction, and read `none`. + +pg_partman missing is `MISSING_DEPENDENCY`, an unrunnable that fails the run, never a pass. CI installs `postgresql-N-partman` from PGDG. + +Removal proof, one mutation per arm, every major on the shell twin: the drop hook keeping the `storage` row reddens `retention leaves no pgcolumnar storage row behind` (20); keeping `row_group` rows reddens `and no row-group row behind` (20); a scan that drops its last row group reddens `and read back as heap does` and `and the rows land in columnar partitions`. A first scan mutation dropped a group only when there were two or more. Every partition here holds one group, so it changed nothing, and was replaced. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_pg_partman_makes_columnar_partitions_with_one_setting` | pg_partman installed; without the setting every partition is heap; with it, a `DEFAULT` partition exists and every partition is columnar; the rows land columnar and read back as heap does; maintenance drops partitions past retention, creates the ones ahead columnar, and leaves no orphaned storage or row-group row; and it holds `AccessExclusiveLock` on the parent | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index d1925428..fe2dabe7 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -547,4 +547,4 @@ guard_tests 405 # rebased tree, not added up. # 495 -> 496 for #1288: test_storage_row_repair.py. Re-derived by collection on # 038600e (after #1303): `496 tests collected`. -cluster_tests 496 +cluster_tests 497 diff --git a/test/pytest/test_partman_columnar.py b/test/pytest/test_partman_columnar.py index e995d543..a83ddd5c 100644 --- a/test/pytest/test_partman_columnar.py +++ b/test/pytest/test_partman_columnar.py @@ -66,7 +66,7 @@ def test_pg_partman_makes_columnar_partitions_with_one_setting(pgc_own_db, expec cur.execute("SELECT pm.create_parent('public.raw_events', 'at', '1 week', p_premake := 2," " p_start_partition := (date_trunc('week', now()) - interval '1 week')::text)") expect.text(" ".join(x.split("=")[0] for x in _am_counts(conn, "raw_events").split()), "heap", - "without default_table_access_method pg_partman makes heap partitions") + "premise: without default_table_access_method pg_partman makes heap partitions") # The recipe. dbname = _one(conn, "SELECT current_database()") @@ -80,10 +80,10 @@ def test_pg_partman_makes_columnar_partitions_with_one_setting(pgc_own_db, expec expect.num(_one(conn, "SELECT count(*) FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid" " WHERE i.inhparent = 'metrics'::regclass" " AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'"), 1, - "pg_partman creates a DEFAULT partition, as docs/how-to.md says") + "premise: pg_partman creates a DEFAULT partition, as docs/how-to.md says") kids = _kids(conn, "metrics") expect.text(_am_counts(conn, "metrics"), f"pgcolumnar={kids}", - "with default_table_access_method every partition pg_partman makes is columnar") + "premise: with default_table_access_method every partition pg_partman makes is columnar") with conn.cursor() as cur: cur.execute("CREATE TABLE metrics_heap (at timestamptz, n bigint) USING heap") @@ -94,7 +94,9 @@ def test_pg_partman_makes_columnar_partitions_with_one_setting(pgc_own_db, expec expect.text(_one(conn, "SELECT string_agg(DISTINCT a.amname, ',') FROM metrics t" " JOIN pg_class c ON c.oid = t.tableoid JOIN pg_am a ON a.oid = c.relam"), "pgcolumnar", "and the rows land in columnar partitions") - agg = "SELECT count(*)::text || '|' || sum(n) || '|' || min(at) || '|' || max(at) FROM {}" + # Every row, in order, hashed: an unfiltered count/sum/min/max is answered from + # zone maps and would not exercise the scan at all. + agg = "SELECT md5(string_agg(at::text || ':' || n, ',' ORDER BY at, n)) FROM {}" expect.text(_one(conn, agg.format("metrics")), _one(conn, agg.format("metrics_heap")), "and read back as heap does") @@ -116,7 +118,7 @@ def test_pg_partman_makes_columnar_partitions_with_one_setting(pgc_own_db, expec expect.text("dropped" if after < before else f"dropped {before - after}", "dropped", "premise: retention dropped partitions") expect.text(_am_counts(conn, "metrics"), f"pgcolumnar={after}", - "the partitions maintenance creates ahead are columnar too") + "premise: the partitions maintenance creates ahead are columnar too") expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.storage s WHERE NOT EXISTS" " (SELECT 1 FROM pg_class c WHERE c.oid = s.relation_oid)"), 0, "retention leaves no pgcolumnar storage row behind") @@ -124,4 +126,4 @@ def test_pg_partman_makes_columnar_partitions_with_one_setting(pgc_own_db, expec " (SELECT 1 FROM pgcolumnar.storage s WHERE s.storage_id = r.storage_id)"), 0, "and no row-group row behind") expect.text(lock, "AccessExclusiveLock", - "run_maintenance holds an exclusive lock on the parent, as docs/how-to.md says") + "premise: run_maintenance holds an exclusive lock on the parent, as docs/how-to.md says")