diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d42d4d90..c53a7497 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -184,7 +184,7 @@ jobs: | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null sudo timeout 300 apt-get -o Acquire::Retries=5 update sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ - postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ + postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman \ liblz4-dev libzstd-dev zlib1g-dev # NOT cached, deliberately. The suites job caches because it runs a matrix for # an hour; a second copy of that cache-key derivation is a thing that goes @@ -580,7 +580,7 @@ jobs: run: | set -euo pipefail v=$(apt-cache policy \ - postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ + postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman \ liblz4-dev libzstd-dev zlib1g-dev python3-pip \ | awk '/Candidate:/ {print $2}' | sort) # $ImageOS, from the shell, where it actually has a value. Reported @@ -628,7 +628,7 @@ jobs: mkdir -p "$HOME/apt-archives/partial" sudo timeout 600 apt-get -o Acquire::Retries=5 \ -o Dir::Cache::archives="$HOME/apt-archives" install -y --no-install-recommends \ - postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ + postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman \ liblz4-dev libzstd-dev zlib1g-dev python3-pip 2>&1 | tee /tmp/apt-install.log # The Arrow and Parquet suites use pyarrow as an independent reader and # skip themselves without it, which would be a silent loss of coverage. diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 41e5059b..0a3ff68e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -126,7 +126,7 @@ jobs: run: | set -euo pipefail v=$(apt-cache policy \ - postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ + postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman \ liblz4-dev libzstd-dev zlib1g-dev python3-pip \ | awk '/Candidate:/ {print $2}' | sort) # $ImageOS, from the shell, where it actually has a value. Reported @@ -156,7 +156,7 @@ jobs: mkdir -p "$HOME/apt-archives/partial" sudo timeout 600 apt-get -o Acquire::Retries=5 \ -o Dir::Cache::archives="$HOME/apt-archives" install -y --no-install-recommends \ - postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ + postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman \ liblz4-dev libzstd-dev zlib1g-dev python3-pip 2>&1 | tee /tmp/apt-install.log # pyarrow for root (the suites run under sudo); the silent-skip trap the # per-PR workflow documents applies here too. @@ -743,7 +743,7 @@ jobs: | sudo tee /etc/apt/sources.list.d/pgdg.list >/dev/null sudo timeout 300 apt-get -o Acquire::Retries=5 update sudo timeout 600 apt-get -o Acquire::Retries=5 install -y --no-install-recommends \ - postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} \ + postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} postgresql-${{ matrix.pg }}-partman \ liblz4-dev libzstd-dev zlib1g-dev # NOT cached, deliberately. The suites job caches because it runs a matrix for # an hour; a second copy of that cache-key derivation is a thing that goes diff --git a/CHANGELOG.md b/CHANGELOG.md index 521f328a..85a87d7a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,6 +51,26 @@ true until the next version shipped. showed a vacuous arm: "under a tenth of the table" passed on empty SQL, because a query that reads nothing reads under a tenth of anything. It now requires at least one block read. +- `docs/how-to.md` documents managing time partitions with pg_partman (#1291), and + `test/partman_columnar.sh` with its pytest twin checks every statement the + section makes. Measured with pg_partman 5.4.1 on PostgreSQL 15 to 19: + + without default_table_access_method every partition is heap, even on 17+ + where the parent carries USING pgcolumnar + with it every partition is columnar, including the + DEFAULT one and those made ahead; rows + land there and read back as heap does + retention partitions dropped, no pgcolumnar + storage or row-group row left behind + run_maintenance holds ACCESS EXCLUSIVE on the parent + + The first arm pins the trap rather than the recipe, so a pg_partman that starts + honouring the parent's access method turns it red instead of leaving the warning + stale. Without the drop hook's catalog cleanup, the retention arm reads 20 + orphaned storage rows. pg_partman is an external extension, so a box without it + FAILS the suite unless `PGC_ALLOW_MISSING_PG_PARTMAN=1` is set. The CI suites + and pytest jobs now install `postgresql-N-partman` from PGDG, in the cache key + as well as the install. ### Changed diff --git a/docs/how-to.md b/docs/how-to.md index ad47e5d6..0fecd064 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -323,6 +323,60 @@ declared retention raises an error rather than doing nothing. `pgcolumnar.reset_options` has no argument for the retention. To stop expiring, set `ttl_interval` to a span no row will reach. +## Manage time partitions with pg_partman + +pg_partman can create and drop columnar partitions. It needs one setting to do +so, and without it the partitions it makes are not columnar. + +Before you start: install pg_partman (PGDG ships it as `postgresql-N-partman`), +and create it in its own schema. + +```sql +CREATE SCHEMA partman; +CREATE EXTENSION pg_partman SCHEMA partman; +``` + +1. Make columnar the default access method where pg_partman runs. A database + setting is the simplest place. + +```sql +ALTER DATABASE mydb SET default_table_access_method = 'pgcolumnar'; +``` + +2. Create the parent and hand it to pg_partman. + +```sql +CREATE TABLE metrics (ts timestamptz NOT NULL, v float8) + PARTITION BY RANGE (ts); +SELECT partman.create_parent('public.metrics', 'ts', '1 day', p_premake := 4); +``` + +3. Set the retention, and run maintenance on a schedule. + +```sql +UPDATE partman.part_config + SET retention = '90 days', retention_keep_table = false + WHERE parent_table = 'public.metrics'; +SELECT partman.run_maintenance('public.metrics'); +``` + +**Without step 1, every partition is a heap table.** pg_partman does not take the +access method from the parent. That holds even on PostgreSQL 17 and later, where +the parent itself can carry `USING pgcolumnar`. Changing the access method of +pg_partman's template table does not help either. Measured with pg_partman 5.4.1 +on PostgreSQL 15 to 19. + +**The setting has a cost.** Every table created in that database becomes +columnar unless it names another access method. + +**Maintenance locks the parent.** `run_maintenance` holds `ACCESS EXCLUSIVE` on +the parent table while it runs, so queries on the table wait for it. pg_partman +also creates a `DEFAULT` partition. + +When a partition is dropped by retention, its pgcolumnar catalog rows go with it. + +`test/partman_columnar.sh` checks each of these statements. + ## Keep tables optimized automatically A background daemon can compact and recluster tables on a schedule. The threshold diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 1903f5f7..0e449de9 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -1759,6 +1759,18 @@ parallel_scan_cost parallel_scan_cost premise: the serial plan has no Gather 15; parallel_scan_cost parallel_scan_cost premise: the serial plan is a columnar scan 15;16;17;18;19 never - parallel_scan_cost parallel_scan_cost premise: the table holds every inserted row 15;16;17;18;19 never - parallel_scan_cost parallel_scan_cost the leader-participation branch changes the divisor 15;16;17;18;19 never - +partman_columnar partman_columnar and no row-group row behind 15;16;17;18;19 2026-10-12 the drop hook stops deleting row_group rows, so they outlive their deleted storage row +partman_columnar partman_columnar and read back as heap does 15;16;17;18;19 2026-10-12 a sequential scan drops the last row group from its list, so every one-group partition reads empty +partman_columnar partman_columnar and the rows land in columnar partitions 15;16;17;18;19 2026-10-12 a sequential scan drops the last row group from its list, so every one-group partition reads empty +partman_columnar partman_columnar premise: pg_partman creates a DEFAULT partition, as docs/how-to.md says 15;16;17;18;19 never - +partman_columnar partman_columnar premise: pg_partman is installed in this database 15;16;17;18;19 never - +partman_columnar partman_columnar premise: retention dropped partitions 15;16;17;18;19 never - +partman_columnar partman_columnar premise: run_maintenance holds an exclusive lock on the parent, as docs/how-to.md says 15;16;17;18;19 never - +partman_columnar partman_columnar premise: the partitions maintenance creates ahead are columnar too 15;16;17;18;19 never - +partman_columnar partman_columnar premise: the rows went in 15;16;17;18;19 2026-10-12 a sequential scan drops the last row group from its list, so every one-group partition reads empty +partman_columnar partman_columnar premise: with default_table_access_method every partition pg_partman makes is columnar 15;16;17;18;19 never - +partman_columnar partman_columnar premise: without default_table_access_method pg_partman makes heap partitions 15;16;17;18;19 never - +partman_columnar partman_columnar retention leaves no pgcolumnar storage row behind 15;16;17;18;19 2026-10-12 the drop hook stops deleting the storage row, so a partition pg_partman drops leaves it behind projection_parallel projection_parallel a covering projection can be a parallel scan 15;16;17;18;19 2026-09-18 - projection_parallel projection_parallel a page cost that makes I/O dominate costs the parallel covering path 15;16;17;18;19 2026-09-23 the partial covering total amortises I/O as well as CPU: (ioRunProj + cpuRunProj) / divisor projection_parallel projection_parallel a parallel covering projection returns the covering rows once 15;16;17;18;19 2026-10-12 a parallel worker never wires the shared row-group counter, so every worker scans every group diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 96f8a404..b28c4209 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1258,4 +1258,22 @@ suites_not_covered 247 # onto #1303's ledger, not merged by hand. # # never 1660 + 4 = 1664 -checks_never_observed_red 1664 +# #1291 (OffgridwithJD): 1664 -> 1671. partman_columnar enters the ledger with +# 12 checks from five-major runs, replayed from the saved logs onto this base. +# Every arm is dated, each by its own mutation: +# +# drop hook keeps the storage row retention leaves no pgcolumnar storage row behind +# drop hook keeps row_group rows and no row-group row behind +# the scan drops its last group and read back as heap does, +# and the rows land in columnar partitions +# (and the premise "the rows went in") +# +# The 7 still `never` are premises: the access method core gives a partition, +# pg_partman's DEFAULT partition and its parent lock. No change to this +# repository can redden them (the owner's #1236 rule). +# +# ledger rows 1883 + 12 = 1895 +# never 1664 + 7 = 1671 +# dated 219 + 5 = 224 +# 1671 + 224 == 1895 +checks_never_observed_red 1671 diff --git a/test/partman_columnar.sh b/test/partman_columnar.sh new file mode 100755 index 00000000..291da24a --- /dev/null +++ b/test/partman_columnar.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# +# pg_partman manages columnar partitions -- with one setting, and not without it (#1291). +# +# The owner's question on #1291 was whether pg_partman works with columnar +# partitions on every major; if it does, we document AND TEST that. Measured with +# pg_partman 5.4.1 on PG 15..19, and asserted here: +# +# OUT OF THE BOX, every partition pg_partman makes is HEAP -- even on 17+, where +# the parent itself carries USING pgcolumnar. pg_partman does not take the access +# method from the parent, and switching its template table does not help either. +# +# WITH default_table_access_method = pgcolumnar where its maintenance runs, every +# partition it makes is columnar, the rows land there, and retention drops +# partitions leaving no pgcolumnar catalog row behind. +# +# run_maintenance holds ACCESS EXCLUSIVE on the parent. That is pg_partman's +# behaviour, not ours; it is asserted because docs/how-to.md tells users so, and a +# pg_partman that changed it should turn this red rather than leave the doc wrong. +# +# The first check pins the trap rather than the recipe, on purpose: it is the reason +# the documentation exists, and a pg_partman that started honouring the parent's +# access method should be noticed, not silently make the warning stale. +# +# CHECKS NO CHANGE TO THIS REPOSITORY CAN REDDEN ARE PREMISES (the owner's rule, +# #1236). Which access method a partition gets, whether pg_partman makes a DEFAULT +# partition, and which lock its maintenance holds are core's and pg_partman's +# behaviour. They are still asserted -- a pg_partman release that changes one +# turns it red, and the document with it -- but as premises, not as arms of this +# repository. The arms are what pgcolumnar decides: what a scan returns, and what +# the drop hook leaves in the catalogs. +# +# pg_partman is an external extension. A box without it FAILS this suite (lib.sh's +# rule for every capability but pyarrow); set PGC_ALLOW_MISSING_PG_PARTMAN=1 to make +# that a named skip. CI installs postgresql-N-partman from PGDG. +# +# Usage: test/partman_columnar.sh [PG_CONFIG] + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +q "CREATE EXTENSION IF NOT EXISTS pgcolumnar;" >/dev/null +if [ "$(q "SELECT count(*) FROM pg_available_extensions WHERE name = 'pg_partman';")" != 1 ]; then + pgc_skip pg_partman "pg_partman is not installed for this major" +fi +q "CREATE SCHEMA partman; CREATE EXTENSION pg_partman SCHEMA partman;" >/dev/null +check_text "premise: pg_partman is installed in this database" \ + "$(q "SELECT count(*) FROM pg_extension WHERE extname = 'pg_partman';")" "1" +echo "-- pg_partman $(q "SELECT extversion FROM pg_extension WHERE extname = 'pg_partman';")" + +# am_counts PARENT -> "am=n am=n" over the partitions pg_partman made +am_counts() { + q "SELECT coalesce(string_agg(amname || '=' || n, ' ' ORDER BY amname), 'none') + FROM (SELECT a.amname, count(*) n FROM pg_inherits i + JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_am a ON a.oid = c.relam + WHERE i.inhparent = '$1'::regclass GROUP BY 1) x;" +} +# On 17+ a partitioned parent can carry USING; give it to them where it can be +# given, so the first arm shows pg_partman ignores even that. +using=""; [ "$PGC_MAJOR" -ge 17 ] && using="USING pgcolumnar" + +# ---- 1. the trap: no setting, heap partitions ------------------------------- +q "CREATE TABLE plain_m (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts) $using;" >/dev/null +q "SELECT partman.create_parent('public.plain_m', 'ts', '1 day', p_premake := 2, + p_start_partition := (date_trunc('day', now()) - interval '2 days')::text);" >/dev/null +check "premise: without default_table_access_method pg_partman makes heap partitions" \ + "$(am_counts plain_m | sed -E 's/=[0-9]+//g')" "heap" + +# ---- 2. the recipe ----------------------------------------------------------- +q "ALTER DATABASE \"$PGC_DB\" SET default_table_access_method = 'pgcolumnar';" >/dev/null +q "CREATE TABLE ts_m (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts) $using;" >/dev/null +q "SELECT partman.create_parent('public.ts_m', 'ts', '1 day', p_premake := 3, + p_start_partition := (date_trunc('day', now()) - interval '40 days')::text);" >/dev/null +check_num "premise: pg_partman creates a DEFAULT partition, as docs/how-to.md says" \ + "$(q "SELECT count(*) FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = 'ts_m'::regclass + AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT';")" "1" +check "premise: with default_table_access_method every partition pg_partman makes is columnar" \ + "$(am_counts ts_m)" "pgcolumnar=$(q "SELECT count(*) FROM pg_inherits WHERE inhparent = 'ts_m'::regclass;")" + +q "CREATE TABLE ts_h (ts timestamptz, v int) USING heap;" >/dev/null +for t in ts_m ts_h; do + q "INSERT INTO $t SELECT date_trunc('day', now()) - interval '40 days' + g * interval '17 min', g + FROM generate_series(1, 3000) g;" >/dev/null +done +check_num "premise: the rows went in" "$(q "SELECT count(*) FROM ts_m;")" "3000" +check "and the rows land in columnar partitions" \ + "$(q "SELECT string_agg(DISTINCT a.amname, ',') FROM ts_m t + JOIN pg_class c ON c.oid = t.tableoid JOIN pg_am a ON a.oid = c.relam;")" "pgcolumnar" +# Every row, in order, hashed: an unfiltered count/sum/min/max is answered from +# zone maps and would not exercise the scan at all. +check "and read back as heap does" \ + "$(q "SELECT md5(string_agg(ts::text || ':' || v, ',' ORDER BY ts, v)) FROM ts_m;")" \ + "$(q "SELECT md5(string_agg(ts::text || ':' || v, ',' ORDER BY ts, v)) FROM ts_h;")" + +# ---- 3. maintenance: ahead, retention, and the parent lock ------------------ +q "UPDATE partman.part_config SET premake = 6, retention = '20 days', + retention_keep_table = false WHERE parent_table = 'public.ts_m';" >/dev/null +st_before="$(q "SELECT count(*) FROM pgcolumnar.storage;")" +kids_before="$(q "SELECT count(*) FROM pg_inherits WHERE inhparent = 'ts_m'::regclass;")" +lock="$(q "BEGIN; SELECT partman.run_maintenance('public.ts_m'); + SELECT coalesce(string_agg(DISTINCT mode, ','), 'none') FROM pg_locks + WHERE relation = 'ts_m'::regclass AND pid = pg_backend_pid(); COMMIT;" | grep -vE '^(BEGIN|COMMIT)$|^$' | tail -1)" +kids_after="$(q "SELECT count(*) FROM pg_inherits WHERE inhparent = 'ts_m'::regclass;")" +echo "-- partitions $kids_before -> $kids_after, storage rows $st_before -> $(q "SELECT count(*) FROM pgcolumnar.storage;")" +dropped=$((kids_before - kids_after)) +check "premise: retention dropped partitions" \ + "$([ "$dropped" -ge 1 ] && echo dropped || echo "dropped $dropped")" "dropped" +check "premise: the partitions maintenance creates ahead are columnar too" \ + "$(am_counts ts_m)" "pgcolumnar=$kids_after" +check_num "retention leaves no pgcolumnar storage row behind" \ + "$(q "SELECT count(*) FROM pgcolumnar.storage s + WHERE NOT EXISTS (SELECT 1 FROM pg_class c WHERE c.oid = s.relation_oid);")" "0" +check_num "and no row-group row behind" \ + "$(q "SELECT count(*) FROM pgcolumnar.row_group r + WHERE NOT EXISTS (SELECT 1 FROM pgcolumnar.storage s WHERE s.storage_id = r.storage_id);")" "0" +check "premise: run_maintenance holds an exclusive lock on the parent, as docs/how-to.md says" \ + "$lock" "AccessExclusiveLock" + +pgc_summary diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index b58705f1..5d9ebc59 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -135,6 +135,7 @@ behaviour, the source of that number is named. - [87. test_latest_reading.py: the latest-reading recipe does what the document says](#87-test_latest_readingpy-the-latest-reading-recipe-does-what-the-document-says) - [88. test_storage_row_repair.py: the alpha5 release notes' repair procedure works as written](#88-test_storage_row_repairpy-the-alpha5-release-notes-repair-procedure-works-as-written) - [89. test_docs_cross_database_query.py: no documented query asks another database for its extensions](#89-test_docs_cross_database_querypy-no-documented-query-asks-another-database-for-its-extensions) +- [90. test_partman_columnar.py: pg_partman makes columnar partitions with one setting](#90-test_partman_columnarpy-pg_partman-makes-columnar-partitions-with-one-setting) ## 1. How to read a test in here @@ -6434,3 +6435,23 @@ Red with the old query restored (got 1, want 0). Its premise is red when the fen | test | what it holds | | --- | --- | | `test_no_documented_query_asks_another_database_for_its_extensions` | at least ten SQL statements were read; none reads `pg_database` and tests `pg_extension` | + +## 90. test_partman_columnar.py: pg_partman makes columnar partitions with one setting + +Port of `partman_columnar.sh` (#1291). The owner's question was whether pg_partman works with columnar partitions on every major; if it does, we document and test that. Measured with pg_partman 5.4.1 on PostgreSQL 15 to 19. Out of the box every partition it makes is heap, even on 17+ where the parent carries `USING pgcolumnar`. With `default_table_access_method = pgcolumnar` every partition is columnar, including the `DEFAULT` one and those made ahead, and retention leaves no pgcolumnar catalog row behind. + +THE TRAP IS PINNED, AS A PREMISE. Which access method core gives a partition, whether pg_partman makes a `DEFAULT` partition, and the `ACCESS EXCLUSIVE` its maintenance holds on the parent are core's and pg_partman's behaviour. No change to this repository can redden them, so by the owner's #1236 rule they are premises (found by @jdatcmd reviewing #1302). They are still asserted, because `docs/how-to.md` states them, and a pg_partman release that changes one must turn the document red with it. The arms are what pgcolumnar decides: what a scan returns and what the drop hook leaves in the catalogs. + +"READ BACK AS HEAP DOES" HASHES EVERY ROW IN ORDER. An unfiltered `count`/`sum`/`min`/`max` is answered from zone maps and would never exercise the scan, so a scan mutation could not have reddened it. + +ON A PRIVATE DATABASE, because the recipe is a database setting and the retention arms count the whole storage catalog. The lock is read inside one explicit `BEGIN`/`COMMIT`. A first draft set `autocommit = False` on the fixture's connection, which did not open a transaction, and read `none`. + +pg_partman missing is `MISSING_DEPENDENCY`, an unrunnable that fails the run, never a pass. CI installs `postgresql-N-partman` from PGDG. + +Removal proof, one mutation per arm, every major on the shell twin: the drop hook keeping the `storage` row reddens `retention leaves no pgcolumnar storage row behind` (20); keeping `row_group` rows reddens `and no row-group row behind` (20); a scan that drops its last row group reddens `and read back as heap does` and `and the rows land in columnar partitions`. A first scan mutation dropped a group only when there were two or more. Every partition here holds one group, so it changed nothing, and was replaced. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_pg_partman_makes_columnar_partitions_with_one_setting` | pg_partman installed; without the setting every partition is heap; with it, a `DEFAULT` partition exists and every partition is columnar; the rows land columnar and read back as heap does; maintenance drops partitions past retention, creates the ones ahead columnar, and leaves no orphaned storage or row-group row; and it holds `AccessExclusiveLock` on the parent | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index d1925428..fe2dabe7 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -547,4 +547,4 @@ guard_tests 405 # rebased tree, not added up. # 495 -> 496 for #1288: test_storage_row_repair.py. Re-derived by collection on # 038600e (after #1303): `496 tests collected`. -cluster_tests 496 +cluster_tests 497 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 14976e0a..cd584275 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -117,6 +117,7 @@ "native_vacuum_race", "parallel_am_scan", "parallel_scan_cost", + "partman_columnar", "projection_drop_column", "projection_parallel", "projection_privilege", diff --git a/test/pytest/test_partman_columnar.py b/test/pytest/test_partman_columnar.py new file mode 100644 index 00000000..a83ddd5c --- /dev/null +++ b/test/pytest/test_partman_columnar.py @@ -0,0 +1,129 @@ +"""pg_partman manages columnar partitions, with one setting and not without it (#1291). + +The owner's question on #1291: does pg_partman work with columnar partitions on +every major? If it does, we document and test that. Measured with pg_partman +5.4.1 on PG 15..19: + +- Out of the box every partition it makes is HEAP, even on 17+ where the parent + itself carries USING pgcolumnar. It does not take the parent's access method. +- With `default_table_access_method = pgcolumnar` where its maintenance runs, + every partition it makes is columnar, the rows land there, and retention drops + partitions leaving no pgcolumnar catalog row behind. +- `run_maintenance` holds ACCESS EXCLUSIVE on the parent. That is pg_partman's + behaviour; it is asserted because docs/how-to.md tells users so. + +The first arm pins the TRAP, on purpose: it is why the documentation exists, and +a pg_partman that began honouring the parent's access method must turn it red +rather than leave the warning stale. + +ON A PRIVATE DATABASE (`pgc_own_db`): the recipe is a database-level setting, and +retention is checked against the whole storage catalog. pg_partman missing is +MISSING_DEPENDENCY (unrunnable, which fails the run), never a pass; CI installs +postgresql-N-partman from PGDG. + +Independent of `test/partman_columnar.sh`: its own tables, partition widths, +dates, retention window and row counts. +""" + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + row = cur.fetchone() + return row[0] if row else None + + +def _am_counts(conn, parent): + return _one(conn, + "SELECT coalesce(string_agg(amname || '=' || n, ' ' ORDER BY amname), 'none')" + " FROM (SELECT a.amname, count(*) n FROM pg_inherits i" + " JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_am a ON a.oid = c.relam" + f" WHERE i.inhparent = '{parent}'::regclass GROUP BY 1) x") + + +def _kids(conn, parent): + return _one(conn, f"SELECT count(*) FROM pg_inherits WHERE inhparent = '{parent}'::regclass") + + +def test_pg_partman_makes_columnar_partitions_with_one_setting(pgc_own_db, expect): + conn = pgc_own_db + if _one(conn, "SELECT count(*) FROM pg_available_extensions WHERE name = 'pg_partman'") != 1: + expect.cannot_run("MISSING_DEPENDENCY", "pg_partman is not installed for this major", + name="pg_partman is not installed for this major") + return + with conn.cursor() as cur: + cur.execute("CREATE SCHEMA pm") + cur.execute("CREATE EXTENSION pg_partman SCHEMA pm") + expect.num(_one(conn, "SELECT count(*) FROM pg_extension WHERE extname = 'pg_partman'"), 1, + "premise: pg_partman is installed in this database") + major = int(_one(conn, "SHOW server_version_num")) // 10000 + using = "USING pgcolumnar" if major >= 17 else "" + + # The trap: no setting, heap partitions. + with conn.cursor() as cur: + cur.execute(f"CREATE TABLE raw_events (at timestamptz NOT NULL, n bigint)" + f" PARTITION BY RANGE (at) {using}") + cur.execute("SELECT pm.create_parent('public.raw_events', 'at', '1 week', p_premake := 2," + " p_start_partition := (date_trunc('week', now()) - interval '1 week')::text)") + expect.text(" ".join(x.split("=")[0] for x in _am_counts(conn, "raw_events").split()), "heap", + "premise: without default_table_access_method pg_partman makes heap partitions") + + # The recipe. + dbname = _one(conn, "SELECT current_database()") + with conn.cursor() as cur: + cur.execute(f'ALTER DATABASE "{dbname}" SET default_table_access_method = \'pgcolumnar\'') + cur.execute("SET default_table_access_method = 'pgcolumnar'") + cur.execute(f"CREATE TABLE metrics (at timestamptz NOT NULL, n bigint)" + f" PARTITION BY RANGE (at) {using}") + cur.execute("SELECT pm.create_parent('public.metrics', 'at', '1 day', p_premake := 2," + " p_start_partition := (date_trunc('day', now()) - interval '30 days')::text)") + expect.num(_one(conn, "SELECT count(*) FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid" + " WHERE i.inhparent = 'metrics'::regclass" + " AND pg_get_expr(c.relpartbound, c.oid) = 'DEFAULT'"), 1, + "premise: pg_partman creates a DEFAULT partition, as docs/how-to.md says") + kids = _kids(conn, "metrics") + expect.text(_am_counts(conn, "metrics"), f"pgcolumnar={kids}", + "premise: with default_table_access_method every partition pg_partman makes is columnar") + + with conn.cursor() as cur: + cur.execute("CREATE TABLE metrics_heap (at timestamptz, n bigint) USING heap") + for t in ("metrics", "metrics_heap"): + cur.execute(f"INSERT INTO {t} SELECT date_trunc('day', now()) - interval '30 days'" + f" + g * interval '11 min', g * 3 FROM generate_series(1, 3600) g") + expect.num(_one(conn, "SELECT count(*) FROM metrics"), 3600, "premise: the rows went in") + expect.text(_one(conn, "SELECT string_agg(DISTINCT a.amname, ',') FROM metrics t" + " JOIN pg_class c ON c.oid = t.tableoid JOIN pg_am a ON a.oid = c.relam"), + "pgcolumnar", "and the rows land in columnar partitions") + # Every row, in order, hashed: an unfiltered count/sum/min/max is answered from + # zone maps and would not exercise the scan at all. + agg = "SELECT md5(string_agg(at::text || ':' || n, ',' ORDER BY at, n)) FROM {}" + expect.text(_one(conn, agg.format("metrics")), _one(conn, agg.format("metrics_heap")), + "and read back as heap does") + + # Maintenance: created ahead, retention, and the parent lock. + with conn.cursor() as cur: + cur.execute("UPDATE pm.part_config SET premake = 5, retention = '12 days'," + " retention_keep_table = false WHERE parent_table = 'public.metrics'") + before = _kids(conn, "metrics") + # One explicit transaction, so the lock is still held when it is read. + with conn.cursor() as cur: + cur.execute("BEGIN") + cur.execute("SELECT pm.run_maintenance('public.metrics')") + cur.execute("SELECT coalesce(string_agg(DISTINCT mode, ','), 'none') FROM pg_locks" + " WHERE relation = 'metrics'::regclass AND pid = pg_backend_pid()") + lock = cur.fetchone()[0] + cur.execute("COMMIT") + after = _kids(conn, "metrics") + print(f"-- partitions {before} -> {after}") + expect.text("dropped" if after < before else f"dropped {before - after}", "dropped", + "premise: retention dropped partitions") + expect.text(_am_counts(conn, "metrics"), f"pgcolumnar={after}", + "premise: the partitions maintenance creates ahead are columnar too") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.storage s WHERE NOT EXISTS" + " (SELECT 1 FROM pg_class c WHERE c.oid = s.relation_oid)"), 0, + "retention leaves no pgcolumnar storage row behind") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.row_group r WHERE NOT EXISTS" + " (SELECT 1 FROM pgcolumnar.storage s WHERE s.storage_id = r.storage_id)"), 0, + "and no row-group row behind") + expect.text(lock, "AccessExclusiveLock", + "premise: run_maintenance holds an exclusive lock on the parent, as docs/how-to.md says") diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 499e3650..a537c3b2 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -248,6 +248,7 @@ SUITES=( parquet_level_width parquet_nested parquet_nested_import + partman_columnar pg19_vacuum_options pg_dump_roundtrip phase2