From 06a46df39f4e66de725bf5b489045804d17a4a6b Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 13:56:06 +0000 Subject: [PATCH 01/12] test: partition_drop_plan, the drop half of the partition lifecycle, red (#1297) Seen red on PG17 at 4c72242: the function is absent (42883), so no plan, no drop, no refusal. Its premises pass: four partitions holding rows, and a DETACH ... CONCURRENTLY cancelled in its second phase leaves the partition pending. Co-Authored-By: Claude Opus 5.5 --- test/partition_lifecycle.sh | 86 +++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/test/partition_lifecycle.sh b/test/partition_lifecycle.sh index 798068013..09a5a0b7b 100755 --- a/test/partition_lifecycle.sh +++ b/test/partition_lifecycle.sh @@ -190,4 +190,90 @@ check "a parent with a DEFAULT partition is refused" \ check "and nothing was created for it" \ "$(kids td)|$(q "SELECT count(*) FROM pg_proc WHERE proname = 'partition_create_ahead';")" "1|1" +# ==== the drop half: partition_drop_plan (#1297) ================================ +# +# partition_drop_plan(parent regclass, cutoff timestamptz) -> SETOF text +# +# The statements that retire every partition whose whole range is older than +# cutoff, in range order: DETACH ... CONCURRENTLY, then DROP. They cannot run +# inside a function (DETACH CONCURRENTLY refuses any function, DO block or +# procedure, on every major: #1297 note 1), so the function RETURNS them and +# the caller runs each as its own top-level statement (\gexec). A partition a +# concurrent detach left pending gets DETACH ... FINALIZE instead. +plan() { q "SELECT string_agg(s, E'\n') FROM pgcolumnar.partition_drop_plan('$1', '$2') s;"; } +run_plan() { # each statement top-level, as \gexec would run it + local line + while IFS= read -r line; do + [ -n "$line" ] && env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -X -q -v ON_ERROR_STOP=1 -c "$line" >/dev/null 2>&1 || echo "failed: $line" + done <<<"$(plan "$1" "$2")" +} +q "CREATE TABLE dl (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts); + CREATE TABLE dl_01 PARTITION OF dl FOR VALUES FROM ('2020-01-01 00:00+00') TO ('2020-02-01 00:00+00') USING pgcolumnar; + CREATE TABLE dl_02 PARTITION OF dl FOR VALUES FROM ('2020-02-01 00:00+00') TO ('2020-03-01 00:00+00') USING pgcolumnar; + CREATE TABLE dl_03 PARTITION OF dl FOR VALUES FROM ('2020-03-01 00:00+00') TO ('2020-04-01 00:00+00') USING pgcolumnar; + CREATE TABLE dl_04 PARTITION OF dl FOR VALUES FROM ('2020-04-01 00:00+00') TO ('2020-05-01 00:00+00') USING pgcolumnar; + INSERT INTO dl SELECT '2020-01-01 00:00+00'::timestamptz + g * interval '1 hour', g FROM generate_series(0, 2903) g;" >/dev/null +check "premise: the drop fixture has four partitions, each holding rows" \ + "$(q "SELECT count(DISTINCT tableoid) FROM dl;")|$(kids dl)" "4|4" +check "the plan retires the partitions wholly before the cutoff, in range order" \ + "$(plan dl '2020-03-15 00:00+00')" \ + "$(printf '%s\n' 'ALTER TABLE dl DETACH PARTITION dl_01 CONCURRENTLY' 'DROP TABLE dl_01' \ + 'ALTER TABLE dl DETACH PARTITION dl_02 CONCURRENTLY' 'DROP TABLE dl_02')" +check "a cutoff on a partition's upper bound includes it" \ + "$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2020-03-01 00:00+00') s WHERE s LIKE 'DROP%';")" "2" +check "a cutoff before every partition plans nothing" \ + "$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2019-06-01 00:00+00');")" "0" +sids="$(q "SELECT string_agg(pgcolumnar.get_storage_id(c)::text, ',') FROM unnest(ARRAY['dl_01','dl_02']::regclass[]) c;")" +run_plan dl '2020-03-15 00:00+00' +check "running the plan leaves only the partitions it did not name" \ + "$(q "SELECT string_agg(c.relname, ',' ORDER BY c.relname) FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = 'dl'::regclass;")|$(q "SELECT count(*) FROM pg_class WHERE relname IN ('dl_01', 'dl_02');")" "dl_03,dl_04|0" +check "and every row the remaining partitions held is still there" \ + "$(q "SELECT count(*) FROM dl;")" "$(q "SELECT count(*) FROM generate_series(0, 2903) g + WHERE '2020-01-01 00:00+00'::timestamptz + g * interval '1 hour' >= '2020-03-01 00:00+00';")" +check "and the dropped partitions' pgcolumnar storage rows are gone" \ + "$(q "SELECT count(*) FROM pgcolumnar.storage WHERE storage_id IN (${sids:-0});")" "0" +check "a second plan for the same cutoff is empty" \ + "$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2020-03-15 00:00+00');")" "0" + +# A timestamp (no zone) key: the cutoff is read in the session's zone, as the key's +# wall-clock time. +q "CREATE TABLE dw (ts timestamp NOT NULL) PARTITION BY RANGE (ts); + CREATE TABLE dw_a PARTITION OF dw FOR VALUES FROM ('2021-01-01') TO ('2021-01-02') USING pgcolumnar; + CREATE TABLE dw_b PARTITION OF dw FOR VALUES FROM ('2021-01-02') TO ('2021-01-03') USING pgcolumnar;" >/dev/null +check "a timestamp key's bounds are compared in the session's wall-clock time" \ + "$(q "SET TimeZone = 'America/New_York'; SELECT string_agg(s, ';') FROM pgcolumnar.partition_drop_plan('dw', '2021-01-02 00:00-05') s;")" \ + "ALTER TABLE dw DETACH PARTITION dw_a CONCURRENTLY;DROP TABLE dw_a" + +# An interrupted concurrent detach leaves the partition pending: a transaction +# holding an older snapshot makes the second phase wait, and a statement timeout +# cancels it there. +q "CREATE TABLE dp (ts timestamptz NOT NULL) PARTITION BY RANGE (ts); + CREATE TABLE dp_1 PARTITION OF dp FOR VALUES FROM ('2020-01-01 00:00+00') TO ('2020-02-01 00:00+00') USING pgcolumnar; + CREATE TABLE dp_2 PARTITION OF dp FOR VALUES FROM ('2020-02-01 00:00+00') TO ('2020-03-01 00:00+00') USING pgcolumnar;" >/dev/null +env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -q \ + -c "BEGIN ISOLATION LEVEL REPEATABLE READ" -c "SELECT count(*) FROM dp" -c "SELECT pg_sleep(8)" -c "COMMIT" >/dev/null 2>&1 & +holder=$! +sleep 1 +env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -q \ + -c "SET statement_timeout = '2s'" -c "ALTER TABLE dp DETACH PARTITION dp_1 CONCURRENTLY" >/dev/null 2>&1 +wait "$holder" +check "premise: the interrupted detach left dp_1 pending" \ + "$(q "SELECT inhdetachpending FROM pg_inherits WHERE inhrelid = 'dp_1'::regclass;")" "t" +check "a pending detach is finalised, not started again" \ + "$(q "SELECT string_agg(s, ';') FROM pgcolumnar.partition_drop_plan('dp', '2020-02-01 00:00+00') s;")" \ + "ALTER TABLE dp DETACH PARTITION dp_1 FINALIZE;DROP TABLE dp_1" +run_plan dp '2020-02-01 00:00+00' +check "and running it drops the pending partition" \ + "$(q "SELECT count(*) FROM pg_class WHERE relname = 'dp_1';")|$(kids dp)" "0|1" + +# Refusals, as partition_create_ahead refuses. +check "a drop plan for a parent with a DEFAULT partition is refused" \ + "$(sqlstate_or "SELECT count(*) FROM pgcolumnar.partition_drop_plan('td', now())")" "55000" +q "CREATE TABLE dn (k int NOT NULL) PARTITION BY RANGE (k); + CREATE TABLE dnl (k int);" >/dev/null +check "a drop plan for a parent not partitioned on a timestamp is refused" \ + "$(sqlstate_or "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dn', now())")|$(sqlstate_or "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dnl', now())")" "55000|55000" + pgc_summary From 95004d8f47aeccbbff141c89807236b866d3682f Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 13:58:47 +0000 Subject: [PATCH 02/12] feat: pgcolumnar.partition_drop_plan returns the statements that retire old partitions (#1297) DETACH ... CONCURRENTLY is refused inside any function, so the drop half returns its statements for the caller to run at top level (\gexec): DETACH CONCURRENTLY then DROP for every partition whose upper bound is at or before the cutoff, in range order, schema-qualified, and DETACH ... FINALIZE for one a concurrent detach left pending. The parent is never locked exclusively. The same refusals and codes as partition_create_ahead. Identical patches in the alpha5 script and the alpha4 upgrade. The arms now expect schema-qualified names, and the timestamp-key arm adds a cutoff that only a wall-clock reading gets right (a UTC reading passed the first version). partition_lifecycle.sh 41/41 on PG17. Co-Authored-By: Claude Opus 5.5 --- pgcolumnar--1.0-alpha4--1.0-alpha5.sql | 86 ++++++++++++++++++++++++++ pgcolumnar--1.0-alpha5.sql | 86 ++++++++++++++++++++++++++ test/partition_lifecycle.sh | 19 +++--- 3 files changed, 183 insertions(+), 8 deletions(-) diff --git a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql index 58c7f0dae..f4c23a10a 100644 --- a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql @@ -963,6 +963,92 @@ BEGIN RETURN made; END; $create_ahead$; + +/* + * pgcolumnar.partition_drop_plan (#1297) + * + * The statements that retire every partition of a RANGE-partitioned parent + * whose whole range is older than cutoff (its upper bound at or before it), in + * range order: ALTER TABLE ... DETACH PARTITION ... CONCURRENTLY, then DROP + * TABLE. A partition a concurrent detach left pending gets DETACH ... FINALIZE + * instead. A partition reaching MAXVALUE is never old. + * + * RETURNED, NOT RUN. DETACH ... CONCURRENTLY is refused inside any function, DO + * block or procedure, on every major (#1297 note 1), and a plain DETACH or a + * DROP of an attached partition takes ACCESS EXCLUSIVE on the parent, which + * blocks every query on the table. The caller runs each statement at top level: + * + * SELECT * FROM pgcolumnar.partition_drop_plan('metrics', now() - interval '90 days') \gexec + * + * The DROP still takes ACCESS EXCLUSIVE on the partition it drops; after the + * detach nothing reaches that partition through the parent. + * + * A `timestamp` key holds wall-clock time, so cutoff is read as the session's + * wall-clock time for it, as partition_create_ahead reads "now". The same + * refusals as partition_create_ahead, with the same codes: a DEFAULT partition + * rules out the concurrent detach, and is refused rather than dropped with the + * exclusive one. + */ +CREATE FUNCTION pgcolumnar.partition_drop_plan(parent regclass, cutoff timestamptz) + RETURNS SETOF text + LANGUAGE plpgsql STABLE + SET search_path = pg_catalog, pg_temp + AS $drop_plan$ +DECLARE + strat "char"; + natts int; + defid oid; + keytype regtype; + p record; +BEGIN + IF cutoff IS NULL THEN + RAISE EXCEPTION 'cutoff must not be NULL' USING ERRCODE = '22023'; + END IF; + SELECT pt.partstrat, pt.partnatts, pt.partdefid, a.atttypid::regtype + INTO strat, natts, defid, keytype + FROM pg_partitioned_table pt + JOIN pg_attribute a ON a.attrelid = pt.partrelid AND a.attnum = pt.partattrs[0] + WHERE pt.partrelid = parent; + IF NOT FOUND OR strat <> 'r' OR natts <> 1 THEN + RAISE EXCEPTION '% is not RANGE-partitioned on one column', parent + USING ERRCODE = '55000'; + END IF; + IF keytype NOT IN ('timestamptz'::regtype, 'timestamp'::regtype) THEN + RAISE EXCEPTION '% is partitioned on %, not on a timestamp', parent, keytype + USING ERRCODE = '55000'; + END IF; + IF defid <> 0 THEN + RAISE EXCEPTION '% has a DEFAULT partition', parent + USING ERRCODE = '55000', + HINT = 'A DEFAULT partition rules out DETACH CONCURRENTLY, so the partition lifecycle refuses to manage this table.'; + END IF; + + FOR p IN + SELECT b.part, b.pending + FROM (SELECT c.oid::regclass AS part, i.inhdetachpending AS pending, + (regexp_match(pg_get_expr(c.relpartbound, c.oid), + 'FROM \(''([^'']+)''\)'))[1] AS lo, + (regexp_match(pg_get_expr(c.relpartbound, c.oid), + 'TO \(''([^'']+)''\)'))[1] AS hi + FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = parent) b + WHERE b.hi IS NOT NULL + AND CASE WHEN keytype = 'timestamp'::regtype + THEN b.hi::timestamp <= cutoff::timestamp + ELSE b.hi::timestamptz <= cutoff END + -- range order; a timestamp bound converts to timestamptz in one zone, monotonically + ORDER BY CASE WHEN keytype = 'timestamp'::regtype + THEN b.hi::timestamp::timestamptz ELSE b.hi::timestamptz END + LOOP + IF p.pending THEN + RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s FINALIZE', parent, p.part); + ELSE + RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s CONCURRENTLY', parent, p.part); + END IF; + RETURN NEXT format('DROP TABLE %s', p.part); + END LOOP; +END +$drop_plan$; /* --------------------------------------------------------------------------- * pgcolumnar.retired_group (#1294) * diff --git a/pgcolumnar--1.0-alpha5.sql b/pgcolumnar--1.0-alpha5.sql index 23fdd601f..664dbf54a 100644 --- a/pgcolumnar--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha5.sql @@ -2208,6 +2208,92 @@ BEGIN END; $create_ahead$; +/* + * pgcolumnar.partition_drop_plan (#1297) + * + * The statements that retire every partition of a RANGE-partitioned parent + * whose whole range is older than cutoff (its upper bound at or before it), in + * range order: ALTER TABLE ... DETACH PARTITION ... CONCURRENTLY, then DROP + * TABLE. A partition a concurrent detach left pending gets DETACH ... FINALIZE + * instead. A partition reaching MAXVALUE is never old. + * + * RETURNED, NOT RUN. DETACH ... CONCURRENTLY is refused inside any function, DO + * block or procedure, on every major (#1297 note 1), and a plain DETACH or a + * DROP of an attached partition takes ACCESS EXCLUSIVE on the parent, which + * blocks every query on the table. The caller runs each statement at top level: + * + * SELECT * FROM pgcolumnar.partition_drop_plan('metrics', now() - interval '90 days') \gexec + * + * The DROP still takes ACCESS EXCLUSIVE on the partition it drops; after the + * detach nothing reaches that partition through the parent. + * + * A `timestamp` key holds wall-clock time, so cutoff is read as the session's + * wall-clock time for it, as partition_create_ahead reads "now". The same + * refusals as partition_create_ahead, with the same codes: a DEFAULT partition + * rules out the concurrent detach, and is refused rather than dropped with the + * exclusive one. + */ +CREATE FUNCTION pgcolumnar.partition_drop_plan(parent regclass, cutoff timestamptz) + RETURNS SETOF text + LANGUAGE plpgsql STABLE + SET search_path = pg_catalog, pg_temp + AS $drop_plan$ +DECLARE + strat "char"; + natts int; + defid oid; + keytype regtype; + p record; +BEGIN + IF cutoff IS NULL THEN + RAISE EXCEPTION 'cutoff must not be NULL' USING ERRCODE = '22023'; + END IF; + SELECT pt.partstrat, pt.partnatts, pt.partdefid, a.atttypid::regtype + INTO strat, natts, defid, keytype + FROM pg_partitioned_table pt + JOIN pg_attribute a ON a.attrelid = pt.partrelid AND a.attnum = pt.partattrs[0] + WHERE pt.partrelid = parent; + IF NOT FOUND OR strat <> 'r' OR natts <> 1 THEN + RAISE EXCEPTION '% is not RANGE-partitioned on one column', parent + USING ERRCODE = '55000'; + END IF; + IF keytype NOT IN ('timestamptz'::regtype, 'timestamp'::regtype) THEN + RAISE EXCEPTION '% is partitioned on %, not on a timestamp', parent, keytype + USING ERRCODE = '55000'; + END IF; + IF defid <> 0 THEN + RAISE EXCEPTION '% has a DEFAULT partition', parent + USING ERRCODE = '55000', + HINT = 'A DEFAULT partition rules out DETACH CONCURRENTLY, so the partition lifecycle refuses to manage this table.'; + END IF; + + FOR p IN + SELECT b.part, b.pending + FROM (SELECT c.oid::regclass AS part, i.inhdetachpending AS pending, + (regexp_match(pg_get_expr(c.relpartbound, c.oid), + 'FROM \(''([^'']+)''\)'))[1] AS lo, + (regexp_match(pg_get_expr(c.relpartbound, c.oid), + 'TO \(''([^'']+)''\)'))[1] AS hi + FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = parent) b + WHERE b.hi IS NOT NULL + AND CASE WHEN keytype = 'timestamp'::regtype + THEN b.hi::timestamp <= cutoff::timestamp + ELSE b.hi::timestamptz <= cutoff END + -- range order; a timestamp bound converts to timestamptz in one zone, monotonically + ORDER BY CASE WHEN keytype = 'timestamp'::regtype + THEN b.hi::timestamp::timestamptz ELSE b.hi::timestamptz END + LOOP + IF p.pending THEN + RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s FINALIZE', parent, p.part); + ELSE + RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s CONCURRENTLY', parent, p.part); + END IF; + RETURN NEXT format('DROP TABLE %s', p.part); + END LOOP; +END +$drop_plan$; + /* --------------------------------------------------------------------------- * Incremental rollups (#1294). * diff --git a/test/partition_lifecycle.sh b/test/partition_lifecycle.sh index 09a5a0b7b..4ba8591ca 100755 --- a/test/partition_lifecycle.sh +++ b/test/partition_lifecycle.sh @@ -194,8 +194,8 @@ check "and nothing was created for it" \ # # partition_drop_plan(parent regclass, cutoff timestamptz) -> SETOF text # -# The statements that retire every partition whose whole range is older than -# cutoff, in range order: DETACH ... CONCURRENTLY, then DROP. They cannot run +# The statements, schema-qualified, that retire every partition whose whole range +# is older than cutoff, in range order: DETACH ... CONCURRENTLY, then DROP. They cannot run # inside a function (DETACH CONCURRENTLY refuses any function, DO block or # procedure, on every major: #1297 note 1), so the function RETURNS them and # the caller runs each as its own top-level statement (\gexec). A partition a @@ -218,8 +218,8 @@ check "premise: the drop fixture has four partitions, each holding rows" \ "$(q "SELECT count(DISTINCT tableoid) FROM dl;")|$(kids dl)" "4|4" check "the plan retires the partitions wholly before the cutoff, in range order" \ "$(plan dl '2020-03-15 00:00+00')" \ - "$(printf '%s\n' 'ALTER TABLE dl DETACH PARTITION dl_01 CONCURRENTLY' 'DROP TABLE dl_01' \ - 'ALTER TABLE dl DETACH PARTITION dl_02 CONCURRENTLY' 'DROP TABLE dl_02')" + "$(printf '%s\n' 'ALTER TABLE public.dl DETACH PARTITION public.dl_01 CONCURRENTLY' 'DROP TABLE public.dl_01' \ + 'ALTER TABLE public.dl DETACH PARTITION public.dl_02 CONCURRENTLY' 'DROP TABLE public.dl_02')" check "a cutoff on a partition's upper bound includes it" \ "$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2020-03-01 00:00+00') s WHERE s LIKE 'DROP%';")" "2" check "a cutoff before every partition plans nothing" \ @@ -238,13 +238,16 @@ check "a second plan for the same cutoff is empty" \ "$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2020-03-15 00:00+00');")" "0" # A timestamp (no zone) key: the cutoff is read in the session's zone, as the key's -# wall-clock time. +# wall-clock time. 23:00 in New York is 04:00 the next day in UTC, so a cutoff read +# in UTC would retire dw_a at the second cutoff too. q "CREATE TABLE dw (ts timestamp NOT NULL) PARTITION BY RANGE (ts); CREATE TABLE dw_a PARTITION OF dw FOR VALUES FROM ('2021-01-01') TO ('2021-01-02') USING pgcolumnar; CREATE TABLE dw_b PARTITION OF dw FOR VALUES FROM ('2021-01-02') TO ('2021-01-03') USING pgcolumnar;" >/dev/null check "a timestamp key's bounds are compared in the session's wall-clock time" \ - "$(q "SET TimeZone = 'America/New_York'; SELECT string_agg(s, ';') FROM pgcolumnar.partition_drop_plan('dw', '2021-01-02 00:00-05') s;")" \ - "ALTER TABLE dw DETACH PARTITION dw_a CONCURRENTLY;DROP TABLE dw_a" + "$(env PGTZ=America/New_York PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -c "SELECT coalesce(string_agg(s, ';'), 'none') FROM pgcolumnar.partition_drop_plan('dw', '2021-01-02 00:00-05') s" \ + -c "SELECT coalesce(string_agg(s, ';'), 'none') FROM pgcolumnar.partition_drop_plan('dw', '2021-01-01 23:00-05') s" 2>&1 | paste -sd'|')" \ + "ALTER TABLE public.dw DETACH PARTITION public.dw_a CONCURRENTLY;DROP TABLE public.dw_a|none" # An interrupted concurrent detach leaves the partition pending: a transaction # holding an older snapshot makes the second phase wait, and a statement timeout @@ -263,7 +266,7 @@ check "premise: the interrupted detach left dp_1 pending" \ "$(q "SELECT inhdetachpending FROM pg_inherits WHERE inhrelid = 'dp_1'::regclass;")" "t" check "a pending detach is finalised, not started again" \ "$(q "SELECT string_agg(s, ';') FROM pgcolumnar.partition_drop_plan('dp', '2020-02-01 00:00+00') s;")" \ - "ALTER TABLE dp DETACH PARTITION dp_1 FINALIZE;DROP TABLE dp_1" + "ALTER TABLE public.dp DETACH PARTITION public.dp_1 FINALIZE;DROP TABLE public.dp_1" run_plan dp '2020-02-01 00:00+00' check "and running it drops the pending partition" \ "$(q "SELECT count(*) FROM pg_class WHERE relname = 'dp_1';")|$(kids dp)" "0|1" From d0320408721aeb8fd282300022a5f12203312d66 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 14:01:47 +0000 Subject: [PATCH 03/12] test, docs: partition_drop_plan's pytest twin, the how-to and the reference (#1297) The twin was seen red on PG17 against main (42883) and passes with the function. Both fixtures now create their partitions out of range order, so a plan in catalog order would fail the order arm. Its own fixture: weekly partitions in 2019, Asia/Tokyo, a private schema. compare_to_bash grades the pair with nothing missing. cluster_tests 525 -> 526 by collection. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 ++ docs/how-to.md | 27 +++++ docs/sql-reference.md | 13 +++ test/partition_lifecycle.sh | 9 +- test/pytest/TESTS.md | 3 +- test/pytest/expected_tests.txt | 5 +- test/pytest/test_partition_lifecycle.py | 133 ++++++++++++++++++++++++ 7 files changed, 191 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03fa3396e..a08512581 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,13 @@ true until the next version shipped. ### Added +- `pgcolumnar.partition_drop_plan(parent, cutoff)` returns the statements that + retire every time partition whose whole range is older than `cutoff`: a + `DETACH PARTITION ... CONCURRENTLY` and a `DROP TABLE` each, in range order, + or `DETACH ... FINALIZE` for a partition an interrupted detach left pending + (#1297). Run them with `\gexec`. The parent is never locked exclusively. + `test/partition_lifecycle.sh` and `test_partition_lifecycle.py`. + - A `pgcolumnar.expire` that has nothing to retire now answers from the retention column's extent in one index lookup, instead of one zone-map probe per row group (#1310). Most passes of the maintenance daemon are such passes. When diff --git a/docs/how-to.md b/docs/how-to.md index 6e73f50fc..1d5550a76 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -482,6 +482,33 @@ partitions, before it creates any. Use a wider width or a shorter lead. `test/partition_lifecycle.sh` checks each of these statements, in sessions west and east of UTC. +## Drop old time partitions + +`pgcolumnar.partition_drop_plan` returns the statements that retire every +partition whose whole range is older than a cutoff. It does not run them, because +`DETACH PARTITION ... CONCURRENTLY` cannot run inside a function. Run each one as +its own statement, for example with psql's `\gexec`: + +```sql +SELECT * FROM pgcolumnar.partition_drop_plan('metrics', now() - interval '90 days') \gexec +``` + +For each old partition, in range order, the plan detaches it concurrently and +then drops it. The concurrent detach takes no exclusive lock on the parent, so +queries and inserts on the table continue. The drop locks only the detached +partition, which no query reaches through the parent any more. A partition whose +upper bound equals the cutoff is old. A partition that straddles the cutoff is +kept. + +If a concurrent detach was interrupted, the partition stays pending. The next +plan finishes that detach with `DETACH PARTITION ... FINALIZE` instead of starting +it again. + +For a `timestamp` key, the cutoff is read as the session's local time. The +function refuses the same tables as `partition_create_ahead`, because a DEFAULT +partition rules out a concurrent detach. A run of the plan drops the +partitions' rows: nothing keeps them. + ## Count distinct values and percentiles approximately For large tables, approximate distinct counts and percentiles cost far less than diff --git a/docs/sql-reference.md b/docs/sql-reference.md index 0c64f8908..6f9fe2926 100644 --- a/docs/sql-reference.md +++ b/docs/sql-reference.md @@ -622,6 +622,19 @@ refused before it creates any (`22023`). The function takes `SHARE UPDATE EXCLUSIVE` on the parent, never `ACCESS EXCLUSIVE`. See [Create time partitions ahead](how-to.md#create-time-partitions-ahead). +### pgcolumnar.partition_drop_plan(parent regclass, cutoff timestamptz) returns setof text + +Returns the statements that retire every partition of `parent` whose upper bound +is at or before `cutoff`, in range order. Each partition gets +`ALTER TABLE ... DETACH PARTITION ... CONCURRENTLY`, then `DROP TABLE`. A +partition that an interrupted concurrent detach left pending gets +`DETACH PARTITION ... FINALIZE` instead. The names are schema-qualified. The +function runs nothing: run each statement at top level, for example with +`\gexec`, because a concurrent detach cannot run inside a function. For a +`timestamp` key, `cutoff` is read as the session's local time. It refuses a +parent with the same codes as `partition_create_ahead` (`55000`), and a NULL +cutoff (`22023`). See [Drop old time partitions](how-to.md#drop-old-time-partitions). + ### pgcolumnar.create_rollup(rollup name, source regclass, ts_column name, width interval, origin timestamptz, group_by name[], agg_functions text[], agg_columns name[], access_method name DEFAULT 'pgcolumnar') returns regclass Creates a rollup of `source` in the source's schema. A rollup is a view of diff --git a/test/partition_lifecycle.sh b/test/partition_lifecycle.sh index 4ba8591ca..d44f142b6 100755 --- a/test/partition_lifecycle.sh +++ b/test/partition_lifecycle.sh @@ -209,12 +209,13 @@ run_plan() { # each statement top-level, as \gexec would run it done <<<"$(plan "$1" "$2")" } q "CREATE TABLE dl (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts); - CREATE TABLE dl_01 PARTITION OF dl FOR VALUES FROM ('2020-01-01 00:00+00') TO ('2020-02-01 00:00+00') USING pgcolumnar; - CREATE TABLE dl_02 PARTITION OF dl FOR VALUES FROM ('2020-02-01 00:00+00') TO ('2020-03-01 00:00+00') USING pgcolumnar; CREATE TABLE dl_03 PARTITION OF dl FOR VALUES FROM ('2020-03-01 00:00+00') TO ('2020-04-01 00:00+00') USING pgcolumnar; + CREATE TABLE dl_02 PARTITION OF dl FOR VALUES FROM ('2020-02-01 00:00+00') TO ('2020-03-01 00:00+00') USING pgcolumnar; CREATE TABLE dl_04 PARTITION OF dl FOR VALUES FROM ('2020-04-01 00:00+00') TO ('2020-05-01 00:00+00') USING pgcolumnar; + CREATE TABLE dl_01 PARTITION OF dl FOR VALUES FROM ('2020-01-01 00:00+00') TO ('2020-02-01 00:00+00') USING pgcolumnar; INSERT INTO dl SELECT '2020-01-01 00:00+00'::timestamptz + g * interval '1 hour', g FROM generate_series(0, 2903) g;" >/dev/null -check "premise: the drop fixture has four partitions, each holding rows" \ +# Created out of range order, so a plan in catalog order is not a plan in range order. +check "premise: the drop fixture has several partitions, each holding rows" \ "$(q "SELECT count(DISTINCT tableoid) FROM dl;")|$(kids dl)" "4|4" check "the plan retires the partitions wholly before the cutoff, in range order" \ "$(plan dl '2020-03-15 00:00+00')" \ @@ -262,7 +263,7 @@ sleep 1 env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -q \ -c "SET statement_timeout = '2s'" -c "ALTER TABLE dp DETACH PARTITION dp_1 CONCURRENTLY" >/dev/null 2>&1 wait "$holder" -check "premise: the interrupted detach left dp_1 pending" \ +check "premise: the interrupted detach left the partition pending" \ "$(q "SELECT inhdetachpending FROM pg_inherits WHERE inhrelid = 'dp_1'::regclass;")" "t" check "a pending detach is finalised, not started again" \ "$(q "SELECT string_agg(s, ';') FROM pgcolumnar.partition_drop_plan('dp', '2020-02-01 00:00+00') s;")" \ diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 3aa9a9ed7..794b6b3cd 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -6536,7 +6536,7 @@ THE MATERIAL'S NODE ID. `set_plan_references` numbers the main tree and then the ## 93. test_partition_lifecycle.py: partition_create_ahead keeps a time-partitioned parent ahead of now -Port of `partition_lifecycle.sh` (#1297). `pgcolumnar.partition_create_ahead(parent, width, ahead)` makes columnar partitions, each `width` wide and contiguous with the highest existing upper bound, until they reach now plus `ahead` widths. It uses a standalone CREATE then ATTACH, so the parent is held SHARE UPDATE EXCLUSIVE, never ACCESS EXCLUSIVE. A DEFAULT partition is refused (`55000`). +Port of `partition_lifecycle.sh` (#1297). The drop half, `partition_drop_plan`, returns its statements rather than running them: `DETACH ... CONCURRENTLY` refuses to run inside any function, so each statement is run at top level in its own autocommit session, as `\gexec` would. `pgcolumnar.partition_create_ahead(parent, width, ahead)` makes columnar partitions, each `width` wide and contiguous with the highest existing upper bound, until they reach now plus `ahead` widths. It uses a standalone CREATE then ATTACH, so the parent is held SHARE UPDATE EXCLUSIVE, never ACCESS EXCLUSIVE. A DEFAULT partition is refused (`55000`). THE SESSION'S ZONE IS WHERE THE PROTOTYPE BROKE. A `timestamp` key holds wall-clock time. The prototype read its bounds as timestamptz in the session's zone and wrote the next ones converted from UTC. In America/New_York that left a four-hour gap between partitions, so a row at the boundary had nowhere to go; and daily partitions made from Tokyo started at 15:00. The arms continue from hand-made partitions in sessions west and east of UTC, and check contiguity, routing at every boundary, coverage of the session's local now, and alignment to the width in the key's own wall clock. This file uses six-hour widths, and Asia/Kolkata for a half-hour offset. @@ -6549,6 +6549,7 @@ Removal proofs, each red in both twins: the prototype's zone handling; `CREATE T | test | what it holds | | --- | --- | | `test_partition_create_ahead` | from nothing it reaches now plus the lead, columnar, contiguous and one width wide; a second call creates nothing, and asking further ahead creates only the difference; rows land in columnar partitions; the parent is never held ACCESS EXCLUSIVE; west of UTC, for both key types, partitions continue a hand-made one contiguously and a row at every boundary routes; east of UTC, a timestamp parent covers the local now and starts on the width's boundary; inputs that broke the first version (a generated column, a 63-character name, more than 1000 partitions, a MAXVALUE bound) are handled or refused before any DDL; a DEFAULT partition is refused and nothing is created | +| `test_partition_drop_plan` | the drop half: `partition_drop_plan` lists `DETACH ... CONCURRENTLY` and `DROP` for the partitions wholly before the cutoff, in range order, from partitions created newest first. A cutoff on an upper bound includes the partition, and one before them all plans nothing. Running the plan, each statement top-level, leaves the rest with their rows and drops the storage rows. A second plan is empty. A `timestamp` key is cut in Tokyo's wall-clock time. A detach cancelled in its second phase is planned as `FINALIZE` and dropped. The DEFAULT and non-timestamp parents are refused (`55000`) | ## 94. test_approx_aggregates.py: hll and tdigest over columnar tables, the compatibility statement diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index f6d1e6f35..ec041c0a2 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -603,4 +603,7 @@ guard_tests 419 # test_a_key_well_under_the_gate_takes_the_node) and one added # (test_on_by_default_and_gated_on_the_estimate), by collection on this branch # rebased onto c90976d (#1329): 945 collected on main, 946 here. -cluster_tests 527 +# +# cluster_tests 527 -> 528 for #1297's test_partition_drop_plan, by collection on +# this branch rebased onto b55cb23 (#1330). +cluster_tests 528 diff --git a/test/pytest/test_partition_lifecycle.py b/test/pytest/test_partition_lifecycle.py index 7cfad89a3..2bfbcc6a5 100644 --- a/test/pytest/test_partition_lifecycle.py +++ b/test/pytest/test_partition_lifecycle.py @@ -230,3 +230,136 @@ def value_or_state(sql): state = e.sqlstate or "?" expect.text(state, "55000", "a parent with a DEFAULT partition is refused") expect.num(_kids(conn, "fallback"), 1, "and nothing was created for it") + + +def test_partition_drop_plan(pgc_conn, pgc_cluster, expect): + """partition_drop_plan returns the statements that retire every partition wholly + older than the cutoff: DETACH ... CONCURRENTLY then DROP, in range order, or + DETACH ... FINALIZE for one a concurrent detach left pending. They are returned, + not run, because DETACH ... CONCURRENTLY refuses to run inside any function + (#1297 note 1); each is run here at top level in its own autocommit session. + + Its own fixture: weekly partitions in 2019 where the shell suite has monthly + ones in 2020, Asia/Tokyo where it has America/New_York, and this test's private + schema, which every statement names. + """ + conn = pgc_conn + schema = _one(conn, "SELECT current_schema()") + dsn = pgc_cluster.dsn(dbname=_one(conn, "SELECT current_database()")) + + def plan(parent, cutoff): + with conn.cursor() as cur: + cur.execute(f"SELECT s FROM pgcolumnar.partition_drop_plan('{parent}', %s) s", (cutoff,)) + return [r[0] for r in cur.fetchall()] + + def run(statements): + with psycopg.connect(dsn, autocommit=True) as c: + for st in statements: + c.execute(st) + + def q(name): + return f"{schema}.{name}" + + weeks = ["2019-03-04", "2019-03-11", "2019-03-18", "2019-03-25", "2019-04-01", "2019-04-08"] + with conn.cursor() as cur: + cur.execute("CREATE TABLE xl (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts)") + # newest first, so catalog order is not range order + for i, (lo, hi) in reversed(list(enumerate(zip(weeks, weeks[1:])))): + cur.execute(f"CREATE TABLE xl_w{i} PARTITION OF xl FOR VALUES FROM ('{lo} 00:00+00')" + f" TO ('{hi} 00:00+00') USING pgcolumnar") + cur.execute("INSERT INTO xl SELECT '2019-03-04 00:00+00'::timestamptz + g * interval '30 min', g" + " FROM generate_series(0, 1679) g") + expect.text(f"{_one(conn, 'SELECT count(DISTINCT tableoid) FROM xl')}|{_kids(conn, 'xl')}", "5|5", + "premise: the drop fixture has several partitions, each holding rows") + + want = [] + for i in range(3): + want += [f"ALTER TABLE {q('xl')} DETACH PARTITION {q(f'xl_w{i}')} CONCURRENTLY", f"DROP TABLE {q(f'xl_w{i}')}"] + expect.text("\n".join(plan("xl", "2019-03-28 12:00+00")), "\n".join(want), + "the plan retires the partitions wholly before the cutoff, in range order") + expect.num(sum(1 for s in plan("xl", "2019-03-18 00:00+00") if s.startswith("DROP")), 2, + "a cutoff on a partition's upper bound includes it") + expect.num(len(plan("xl", "2018-12-31 00:00+00")), 0, "a cutoff before every partition plans nothing") + + sids = [_one(conn, f"SELECT pgcolumnar.get_storage_id('xl_w{i}')") for i in range(3)] + run(plan("xl", "2019-03-28 12:00+00")) + left = _one(conn, "SELECT string_agg(c.relname, ',' ORDER BY c.relname) FROM pg_inherits i" + " JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'xl'::regclass") + gone = _one(conn, "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace" + f" WHERE n.nspname = '{schema}' AND c.relname IN ('xl_w0', 'xl_w1', 'xl_w2')") + expect.text(f"{left}|{gone}", "xl_w3,xl_w4|0", "running the plan leaves only the partitions it did not name") + expect.num(_one(conn, "SELECT count(*) FROM xl"), + _one(conn, "SELECT count(*) FROM generate_series(0, 1679) g" + " WHERE '2019-03-04 00:00+00'::timestamptz + g * interval '30 min' >= '2019-03-25 00:00+00'"), + "and every row the remaining partitions held is still there") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.storage WHERE storage_id = ANY (%s)" % ( + "ARRAY[" + ",".join(str(s) for s in sids) + "]::bigint[]")), 0, + "and the dropped partitions' pgcolumnar storage rows are gone") + expect.num(len(plan("xl", "2019-03-28 12:00+00")), 0, "a second plan for the same cutoff is empty") + + # A timestamp key: the cutoff is the session's wall-clock time. 06:00 in Tokyo is + # 21:00 the previous day in UTC, so a UTC reading would retire nothing at the + # first cutoff. + with conn.cursor() as cur: + cur.execute("CREATE TABLE xw (ts timestamp NOT NULL) PARTITION BY RANGE (ts)") + cur.execute("CREATE TABLE xw_a PARTITION OF xw FOR VALUES FROM ('2019-05-01 00:00') TO ('2019-05-01 06:00') USING pgcolumnar") + cur.execute("CREATE TABLE xw_b PARTITION OF xw FOR VALUES FROM ('2019-05-01 06:00') TO ('2019-05-01 12:00') USING pgcolumnar") + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(f"SET search_path TO {schema}") + c.execute("SET TimeZone = 'Asia/Tokyo'") + got = [] + for cut in ("2019-05-01 06:00+09", "2019-05-01 05:00+09"): + row = c.execute("SELECT coalesce(string_agg(s, ';'), 'none') FROM pgcolumnar.partition_drop_plan('xw', %s) s", + (cut,)).fetchone() + got.append(row[0]) + expect.text("|".join(got), + f"ALTER TABLE {q('xw')} DETACH PARTITION {q('xw_a')} CONCURRENTLY;DROP TABLE {q('xw_a')}|none", + "a timestamp key's bounds are compared in the session's wall-clock time") + + # A pending detach: an older snapshot holds the second phase, a timeout cancels it. + with conn.cursor() as cur: + cur.execute("CREATE TABLE xp (ts timestamptz NOT NULL) PARTITION BY RANGE (ts)") + cur.execute("CREATE TABLE xp_1 PARTITION OF xp FOR VALUES FROM ('2019-06-01 00:00+00') TO ('2019-06-08 00:00+00') USING pgcolumnar") + cur.execute("CREATE TABLE xp_2 PARTITION OF xp FOR VALUES FROM ('2019-06-08 00:00+00') TO ('2019-06-15 00:00+00') USING pgcolumnar") + with psycopg.connect(dsn) as holder: + holder.execute(f"SET search_path TO {schema}") + holder.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") + holder.execute("SELECT count(*) FROM xp").fetchone() + with psycopg.connect(dsn, autocommit=True) as d: + d.execute(f"SET search_path TO {schema}") + d.execute("SET statement_timeout = '1500ms'") + try: + d.execute("ALTER TABLE xp DETACH PARTITION xp_1 CONCURRENTLY") + except psycopg.errors.QueryCanceled: + pass + holder.rollback() + expect.text(str(_one(conn, "SELECT inhdetachpending FROM pg_inherits WHERE inhrelid = 'xp_1'::regclass")), + "True", "premise: the interrupted detach left the partition pending") + pending_plan = plan("xp", "2019-06-08 00:00+00") + expect.text(";".join(pending_plan), + f"ALTER TABLE {q('xp')} DETACH PARTITION {q('xp_1')} FINALIZE;DROP TABLE {q('xp_1')}", + "a pending detach is finalised, not started again") + run(pending_plan) + gone = _one(conn, "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace" + f" WHERE n.nspname = '{schema}' AND c.relname = 'xp_1'") + expect.text(f"{gone}|{_kids(conn, 'xp')}", "0|1", "and running it drops the pending partition") + + def sqlstate(sql): + try: + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(f"SET search_path TO {schema}") + c.execute(sql).fetchall() + return "no error" + except psycopg.Error as e: + return e.sqlstate + + with conn.cursor() as cur: + cur.execute("CREATE TABLE xd (ts timestamptz NOT NULL) PARTITION BY RANGE (ts)") + cur.execute("CREATE TABLE xd_default PARTITION OF xd DEFAULT USING pgcolumnar") + cur.execute("CREATE TABLE xn (k bigint NOT NULL) PARTITION BY RANGE (k)") + cur.execute("CREATE TABLE xnl (k int)") + expect.text(sqlstate("SELECT * FROM pgcolumnar.partition_drop_plan('xd', now())"), "55000", + "a drop plan for a parent with a DEFAULT partition is refused") + expect.text(sqlstate("SELECT * FROM pgcolumnar.partition_drop_plan('xn', now())") + "|" + + sqlstate("SELECT * FROM pgcolumnar.partition_drop_plan('xnl', now())"), "55000|55000", + "a drop plan for a parent not partitioned on a timestamp is refused") From 565eed995017ef2efd36810b20373d81a245f0fb Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 14:35:40 +0000 Subject: [PATCH 04/12] test: 'a second plan for the same cutoff is empty' is a premise (#1297) No mutation of partition_drop_plan reddens it: dropped tables are gone, so a second plan is empty by construction. Renamed in both twins (#1236). Co-Authored-By: Claude Opus 5.5 --- test/partition_lifecycle.sh | 2 +- test/pytest/test_partition_lifecycle.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/partition_lifecycle.sh b/test/partition_lifecycle.sh index d44f142b6..3231386f7 100755 --- a/test/partition_lifecycle.sh +++ b/test/partition_lifecycle.sh @@ -235,7 +235,7 @@ check "and every row the remaining partitions held is still there" \ WHERE '2020-01-01 00:00+00'::timestamptz + g * interval '1 hour' >= '2020-03-01 00:00+00';")" check "and the dropped partitions' pgcolumnar storage rows are gone" \ "$(q "SELECT count(*) FROM pgcolumnar.storage WHERE storage_id IN (${sids:-0});")" "0" -check "a second plan for the same cutoff is empty" \ +check "premise: a second plan for the same cutoff is empty" \ "$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2020-03-15 00:00+00');")" "0" # A timestamp (no zone) key: the cutoff is read in the session's zone, as the key's diff --git a/test/pytest/test_partition_lifecycle.py b/test/pytest/test_partition_lifecycle.py index 2bfbcc6a5..f46b52bb6 100644 --- a/test/pytest/test_partition_lifecycle.py +++ b/test/pytest/test_partition_lifecycle.py @@ -295,7 +295,7 @@ def q(name): expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.storage WHERE storage_id = ANY (%s)" % ( "ARRAY[" + ",".join(str(s) for s in sids) + "]::bigint[]")), 0, "and the dropped partitions' pgcolumnar storage rows are gone") - expect.num(len(plan("xl", "2019-03-28 12:00+00")), 0, "a second plan for the same cutoff is empty") + expect.num(len(plan("xl", "2019-03-28 12:00+00")), 0, "premise: a second plan for the same cutoff is empty") # A timestamp key: the cutoff is the session's wall-clock time. 06:00 in Tokyo is # 21:00 the previous day in UTC, so a UTC reading would retire nothing at the From 59bb3fe193cb302d8bb1e02b9ce330adf20d5342 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 14:18:49 +0000 Subject: [PATCH 05/12] test: partition_drop_plan arms from jdatcmd's review of the design, red (#1297) Red on PG17: a cutoff of the other timestamp type is planned rather than refused (42804), and a tiered partition is planned with DROP TABLE, which fails on a foreign table. Green already, as guards: names that need quoting, MINVALUE and MAXVALUE bounds, a partition that is itself partitioned. Co-Authored-By: Claude Opus 5.5 --- test/partition_lifecycle.sh | 78 +++++++++++++++++++++++++++++++++---- 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/test/partition_lifecycle.sh b/test/partition_lifecycle.sh index 3231386f7..1b6ea4ca3 100755 --- a/test/partition_lifecycle.sh +++ b/test/partition_lifecycle.sh @@ -238,17 +238,79 @@ check "and the dropped partitions' pgcolumnar storage rows are gone" \ check "premise: a second plan for the same cutoff is empty" \ "$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2020-03-15 00:00+00');")" "0" -# A timestamp (no zone) key: the cutoff is read in the session's zone, as the key's -# wall-clock time. 23:00 in New York is 04:00 the next day in UTC, so a cutoff read -# in UTC would retire dw_a at the second cutoff too. +# The cutoff's type must be the key's (jdatcmd, #1297): a timestamp bound compared +# with a timestamptz cutoff converts in the session's zone, so the same call could +# plan differently from two sessions. A cutoff of the other type is refused +# (42804), and a matching one plans the same everywhere. q "CREATE TABLE dw (ts timestamp NOT NULL) PARTITION BY RANGE (ts); CREATE TABLE dw_a PARTITION OF dw FOR VALUES FROM ('2021-01-01') TO ('2021-01-02') USING pgcolumnar; CREATE TABLE dw_b PARTITION OF dw FOR VALUES FROM ('2021-01-02') TO ('2021-01-03') USING pgcolumnar;" >/dev/null -check "a timestamp key's bounds are compared in the session's wall-clock time" \ - "$(env PGTZ=America/New_York PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ - -c "SELECT coalesce(string_agg(s, ';'), 'none') FROM pgcolumnar.partition_drop_plan('dw', '2021-01-02 00:00-05') s" \ - -c "SELECT coalesce(string_agg(s, ';'), 'none') FROM pgcolumnar.partition_drop_plan('dw', '2021-01-01 23:00-05') s" 2>&1 | paste -sd'|')" \ - "ALTER TABLE public.dw DETACH PARTITION public.dw_a CONCURRENTLY;DROP TABLE public.dw_a|none" +plan_in() { # plan_in ZONE PARENT CUTOFF-EXPRESSION + env PGTZ="$1" PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -c "SELECT coalesce(string_agg(s, ';'), 'none') FROM pgcolumnar.partition_drop_plan('$2', $3) s" 2>&1 +} +check "a timestamp key's plan for a timestamp cutoff is the same from sessions west and east of UTC" \ + "$(plan_in America/Los_Angeles dw "'2021-01-02 00:00'::timestamp")|$(plan_in Asia/Kolkata dw "'2021-01-02 00:00'::timestamp")" \ + "ALTER TABLE public.dw DETACH PARTITION public.dw_a CONCURRENTLY;DROP TABLE public.dw_a|ALTER TABLE public.dw DETACH PARTITION public.dw_a CONCURRENTLY;DROP TABLE public.dw_a" +check "a cutoff of the other timestamp type is refused" \ + "$(sqlstate_or "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dw', '2021-01-02 00:00+00'::timestamptz)")|$(sqlstate_or "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2020-03-01 00:00'::timestamp)")" \ + "42804|42804" + +# Bounds at the ends of the range: MINVALUE below is listed once its upper bound +# is past, MAXVALUE above never is. +q "CREATE TABLE dm (ts timestamptz NOT NULL) PARTITION BY RANGE (ts); + CREATE TABLE dm_lo PARTITION OF dm FOR VALUES FROM (MINVALUE) TO ('2020-01-01 00:00+00') USING pgcolumnar; + CREATE TABLE dm_mid PARTITION OF dm FOR VALUES FROM ('2020-01-01 00:00+00') TO ('2020-02-01 00:00+00') USING pgcolumnar; + CREATE TABLE dm_hi PARTITION OF dm FOR VALUES FROM ('2020-02-01 00:00+00') TO (MAXVALUE) USING pgcolumnar;" >/dev/null +check "a partition from MINVALUE is listed once its upper bound is past, one to MAXVALUE never" \ + "$(q "SELECT string_agg(s, ';') FROM pgcolumnar.partition_drop_plan('dm', '2020-01-01 00:00+00') s WHERE s LIKE 'DROP%';")|$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dm', 'infinity') s WHERE s LIKE '%dm_hi%';")" \ + "DROP TABLE public.dm_lo|0" + +# Names that need quoting, in a schema that needs quoting: the plan must run as +# returned (#1317 was caught by exactly this). +q "CREATE SCHEMA \"Drop Zone\"; + CREATE TABLE \"Drop Zone\".\"Metrics\" (ts timestamptz NOT NULL) PARTITION BY RANGE (ts); + CREATE TABLE \"Drop Zone\".\"Old Part\" PARTITION OF \"Drop Zone\".\"Metrics\" + FOR VALUES FROM ('2020-01-01 00:00+00') TO ('2020-02-01 00:00+00') USING pgcolumnar; + CREATE TABLE \"Drop Zone\".\"New Part\" PARTITION OF \"Drop Zone\".\"Metrics\" + FOR VALUES FROM ('2020-02-01 00:00+00') TO ('2020-03-01 00:00+00') USING pgcolumnar;" >/dev/null +run_plan '"Drop Zone"."Metrics"' '2020-02-01 00:00+00' +check "a plan for names that need quoting runs as returned" \ + "$(q "SELECT string_agg(c.relname, ',') FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'Drop Zone' AND c.relkind IN ('r', 'p');")" "Metrics,New Part" + +# A partition that is itself partitioned is listed whole: the concurrent detach +# takes it with its children, and DROP drops them. +q "CREATE TABLE ds (ts timestamptz NOT NULL) PARTITION BY RANGE (ts); + CREATE TABLE ds_q1 PARTITION OF ds FOR VALUES FROM ('2020-01-01 00:00+00') TO ('2020-04-01 00:00+00') PARTITION BY RANGE (ts); + CREATE TABLE ds_q1_a PARTITION OF ds_q1 FOR VALUES FROM ('2020-01-01 00:00+00') TO ('2020-02-15 00:00+00') USING pgcolumnar; + CREATE TABLE ds_q1_b PARTITION OF ds_q1 FOR VALUES FROM ('2020-02-15 00:00+00') TO ('2020-04-01 00:00+00') USING pgcolumnar; + CREATE TABLE ds_q2 PARTITION OF ds FOR VALUES FROM ('2020-04-01 00:00+00') TO ('2020-07-01 00:00+00') USING pgcolumnar; + INSERT INTO ds SELECT '2020-01-01 00:00+00'::timestamptz + g * interval '1 day' FROM generate_series(0, 180) g;" >/dev/null +check "a partition that is itself partitioned is planned whole" \ + "$(plan ds '2020-04-01 00:00+00' | paste -sd';')" \ + "ALTER TABLE public.ds DETACH PARTITION public.ds_q1 CONCURRENTLY;DROP TABLE public.ds_q1" +run_plan ds '2020-04-01 00:00+00' +check "and running the plan drops it with its children" \ + "$(q "SELECT count(*) FROM pg_class WHERE relname IN ('ds_q1', 'ds_q1_a', 'ds_q1_b');")|$(q "SELECT count(*) FROM ds;")" \ + "0|$(q "SELECT count(*) FROM generate_series(0, 180) g WHERE '2020-01-01 00:00+00'::timestamptz + g * interval '1 day' >= '2020-04-01 00:00+00';")" + +# A partition tier_partition moved to Parquet (#1295) is a foreign table: it is +# listed like any other, dropped with DROP FOREIGN TABLE, and its file stays. +psql_run "CREATE SERVER drop_pq FOREIGN DATA WRAPPER pgcolumnar_parquet;" +q "CREATE TABLE dt (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts); + CREATE TABLE dt_jan PARTITION OF dt FOR VALUES FROM ('2020-01-01 00:00+00') TO ('2020-02-01 00:00+00') USING pgcolumnar; + CREATE TABLE dt_feb PARTITION OF dt FOR VALUES FROM ('2020-02-01 00:00+00') TO ('2020-03-01 00:00+00') USING pgcolumnar; + INSERT INTO dt SELECT '2020-01-01 00:00+00'::timestamptz + g * interval '1 hour', g FROM generate_series(0, 1400) g;" >/dev/null +mkdir -p "$PGC_WORKDIR/drop" && chmod 777 "$PGC_WORKDIR/drop" +check "premise: the January partition was tiered to Parquet" \ + "$(q "SELECT pgcolumnar.tier_partition('dt_jan', 'drop_pq', '$PGC_WORKDIR/drop/dt_jan.parquet');")|$(q "SELECT relkind FROM pg_class WHERE relname = 'dt_jan';")" "744|f" +check "a tiered partition is planned with DROP FOREIGN TABLE" \ + "$(plan dt '2020-02-01 00:00+00' | paste -sd';')" \ + "ALTER TABLE public.dt DETACH PARTITION public.dt_jan CONCURRENTLY;DROP FOREIGN TABLE public.dt_jan" +run_plan dt '2020-02-01 00:00+00' +check "and running it drops the foreign table and leaves its Parquet file" \ + "$(q "SELECT count(*) FROM pg_class WHERE relname = 'dt_jan';")|$([ -s "$PGC_WORKDIR/drop/dt_jan.parquet" ] && echo file-kept || echo file-gone)" "0|file-kept" # An interrupted concurrent detach leaves the partition pending: a transaction # holding an older snapshot makes the second phase wait, and a statement timeout From 35b5c14c6154ef548866cc7d6782eacde72de0f6 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 14:20:16 +0000 Subject: [PATCH 06/12] feat: partition_drop_plan takes a cutoff of the key's own type, drops tiered partitions as foreign tables (#1297) From jdatcmd's review: a timestamp bound compared with a timestamptz cutoff converts in the session's zone, so the same call could plan differently from two sessions. Two overloads now, timestamptz and timestamp, and a key of the other type is refused (42804, with a hint naming the right expression). A partition tier_partition moved to Parquet is planned with DROP FOREIGN TABLE; its file stays. The checks the overloads share live in partition_drop_candidates(parent, cutoff_type). partition_lifecycle.sh 49/49 on PG17; identical patches in both scripts. Co-Authored-By: Claude Opus 5.5 --- pgcolumnar--1.0-alpha4--1.0-alpha5.sql | 110 +++++++++++++++++-------- pgcolumnar--1.0-alpha5.sql | 110 +++++++++++++++++-------- 2 files changed, 148 insertions(+), 72 deletions(-) diff --git a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql index f4c23a10a..7c6e3fd93 100644 --- a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql @@ -970,8 +970,10 @@ $create_ahead$; * The statements that retire every partition of a RANGE-partitioned parent * whose whole range is older than cutoff (its upper bound at or before it), in * range order: ALTER TABLE ... DETACH PARTITION ... CONCURRENTLY, then DROP - * TABLE. A partition a concurrent detach left pending gets DETACH ... FINALIZE - * instead. A partition reaching MAXVALUE is never old. + * TABLE (DROP FOREIGN TABLE for one tier_partition moved to Parquet, whose file + * stays). A partition a concurrent detach left pending gets DETACH ... FINALIZE + * instead. A partition reaching MAXVALUE is never old; one from MINVALUE is, once + * its upper bound is past. A partition that is itself partitioned is listed whole. * * RETURNED, NOT RUN. DETACH ... CONCURRENTLY is refused inside any function, DO * block or procedure, on every major (#1297 note 1), and a plain DETACH or a @@ -980,30 +982,27 @@ $create_ahead$; * * SELECT * FROM pgcolumnar.partition_drop_plan('metrics', now() - interval '90 days') \gexec * - * The DROP still takes ACCESS EXCLUSIVE on the partition it drops; after the - * detach nothing reaches that partition through the parent. + * THE CUTOFF'S TYPE IS THE KEY'S (jdatcmd, #1297). A timestamp bound compared + * with a timestamptz cutoff converts in the session's zone, so one call could + * plan differently from two sessions. Two overloads, each refusing the other + * type of key (42804); an untyped literal resolves to timestamptz. * - * A `timestamp` key holds wall-clock time, so cutoff is read as the session's - * wall-clock time for it, as partition_create_ahead reads "now". The same - * refusals as partition_create_ahead, with the same codes: a DEFAULT partition - * rules out the concurrent detach, and is refused rather than dropped with the - * exclusive one. + * The same refusals as partition_create_ahead, with the same codes: a DEFAULT + * partition rules out the concurrent detach, and is refused rather than dropped + * with the exclusive one. No ownership check: this reads catalogs the caller + * can see, and DETACH checks ownership when the statements run. */ -CREATE FUNCTION pgcolumnar.partition_drop_plan(parent regclass, cutoff timestamptz) - RETURNS SETOF text +CREATE FUNCTION pgcolumnar.partition_drop_candidates(parent regclass, cutoff_type regtype) + RETURNS TABLE (part regclass, pending boolean, foreign_table boolean, upper_bound text) LANGUAGE plpgsql STABLE SET search_path = pg_catalog, pg_temp - AS $drop_plan$ + AS $drop_candidates$ DECLARE strat "char"; natts int; defid oid; keytype regtype; - p record; BEGIN - IF cutoff IS NULL THEN - RAISE EXCEPTION 'cutoff must not be NULL' USING ERRCODE = '22023'; - END IF; SELECT pt.partstrat, pt.partnatts, pt.partdefid, a.atttypid::regtype INTO strat, natts, defid, keytype FROM pg_partitioned_table pt @@ -1022,33 +1021,72 @@ BEGIN USING ERRCODE = '55000', HINT = 'A DEFAULT partition rules out DETACH CONCURRENTLY, so the partition lifecycle refuses to manage this table.'; END IF; - - FOR p IN - SELECT b.part, b.pending + IF keytype <> cutoff_type THEN + RAISE EXCEPTION '% is partitioned on %, so its cutoff must be a %, not a %', + parent, keytype, keytype, cutoff_type + USING ERRCODE = '42804', + HINT = CASE WHEN keytype = 'timestamp'::regtype + THEN 'Pass a timestamp, for example localtimestamp - interval ''90 days''.' + ELSE 'Pass a timestamptz, for example now() - interval ''90 days''.' END; + END IF; + -- MAXVALUE above has no upper bound to read, and is never old + RETURN QUERY + SELECT b.part, b.pending, b.foreign_table, b.upper_bound FROM (SELECT c.oid::regclass AS part, i.inhdetachpending AS pending, + c.relkind = 'f' AS foreign_table, (regexp_match(pg_get_expr(c.relpartbound, c.oid), - 'FROM \(''([^'']+)''\)'))[1] AS lo, - (regexp_match(pg_get_expr(c.relpartbound, c.oid), - 'TO \(''([^'']+)''\)'))[1] AS hi + 'TO \(''([^'']+)''\)'))[1] AS upper_bound FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = parent) b - WHERE b.hi IS NOT NULL - AND CASE WHEN keytype = 'timestamp'::regtype - THEN b.hi::timestamp <= cutoff::timestamp - ELSE b.hi::timestamptz <= cutoff END - -- range order; a timestamp bound converts to timestamptz in one zone, monotonically - ORDER BY CASE WHEN keytype = 'timestamp'::regtype - THEN b.hi::timestamp::timestamptz ELSE b.hi::timestamptz END + WHERE b.upper_bound IS NOT NULL; +END +$drop_candidates$; + +CREATE FUNCTION pgcolumnar.partition_drop_plan(parent regclass, cutoff timestamptz) + RETURNS SETOF text + LANGUAGE plpgsql STABLE + SET search_path = pg_catalog, pg_temp + AS $drop_plan_tz$ +DECLARE + p record; +BEGIN + IF cutoff IS NULL THEN + RAISE EXCEPTION 'cutoff must not be NULL' USING ERRCODE = '22023'; + END IF; + FOR p IN + SELECT d.* FROM pgcolumnar.partition_drop_candidates(parent, 'timestamptz') d + WHERE d.upper_bound::timestamptz <= cutoff + ORDER BY d.upper_bound::timestamptz LOOP - IF p.pending THEN - RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s FINALIZE', parent, p.part); - ELSE - RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s CONCURRENTLY', parent, p.part); - END IF; - RETURN NEXT format('DROP TABLE %s', p.part); + RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s %s', parent, p.part, + CASE WHEN p.pending THEN 'FINALIZE' ELSE 'CONCURRENTLY' END); + RETURN NEXT format('DROP %s %s', CASE WHEN p.foreign_table THEN 'FOREIGN TABLE' ELSE 'TABLE' END, p.part); + END LOOP; +END +$drop_plan_tz$; + +CREATE FUNCTION pgcolumnar.partition_drop_plan(parent regclass, cutoff timestamp) + RETURNS SETOF text + LANGUAGE plpgsql STABLE + SET search_path = pg_catalog, pg_temp + AS $drop_plan_ts$ +DECLARE + p record; +BEGIN + IF cutoff IS NULL THEN + RAISE EXCEPTION 'cutoff must not be NULL' USING ERRCODE = '22023'; + END IF; + FOR p IN + SELECT d.* FROM pgcolumnar.partition_drop_candidates(parent, 'timestamp') d + WHERE d.upper_bound::timestamp <= cutoff + ORDER BY d.upper_bound::timestamp + LOOP + RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s %s', parent, p.part, + CASE WHEN p.pending THEN 'FINALIZE' ELSE 'CONCURRENTLY' END); + RETURN NEXT format('DROP %s %s', CASE WHEN p.foreign_table THEN 'FOREIGN TABLE' ELSE 'TABLE' END, p.part); END LOOP; END -$drop_plan$; +$drop_plan_ts$; /* --------------------------------------------------------------------------- * pgcolumnar.retired_group (#1294) * diff --git a/pgcolumnar--1.0-alpha5.sql b/pgcolumnar--1.0-alpha5.sql index 664dbf54a..fc1c150c7 100644 --- a/pgcolumnar--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha5.sql @@ -2214,8 +2214,10 @@ $create_ahead$; * The statements that retire every partition of a RANGE-partitioned parent * whose whole range is older than cutoff (its upper bound at or before it), in * range order: ALTER TABLE ... DETACH PARTITION ... CONCURRENTLY, then DROP - * TABLE. A partition a concurrent detach left pending gets DETACH ... FINALIZE - * instead. A partition reaching MAXVALUE is never old. + * TABLE (DROP FOREIGN TABLE for one tier_partition moved to Parquet, whose file + * stays). A partition a concurrent detach left pending gets DETACH ... FINALIZE + * instead. A partition reaching MAXVALUE is never old; one from MINVALUE is, once + * its upper bound is past. A partition that is itself partitioned is listed whole. * * RETURNED, NOT RUN. DETACH ... CONCURRENTLY is refused inside any function, DO * block or procedure, on every major (#1297 note 1), and a plain DETACH or a @@ -2224,30 +2226,27 @@ $create_ahead$; * * SELECT * FROM pgcolumnar.partition_drop_plan('metrics', now() - interval '90 days') \gexec * - * The DROP still takes ACCESS EXCLUSIVE on the partition it drops; after the - * detach nothing reaches that partition through the parent. + * THE CUTOFF'S TYPE IS THE KEY'S (jdatcmd, #1297). A timestamp bound compared + * with a timestamptz cutoff converts in the session's zone, so one call could + * plan differently from two sessions. Two overloads, each refusing the other + * type of key (42804); an untyped literal resolves to timestamptz. * - * A `timestamp` key holds wall-clock time, so cutoff is read as the session's - * wall-clock time for it, as partition_create_ahead reads "now". The same - * refusals as partition_create_ahead, with the same codes: a DEFAULT partition - * rules out the concurrent detach, and is refused rather than dropped with the - * exclusive one. + * The same refusals as partition_create_ahead, with the same codes: a DEFAULT + * partition rules out the concurrent detach, and is refused rather than dropped + * with the exclusive one. No ownership check: this reads catalogs the caller + * can see, and DETACH checks ownership when the statements run. */ -CREATE FUNCTION pgcolumnar.partition_drop_plan(parent regclass, cutoff timestamptz) - RETURNS SETOF text +CREATE FUNCTION pgcolumnar.partition_drop_candidates(parent regclass, cutoff_type regtype) + RETURNS TABLE (part regclass, pending boolean, foreign_table boolean, upper_bound text) LANGUAGE plpgsql STABLE SET search_path = pg_catalog, pg_temp - AS $drop_plan$ + AS $drop_candidates$ DECLARE strat "char"; natts int; defid oid; keytype regtype; - p record; BEGIN - IF cutoff IS NULL THEN - RAISE EXCEPTION 'cutoff must not be NULL' USING ERRCODE = '22023'; - END IF; SELECT pt.partstrat, pt.partnatts, pt.partdefid, a.atttypid::regtype INTO strat, natts, defid, keytype FROM pg_partitioned_table pt @@ -2266,33 +2265,72 @@ BEGIN USING ERRCODE = '55000', HINT = 'A DEFAULT partition rules out DETACH CONCURRENTLY, so the partition lifecycle refuses to manage this table.'; END IF; - - FOR p IN - SELECT b.part, b.pending + IF keytype <> cutoff_type THEN + RAISE EXCEPTION '% is partitioned on %, so its cutoff must be a %, not a %', + parent, keytype, keytype, cutoff_type + USING ERRCODE = '42804', + HINT = CASE WHEN keytype = 'timestamp'::regtype + THEN 'Pass a timestamp, for example localtimestamp - interval ''90 days''.' + ELSE 'Pass a timestamptz, for example now() - interval ''90 days''.' END; + END IF; + -- MAXVALUE above has no upper bound to read, and is never old + RETURN QUERY + SELECT b.part, b.pending, b.foreign_table, b.upper_bound FROM (SELECT c.oid::regclass AS part, i.inhdetachpending AS pending, + c.relkind = 'f' AS foreign_table, (regexp_match(pg_get_expr(c.relpartbound, c.oid), - 'FROM \(''([^'']+)''\)'))[1] AS lo, - (regexp_match(pg_get_expr(c.relpartbound, c.oid), - 'TO \(''([^'']+)''\)'))[1] AS hi + 'TO \(''([^'']+)''\)'))[1] AS upper_bound FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = parent) b - WHERE b.hi IS NOT NULL - AND CASE WHEN keytype = 'timestamp'::regtype - THEN b.hi::timestamp <= cutoff::timestamp - ELSE b.hi::timestamptz <= cutoff END - -- range order; a timestamp bound converts to timestamptz in one zone, monotonically - ORDER BY CASE WHEN keytype = 'timestamp'::regtype - THEN b.hi::timestamp::timestamptz ELSE b.hi::timestamptz END + WHERE b.upper_bound IS NOT NULL; +END +$drop_candidates$; + +CREATE FUNCTION pgcolumnar.partition_drop_plan(parent regclass, cutoff timestamptz) + RETURNS SETOF text + LANGUAGE plpgsql STABLE + SET search_path = pg_catalog, pg_temp + AS $drop_plan_tz$ +DECLARE + p record; +BEGIN + IF cutoff IS NULL THEN + RAISE EXCEPTION 'cutoff must not be NULL' USING ERRCODE = '22023'; + END IF; + FOR p IN + SELECT d.* FROM pgcolumnar.partition_drop_candidates(parent, 'timestamptz') d + WHERE d.upper_bound::timestamptz <= cutoff + ORDER BY d.upper_bound::timestamptz LOOP - IF p.pending THEN - RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s FINALIZE', parent, p.part); - ELSE - RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s CONCURRENTLY', parent, p.part); - END IF; - RETURN NEXT format('DROP TABLE %s', p.part); + RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s %s', parent, p.part, + CASE WHEN p.pending THEN 'FINALIZE' ELSE 'CONCURRENTLY' END); + RETURN NEXT format('DROP %s %s', CASE WHEN p.foreign_table THEN 'FOREIGN TABLE' ELSE 'TABLE' END, p.part); + END LOOP; +END +$drop_plan_tz$; + +CREATE FUNCTION pgcolumnar.partition_drop_plan(parent regclass, cutoff timestamp) + RETURNS SETOF text + LANGUAGE plpgsql STABLE + SET search_path = pg_catalog, pg_temp + AS $drop_plan_ts$ +DECLARE + p record; +BEGIN + IF cutoff IS NULL THEN + RAISE EXCEPTION 'cutoff must not be NULL' USING ERRCODE = '22023'; + END IF; + FOR p IN + SELECT d.* FROM pgcolumnar.partition_drop_candidates(parent, 'timestamp') d + WHERE d.upper_bound::timestamp <= cutoff + ORDER BY d.upper_bound::timestamp + LOOP + RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s %s', parent, p.part, + CASE WHEN p.pending THEN 'FINALIZE' ELSE 'CONCURRENTLY' END); + RETURN NEXT format('DROP %s %s', CASE WHEN p.foreign_table THEN 'FOREIGN TABLE' ELSE 'TABLE' END, p.part); END LOOP; END -$drop_plan$; +$drop_plan_ts$; /* --------------------------------------------------------------------------- * Incremental rollups (#1294). From be9d7dd666f3d1f878c113b1a6b372c0d8a61808 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 14:21:56 +0000 Subject: [PATCH 07/12] test, docs: the twin's arms for jdatcmd's review points; the cutoff type rule, tiered and sub-partitioned partitions documented (#1297) test_partition_lifecycle.py matches the shell suite's 49 checks by name (compare_to_bash: nothing missing), with its own fixture: Tokyo and Denver, a quoted schema of its own, half-year sub-partitions, a tiered January in 2019. Both twins green on PG17. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 5 +- docs/how-to.md | 16 +++- docs/sql-reference.md | 19 ++-- test/pytest/TESTS.md | 2 +- test/pytest/test_partition_lifecycle.py | 122 +++++++++++++++++++----- 5 files changed, 128 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a08512581..a13222c83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,10 @@ true until the next version shipped. retire every time partition whose whole range is older than `cutoff`: a `DETACH PARTITION ... CONCURRENTLY` and a `DROP TABLE` each, in range order, or `DETACH ... FINALIZE` for a partition an interrupted detach left pending - (#1297). Run them with `\gexec`. The parent is never locked exclusively. + (#1297). Run them with `\gexec`. The parent is never locked exclusively. The + cutoff has the key's own type, so a plan does not depend on the session's + time zone. A partition moved to Parquet by `tier_partition` is dropped as a + foreign table, and its file stays. `test/partition_lifecycle.sh` and `test_partition_lifecycle.py`. - A `pgcolumnar.expire` that has nothing to retire now answers from the diff --git a/docs/how-to.md b/docs/how-to.md index 1d5550a76..dab27002a 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -504,9 +504,19 @@ If a concurrent detach was interrupted, the partition stays pending. The next plan finishes that detach with `DETACH PARTITION ... FINALIZE` instead of starting it again. -For a `timestamp` key, the cutoff is read as the session's local time. The -function refuses the same tables as `partition_create_ahead`, because a DEFAULT -partition rules out a concurrent detach. A run of the plan drops the +The cutoff has the key's type: a `timestamptz` for a `timestamptz` key, as in +the example, and a `timestamp` for a `timestamp` key, such as +`localtimestamp - interval '90 days'`. A cutoff of the other type is refused +(`42804`), because comparing across the two depends on the session's time zone, +and the same call would plan differently from two sessions. + +A partition that is itself partitioned is planned as one, and the `DROP` takes +its children with it. A partition that `tier_partition` moved to Parquet is +planned with `DROP FOREIGN TABLE`. That removes the foreign table, and the +Parquet file stays where it is: delete it yourself when you no longer need it. + +The function refuses the same tables as `partition_create_ahead`, because a +DEFAULT partition rules out a concurrent detach. A run of the plan drops the partitions' rows: nothing keeps them. ## Count distinct values and percentiles approximately diff --git a/docs/sql-reference.md b/docs/sql-reference.md index 6f9fe2926..f78d06406 100644 --- a/docs/sql-reference.md +++ b/docs/sql-reference.md @@ -622,18 +622,21 @@ refused before it creates any (`22023`). The function takes `SHARE UPDATE EXCLUSIVE` on the parent, never `ACCESS EXCLUSIVE`. See [Create time partitions ahead](how-to.md#create-time-partitions-ahead). -### pgcolumnar.partition_drop_plan(parent regclass, cutoff timestamptz) returns setof text +### pgcolumnar.partition_drop_plan(parent regclass, cutoff timestamptz | timestamp) returns setof text Returns the statements that retire every partition of `parent` whose upper bound is at or before `cutoff`, in range order. Each partition gets -`ALTER TABLE ... DETACH PARTITION ... CONCURRENTLY`, then `DROP TABLE`. A -partition that an interrupted concurrent detach left pending gets -`DETACH PARTITION ... FINALIZE` instead. The names are schema-qualified. The +`ALTER TABLE ... DETACH PARTITION ... CONCURRENTLY`, then `DROP TABLE`, or `DROP FOREIGN TABLE` for a partition `tier_partition` moved to +Parquet (the file stays). A partition that an interrupted concurrent detach +left pending gets `DETACH PARTITION ... FINALIZE` instead. A partition that is +itself partitioned is listed whole. The names are schema-qualified. The function runs nothing: run each statement at top level, for example with -`\gexec`, because a concurrent detach cannot run inside a function. For a -`timestamp` key, `cutoff` is read as the session's local time. It refuses a -parent with the same codes as `partition_create_ahead` (`55000`), and a NULL -cutoff (`22023`). See [Drop old time partitions](how-to.md#drop-old-time-partitions). +`\gexec`, because a concurrent detach cannot run inside a function. `cutoff` +must have the key's type; the other type is refused (`42804`), and an untyped +literal is a `timestamptz`. It refuses a parent with the same codes as +`partition_create_ahead` (`55000`), and a NULL cutoff (`22023`). +`pgcolumnar.partition_drop_candidates(parent, cutoff_type)` holds the checks +the two overloads share. See [Drop old time partitions](how-to.md#drop-old-time-partitions). ### pgcolumnar.create_rollup(rollup name, source regclass, ts_column name, width interval, origin timestamptz, group_by name[], agg_functions text[], agg_columns name[], access_method name DEFAULT 'pgcolumnar') returns regclass diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 794b6b3cd..07981df9d 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -6549,7 +6549,7 @@ Removal proofs, each red in both twins: the prototype's zone handling; `CREATE T | test | what it holds | | --- | --- | | `test_partition_create_ahead` | from nothing it reaches now plus the lead, columnar, contiguous and one width wide; a second call creates nothing, and asking further ahead creates only the difference; rows land in columnar partitions; the parent is never held ACCESS EXCLUSIVE; west of UTC, for both key types, partitions continue a hand-made one contiguously and a row at every boundary routes; east of UTC, a timestamp parent covers the local now and starts on the width's boundary; inputs that broke the first version (a generated column, a 63-character name, more than 1000 partitions, a MAXVALUE bound) are handled or refused before any DDL; a DEFAULT partition is refused and nothing is created | -| `test_partition_drop_plan` | the drop half: `partition_drop_plan` lists `DETACH ... CONCURRENTLY` and `DROP` for the partitions wholly before the cutoff, in range order, from partitions created newest first. A cutoff on an upper bound includes the partition, and one before them all plans nothing. Running the plan, each statement top-level, leaves the rest with their rows and drops the storage rows. A second plan is empty. A `timestamp` key is cut in Tokyo's wall-clock time. A detach cancelled in its second phase is planned as `FINALIZE` and dropped. The DEFAULT and non-timestamp parents are refused (`55000`) | +| `test_partition_drop_plan` | the drop half: `partition_drop_plan` lists `DETACH ... CONCURRENTLY` and `DROP` for the partitions wholly before the cutoff, in range order, from partitions created newest first. A cutoff on an upper bound includes the partition, and one before them all plans nothing. Running the plan, each statement top-level, leaves the rest with their rows and drops the storage rows. A `timestamp` key with a `timestamp` cutoff plans the same in Tokyo and Denver, and a cutoff of the other type is refused (`42804`). MINVALUE is listed once past, and MAXVALUE never. Quoted schema and partition names run as returned. A sub-partitioned partition is planned whole and dropped with its children. A tiered partition is planned with `DROP FOREIGN TABLE`, and its Parquet file stays. A detach cancelled in its second phase is planned as `FINALIZE` and dropped. The DEFAULT and non-timestamp parents are refused (`55000`) | ## 94. test_approx_aggregates.py: hll and tdigest over columnar tables, the compatibility statement diff --git a/test/pytest/test_partition_lifecycle.py b/test/pytest/test_partition_lifecycle.py index f46b52bb6..ff022db17 100644 --- a/test/pytest/test_partition_lifecycle.py +++ b/test/pytest/test_partition_lifecycle.py @@ -260,6 +260,16 @@ def run(statements): def q(name): return f"{schema}.{name}" + def sqlstate(sql): + try: + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(f"SET search_path TO {schema}") + c.execute(sql).fetchall() + return "no error" + except psycopg.Error as e: + return e.sqlstate + + weeks = ["2019-03-04", "2019-03-11", "2019-03-18", "2019-03-25", "2019-04-01", "2019-04-08"] with conn.cursor() as cur: cur.execute("CREATE TABLE xl (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts)") @@ -297,24 +307,99 @@ def q(name): "and the dropped partitions' pgcolumnar storage rows are gone") expect.num(len(plan("xl", "2019-03-28 12:00+00")), 0, "premise: a second plan for the same cutoff is empty") - # A timestamp key: the cutoff is the session's wall-clock time. 06:00 in Tokyo is - # 21:00 the previous day in UTC, so a UTC reading would retire nothing at the - # first cutoff. + # The cutoff's type must be the key's: a timestamp key with a timestamp cutoff + # plans the same in every session, and a cutoff of the other type is refused. with conn.cursor() as cur: cur.execute("CREATE TABLE xw (ts timestamp NOT NULL) PARTITION BY RANGE (ts)") cur.execute("CREATE TABLE xw_a PARTITION OF xw FOR VALUES FROM ('2019-05-01 00:00') TO ('2019-05-01 06:00') USING pgcolumnar") cur.execute("CREATE TABLE xw_b PARTITION OF xw FOR VALUES FROM ('2019-05-01 06:00') TO ('2019-05-01 12:00') USING pgcolumnar") - with psycopg.connect(dsn, autocommit=True) as c: - c.execute(f"SET search_path TO {schema}") - c.execute("SET TimeZone = 'Asia/Tokyo'") - got = [] - for cut in ("2019-05-01 06:00+09", "2019-05-01 05:00+09"): - row = c.execute("SELECT coalesce(string_agg(s, ';'), 'none') FROM pgcolumnar.partition_drop_plan('xw', %s) s", - (cut,)).fetchone() - got.append(row[0]) - expect.text("|".join(got), - f"ALTER TABLE {q('xw')} DETACH PARTITION {q('xw_a')} CONCURRENTLY;DROP TABLE {q('xw_a')}|none", - "a timestamp key's bounds are compared in the session's wall-clock time") + got = [] + for zone in ("Asia/Tokyo", "America/Denver"): + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(f"SET search_path TO {schema}") + c.execute(f"SET TimeZone = '{zone}'") + got.append(c.execute("SELECT coalesce(string_agg(s, ';'), 'none')" + " FROM pgcolumnar.partition_drop_plan('xw', '2019-05-01 06:00'::timestamp) s").fetchone()[0]) + one = f"ALTER TABLE {q('xw')} DETACH PARTITION {q('xw_a')} CONCURRENTLY;DROP TABLE {q('xw_a')}" + expect.text("|".join(got), f"{one}|{one}", + "a timestamp key's plan for a timestamp cutoff is the same from sessions west and east of UTC") + expect.text(sqlstate("SELECT * FROM pgcolumnar.partition_drop_plan('xw', '2019-05-01 06:00+00'::timestamptz)") + "|" + + sqlstate("SELECT * FROM pgcolumnar.partition_drop_plan('xl', '2019-04-01 00:00'::timestamp)"), + "42804|42804", "a cutoff of the other timestamp type is refused") + + # MINVALUE below is listed once its upper bound is past; MAXVALUE above never. + with conn.cursor() as cur: + cur.execute("CREATE TABLE xm (ts timestamptz NOT NULL) PARTITION BY RANGE (ts)") + cur.execute("CREATE TABLE xm_lo PARTITION OF xm FOR VALUES FROM (MINVALUE) TO ('2019-07-01 00:00+00') USING pgcolumnar") + cur.execute("CREATE TABLE xm_hi PARTITION OF xm FOR VALUES FROM ('2019-07-01 00:00+00') TO (MAXVALUE) USING pgcolumnar") + lo = [s for s in plan("xm", "2019-07-01 00:00+00") if s.startswith("DROP")] + hi = [s for s in plan("xm", "infinity") if "xm_hi" in s] + expect.text(f"{';'.join(lo)}|{len(hi)}", f"DROP TABLE {q('xm_lo')}|0", + "a partition from MINVALUE is listed once its upper bound is past, one to MAXVALUE never") + + # Names that need quoting, in a schema that needs quoting. + with conn.cursor() as cur: + cur.execute('CREATE SCHEMA "Py Drop"') + cur.execute('CREATE TABLE "Py Drop"."Readings" (ts timestamptz NOT NULL) PARTITION BY RANGE (ts)') + cur.execute('CREATE TABLE "Py Drop"."Week One" PARTITION OF "Py Drop"."Readings"' + " FOR VALUES FROM ('2019-08-05 00:00+00') TO ('2019-08-12 00:00+00') USING pgcolumnar") + cur.execute('CREATE TABLE "Py Drop"."Week Two" PARTITION OF "Py Drop"."Readings"' + " FOR VALUES FROM ('2019-08-12 00:00+00') TO ('2019-08-19 00:00+00') USING pgcolumnar") + try: + run(plan('"Py Drop"."Readings"', "2019-08-12 00:00+00")) + left = _one(conn, "SELECT string_agg(c.relname, ',' ORDER BY c.relname) FROM pg_class c" + " JOIN pg_namespace n ON n.oid = c.relnamespace" + " WHERE n.nspname = 'Py Drop' AND c.relkind IN ('r', 'p')") + finally: + conn.execute('DROP SCHEMA "Py Drop" CASCADE') + expect.text(left, "Readings,Week Two", "a plan for names that need quoting runs as returned") + + # A partition that is itself partitioned is listed whole and dropped with its children. + with conn.cursor() as cur: + cur.execute("CREATE TABLE xs (ts timestamptz NOT NULL) PARTITION BY RANGE (ts)") + cur.execute("CREATE TABLE xs_h1 PARTITION OF xs FOR VALUES FROM ('2019-01-01 00:00+00') TO ('2019-07-01 00:00+00')" + " PARTITION BY RANGE (ts)") + for name, lo_, hi_ in (("xs_h1_a", "2019-01-01", "2019-04-01"), ("xs_h1_b", "2019-04-01", "2019-07-01")): + cur.execute(f"CREATE TABLE {name} PARTITION OF xs_h1 FOR VALUES FROM ('{lo_} 00:00+00') TO ('{hi_} 00:00+00')" + " USING pgcolumnar") + cur.execute("CREATE TABLE xs_h2 PARTITION OF xs FOR VALUES FROM ('2019-07-01 00:00+00') TO ('2020-01-01 00:00+00')" + " USING pgcolumnar") + cur.execute("INSERT INTO xs SELECT '2019-01-01 00:00+00'::timestamptz + g * interval '1 day' FROM generate_series(0, 364) g") + expect.text(";".join(plan("xs", "2019-07-01 00:00+00")), + f"ALTER TABLE {q('xs')} DETACH PARTITION {q('xs_h1')} CONCURRENTLY;DROP TABLE {q('xs_h1')}", + "a partition that is itself partitioned is planned whole") + run(plan("xs", "2019-07-01 00:00+00")) + gone = _one(conn, "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace" + f" WHERE n.nspname = '{schema}' AND c.relname IN ('xs_h1', 'xs_h1_a', 'xs_h1_b')") + kept = _one(conn, "SELECT count(*) FROM generate_series(0, 364) g" + " WHERE '2019-01-01 00:00+00'::timestamptz + g * interval '1 day' >= '2019-07-01 00:00+00'") + expect.text(f"{gone}|{_one(conn, 'SELECT count(*) FROM xs')}", f"0|{kept}", + "and running the plan drops it with its children") + + # A partition tier_partition moved to Parquet: DROP FOREIGN TABLE, and its file stays. + path = _one(conn, "SELECT current_setting('data_directory')") + f"/{schema}_xt_jan.parquet" + with conn.cursor() as cur: + cur.execute(f"CREATE SERVER {schema}_pq FOREIGN DATA WRAPPER pgcolumnar_parquet") + cur.execute("CREATE TABLE xt (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts)") + cur.execute("CREATE TABLE xt_jan PARTITION OF xt FOR VALUES FROM ('2019-01-01 00:00+00') TO ('2019-02-01 00:00+00') USING pgcolumnar") + cur.execute("CREATE TABLE xt_feb PARTITION OF xt FOR VALUES FROM ('2019-02-01 00:00+00') TO ('2019-03-01 00:00+00') USING pgcolumnar") + cur.execute("INSERT INTO xt SELECT '2019-01-01 00:00+00'::timestamptz + g * interval '2 hours', g FROM generate_series(0, 670) g") + try: + moved = _one(conn, f"SELECT pgcolumnar.tier_partition('xt_jan', '{schema}_pq', '{path}')") + kind = _one(conn, "SELECT relkind FROM pg_class WHERE oid = 'xt_jan'::regclass") + expect.text(f"{moved}|{kind}", "372|f", "premise: the January partition was tiered to Parquet") + tiered = plan("xt", "2019-02-01 00:00+00") + expect.text(";".join(tiered), + f"ALTER TABLE {q('xt')} DETACH PARTITION {q('xt_jan')} CONCURRENTLY;DROP FOREIGN TABLE {q('xt_jan')}", + "a tiered partition is planned with DROP FOREIGN TABLE") + run(tiered) + left = _one(conn, "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace" + f" WHERE n.nspname = '{schema}' AND c.relname = 'xt_jan'") + size = _one(conn, f"SELECT (pg_stat_file('{path}', true)).size") + expect.text(f"{left}|{'file-kept' if size else 'file-gone'}", "0|file-kept", + "and running it drops the foreign table and leaves its Parquet file") + finally: + conn.execute(f"DROP SERVER {schema}_pq CASCADE") # A pending detach: an older snapshot holds the second phase, a timeout cancels it. with conn.cursor() as cur: @@ -344,15 +429,6 @@ def q(name): f" WHERE n.nspname = '{schema}' AND c.relname = 'xp_1'") expect.text(f"{gone}|{_kids(conn, 'xp')}", "0|1", "and running it drops the pending partition") - def sqlstate(sql): - try: - with psycopg.connect(dsn, autocommit=True) as c: - c.execute(f"SET search_path TO {schema}") - c.execute(sql).fetchall() - return "no error" - except psycopg.Error as e: - return e.sqlstate - with conn.cursor() as cur: cur.execute("CREATE TABLE xd (ts timestamptz NOT NULL) PARTITION BY RANGE (ts)") cur.execute("CREATE TABLE xd_default PARTITION OF xd DEFAULT USING pgcolumnar") From 78d03ce36b2a8fca8843f3f3942112c4a654c4aa Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 14:32:24 +0000 Subject: [PATCH 08/12] test, docs: the quoting arm requires every planned statement to run; how-to sentences shortened (#1297) A plan with unquoted names failed its DETACH, and the DROP still removed the attached partition (under ACCESS EXCLUSIVE on the parent), so the tables left could not show the bug: the unquoted-names mutation survived. The arm now also requires that no planned statement failed, and is red under that mutation. docs_style's sentence-length rule caught two sentences in the how-to. Co-Authored-By: Claude Opus 5.5 --- docs/how-to.md | 10 +++++----- test/partition_lifecycle.sh | 9 ++++++--- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/docs/how-to.md b/docs/how-to.md index dab27002a..ed4fb1756 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -504,11 +504,11 @@ If a concurrent detach was interrupted, the partition stays pending. The next plan finishes that detach with `DETACH PARTITION ... FINALIZE` instead of starting it again. -The cutoff has the key's type: a `timestamptz` for a `timestamptz` key, as in -the example, and a `timestamp` for a `timestamp` key, such as -`localtimestamp - interval '90 days'`. A cutoff of the other type is refused -(`42804`), because comparing across the two depends on the session's time zone, -and the same call would plan differently from two sessions. +The cutoff must have the key's type. A `timestamptz` key takes a `timestamptz` +cutoff, as in the example. A `timestamp` key takes a `timestamp` cutoff, such +as `localtimestamp - interval '90 days'`. A cutoff of the other type is refused +(`42804`). Comparing across the two types depends on the session's time zone, +so the same call could plan differently from two sessions. A partition that is itself partitioned is planned as one, and the `DROP` takes its children with it. A partition that `tier_partition` moved to Parquet is diff --git a/test/partition_lifecycle.sh b/test/partition_lifecycle.sh index 1b6ea4ca3..686234951 100755 --- a/test/partition_lifecycle.sh +++ b/test/partition_lifecycle.sh @@ -274,10 +274,13 @@ q "CREATE SCHEMA \"Drop Zone\"; FOR VALUES FROM ('2020-01-01 00:00+00') TO ('2020-02-01 00:00+00') USING pgcolumnar; CREATE TABLE \"Drop Zone\".\"New Part\" PARTITION OF \"Drop Zone\".\"Metrics\" FOR VALUES FROM ('2020-02-01 00:00+00') TO ('2020-03-01 00:00+00') USING pgcolumnar;" >/dev/null -run_plan '"Drop Zone"."Metrics"' '2020-02-01 00:00+00' +# Every statement must run: a DETACH that fails on a bad name still leaves the +# DROP to remove an attached partition (under ACCESS EXCLUSIVE on the parent), so +# the tables left alone cannot tell a quoting bug apart. +quoted_failures="$(run_plan '"Drop Zone"."Metrics"' '2020-02-01 00:00+00')" check "a plan for names that need quoting runs as returned" \ - "$(q "SELECT string_agg(c.relname, ',') FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = 'Drop Zone' AND c.relkind IN ('r', 'p');")" "Metrics,New Part" + "${quoted_failures:-every statement ran}|$(q "SELECT string_agg(c.relname, ',' ORDER BY c.relname) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'Drop Zone' AND c.relkind IN ('r', 'p');")" "every statement ran|Metrics,New Part" # A partition that is itself partitioned is listed whole: the concurrent detach # takes it with its children, and DROP drops them. From 6d976d3f59e0ee8be613e6ec5d02f6b317469bf5 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 15:19:19 +0000 Subject: [PATCH 09/12] test: a DROP after a failed DETACH must refuse, red; the second-plan check is an arm (#1331 review) Red on PG17: a DETACH that times out on the parent's lock, then the plan's DROP succeeds and removes the still-attached partition, under ACCESS EXCLUSIVE on the parent. The expectations now spell the guarded drop (gdrop). 'a second plan for the same cutoff is empty' also requires the kept partitions still attached, which makes it behaviour a mutation reaches (#1236). Co-Authored-By: Claude Opus 5.5 --- test/partition_lifecycle.sh | 57 ++++++++++++++++++++++++++++++------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/test/partition_lifecycle.sh b/test/partition_lifecycle.sh index 686234951..4eb32c991 100755 --- a/test/partition_lifecycle.sh +++ b/test/partition_lifecycle.sh @@ -201,6 +201,14 @@ check "and nothing was created for it" \ # the caller runs each as its own top-level statement (\gexec). A partition a # concurrent detach left pending gets DETACH ... FINALIZE instead. plan() { q "SELECT string_agg(s, E'\n') FROM pgcolumnar.partition_drop_plan('$1', '$2') s;"; } +# The drop each partition gets: a DO block that refuses (55000) while the partition +# is still attached, so a DETACH that failed cannot be followed by a DROP that takes +# ACCESS EXCLUSIVE on the parent (jdatcmd, #1331 review). +gdrop() { # gdrop 'TABLE'|'FOREIGN TABLE' QUALIFIED-NAME -> the planned statement + local lit="'${2//\'/\'\'}'" + printf "DO \$pgcolumnar_drop\$ BEGIN IF EXISTS (SELECT 1 FROM pg_catalog.pg_inherits WHERE inhrelid = %s::regclass) THEN RAISE EXCEPTION '%% is still attached to its parent; its DETACH did not finish', %s USING ERRCODE = '55000'; END IF; DROP %s %s; END \$pgcolumnar_drop\$" \ + "$lit" "$lit" "$1" "$2" +} run_plan() { # each statement top-level, as \gexec would run it local line while IFS= read -r line; do @@ -219,10 +227,10 @@ check "premise: the drop fixture has several partitions, each holding rows" \ "$(q "SELECT count(DISTINCT tableoid) FROM dl;")|$(kids dl)" "4|4" check "the plan retires the partitions wholly before the cutoff, in range order" \ "$(plan dl '2020-03-15 00:00+00')" \ - "$(printf '%s\n' 'ALTER TABLE public.dl DETACH PARTITION public.dl_01 CONCURRENTLY' 'DROP TABLE public.dl_01' \ - 'ALTER TABLE public.dl DETACH PARTITION public.dl_02 CONCURRENTLY' 'DROP TABLE public.dl_02')" + "$(printf '%s\n' 'ALTER TABLE public.dl DETACH PARTITION public.dl_01 CONCURRENTLY' "$(gdrop TABLE public.dl_01)" \ + 'ALTER TABLE public.dl DETACH PARTITION public.dl_02 CONCURRENTLY' "$(gdrop TABLE public.dl_02)")" check "a cutoff on a partition's upper bound includes it" \ - "$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2020-03-01 00:00+00') s WHERE s LIKE 'DROP%';")" "2" + "$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2020-03-01 00:00+00') s WHERE s LIKE 'DO %';")" "2" check "a cutoff before every partition plans nothing" \ "$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2019-06-01 00:00+00');")" "0" sids="$(q "SELECT string_agg(pgcolumnar.get_storage_id(c)::text, ',') FROM unnest(ARRAY['dl_01','dl_02']::regclass[]) c;")" @@ -235,8 +243,8 @@ check "and every row the remaining partitions held is still there" \ WHERE '2020-01-01 00:00+00'::timestamptz + g * interval '1 hour' >= '2020-03-01 00:00+00';")" check "and the dropped partitions' pgcolumnar storage rows are gone" \ "$(q "SELECT count(*) FROM pgcolumnar.storage WHERE storage_id IN (${sids:-0});")" "0" -check "premise: a second plan for the same cutoff is empty" \ - "$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2020-03-15 00:00+00');")" "0" +check "a second plan for the same cutoff is empty, and the partitions it kept are still attached" \ + "$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2020-03-15 00:00+00');")|$(kids dl)" "0|2" # The cutoff's type must be the key's (jdatcmd, #1297): a timestamp bound compared # with a timestamptz cutoff converts in the session's zone, so the same call could @@ -251,7 +259,7 @@ plan_in() { # plan_in ZONE PARENT CUTOFF-EXPRESSION } check "a timestamp key's plan for a timestamp cutoff is the same from sessions west and east of UTC" \ "$(plan_in America/Los_Angeles dw "'2021-01-02 00:00'::timestamp")|$(plan_in Asia/Kolkata dw "'2021-01-02 00:00'::timestamp")" \ - "ALTER TABLE public.dw DETACH PARTITION public.dw_a CONCURRENTLY;DROP TABLE public.dw_a|ALTER TABLE public.dw DETACH PARTITION public.dw_a CONCURRENTLY;DROP TABLE public.dw_a" + "ALTER TABLE public.dw DETACH PARTITION public.dw_a CONCURRENTLY;$(gdrop TABLE public.dw_a)|ALTER TABLE public.dw DETACH PARTITION public.dw_a CONCURRENTLY;$(gdrop TABLE public.dw_a)" check "a cutoff of the other timestamp type is refused" \ "$(sqlstate_or "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dw', '2021-01-02 00:00+00'::timestamptz)")|$(sqlstate_or "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dl', '2020-03-01 00:00'::timestamp)")" \ "42804|42804" @@ -263,8 +271,8 @@ q "CREATE TABLE dm (ts timestamptz NOT NULL) PARTITION BY RANGE (ts); CREATE TABLE dm_mid PARTITION OF dm FOR VALUES FROM ('2020-01-01 00:00+00') TO ('2020-02-01 00:00+00') USING pgcolumnar; CREATE TABLE dm_hi PARTITION OF dm FOR VALUES FROM ('2020-02-01 00:00+00') TO (MAXVALUE) USING pgcolumnar;" >/dev/null check "a partition from MINVALUE is listed once its upper bound is past, one to MAXVALUE never" \ - "$(q "SELECT string_agg(s, ';') FROM pgcolumnar.partition_drop_plan('dm', '2020-01-01 00:00+00') s WHERE s LIKE 'DROP%';")|$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dm', 'infinity') s WHERE s LIKE '%dm_hi%';")" \ - "DROP TABLE public.dm_lo|0" + "$(q "SELECT string_agg(s, ';') FROM pgcolumnar.partition_drop_plan('dm', '2020-01-01 00:00+00') s WHERE s LIKE 'DO %';")|$(q "SELECT count(*) FROM pgcolumnar.partition_drop_plan('dm', 'infinity') s WHERE s LIKE '%dm_hi%';")" \ + "$(gdrop TABLE public.dm_lo)|0" # Names that need quoting, in a schema that needs quoting: the plan must run as # returned (#1317 was caught by exactly this). @@ -292,7 +300,7 @@ q "CREATE TABLE ds (ts timestamptz NOT NULL) PARTITION BY RANGE (ts); INSERT INTO ds SELECT '2020-01-01 00:00+00'::timestamptz + g * interval '1 day' FROM generate_series(0, 180) g;" >/dev/null check "a partition that is itself partitioned is planned whole" \ "$(plan ds '2020-04-01 00:00+00' | paste -sd';')" \ - "ALTER TABLE public.ds DETACH PARTITION public.ds_q1 CONCURRENTLY;DROP TABLE public.ds_q1" + "ALTER TABLE public.ds DETACH PARTITION public.ds_q1 CONCURRENTLY;$(gdrop TABLE public.ds_q1)" run_plan ds '2020-04-01 00:00+00' check "and running the plan drops it with its children" \ "$(q "SELECT count(*) FROM pg_class WHERE relname IN ('ds_q1', 'ds_q1_a', 'ds_q1_b');")|$(q "SELECT count(*) FROM ds;")" \ @@ -310,7 +318,7 @@ check "premise: the January partition was tiered to Parquet" \ "$(q "SELECT pgcolumnar.tier_partition('dt_jan', 'drop_pq', '$PGC_WORKDIR/drop/dt_jan.parquet');")|$(q "SELECT relkind FROM pg_class WHERE relname = 'dt_jan';")" "744|f" check "a tiered partition is planned with DROP FOREIGN TABLE" \ "$(plan dt '2020-02-01 00:00+00' | paste -sd';')" \ - "ALTER TABLE public.dt DETACH PARTITION public.dt_jan CONCURRENTLY;DROP FOREIGN TABLE public.dt_jan" + "ALTER TABLE public.dt DETACH PARTITION public.dt_jan CONCURRENTLY;$(gdrop 'FOREIGN TABLE' public.dt_jan)" run_plan dt '2020-02-01 00:00+00' check "and running it drops the foreign table and leaves its Parquet file" \ "$(q "SELECT count(*) FROM pg_class WHERE relname = 'dt_jan';")|$([ -s "$PGC_WORKDIR/drop/dt_jan.parquet" ] && echo file-kept || echo file-gone)" "0|file-kept" @@ -332,11 +340,38 @@ check "premise: the interrupted detach left the partition pending" \ "$(q "SELECT inhdetachpending FROM pg_inherits WHERE inhrelid = 'dp_1'::regclass;")" "t" check "a pending detach is finalised, not started again" \ "$(q "SELECT string_agg(s, ';') FROM pgcolumnar.partition_drop_plan('dp', '2020-02-01 00:00+00') s;")" \ - "ALTER TABLE public.dp DETACH PARTITION public.dp_1 FINALIZE;DROP TABLE public.dp_1" + "ALTER TABLE public.dp DETACH PARTITION public.dp_1 FINALIZE;$(gdrop TABLE public.dp_1)" run_plan dp '2020-02-01 00:00+00' check "and running it drops the pending partition" \ "$(q "SELECT count(*) FROM pg_class WHERE relname = 'dp_1';")|$(kids dp)" "0|1" +# A DETACH that fails -- here a lock_timeout while another session holds the +# parent -- must not be followed by a DROP of the still-attached partition, which +# would take ACCESS EXCLUSIVE on the parent and block every query on it. \gexec +# carries on after an error unless ON_ERROR_STOP is set, so the plan itself has +# to be safe. Each statement's SQLSTATE, in order. +q "CREATE TABLE df (ts timestamptz NOT NULL) PARTITION BY RANGE (ts); + CREATE TABLE df_1 PARTITION OF df FOR VALUES FROM ('2020-01-01 00:00+00') TO ('2020-02-01 00:00+00') USING pgcolumnar; + CREATE TABLE df_2 PARTITION OF df FOR VALUES FROM ('2020-02-01 00:00+00') TO ('2020-03-01 00:00+00') USING pgcolumnar;" >/dev/null +env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -q \ + -c "BEGIN" -c "LOCK TABLE df IN SHARE UPDATE EXCLUSIVE MODE" -c "SELECT pg_sleep(4)" -c "COMMIT" >/dev/null 2>&1 & +holder=$! +sleep 1 +stmt_state() { # stmt_state SQL [PGOPTIONS] -> ok, or the SQLSTATE it failed with + local out + out="$(env PGOPTIONS="${2:-}" PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -X -q -v ON_ERROR_STOP=1 -v VERBOSITY=sqlstate -c "$1" 2>&1 >/dev/null)" + [ -z "$out" ] && echo ok || sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p' <<<"$out" | head -1 +} +mapfile -t df_plan <<<"$(plan df '2020-02-01 00:00+00')" +# The DETACH times out while the holder has the parent; the holder then lets go, +# so the DROP that follows finds nothing in its way but its own guard. +states="$(stmt_state "${df_plan[0]}" '-c lock_timeout=1500')" +wait "$holder" +states="$states;$(stmt_state "${df_plan[1]}")" +check "after a DETACH that fails, the plan's DROP refuses and the partition stays attached" \ + "$states|$(kids df)|$(q "SELECT count(*) FROM pg_inherits WHERE inhrelid = 'df_1'::regclass;")" "55P03;55000|2|1" + # Refusals, as partition_create_ahead refuses. check "a drop plan for a parent with a DEFAULT partition is refused" \ "$(sqlstate_or "SELECT count(*) FROM pgcolumnar.partition_drop_plan('td', now())")" "55000" From 9494d7c7e5732c61a0912e83dc6ea515e903b066 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 15:20:05 +0000 Subject: [PATCH 10/12] fix: partition_drop_plan's DROP refuses while the partition is still attached (#1331 review) Each drop is now a DO block that raises 55000 while the partition is in pg_inherits, so a DETACH that failed (a lock_timeout, a cancelled concurrent wait, a permission error) cannot be followed by a DROP that takes ACCESS EXCLUSIVE on the parent: \gexec carries on after an error. A pending partition is refused too; the next plan finalises it. partition_lifecycle.sh 50/50 on PG17; identical patches in both scripts. Co-Authored-By: Claude Opus 5.5 --- pgcolumnar--1.0-alpha4--1.0-alpha5.sql | 16 ++++++++++++++-- pgcolumnar--1.0-alpha5.sql | 16 ++++++++++++++-- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql index 7c6e3fd93..da7c58dd7 100644 --- a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql @@ -987,6 +987,14 @@ $create_ahead$; * plan differently from two sessions. Two overloads, each refusing the other * type of key (42804); an untyped literal resolves to timestamptz. * + * THE DROP REFUSES WHILE THE PARTITION IS ATTACHED (jdatcmd, #1331 review). Any + * DETACH can fail -- a lock_timeout, a cancelled concurrent wait, a permission + * error -- and \gexec carries on unless ON_ERROR_STOP is set. A plain DROP of an + * attached partition would then take ACCESS EXCLUSIVE on the parent. So each + * drop is a DO block that raises 55000 while the partition is still in + * pg_inherits, a pending one included; the next plan's FINALIZE picks that up. + * A DROP may run inside DO; only DETACH ... CONCURRENTLY must be top-level. + * * The same refusals as partition_create_ahead, with the same codes: a DEFAULT * partition rules out the concurrent detach, and is refused rather than dropped * with the exclusive one. No ownership check: this reads catalogs the caller @@ -1060,7 +1068,9 @@ BEGIN LOOP RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s %s', parent, p.part, CASE WHEN p.pending THEN 'FINALIZE' ELSE 'CONCURRENTLY' END); - RETURN NEXT format('DROP %s %s', CASE WHEN p.foreign_table THEN 'FOREIGN TABLE' ELSE 'TABLE' END, p.part); + RETURN NEXT format('DO $pgcolumnar_drop$ BEGIN IF EXISTS (SELECT 1 FROM pg_catalog.pg_inherits WHERE inhrelid = %L::regclass) THEN RAISE EXCEPTION ''%% is still attached to its parent; its DETACH did not finish'', %L USING ERRCODE = ''55000''; END IF; DROP %s %s; END $pgcolumnar_drop$', + p.part::text, p.part::text, + CASE WHEN p.foreign_table THEN 'FOREIGN TABLE' ELSE 'TABLE' END, p.part); END LOOP; END $drop_plan_tz$; @@ -1083,7 +1093,9 @@ BEGIN LOOP RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s %s', parent, p.part, CASE WHEN p.pending THEN 'FINALIZE' ELSE 'CONCURRENTLY' END); - RETURN NEXT format('DROP %s %s', CASE WHEN p.foreign_table THEN 'FOREIGN TABLE' ELSE 'TABLE' END, p.part); + RETURN NEXT format('DO $pgcolumnar_drop$ BEGIN IF EXISTS (SELECT 1 FROM pg_catalog.pg_inherits WHERE inhrelid = %L::regclass) THEN RAISE EXCEPTION ''%% is still attached to its parent; its DETACH did not finish'', %L USING ERRCODE = ''55000''; END IF; DROP %s %s; END $pgcolumnar_drop$', + p.part::text, p.part::text, + CASE WHEN p.foreign_table THEN 'FOREIGN TABLE' ELSE 'TABLE' END, p.part); END LOOP; END $drop_plan_ts$; diff --git a/pgcolumnar--1.0-alpha5.sql b/pgcolumnar--1.0-alpha5.sql index fc1c150c7..2d7b0b333 100644 --- a/pgcolumnar--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha5.sql @@ -2231,6 +2231,14 @@ $create_ahead$; * plan differently from two sessions. Two overloads, each refusing the other * type of key (42804); an untyped literal resolves to timestamptz. * + * THE DROP REFUSES WHILE THE PARTITION IS ATTACHED (jdatcmd, #1331 review). Any + * DETACH can fail -- a lock_timeout, a cancelled concurrent wait, a permission + * error -- and \gexec carries on unless ON_ERROR_STOP is set. A plain DROP of an + * attached partition would then take ACCESS EXCLUSIVE on the parent. So each + * drop is a DO block that raises 55000 while the partition is still in + * pg_inherits, a pending one included; the next plan's FINALIZE picks that up. + * A DROP may run inside DO; only DETACH ... CONCURRENTLY must be top-level. + * * The same refusals as partition_create_ahead, with the same codes: a DEFAULT * partition rules out the concurrent detach, and is refused rather than dropped * with the exclusive one. No ownership check: this reads catalogs the caller @@ -2304,7 +2312,9 @@ BEGIN LOOP RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s %s', parent, p.part, CASE WHEN p.pending THEN 'FINALIZE' ELSE 'CONCURRENTLY' END); - RETURN NEXT format('DROP %s %s', CASE WHEN p.foreign_table THEN 'FOREIGN TABLE' ELSE 'TABLE' END, p.part); + RETURN NEXT format('DO $pgcolumnar_drop$ BEGIN IF EXISTS (SELECT 1 FROM pg_catalog.pg_inherits WHERE inhrelid = %L::regclass) THEN RAISE EXCEPTION ''%% is still attached to its parent; its DETACH did not finish'', %L USING ERRCODE = ''55000''; END IF; DROP %s %s; END $pgcolumnar_drop$', + p.part::text, p.part::text, + CASE WHEN p.foreign_table THEN 'FOREIGN TABLE' ELSE 'TABLE' END, p.part); END LOOP; END $drop_plan_tz$; @@ -2327,7 +2337,9 @@ BEGIN LOOP RETURN NEXT format('ALTER TABLE %s DETACH PARTITION %s %s', parent, p.part, CASE WHEN p.pending THEN 'FINALIZE' ELSE 'CONCURRENTLY' END); - RETURN NEXT format('DROP %s %s', CASE WHEN p.foreign_table THEN 'FOREIGN TABLE' ELSE 'TABLE' END, p.part); + RETURN NEXT format('DO $pgcolumnar_drop$ BEGIN IF EXISTS (SELECT 1 FROM pg_catalog.pg_inherits WHERE inhrelid = %L::regclass) THEN RAISE EXCEPTION ''%% is still attached to its parent; its DETACH did not finish'', %L USING ERRCODE = ''55000''; END IF; DROP %s %s; END $pgcolumnar_drop$', + p.part::text, p.part::text, + CASE WHEN p.foreign_table THEN 'FOREIGN TABLE' ELSE 'TABLE' END, p.part); END LOOP; END $drop_plan_ts$; From 814ca7e27d74651ec0a7cb168a02b3de5fcccb4b Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 15:21:18 +0000 Subject: [PATCH 11/12] test, docs: the twin's guarded drop and failed-DETACH arm; ON_ERROR_STOP and the drop's refusal documented (#1331 review) test_partition_lifecycle.py matches the shell suite's 50 checks by name. The how-to's example sets ON_ERROR_STOP and says what the drop's refusal guards. Both twins green on PG17. Co-Authored-By: Claude Opus 5.5 --- docs/how-to.md | 6 +++ docs/sql-reference.md | 7 ++- test/pytest/test_partition_lifecycle.py | 59 +++++++++++++++++++++---- 3 files changed, 61 insertions(+), 11 deletions(-) diff --git a/docs/how-to.md b/docs/how-to.md index ed4fb1756..dfcea6e1c 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -490,6 +490,7 @@ partition whose whole range is older than a cutoff. It does not run them, becaus its own statement, for example with psql's `\gexec`: ```sql +\set ON_ERROR_STOP on SELECT * FROM pgcolumnar.partition_drop_plan('metrics', now() - interval '90 days') \gexec ``` @@ -504,6 +505,11 @@ If a concurrent detach was interrupted, the partition stays pending. The next plan finishes that detach with `DETACH PARTITION ... FINALIZE` instead of starting it again. +The drop refuses while its partition is still attached (`55000`). So if a detach +fails, for example on a lock timeout, the drop after it does not lock the parent +or remove the partition. A later plan retries it. `ON_ERROR_STOP` stops the run +at the first failure, which also keeps the plan's other statements from running. + The cutoff must have the key's type. A `timestamptz` key takes a `timestamptz` cutoff, as in the example. A `timestamp` key takes a `timestamp` cutoff, such as `localtimestamp - interval '90 days'`. A cutoff of the other type is refused diff --git a/docs/sql-reference.md b/docs/sql-reference.md index f78d06406..085275157 100644 --- a/docs/sql-reference.md +++ b/docs/sql-reference.md @@ -626,8 +626,11 @@ refused before it creates any (`22023`). The function takes Returns the statements that retire every partition of `parent` whose upper bound is at or before `cutoff`, in range order. Each partition gets -`ALTER TABLE ... DETACH PARTITION ... CONCURRENTLY`, then `DROP TABLE`, or `DROP FOREIGN TABLE` for a partition `tier_partition` moved to -Parquet (the file stays). A partition that an interrupted concurrent detach +`ALTER TABLE ... DETACH PARTITION ... CONCURRENTLY`, then a `DO` block that runs +`DROP TABLE`, or `DROP FOREIGN TABLE` for a partition `tier_partition` moved to +Parquet (the file stays). The block refuses (`55000`) while the partition is +still attached, so a failed detach is never followed by a drop that locks the +parent. A partition that an interrupted concurrent detach left pending gets `DETACH PARTITION ... FINALIZE` instead. A partition that is itself partitioned is listed whole. The names are schema-qualified. The function runs nothing: run each statement at top level, for example with diff --git a/test/pytest/test_partition_lifecycle.py b/test/pytest/test_partition_lifecycle.py index ff022db17..d08503bcd 100644 --- a/test/pytest/test_partition_lifecycle.py +++ b/test/pytest/test_partition_lifecycle.py @@ -260,6 +260,28 @@ def run(statements): def q(name): return f"{schema}.{name}" + def gdrop(kind, name): + """The drop the plan gives a partition: a DO block that refuses (55000) + while it is still attached, so a failed DETACH is never followed by a DROP + that locks the parent (#1331 review).""" + lit = "'" + name.replace("'", "''") + "'" + return ("DO $pgcolumnar_drop$ BEGIN IF EXISTS (SELECT 1 FROM pg_catalog.pg_inherits" + f" WHERE inhrelid = {lit}::regclass) THEN RAISE EXCEPTION '% is still attached to its parent;" + f" its DETACH did not finish', {lit} USING ERRCODE = '55000'; END IF; DROP {kind} {name};" + " END $pgcolumnar_drop$") + + def sqlstate_with(sql, setup=None): + """'ok', or the SQLSTATE the statement failed with, in its own session.""" + try: + with psycopg.connect(dsn, autocommit=True) as c: + c.execute(f"SET search_path TO {schema}") + if setup: + c.execute(setup) + c.execute(sql) + return "ok" + except psycopg.Error as e: + return e.sqlstate + def sqlstate(sql): try: with psycopg.connect(dsn, autocommit=True) as c: @@ -284,10 +306,10 @@ def sqlstate(sql): want = [] for i in range(3): - want += [f"ALTER TABLE {q('xl')} DETACH PARTITION {q(f'xl_w{i}')} CONCURRENTLY", f"DROP TABLE {q(f'xl_w{i}')}"] + want += [f"ALTER TABLE {q('xl')} DETACH PARTITION {q(f'xl_w{i}')} CONCURRENTLY", gdrop("TABLE", q(f"xl_w{i}"))] expect.text("\n".join(plan("xl", "2019-03-28 12:00+00")), "\n".join(want), "the plan retires the partitions wholly before the cutoff, in range order") - expect.num(sum(1 for s in plan("xl", "2019-03-18 00:00+00") if s.startswith("DROP")), 2, + expect.num(sum(1 for s in plan("xl", "2019-03-18 00:00+00") if s.startswith("DO ")), 2, "a cutoff on a partition's upper bound includes it") expect.num(len(plan("xl", "2018-12-31 00:00+00")), 0, "a cutoff before every partition plans nothing") @@ -305,7 +327,8 @@ def sqlstate(sql): expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.storage WHERE storage_id = ANY (%s)" % ( "ARRAY[" + ",".join(str(s) for s in sids) + "]::bigint[]")), 0, "and the dropped partitions' pgcolumnar storage rows are gone") - expect.num(len(plan("xl", "2019-03-28 12:00+00")), 0, "premise: a second plan for the same cutoff is empty") + expect.text(f"{len(plan('xl', '2019-03-28 12:00+00'))}|{_kids(conn, 'xl')}", "0|2", + "a second plan for the same cutoff is empty, and the partitions it kept are still attached") # The cutoff's type must be the key's: a timestamp key with a timestamp cutoff # plans the same in every session, and a cutoff of the other type is refused. @@ -320,7 +343,7 @@ def sqlstate(sql): c.execute(f"SET TimeZone = '{zone}'") got.append(c.execute("SELECT coalesce(string_agg(s, ';'), 'none')" " FROM pgcolumnar.partition_drop_plan('xw', '2019-05-01 06:00'::timestamp) s").fetchone()[0]) - one = f"ALTER TABLE {q('xw')} DETACH PARTITION {q('xw_a')} CONCURRENTLY;DROP TABLE {q('xw_a')}" + one = f"ALTER TABLE {q('xw')} DETACH PARTITION {q('xw_a')} CONCURRENTLY;" + gdrop("TABLE", q("xw_a")) expect.text("|".join(got), f"{one}|{one}", "a timestamp key's plan for a timestamp cutoff is the same from sessions west and east of UTC") expect.text(sqlstate("SELECT * FROM pgcolumnar.partition_drop_plan('xw', '2019-05-01 06:00+00'::timestamptz)") + "|" @@ -332,9 +355,9 @@ def sqlstate(sql): cur.execute("CREATE TABLE xm (ts timestamptz NOT NULL) PARTITION BY RANGE (ts)") cur.execute("CREATE TABLE xm_lo PARTITION OF xm FOR VALUES FROM (MINVALUE) TO ('2019-07-01 00:00+00') USING pgcolumnar") cur.execute("CREATE TABLE xm_hi PARTITION OF xm FOR VALUES FROM ('2019-07-01 00:00+00') TO (MAXVALUE) USING pgcolumnar") - lo = [s for s in plan("xm", "2019-07-01 00:00+00") if s.startswith("DROP")] + lo = [s for s in plan("xm", "2019-07-01 00:00+00") if s.startswith("DO ")] hi = [s for s in plan("xm", "infinity") if "xm_hi" in s] - expect.text(f"{';'.join(lo)}|{len(hi)}", f"DROP TABLE {q('xm_lo')}|0", + expect.text(f"{';'.join(lo)}|{len(hi)}", gdrop("TABLE", q("xm_lo")) + "|0", "a partition from MINVALUE is listed once its upper bound is past, one to MAXVALUE never") # Names that need quoting, in a schema that needs quoting. @@ -366,7 +389,7 @@ def sqlstate(sql): " USING pgcolumnar") cur.execute("INSERT INTO xs SELECT '2019-01-01 00:00+00'::timestamptz + g * interval '1 day' FROM generate_series(0, 364) g") expect.text(";".join(plan("xs", "2019-07-01 00:00+00")), - f"ALTER TABLE {q('xs')} DETACH PARTITION {q('xs_h1')} CONCURRENTLY;DROP TABLE {q('xs_h1')}", + f"ALTER TABLE {q('xs')} DETACH PARTITION {q('xs_h1')} CONCURRENTLY;" + gdrop("TABLE", q("xs_h1")), "a partition that is itself partitioned is planned whole") run(plan("xs", "2019-07-01 00:00+00")) gone = _one(conn, "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace" @@ -390,7 +413,7 @@ def sqlstate(sql): expect.text(f"{moved}|{kind}", "372|f", "premise: the January partition was tiered to Parquet") tiered = plan("xt", "2019-02-01 00:00+00") expect.text(";".join(tiered), - f"ALTER TABLE {q('xt')} DETACH PARTITION {q('xt_jan')} CONCURRENTLY;DROP FOREIGN TABLE {q('xt_jan')}", + f"ALTER TABLE {q('xt')} DETACH PARTITION {q('xt_jan')} CONCURRENTLY;" + gdrop("FOREIGN TABLE", q("xt_jan")), "a tiered partition is planned with DROP FOREIGN TABLE") run(tiered) left = _one(conn, "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace" @@ -422,13 +445,31 @@ def sqlstate(sql): "True", "premise: the interrupted detach left the partition pending") pending_plan = plan("xp", "2019-06-08 00:00+00") expect.text(";".join(pending_plan), - f"ALTER TABLE {q('xp')} DETACH PARTITION {q('xp_1')} FINALIZE;DROP TABLE {q('xp_1')}", + f"ALTER TABLE {q('xp')} DETACH PARTITION {q('xp_1')} FINALIZE;" + gdrop("TABLE", q("xp_1")), "a pending detach is finalised, not started again") run(pending_plan) gone = _one(conn, "SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace" f" WHERE n.nspname = '{schema}' AND c.relname = 'xp_1'") expect.text(f"{gone}|{_kids(conn, 'xp')}", "0|1", "and running it drops the pending partition") + # A DETACH that fails on the parent's lock, and then the rest of the plan once + # the lock is gone: the DROP must refuse, and the partition stay attached. + with conn.cursor() as cur: + cur.execute("CREATE TABLE xf (ts timestamptz NOT NULL) PARTITION BY RANGE (ts)") + cur.execute("CREATE TABLE xf_1 PARTITION OF xf FOR VALUES FROM ('2019-09-01 00:00+00') TO ('2019-09-08 00:00+00') USING pgcolumnar") + cur.execute("CREATE TABLE xf_2 PARTITION OF xf FOR VALUES FROM ('2019-09-08 00:00+00') TO ('2019-09-15 00:00+00') USING pgcolumnar") + detach, drop = plan("xf", "2019-09-08 00:00+00") + states = [] + with psycopg.connect(dsn) as holder: + holder.execute(f"SET search_path TO {schema}") + holder.execute("LOCK TABLE xf IN SHARE UPDATE EXCLUSIVE MODE") + states.append(sqlstate_with(detach, "SET lock_timeout = '1500ms'")) + holder.rollback() + states.append(sqlstate_with(drop)) + attached = _one(conn, "SELECT count(*) FROM pg_inherits WHERE inhrelid = 'xf_1'::regclass") + expect.text(f"{';'.join(states)}|{_kids(conn, 'xf')}|{attached}", "55P03;55000|2|1", + "after a DETACH that fails, the plan's DROP refuses and the partition stays attached") + with conn.cursor() as cur: cur.execute("CREATE TABLE xd (ts timestamptz NOT NULL) PARTITION BY RANGE (ts)") cur.execute("CREATE TABLE xd_default PARTITION OF xd DEFAULT USING pgcolumnar") From d7effdc82ec82852d3de73ec940cd941ea5693c9 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 15:33:40 +0000 Subject: [PATCH 12/12] test: partition_drop_plan's 23 checks in the ledger, 20 dated by seventeen mutations (#1297) partition_lifecycle.sh 50/50 and test_partition_lifecycle.py on each of PG15-19, main at b55cb23. Every never row of the suite is a premise. Ledger 2309 / 1804, gate rc=0. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 23 +++++++++++++++++++++++ test/check_ledger_budget.txt | 21 ++++++++++++++++++++- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index b19f8618f..53ea84c43 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -1855,19 +1855,37 @@ parallel_scan_cost parallel_scan_cost premise: the serial plan is a columnar sca parallel_scan_cost parallel_scan_cost premise: the table holds every inserted row 15;16;17;18;19 never - parallel_scan_cost parallel_scan_cost the leader-participation branch changes the divisor 15;16;17;18;19 never - partition_lifecycle partition_lifecycle a call that would create more than 1000 partitions is refused before any DDL 15;16;17;18;19 2026-10-12 partition_create_ahead without the 1000-partition bound +partition_lifecycle partition_lifecycle a cutoff before every partition plans nothing 15;16;17;18;19 2026-10-13 the cutoff filter removed: every partition planned +partition_lifecycle partition_lifecycle a cutoff of the other timestamp type is refused 15;16;17;18;19 2026-10-13 a cutoff of the other timestamp type planned instead of refused +partition_lifecycle partition_lifecycle a cutoff on a partition's upper bound includes it 15;16;17;18;19 2026-10-13 a partition whose upper bound equals the cutoff left out;a plain DROP, not refusing while the partition is still attached;the cutoff filter removed: every partition planned;the lower bound read where the upper is meant;the plan detaches but drops nothing +partition_lifecycle partition_lifecycle a drop plan for a parent not partitioned on a timestamp is refused 15;16;17;18;19 2026-10-13 a parent partitioned on a non-timestamp key planned instead of refused +partition_lifecycle partition_lifecycle a drop plan for a parent with a DEFAULT partition is refused 15;16;17;18;19 2026-10-13 a parent with a DEFAULT partition planned instead of refused partition_lifecycle partition_lifecycle a parent with a 63-character name gets partitions whose names keep their dates 15;16;17;18;19 2026-10-12 partition_create_ahead stopping one width short of now plus the lead;partition_create_ahead without shortening a long parent name partition_lifecycle partition_lifecycle a parent with a DEFAULT partition is refused 15;16;17;18;19 2026-10-12 partition_create_ahead without the DEFAULT-partition refusal partition_lifecycle partition_lifecycle a parent with a generated column gets its partitions, and the column stays generated 15;16;17;18;19 2026-10-12 partition_create_ahead stopping one width short of now plus the lead;partition_create_ahead without INCLUDING GENERATED partition_lifecycle partition_lifecycle a parent with a partition bounded by MAXVALUE is refused, and nothing is created 15;16;17;18;19 2026-10-12 partition_create_ahead without the MAXVALUE refusal +partition_lifecycle partition_lifecycle a partition from MINVALUE is listed once its upper bound is past, one to MAXVALUE never 15;16;17;18;19 2026-10-13 a partition planned only when it has a lower bound, so never one from MINVALUE;a partition whose upper bound equals the cutoff left out;a plain DROP, not refusing while the partition is still attached;the cutoff filter removed: every partition planned;the lower bound read where the upper is meant;the plan detaches but drops nothing +partition_lifecycle partition_lifecycle a partition that is itself partitioned is planned whole 15;16;17;18;19 2026-10-13 a partition that is itself partitioned left out of the plan;a partition whose upper bound equals the cutoff left out;a plain DROP, not refusing while the partition is still attached;the cutoff filter removed: every partition planned;the lower bound read where the upper is meant;the plan detaches but drops nothing;the plan detaches without CONCURRENTLY +partition_lifecycle partition_lifecycle a pending detach is finalised, not started again 15;16;17;18;19 2026-10-13 a partition whose upper bound equals the cutoff left out;a pending detach planned as a new concurrent detach, not FINALIZE;a plain DROP, not refusing while the partition is still attached;the cutoff filter removed: every partition planned;the lower bound read where the upper is meant;the plan detaches but drops nothing +partition_lifecycle partition_lifecycle a plan for names that need quoting runs as returned 15;16;17;18;19 2026-10-13 a partition whose upper bound equals the cutoff left out;names in the plan with their quotes removed;the cutoff filter removed: every partition planned;the lower bound read where the upper is meant;the plan detaches but drops nothing partition_lifecycle partition_lifecycle a second call creates nothing 15;16;17;18;19 2026-10-12 partition_create_ahead starting from now, not the highest upper bound +partition_lifecycle partition_lifecycle a second plan for the same cutoff is empty, and the partitions it kept are still attached 15;16;17;18;19 2026-10-13 the cutoff filter removed: every partition planned;the lower bound read where the upper is meant +partition_lifecycle partition_lifecycle a tiered partition is planned with DROP FOREIGN TABLE 15;16;17;18;19 2026-10-13 a partition whose upper bound equals the cutoff left out;a plain DROP, not refusing while the partition is still attached;a tiered partition planned with DROP TABLE;the cutoff filter removed: every partition planned;the lower bound read where the upper is meant;the plan detaches but drops nothing;the plan detaches without CONCURRENTLY +partition_lifecycle partition_lifecycle a timestamp key's plan for a timestamp cutoff is the same from sessions west and east of UTC 15;16;17;18;19 2026-10-13 a plain DROP, not refusing while the partition is still attached;the lower bound read where the upper is meant;the plan detaches but drops nothing;the plan detaches without CONCURRENTLY;the timestamp overload compares the bound read as UTC with the cutoff converted in the session's zone partition_lifecycle partition_lifecycle a timestamp parent made east of UTC covers the local now plus the lead 15;16;17;18;19 2026-10-12 partition_create_ahead stopping one width short of now plus the lead partition_lifecycle partition_lifecycle a timestamp parent's partitions are contiguous across a non-UTC session 15;16;17;18;19 2026-10-12 partition_create_ahead leaving a minute between partitions;partition_create_ahead reading a timestamp key's bounds in the session's zone and writing them from UTC (the prototype);partition_create_ahead starting from now, not the highest upper bound partition_lifecycle partition_lifecycle a timestamptz parent's partitions are contiguous across a non-UTC session 15;16;17;18;19 2026-10-12 partition_create_ahead leaving a minute between partitions;partition_create_ahead starting from now, not the highest upper bound +partition_lifecycle partition_lifecycle after a DETACH that fails, the plan's DROP refuses and the partition stays attached 15;16;17;18;19 2026-10-13 a partition whose upper bound equals the cutoff left out;a plain DROP, not refusing while the partition is still attached;the plan detaches but drops nothing partition_lifecycle partition_lifecycle and a row at every boundary has exactly one partition to go to (timestamp) 15;16;17;18;19 2026-10-12 partition_create_ahead leaving a minute between partitions;partition_create_ahead reading a timestamp key's bounds in the session's zone and writing them from UTC (the prototype);partition_create_ahead starting from now, not the highest upper bound partition_lifecycle partition_lifecycle and a row at every boundary has exactly one partition to go to (timestamptz) 15;16;17;18;19 2026-10-12 partition_create_ahead leaving a minute between partitions;partition_create_ahead starting from now, not the highest upper bound +partition_lifecycle partition_lifecycle and every row the remaining partitions held is still there 15;16;17;18;19 2026-10-13 the cutoff filter removed: every partition planned;the lower bound read where the upper is meant partition_lifecycle partition_lifecycle and its partitions start on the width's boundary in the key's own wall clock 15;16;17;18;19 2026-10-12 partition_create_ahead leaving a minute between partitions;partition_create_ahead reading a timestamp key's bounds in the session's zone and writing them from UTC (the prototype) partition_lifecycle partition_lifecycle and land in columnar partitions 15;16;17;18;19 2026-10-12 partition_create_ahead creating heap partitions;partition_create_ahead starting from now, not the highest upper bound;partition_create_ahead stopping one width short of now plus the lead partition_lifecycle partition_lifecycle and nothing was created for it 15;16;17;18;19 2026-10-12 partition_create_ahead without the DEFAULT-partition refusal +partition_lifecycle partition_lifecycle and running it drops the foreign table and leaves its Parquet file 15;16;17;18;19 2026-10-13 a partition whose upper bound equals the cutoff left out;a tiered partition planned with DROP TABLE;the plan detaches but drops nothing +partition_lifecycle partition_lifecycle and running it drops the pending partition 15;16;17;18;19 2026-10-13 a partition whose upper bound equals the cutoff left out;a pending detach planned as a new concurrent detach, not FINALIZE;the cutoff filter removed: every partition planned;the lower bound read where the upper is meant;the plan detaches but drops nothing +partition_lifecycle partition_lifecycle and running the plan drops it with its children 15;16;17;18;19 2026-10-13 a partition that is itself partitioned left out of the plan;a partition whose upper bound equals the cutoff left out;the cutoff filter removed: every partition planned;the lower bound read where the upper is meant;the plan detaches but drops nothing +partition_lifecycle partition_lifecycle and the dropped partitions' pgcolumnar storage rows are gone 15;16;17;18;19 2026-10-13 the drop hook leaves a dropped table's pgcolumnar storage behind;the plan detaches but drops nothing partition_lifecycle partition_lifecycle and they cover now() plus the lead 15;16;17;18;19 2026-10-12 partition_create_ahead stopping one width short of now plus the lead partition_lifecycle partition_lifecycle asking further ahead creates only the difference 15;16;17;18;19 2026-10-12 partition_create_ahead starting from now, not the highest upper bound partition_lifecycle partition_lifecycle every partition it creates is columnar 15;16;17;18;19 2026-10-12 partition_create_ahead creating heap partitions @@ -1876,10 +1894,15 @@ partition_lifecycle partition_lifecycle it never takes AccessExclusiveLock on th partition_lifecycle partition_lifecycle premise: a timestamp parent continued from a hand-made partition, in a session west of UTC 15;16;17;18;19 never - partition_lifecycle partition_lifecycle premise: a timestamp parent was given partitions from a session east of UTC 15;16;17;18;19 never - partition_lifecycle partition_lifecycle premise: a timestamptz parent continued from a hand-made partition, in a session west of UTC 15;16;17;18;19 never - +partition_lifecycle partition_lifecycle premise: the January partition was tiered to Parquet 15;16;17;18;19 never - partition_lifecycle partition_lifecycle premise: the call inside the transaction created partitions 15;16;17;18;19 never - +partition_lifecycle partition_lifecycle premise: the drop fixture has several partitions, each holding rows 15;16;17;18;19 never - partition_lifecycle partition_lifecycle premise: the function exists 15;16;17;18;19 never - +partition_lifecycle partition_lifecycle premise: the interrupted detach left the partition pending 15;16;17;18;19 never - partition_lifecycle partition_lifecycle premise: the parent now has the partitions it reported 15;16;17;18;19 never - partition_lifecycle partition_lifecycle rows within the covered range go in 15;16;17;18;19 2026-10-12 partition_create_ahead starting from now, not the highest upper bound;partition_create_ahead stopping one width short of now plus the lead +partition_lifecycle partition_lifecycle running the plan leaves only the partitions it did not name 15;16;17;18;19 2026-10-13 the cutoff filter removed: every partition planned;the lower bound read where the upper is meant;the plan detaches but drops nothing +partition_lifecycle partition_lifecycle the plan retires the partitions wholly before the cutoff, in range order 15;16;17;18;19 2026-10-13 a plain DROP, not refusing while the partition is still attached;the cutoff filter removed: every partition planned;the lower bound read where the upper is meant;the plan detaches but drops nothing;the plan detaches without CONCURRENTLY;the plan in catalog order, not range order partition_lifecycle partition_lifecycle they are contiguous, each one width wide 15;16;17;18;19 2026-10-12 partition_create_ahead leaving a minute between partitions partman_columnar partman_columnar and no row-group row behind 15;16;17;18;19 2026-10-12 the drop hook stops deleting row_group rows, so they outlive their deleted storage row partman_columnar partman_columnar and read back as heap does 15;16;17;18;19 2026-10-12 a sequential scan drops the last row group from its list, so every one-group partition reads empty diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index e474aa664..3649f2b7a 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1606,4 +1606,23 @@ suites_not_covered 246 # never 1800 + 1 = 1801 # dated 484 + 1 = 485 # 1801 + 485 == 2286 -checks_never_observed_red 1801 +# +# #1297, the drop half: partition_drop_plan (OffgridwithJD): 1801 -> 1804. +# partition_lifecycle, already covered, gains 23 checks from a five-major run +# (partition_lifecycle.sh 50/50 on each of 15-19). Twenty arms are dated by +# seventeen mutations, each merged one log at a time with --mutation and +# --target: a pending detach not finalised, the cutoff made strict, the cutoff +# type check removed, the timestamp overload comparing across zones, the DEFAULT +# and key-type refusals removed, catalog order, no DROP, a DROP that does not +# refuse while the partition is attached (#1331 review), a plain DETACH, no +# cutoff filter, the lower bound read for the upper, MINVALUE left out, a tiered +# partition dropped as a table, names unquoted, sub-partitioned partitions left +# out, and the drop hook leaving storage behind. The three never rows are +# premises about the fixtures: the partitions, the pending detach, and the +# tiered January. +# +# ledger rows 2286 + 23 = 2309 +# never 1801 + 3 = 1804 +# dated 485 + 20 = 505 +# 1804 + 505 == 2309 +checks_never_observed_red 1804