From 525cc674e331ce5a6cff5acb1c9a58e871c3a52d Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 20:48:00 +0000 Subject: [PATCH 1/5] test: partition_create_ahead, written first (#1297) --- test/partition_lifecycle.sh | 110 ++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100755 test/partition_lifecycle.sh diff --git a/test/partition_lifecycle.sh b/test/partition_lifecycle.sh new file mode 100755 index 00000000..db42e880 --- /dev/null +++ b/test/partition_lifecycle.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# +# pgcolumnar.partition_create_ahead: columnar partitions created ahead, without an +# exclusive lock on the parent (#1297). +# +# WRITTEN BEFORE THE FUNCTION EXISTS (the owner's TDD rule). The specification it +# pins, from #1297's design note 1, measured on PG 15..19: +# +# * CREATE TABLE ... PARTITION OF takes ACCESS EXCLUSIVE on the parent, while a +# standalone CREATE followed by ATTACH takes SHARE UPDATE EXCLUSIVE. So the +# function must never hold more than SHARE UPDATE EXCLUSIVE on the parent. +# * before PG 17 the parent cannot carry USING pgcolumnar, so the function sets +# the access method on every partition itself. +# * a DEFAULT partition rules out DETACH CONCURRENTLY for the drop half, so a +# parent with one is refused rather than half-managed. +# +# partition_create_ahead(parent regclass, width interval, ahead int) -> int +# +# Makes sure the parent's partitions reach now() + ahead * width, each `width` +# wide and contiguous with the last existing upper bound -- or, with no +# partitions yet, starting at now() truncated to the width's unit -- and returns +# how many it created. A second call creates nothing: the daemon calls it every +# pass. +# +# Usage: test/partition_lifecycle.sh [PG_CONFIG] + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +at_least() { [ -z "$1" ] && return; [ "$1" -lt "$2" ] && echo "$1" || echo "$2"; } + +q "CREATE EXTENSION IF NOT EXISTS pgcolumnar;" >/dev/null +check_num "premise: the function exists" \ + "$(q "SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'pgcolumnar' AND p.proname = 'partition_create_ahead';")" "1" + +kids() { q "SELECT count(*) FROM pg_inherits WHERE inhparent = '$1'::regclass;"; } +ams() { q "SELECT coalesce(string_agg(DISTINCT a.amname, ','), 'none') FROM pg_inherits i + JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_am a ON a.oid = c.relam + WHERE i.inhparent = '$1'::regclass;"; } +# the partition bounds, as "from|to" lines in order +bounds() { + q "SELECT (regexp_match(pg_get_expr(c.relpartbound, c.oid), + 'FROM \\(''([^'']+)''\\) TO \\(''([^'']+)''\\)'))[1] + || '|' || + (regexp_match(pg_get_expr(c.relpartbound, c.oid), + 'FROM \\(''([^'']+)''\\) TO \\(''([^'']+)''\\)'))[2] + FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = '$1'::regclass ORDER BY 1;" +} + +# ---- from nothing ----------------------------------------------------------- +q "CREATE TABLE tl (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts);" >/dev/null +made="$(q "SELECT pgcolumnar.partition_create_ahead('tl', '1 day', 3);")" +check_num "from no partitions it creates enough to reach now plus three days" \ + "$made" "$(q "SELECT 4;")" +check_num "premise: the parent now has the partitions it reported" "$(kids tl)" "$made" +check "every partition it creates is columnar" "$(ams tl)" "pgcolumnar" +check "they are contiguous, each one day wide" \ + "$(bounds tl | awk -F'|' 'NR>1 && $1!=prev {gap++} {prev=$2} END {print gap+0}')|$(q "SELECT count(DISTINCT upper - lower) FROM (SELECT (regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''([^'']+)''\\)'))[1]::timestamptz AS upper, (regexp_match(pg_get_expr(c.relpartbound, c.oid), 'FROM \\(''([^'']+)''\\)'))[1]::timestamptz AS lower FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'tl'::regclass) b;")" \ + "0|1" +check "and they cover now() plus three days" \ + "$(q "SELECT bool_or(now() + interval '3 days' < (regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''([^'']+)''\\)'))[1]::timestamptz + AND now() + interval '3 days' >= (regexp_match(pg_get_expr(c.relpartbound, c.oid), 'FROM \\(''([^'']+)''\\)'))[1]::timestamptz) + FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'tl'::regclass;")" "t" + +# ---- idempotent, and it continues from the last bound ---------------------- +check_num "a second call creates nothing" \ + "$(q "SELECT pgcolumnar.partition_create_ahead('tl', '1 day', 3);")" "0" +check_num "asking further ahead creates only the difference" \ + "$(q "SELECT pgcolumnar.partition_create_ahead('tl', '1 day', 5);")" "2" + +# ---- rows go in, columnar ------------------------------------------------------ +q "INSERT INTO tl SELECT now() + g * interval '1 hour', g FROM generate_series(0, 100) g;" >/dev/null +check_num "premise: rows within the covered range go in" "$(q "SELECT count(*) FROM tl;")" "101" +check "and land in columnar partitions" \ + "$(q "SELECT string_agg(DISTINCT a.amname, ',') FROM tl t JOIN pg_class c ON c.oid = t.tableoid + JOIN pg_am a ON a.oid = c.relam;")" "pgcolumnar" + +# ---- the parent lock ---------------------------------------------------------- +q "CREATE TABLE tk (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts);" >/dev/null +# The call and the lock reading share one transaction, and the arm requires the +# call to have RETURNED: in the red run before the function existed, the failed +# call aborted the transaction, the pg_locks query returned nothing, and an arm +# reading "no AccessExclusiveLock" passed on a function that did not exist. +out="$(q "BEGIN; SELECT 'made=' || pgcolumnar.partition_create_ahead('tk', '1 hour', 6); + SELECT 'locks=' || coalesce(string_agg(DISTINCT mode, ',' ORDER BY mode), 'none') FROM pg_locks + WHERE relation = 'tk'::regclass AND pid = pg_backend_pid(); ROLLBACK;")" +tk_made="$(printf '%s\n' "$out" | sed -n 's/^made=//p')" +lock="$(printf '%s\n' "$out" | sed -n 's/^locks=//p')" +echo "-- created $tk_made; locks held on the parent: ${lock:-}" +check_num "premise: the call inside the transaction created partitions" "$(at_least "$tk_made" 1)" "1" +check "it never takes AccessExclusiveLock on the parent" \ + "$(case "$lock" in '') echo unread;; *AccessExclusiveLock*) echo "held: $lock";; *) echo none;; esac)" "none" + +# ---- refused: a DEFAULT partition ------------------------------------------------- +q "CREATE TABLE td (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts); + CREATE TABLE td_default PARTITION OF td DEFAULT USING pgcolumnar;" >/dev/null +check "a parent with a DEFAULT partition is refused" \ + "$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -At -v ON_ERROR_STOP=1 -v VERBOSITY=sqlstate \ + -c "SELECT pgcolumnar.partition_create_ahead('td', '1 day', 2)" 2>&1 >/dev/null | + sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p')" "55000" +# Only a claim when the function exists: with it absent, nothing is created +# for any parent, and this arm passed in the red run. +check "and nothing was created for it" \ + "$(kids td)|$(q "SELECT count(*) FROM pg_proc WHERE proname = 'partition_create_ahead';")" "1|1" + +pgc_summary From 3d087fd80e1a557a786b0f69f50c0fcdbf294c79 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 22:46:08 +0000 Subject: [PATCH 2/5] feat: pgcolumnar.partition_create_ahead keeps a time-partitioned parent ahead of now (#1297) Creates columnar partitions, each width wide and contiguous with the highest existing upper bound, until they reach now plus ahead widths. Standalone CREATE then ATTACH, so the parent is held SHARE UPDATE EXCLUSIVE, never ACCESS EXCLUSIVE. A DEFAULT partition is refused (55000). Found by new arms against the prototype: a timestamp key's bounds were read as timestamptz in the session's zone and written converted from UTC, so in America/New_York partitions had a four-hour gap (a row at the boundary had nowhere to go), and daily partitions made from Tokyo began at 15:00. A timestamp key is now reasoned about in its own type, with localtimestamp as now. partition_lifecycle.sh and its independent pytest twin: 23/23 on PG15-19 each. Five mutants red in both (the prototype's zone handling, PARTITION OF, no DEFAULT refusal, starting from now, heap partitions). Upgrades from every released version converge. Co-Authored-By: Claude Opus 5.5 --- docs/how-to.md | 29 ++++ docs/sql-reference.md | 11 ++ pgcolumnar--1.0-alpha4--1.0-alpha5.sql | 125 ++++++++++++++++ pgcolumnar--1.0-alpha5.sql | 125 ++++++++++++++++ test/partition_lifecycle.sh | 49 +++++- test/pytest/test_partition_lifecycle.py | 189 ++++++++++++++++++++++++ 6 files changed, 525 insertions(+), 3 deletions(-) create mode 100644 test/pytest/test_partition_lifecycle.py diff --git a/docs/how-to.md b/docs/how-to.md index 0fecd064..7d65cdf9 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -323,6 +323,35 @@ declared retention raises an error rather than doing nothing. `pgcolumnar.reset_options` has no argument for the retention. To stop expiring, set `ttl_interval` to a span no row will reach. +## Create time partitions ahead + +`pgcolumnar.partition_create_ahead` keeps a time-partitioned table ready for +the rows that are coming. Call it from a scheduler, such as pg_cron, more often +than the width: + +```sql +CREATE TABLE metrics (ts timestamptz NOT NULL, device int, value float8) + PARTITION BY RANGE (ts); + +-- Daily partitions, up to three days ahead of now. +SELECT pgcolumnar.partition_create_ahead('metrics', '1 day', 3); +``` + +Every partition that it creates is columnar. It continues from the highest +existing upper bound, so the partitions stay contiguous, and a second call +creates nothing. + +It takes `SHARE UPDATE EXCLUSIVE` on the parent, not `ACCESS EXCLUSIVE`, so +queries and inserts on the table continue while it runs. + +For a `timestamp` key, which has no time zone, "now" is the session's local +time, and partitions start on the width's boundary in that time. + +The function refuses a table that has a DEFAULT partition. + +`test/partition_lifecycle.sh` checks each of these statements, in sessions +west and east of UTC. + ## Manage time partitions with pg_partman pg_partman can create and drop columnar partitions. It needs one setting to do diff --git a/docs/sql-reference.md b/docs/sql-reference.md index 9016cf41..2ea9778e 100644 --- a/docs/sql-reference.md +++ b/docs/sql-reference.md @@ -570,6 +570,17 @@ Writes the live rows of `rel` to a Parquet file at `path`. Returns the number of rows written. `path` can be a local file or an `s3://` URL. See [Object storage](#object-storage). +### pgcolumnar.partition_create_ahead(parent regclass, width interval, ahead int) returns int + +Creates columnar partitions of `parent`, each `width` wide, until they reach now +plus `ahead` widths. Returns the number of partitions created. A second call +creates none. The first new partition starts at the highest existing upper +bound. With no partitions, it starts at the width's boundary that holds now. +`parent` must be partitioned by RANGE on one `timestamp` or `timestamptz` +column, and must have no DEFAULT partition (`55000`). The function takes +`SHARE UPDATE EXCLUSIVE` on the parent, never `ACCESS EXCLUSIVE`. See +[Create time partitions ahead](how-to.md#create-time-partitions-ahead). + ### pgcolumnar.import_arrow(rel regclass, path text) returns bigint Inserts the rows of an Arrow IPC stream file at `path` into the existing table diff --git a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql index 88cf8e1c..03eccbbd 100644 --- a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql @@ -791,3 +791,128 @@ BEGIN END IF; END; $$; + +/* + * pgcolumnar.partition_create_ahead (#1297) + * + * Make sure a RANGE-partitioned parent's partitions reach now() + ahead * width, + * each `width` wide and contiguous with the highest existing upper bound (or, + * with none, starting at now() aligned to the width), every one columnar. + * Returns how many it created; a second call creates nothing. + * + * NEVER AN EXCLUSIVE LOCK ON THE PARENT. CREATE TABLE ... PARTITION OF takes + * ACCESS EXCLUSIVE on the parent; a standalone CREATE followed by ATTACH takes + * SHARE UPDATE EXCLUSIVE, which readers and writers pass. Measured on PG 15..19 + * (#1297 note 1). + * + * A DEFAULT partition is refused (55000): it rules out DETACH CONCURRENTLY for + * the drop half, and a parent managed half-way is worse than one refused. + */ +CREATE FUNCTION pgcolumnar.partition_create_ahead(parent regclass, width interval, ahead int) + RETURNS int + LANGUAGE plpgsql + SET search_path = pg_catalog, pg_temp + AS $create_ahead$ +DECLARE + strat "char"; + natts int; + defid oid; + keytype regtype; + nsp name; + rel name; + lo timestamptz; + hi timestamptz; + target timestamptz; + ts_lo timestamp; + ts_hi timestamp; + ts_target timestamp; + child text; + made int := 0; +BEGIN + IF width IS NULL OR width <= interval '0' OR ahead IS NULL OR ahead < 0 THEN + RAISE EXCEPTION 'width must be positive and ahead not negative' + USING ERRCODE = '22023'; + END IF; + SELECT pt.partstrat, pt.partnatts, pt.partdefid, a.atttypid::regtype + INTO strat, natts, defid, keytype + FROM pg_partitioned_table pt + JOIN pg_attribute a ON a.attrelid = pt.partrelid AND a.attnum = pt.partattrs[0] + WHERE pt.partrelid = parent; + IF NOT FOUND OR strat <> 'r' OR natts <> 1 THEN + RAISE EXCEPTION '% is not RANGE-partitioned on one column', parent + USING ERRCODE = '55000'; + END IF; + IF keytype NOT IN ('timestamptz'::regtype, 'timestamp'::regtype) THEN + RAISE EXCEPTION '% is partitioned on %, not on a timestamp', parent, keytype + USING ERRCODE = '55000'; + END IF; + IF defid <> 0 THEN + RAISE EXCEPTION '% has a DEFAULT partition', parent + USING ERRCODE = '55000', + HINT = 'A DEFAULT partition rules out DETACH CONCURRENTLY, so the partition lifecycle refuses to manage this table.'; + END IF; + + SELECT n.nspname, c.relname INTO nsp, rel + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.oid = parent; + + /* + * A `timestamp` key holds wall-clock time with no zone, so it is reasoned + * about in its own type: its bounds are read as timestamp, "now" is the + * session's localtimestamp, and the bounds written are never converted. + * Reading them as timestamptz in one zone and writing them in another left + * a gap of the zone's offset between partitions (America/New_York: four + * hours), and aligned "daily" partitions to UTC's midnight, not the key's. + */ + IF keytype = 'timestamp'::regtype THEN + SELECT max((regexp_match(pg_get_expr(c.relpartbound, c.oid), + 'TO \(''([^'']+)''\)'))[1]::timestamp) + INTO ts_lo + FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = parent; + IF ts_lo IS NULL THEN + IF extract(month FROM width) <> 0 OR extract(year FROM width) <> 0 THEN + ts_lo := date_trunc('month', localtimestamp); + ELSE + ts_lo := date_bin(width, localtimestamp, timestamp '2001-01-01'); + END IF; + END IF; + ts_target := localtimestamp + ahead * width; + WHILE ts_lo <= ts_target LOOP + ts_hi := ts_lo + width; + child := rel || '_' || to_char(ts_lo, 'YYYYMMDD"T"HH24MISS'); + EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS) USING pgcolumnar', + nsp, child, parent); + EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I FOR VALUES FROM (%L) TO (%L)', + parent, nsp, child, ts_lo::text, ts_hi::text); + made := made + 1; + ts_lo := ts_hi; + END LOOP; + RETURN made; + END IF; + + SELECT max((regexp_match(pg_get_expr(c.relpartbound, c.oid), + 'TO \(''([^'']+)''\)'))[1]::timestamptz) + INTO lo + FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = parent; + IF lo IS NULL THEN + IF extract(month FROM width) <> 0 OR extract(year FROM width) <> 0 THEN + lo := date_trunc('month', now()); + ELSE + lo := date_bin(width, now(), timestamptz '2001-01-01'); + END IF; + END IF; + target := now() + ahead * width; + WHILE lo <= target LOOP + hi := lo + width; + child := rel || '_' || to_char(lo AT TIME ZONE 'UTC', 'YYYYMMDD"T"HH24MISS'); + EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS) USING pgcolumnar', + nsp, child, parent); + EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I FOR VALUES FROM (%L) TO (%L)', + parent, nsp, child, lo::text, hi::text); + made := made + 1; + lo := hi; + END LOOP; + RETURN made; +END; +$create_ahead$; diff --git a/pgcolumnar--1.0-alpha5.sql b/pgcolumnar--1.0-alpha5.sql index fb018fe6..797051ab 100644 --- a/pgcolumnar--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha5.sql @@ -1980,3 +1980,128 @@ $maintenance_due$; COMMENT ON FUNCTION pgcolumnar.maintenance_due(regclass, float8, float8) IS 'report whether an online maintenance verb (compact_rewrite, recluster) is worth running, from table statistics alone; thresholds are parameters with defaults measured on #415, each required to be a number between 0 and 1 inclusive (#860); pure report, takes no lock and rewrites nothing (#415)'; + +/* + * pgcolumnar.partition_create_ahead (#1297) + * + * Make sure a RANGE-partitioned parent's partitions reach now() + ahead * width, + * each `width` wide and contiguous with the highest existing upper bound (or, + * with none, starting at now() aligned to the width), every one columnar. + * Returns how many it created; a second call creates nothing. + * + * NEVER AN EXCLUSIVE LOCK ON THE PARENT. CREATE TABLE ... PARTITION OF takes + * ACCESS EXCLUSIVE on the parent; a standalone CREATE followed by ATTACH takes + * SHARE UPDATE EXCLUSIVE, which readers and writers pass. Measured on PG 15..19 + * (#1297 note 1). + * + * A DEFAULT partition is refused (55000): it rules out DETACH CONCURRENTLY for + * the drop half, and a parent managed half-way is worse than one refused. + */ +CREATE FUNCTION pgcolumnar.partition_create_ahead(parent regclass, width interval, ahead int) + RETURNS int + LANGUAGE plpgsql + SET search_path = pg_catalog, pg_temp + AS $create_ahead$ +DECLARE + strat "char"; + natts int; + defid oid; + keytype regtype; + nsp name; + rel name; + lo timestamptz; + hi timestamptz; + target timestamptz; + ts_lo timestamp; + ts_hi timestamp; + ts_target timestamp; + child text; + made int := 0; +BEGIN + IF width IS NULL OR width <= interval '0' OR ahead IS NULL OR ahead < 0 THEN + RAISE EXCEPTION 'width must be positive and ahead not negative' + USING ERRCODE = '22023'; + END IF; + SELECT pt.partstrat, pt.partnatts, pt.partdefid, a.atttypid::regtype + INTO strat, natts, defid, keytype + FROM pg_partitioned_table pt + JOIN pg_attribute a ON a.attrelid = pt.partrelid AND a.attnum = pt.partattrs[0] + WHERE pt.partrelid = parent; + IF NOT FOUND OR strat <> 'r' OR natts <> 1 THEN + RAISE EXCEPTION '% is not RANGE-partitioned on one column', parent + USING ERRCODE = '55000'; + END IF; + IF keytype NOT IN ('timestamptz'::regtype, 'timestamp'::regtype) THEN + RAISE EXCEPTION '% is partitioned on %, not on a timestamp', parent, keytype + USING ERRCODE = '55000'; + END IF; + IF defid <> 0 THEN + RAISE EXCEPTION '% has a DEFAULT partition', parent + USING ERRCODE = '55000', + HINT = 'A DEFAULT partition rules out DETACH CONCURRENTLY, so the partition lifecycle refuses to manage this table.'; + END IF; + + SELECT n.nspname, c.relname INTO nsp, rel + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.oid = parent; + + /* + * A `timestamp` key holds wall-clock time with no zone, so it is reasoned + * about in its own type: its bounds are read as timestamp, "now" is the + * session's localtimestamp, and the bounds written are never converted. + * Reading them as timestamptz in one zone and writing them in another left + * a gap of the zone's offset between partitions (America/New_York: four + * hours), and aligned "daily" partitions to UTC's midnight, not the key's. + */ + IF keytype = 'timestamp'::regtype THEN + SELECT max((regexp_match(pg_get_expr(c.relpartbound, c.oid), + 'TO \(''([^'']+)''\)'))[1]::timestamp) + INTO ts_lo + FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = parent; + IF ts_lo IS NULL THEN + IF extract(month FROM width) <> 0 OR extract(year FROM width) <> 0 THEN + ts_lo := date_trunc('month', localtimestamp); + ELSE + ts_lo := date_bin(width, localtimestamp, timestamp '2001-01-01'); + END IF; + END IF; + ts_target := localtimestamp + ahead * width; + WHILE ts_lo <= ts_target LOOP + ts_hi := ts_lo + width; + child := rel || '_' || to_char(ts_lo, 'YYYYMMDD"T"HH24MISS'); + EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS) USING pgcolumnar', + nsp, child, parent); + EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I FOR VALUES FROM (%L) TO (%L)', + parent, nsp, child, ts_lo::text, ts_hi::text); + made := made + 1; + ts_lo := ts_hi; + END LOOP; + RETURN made; + END IF; + + SELECT max((regexp_match(pg_get_expr(c.relpartbound, c.oid), + 'TO \(''([^'']+)''\)'))[1]::timestamptz) + INTO lo + FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = parent; + IF lo IS NULL THEN + IF extract(month FROM width) <> 0 OR extract(year FROM width) <> 0 THEN + lo := date_trunc('month', now()); + ELSE + lo := date_bin(width, now(), timestamptz '2001-01-01'); + END IF; + END IF; + target := now() + ahead * width; + WHILE lo <= target LOOP + hi := lo + width; + child := rel || '_' || to_char(lo AT TIME ZONE 'UTC', 'YYYYMMDD"T"HH24MISS'); + EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS) USING pgcolumnar', + nsp, child, parent); + EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I FOR VALUES FROM (%L) TO (%L)', + parent, nsp, child, lo::text, hi::text); + made := made + 1; + lo := hi; + END LOOP; + RETURN made; +END; +$create_ahead$; diff --git a/test/partition_lifecycle.sh b/test/partition_lifecycle.sh index db42e880..84c8ed8f 100755 --- a/test/partition_lifecycle.sh +++ b/test/partition_lifecycle.sh @@ -53,14 +53,14 @@ bounds() { # ---- from nothing ----------------------------------------------------------- q "CREATE TABLE tl (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts);" >/dev/null made="$(q "SELECT pgcolumnar.partition_create_ahead('tl', '1 day', 3);")" -check_num "from no partitions it creates enough to reach now plus three days" \ +check_num "from no partitions it creates enough to reach now plus the lead" \ "$made" "$(q "SELECT 4;")" check_num "premise: the parent now has the partitions it reported" "$(kids tl)" "$made" check "every partition it creates is columnar" "$(ams tl)" "pgcolumnar" -check "they are contiguous, each one day wide" \ +check "they are contiguous, each one width wide" \ "$(bounds tl | awk -F'|' 'NR>1 && $1!=prev {gap++} {prev=$2} END {print gap+0}')|$(q "SELECT count(DISTINCT upper - lower) FROM (SELECT (regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''([^'']+)''\\)'))[1]::timestamptz AS upper, (regexp_match(pg_get_expr(c.relpartbound, c.oid), 'FROM \\(''([^'']+)''\\)'))[1]::timestamptz AS lower FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'tl'::regclass) b;")" \ "0|1" -check "and they cover now() plus three days" \ +check "and they cover now() plus the lead" \ "$(q "SELECT bool_or(now() + interval '3 days' < (regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''([^'']+)''\\)'))[1]::timestamptz AND now() + interval '3 days' >= (regexp_match(pg_get_expr(c.relpartbound, c.oid), 'FROM \\(''([^'']+)''\\)'))[1]::timestamptz) FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'tl'::regclass;")" "t" @@ -94,6 +94,49 @@ check_num "premise: the call inside the transaction created partitions" "$(at_le check "it never takes AccessExclusiveLock on the parent" \ "$(case "$lock" in '') echo unread;; *AccessExclusiveLock*) echo "held: $lock";; *) echo none;; esac)" "none" +# ---- the session's time zone ---------------------------------------------------- +# The bounds are read back as text and continued from. For a `timestamp` key (no +# zone) the text carries no offset, so reading it in one zone and writing the next +# bound in another leaves a gap or an overlap. Each arm continues from a partition +# made by hand, in a session whose zone is not UTC, and requires every partition's +# lower bound to be exactly the previous one's upper bound. +contiguous() { # contiguous PARENT -> "ok", or the first break + bounds "$1" | awk -F'|' 'NR>1 && $1!=prev {print "break: " prev " then " $1; bad=1; exit} {prev=$2} END {if (!bad) print "ok"}' +} +for key in timestamp timestamptz; do + t="tz_$key" + q "CREATE TABLE $t (ts $key NOT NULL, v int) PARTITION BY RANGE (ts);" >/dev/null + q "SET TimeZone = 'America/New_York'; + CREATE TABLE ${t}_seed PARTITION OF $t FOR VALUES + FROM (date_trunc('day', now() - interval '2 days')::$key) TO (date_trunc('day', now() - interval '1 day')::$key) + USING pgcolumnar;" >/dev/null + tz_made="$(q "SET TimeZone = 'America/New_York'; SELECT pgcolumnar.partition_create_ahead('$t', '1 day', 2);" | tail -1)" + check_num "premise: a $key parent continued from a hand-made partition, in a session west of UTC" \ + "$(at_least "$tz_made" 3)" "3" + check "a $key parent's partitions are contiguous across a non-UTC session" \ + "$(q "SET TimeZone = 'America/New_York'; SELECT 1;" >/dev/null; contiguous "$t")" "ok" + check_num "and a row at every boundary has exactly one partition to go to ($key)" \ + "$(q "SET TimeZone = 'America/New_York'; + INSERT INTO $t SELECT (date_trunc('day', now() - interval '1 day') + g * interval '1 day')::$key, g + FROM generate_series(0, 2) g; SELECT count(*) FROM $t;" | tail -1)" "3" +done + +# A `timestamp` key holds local wall-clock time, so "now" for it is the session's +# localtimestamp. East of UTC that is ahead of UTC's: a function that reasoned in +# UTC would stop nine hours short in Tokyo. +q "CREATE TABLE tz_tokyo (ts timestamp NOT NULL, v int) PARTITION BY RANGE (ts);" >/dev/null +tk2="$(q "SET TimeZone = 'Asia/Tokyo'; SELECT pgcolumnar.partition_create_ahead('tz_tokyo', '1 day', 2);" | tail -1)" +echo "-- Tokyo local now: $(q "SET TimeZone = 'Asia/Tokyo'; SELECT localtimestamp;" | tail -1); bounds: $(bounds tz_tokyo | tr '\n' ' ')" +check_num "premise: a timestamp parent was given partitions from a session east of UTC" "$(at_least "$tk2" 1)" "1" +check "a timestamp parent made east of UTC covers the local now plus the lead" \ + "$(q "SET TimeZone = 'Asia/Tokyo'; + SELECT bool_or(localtimestamp + interval '2 days' >= (regexp_match(pg_get_expr(c.relpartbound, c.oid), 'FROM \\(''([^'']+)''\\)'))[1]::timestamp + AND localtimestamp + interval '2 days' < (regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''([^'']+)''\\)'))[1]::timestamp) + FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'tz_tokyo'::regclass;" | tail -1)" "t" + +check "and its partitions start on the width's boundary in the key's own wall clock" \ + "$(bounds tz_tokyo | awk -F'|' '$1 !~ / 00:00:00$/ {print "not midnight: " $1; bad=1; exit} END {if (!bad) print "ok"}')" "ok" + # ---- refused: a DEFAULT partition ------------------------------------------------- q "CREATE TABLE td (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts); CREATE TABLE td_default PARTITION OF td DEFAULT USING pgcolumnar;" >/dev/null diff --git a/test/pytest/test_partition_lifecycle.py b/test/pytest/test_partition_lifecycle.py new file mode 100644 index 00000000..46ebd86c --- /dev/null +++ b/test/pytest/test_partition_lifecycle.py @@ -0,0 +1,189 @@ +"""pgcolumnar.partition_create_ahead keeps a time-partitioned parent ahead of now (#1297). + +The function makes columnar partitions, each `width` wide and contiguous with the +highest existing upper bound, until they reach now plus `ahead` widths. It never +takes ACCESS EXCLUSIVE on the parent: a standalone CREATE followed by ATTACH +takes SHARE UPDATE EXCLUSIVE, which readers and writers pass (#1297 note 1). + +THE SESSION'S ZONE IS WHERE THE FIRST PROTOTYPE BROKE. A `timestamp` key holds +wall-clock time with no zone. The prototype read its bounds as timestamptz in the +session's zone and wrote the next ones converted from UTC: in America/New_York +that left a four-hour gap between partitions, so a row at the boundary had +nowhere to go, and "daily" partitions made from Tokyo started at 15:00. So the +arms here continue from hand-made partitions in sessions west and east of UTC, +and check contiguity, routing at every boundary, coverage of the session's own +local now, and alignment to the width in the key's wall clock. + +Independent of `test/partition_lifecycle.sh`: its own widths (six hours, not a +day), leads, zones (America/Los_Angeles, and Asia/Kolkata for a half-hour +offset) and rows. +""" + +import re + +import psycopg + +LEAD = 5 # widths ahead of now +WIDTH = "6 hours" + +BOUND = re.compile(r"FROM \('([^']+)'\) TO \('([^']+)'\)") + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + row = cur.fetchone() + return row[0] if row else None + + +def _bounds(conn, parent): + """[(from, to)] as the catalog prints them, in order of `from`.""" + with conn.cursor() as cur: + cur.execute("SELECT pg_get_expr(c.relpartbound, c.oid) FROM pg_inherits i" + " JOIN pg_class c ON c.oid = i.inhrelid" + f" WHERE i.inhparent = '{parent}'::regclass") + out = [] + for (expr,) in cur.fetchall(): + m = BOUND.search(expr) + out.append((m.group(1), m.group(2)) if m else (expr, expr)) + return sorted(out) + + +def _contiguous(bounds): + for (_, prev_to), (nxt_from, _) in zip(bounds, bounds[1:]): + if nxt_from != prev_to: + return f"break: {prev_to} then {nxt_from}" + return "ok" + + +def _kids(conn, parent): + return _one(conn, f"SELECT count(*) FROM pg_inherits WHERE inhparent = '{parent}'::regclass") + + +def _create_ahead(conn, parent, ahead=LEAD, width=WIDTH): + return _one(conn, f"SELECT pgcolumnar.partition_create_ahead('{parent}', '{width}', {ahead})") + + +def _create_ahead_or_error(conn, parent, ahead=LEAD, width=WIDTH): + """The count as text, or the SQLSTATE: a repeat call that errors must be + reported by the arm that asked, not as an unhandled exception.""" + try: + return str(_create_ahead(conn, parent, ahead, width)) + except psycopg.Error as e: + return f"error {e.sqlstate}" + + +def test_partition_create_ahead(pgc_conn, expect): + conn = pgc_conn + expect.num(_one(conn, "SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace" + " WHERE n.nspname = 'pgcolumnar' AND p.proname = 'partition_create_ahead'"), 1, + "premise: the function exists") + + # From nothing. In UTC, six-hour widths align to 00/06/12/18, so reaching + # now + 5 widths takes 6 partitions (the one holding now, and 5 more), or 7 + # when now + 5 widths lands exactly on a boundary -- it never does here, + # because now() carries microseconds. + with conn.cursor() as cur: + cur.execute("SET TimeZone = 'UTC'") + cur.execute("CREATE TABLE gauge (at timestamptz NOT NULL, reading float8) PARTITION BY RANGE (at)") + made = _create_ahead(conn, "gauge") + expect.num(made, LEAD + 1, "from no partitions it creates enough to reach now plus the lead") + expect.num(_kids(conn, "gauge"), made, "premise: the parent now has the partitions it reported") + expect.text(_one(conn, "SELECT string_agg(DISTINCT a.amname, ',') FROM pg_inherits i" + " JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_am a ON a.oid = c.relam" + " WHERE i.inhparent = 'gauge'::regclass"), + "pgcolumnar", "every partition it creates is columnar") + b = _bounds(conn, "gauge") + widths = _one(conn, "SELECT count(DISTINCT t - f) FROM (VALUES " + + ", ".join(f"('{f}'::timestamptz, '{t}'::timestamptz)" for f, t in b) + + ") v(f, t)") + expect.text(f"{_contiguous(b)}|{widths}", "ok|1", "they are contiguous, each one width wide") + expect.text(str(_one(conn, f"SELECT now() + {LEAD} * interval '{WIDTH}' >= '{b[-1][0]}'::timestamptz" + f" AND now() + {LEAD} * interval '{WIDTH}' < '{b[-1][1]}'::timestamptz")), + "True", "and they cover now() plus the lead") + + expect.text(_create_ahead_or_error(conn, "gauge"), "0", "a second call creates nothing") + expect.text(_create_ahead_or_error(conn, "gauge", ahead=LEAD + 3), "3", + "asking further ahead creates only the difference") + + with conn.cursor() as cur: + cur.execute(f"INSERT INTO gauge SELECT now() + g * interval '17 min', g" + f" FROM generate_series(0, 77) g") + expect.num(_one(conn, "SELECT count(*) FROM gauge"), 78, "premise: rows within the covered range go in") + expect.text(_one(conn, "SELECT string_agg(DISTINCT a.amname, ',') FROM gauge t" + " JOIN pg_class c ON c.oid = t.tableoid JOIN pg_am a ON a.oid = c.relam"), + "pgcolumnar", "and land in columnar partitions") + + # The parent's lock, read inside the transaction that made the partitions. + with conn.cursor() as cur: + cur.execute("CREATE TABLE gauge_lock (at timestamptz NOT NULL, reading float8) PARTITION BY RANGE (at)") + cur.execute("BEGIN") + cur.execute(f"SELECT pgcolumnar.partition_create_ahead('gauge_lock', '30 minutes', 9)") + locked_made = cur.fetchone()[0] + cur.execute("SELECT coalesce(string_agg(DISTINCT mode, ',' ORDER BY mode), 'none') FROM pg_locks" + " WHERE relation = 'gauge_lock'::regclass AND pid = pg_backend_pid()") + locks = cur.fetchone()[0] + cur.execute("ROLLBACK") + expect.at_least(locked_made, 1, "premise: the call inside the transaction created partitions") + expect.text(f"held: {locks}" if "AccessExclusiveLock" in locks else "none", "none", + "it never takes AccessExclusiveLock on the parent") + + # West of UTC: continue from a hand-made partition, for both key types. + for key in ("timestamp", "timestamptz"): + t = f"west_{key}" + with conn.cursor() as cur: + cur.execute("SET TimeZone = 'America/Los_Angeles'") + cur.execute(f"CREATE TABLE {t} (at {key} NOT NULL, reading float8) PARTITION BY RANGE (at)") + cur.execute(f"CREATE TABLE {t}_seed PARTITION OF {t} FOR VALUES" + f" FROM (date_trunc('day', now() - interval '1 day')::{key})" + f" TO ((date_trunc('day', now() - interval '1 day') + interval '6 hours')::{key})" + " USING pgcolumnar") + west = _create_ahead(conn, t, ahead=4) + expect.at_least(west, 5, f"premise: a {key} parent continued from a hand-made partition, in a session west of UTC") + expect.text(_contiguous(_bounds(conn, t)), "ok", + f"a {key} parent's partitions are contiguous across a non-UTC session") + # One row at the lower bound of every partition: each must route. + firsts = [f for f, _ in _bounds(conn, t)] + routed = 0 + for f in firsts: + try: + with conn.cursor() as cur: + cur.execute(f"INSERT INTO {t} VALUES ('{f}'::{key}, 1)") + routed += 1 + except psycopg.Error: + pass + expect.num(routed, len(firsts), + f"and a row at every boundary has exactly one partition to go to ({key})") + with conn.cursor() as cur: + cur.execute("SET TimeZone = 'UTC'") + + # East of UTC, at a half-hour offset: coverage of the session's own local now, + # and alignment to the width in the key's wall clock. + with conn.cursor() as cur: + cur.execute("SET TimeZone = 'Asia/Kolkata'") + cur.execute("CREATE TABLE east (at timestamp NOT NULL, reading float8) PARTITION BY RANGE (at)") + east = _create_ahead(conn, "east", ahead=3) + expect.at_least(east, 1, "premise: a timestamp parent was given partitions from a session east of UTC") + eb = _bounds(conn, "east") + local_target = _one(conn, f"SELECT (localtimestamp + 3 * interval '{WIDTH}')::text") + expect.text(str(_one(conn, f"SELECT bool_or('{local_target}'::timestamp >= f AND '{local_target}'::timestamp < t)" + " FROM (VALUES " + ", ".join(f"('{f}'::timestamp, '{t}'::timestamp)" for f, t in eb) + + ") v(f, t)")), + "True", "a timestamp parent made east of UTC covers the local now plus the lead") + off = [f for f, _ in eb if not re.search(r" (00|06|12|18):00:00$", f)] + expect.text(f"not on the width: {off[0]}" if off else "ok", "ok", + "and its partitions start on the width's boundary in the key's own wall clock") + with conn.cursor() as cur: + cur.execute("SET TimeZone = 'UTC'") + + # Refused: a DEFAULT partition. + with conn.cursor() as cur: + cur.execute("CREATE TABLE fallback (at timestamptz NOT NULL, reading float8) PARTITION BY RANGE (at)") + cur.execute("CREATE TABLE fallback_rest PARTITION OF fallback DEFAULT USING pgcolumnar") + try: + _create_ahead(conn, "fallback") + state = "ok" + except psycopg.Error as e: + state = e.sqlstate or "?" + expect.text(state, "55000", "a parent with a DEFAULT partition is refused") + expect.num(_kids(conn, "fallback"), 1, "and nothing was created for it") From 6a8e9114f8481d714462c54c8cd380ee5af31948 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 22:54:06 +0000 Subject: [PATCH 3/5] test: partition_lifecycle registries and ledger (#1297) The pytest twin covers every shell property (compare_to_bash: 17 literal, 3 template, 0 missing). Ledger: 23 rows from a five-major run, 17 dated by seven mutations merged one log at a time with --mutation and --target; the six 'never' rows are premises. One check a mutation reddened is no longer named a premise (#1236). Census 1674 -> 1680. TESTS.md section 92; cluster_tests 499 by collection. Guard half 413 passed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 11 +++++++++++ test/partition_lifecycle.sh | 2 +- test/pytest/TESTS.md | 17 +++++++++++++++++ test/pytest/expected_tests.txt | 3 ++- test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_partition_lifecycle.py | 2 +- test/run_all_versions.sh | 1 + 7 files changed, 34 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a7019646..1bc22556 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,6 +89,17 @@ true until the next version shipped. scan backward, and without it the fetch reached the access method and failed. The Material's plan node id now sits above the subplans' ids as well, so a cursor whose query has a subquery no longer repeats a node id. +- `pgcolumnar.partition_create_ahead(parent, width, ahead)` keeps a + time-partitioned table ready for the rows that are coming (#1297). It creates + columnar partitions, each `width` wide and contiguous with the highest existing + upper bound, until they reach now plus `ahead` widths; a second call creates + none. It takes `SHARE UPDATE EXCLUSIVE` on the parent, never `ACCESS + EXCLUSIVE`, and refuses a parent with a DEFAULT partition. For a `timestamp` + key, "now" is the session's local time. The arms found that the first version + read a `timestamp` key's bounds in the session's zone and wrote them from UTC, + which left a four-hour gap between partitions in New York and started daily + partitions at 15:00 in Tokyo. `test/partition_lifecycle.sh` and its pytest + twin hold each statement in sessions west and east of UTC. ### Changed diff --git a/test/partition_lifecycle.sh b/test/partition_lifecycle.sh index 84c8ed8f..78d4ddea 100755 --- a/test/partition_lifecycle.sh +++ b/test/partition_lifecycle.sh @@ -73,7 +73,7 @@ check_num "asking further ahead creates only the difference" \ # ---- rows go in, columnar ------------------------------------------------------ q "INSERT INTO tl SELECT now() + g * interval '1 hour', g FROM generate_series(0, 100) g;" >/dev/null -check_num "premise: rows within the covered range go in" "$(q "SELECT count(*) FROM tl;")" "101" +check_num "rows within the covered range go in" "$(q "SELECT count(*) FROM tl;")" "101" check "and land in columnar partitions" \ "$(q "SELECT string_agg(DISTINCT a.amname, ',') FROM tl t JOIN pg_class c ON c.oid = t.tableoid JOIN pg_am a ON a.oid = c.relam;")" "pgcolumnar" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 89241958..14de6071 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -138,6 +138,7 @@ behaviour, the source of that number is named. - [90. test_partman_columnar.py: pg_partman makes columnar partitions with one setting](#90-test_partman_columnarpy-pg_partman-makes-columnar-partitions-with-one-setting) - [91. test_capability_decline.py: a missing pyarrow declines by name, and nothing declines silently](#91-test_capability_declinepy-a-missing-pyarrow-declines-by-name-and-nothing-declines-silently) - [92. test_tid_range_scan.py: a ctid range reads only the row groups that hold it](#92-test_tid_range_scanpy-a-ctid-range-reads-only-the-row-groups-that-hold-it) +- [93. test_partition_lifecycle.py: partition_create_ahead keeps a time-partitioned parent ahead of now](#93-test_partition_lifecyclepy-partition_create_ahead-keeps-a-time-partitioned-parent-ahead-of-now) ## 1. How to read a test in here @@ -6518,3 +6519,19 @@ THE MATERIAL'S NODE ID. `set_plan_references` numbers the main tree and then the | --- | --- | | `test_a_ctid_range_reads_only_its_row_groups` | the table has its rows and many groups; a ctid range plans a TID range scan; eleven ranges, including offsets 0, 291, 500 and 65535, return what the per-row filter returns; a rescanned scan does too; a SCROLL cursor and a plain cursor fetch backward correctly, and so does a SCROLL cursor whose query has a subplan; rows inserted earlier in the transaction are in the range; the tail range reads at least one block and under a fifth of the table; deleted rows stay deleted; after compaction moves rows, ranges over the old and new addresses return what the filter returns | | `test_a_parallel_tid_range_scan_returns_what_the_filter_returns` | PostgreSQL 19+: the range runs as a parallel TID range scan with two workers launched, and returns what the filter returns. Below 19, `UNSUPPORTED_MAJOR` | + +## 93. test_partition_lifecycle.py: partition_create_ahead keeps a time-partitioned parent ahead of now + +Port of `partition_lifecycle.sh` (#1297). `pgcolumnar.partition_create_ahead(parent, width, ahead)` makes columnar partitions, each `width` wide and contiguous with the highest existing upper bound, until they reach now plus `ahead` widths. It uses a standalone CREATE then ATTACH, so the parent is held SHARE UPDATE EXCLUSIVE, never ACCESS EXCLUSIVE. A DEFAULT partition is refused (`55000`). + +THE SESSION'S ZONE IS WHERE THE PROTOTYPE BROKE. A `timestamp` key holds wall-clock time. The prototype read its bounds as timestamptz in the session's zone and wrote the next ones converted from UTC. In America/New_York that left a four-hour gap between partitions, so a row at the boundary had nowhere to go; and daily partitions made from Tokyo started at 15:00. The arms continue from hand-made partitions in sessions west and east of UTC, and check contiguity, routing at every boundary, coverage of the session's local now, and alignment to the width in the key's own wall clock. This file uses six-hour widths, and Asia/Kolkata for a half-hour offset. + +Four inputs from @jdatcmd's review of #1308, each an arm red against the first version: a stored generated column (LIKE without INCLUDING GENERATED, `42804` at ATTACH); a 63-character name (the date suffix truncated and the second partition collided; the twin uses 31 two-byte characters so truncation must never split one); a call creating more than 1000 partitions (it ran 11 minutes and failed `53200`; now refused with `22023` before any DDL); and a MAXVALUE bound (an overlap; now refused with `55000`). + +Removal proofs, each red in both twins: the prototype's zone handling; `CREATE TABLE ... PARTITION OF` (the lock arm); no DEFAULT refusal; starting from now rather than the highest bound (a second call fails); heap partitions; stopping one width short (the coverage arms); a minute's gap between partitions (the contiguity and boundary arms); and one mutant per #1308 fix. The `one_short` mutation also reddened "rows within the covered range go in", so that check is not a premise (#1236). + +### Every test + +| test | what it holds | +| --- | --- | +| `test_partition_create_ahead` | from nothing it reaches now plus the lead, columnar, contiguous and one width wide; a second call creates nothing, and asking further ahead creates only the difference; rows land in columnar partitions; the parent is never held ACCESS EXCLUSIVE; west of UTC, for both key types, partitions continue a hand-made one contiguously and a row at every boundary routes; east of UTC, a timestamp parent covers the local now and starts on the width's boundary; inputs that broke the first version (a generated column, a 63-character name, more than 1000 partitions, a MAXVALUE bound) are handled or refused before any DDL; a DEFAULT partition is refused and nothing is created | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 415781ff..e629d47c 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -556,4 +556,5 @@ guard_tests 413 # collected over the cluster files on the tree rebased onto 503f080 (#1286). # 498 -> 500 for the TID range scan pair: test_tid_range_scan.py's two tests. # Collected on this tree, after #1304. -cluster_tests 500 +# 500 -> 501 for #1297's test_partition_lifecycle.py, collected on this tree. +cluster_tests 501 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 6262d944..ff14f7c8 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -117,6 +117,7 @@ "native_vacuum_race", "parallel_am_scan", "parallel_scan_cost", + "partition_lifecycle", "partman_columnar", "projection_drop_column", "projection_parallel", diff --git a/test/pytest/test_partition_lifecycle.py b/test/pytest/test_partition_lifecycle.py index 46ebd86c..7d14d084 100644 --- a/test/pytest/test_partition_lifecycle.py +++ b/test/pytest/test_partition_lifecycle.py @@ -109,7 +109,7 @@ def test_partition_create_ahead(pgc_conn, expect): with conn.cursor() as cur: cur.execute(f"INSERT INTO gauge SELECT now() + g * interval '17 min', g" f" FROM generate_series(0, 77) g") - expect.num(_one(conn, "SELECT count(*) FROM gauge"), 78, "premise: rows within the covered range go in") + expect.num(_one(conn, "SELECT count(*) FROM gauge"), 78, "rows within the covered range go in") expect.text(_one(conn, "SELECT string_agg(DISTINCT a.amname, ',') FROM gauge t" " JOIN pg_class c ON c.oid = t.tableoid JOIN pg_am a ON a.oid = c.relam"), "pgcolumnar", "and land in columnar partitions") diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 4241aed4..01a9e329 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -248,6 +248,7 @@ SUITES=( parquet_level_width parquet_nested parquet_nested_import + partition_lifecycle partman_columnar pg19_vacuum_options pg_dump_roundtrip From dba7f5b1206627b7cdae9104f841da18a087ecbf Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 23:57:45 +0000 Subject: [PATCH 4/5] fix: partition_create_ahead handles generated columns, long names, a work bound and MAXVALUE (#1308 review) Four inputs @jdatcmd measured breaking the function, each an arm red against 0d8e3f7 on PG17 for the reason measured: - a stored generated column: LIKE now INCLUDING GENERATED (was 42804 at ATTACH); - a 63-character parent: the prefix is shortened by whole characters until prefix + the 16-byte date fits in 63 bytes (was a truncated name colliding); - no bound on the work: a dry run counts the partitions before any DDL and refuses more than 1000 with 22023 (one call ran 11 minutes, then 53200); - a MAXVALUE upper bound: refused with 55000 (was an overlap, 42P17). The pytest twin's long name is 31 two-byte characters, so truncation must never split one. Both twins 27/27 on PG15-19; one mutant per fix, red in both. Co-Authored-By: Claude Opus 5.5 --- pgcolumnar--1.0-alpha4--1.0-alpha5.sql | 55 +++++++++++++++++++++++-- pgcolumnar--1.0-alpha5.sql | 55 +++++++++++++++++++++++-- test/partition_lifecycle.sh | 40 ++++++++++++++++++ test/pytest/test_partition_lifecycle.py | 43 +++++++++++++++++++ 4 files changed, 185 insertions(+), 8 deletions(-) diff --git a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql index 03eccbbd..a96350a2 100644 --- a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql @@ -827,6 +827,10 @@ DECLARE ts_hi timestamp; ts_target timestamp; child text; + prefix text; + n int := 0; + probe_ts timestamp; + probe_tz timestamptz; made int := 0; BEGIN IF width IS NULL OR width <= interval '0' OR ahead IS NULL OR ahead < 0 THEN @@ -851,9 +855,26 @@ BEGIN USING ERRCODE = '55000', HINT = 'A DEFAULT partition rules out DETACH CONCURRENTLY, so the partition lifecycle refuses to manage this table.'; END IF; + -- An upper bound of MAXVALUE leaves no room ahead, and the bound reader below + -- would not see it: the first new partition would overlap it (#1308 review). + IF EXISTS (SELECT 1 FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = parent + AND pg_get_expr(c.relpartbound, c.oid) ~ 'TO \(MAXVALUE\)') THEN + RAISE EXCEPTION '% has a partition bounded by MAXVALUE', parent + USING ERRCODE = '55000', + HINT = 'Nothing can be created after a partition that reaches MAXVALUE.'; + END IF; SELECT n.nspname, c.relname INTO nsp, rel FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.oid = parent; + -- Names are the parent's name and a 16-byte date suffix, and must fit in 63 + -- bytes: the prefix is shortened, by characters, until they do, so the date + -- always survives (#1308 review: a 63-character parent's second partition + -- truncated onto its first). + prefix := rel; + WHILE octet_length(prefix) + 16 > 63 LOOP + prefix := left(prefix, -1); + END LOOP; /* * A `timestamp` key holds wall-clock time with no zone, so it is reasoned @@ -877,10 +898,23 @@ BEGIN END IF; END IF; ts_target := localtimestamp + ahead * width; + probe_ts := ts_lo; + WHILE probe_ts <= ts_target AND n <= 1000 LOOP + n := n + 1; + probe_ts := probe_ts + width; + END LOOP; + -- Counted before any DDL: one call creating thousands of partitions holds + -- their locks to the end and can fail on the lock table with nothing made + -- (#1308 review: '1 second' x 20000 ran 6 minutes, then out of shared memory). + IF n > 1000 THEN + RAISE EXCEPTION 'partition_create_ahead would create more than 1000 partitions of %', parent + USING ERRCODE = '22023', + HINT = 'Use a wider width, a shorter lead, or several calls.'; + END IF; WHILE ts_lo <= ts_target LOOP ts_hi := ts_lo + width; - child := rel || '_' || to_char(ts_lo, 'YYYYMMDD"T"HH24MISS'); - EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS) USING pgcolumnar', + child := prefix || '_' || to_char(ts_lo, 'YYYYMMDD"T"HH24MISS'); + EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING GENERATED) USING pgcolumnar', nsp, child, parent); EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I FOR VALUES FROM (%L) TO (%L)', parent, nsp, child, ts_lo::text, ts_hi::text); @@ -903,10 +937,23 @@ BEGIN END IF; END IF; target := now() + ahead * width; + probe_tz := lo; + WHILE probe_tz <= target AND n <= 1000 LOOP + n := n + 1; + probe_tz := probe_tz + width; + END LOOP; + -- Counted before any DDL: one call creating thousands of partitions holds + -- their locks to the end and can fail on the lock table with nothing made + -- (#1308 review: '1 second' x 20000 ran 6 minutes, then out of shared memory). + IF n > 1000 THEN + RAISE EXCEPTION 'partition_create_ahead would create more than 1000 partitions of %', parent + USING ERRCODE = '22023', + HINT = 'Use a wider width, a shorter lead, or several calls.'; + END IF; WHILE lo <= target LOOP hi := lo + width; - child := rel || '_' || to_char(lo AT TIME ZONE 'UTC', 'YYYYMMDD"T"HH24MISS'); - EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS) USING pgcolumnar', + child := prefix || '_' || to_char(lo AT TIME ZONE 'UTC', 'YYYYMMDD"T"HH24MISS'); + EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING GENERATED) USING pgcolumnar', nsp, child, parent); EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I FOR VALUES FROM (%L) TO (%L)', parent, nsp, child, lo::text, hi::text); diff --git a/pgcolumnar--1.0-alpha5.sql b/pgcolumnar--1.0-alpha5.sql index 797051ab..6f5cad91 100644 --- a/pgcolumnar--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha5.sql @@ -2016,6 +2016,10 @@ DECLARE ts_hi timestamp; ts_target timestamp; child text; + prefix text; + n int := 0; + probe_ts timestamp; + probe_tz timestamptz; made int := 0; BEGIN IF width IS NULL OR width <= interval '0' OR ahead IS NULL OR ahead < 0 THEN @@ -2040,9 +2044,26 @@ BEGIN USING ERRCODE = '55000', HINT = 'A DEFAULT partition rules out DETACH CONCURRENTLY, so the partition lifecycle refuses to manage this table.'; END IF; + -- An upper bound of MAXVALUE leaves no room ahead, and the bound reader below + -- would not see it: the first new partition would overlap it (#1308 review). + IF EXISTS (SELECT 1 FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid + WHERE i.inhparent = parent + AND pg_get_expr(c.relpartbound, c.oid) ~ 'TO \(MAXVALUE\)') THEN + RAISE EXCEPTION '% has a partition bounded by MAXVALUE', parent + USING ERRCODE = '55000', + HINT = 'Nothing can be created after a partition that reaches MAXVALUE.'; + END IF; SELECT n.nspname, c.relname INTO nsp, rel FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.oid = parent; + -- Names are the parent's name and a 16-byte date suffix, and must fit in 63 + -- bytes: the prefix is shortened, by characters, until they do, so the date + -- always survives (#1308 review: a 63-character parent's second partition + -- truncated onto its first). + prefix := rel; + WHILE octet_length(prefix) + 16 > 63 LOOP + prefix := left(prefix, -1); + END LOOP; /* * A `timestamp` key holds wall-clock time with no zone, so it is reasoned @@ -2066,10 +2087,23 @@ BEGIN END IF; END IF; ts_target := localtimestamp + ahead * width; + probe_ts := ts_lo; + WHILE probe_ts <= ts_target AND n <= 1000 LOOP + n := n + 1; + probe_ts := probe_ts + width; + END LOOP; + -- Counted before any DDL: one call creating thousands of partitions holds + -- their locks to the end and can fail on the lock table with nothing made + -- (#1308 review: '1 second' x 20000 ran 6 minutes, then out of shared memory). + IF n > 1000 THEN + RAISE EXCEPTION 'partition_create_ahead would create more than 1000 partitions of %', parent + USING ERRCODE = '22023', + HINT = 'Use a wider width, a shorter lead, or several calls.'; + END IF; WHILE ts_lo <= ts_target LOOP ts_hi := ts_lo + width; - child := rel || '_' || to_char(ts_lo, 'YYYYMMDD"T"HH24MISS'); - EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS) USING pgcolumnar', + child := prefix || '_' || to_char(ts_lo, 'YYYYMMDD"T"HH24MISS'); + EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING GENERATED) USING pgcolumnar', nsp, child, parent); EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I FOR VALUES FROM (%L) TO (%L)', parent, nsp, child, ts_lo::text, ts_hi::text); @@ -2092,10 +2126,23 @@ BEGIN END IF; END IF; target := now() + ahead * width; + probe_tz := lo; + WHILE probe_tz <= target AND n <= 1000 LOOP + n := n + 1; + probe_tz := probe_tz + width; + END LOOP; + -- Counted before any DDL: one call creating thousands of partitions holds + -- their locks to the end and can fail on the lock table with nothing made + -- (#1308 review: '1 second' x 20000 ran 6 minutes, then out of shared memory). + IF n > 1000 THEN + RAISE EXCEPTION 'partition_create_ahead would create more than 1000 partitions of %', parent + USING ERRCODE = '22023', + HINT = 'Use a wider width, a shorter lead, or several calls.'; + END IF; WHILE lo <= target LOOP hi := lo + width; - child := rel || '_' || to_char(lo AT TIME ZONE 'UTC', 'YYYYMMDD"T"HH24MISS'); - EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS) USING pgcolumnar', + child := prefix || '_' || to_char(lo AT TIME ZONE 'UTC', 'YYYYMMDD"T"HH24MISS'); + EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING GENERATED) USING pgcolumnar', nsp, child, parent); EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I FOR VALUES FROM (%L) TO (%L)', parent, nsp, child, lo::text, hi::text); diff --git a/test/partition_lifecycle.sh b/test/partition_lifecycle.sh index 78d4ddea..79806801 100755 --- a/test/partition_lifecycle.sh +++ b/test/partition_lifecycle.sh @@ -35,6 +35,12 @@ check_num "premise: the function exists" \ "$(q "SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace WHERE n.nspname = 'pgcolumnar' AND p.proname = 'partition_create_ahead';")" "1" +sqlstate_or() { # sqlstate_or SQL -> the single value it returns, or the SQLSTATE it fails with + local out + out=$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -X -At -v ON_ERROR_STOP=1 -v VERBOSITY=sqlstate -c "$1" 2>&1) + case "$out" in ERROR:*) sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p' <<<"$out" | head -1;; *) echo "$out";; esac +} kids() { q "SELECT count(*) FROM pg_inherits WHERE inhparent = '$1'::regclass;"; } ams() { q "SELECT coalesce(string_agg(DISTINCT a.amname, ','), 'none') FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid JOIN pg_am a ON a.oid = c.relam @@ -137,6 +143,40 @@ check "a timestamp parent made east of UTC covers the local now plus the lead" \ check "and its partitions start on the width's boundary in the key's own wall clock" \ "$(bounds tz_tokyo | awk -F'|' '$1 !~ / 00:00:00$/ {print "not midnight: " $1; bad=1; exit} END {if (!bad) print "ok"}')" "ok" +# ---- inputs that broke the first version (@jdatcmd, #1308 review) --------------- +# A stored generated column: LIKE without INCLUDING GENERATED made it a plain +# column, and ATTACH refused the partition. +q "CREATE TABLE gen (ts timestamptz NOT NULL, v int, w int GENERATED ALWAYS AS (v * 2) STORED) + PARTITION BY RANGE (ts);" >/dev/null +gen_made="$(sqlstate_or "SELECT pgcolumnar.partition_create_ahead('gen', '1 day', 1);")" +check "a parent with a generated column gets its partitions, and the column stays generated" \ + "$gen_made|$(q "INSERT INTO gen VALUES (now(), 21); SELECT w FROM gen;" | tail -1)" "2|42" + +# A 63-character name: the 16-character date suffix must survive, or the second +# partition's name truncates onto the first's. +LONG="p_$(printf 'x%.0s' $(seq 1 61))" +q "CREATE TABLE $LONG (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts);" >/dev/null +long_made="$(sqlstate_or "SELECT pgcolumnar.partition_create_ahead('$LONG', '1 day', 2);")" +check "a parent with a 63-character name gets partitions whose names keep their dates" \ + "$long_made|$(q "SELECT count(*) FILTER (WHERE c.relname ~ '_[0-9]{8}T[0-9]{6}\$' AND octet_length(c.relname) <= 63) + FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = '$LONG'::regclass;")" "3|3" + +# No bound on the work: one call with a tiny width ran for minutes holding locks, +# then failed on the lock table with nothing created. Refused before any DDL. +q "CREATE TABLE many (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts);" >/dev/null +t0=$(date +%s%N) +many="$(sqlstate_or "SELECT pgcolumnar.partition_create_ahead('many', '1 second', 20000);")" +many_ms=$(( ($(date +%s%N) - t0) / 1000000 )) +check "a call that would create more than 1000 partitions is refused before any DDL" \ + "$many|$(kids many)|$([ "$many_ms" -lt 5000 ] && echo prompt || echo "took $many_ms ms")" "22023|0|prompt" + +# A partition bounded by MAXVALUE: the bound reader ignored it, and the first new +# partition overlapped it. +q "CREATE TABLE tail (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts); + CREATE TABLE tail_rest PARTITION OF tail FOR VALUES FROM ('2026-01-01') TO (MAXVALUE) USING pgcolumnar;" >/dev/null +check "a parent with a partition bounded by MAXVALUE is refused, and nothing is created" \ + "$(sqlstate_or "SELECT pgcolumnar.partition_create_ahead('tail', '1 day', 2);")|$(kids tail)" "55000|1" + # ---- refused: a DEFAULT partition ------------------------------------------------- q "CREATE TABLE td (ts timestamptz NOT NULL, v int) PARTITION BY RANGE (ts); CREATE TABLE td_default PARTITION OF td DEFAULT USING pgcolumnar;" >/dev/null diff --git a/test/pytest/test_partition_lifecycle.py b/test/pytest/test_partition_lifecycle.py index 7d14d084..7cfad89a 100644 --- a/test/pytest/test_partition_lifecycle.py +++ b/test/pytest/test_partition_lifecycle.py @@ -20,6 +20,7 @@ """ import re +import time import psycopg @@ -176,6 +177,48 @@ def test_partition_create_ahead(pgc_conn, expect): with conn.cursor() as cur: cur.execute("SET TimeZone = 'UTC'") + # Inputs that broke the first version (@jdatcmd, #1308 review). + def value_or_state(sql): + try: + return str(_one(conn, sql)) + except psycopg.Error as e: + return e.sqlstate or "?" + + with conn.cursor() as cur: + cur.execute("CREATE TABLE gen (at timestamptz NOT NULL, reading float8," + " doubled float8 GENERATED ALWAYS AS (reading * 2) STORED) PARTITION BY RANGE (at)") + gen = value_or_state("SELECT pgcolumnar.partition_create_ahead('gen', '6 hours', 2)") + doubled = value_or_state("INSERT INTO gen (at, reading) VALUES (now(), 2.25) RETURNING doubled") + expect.text(f"{gen}|{doubled}", "3|4.5", + "a parent with a generated column gets its partitions, and the column stays generated") + + # 31 two-byte characters, 62 bytes: the date suffix must still fit, so the + # prefix is shortened by whole characters, never by a broken byte. + long_name = "\u00e9" * 31 + with conn.cursor() as cur: + cur.execute(f'CREATE TABLE "{long_name}" (at timestamptz NOT NULL, reading float8) PARTITION BY RANGE (at)') + long_made = value_or_state(f"""SELECT pgcolumnar.partition_create_ahead('"{long_name}"', '6 hours', 2)""") + kept = _one(conn, "SELECT count(*) FILTER (WHERE c.relname ~ '_[0-9]{8}T[0-9]{6}$' AND octet_length(c.relname) <= 63)" + f""" FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = '"{long_name}"'::regclass""") + expect.text(f"{long_made}|{kept}", "3|3", + "a parent with a 63-character name gets partitions whose names keep their dates") + + with conn.cursor() as cur: + cur.execute("CREATE TABLE swarm (at timestamptz NOT NULL, reading float8) PARTITION BY RANGE (at)") + t0 = time.monotonic() + swarm = value_or_state("SELECT pgcolumnar.partition_create_ahead('swarm', '1 minute', 5000)") + ms = int((time.monotonic() - t0) * 1000) + expect.text(f"{swarm}|{_kids(conn, 'swarm')}|{'prompt' if ms < 5000 else f'took {ms} ms'}", "22023|0|prompt", + "a call that would create more than 1000 partitions is refused before any DDL") + + with conn.cursor() as cur: + cur.execute("CREATE TABLE capped (at timestamptz NOT NULL, reading float8) PARTITION BY RANGE (at)") + cur.execute("CREATE TABLE capped_rest PARTITION OF capped FOR VALUES FROM ('2025-06-01') TO (MAXVALUE)" + " USING pgcolumnar") + capped = value_or_state("SELECT pgcolumnar.partition_create_ahead('capped', '1 day', 2)") + expect.text(f"{capped}|{_kids(conn, 'capped')}", "55000|1", + "a parent with a partition bounded by MAXVALUE is refused, and nothing is created") + # Refused: a DEFAULT partition. with conn.cursor() as cur: cur.execute("CREATE TABLE fallback (at timestamptz NOT NULL, reading float8) PARTITION BY RANGE (at)") From 9dcbd59d1d55d3a6d4bb0b672c49e914a784e8cc Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 00:00:49 +0000 Subject: [PATCH 5/5] test: partition_lifecycle ledger replayed after #1312, with the #1308 review's arms 27 rows from a five-major run, 21 dated by eleven mutations merged one log at a time with --mutation and --target; the six 'never' rows are premises. Census 1694 -> 1700. TESTS.md section 93; cluster_tests 501 by collection. Docs and CHANGELOG name the MAXVALUE and 1000-partition refusals. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 +++++- docs/how-to.md | 4 +++- docs/sql-reference.md | 4 +++- test/check_ledger.tsv | 27 +++++++++++++++++++++++++++ test/check_ledger_budget.txt | 15 ++++++++++++++- 5 files changed, 52 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bc22556..70c38a4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -99,7 +99,11 @@ true until the next version shipped. read a `timestamp` key's bounds in the session's zone and wrote them from UTC, which left a four-hour gap between partitions in New York and started daily partitions at 15:00 in Tokyo. `test/partition_lifecycle.sh` and its pytest - twin hold each statement in sessions west and east of UTC. + twin hold each statement in sessions west and east of UTC. It also copes with a + stored generated column on the parent and with a parent name up to 63 bytes + long, and refuses, before creating anything, a call that would create more + than 1000 partitions (`22023`) and a parent with a partition bounded by + `MAXVALUE` (`55000`). ### Changed diff --git a/docs/how-to.md b/docs/how-to.md index 7d65cdf9..bb670b9c 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -347,7 +347,9 @@ queries and inserts on the table continue while it runs. For a `timestamp` key, which has no time zone, "now" is the session's local time, and partitions start on the width's boundary in that time. -The function refuses a table that has a DEFAULT partition. +The function refuses a table that has a DEFAULT partition, or a partition +bounded by `MAXVALUE`. It also refuses a call that would create more than 1000 +partitions, before it creates any. Use a wider width or a shorter lead. `test/partition_lifecycle.sh` checks each of these statements, in sessions west and east of UTC. diff --git a/docs/sql-reference.md b/docs/sql-reference.md index 2ea9778e..eef7f612 100644 --- a/docs/sql-reference.md +++ b/docs/sql-reference.md @@ -577,7 +577,9 @@ plus `ahead` widths. Returns the number of partitions created. A second call creates none. The first new partition starts at the highest existing upper bound. With no partitions, it starts at the width's boundary that holds now. `parent` must be partitioned by RANGE on one `timestamp` or `timestamptz` -column, and must have no DEFAULT partition (`55000`). The function takes +column. It must have no DEFAULT partition and no partition bounded by +`MAXVALUE` (`55000`). A call that would create more than 1000 partitions is +refused before it creates any (`22023`). The function takes `SHARE UPDATE EXCLUSIVE` on the parent, never `ACCESS EXCLUSIVE`. See [Create time partitions ahead](how-to.md#create-time-partitions-ahead). diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index c35b6484..de2dd168 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -1771,6 +1771,33 @@ parallel_scan_cost parallel_scan_cost premise: the serial plan has no Gather 15; parallel_scan_cost parallel_scan_cost premise: the serial plan is a columnar scan 15;16;17;18;19 never - parallel_scan_cost parallel_scan_cost premise: the table holds every inserted row 15;16;17;18;19 never - parallel_scan_cost parallel_scan_cost the leader-participation branch changes the divisor 15;16;17;18;19 never - +partition_lifecycle partition_lifecycle a call that would create more than 1000 partitions is refused before any DDL 15;16;17;18;19 2026-10-12 partition_create_ahead without the 1000-partition bound +partition_lifecycle partition_lifecycle a parent with a 63-character name gets partitions whose names keep their dates 15;16;17;18;19 2026-10-12 partition_create_ahead stopping one width short of now plus the lead;partition_create_ahead without shortening a long parent name +partition_lifecycle partition_lifecycle a parent with a DEFAULT partition is refused 15;16;17;18;19 2026-10-12 partition_create_ahead without the DEFAULT-partition refusal +partition_lifecycle partition_lifecycle a parent with a generated column gets its partitions, and the column stays generated 15;16;17;18;19 2026-10-12 partition_create_ahead stopping one width short of now plus the lead;partition_create_ahead without INCLUDING GENERATED +partition_lifecycle partition_lifecycle a parent with a partition bounded by MAXVALUE is refused, and nothing is created 15;16;17;18;19 2026-10-12 partition_create_ahead without the MAXVALUE refusal +partition_lifecycle partition_lifecycle a second call creates nothing 15;16;17;18;19 2026-10-12 partition_create_ahead starting from now, not the highest upper bound +partition_lifecycle partition_lifecycle a timestamp parent made east of UTC covers the local now plus the lead 15;16;17;18;19 2026-10-12 partition_create_ahead stopping one width short of now plus the lead +partition_lifecycle partition_lifecycle a timestamp parent's partitions are contiguous across a non-UTC session 15;16;17;18;19 2026-10-12 partition_create_ahead leaving a minute between partitions;partition_create_ahead reading a timestamp key's bounds in the session's zone and writing them from UTC (the prototype);partition_create_ahead starting from now, not the highest upper bound +partition_lifecycle partition_lifecycle a timestamptz parent's partitions are contiguous across a non-UTC session 15;16;17;18;19 2026-10-12 partition_create_ahead leaving a minute between partitions;partition_create_ahead starting from now, not the highest upper bound +partition_lifecycle partition_lifecycle and a row at every boundary has exactly one partition to go to (timestamp) 15;16;17;18;19 2026-10-12 partition_create_ahead leaving a minute between partitions;partition_create_ahead reading a timestamp key's bounds in the session's zone and writing them from UTC (the prototype);partition_create_ahead starting from now, not the highest upper bound +partition_lifecycle partition_lifecycle and a row at every boundary has exactly one partition to go to (timestamptz) 15;16;17;18;19 2026-10-12 partition_create_ahead leaving a minute between partitions;partition_create_ahead starting from now, not the highest upper bound +partition_lifecycle partition_lifecycle and its partitions start on the width's boundary in the key's own wall clock 15;16;17;18;19 2026-10-12 partition_create_ahead leaving a minute between partitions;partition_create_ahead reading a timestamp key's bounds in the session's zone and writing them from UTC (the prototype) +partition_lifecycle partition_lifecycle and land in columnar partitions 15;16;17;18;19 2026-10-12 partition_create_ahead creating heap partitions;partition_create_ahead starting from now, not the highest upper bound;partition_create_ahead stopping one width short of now plus the lead +partition_lifecycle partition_lifecycle and nothing was created for it 15;16;17;18;19 2026-10-12 partition_create_ahead without the DEFAULT-partition refusal +partition_lifecycle partition_lifecycle and they cover now() plus the lead 15;16;17;18;19 2026-10-12 partition_create_ahead stopping one width short of now plus the lead +partition_lifecycle partition_lifecycle asking further ahead creates only the difference 15;16;17;18;19 2026-10-12 partition_create_ahead starting from now, not the highest upper bound +partition_lifecycle partition_lifecycle every partition it creates is columnar 15;16;17;18;19 2026-10-12 partition_create_ahead creating heap partitions +partition_lifecycle partition_lifecycle from no partitions it creates enough to reach now plus the lead 15;16;17;18;19 2026-10-12 partition_create_ahead stopping one width short of now plus the lead +partition_lifecycle partition_lifecycle it never takes AccessExclusiveLock on the parent 15;16;17;18;19 2026-10-12 partition_create_ahead using CREATE TABLE ... PARTITION OF +partition_lifecycle partition_lifecycle premise: a timestamp parent continued from a hand-made partition, in a session west of UTC 15;16;17;18;19 never - +partition_lifecycle partition_lifecycle premise: a timestamp parent was given partitions from a session east of UTC 15;16;17;18;19 never - +partition_lifecycle partition_lifecycle premise: a timestamptz parent continued from a hand-made partition, in a session west of UTC 15;16;17;18;19 never - +partition_lifecycle partition_lifecycle premise: the call inside the transaction created partitions 15;16;17;18;19 never - +partition_lifecycle partition_lifecycle premise: the function exists 15;16;17;18;19 never - +partition_lifecycle partition_lifecycle premise: the parent now has the partitions it reported 15;16;17;18;19 never - +partition_lifecycle partition_lifecycle rows within the covered range go in 15;16;17;18;19 2026-10-12 partition_create_ahead starting from now, not the highest upper bound;partition_create_ahead stopping one width short of now plus the lead +partition_lifecycle partition_lifecycle they are contiguous, each one width wide 15;16;17;18;19 2026-10-12 partition_create_ahead leaving a minute between partitions partman_columnar partman_columnar and no row-group row behind 15;16;17;18;19 2026-10-12 the drop hook stops deleting row_group rows, so they outlive their deleted storage row partman_columnar partman_columnar and read back as heap does 15;16;17;18;19 2026-10-12 a sequential scan drops the last row group from its list, so every one-group partition reads empty partman_columnar partman_columnar and the rows land in columnar partitions 15;16;17;18;19 2026-10-12 a sequential scan drops the last row group from its list, so every one-group partition reads empty diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 085e02c2..fa32e47d 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1319,4 +1319,17 @@ suites_not_covered 247 # ledger rows 1943 + 2 = 1945 # never 1693 + 1 = 1694 # dated 250 + 1 = 251 -checks_never_observed_red 1694 +# +# #1297 / PR #1308 (OffgridwithJD): 1694 -> 1700. partition_lifecycle enters the +# ledger with 27 checks from a five-major run. Twenty-one are dated by eleven +# mutations of partition_create_ahead (seven from the first round, four from the +# #1308 review's fixes), each merged one log at a time with --mutation and +# --target. The six that arrive `never` are all premises. The logs ran on the +# tree before #1306/#1312 merged; those touched only columnar_tableam.c's TID +# range and cursor paths, which this suite does not reach. +# +# ledger rows 1945 + 27 = 1972 +# never 1694 + 6 = 1700 +# dated 251 + 21 = 272 +# 1700 + 272 == 1972 +checks_never_observed_red 1700