feat: partition_create_ahead keeps a time-partitioned parent ahead of now (#1297)

CHANGELOG.md

@@ -89,6 +89,21 @@ true until the next version shipped.
   scan backward, and without it the fetch reached the access method and failed.
   The Material's plan node id now sits above the subplans' ids as well, so a
   cursor whose query has a subquery no longer repeats a node id.
+- `pgcolumnar.partition_create_ahead(parent, width, ahead)` keeps a
+  time-partitioned table ready for the rows that are coming (#1297). It creates
+  columnar partitions, each `width` wide and contiguous with the highest existing
+  upper bound, until they reach now plus `ahead` widths; a second call creates
+  none. It takes `SHARE UPDATE EXCLUSIVE` on the parent, never `ACCESS
+  EXCLUSIVE`, and refuses a parent with a DEFAULT partition. For a `timestamp`
+  key, "now" is the session's local time. The arms found that the first version
+  read a `timestamp` key's bounds in the session's zone and wrote them from UTC,
+  which left a four-hour gap between partitions in New York and started daily
+  partitions at 15:00 in Tokyo. `test/partition_lifecycle.sh` and its pytest
+  twin hold each statement in sessions west and east of UTC. It also copes with a
+  stored generated column on the parent and with a parent name up to 63 bytes
+  long, and refuses, before creating anything, a call that would create more
+  than 1000 partitions (`22023`) and a parent with a partition bounded by
+  `MAXVALUE` (`55000`).
 
 ### Changed
 

docs/how-to.md

@@ -323,6 +323,37 @@ declared retention raises an error rather than doing nothing.
 `pgcolumnar.reset_options` has no argument for the retention. To stop expiring,
 set `ttl_interval` to a span no row will reach.
 
+## Create time partitions ahead
+
+`pgcolumnar.partition_create_ahead` keeps a time-partitioned table ready for
+the rows that are coming. Call it from a scheduler, such as pg_cron, more often
+than the width:
+
+```sql
+CREATE TABLE metrics (ts timestamptz NOT NULL, device int, value float8)
+    PARTITION BY RANGE (ts);
+
+-- Daily partitions, up to three days ahead of now.
+SELECT pgcolumnar.partition_create_ahead('metrics', '1 day', 3);
+```
+
+Every partition that it creates is columnar. It continues from the highest
+existing upper bound, so the partitions stay contiguous, and a second call
+creates nothing.
+
+It takes `SHARE UPDATE EXCLUSIVE` on the parent, not `ACCESS EXCLUSIVE`, so
+queries and inserts on the table continue while it runs.
+
+For a `timestamp` key, which has no time zone, "now" is the session's local
+time, and partitions start on the width's boundary in that time.
+
+The function refuses a table that has a DEFAULT partition, or a partition
+bounded by `MAXVALUE`. It also refuses a call that would create more than 1000
+partitions, before it creates any. Use a wider width or a shorter lead.
+
+`test/partition_lifecycle.sh` checks each of these statements, in sessions
+west and east of UTC.
+
 ## Manage time partitions with pg_partman
 
 pg_partman can create and drop columnar partitions. It needs one setting to do

docs/sql-reference.md

@@ -570,6 +570,19 @@ Writes the live rows of `rel` to a Parquet file at `path`. Returns the number of
 rows written. `path` can be a local file or an `s3://` URL. See
 [Object storage](#object-storage).
 
+### pgcolumnar.partition_create_ahead(parent regclass, width interval, ahead int) returns int
+
+Creates columnar partitions of `parent`, each `width` wide, until they reach now
+plus `ahead` widths. Returns the number of partitions created. A second call
+creates none. The first new partition starts at the highest existing upper
+bound. With no partitions, it starts at the width's boundary that holds now.
+`parent` must be partitioned by RANGE on one `timestamp` or `timestamptz`
+column. It must have no DEFAULT partition and no partition bounded by
+`MAXVALUE` (`55000`). A call that would create more than 1000 partitions is
+refused before it creates any (`22023`). The function takes
+`SHARE UPDATE EXCLUSIVE` on the parent, never `ACCESS EXCLUSIVE`. See
+[Create time partitions ahead](how-to.md#create-time-partitions-ahead).
+
 ### pgcolumnar.import_arrow(rel regclass, path text) returns bigint
 
 Inserts the rows of an Arrow IPC stream file at `path` into the existing table

pgcolumnar--1.0-alpha4--1.0-alpha5.sql

@@ -791,3 +791,175 @@ BEGIN
 	END IF;
 END;
 $$;
+
+/*
+ * pgcolumnar.partition_create_ahead (#1297)
+ *
+ * Make sure a RANGE-partitioned parent's partitions reach now() + ahead * width,
+ * each `width` wide and contiguous with the highest existing upper bound (or,
+ * with none, starting at now() aligned to the width), every one columnar.
+ * Returns how many it created; a second call creates nothing.
+ *
+ * NEVER AN EXCLUSIVE LOCK ON THE PARENT. CREATE TABLE ... PARTITION OF takes
+ * ACCESS EXCLUSIVE on the parent; a standalone CREATE followed by ATTACH takes
+ * SHARE UPDATE EXCLUSIVE, which readers and writers pass. Measured on PG 15..19
+ * (#1297 note 1).
+ *
+ * A DEFAULT partition is refused (55000): it rules out DETACH CONCURRENTLY for
+ * the drop half, and a parent managed half-way is worse than one refused.
+ */
+CREATE FUNCTION pgcolumnar.partition_create_ahead(parent regclass, width interval, ahead int)
+	RETURNS int
+	LANGUAGE plpgsql
+	SET search_path = pg_catalog, pg_temp
+	AS $create_ahead$
+DECLARE
+	strat     "char";
+	natts     int;
+	defid     oid;
+	keytype   regtype;
+	nsp       name;
+	rel       name;
+	lo        timestamptz;
+	hi        timestamptz;
+	target    timestamptz;
+	ts_lo     timestamp;
+	ts_hi     timestamp;
+	ts_target timestamp;
+	child     text;
+	prefix    text;
+	n         int := 0;
+	probe_ts  timestamp;
+	probe_tz  timestamptz;
+	made      int := 0;
+BEGIN
+	IF width IS NULL OR width <= interval '0' OR ahead IS NULL OR ahead < 0 THEN
+		RAISE EXCEPTION 'width must be positive and ahead not negative'
+			USING ERRCODE = '22023';
+	END IF;
+	SELECT pt.partstrat, pt.partnatts, pt.partdefid, a.atttypid::regtype
+	  INTO strat, natts, defid, keytype
+	  FROM pg_partitioned_table pt
+	  JOIN pg_attribute a ON a.attrelid = pt.partrelid AND a.attnum = pt.partattrs[0]
+	 WHERE pt.partrelid = parent;
+	IF NOT FOUND OR strat <> 'r' OR natts <> 1 THEN
+		RAISE EXCEPTION '% is not RANGE-partitioned on one column', parent
+			USING ERRCODE = '55000';
+	END IF;
+	IF keytype NOT IN ('timestamptz'::regtype, 'timestamp'::regtype) THEN
+		RAISE EXCEPTION '% is partitioned on %, not on a timestamp', parent, keytype
+			USING ERRCODE = '55000';
+	END IF;
+	IF defid <> 0 THEN
+		RAISE EXCEPTION '% has a DEFAULT partition', parent
+			USING ERRCODE = '55000',
+			      HINT = 'A DEFAULT partition rules out DETACH CONCURRENTLY, so the partition lifecycle refuses to manage this table.';
+	END IF;
+	-- An upper bound of MAXVALUE leaves no room ahead, and the bound reader below
+	-- would not see it: the first new partition would overlap it (#1308 review).
+	IF EXISTS (SELECT 1 FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid
+			   WHERE i.inhparent = parent
+				 AND pg_get_expr(c.relpartbound, c.oid) ~ 'TO \(MAXVALUE\)') THEN
+		RAISE EXCEPTION '% has a partition bounded by MAXVALUE', parent
+			USING ERRCODE = '55000',
+			      HINT = 'Nothing can be created after a partition that reaches MAXVALUE.';
+	END IF;
+
+	SELECT n.nspname, c.relname INTO nsp, rel
+	  FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.oid = parent;
+	-- Names are the parent's name and a 16-byte date suffix, and must fit in 63
+	-- bytes: the prefix is shortened, by characters, until they do, so the date
+	-- always survives (#1308 review: a 63-character parent's second partition
+	-- truncated onto its first).
+	prefix := rel;
+	WHILE octet_length(prefix) + 16 > 63 LOOP
+		prefix := left(prefix, -1);
+	END LOOP;
+
+	/*
+	 * A `timestamp` key holds wall-clock time with no zone, so it is reasoned
+	 * about in its own type: its bounds are read as timestamp, "now" is the
+	 * session's localtimestamp, and the bounds written are never converted.
+	 * Reading them as timestamptz in one zone and writing them in another left
+	 * a gap of the zone's offset between partitions (America/New_York: four
+	 * hours), and aligned "daily" partitions to UTC's midnight, not the key's.
+	 */
+	IF keytype = 'timestamp'::regtype THEN
+		SELECT max((regexp_match(pg_get_expr(c.relpartbound, c.oid),
+		                         'TO \(''([^'']+)''\)'))[1]::timestamp)
+		  INTO ts_lo
+		  FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid
+		 WHERE i.inhparent = parent;
+		IF ts_lo IS NULL THEN
+			IF extract(month FROM width) <> 0 OR extract(year FROM width) <> 0 THEN
+				ts_lo := date_trunc('month', localtimestamp);
+			ELSE
+				ts_lo := date_bin(width, localtimestamp, timestamp '2001-01-01');
+			END IF;
+		END IF;
+		ts_target := localtimestamp + ahead * width;
+		probe_ts := ts_lo;
+		WHILE probe_ts <= ts_target AND n <= 1000 LOOP
+			n := n + 1;
+			probe_ts := probe_ts + width;
+		END LOOP;
+		-- Counted before any DDL: one call creating thousands of partitions holds
+		-- their locks to the end and can fail on the lock table with nothing made
+		-- (#1308 review: '1 second' x 20000 ran 6 minutes, then out of shared memory).
+		IF n > 1000 THEN
+			RAISE EXCEPTION 'partition_create_ahead would create more than 1000 partitions of %', parent
+				USING ERRCODE = '22023',
+				      HINT = 'Use a wider width, a shorter lead, or several calls.';
+		END IF;
+		WHILE ts_lo <= ts_target LOOP
+			ts_hi := ts_lo + width;
+			child := prefix || '_' || to_char(ts_lo, 'YYYYMMDD"T"HH24MISS');
+			EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING GENERATED) USING pgcolumnar',
+			               nsp, child, parent);
+			EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I FOR VALUES FROM (%L) TO (%L)',
+			               parent, nsp, child, ts_lo::text, ts_hi::text);
+			made := made + 1;
+			ts_lo := ts_hi;
+		END LOOP;
+		RETURN made;
+	END IF;
+
+	SELECT max((regexp_match(pg_get_expr(c.relpartbound, c.oid),
+	                         'TO \(''([^'']+)''\)'))[1]::timestamptz)
+	  INTO lo
+	  FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid
+	 WHERE i.inhparent = parent;
+	IF lo IS NULL THEN
+		IF extract(month FROM width) <> 0 OR extract(year FROM width) <> 0 THEN
+			lo := date_trunc('month', now());
+		ELSE
+			lo := date_bin(width, now(), timestamptz '2001-01-01');
+		END IF;
+	END IF;
+	target := now() + ahead * width;
+	probe_tz := lo;
+	WHILE probe_tz <= target AND n <= 1000 LOOP
+		n := n + 1;
+		probe_tz := probe_tz + width;
+	END LOOP;
+	-- Counted before any DDL: one call creating thousands of partitions holds
+	-- their locks to the end and can fail on the lock table with nothing made
+	-- (#1308 review: '1 second' x 20000 ran 6 minutes, then out of shared memory).
+	IF n > 1000 THEN
+		RAISE EXCEPTION 'partition_create_ahead would create more than 1000 partitions of %', parent
+			USING ERRCODE = '22023',
+			      HINT = 'Use a wider width, a shorter lead, or several calls.';
+	END IF;
+	WHILE lo <= target LOOP
+		hi := lo + width;
+		child := prefix || '_' || to_char(lo AT TIME ZONE 'UTC', 'YYYYMMDD"T"HH24MISS');
+		EXECUTE format('CREATE TABLE %I.%I (LIKE %s INCLUDING DEFAULTS INCLUDING CONSTRAINTS INCLUDING GENERATED) USING pgcolumnar',
+		               nsp, child, parent);
+		EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I FOR VALUES FROM (%L) TO (%L)',
+		               parent, nsp, child, lo::text, hi::text);
+		made := made + 1;
+		lo := hi;
+	END LOOP;
+	RETURN made;
+END;
+$create_ahead$;