From 67dbf052e0f571cbce4d145f01d59516f7a0951d Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 22:19:44 +0000 Subject: [PATCH 1/9] test: tier_partition moves a cold partition to Parquet, exactly once at every failure, written first (#1295) Against the API proposed on #1295. Red on main: 21 arms need the function or the fault GUC; every fixture premise is green. Each fault arm first requires that the injected fault is the error raised, and the rollback arm that the call ran. Co-Authored-By: Claude Opus 5.5 --- test/tier_partition.sh | 162 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100755 test/tier_partition.sh diff --git a/test/tier_partition.sh b/test/tier_partition.sh new file mode 100755 index 000000000..63726cb9b --- /dev/null +++ b/test/tier_partition.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# +# pgcolumnar.tier_partition moves a cold columnar partition to Parquet, in one +# transaction, and a failure at any step leaves the rows exactly once (#1295). +# +# WRITTEN BEFORE THE CODE, against the API proposed on #1295: +# +# pgcolumnar.tier_partition(partition regclass, server name, path text) -> rows +# +# The owner's decision (#1295, 2026-10-12): atomic, with a brief ACCESS EXCLUSIVE +# on the parent. Export under the partition's SHARE lock, then detach, drop, +# create the foreign table under the partition's own name, and attach it with the +# partition's own bounds. +# +# "EXACTLY ONCE" IS TESTED, NOT ARGUED. pgcolumnar.tier_fault_after raises an +# ERROR right after the step it names. Each fault arm first requires that the +# error raised IS the injected one: a missing function also fails, and leaves +# the rows where they were, so "the rows are still local" alone proves nothing. +# +# A local path stands in for object storage; `s3://` is the same writer with +# another path and is not exercised here. +# +# Usage: test/tier_partition.sh [PG_CONFIG] + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +W="$PGC_WORKDIR/tier" +mkdir -p "$W" && chmod 777 "$W" + +# sqlstate SQL... -> the SQLSTATE the last statement fails with, or ok +sqlstate() { + local out + out=$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -X -At -v ON_ERROR_STOP=1 -v VERBOSITY=sqlstate "${@/#/-c}" 2>&1 >/dev/null) + [ -z "$out" ] && { echo ok; return; } + echo "$out" | sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p' | head -1 +} +# errtext SQL... -> the first ERROR line's message, or ok +errtext() { + local out + out=$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -X -At -v ON_ERROR_STOP=1 "${@/#/-c}" 2>&1 >/dev/null) + [ -z "$out" ] && { echo ok; return; } + echo "$out" | sed -n 's/^ERROR: *//p' | head -1 +} + +q "CREATE EXTENSION IF NOT EXISTS pgcolumnar;" >/dev/null +psql_run "CREATE SERVER tier_pq FOREIGN DATA WRAPPER pgcolumnar_parquet;" + +# build NAME: a parent with a January and a February columnar partition +build() { + psql_run "DROP TABLE IF EXISTS $1 CASCADE;" + psql_run "CREATE TABLE $1 (ts timestamptz NOT NULL, dev int, v bigint, note text) + PARTITION BY RANGE (ts);" + psql_run "CREATE TABLE ${1}_jan PARTITION OF $1 + FOR VALUES FROM ('2026-01-01') TO ('2026-02-01') USING pgcolumnar;" + psql_run "CREATE TABLE ${1}_feb PARTITION OF $1 + FOR VALUES FROM ('2026-02-01') TO ('2026-03-01') USING pgcolumnar;" + psql_run "INSERT INTO $1 SELECT '2026-01-01'::timestamptz + g * interval '5 min', g % 13, + (hashint4(g)::bigint * 7) % 100003, md5(g::text) + FROM generate_series(0, 16991) g;" +} +# every row, decoded: an unfiltered count/sum is answered from zone maps +digest() { q "SELECT count(*) || '|' || md5(string_agg(ts::text || dev || v || note, ',' ORDER BY ts)) FROM $1;"; } +# where the January range lives: relkind, access method, and its bound +jan_shape() { + q "SELECT c.relkind::text || ':' || coalesce(a.amname, '-') || ':' || pg_get_expr(c.relpartbound, c.oid) + FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid LEFT JOIN pg_am a ON a.oid = c.relam + WHERE i.inhparent = '$1'::regclass AND c.relname = '${1}_jan';" +} +JAN_BOUND="FOR VALUES FROM ('2026-01-01 00:00:00+00') TO ('2026-02-01 00:00:00+00')" + +psql_run "SET timezone = 'UTC'; ALTER DATABASE \"$PGC_DB\" SET timezone = 'UTC';" +build m +check_num "premise: the parent holds every row" "$(q "SELECT count(*) FROM m;")" "16992" +check_num "premise: January holds its share" "$(q "SELECT count(*) FROM m_jan;")" "8928" +before="$(digest m)" +check "premise: January is a local columnar partition with its bound" "$(jan_shape m)" "r:pgcolumnar:$JAN_BOUND" +jan_storage="$(q "SELECT pgcolumnar.get_storage_id('m_jan');")" + +# ---- the move --------------------------------------------------------------- +check "tier_partition moves January and returns the rows it moved" \ + "$(q "SELECT pgcolumnar.tier_partition('m_jan', 'tier_pq', '$W/m_jan.parquet');")" "8928" +# The relkind rides along: with nothing moved the digest is trivially unchanged. +check "the answers over the parent are the same after the move" \ + "$(digest m)|$(q "SELECT relkind FROM pg_class WHERE oid = 'm_jan'::regclass;")" "$before|f" +check "January is now a foreign partition with its own name and bound" "$(jan_shape m)" "f:-:$JAN_BOUND" +check "the foreign partition reads the exported file" \ + "$(q "SELECT srvname || ':' || array_to_string(ftoptions, ',') FROM pg_foreign_table t + JOIN pg_foreign_server s ON s.oid = t.ftserver WHERE t.ftrelid = 'm_jan'::regclass;")" \ + "tier_pq:path=$W/m_jan.parquet" +check_num "the local partition leaves no pgcolumnar storage row behind" \ + "$(q "SELECT count(*) FROM pgcolumnar.storage WHERE storage_id = ${jan_storage:-0};")" "0" +check_num "premise: the storage id was read before the move" "$([ -n "$jan_storage" ] && echo 1 || echo 0)" "1" +check_num "nor any row-group row" \ + "$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = ${jan_storage:-0};")" "0" + +# ---- pruning over the mixed parent ------------------------------------------ +plan_jan="$(q "EXPLAIN SELECT max(v) FROM m WHERE ts < '2026-01-20';")" +plan_feb="$(q "EXPLAIN SELECT max(v) FROM m WHERE ts >= '2026-02-10';")" +check "a January query scans only the foreign partition" \ + "$(grep -c 'Foreign Scan' <<<"$plan_jan")|$(grep -c 'm_feb' <<<"$plan_jan")" "1|0" +check "a February query scans only the local partition, beside a foreign one" \ + "$(grep -c 'Foreign Scan' <<<"$plan_feb")|$(grep -c 'm_feb' <<<"$plan_feb")|$(grep -c 'Foreign Scan' <<<"$plan_jan")" "0|1|1" +check "an insert routed to the tiered range is refused" \ + "$(sqlstate "INSERT INTO m VALUES ('2026-01-15', 1, 1, 'x');")" "0A000" + +# ---- refusals, before anything moves ---------------------------------------- +# Measured with export_parquet on main: an existing path is overwritten without a +# word, which is why tier_partition must refuse one. +rb="$(digest m)" +psql_run "CREATE TABLE lone (ts timestamptz, v int) USING pgcolumnar;" +check "a table that is not a partition is refused" \ + "$(sqlstate "SELECT pgcolumnar.tier_partition('lone', 'tier_pq', '$W/lone.parquet');")" "42809" +psql_run "CREATE TABLE m_mar PARTITION OF m FOR VALUES FROM ('2026-03-01') TO ('2026-04-01') USING heap;" +check "a partition that is not columnar is refused" \ + "$(sqlstate "SELECT pgcolumnar.tier_partition('m_mar', 'tier_pq', '$W/m_mar.parquet');")" "42809" +psql_run "CREATE TABLE m_def PARTITION OF m DEFAULT USING pgcolumnar;" +check "a DEFAULT partition is refused" \ + "$(sqlstate "SELECT pgcolumnar.tier_partition('m_def', 'tier_pq', '$W/m_def.parquet');")" "0A000" +psql_run "CREATE SERVER tier_ice FOREIGN DATA WRAPPER pgcolumnar_iceberg;" +check "a server of another wrapper is refused" \ + "$(sqlstate "SELECT pgcolumnar.tier_partition('m_feb', 'tier_ice', '$W/m_feb.parquet');")" "42809" +check "an existing path is refused, so no file a table reads is overwritten" \ + "$(sqlstate "SELECT pgcolumnar.tier_partition('m_feb', 'tier_pq', '$W/m_jan.parquet');")" "58P02" +check "and the refusals moved nothing" "$(digest m)" "$rb" +check "premise: February is still local after the refusals" \ + "$(q "SELECT relkind FROM pg_class WHERE oid = 'm_feb'::regclass;")" "r" + +# ---- the parent's lock ------------------------------------------------------ +build k +kb="$(digest k)" +lock="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -c "BEGIN" -c "SELECT pgcolumnar.tier_partition('k_jan', 'tier_pq', '$W/k_jan.parquet')" \ + -c "SELECT coalesce(string_agg(mode, ',' ORDER BY mode), 'none') FROM pg_locks + WHERE relation = 'k'::regclass AND pid = pg_backend_pid() AND granted" \ + -c "ROLLBACK" 2>&1)" +# Line 2 is the call's result: a rollback of a call that never ran proves nothing. +check "premise: inside the transaction the call moved January" "$(sed -n 2p <<<"$lock")" "8928" +check "the call holds ACCESS EXCLUSIVE on the parent until commit, as the docs say" \ + "$(grep -o 'AccessExclusiveLock' <<<"$lock")" "AccessExclusiveLock" +check "a rollback after the whole call leaves the rows local, exactly once" \ + "$(digest k)|$(jan_shape k)" "$kb|r:pgcolumnar:$JAN_BOUND" + +# ---- a failure after each step ---------------------------------------------- +for step in export detach drop create attach; do + build f + fb="$(digest f)" + err="$(errtext "SET pgcolumnar.tier_fault_after = '$step'" \ + "SELECT pgcolumnar.tier_partition('f_jan', 'tier_pq', '$W/f_$step.parquet')")" + check "premise: the call fails with the fault injected after $step" \ + "$(grep -o "injected fault after $step" <<<"$err" || echo "$err")" "injected fault after $step" + check "a failure after $step leaves the rows exactly once, still local" \ + "$(digest f)|$(jan_shape f)" "$fb|r:pgcolumnar:$JAN_BOUND" + check_num "and no foreign table behind it after $step" \ + "$(q "SELECT count(*) FROM pg_foreign_table t JOIN pg_class c ON c.oid = t.ftrelid + WHERE c.relname LIKE 'f\\_%';")" "0" +done + +pgc_summary From 5d0a2eaf718d3104b13bf0e9fccf704ab6c401be Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 23:23:53 +0000 Subject: [PATCH 2/9] feat: pgcolumnar.tier_partition moves a cold partition to Parquet in one transaction (#1295) The owner's decision on #1295: atomic, with a brief ACCESS EXCLUSIVE on the parent. Export under the partition's SHARE lock, then detach, drop, create the foreign table with the partition's name, columns, owner, grants and comment, and attach it with the partition's own bounds. pgcolumnar.tier_fault_after (superuser, developer) raises an error after a named step, or pauses, so exactly-once and the race are tested. @jdatcmd's review of the proposal, each point an arm red against the first version: READ COMMITTED only (at REPEATABLE READ the first version dropped a row committed between the caller's snapshot and the lock); a UNIQUE parent and dependents of the partition (view, policy, own trigger, own foreign key) refused before anything moves; grants and comment carried; every lock bounded by lock_wait, with the caller's lock_timeout restored; a path a foreign table in this database reads refused, and a retry over a failed attempt's file allowed. The deadlock the review predicted is real; one side gets 40P01 and every row is exactly once. pgcolumnar_parquet has no AnalyzeForeignTable, so no ANALYZE. tier_partition.sh and its pytest twin: 65/65 on PG15-19 each. Fourteen mutants, each red on its arm; the new seven red in the twin too. Co-Authored-By: Claude Opus 5.5 --- docs/how-to.md | 69 ++++ docs/sql-reference.md | 22 ++ pgcolumnar--1.0-alpha4--1.0-alpha5.sql | 153 +++++++++ pgcolumnar--1.0-alpha5.sql | 153 +++++++++ src/columnar_tableam.c | 31 ++ test/pytest/TESTS.md | 24 ++ test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_tier_partition.py | 432 +++++++++++++++++++++++++ test/run_all_versions.sh | 1 + test/tier_partition.sh | 191 ++++++++++- 10 files changed, 1069 insertions(+), 8 deletions(-) create mode 100644 test/pytest/test_tier_partition.py diff --git a/docs/how-to.md b/docs/how-to.md index 7d66f4272..c402299d3 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -616,6 +616,75 @@ and refreshes only the rollups that it owns. `test/rollup.sh` checks each of these statements. +## Move cold partitions to Parquet + +`pgcolumnar.tier_partition` moves one columnar partition into a Parquet file. +It then attaches the file as a foreign partition with the same name, bounds, +grants and comment. Queries on the parent return the same rows before and after +the move. + +```sql +CREATE SERVER cold FOREIGN DATA WRAPPER pgcolumnar_parquet; + +SELECT pgcolumnar.tier_partition('metrics_2026_01', 'cold', + 's3://archive/metrics/2026_01.parquet'); +``` + +The function returns the number of rows that it moved. It runs as one +transaction: if it fails at any step, the rows stay in the columnar partition, +exactly once. + +**Run it at READ COMMITTED.** The function refuses `REPEATABLE READ` and +`SERIALIZABLE` (`25000`). An older snapshot does not see rows that other +sessions committed just before the move. Those rows would be dropped with the +partition, not exported. + +**It locks the parent briefly.** The export runs under a `SHARE` lock on the +partition, so writes to that partition wait and reads continue. Then the +function holds `ACCESS EXCLUSIVE` on the parent until it commits. That lock +covers the catalog changes only, not the export. + +**Every lock waits at most `lock_wait`.** The default is 10 seconds. A long +query on the parent would otherwise make every new query wait behind the move. +When the wait runs out, the move fails with `55P03` and the rows stay where they +were: + +```sql +SELECT pgcolumnar.tier_partition('metrics_2026_01', 'cold', + 's3://archive/metrics/2026_01.parquet', + lock_wait => '2 seconds'); +``` + +An insert into the partition's range that runs at the same moment can deadlock +with the move. PostgreSQL then cancels one of the two (`40P01`). Either way, +every row is stored exactly once. + +**A failed attempt leaves its file behind.** A transaction cannot remove a file +that it wrote. You can retry with the same `path`: the function overwrites a +file that no foreign table in the database reads. It refuses a `path` that a +foreign table in this database reads (`58P02`). It cannot see readers outside +the database, so do not reuse a path that another system reads. + +The function refuses these partitions before it moves anything: + +- A `DEFAULT` partition, a partition that is not columnar, or a table that is + not a partition. +- A partition whose parent has a `UNIQUE` index or a primary key. A foreign + table cannot be attached under one. +- A partition that a view depends on (`2BP01`). +- A partition that has a row-level policy, a trigger of its own, or a foreign + key of its own (`2BP01`). Dropping the partition would remove these objects. + +After the move: + +- An insert into the moved range fails, because a Parquet foreign table is + read-only. +- Partition pruning works as before. A query that touches only the local + partitions does not read the Parquet file. + +`test/tier_partition.sh` checks each of these statements. It uses a local path, +not object storage. + ## Keep tables optimized automatically A background daemon can compact and recluster tables on a schedule. The threshold diff --git a/docs/sql-reference.md b/docs/sql-reference.md index c4f25322a..a991ed48a 100644 --- a/docs/sql-reference.md +++ b/docs/sql-reference.md @@ -622,6 +622,28 @@ Records the source's current row groups as folded, without changing the rollup, and returns the number of row groups recorded. Use it after a rewrite of the source, when the rollup was current before the rewrite. +### pgcolumnar.tier_partition(partition regclass, server name, path text, lock_wait interval DEFAULT '10 seconds') returns bigint + +Moves the columnar partition `partition` to a Parquet file at `path`. Then +attaches the file as a foreign table of `server`, which must use the +`pgcolumnar_parquet` wrapper. The foreign table takes the partition's name, +columns, owner, bounds, grants and comment. Returns the number of rows moved. + +The function runs as one transaction, at READ COMMITTED only (`25000` +otherwise). It holds `ACCESS EXCLUSIVE` on the parent from the detach until +commit. Every lock waits at most `lock_wait` (`55P03`). + +The function refuses these: + +- a `path` that a foreign table in this database reads (`58P02`); +- a `DEFAULT` partition (`0A000`); +- a parent with a `UNIQUE` index or primary key (`0A000`); +- a partition that other objects depend on (`2BP01`); +- a relation that is not a columnar partition (`42809`). + +See +[Move cold partitions to Parquet](how-to.md#move-cold-partitions-to-parquet). + ### pgcolumnar.import_arrow(rel regclass, path text) returns bigint Inserts the rows of an Arrow IPC stream file at `path` into the existing table diff --git a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql index 8fd0c65cc..ed68dc3d4 100644 --- a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql @@ -1413,3 +1413,156 @@ $rb$; COMMENT ON FUNCTION pgcolumnar.rebase_rollup(regclass) IS 'record a rollup''s source row groups as folded without touching the rollup, after the source was rewritten; the caller asserts the rollup is current (#1294)'; + +-- #1295: move a cold columnar partition to Parquet in one transaction. +CREATE FUNCTION pgcolumnar.tier_partition(partition regclass, server name, path text, + lock_wait interval DEFAULT '10 seconds') + RETURNS bigint + LANGUAGE plpgsql + SET search_path = pg_catalog, pg_temp +AS $tier$ +DECLARE + fault text := coalesce(current_setting('pgcolumnar.tier_fault_after', true), ''); + rel pg_class; + parent regclass; + bound text; + fdw name; + cols text; + deps text; + note text; + acl record; + moved bigint; + seen bigint; + callers_timeout text := current_setting('lock_timeout'); +BEGIN + -- A REPEATABLE READ or SERIALIZABLE snapshot predates the lock taken below, so + -- the export would miss rows committed in between, and DROP would take them + -- (#1295, measured by @jdatcmd). Under READ COMMITTED every statement here + -- takes a fresh snapshot, so the export sees everything the lock let finish. + IF current_setting('transaction_isolation') <> 'read committed' THEN + RAISE EXCEPTION 'tier_partition runs only at READ COMMITTED, not %', + upper(current_setting('transaction_isolation')) + USING ERRCODE = 'invalid_transaction_state', + DETAIL = 'An older snapshot would export fewer rows than the partition holds, and the rest would be dropped with it.'; + END IF; + + SELECT * INTO rel FROM pg_class WHERE oid = partition; + IF NOT rel.relispartition THEN + RAISE EXCEPTION '"%" is not a partition', partition + USING ERRCODE = 'wrong_object_type'; + END IF; + -- A partition that is not columnar is refused by export_parquet, before + -- anything moves; a check of its own here was measured redundant. + bound := pg_get_expr(rel.relpartbound, rel.oid); + IF bound = 'DEFAULT' THEN + RAISE EXCEPTION 'the DEFAULT partition "%" cannot be tiered', partition + USING ERRCODE = 'feature_not_supported', + DETAIL = 'A foreign partition cannot take the rows no other partition accepts.'; + END IF; + SELECT w.fdwname INTO fdw FROM pg_foreign_server s + JOIN pg_foreign_data_wrapper w ON w.oid = s.srvfdw WHERE s.srvname = server; + IF fdw IS NULL THEN + RAISE EXCEPTION 'server "%" does not exist', server USING ERRCODE = 'undefined_object'; + ELSIF fdw <> 'pgcolumnar_parquet' THEN + RAISE EXCEPTION 'server "%" uses the % wrapper, not pgcolumnar_parquet', server, fdw + USING ERRCODE = 'wrong_object_type'; + END IF; + + SELECT i.inhparent INTO parent FROM pg_inherits i WHERE i.inhrelid = partition; + -- ATTACH of a foreign table is not allowed under a UNIQUE index; refused here, + -- before the export, rather than at step 5 with an orphan file behind it. + IF EXISTS (SELECT 1 FROM pg_index WHERE indrelid = parent AND (indisunique OR indisprimary)) THEN + RAISE EXCEPTION '"%" has a UNIQUE index or primary key, so a foreign partition cannot be attached to it', parent + USING ERRCODE = 'feature_not_supported'; + END IF; + -- DROP would fail on a dependent, or remove it silently with CASCADE. Allowed: + -- the partition's own row type, TOAST table and indexes, its own constraints + -- other than foreign keys, its defaults, and triggers cloned from the parent, + -- all of which the foreign partition does not need or gets again on ATTACH. + SELECT string_agg(pg_describe_object(d.classid, d.objid, d.objsubid), ', ' ORDER BY 1) + INTO deps + FROM pg_depend d + WHERE d.refclassid = 'pg_class'::regclass AND d.refobjid = partition + AND NOT (d.classid = 'pg_type'::regclass + OR d.classid = 'pg_attrdef'::regclass + OR (d.classid = 'pg_class'::regclass AND + (SELECT c.relkind FROM pg_class c WHERE c.oid = d.objid) IN ('t', 'i', 'I')) + OR (d.classid = 'pg_constraint'::regclass AND EXISTS + (SELECT 1 FROM pg_constraint k WHERE k.oid = d.objid + AND k.conrelid = partition AND k.contype <> 'f')) + OR (d.classid = 'pg_trigger'::regclass AND EXISTS + (SELECT 1 FROM pg_trigger t WHERE t.oid = d.objid AND t.tgparentid <> 0))); + IF deps IS NOT NULL THEN + RAISE EXCEPTION 'other objects depend on partition "%"', partition + USING ERRCODE = 'dependent_objects_still_exist', + DETAIL = format('Dropping the partition would fail on, or remove: %s.', deps); + END IF; + -- export_parquet overwrites an existing file without a word. A file another + -- foreign table in this database reads is refused; one only a failed earlier + -- attempt left behind is overwritten, so a retry can use the same path. + IF EXISTS (SELECT 1 FROM pg_foreign_table t + WHERE ('path=' || path) = ANY (t.ftoptions)) THEN + RAISE EXCEPTION 'file "%" is read by a foreign table in this database', path + USING ERRCODE = 'duplicate_file'; + END IF; + + SELECT string_agg(format('%I %s%s', a.attname, format_type(a.atttypid, a.atttypmod), + CASE WHEN a.attnotnull THEN ' NOT NULL' ELSE '' END), + ', ' ORDER BY a.attnum) + INTO cols FROM pg_attribute a + WHERE a.attrelid = partition AND a.attnum > 0 AND NOT a.attisdropped; + note := obj_description(partition, 'pg_class'); + + -- Every lock this needs waits at most lock_wait: a long reader on the parent + -- would otherwise queue every new query behind the move's request. + PERFORM set_config('lock_timeout', + (extract(epoch FROM lock_wait) * 1000)::bigint::text || 'ms', true); + + -- Writes to the cold partition wait; reads go on. + EXECUTE format('LOCK TABLE %s IN SHARE MODE', partition); + moved := pgcolumnar.export_parquet(partition, path); + -- Under the lock nothing can commit to the partition, so a statement taken + -- now sees what the export saw. A difference is an error, never a loss. + EXECUTE format('SELECT count(*) FROM %s', partition) INTO seen; + IF seen <> moved THEN + RAISE EXCEPTION 'exported % rows but partition "%" holds %', moved, partition, seen + USING ERRCODE = 'internal_error'; + END IF; + IF fault = 'export' THEN RAISE EXCEPTION 'injected fault after export'; END IF; + IF fault = 'pause' THEN PERFORM pg_sleep(2); END IF; + + -- From here the parent is held ACCESS EXCLUSIVE until commit: catalog work only. + EXECUTE format('ALTER TABLE %s DETACH PARTITION %s', parent, partition); + IF fault = 'detach' THEN RAISE EXCEPTION 'injected fault after detach'; END IF; + EXECUTE format('DROP TABLE %s', partition); + IF fault = 'drop' THEN RAISE EXCEPTION 'injected fault after drop'; END IF; + + EXECUTE format('CREATE FOREIGN TABLE %I.%I (%s) SERVER %I OPTIONS (path %L)', + rel.relnamespace::regnamespace, rel.relname, cols, server, path); + EXECUTE format('ALTER FOREIGN TABLE %I.%I OWNER TO %I', + rel.relnamespace::regnamespace, rel.relname, rel.relowner::regrole); + FOR acl IN SELECT * FROM aclexplode(rel.relacl) x WHERE x.grantee <> rel.relowner LOOP + EXECUTE format('GRANT %s ON %I.%I TO %s%s', acl.privilege_type, + rel.relnamespace::regnamespace, rel.relname, + CASE WHEN acl.grantee = 0 THEN 'PUBLIC' ELSE quote_ident(acl.grantee::regrole::text) END, + CASE WHEN acl.is_grantable THEN ' WITH GRANT OPTION' ELSE '' END); + END LOOP; + IF note IS NOT NULL THEN + EXECUTE format('COMMENT ON FOREIGN TABLE %I.%I IS %L', + rel.relnamespace::regnamespace, rel.relname, note); + END IF; + IF fault = 'create' THEN RAISE EXCEPTION 'injected fault after create'; END IF; + EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I %s', + parent, rel.relnamespace::regnamespace, rel.relname, bound); + IF fault = 'attach' THEN RAISE EXCEPTION 'injected fault after attach'; END IF; + -- No ANALYZE: pgcolumnar_parquet has no AnalyzeForeignTable, so there is no + -- sample to take (checked on #1295). + -- The caller's lock_timeout again, or it would last to the end of their + -- transaction: set_config(..., true) is transaction-local, not call-local. + PERFORM set_config('lock_timeout', callers_timeout, true); + RETURN moved; +END +$tier$; + +COMMENT ON FUNCTION pgcolumnar.tier_partition(regclass, name, text, interval) + IS 'move a cold columnar partition to a Parquet file, attached in its place as a foreign partition with the same name, bounds, grants and comment, in one transaction at READ COMMITTED (#1295). Holds ACCESS EXCLUSIVE on the parent for the catalog steps; every lock waits at most lock_wait. Returns rows moved'; diff --git a/pgcolumnar--1.0-alpha5.sql b/pgcolumnar--1.0-alpha5.sql index 0df96cd1f..0d72b8e2a 100644 --- a/pgcolumnar--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha5.sql @@ -2603,3 +2603,156 @@ $rb$; COMMENT ON FUNCTION pgcolumnar.rebase_rollup(regclass) IS 'record a rollup''s source row groups as folded without touching the rollup, after the source was rewritten; the caller asserts the rollup is current (#1294)'; + +-- #1295: move a cold columnar partition to Parquet in one transaction. +CREATE FUNCTION pgcolumnar.tier_partition(partition regclass, server name, path text, + lock_wait interval DEFAULT '10 seconds') + RETURNS bigint + LANGUAGE plpgsql + SET search_path = pg_catalog, pg_temp +AS $tier$ +DECLARE + fault text := coalesce(current_setting('pgcolumnar.tier_fault_after', true), ''); + rel pg_class; + parent regclass; + bound text; + fdw name; + cols text; + deps text; + note text; + acl record; + moved bigint; + seen bigint; + callers_timeout text := current_setting('lock_timeout'); +BEGIN + -- A REPEATABLE READ or SERIALIZABLE snapshot predates the lock taken below, so + -- the export would miss rows committed in between, and DROP would take them + -- (#1295, measured by @jdatcmd). Under READ COMMITTED every statement here + -- takes a fresh snapshot, so the export sees everything the lock let finish. + IF current_setting('transaction_isolation') <> 'read committed' THEN + RAISE EXCEPTION 'tier_partition runs only at READ COMMITTED, not %', + upper(current_setting('transaction_isolation')) + USING ERRCODE = 'invalid_transaction_state', + DETAIL = 'An older snapshot would export fewer rows than the partition holds, and the rest would be dropped with it.'; + END IF; + + SELECT * INTO rel FROM pg_class WHERE oid = partition; + IF NOT rel.relispartition THEN + RAISE EXCEPTION '"%" is not a partition', partition + USING ERRCODE = 'wrong_object_type'; + END IF; + -- A partition that is not columnar is refused by export_parquet, before + -- anything moves; a check of its own here was measured redundant. + bound := pg_get_expr(rel.relpartbound, rel.oid); + IF bound = 'DEFAULT' THEN + RAISE EXCEPTION 'the DEFAULT partition "%" cannot be tiered', partition + USING ERRCODE = 'feature_not_supported', + DETAIL = 'A foreign partition cannot take the rows no other partition accepts.'; + END IF; + SELECT w.fdwname INTO fdw FROM pg_foreign_server s + JOIN pg_foreign_data_wrapper w ON w.oid = s.srvfdw WHERE s.srvname = server; + IF fdw IS NULL THEN + RAISE EXCEPTION 'server "%" does not exist', server USING ERRCODE = 'undefined_object'; + ELSIF fdw <> 'pgcolumnar_parquet' THEN + RAISE EXCEPTION 'server "%" uses the % wrapper, not pgcolumnar_parquet', server, fdw + USING ERRCODE = 'wrong_object_type'; + END IF; + + SELECT i.inhparent INTO parent FROM pg_inherits i WHERE i.inhrelid = partition; + -- ATTACH of a foreign table is not allowed under a UNIQUE index; refused here, + -- before the export, rather than at step 5 with an orphan file behind it. + IF EXISTS (SELECT 1 FROM pg_index WHERE indrelid = parent AND (indisunique OR indisprimary)) THEN + RAISE EXCEPTION '"%" has a UNIQUE index or primary key, so a foreign partition cannot be attached to it', parent + USING ERRCODE = 'feature_not_supported'; + END IF; + -- DROP would fail on a dependent, or remove it silently with CASCADE. Allowed: + -- the partition's own row type, TOAST table and indexes, its own constraints + -- other than foreign keys, its defaults, and triggers cloned from the parent, + -- all of which the foreign partition does not need or gets again on ATTACH. + SELECT string_agg(pg_describe_object(d.classid, d.objid, d.objsubid), ', ' ORDER BY 1) + INTO deps + FROM pg_depend d + WHERE d.refclassid = 'pg_class'::regclass AND d.refobjid = partition + AND NOT (d.classid = 'pg_type'::regclass + OR d.classid = 'pg_attrdef'::regclass + OR (d.classid = 'pg_class'::regclass AND + (SELECT c.relkind FROM pg_class c WHERE c.oid = d.objid) IN ('t', 'i', 'I')) + OR (d.classid = 'pg_constraint'::regclass AND EXISTS + (SELECT 1 FROM pg_constraint k WHERE k.oid = d.objid + AND k.conrelid = partition AND k.contype <> 'f')) + OR (d.classid = 'pg_trigger'::regclass AND EXISTS + (SELECT 1 FROM pg_trigger t WHERE t.oid = d.objid AND t.tgparentid <> 0))); + IF deps IS NOT NULL THEN + RAISE EXCEPTION 'other objects depend on partition "%"', partition + USING ERRCODE = 'dependent_objects_still_exist', + DETAIL = format('Dropping the partition would fail on, or remove: %s.', deps); + END IF; + -- export_parquet overwrites an existing file without a word. A file another + -- foreign table in this database reads is refused; one only a failed earlier + -- attempt left behind is overwritten, so a retry can use the same path. + IF EXISTS (SELECT 1 FROM pg_foreign_table t + WHERE ('path=' || path) = ANY (t.ftoptions)) THEN + RAISE EXCEPTION 'file "%" is read by a foreign table in this database', path + USING ERRCODE = 'duplicate_file'; + END IF; + + SELECT string_agg(format('%I %s%s', a.attname, format_type(a.atttypid, a.atttypmod), + CASE WHEN a.attnotnull THEN ' NOT NULL' ELSE '' END), + ', ' ORDER BY a.attnum) + INTO cols FROM pg_attribute a + WHERE a.attrelid = partition AND a.attnum > 0 AND NOT a.attisdropped; + note := obj_description(partition, 'pg_class'); + + -- Every lock this needs waits at most lock_wait: a long reader on the parent + -- would otherwise queue every new query behind the move's request. + PERFORM set_config('lock_timeout', + (extract(epoch FROM lock_wait) * 1000)::bigint::text || 'ms', true); + + -- Writes to the cold partition wait; reads go on. + EXECUTE format('LOCK TABLE %s IN SHARE MODE', partition); + moved := pgcolumnar.export_parquet(partition, path); + -- Under the lock nothing can commit to the partition, so a statement taken + -- now sees what the export saw. A difference is an error, never a loss. + EXECUTE format('SELECT count(*) FROM %s', partition) INTO seen; + IF seen <> moved THEN + RAISE EXCEPTION 'exported % rows but partition "%" holds %', moved, partition, seen + USING ERRCODE = 'internal_error'; + END IF; + IF fault = 'export' THEN RAISE EXCEPTION 'injected fault after export'; END IF; + IF fault = 'pause' THEN PERFORM pg_sleep(2); END IF; + + -- From here the parent is held ACCESS EXCLUSIVE until commit: catalog work only. + EXECUTE format('ALTER TABLE %s DETACH PARTITION %s', parent, partition); + IF fault = 'detach' THEN RAISE EXCEPTION 'injected fault after detach'; END IF; + EXECUTE format('DROP TABLE %s', partition); + IF fault = 'drop' THEN RAISE EXCEPTION 'injected fault after drop'; END IF; + + EXECUTE format('CREATE FOREIGN TABLE %I.%I (%s) SERVER %I OPTIONS (path %L)', + rel.relnamespace::regnamespace, rel.relname, cols, server, path); + EXECUTE format('ALTER FOREIGN TABLE %I.%I OWNER TO %I', + rel.relnamespace::regnamespace, rel.relname, rel.relowner::regrole); + FOR acl IN SELECT * FROM aclexplode(rel.relacl) x WHERE x.grantee <> rel.relowner LOOP + EXECUTE format('GRANT %s ON %I.%I TO %s%s', acl.privilege_type, + rel.relnamespace::regnamespace, rel.relname, + CASE WHEN acl.grantee = 0 THEN 'PUBLIC' ELSE quote_ident(acl.grantee::regrole::text) END, + CASE WHEN acl.is_grantable THEN ' WITH GRANT OPTION' ELSE '' END); + END LOOP; + IF note IS NOT NULL THEN + EXECUTE format('COMMENT ON FOREIGN TABLE %I.%I IS %L', + rel.relnamespace::regnamespace, rel.relname, note); + END IF; + IF fault = 'create' THEN RAISE EXCEPTION 'injected fault after create'; END IF; + EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I %s', + parent, rel.relnamespace::regnamespace, rel.relname, bound); + IF fault = 'attach' THEN RAISE EXCEPTION 'injected fault after attach'; END IF; + -- No ANALYZE: pgcolumnar_parquet has no AnalyzeForeignTable, so there is no + -- sample to take (checked on #1295). + -- The caller's lock_timeout again, or it would last to the end of their + -- transaction: set_config(..., true) is transaction-local, not call-local. + PERFORM set_config('lock_timeout', callers_timeout, true); + RETURN moved; +END +$tier$; + +COMMENT ON FUNCTION pgcolumnar.tier_partition(regclass, name, text, interval) + IS 'move a cold columnar partition to a Parquet file, attached in its place as a foreign partition with the same name, bounds, grants and comment, in one transaction at READ COMMITTED (#1295). Holds ACCESS EXCLUSIVE on the parent for the catalog steps; every lock waits at most lock_wait. Returns rows moved'; diff --git a/src/columnar_tableam.c b/src/columnar_tableam.c index d72f71e7c..54df9b2a3 100644 --- a/src/columnar_tableam.c +++ b/src/columnar_tableam.c @@ -3666,6 +3666,21 @@ pgcolumnar_get_relation_info(PlannerInfo *root, Oid relationObjectId, /* pgcolumnar.rollup_pause (#1294): read by refresh_rollup through current_setting */ static bool pgcolumnar_rollup_pause = false; +/* pgcolumnar.tier_fault_after (#1295): read by tier_partition through current_setting */ +static char *pgcolumnar_tier_fault_after = NULL; + +static bool +pgcolumnar_check_tier_fault_after(char **newval, void **extra, GucSource source) +{ + static const char *const steps[] = {"", "export", "detach", "drop", "create", "attach", "pause"}; + + for (int i = 0; i < lengthof(steps); i++) + if (strcmp(*newval, steps[i]) == 0) + return true; + GUC_check_errdetail("Valid values are export, detach, drop, create, attach, pause, or empty."); + return false; +} + void _PG_init(void) { @@ -4110,6 +4125,22 @@ _PG_init(void) GUC_NOT_IN_SAMPLE, NULL, NULL, NULL); + /* + * A developer fault for pgcolumnar.tier_partition (#1295): the step named + * here raises an ERROR right after it runs, so the tests can show that a + * failure at every step leaves the rows exactly once. Superuser-only and + * empty by default; any value but a step name is refused when set, so a typo + * cannot leave a test believing it injected a fault it did not. + */ + DefineCustomStringVariable("pgcolumnar.tier_fault_after", + "Developer fault: raise an error after this tier_partition step.", + "One of export, detach, drop, create, attach; pause sleeps two seconds after export; empty for none.", + &pgcolumnar_tier_fault_after, + "", + PGC_SUSET, + GUC_NOT_IN_SAMPLE, + pgcolumnar_check_tier_fault_after, NULL, NULL); + DefineCustomStringVariable("pgcolumnar.objstore_allowed_endpoints", "Remote endpoints the object-store module may connect to.", "Comma-separated host or host:port entries; empty refuses all remote access.", diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 20b3672de..db5e0daca 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -145,6 +145,7 @@ behaviour, the source of that number is named. - [97. test_rollup.py: incremental rollups fold appended row groups, and equal a full recompute](#97-test_rolluppy-incremental-rollups-fold-appended-row-groups-and-equal-a-full-recompute) - [98. test_timeseries_recipes.py: the gap-fill, LOCF and interpolation recipes answer what they claim](#98-test_timeseries_recipespy-the-gap-fill-locf-and-interpolation-recipes-answer-what-they-claim) - [99. test_suite_exit_trap.py: a suite's own EXIT trap must still stop its cluster](#99-test_suite_exit_trappy-a-suites-own-exit-trap-must-still-stop-its-cluster) +- [100. test_tier_partition.py: tier_partition moves a cold partition to Parquet, exactly once](#100-test_tier_partitionpy-tier_partition-moves-a-cold-partition-to-parquet-exactly-once) ## 1. How to read a test in here @@ -6657,3 +6658,26 @@ Written red first: on the tree before the fix, the population arm fails naming ` | `test_the_detector_is_not_satisfied_by_a_comment_or_the_next_function` | the teardown named only in a comment, or only in the function after a one-line handler, does not count | | `test_the_detector_passes_a_handler_that_chains` | a handler calling `pgc_on_exit`, one calling `pgc_teardown`, a quoted handler, and a suite without `pgc_setup` are not reported | | `test_no_suites_own_exit_trap_leaves_its_cluster_running` | `pgc_setup` still arms `trap pgc_on_exit EXIT` (premise), and no suite in `test/` is reported | + +## 100. test_tier_partition.py: tier_partition moves a cold partition to Parquet, exactly once + +Port of `tier_partition.sh` (#1295). The owner's decision on #1295: the move is atomic, with a brief ACCESS EXCLUSIVE on the parent. The export runs under the partition's SHARE lock. Then the function detaches and drops the partition, creates a foreign table with the partition's own name, columns, owner, grants and comment, and attaches it with the partition's own bounds. + +EXACTLY ONCE IS TESTED, NOT ARGUED. The developer GUC `pgcolumnar.tier_fault_after` raises an error right after the step it names (`pause` sleeps instead, for the race arm). Each fault arm first requires that the error raised is the injected one: a missing function also fails and leaves the rows where they were. + +What the arms found, in order: +- `pg_stat_file(...) IS NOT NULL` is false for an existing file, because `creation` is NULL on Linux, so the first existing-path refusal never fired, and a second move overwrote a file another partition was reading. +- Without the SHARE lock, a write in flight when the move started was dropped with the partition. +- @jdatcmd's review: at REPEATABLE READ or SERIALIZABLE the SHARE lock does not refresh the snapshot. The first version, at those levels, exported without a row that another session committed between the caller's snapshot and the lock, and dropped it: `16992|0` where `16993|1` was owed. Those levels are now refused (`25000`). +- The first version silently moved a partition with a row-level policy, dropping the policy, and left an orphan file behind a UNIQUE index on the parent. Both are refused before anything moves now, as are a view, a trigger of its own and a foreign key of its own. +- A foreign key REFERENCING a columnar table is refused by pgcolumnar itself, so it cannot arise. In the first draft of that arm the setup failed, rolled back the previous case's DROP with it (a multi-statement string is one transaction), and the arm passed on a leftover trigger. Each dependent is now made in statements of its own, with a premise that it exists, and a control arm shows the same partition moves once they are gone. +- The deadlock @jdatcmd predicted is real: an insert through the parent racing the move ends with one side cancelled (`40P01`), and every row is still exactly once. + +On a private database, because foreign servers are per database. A local path stands in for object storage. The pytest half uses connections in threads where the shell half uses background sessions. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_tier_partition_moves_a_partition_exactly_once` | the move returns the rows moved; the parent's answers are the same; the cold range is a foreign partition with the same name and bound, reading the exported file; no pgcolumnar catalog rows are left; pruning keeps the ranges apart; an insert into the tiered range is refused; five refusals move nothing; the parent is held ACCESS EXCLUSIVE until commit, and a rollback leaves the rows local; a write in flight is exported; a fault after each of the five steps leaves the rows exactly once, still local | +| `test_tier_partition_refuses_what_it_cannot_move_and_survives_concurrency` | at each isolation level a row committed between the snapshot and the lock survives, and only READ COMMITTED moves; a UNIQUE parent and four kinds of dependent are refused with no file left, and the partition moves once they are gone; grants and comment carry over; a racing insert leaves every row once; lock_wait times a held parent out with nothing attached; the caller's lock_timeout survives; a retry over a failed attempt's file moves the partition | diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 49c1cec9b..24fdf9350 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -136,6 +136,7 @@ "storage_row_repair", "temporal", "tid_range_scan", + "tier_partition", "timeseries_recipes", "ttl_expire", "validity_elision", diff --git a/test/pytest/test_tier_partition.py b/test/pytest/test_tier_partition.py new file mode 100644 index 000000000..d88e0cd25 --- /dev/null +++ b/test/pytest/test_tier_partition.py @@ -0,0 +1,432 @@ +"""pgcolumnar.tier_partition moves a cold partition to Parquet, exactly once (#1295). + +The owner's decision on #1295: atomic, with a brief ACCESS EXCLUSIVE on the +parent. Export under the partition's SHARE lock, then detach, drop, create the +foreign table under the partition's own name, and attach it with its own bounds. + +"EXACTLY ONCE" IS TESTED, NOT ARGUED. `pgcolumnar.tier_fault_after` raises an +error right after the step it names, and each fault arm first requires that the +error raised IS the injected one: a missing function also fails and leaves the +rows where they were. + +Two findings the shell twin made while the code was written, held here too: +- `pg_stat_file(...) IS NOT NULL` is false for an existing file, so the + existing-path refusal never fired, and the export overwrote a file a foreign + partition was reading. +- without the SHARE lock, a write in flight when the move starts was lost: the + export did not see it, DETACH waited for it, DROP took it with the partition. + +ON A PRIVATE DATABASE (`pgc_own_db`): foreign servers are per database. A local +path stands in for object storage. Independent of `test/tier_partition.sh`: its +own columns, months, row counts, paths and in-flight row. +""" + +import threading +import time + +import psycopg + +BOUND = "FOR VALUES FROM ('2026-03-01 00:00:00+00') TO ('2026-04-01 00:00:00+00')" +STEPS = ("export", "detach", "drop", "create", "attach") + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + row = cur.fetchone() + return row[0] if row else None + + +def _sqlstate(conn, sql): + try: + with conn.cursor() as cur: + cur.execute(sql) + return "ok" + except psycopg.Error as e: + return e.sqlstate or "?" + + +def _errtext(conn, sql): + try: + with conn.cursor() as cur: + cur.execute(sql) + return "ok" + except psycopg.Error as e: + return str(e).splitlines()[0] + + +def _build(conn, t): + with conn.cursor() as cur: + cur.execute(f"DROP TABLE IF EXISTS {t} CASCADE") + cur.execute(f"CREATE TABLE {t} (at timestamptz NOT NULL, sensor int, reading float8," + " label text) PARTITION BY RANGE (at)") + cur.execute(f"CREATE TABLE {t}_mar PARTITION OF {t} FOR VALUES FROM ('2026-03-01')" + " TO ('2026-04-01') USING pgcolumnar") + cur.execute(f"CREATE TABLE {t}_apr PARTITION OF {t} FOR VALUES FROM ('2026-04-01')" + " TO ('2026-05-01') USING pgcolumnar") + cur.execute(f"INSERT INTO {t} SELECT '2026-03-01'::timestamptz + g * interval '7 min'," + " g % 17, (g * 31 % 997) / 7.0, 'r' || (g * 13 % 101)" + " FROM generate_series(0, 12340) g") + + +def _digest(conn, t): + # Every row decoded: an unfiltered count/sum is answered from the zone maps. + return _one(conn, f"SELECT count(*) || '|' || md5(string_agg(at::text || sensor || reading" + f" || label, ',' ORDER BY at)) FROM {t}") + + +def _mar_shape(conn, t): + return _one(conn, "SELECT c.relkind::text || ':' || coalesce(a.amname, '-') || ':'" + " || pg_get_expr(c.relpartbound, c.oid) FROM pg_inherits i" + " JOIN pg_class c ON c.oid = i.inhrelid LEFT JOIN pg_am a ON a.oid = c.relam" + f" WHERE i.inhparent = '{t}'::regclass AND c.relname = '{t}_mar'") + + +def _tier(t, path): + return f"SELECT pgcolumnar.tier_partition('{t}_mar', 'tq', '{path}')" + + +def test_tier_partition_moves_a_partition_exactly_once(pgc_own_db, pgc_cluster, server_dir, expect): + conn = pgc_own_db + with conn.cursor() as cur: + cur.execute("SET timezone = 'UTC'") + cur.execute("CREATE SERVER tq FOREIGN DATA WRAPPER pgcolumnar_parquet") + _build(conn, "s") + expect.num(_one(conn, "SELECT count(*) FROM s"), 12341, "premise: the parent holds every row") + mar = _one(conn, "SELECT count(*) FROM s_mar") + expect.num(mar, 6378, "premise: the cold partition holds its share") + before = _digest(conn, "s") + expect.text(_mar_shape(conn, "s"), f"r:pgcolumnar:{BOUND}", + "premise: the cold partition is local and columnar, with its bound") + sid = _one(conn, "SELECT pgcolumnar.get_storage_id('s_mar')") + + path = server_dir / "s_mar.parquet" + expect.text(str(_one(conn, _tier("s", path))), str(mar), + "tier_partition moves the cold partition and returns the rows it moved") + kind = _one(conn, "SELECT coalesce((SELECT relkind::text FROM pg_class" + " WHERE oid = to_regclass('s_mar')), 'absent')") + # The relkind rides along: with nothing moved the digest is trivially unchanged. + expect.text(f"{_digest(conn, 's')}|{kind}", f"{before}|f", + "the answers over the parent are the same after the move") + expect.text(_mar_shape(conn, "s"), f"f:-:{BOUND}", + "the cold partition is now a foreign partition with its own name and bound") + expect.text(_one(conn, "SELECT srvname || ':' || array_to_string(ftoptions, ',')" + " FROM pg_foreign_table t JOIN pg_foreign_server s ON s.oid = t.ftserver" + " WHERE t.ftrelid = 's_mar'::regclass"), + f"tq:path={path}", "the foreign partition reads the exported file") + expect.text("read" if sid is not None else "unread", "read", + "premise: the storage id was read before the move") + expect.num(_one(conn, f"SELECT count(*) FROM pgcolumnar.storage WHERE storage_id = {sid or 0}"), 0, + "the local partition leaves no pgcolumnar storage row behind") + expect.num(_one(conn, f"SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = {sid or 0}"), 0, + "nor any row-group row") + + def plan(sql): + with conn.cursor() as cur: + cur.execute("EXPLAIN " + sql) + return "\n".join(r[0] for r in cur.fetchall()) + p_mar = plan("SELECT min(reading) FROM s WHERE at < '2026-03-15'") + p_apr = plan("SELECT min(reading) FROM s WHERE at >= '2026-04-12'") + expect.text(f"{p_mar.count('Foreign Scan')}|{p_mar.count('s_apr')}", "1|0", + "a query on the tiered range scans only the foreign partition") + expect.text(f"{p_apr.count('Foreign Scan')}|{p_apr.count('s_apr')}|{p_mar.count('Foreign Scan')}", + "0|1|1", "a query on the other range scans only the local partition, beside a foreign one") + expect.text(_sqlstate(conn, "INSERT INTO s VALUES ('2026-03-09', 1, 1, 'x')"), "0A000", + "an insert routed to the tiered range is refused") + + # Refusals, before anything moves. + rb = _digest(conn, "s") + with conn.cursor() as cur: + cur.execute("CREATE TABLE solo (at timestamptz, v int) USING pgcolumnar") + cur.execute("CREATE TABLE s_may PARTITION OF s FOR VALUES FROM ('2026-05-01')" + " TO ('2026-06-01') USING heap") + cur.execute("CREATE TABLE s_rest PARTITION OF s DEFAULT USING pgcolumnar") + cur.execute("CREATE SERVER ti FOREIGN DATA WRAPPER pgcolumnar_iceberg") + expect.text(_sqlstate(conn, f"SELECT pgcolumnar.tier_partition('solo', 'tq', '{server_dir}/solo.parquet')"), + "42809", "a table that is not a partition is refused") + expect.text(_sqlstate(conn, f"SELECT pgcolumnar.tier_partition('s_may', 'tq', '{server_dir}/may.parquet')"), + "42809", "a partition that is not columnar is refused") + expect.text(_sqlstate(conn, f"SELECT pgcolumnar.tier_partition('s_rest', 'tq', '{server_dir}/rest.parquet')"), + "0A000", "a DEFAULT partition is refused") + expect.text(_sqlstate(conn, f"SELECT pgcolumnar.tier_partition('s_apr', 'ti', '{server_dir}/apr.parquet')"), + "42809", "a server of another wrapper is refused") + expect.text(_sqlstate(conn, f"SELECT pgcolumnar.tier_partition('s_apr', 'tq', '{path}')"), + "58P02", "an existing path is refused, so no file a table reads is overwritten") + expect.text(_digest(conn, "s"), rb, "and the refusals moved nothing") + expect.text(_one(conn, "SELECT relkind::text FROM pg_class WHERE oid = 's_apr'::regclass"), "r", + "premise: the other partition is still local after the refusals") + + # The parent's lock, and a rollback of the whole call. + _build(conn, "k") + kb = _digest(conn, "k") + with conn.cursor() as cur: + cur.execute("BEGIN") + cur.execute(_tier("k", server_dir / "k_mar.parquet")) + moved = cur.fetchone()[0] + cur.execute("SELECT coalesce(string_agg(mode, ',' ORDER BY mode), 'none') FROM pg_locks" + " WHERE relation = 'k'::regclass AND pid = pg_backend_pid() AND granted") + locks = cur.fetchone()[0] + cur.execute("ROLLBACK") + expect.num(moved, mar, "premise: inside the transaction the call moved the cold partition") + expect.text("AccessExclusiveLock" if "AccessExclusiveLock" in locks else locks, "AccessExclusiveLock", + "the call holds ACCESS EXCLUSIVE on the parent until commit, as the docs say") + expect.text(f"{_digest(conn, 'k')}|{_mar_shape(conn, 'k')}", f"{kb}|r:pgcolumnar:{BOUND}", + "a rollback after the whole call leaves the rows local, exactly once") + + # A write in flight when the move starts. + _build(conn, "w") + wb = _one(conn, "SELECT count(*) FROM w") + writer = psycopg.connect(pgc_cluster.dsn(dbname=_one(conn, "SELECT current_database()"))) + try: + writer.execute("INSERT INTO w_mar VALUES ('2026-03-31 23:58:00+00', 5, 5.5, 'in flight')") + out = {} + + def mover(): + t0 = time.monotonic() + out["moved"] = _one(conn, _tier("w", server_dir / "w_mar.parquet")) + out["ms"] = int((time.monotonic() - t0) * 1000) + th = threading.Thread(target=mover) + th.start() + time.sleep(1.5) + writer.commit() + th.join(60) + finally: + writer.close() + print(f"-- the move waited {out.get('ms')} ms for the writer") + expect.at_least(out.get("ms", 0), 1000, + "premise: the move started while the insert was in flight, and waited for it") + total = _one(conn, "SELECT count(*) FROM w") + flight = _one(conn, "SELECT count(*) FROM w WHERE label = 'in flight'") + expect.text(f"{out.get('moved')}|{total}|{flight}", + f"{mar + 1}|{wb + 1}|1", "a write in flight when the move starts is in the exported rows") + + # A failure after each step. + for step in STEPS: + _build(conn, "f") + fb = _digest(conn, "f") + with conn.cursor() as cur: + cur.execute(f"SET pgcolumnar.tier_fault_after = '{step}'") + err = _errtext(conn, _tier("f", server_dir / f"f_{step}.parquet")) + with conn.cursor() as cur: + cur.execute("RESET pgcolumnar.tier_fault_after") + expect.text(f"injected fault after {step}" if f"injected fault after {step}" in err else err, + f"injected fault after {step}", + f"premise: the call fails with the fault injected after {step}") + expect.text(f"{_digest(conn, 'f')}|{_mar_shape(conn, 'f')}", f"{fb}|r:pgcolumnar:{BOUND}", + f"a failure after {step} leaves the rows exactly once, still local") + expect.num(_one(conn, "SELECT count(*) FROM pg_foreign_table t JOIN pg_class c ON c.oid = t.ftrelid" + " WHERE c.relname LIKE 'f\\_%'"), 0, + f"and no foreign table behind it after {step}") + + +def _dsn(pgc_cluster, conn): + return pgc_cluster.dsn(dbname=_one(conn, "SELECT current_database()")) + + +def _relkind(conn, t): + return _one(conn, f"SELECT coalesce((SELECT relkind::text FROM pg_class WHERE oid = to_regclass('{t}_mar')), 'absent')") + + +def test_tier_partition_refuses_what_it_cannot_move_and_survives_concurrency( + pgc_own_db, pgc_cluster, server_dir, expect): + """The #1295 review's five points, each an arm (@jdatcmd). + + 1. A writer that commits between the caller's snapshot and the lock: at + REPEATABLE READ or SERIALIZABLE the export would miss its row and DROP + would take it, so those levels are refused; at READ COMMITTED it moves. + 2. A UNIQUE index on the parent, and 3. anything that depends on the + partition, are refused before anything moves, leaving no file. + 4. An insert racing the swap, and a parent held by a long reader. + 5. A retry over the file a failed attempt left. + """ + conn = pgc_own_db + dsn = _dsn(pgc_cluster, conn) + with conn.cursor() as cur: + cur.execute("SET timezone = 'UTC'") + cur.execute("CREATE SERVER tq FOREIGN DATA WRAPPER pgcolumnar_parquet") + mar = 6378 + + # 1. Isolation. + for n, lvl in enumerate(("read committed", "repeatable read", "serializable")): + t = f"iso{n}" + _build(conn, t) + before = _one(conn, f"SELECT count(*) FROM {t}") + out = {} + + def mover(): + with psycopg.connect(dsn, autocommit=True) as m: + m.execute(f"BEGIN ISOLATION LEVEL {lvl}") + out["seen"] = m.execute(f"SELECT count(*) FROM {t}_mar").fetchone()[0] + time.sleep(1.5) + try: + out["moved"] = m.execute(_tier(t, server_dir / f"{t}.parquet")).fetchone()[0] + m.execute("COMMIT") + except psycopg.Error as e: + out["state"] = e.sqlstate + m.execute("ROLLBACK") + th = threading.Thread(target=mover) + th.start() + time.sleep(0.7) + with conn.cursor() as cur: + cur.execute(f"INSERT INTO {t}_mar VALUES ('2026-03-20 09:30:00+00', 4, 4.5, 'committed between')") + th.join(60) + expect.num(out.get("seen"), mar, f"premise: at {lvl} the caller's snapshot was taken before the writer committed") + expect.text(f"{_one(conn, f'SELECT count(*) FROM {t}')}|" + f"{_one(conn, f'SELECT count(*) FROM {t} WHERE label = ' + chr(39) + 'committed between' + chr(39))}", + f"{before + 1}|1", f"at {lvl} a row committed between the caller's snapshot and the lock survives") + if lvl == "read committed": + expect.text(f"{out.get('moved')}|{_relkind(conn, t)}", f"{mar + 1}|f", + "at read committed the move goes ahead and exports that row") + else: + expect.text(f"{out.get('state')}|{_relkind(conn, t)}", "25000|r", + f"at {lvl} the move is refused, and the rows stay local") + + # 2 and 3. Refused before anything moves, and no file left. + def refused(t, path): + return f"{_digest(conn, t)}|{_relkind(conn, t)}|{'file' if path.exists() else 'nofile'}" + + _build(conn, "uq") + with conn.cursor() as cur: + cur.execute("CREATE UNIQUE INDEX uq_key ON uq (at)") + ub = _digest(conn, "uq") + p = server_dir / "uq.parquet" + expect.text(f"{_sqlstate(conn, _tier('uq', p))}|{refused('uq', p)}", f"0A000|{ub}|r|nofile", + "a parent with a UNIQUE index is refused before anything moves") + + _build(conn, "dep") + db = _digest(conn, "dep") + # Each dependent made and removed one statement at a time, with a premise that + # it exists: a multi-statement string is one transaction, and a failed CREATE + # once rolled back the previous case's DROP, so an arm passed on a leftover. + # Names WRITTEN AT THE CALL: compare_to_bash resolves string literals there. + def run_each(stmts): + for st in stmts: + with conn.cursor() as cur: + cur.execute(st) + + def dep_case(i, make, premise_sql, undo): + run_each(make) + p = server_dir / f"dep{i}.parquet" + got = f"{_one(conn, premise_sql)}|{_sqlstate(conn, _tier('dep', p))}|{refused('dep', p)}" + run_each(undo) + return got + + expect.text(dep_case(0, ["CREATE VIEW dep_view AS SELECT at, reading FROM dep_mar"], + "SELECT count(*) FROM pg_views WHERE viewname = 'dep_view'", ["DROP VIEW dep_view"]), + f"1|2BP01|{db}|r|nofile", "a partition a view depends on is refused before anything moves") + expect.text(dep_case(1, ["ALTER TABLE dep_mar ENABLE ROW LEVEL SECURITY", + "CREATE POLICY dep_pol ON dep_mar USING (sensor <> 3)"], + "SELECT count(*) FROM pg_policy WHERE polname = 'dep_pol'", + ["DROP POLICY dep_pol ON dep_mar", "ALTER TABLE dep_mar DISABLE ROW LEVEL SECURITY"]), + f"1|2BP01|{db}|r|nofile", "a partition with a row-level policy is refused before anything moves") + expect.text(dep_case(2, ["CREATE FUNCTION dep_trg() RETURNS trigger LANGUAGE plpgsql AS $$BEGIN RETURN NEW; END$$", + "CREATE TRIGGER dep_own BEFORE DELETE ON dep_mar FOR EACH ROW EXECUTE FUNCTION dep_trg()"], + "SELECT count(*) FROM pg_trigger WHERE tgname = 'dep_own'", ["DROP TRIGGER dep_own ON dep_mar"]), + f"1|2BP01|{db}|r|nofile", "a partition with a trigger of its own is refused before anything moves") + # A foreign key REFERENCING a columnar table is refused by pgcolumnar itself, + # so the case that arises is one the partition holds. + expect.text(dep_case(3, ["CREATE TABLE dep_sensor (id int PRIMARY KEY) USING heap", + "INSERT INTO dep_sensor SELECT g FROM generate_series(0, 16) g", + "ALTER TABLE dep_mar ADD CONSTRAINT dep_fk FOREIGN KEY (sensor) REFERENCES dep_sensor (id)"], + "SELECT count(*) FROM pg_constraint WHERE conname = 'dep_fk'", + ["ALTER TABLE dep_mar DROP CONSTRAINT dep_fk"]), + f"1|2BP01|{db}|r|nofile", "a partition with a foreign key of its own is refused before anything moves") + expect.num(_one(conn, _tier("dep", server_dir / "dep_free.parquet")), mar, + "with the dependents gone, the same partition moves") + + # What the foreign table carries over. + _build(conn, "acl") + with conn.cursor() as cur: + cur.execute("DO $$ BEGIN CREATE ROLE tier_reader_py; EXCEPTION WHEN duplicate_object THEN NULL; END $$") + cur.execute("GRANT SELECT ON acl_mar TO tier_reader_py") + cur.execute("COMMENT ON TABLE acl_mar IS 'March, archived'") + expect.num(_one(conn, _tier("acl", server_dir / "acl.parquet")), mar, + "premise: the cold partition was moved with its grant and comment set") + expect.text(_one(conn, "SELECT relkind::text || '|' || has_table_privilege('tier_reader_py', 'acl_mar', 'SELECT')" + " || '|' || coalesce(obj_description('acl_mar'::regclass, 'pg_class'), '-')" + " FROM pg_class WHERE oid = 'acl_mar'::regclass"), + "f|true|March, archived", "the foreign partition keeps the partition's grants and comment") + with conn.cursor() as cur: + cur.execute("REVOKE ALL ON acl_mar FROM tier_reader_py") + + # 4. An insert through the parent racing the swap. + _build(conn, "race") + rb = _one(conn, "SELECT count(*) FROM race") + out = {} + + def racer(): + with psycopg.connect(dsn, autocommit=True) as m: + m.execute("SET pgcolumnar.tier_fault_after = 'pause'") + try: + out["moved"] = m.execute(_tier("race", server_dir / "race.parquet")).fetchone()[0] + except psycopg.Error as e: + out["mover"] = e.sqlstate + th = threading.Thread(target=racer) + th.start() + time.sleep(0.8) + try: + with psycopg.connect(dsn, autocommit=True) as w: + w.execute("INSERT INTO race VALUES ('2026-03-28 18:00:00+00', 9, 9.9, 'raced')") + out["writer"] = "ok" + except psycopg.Error as e: + out["writer"] = e.sqlstate + th.join(60) + print(f"-- race: {out}") + raced = _one(conn, "SELECT count(*) FROM race WHERE label = 'raced'") + expect.num(1 if ("moved" in out) != ("mover" in out) else 0, 1, + "premise: one side of the race gave way or both went through, and nothing hung") + expect.text(f"{_one(conn, 'SELECT count(*) FROM race')}|{raced}", f"{rb + raced}|{raced}", + "an insert racing the move leaves every row exactly once, whichever side gives way") + + # A parent held by a long reader, and lock_wait. + _build(conn, "held") + hb = _digest(conn, "held") + reader = psycopg.connect(dsn, autocommit=True) + try: + reader.execute("BEGIN") + reader.execute("SELECT count(*) FROM held").fetchone() + t0 = time.monotonic() + state = _sqlstate(conn, "SELECT pgcolumnar.tier_partition('held_mar', 'tq', " + f"'{server_dir}/held.parquet', lock_wait => '800 milliseconds')") + ms = int((time.monotonic() - t0) * 1000) + reader.execute("ROLLBACK") + finally: + reader.close() + print(f"-- gave up after {ms} ms") + expect.text(f"{state}|{'prompt' if ms < 3500 else f'took {ms} ms'}", "55P03|prompt", + "a parent held by a long reader times the move out, as lock_wait says") + expect.text(f"{_digest(conn, 'held')}|{_relkind(conn, 'held')}", f"{hb}|r", + "and the timed-out move leaves the rows local, with nothing attached") + + # The caller's lock_timeout survives the call. + _build(conn, "lt") + with conn.cursor() as cur: + cur.execute("BEGIN") + cur.execute("SET LOCAL lock_timeout = '13s'") + cur.execute("SELECT pgcolumnar.tier_partition('lt_mar', 'tq', " + f"'{server_dir}/lt.parquet', lock_wait => '2 seconds')") + lt_moved = cur.fetchone()[0] + cur.execute("SHOW lock_timeout") + lt = cur.fetchone()[0] + cur.execute("ROLLBACK") + expect.num(lt_moved, mar, "premise: the move ran inside the caller's transaction") + expect.text(lt, "13s", "after the move the caller's lock_timeout is the caller's again") + + # 5. A retry over the file a failed attempt left. + _build(conn, "again") + ab = _digest(conn, "again") + p = server_dir / "again.parquet" + with conn.cursor() as cur: + cur.execute("SET pgcolumnar.tier_fault_after = 'export'") + err = _errtext(conn, _tier("again", p)) + with conn.cursor() as cur: + cur.execute("RESET pgcolumnar.tier_fault_after") + expect.text(f"{'injected fault after export' if 'injected fault after export' in err else err}|" + f"{'file' if p.exists() else 'nofile'}", "injected fault after export|file", + "premise: the first attempt failed after export and left its file") + try: + retried = str(_one(conn, _tier("again", p))) + except psycopg.Error as e: + retried = f"error {e.sqlstate}" + expect.text(f"{retried}|{_digest(conn, 'again')}|{_relkind(conn, 'again')}", + f"{mar}|{ab}|f", "a retry over the failed attempt's file moves the partition, exactly once") diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 3ed98890a..5bc262fa3 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -298,6 +298,7 @@ SUITES=( tablesample temporal tid_range_scan + tier_partition timeseries_recipes truncate_cleanup ttl_expire diff --git a/test/tier_partition.sh b/test/tier_partition.sh index 63726cb9b..89229ea48 100755 --- a/test/tier_partition.sh +++ b/test/tier_partition.sh @@ -75,18 +75,18 @@ JAN_BOUND="FOR VALUES FROM ('2026-01-01 00:00:00+00') TO ('2026-02-01 00:00:00+0 psql_run "SET timezone = 'UTC'; ALTER DATABASE \"$PGC_DB\" SET timezone = 'UTC';" build m check_num "premise: the parent holds every row" "$(q "SELECT count(*) FROM m;")" "16992" -check_num "premise: January holds its share" "$(q "SELECT count(*) FROM m_jan;")" "8928" +check_num "premise: the cold partition holds its share" "$(q "SELECT count(*) FROM m_jan;")" "8928" before="$(digest m)" -check "premise: January is a local columnar partition with its bound" "$(jan_shape m)" "r:pgcolumnar:$JAN_BOUND" +check "premise: the cold partition is local and columnar, with its bound" "$(jan_shape m)" "r:pgcolumnar:$JAN_BOUND" jan_storage="$(q "SELECT pgcolumnar.get_storage_id('m_jan');")" # ---- the move --------------------------------------------------------------- -check "tier_partition moves January and returns the rows it moved" \ +check "tier_partition moves the cold partition and returns the rows it moved" \ "$(q "SELECT pgcolumnar.tier_partition('m_jan', 'tier_pq', '$W/m_jan.parquet');")" "8928" # The relkind rides along: with nothing moved the digest is trivially unchanged. check "the answers over the parent are the same after the move" \ "$(digest m)|$(q "SELECT relkind FROM pg_class WHERE oid = 'm_jan'::regclass;")" "$before|f" -check "January is now a foreign partition with its own name and bound" "$(jan_shape m)" "f:-:$JAN_BOUND" +check "the cold partition is now a foreign partition with its own name and bound" "$(jan_shape m)" "f:-:$JAN_BOUND" check "the foreign partition reads the exported file" \ "$(q "SELECT srvname || ':' || array_to_string(ftoptions, ',') FROM pg_foreign_table t JOIN pg_foreign_server s ON s.oid = t.ftserver WHERE t.ftrelid = 'm_jan'::regclass;")" \ @@ -100,9 +100,9 @@ check_num "nor any row-group row" \ # ---- pruning over the mixed parent ------------------------------------------ plan_jan="$(q "EXPLAIN SELECT max(v) FROM m WHERE ts < '2026-01-20';")" plan_feb="$(q "EXPLAIN SELECT max(v) FROM m WHERE ts >= '2026-02-10';")" -check "a January query scans only the foreign partition" \ +check "a query on the tiered range scans only the foreign partition" \ "$(grep -c 'Foreign Scan' <<<"$plan_jan")|$(grep -c 'm_feb' <<<"$plan_jan")" "1|0" -check "a February query scans only the local partition, beside a foreign one" \ +check "a query on the other range scans only the local partition, beside a foreign one" \ "$(grep -c 'Foreign Scan' <<<"$plan_feb")|$(grep -c 'm_feb' <<<"$plan_feb")|$(grep -c 'Foreign Scan' <<<"$plan_jan")" "0|1|1" check "an insert routed to the tiered range is refused" \ "$(sqlstate "INSERT INTO m VALUES ('2026-01-15', 1, 1, 'x');")" "0A000" @@ -115,6 +115,8 @@ psql_run "CREATE TABLE lone (ts timestamptz, v int) USING pgcolumnar;" check "a table that is not a partition is refused" \ "$(sqlstate "SELECT pgcolumnar.tier_partition('lone', 'tier_pq', '$W/lone.parquet');")" "42809" psql_run "CREATE TABLE m_mar PARTITION OF m FOR VALUES FROM ('2026-03-01') TO ('2026-04-01') USING heap;" +# Refused by export_parquet, the first step, so nothing has moved: the arm holds +# the property, not which line of tier_partition raises it. check "a partition that is not columnar is refused" \ "$(sqlstate "SELECT pgcolumnar.tier_partition('m_mar', 'tier_pq', '$W/m_mar.parquet');")" "42809" psql_run "CREATE TABLE m_def PARTITION OF m DEFAULT USING pgcolumnar;" @@ -126,7 +128,7 @@ check "a server of another wrapper is refused" \ check "an existing path is refused, so no file a table reads is overwritten" \ "$(sqlstate "SELECT pgcolumnar.tier_partition('m_feb', 'tier_pq', '$W/m_jan.parquet');")" "58P02" check "and the refusals moved nothing" "$(digest m)" "$rb" -check "premise: February is still local after the refusals" \ +check "premise: the other partition is still local after the refusals" \ "$(q "SELECT relkind FROM pg_class WHERE oid = 'm_feb'::regclass;")" "r" # ---- the parent's lock ------------------------------------------------------ @@ -138,12 +140,185 @@ lock="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgre WHERE relation = 'k'::regclass AND pid = pg_backend_pid() AND granted" \ -c "ROLLBACK" 2>&1)" # Line 2 is the call's result: a rollback of a call that never ran proves nothing. -check "premise: inside the transaction the call moved January" "$(sed -n 2p <<<"$lock")" "8928" +check "premise: inside the transaction the call moved the cold partition" "$(sed -n 2p <<<"$lock")" "8928" check "the call holds ACCESS EXCLUSIVE on the parent until commit, as the docs say" \ "$(grep -o 'AccessExclusiveLock' <<<"$lock")" "AccessExclusiveLock" check "a rollback after the whole call leaves the rows local, exactly once" \ "$(digest k)|$(jan_shape k)" "$kb|r:pgcolumnar:$JAN_BOUND" +# ---- a write in flight when the move starts --------------------------------- +# Without the SHARE lock the export does not see an uncommitted insert, DETACH +# then waits for it to commit, and DROP takes the row with the partition: lost. +# With it, the move waits for the insert and exports the row. +build w +wb="$(q "SELECT count(*) FROM w;")"; wj="$(q "SELECT count(*) FROM w_jan;")" +env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -c "BEGIN" -c "INSERT INTO w_jan VALUES ('2026-01-31 23:59:00+00', 99, 4242, 'in flight')" \ + -c "SELECT pg_sleep(3)" -c "COMMIT" >/dev/null 2>&1 & +writer=$! +sleep 1 +t0=$(date +%s%N) +moved="$(q "SELECT pgcolumnar.tier_partition('w_jan', 'tier_pq', '$W/w_jan.parquet');")" +waited=$(( ($(date +%s%N) - t0) / 1000000 )) +wait "$writer" +echo "-- the move waited ${waited} ms for the writer" +check_num "premise: the move started while the insert was in flight, and waited for it" \ + "$([ "$waited" -ge 1000 ] && echo 1 || echo "waited only $waited ms")" "1" +check "a write in flight when the move starts is in the exported rows" \ + "$moved|$(q "SELECT count(*) FROM w;")|$(q "SELECT count(*) FROM w WHERE note = 'in flight';")" \ + "$((wj + 1))|$((wb + 1))|1" + +# ---- isolation: a writer that commits between the caller's snapshot and the lock +# Measured by @jdatcmd on #1295: the SHARE lock does not refresh a REPEATABLE READ +# snapshot, so the export missed rows that DROP then took with the partition. +iso_sqlstate() { sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p' "$1" | head -1; } +n=0 +for lvl in "read committed" "repeatable read" "serializable"; do + n=$((n + 1)); build i + ib="$(q "SELECT count(*) FROM i;")"; ij="$(q "SELECT count(*) FROM i_jan;")" + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -v VERBOSITY=sqlstate -c "BEGIN ISOLATION LEVEL $lvl" -c "SELECT 'seen=' || count(*) FROM i_jan" \ + -c "SELECT pg_sleep(2)" \ + -c "SELECT 'moved=' || pgcolumnar.tier_partition('i_jan', 'tier_pq', '$W/i_$n.parquet')" \ + -c "COMMIT" > "$W/iso_$n.out" 2>&1 & + mover=$! + sleep 1 + q "INSERT INTO i_jan VALUES ('2026-01-20 12:00:00+00', 7, 7, 'committed between');" >/dev/null + wait "$mover" + check_num "premise: at $lvl the caller's snapshot was taken before the writer committed" \ + "$(sed -n 's/^seen=//p' "$W/iso_$n.out")" "$ij" + check "at $lvl a row committed between the caller's snapshot and the lock survives" \ + "$(q "SELECT count(*) FROM i;")|$(q "SELECT count(*) FROM i WHERE note = 'committed between';")" "$((ib + 1))|1" + if [ "$lvl" = "read committed" ]; then + check "at read committed the move goes ahead and exports that row" \ + "$(sed -n 's/^moved=//p' "$W/iso_$n.out")|$(q "SELECT relkind FROM pg_class WHERE oid = 'i_jan'::regclass;")" "$((ij + 1))|f" + else + check "at $lvl the move is refused, and the rows stay local" \ + "$(iso_sqlstate "$W/iso_$n.out")|$(q "SELECT relkind FROM pg_class WHERE oid = 'i_jan'::regclass;")" "25000|r" + fi +done + +# ---- refused up front: what ATTACH or DROP would trip over -------------------- +# Each refusal leaves the rows local and writes no file: before this, a UNIQUE +# index on the parent surfaced at ATTACH, after the export, as an orphan file. +refused_cleanly() { # refused_cleanly TABLE PATH -> "rows|relkind|file" + echo "$(digest "$1")|$(q "SELECT relkind FROM pg_class WHERE oid = '${1}_jan'::regclass;")|$([ -e "$2" ] && echo file || echo nofile)" +} +build u +psql_run "CREATE UNIQUE INDEX u_key ON u (ts);" +ub="$(digest u)" +check "a parent with a UNIQUE index is refused before anything moves" \ + "$(sqlstate "SELECT pgcolumnar.tier_partition('u_jan', 'tier_pq', '$W/u.parquet');")|$(refused_cleanly u "$W/u.parquet")" \ + "0A000|$ub|r|nofile" + +build d +db="$(digest d)" +# Each dependent made and removed in statements of its own, with a premise that it +# exists: a multi-statement -c is one transaction, and in the first draft a failed +# CREATE rolled back the DROP of the case before, so the foreign-key arm passed +# on a leftover trigger and tested nothing. +dep_case() { # dep_case PATH PREMISE_SQL -> "premise|sqlstate|rows|relkind|file" + echo "$(q "$2")|$(sqlstate "SELECT pgcolumnar.tier_partition('d_jan', 'tier_pq', '$1');")|$(refused_cleanly d "$1")" +} +psql_run "CREATE VIEW d_view AS SELECT * FROM d_jan;" +check "a partition a view depends on is refused before anything moves" \ + "$(dep_case "$W/d1.parquet" "SELECT count(*) FROM pg_views WHERE viewname = 'd_view';")" "1|2BP01|$db|r|nofile" +psql_run "DROP VIEW d_view;" +psql_run "ALTER TABLE d_jan ENABLE ROW LEVEL SECURITY;" +psql_run "CREATE POLICY d_pol ON d_jan USING (dev > 0);" +check "a partition with a row-level policy is refused before anything moves" \ + "$(dep_case "$W/d2.parquet" "SELECT count(*) FROM pg_policy WHERE polname = 'd_pol';")" "1|2BP01|$db|r|nofile" +psql_run "DROP POLICY d_pol ON d_jan;" +psql_run "ALTER TABLE d_jan DISABLE ROW LEVEL SECURITY;" +psql_run "CREATE FUNCTION d_trg() RETURNS trigger LANGUAGE plpgsql AS \$\$BEGIN RETURN NEW; END\$\$;" +psql_run "CREATE TRIGGER d_local BEFORE UPDATE ON d_jan FOR EACH ROW EXECUTE FUNCTION d_trg();" +check "a partition with a trigger of its own is refused before anything moves" \ + "$(dep_case "$W/d3.parquet" "SELECT count(*) FROM pg_trigger WHERE tgname = 'd_local';")" "1|2BP01|$db|r|nofile" +psql_run "DROP TRIGGER d_local ON d_jan;" +# A foreign key REFERENCING a columnar table is refused by pgcolumnar itself (row +# locking), so the case that can arise is one the partition holds: a foreign +# table cannot carry it, and DROP would take it. +psql_run "CREATE TABLE d_dev (k int PRIMARY KEY) USING heap;" +psql_run "INSERT INTO d_dev SELECT g FROM generate_series(0, 12) g;" +psql_run "ALTER TABLE d_jan ADD CONSTRAINT d_fk FOREIGN KEY (dev) REFERENCES d_dev (k);" +check "a partition with a foreign key of its own is refused before anything moves" \ + "$(dep_case "$W/d4.parquet" "SELECT count(*) FROM pg_constraint WHERE conname = 'd_fk';")" "1|2BP01|$db|r|nofile" +psql_run "ALTER TABLE d_jan DROP CONSTRAINT d_fk;" +check_num "with the dependents gone, the same partition moves" \ + "$(q "SELECT pgcolumnar.tier_partition('d_jan', 'tier_pq', '$W/d5.parquet');")" "8928" + +# ---- what the foreign table carries over -------------------------------------- +build a +psql_run "CREATE ROLE tier_reader; GRANT SELECT ON a_jan TO tier_reader; + COMMENT ON TABLE a_jan IS 'January, cold';" +check_num "premise: the cold partition was moved with its grant and comment set" \ + "$(q "SELECT pgcolumnar.tier_partition('a_jan', 'tier_pq', '$W/a.parquet');")" "8928" +check "the foreign partition keeps the partition's grants and comment" \ + "$(q "SELECT relkind::text || '|' || has_table_privilege('tier_reader', 'a_jan', 'SELECT') || '|' || + coalesce(obj_description('a_jan'::regclass, 'pg_class'), '-') FROM pg_class WHERE oid = 'a_jan'::regclass;")" \ + "f|true|January, cold" + +# ---- an insert routed to the cold partition, racing the swap ------------------ +# The mover holds SHARE on the partition and then asks for ACCESS EXCLUSIVE on the +# parent; an insert through the parent holds ROW EXCLUSIVE on the parent and waits +# for the partition. Whichever side the deadlock detector aborts, every row must +# be exactly once. +build r +rb="$(q "SELECT count(*) FROM r;")" +env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At -v VERBOSITY=sqlstate \ + -c "SET pgcolumnar.tier_fault_after = 'pause'" \ + -c "SELECT 'moved=' || pgcolumnar.tier_partition('r_jan', 'tier_pq', '$W/r.parquet')" > "$W/race_mover.out" 2>&1 & +mover=$! +sleep 1 +env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At -v VERBOSITY=sqlstate \ + -c "INSERT INTO r VALUES ('2026-01-26 08:00:00+00', 3, 3, 'raced')" > "$W/race_writer.out" 2>&1 +wait "$mover" +echo "-- race: mover [$(tr '\n' ' ' < "$W/race_mover.out")] writer [$(tr '\n' ' ' < "$W/race_writer.out")]" +raced="$(q "SELECT count(*) FROM r WHERE note = 'raced';")" +check_num "premise: one side of the race gave way or both went through, and nothing hung" \ + "$(grep -c 'moved=\|ERROR' "$W/race_mover.out")" "1" +check "an insert racing the move leaves every row exactly once, whichever side gives way" \ + "$(q "SELECT count(*) FROM r;")|$raced" "$((rb + raced))|$raced" + +# ---- a bounded wait for the parent's lock ------------------------------------- +build l +lb="$(digest l)" +env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -c "BEGIN" -c "SELECT count(*) FROM l" -c "SELECT pg_sleep(6)" -c "COMMIT" >/dev/null 2>&1 & +reader=$! +sleep 1 +t0=$(date +%s%N) +lk="$(sqlstate "SELECT pgcolumnar.tier_partition('l_jan', 'tier_pq', '$W/l.parquet', lock_wait => '1 second');")" +lms=$(( ($(date +%s%N) - t0) / 1000000 )) +wait "$reader" +echo "-- the move gave up on the parent's lock after ${lms} ms" +check "a parent held by a long reader times the move out, as lock_wait says" \ + "$lk|$([ "$lms" -lt 4000 ] && echo prompt || echo "took $lms ms")" "55P03|prompt" +check "and the timed-out move leaves the rows local, with nothing attached" \ + "$(digest l)|$(q "SELECT relkind FROM pg_class WHERE oid = 'l_jan'::regclass;")" "$lb|r" + +# The bound is set for the move's own locks only: set_config(..., true) would +# otherwise last to the end of the caller's transaction. +build t +lt="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -c "BEGIN" -c "SET LOCAL lock_timeout = '7s'" \ + -c "SELECT 'moved=' || pgcolumnar.tier_partition('t_jan', 'tier_pq', '$W/t.parquet', lock_wait => '3 seconds')" \ + -c "SHOW lock_timeout" -c "ROLLBACK" 2>&1)" +check "premise: the move ran inside the caller's transaction" "$(sed -n 's/^moved=//p' <<<"$lt")" "8928" +check "after the move the caller's lock_timeout is the caller's again" "$(sed -n '/^moved=/{n;p}' <<<"$lt")" "7s" + +# ---- a retry after a failure, over the file the failure left ------------------ +build o +ob="$(digest o)" +o_err="$(errtext "SET pgcolumnar.tier_fault_after = 'export'" \ + "SELECT pgcolumnar.tier_partition('o_jan', 'tier_pq', '$W/o.parquet')")" +check "premise: the first attempt failed after export and left its file" \ + "$(grep -o 'injected fault after export' <<<"$o_err")|$([ -e "$W/o.parquet" ] && echo file || echo nofile)" \ + "injected fault after export|file" +check "a retry over the failed attempt's file moves the partition, exactly once" \ + "$(q "SELECT pgcolumnar.tier_partition('o_jan', 'tier_pq', '$W/o.parquet');")|$(digest o)|$(q "SELECT relkind FROM pg_class WHERE oid = 'o_jan'::regclass;")" \ + "8928|$ob|f" + # ---- a failure after each step ---------------------------------------------- for step in export detach drop create attach; do build f From 83993b132eb2440aeec3ba90df1c3cda00ef8ba1 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 00:03:46 +0000 Subject: [PATCH 3/9] test: four tier_partition checks a mutation reddens are not premises (#1236) 'the cold partition was moved with its grant and comment set' and 'the other partition is still local after the refusals' are reddened by the suffix and path-check mutations; 'inside the transaction the call moved the cold partition' and 'the move ran inside the caller's transaction' by return_zero. A check a repository change can redden is an arm. Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_tier_partition.py | 8 ++++---- test/tier_partition.sh | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/test/pytest/test_tier_partition.py b/test/pytest/test_tier_partition.py index d88e0cd25..60b60dab2 100644 --- a/test/pytest/test_tier_partition.py +++ b/test/pytest/test_tier_partition.py @@ -154,7 +154,7 @@ def plan(sql): "58P02", "an existing path is refused, so no file a table reads is overwritten") expect.text(_digest(conn, "s"), rb, "and the refusals moved nothing") expect.text(_one(conn, "SELECT relkind::text FROM pg_class WHERE oid = 's_apr'::regclass"), "r", - "premise: the other partition is still local after the refusals") + "the other partition is still local after the refusals") # The parent's lock, and a rollback of the whole call. _build(conn, "k") @@ -167,7 +167,7 @@ def plan(sql): " WHERE relation = 'k'::regclass AND pid = pg_backend_pid() AND granted") locks = cur.fetchone()[0] cur.execute("ROLLBACK") - expect.num(moved, mar, "premise: inside the transaction the call moved the cold partition") + expect.num(moved, mar, "inside the transaction the call moved the cold partition") expect.text("AccessExclusiveLock" if "AccessExclusiveLock" in locks else locks, "AccessExclusiveLock", "the call holds ACCESS EXCLUSIVE on the parent until commit, as the docs say") expect.text(f"{_digest(conn, 'k')}|{_mar_shape(conn, 'k')}", f"{kb}|r:pgcolumnar:{BOUND}", @@ -341,7 +341,7 @@ def dep_case(i, make, premise_sql, undo): cur.execute("GRANT SELECT ON acl_mar TO tier_reader_py") cur.execute("COMMENT ON TABLE acl_mar IS 'March, archived'") expect.num(_one(conn, _tier("acl", server_dir / "acl.parquet")), mar, - "premise: the cold partition was moved with its grant and comment set") + "the cold partition was moved with its grant and comment set") expect.text(_one(conn, "SELECT relkind::text || '|' || has_table_privilege('tier_reader_py', 'acl_mar', 'SELECT')" " || '|' || coalesce(obj_description('acl_mar'::regclass, 'pg_class'), '-')" " FROM pg_class WHERE oid = 'acl_mar'::regclass"), @@ -409,7 +409,7 @@ def racer(): cur.execute("SHOW lock_timeout") lt = cur.fetchone()[0] cur.execute("ROLLBACK") - expect.num(lt_moved, mar, "premise: the move ran inside the caller's transaction") + expect.num(lt_moved, mar, "the move ran inside the caller's transaction") expect.text(lt, "13s", "after the move the caller's lock_timeout is the caller's again") # 5. A retry over the file a failed attempt left. diff --git a/test/tier_partition.sh b/test/tier_partition.sh index 89229ea48..4c13ac35e 100755 --- a/test/tier_partition.sh +++ b/test/tier_partition.sh @@ -128,7 +128,7 @@ check "a server of another wrapper is refused" \ check "an existing path is refused, so no file a table reads is overwritten" \ "$(sqlstate "SELECT pgcolumnar.tier_partition('m_feb', 'tier_pq', '$W/m_jan.parquet');")" "58P02" check "and the refusals moved nothing" "$(digest m)" "$rb" -check "premise: the other partition is still local after the refusals" \ +check "the other partition is still local after the refusals" \ "$(q "SELECT relkind FROM pg_class WHERE oid = 'm_feb'::regclass;")" "r" # ---- the parent's lock ------------------------------------------------------ @@ -140,7 +140,7 @@ lock="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgre WHERE relation = 'k'::regclass AND pid = pg_backend_pid() AND granted" \ -c "ROLLBACK" 2>&1)" # Line 2 is the call's result: a rollback of a call that never ran proves nothing. -check "premise: inside the transaction the call moved the cold partition" "$(sed -n 2p <<<"$lock")" "8928" +check "inside the transaction the call moved the cold partition" "$(sed -n 2p <<<"$lock")" "8928" check "the call holds ACCESS EXCLUSIVE on the parent until commit, as the docs say" \ "$(grep -o 'AccessExclusiveLock' <<<"$lock")" "AccessExclusiveLock" check "a rollback after the whole call leaves the rows local, exactly once" \ @@ -251,7 +251,7 @@ check_num "with the dependents gone, the same partition moves" \ build a psql_run "CREATE ROLE tier_reader; GRANT SELECT ON a_jan TO tier_reader; COMMENT ON TABLE a_jan IS 'January, cold';" -check_num "premise: the cold partition was moved with its grant and comment set" \ +check_num "the cold partition was moved with its grant and comment set" \ "$(q "SELECT pgcolumnar.tier_partition('a_jan', 'tier_pq', '$W/a.parquet');")" "8928" check "the foreign partition keeps the partition's grants and comment" \ "$(q "SELECT relkind::text || '|' || has_table_privilege('tier_reader', 'a_jan', 'SELECT') || '|' || @@ -304,7 +304,7 @@ lt="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -c "BEGIN" -c "SET LOCAL lock_timeout = '7s'" \ -c "SELECT 'moved=' || pgcolumnar.tier_partition('t_jan', 'tier_pq', '$W/t.parquet', lock_wait => '3 seconds')" \ -c "SHOW lock_timeout" -c "ROLLBACK" 2>&1)" -check "premise: the move ran inside the caller's transaction" "$(sed -n 's/^moved=//p' <<<"$lt")" "8928" +check "the move ran inside the caller's transaction" "$(sed -n 's/^moved=//p' <<<"$lt")" "8928" check "after the move the caller's lock_timeout is the caller's again" "$(sed -n '/^moved=/{n;p}' <<<"$lt")" "7s" # ---- a retry after a failure, over the file the failure left ------------------ From d28a74a75fff9d65c74f1dbaaa90236b3f501a04 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 00:04:20 +0000 Subject: [PATCH 4/9] test, docs: register tier_partition and describe it (#1295) cluster_tests 500 -> 502 by collection. The ledger rows come from a five-major run and sixteen mutations on this tree, in a later commit. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f49d40fe0..f4e171c8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -171,6 +171,19 @@ true until the next version shipped. pulled up into a join read every row group (114 blocks of a 103-block table for one new group), because PostgreSQL builds a TID range scan only from a relation's own restrictions; `OFFSET 0` keeps it a scan (15 blocks). +- `pgcolumnar.tier_partition(partition, server, path, lock_wait)` moves a cold + columnar partition to a Parquet file and attaches the file in its place as a + foreign partition with the same name, bounds, grants and comment (#1295). It + runs as one transaction at READ COMMITTED: the export holds a `SHARE` lock on + the partition, and the parent is held `ACCESS EXCLUSIVE` from the detach until + commit, which covers catalog work only. Every lock waits at most `lock_wait`. + A failure at any step leaves the rows exactly once, still local; the tests + prove it with a developer fault after each step. It refuses, before anything + moves, `REPEATABLE READ` and `SERIALIZABLE` (an older snapshot would drop + rows committed just before the lock), a parent with a `UNIQUE` index, and a + partition that a view, a policy, its own trigger or its own foreign key + depends on. A path a foreign table in the database reads is refused; a retry + over a failed attempt's file is allowed. ### Changed From 9ba980597018ed7fb5b031a70113b2d7ffddc31d Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 00:17:36 +0000 Subject: [PATCH 5/9] test: tier_partition ledger, from a five-major run and sixteen mutations on this tree (#1295) 65 rows; 32 dated by sixteen mutations merged one log at a time with --mutation and --target, no premise among them; 33 'never' (15 premises, 18 arms the budget comment names). Census 1694 -> 1727. The twin's loops iterate literals, so the loop-coverage guard sees them non-empty by construction. Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_tier_partition.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/pytest/test_tier_partition.py b/test/pytest/test_tier_partition.py index 60b60dab2..93fbfd671 100644 --- a/test/pytest/test_tier_partition.py +++ b/test/pytest/test_tier_partition.py @@ -27,7 +27,6 @@ import psycopg BOUND = "FOR VALUES FROM ('2026-03-01 00:00:00+00') TO ('2026-04-01 00:00:00+00')" -STEPS = ("export", "detach", "drop", "create", "attach") def _one(conn, sql): @@ -201,7 +200,7 @@ def mover(): f"{mar + 1}|{wb + 1}|1", "a write in flight when the move starts is in the exported rows") # A failure after each step. - for step in STEPS: + for step in ("export", "detach", "drop", "create", "attach"): _build(conn, "f") fb = _digest(conn, "f") with conn.cursor() as cur: @@ -247,7 +246,7 @@ def test_tier_partition_refuses_what_it_cannot_move_and_survives_concurrency( mar = 6378 # 1. Isolation. - for n, lvl in enumerate(("read committed", "repeatable read", "serializable")): + for n, lvl in ((0, "read committed"), (1, "repeatable read"), (2, "serializable")): t = f"iso{n}" _build(conn, t) before = _one(conn, f"SELECT count(*) FROM {t}") From cae84af7c861b35012a6464a77ba8c08719dd4d1 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 00:58:55 +0000 Subject: [PATCH 6/9] fix: tier_partition quotes reg* names once, and carries collation and generated columns (#1317 review) Four inputs @jdatcmd measured failing after the export, each leaving an orphan file, each now an arm red on 5eb86f2 and green: - a schema, owner or grantee whose name needs quoting: %I over regnamespace or regrole quoted them twice (3F000, 42704); reg* casts now go through %s; - a column with its own collation (42P21 at ATTACH): COLLATE from attcollation when it is not the type's own; - a stored generated column (42804 at ATTACH), and on PostgreSQL 18 a virtual one: the foreign table declares the same generation expression. Measured on 17 and 18 that ATTACH accepts both, and that an identity column needs nothing. The docs say the path check compares the text of the path. tier_partition.sh 71/71 and the twin 71 on PG18; the virtual arm is a named SKIP / UNRUN below 18. Co-Authored-By: Claude Opus 5.5 --- docs/how-to.md | 6 +- docs/sql-reference.md | 3 +- pgcolumnar--1.0-alpha4--1.0-alpha5.sql | 29 ++++++--- pgcolumnar--1.0-alpha5.sql | 29 ++++++--- test/pytest/expected_unrunnable.txt | 5 ++ test/pytest/test_tier_partition.py | 81 ++++++++++++++++++++++++++ test/tier_partition.sh | 56 ++++++++++++++++++ 7 files changed, 192 insertions(+), 17 deletions(-) diff --git a/docs/how-to.md b/docs/how-to.md index c402299d3..c266458e5 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -662,8 +662,10 @@ every row is stored exactly once. **A failed attempt leaves its file behind.** A transaction cannot remove a file that it wrote. You can retry with the same `path`: the function overwrites a file that no foreign table in the database reads. It refuses a `path` that a -foreign table in this database reads (`58P02`). It cannot see readers outside -the database, so do not reuse a path that another system reads. +foreign table in this database reads (`58P02`). The check compares the text of +the path, so the same file spelled another way, for example through a symbolic +link, is not recognised. It also cannot see readers outside the database, so do +not reuse a path that another system reads. The function refuses these partitions before it moves anything: diff --git a/docs/sql-reference.md b/docs/sql-reference.md index a991ed48a..acba89adc 100644 --- a/docs/sql-reference.md +++ b/docs/sql-reference.md @@ -627,7 +627,8 @@ source, when the rollup was current before the rewrite. Moves the columnar partition `partition` to a Parquet file at `path`. Then attaches the file as a foreign table of `server`, which must use the `pgcolumnar_parquet` wrapper. The foreign table takes the partition's name, -columns, owner, bounds, grants and comment. Returns the number of rows moved. +columns (with their collations and generated expressions), owner, bounds, +grants and comment. Returns the number of rows moved. The function runs as one transaction, at READ COMMITTED only (`25000` otherwise). It holds `ACCESS EXCLUSIVE` on the parent from the detach until diff --git a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql index ed68dc3d4..f805c3186 100644 --- a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql @@ -1506,10 +1506,22 @@ BEGIN USING ERRCODE = 'duplicate_file'; END IF; - SELECT string_agg(format('%I %s%s', a.attname, format_type(a.atttypid, a.atttypmod), + -- Every column as ATTACH will compare it: type, collation where it is not + -- the type's own, a stored or virtual generation expression, and NOT NULL + -- (#1317 review: each of collation and generation failed at ATTACH, after + -- the export, leaving an orphan file). + SELECT string_agg(format('%I %s%s%s%s', a.attname, format_type(a.atttypid, a.atttypmod), + CASE WHEN a.attcollation <> 0 AND a.attcollation <> t.typcollation + THEN ' COLLATE ' || a.attcollation::regcollation::text ELSE '' END, + CASE a.attgenerated + WHEN 's' THEN format(' GENERATED ALWAYS AS (%s) STORED', pg_get_expr(d.adbin, d.adrelid)) + WHEN 'v' THEN format(' GENERATED ALWAYS AS (%s) VIRTUAL', pg_get_expr(d.adbin, d.adrelid)) + ELSE '' END, CASE WHEN a.attnotnull THEN ' NOT NULL' ELSE '' END), ', ' ORDER BY a.attnum) INTO cols FROM pg_attribute a + JOIN pg_type t ON t.oid = a.atttypid + LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum WHERE a.attrelid = partition AND a.attnum > 0 AND NOT a.attisdropped; note := obj_description(partition, 'pg_class'); @@ -1537,22 +1549,25 @@ BEGIN EXECUTE format('DROP TABLE %s', partition); IF fault = 'drop' THEN RAISE EXCEPTION 'injected fault after drop'; END IF; - EXECUTE format('CREATE FOREIGN TABLE %I.%I (%s) SERVER %I OPTIONS (path %L)', + -- regnamespace and regrole quote themselves when needed, so %s, never %I: + -- %I quoted them a second time (#1317 review: a schema "Tier" or an owner + -- "Owner X" failed after the export). + EXECUTE format('CREATE FOREIGN TABLE %s.%I (%s) SERVER %I OPTIONS (path %L)', rel.relnamespace::regnamespace, rel.relname, cols, server, path); - EXECUTE format('ALTER FOREIGN TABLE %I.%I OWNER TO %I', + EXECUTE format('ALTER FOREIGN TABLE %s.%I OWNER TO %s', rel.relnamespace::regnamespace, rel.relname, rel.relowner::regrole); FOR acl IN SELECT * FROM aclexplode(rel.relacl) x WHERE x.grantee <> rel.relowner LOOP - EXECUTE format('GRANT %s ON %I.%I TO %s%s', acl.privilege_type, + EXECUTE format('GRANT %s ON %s.%I TO %s%s', acl.privilege_type, rel.relnamespace::regnamespace, rel.relname, - CASE WHEN acl.grantee = 0 THEN 'PUBLIC' ELSE quote_ident(acl.grantee::regrole::text) END, + CASE WHEN acl.grantee = 0 THEN 'PUBLIC' ELSE acl.grantee::regrole::text END, CASE WHEN acl.is_grantable THEN ' WITH GRANT OPTION' ELSE '' END); END LOOP; IF note IS NOT NULL THEN - EXECUTE format('COMMENT ON FOREIGN TABLE %I.%I IS %L', + EXECUTE format('COMMENT ON FOREIGN TABLE %s.%I IS %L', rel.relnamespace::regnamespace, rel.relname, note); END IF; IF fault = 'create' THEN RAISE EXCEPTION 'injected fault after create'; END IF; - EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I %s', + EXECUTE format('ALTER TABLE %s ATTACH PARTITION %s.%I %s', parent, rel.relnamespace::regnamespace, rel.relname, bound); IF fault = 'attach' THEN RAISE EXCEPTION 'injected fault after attach'; END IF; -- No ANALYZE: pgcolumnar_parquet has no AnalyzeForeignTable, so there is no diff --git a/pgcolumnar--1.0-alpha5.sql b/pgcolumnar--1.0-alpha5.sql index 0d72b8e2a..05e156e74 100644 --- a/pgcolumnar--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha5.sql @@ -2696,10 +2696,22 @@ BEGIN USING ERRCODE = 'duplicate_file'; END IF; - SELECT string_agg(format('%I %s%s', a.attname, format_type(a.atttypid, a.atttypmod), + -- Every column as ATTACH will compare it: type, collation where it is not + -- the type's own, a stored or virtual generation expression, and NOT NULL + -- (#1317 review: each of collation and generation failed at ATTACH, after + -- the export, leaving an orphan file). + SELECT string_agg(format('%I %s%s%s%s', a.attname, format_type(a.atttypid, a.atttypmod), + CASE WHEN a.attcollation <> 0 AND a.attcollation <> t.typcollation + THEN ' COLLATE ' || a.attcollation::regcollation::text ELSE '' END, + CASE a.attgenerated + WHEN 's' THEN format(' GENERATED ALWAYS AS (%s) STORED', pg_get_expr(d.adbin, d.adrelid)) + WHEN 'v' THEN format(' GENERATED ALWAYS AS (%s) VIRTUAL', pg_get_expr(d.adbin, d.adrelid)) + ELSE '' END, CASE WHEN a.attnotnull THEN ' NOT NULL' ELSE '' END), ', ' ORDER BY a.attnum) INTO cols FROM pg_attribute a + JOIN pg_type t ON t.oid = a.atttypid + LEFT JOIN pg_attrdef d ON d.adrelid = a.attrelid AND d.adnum = a.attnum WHERE a.attrelid = partition AND a.attnum > 0 AND NOT a.attisdropped; note := obj_description(partition, 'pg_class'); @@ -2727,22 +2739,25 @@ BEGIN EXECUTE format('DROP TABLE %s', partition); IF fault = 'drop' THEN RAISE EXCEPTION 'injected fault after drop'; END IF; - EXECUTE format('CREATE FOREIGN TABLE %I.%I (%s) SERVER %I OPTIONS (path %L)', + -- regnamespace and regrole quote themselves when needed, so %s, never %I: + -- %I quoted them a second time (#1317 review: a schema "Tier" or an owner + -- "Owner X" failed after the export). + EXECUTE format('CREATE FOREIGN TABLE %s.%I (%s) SERVER %I OPTIONS (path %L)', rel.relnamespace::regnamespace, rel.relname, cols, server, path); - EXECUTE format('ALTER FOREIGN TABLE %I.%I OWNER TO %I', + EXECUTE format('ALTER FOREIGN TABLE %s.%I OWNER TO %s', rel.relnamespace::regnamespace, rel.relname, rel.relowner::regrole); FOR acl IN SELECT * FROM aclexplode(rel.relacl) x WHERE x.grantee <> rel.relowner LOOP - EXECUTE format('GRANT %s ON %I.%I TO %s%s', acl.privilege_type, + EXECUTE format('GRANT %s ON %s.%I TO %s%s', acl.privilege_type, rel.relnamespace::regnamespace, rel.relname, - CASE WHEN acl.grantee = 0 THEN 'PUBLIC' ELSE quote_ident(acl.grantee::regrole::text) END, + CASE WHEN acl.grantee = 0 THEN 'PUBLIC' ELSE acl.grantee::regrole::text END, CASE WHEN acl.is_grantable THEN ' WITH GRANT OPTION' ELSE '' END); END LOOP; IF note IS NOT NULL THEN - EXECUTE format('COMMENT ON FOREIGN TABLE %I.%I IS %L', + EXECUTE format('COMMENT ON FOREIGN TABLE %s.%I IS %L', rel.relnamespace::regnamespace, rel.relname, note); END IF; IF fault = 'create' THEN RAISE EXCEPTION 'injected fault after create'; END IF; - EXECUTE format('ALTER TABLE %s ATTACH PARTITION %I.%I %s', + EXECUTE format('ALTER TABLE %s ATTACH PARTITION %s.%I %s', parent, rel.relnamespace::regnamespace, rel.relname, bound); IF fault = 'attach' THEN RAISE EXCEPTION 'injected fault after attach'; END IF; -- No ANALYZE: pgcolumnar_parquet has no AnalyzeForeignTable, so there is no diff --git a/test/pytest/expected_unrunnable.txt b/test/pytest/expected_unrunnable.txt index caaa772d3..fff841aca 100644 --- a/test/pytest/expected_unrunnable.txt +++ b/test/pytest/expected_unrunnable.txt @@ -55,3 +55,8 @@ # hll 2.19 does not build against PostgreSQL 19 (FuncnameGetCandidates gained an # argument), so the hll half of #1296's compatibility test is UNSUPPORTED_MAJOR there. 19 test_approx_aggregates.py::test_hll_over_columnar + +# PostgreSQL 15 to 17: virtual generated columns arrived in PostgreSQL 18. +15 test_tier_partition.py::test_tier_partition_moves_a_virtual_generated_column +16 test_tier_partition.py::test_tier_partition_moves_a_virtual_generated_column +17 test_tier_partition.py::test_tier_partition_moves_a_virtual_generated_column diff --git a/test/pytest/test_tier_partition.py b/test/pytest/test_tier_partition.py index 93fbfd671..3321a5b99 100644 --- a/test/pytest/test_tier_partition.py +++ b/test/pytest/test_tier_partition.py @@ -411,6 +411,60 @@ def racer(): expect.num(lt_moved, mar, "the move ran inside the caller's transaction") expect.text(lt, "13s", "after the move the caller's lock_timeout is the caller's again") + # Names and columns the foreign table must reproduce (#1317 review): each + # failed after the export in the first version. Each moves a small partition, + # which must then read back the same through its parent. + def mini(t, extra=""): + with conn.cursor() as cur: + cur.execute(f"CREATE TABLE {t} (at timestamptz NOT NULL, reading int{extra}) PARTITION BY RANGE (at)") + cur.execute(f"CREATE TABLE {t}_mar PARTITION OF {t} FOR VALUES FROM ('2026-03-01') TO ('2026-04-01')" + " USING pgcolumnar") + cur.execute(f"INSERT INTO {t} (at, reading) SELECT '2026-03-01'::timestamptz + g * interval '5 h', g * 7" + " FROM generate_series(0, 119) g") + + def moved_reads(t, cols): + before = _one(conn, f"SELECT md5(string_agg({cols}, ',' ORDER BY at)) FROM {t}") + try: + moved = str(_one(conn, f"SELECT pgcolumnar.tier_partition('{t}_mar', 'tq', '{server_dir}/{t.replace(chr(34), '').replace(' ', '_').replace('.', '_')}.parquet')")) + except psycopg.Error as e: + moved = e.sqlstate or "?" + after = _one(conn, f"SELECT md5(string_agg({cols}, ',' ORDER BY at)) FROM {t}") + kind = _one(conn, f"SELECT relkind::text FROM pg_class WHERE oid = '{t}_mar'::regclass") + return f"{moved}|{kind}|{'same' if before and before == after else 'differs'}" + + with conn.cursor() as cur: + cur.execute('CREATE SCHEMA "Cold Store"') + mini('"Cold Store".cs') + expect.text(moved_reads('"Cold Store".cs', "at::text || reading"), "120|f|same", + "a partition in a schema whose name needs quoting moves and reads back the same") + with conn.cursor() as cur: + cur.execute('DO $$ BEGIN CREATE ROLE "Archive Owner"; EXCEPTION WHEN duplicate_object THEN NULL; END $$') + mini("co") + with conn.cursor() as cur: + cur.execute('ALTER TABLE co_mar OWNER TO "Archive Owner"') + expect.text(f"{moved_reads('co', 'at::text || reading')}|" + f"{_one(conn, 'SELECT relowner::regrole::text FROM pg_class WHERE oid = ' + repr('co_mar') + '::regclass')}", + '120|f|same|"Archive Owner"', + "a partition owned by a role whose name needs quoting moves, and keeps that owner") + with conn.cursor() as cur: + cur.execute('DO $$ BEGIN CREATE ROLE "Audit Reader"; EXCEPTION WHEN duplicate_object THEN NULL; END $$') + mini("cg") + with conn.cursor() as cur: + cur.execute('GRANT SELECT ON cg_mar TO "Audit Reader"') + expect.text(f"{moved_reads('cg', 'at::text || reading')}|" + f"{_one(conn, 'SELECT has_table_privilege(' + repr('Audit Reader') + ', ' + repr('cg_mar') + ', ' + repr('SELECT') + ')')}", + "120|f|same|True", "a grant to a role whose name needs quoting is carried to the foreign partition") + with conn.cursor() as cur: + cur.execute('REVOKE ALL ON cg_mar FROM "Audit Reader"') + mini("cc", ', label text COLLATE "C"') + with conn.cursor() as cur: + cur.execute("UPDATE cc SET label = 'L' || reading") + expect.text(moved_reads("cc", "at::text || reading || label"), "120|f|same", + "a column with its own collation moves and reads back the same") + mini("cs2", ", doubled int GENERATED ALWAYS AS (reading * 2) STORED") + expect.text(moved_reads("cs2", "at::text || reading || doubled"), "120|f|same", + "a stored generated column moves and reads back the same") + # 5. A retry over the file a failed attempt left. _build(conn, "again") ab = _digest(conn, "again") @@ -429,3 +483,30 @@ def racer(): retried = f"error {e.sqlstate}" expect.text(f"{retried}|{_digest(conn, 'again')}|{_relkind(conn, 'again')}", f"{mar}|{ab}|f", "a retry over the failed attempt's file moves the partition, exactly once") + + +def test_tier_partition_moves_a_virtual_generated_column(pgc_own_db, server_dir, expect): + """PostgreSQL 18's virtual generated columns: the foreign table declares the + same expression, and the moved rows read back the same (#1317 review).""" + conn = pgc_own_db + major = int(_one(conn, "SHOW server_version_num")) // 10000 + if major < 18: + expect.cannot_run( + "UNSUPPORTED_MAJOR", + f"virtual generated columns arrived in PostgreSQL 18; this server is {major}", + name="a virtual generated column moves and reads back the same") + return + with conn.cursor() as cur: + cur.execute("CREATE SERVER tq FOREIGN DATA WRAPPER pgcolumnar_parquet") + cur.execute("CREATE TABLE cv (at timestamptz NOT NULL, reading int," + " tripled int GENERATED ALWAYS AS (reading * 3) VIRTUAL) PARTITION BY RANGE (at)") + cur.execute("CREATE TABLE cv_mar PARTITION OF cv FOR VALUES FROM ('2026-03-01') TO ('2026-04-01')" + " USING pgcolumnar") + cur.execute("INSERT INTO cv (at, reading) SELECT '2026-03-01'::timestamptz + g * interval '9 h', g" + " FROM generate_series(0, 70) g") + digest = "SELECT md5(string_agg(at::text || reading || tripled, ',' ORDER BY at)) FROM cv" + before = _one(conn, digest) + moved = _one(conn, f"SELECT pgcolumnar.tier_partition('cv_mar', 'tq', '{server_dir}/cv.parquet')") + kind = _one(conn, "SELECT relkind::text FROM pg_class WHERE oid = 'cv_mar'::regclass") + expect.text(f"{moved}|{kind}|{'same' if before and before == _one(conn, digest) else 'differs'}", + "71|f|same", "a virtual generated column moves and reads back the same") diff --git a/test/tier_partition.sh b/test/tier_partition.sh index 4c13ac35e..9d5f4df0e 100755 --- a/test/tier_partition.sh +++ b/test/tier_partition.sh @@ -319,6 +319,62 @@ check "a retry over the failed attempt's file moves the partition, exactly once" "$(q "SELECT pgcolumnar.tier_partition('o_jan', 'tier_pq', '$W/o.parquet');")|$(digest o)|$(q "SELECT relkind FROM pg_class WHERE oid = 'o_jan'::regclass;")" \ "8928|$ob|f" +# ---- names and columns the foreign table must reproduce (#1317 review) --------- +# Each failed after the export in the first version: a quoted schema, owner or +# grantee was quoted twice (%I over a reg* cast that quotes itself), and ATTACH +# refused a column whose collation or generation the foreign table lacked. +# Each moves a small partition, which must then read back as before. +mini() { # mini SCHEMA.TABLE [COLUMNS]: a 100-row parent with one January partition + psql_run "CREATE TABLE $1 (ts timestamptz NOT NULL, v int${2:-}) PARTITION BY RANGE (ts);" + psql_run "CREATE TABLE ${1}_jan PARTITION OF $1 FOR VALUES FROM ('2026-01-01') TO ('2026-02-01') USING pgcolumnar;" + psql_run "INSERT INTO $1 (ts, v) SELECT '2026-01-01'::timestamptz + g * interval '7 h', g FROM generate_series(0, 99) g;" +} +moved_reads() { # moved_reads SCHEMA.TABLE PATH COLUMNS-TO-READ -> rows moved|relkind|digest-equal + local before after + before="$(q "SELECT md5(string_agg($3, ',' ORDER BY ts)) FROM $1;")" + out="$(sqlstate_or "SELECT pgcolumnar.tier_partition('${1}_jan', 'tier_pq', '$2');")" + after="$(q "SELECT md5(string_agg($3, ',' ORDER BY ts)) FROM $1;")" + echo "$out|$(q "SELECT relkind FROM pg_class WHERE oid = '${1}_jan'::regclass;")|$([ -n "$before" ] && [ "$before" = "$after" ] && echo same || echo "differs")" +} +sqlstate_or() { + local o + o=$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -v ON_ERROR_STOP=1 -v VERBOSITY=sqlstate -c "$1" 2>&1) + case "$o" in ERROR:*) sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p' <<<"$o" | head -1;; *) echo "$o";; esac +} +psql_run 'CREATE SCHEMA "Tier Q";' +mini '"Tier Q".qs' +check "a partition in a schema whose name needs quoting moves and reads back the same" \ + "$(moved_reads '"Tier Q".qs' "$W/qs.parquet" "ts::text || v")" "100|f|same" +psql_run 'CREATE ROLE "Owner X";' +mini qo +psql_run 'ALTER TABLE qo_jan OWNER TO "Owner X";' +check "a partition owned by a role whose name needs quoting moves, and keeps that owner" \ + "$(moved_reads qo "$W/qo.parquet" "ts::text || v")|$(q "SELECT relowner::regrole FROM pg_class WHERE oid = 'qo_jan'::regclass;")" \ + '100|f|same|"Owner X"' +psql_run 'CREATE ROLE "Reader Y";' +mini qg +psql_run 'GRANT SELECT ON qg_jan TO "Reader Y";' +check "a grant to a role whose name needs quoting is carried to the foreign partition" \ + "$(moved_reads qg "$W/qg.parquet" "ts::text || v")|$(q "SELECT has_table_privilege('Reader Y', 'qg_jan', 'SELECT');")" \ + "100|f|same|t" +mini qc ', t text COLLATE "C"' +psql_run "UPDATE qc SET t = 'r' || v;" +check "a column with its own collation moves and reads back the same" \ + "$(moved_reads qc "$W/qc.parquet" "ts::text || v || t")" "100|f|same" +mini qgs ', w int GENERATED ALWAYS AS (v * 3) STORED' +check "a stored generated column moves and reads back the same" \ + "$(moved_reads qgs "$W/qgs.parquet" "ts::text || v || w")" "100|f|same" +if [ "$PGC_MAJOR" -ge 18 ]; then + mini qgv ', w int GENERATED ALWAYS AS (v * 5) VIRTUAL' + check "a virtual generated column moves and reads back the same" \ + "$(moved_reads qgv "$W/qgv.parquet" "ts::text || v || w")" "100|f|same" +else + check_skip "a virtual generated column moves and reads back the same" \ + "SKIP a virtual generated column moves and reads back the same (PostgreSQL $PGC_MAJOR has none; 18 added them)" \ + "PostgreSQL $PGC_MAJOR has no virtual generated columns" +fi + # ---- a failure after each step ---------------------------------------------- for step in export detach drop create attach; do build f From 507620cb53191b51f05efa1fa9c39aa6690fae42 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 01:15:54 +0000 Subject: [PATCH 7/9] test: tier_partition ledger after the #1317 review, from five majors and nineteen mutations (#1295) 71 rows; 38 dated, no premise among them; 33 never (15 premises, 18 arms the budget comment names). Census 1705 -> 1738. cluster_tests 503 -> 506 by collection: the virtual-column test is the third in the file. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 71 ++++++++++++++++++++++++++++++++++ test/check_ledger_budget.txt | 21 +++++++++- test/pytest/expected_tests.txt | 5 ++- 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 5e307f09a..53fe01b4a 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2037,6 +2037,77 @@ tid_range_scan tid_range_scan premise: the subquery cursor plans a TID range sca tid_range_scan tid_range_scan premise: the table has many row groups, so skipping can show 15;16;17;18;19 never - tid_range_scan tid_range_scan premise: the table holds every row 15;16;17;18;19 never - tid_range_scan tid_range_scan rows inserted earlier in the same transaction are in the range, as the filter sees them 15;16;17;18;19 never - +tier_partition tier_partition a DEFAULT partition is refused 15;16;17;18;19 2026-10-13 tier_partition without the DEFAULT refusal +tier_partition tier_partition a column with its own collation moves and reads back the same 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition returning 0 rows moved;tier_partition's foreign table without the columns' collations +tier_partition tier_partition a failure after attach leaves the rows exactly once, still local 15;16;17;18;19 never - +tier_partition tier_partition a failure after create leaves the rows exactly once, still local 15;16;17;18;19 never - +tier_partition tier_partition a failure after detach leaves the rows exactly once, still local 15;16;17;18;19 never - +tier_partition tier_partition a failure after drop leaves the rows exactly once, still local 15;16;17;18;19 never - +tier_partition tier_partition a failure after export leaves the rows exactly once, still local 15;16;17;18;19 never - +tier_partition tier_partition a grant to a role whose name needs quoting is carried to the foreign partition 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition not carrying the grants;tier_partition returning 0 rows moved +tier_partition tier_partition a parent held by a long reader times the move out, as lock_wait says 15;16;17;18;19 2026-10-13 tier_partition without the lock_wait bound +tier_partition tier_partition a parent with a UNIQUE index is refused before anything moves 15;16;17;18;19 2026-10-13 tier_partition without the UNIQUE-parent refusal +tier_partition tier_partition a partition a view depends on is refused before anything moves 15;16;17;18;19 2026-10-13 tier_partition without the dependents check +tier_partition tier_partition a partition in a schema whose name needs quoting moves and reads back the same 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition quoting the schema with %I over regnamespace (twice);tier_partition returning 0 rows moved +tier_partition tier_partition a partition owned by a role whose name needs quoting moves, and keeps that owner 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition returning 0 rows moved +tier_partition tier_partition a partition that is not columnar is refused 15;16;17;18;19 never - +tier_partition tier_partition a partition with a foreign key of its own is refused before anything moves 15;16;17;18;19 2026-10-13 tier_partition without the dependents check +tier_partition tier_partition a partition with a row-level policy is refused before anything moves 15;16;17;18;19 2026-10-13 tier_partition without the dependents check +tier_partition tier_partition a partition with a trigger of its own is refused before anything moves 15;16;17;18;19 2026-10-13 tier_partition without the dependents check +tier_partition tier_partition a query on the other range scans only the local partition, beside a foreign one 15;16;17;18;19 never - +tier_partition tier_partition a query on the tiered range scans only the foreign partition 15;16;17;18;19 never - +tier_partition tier_partition a retry over the failed attempt's file moves the partition, exactly once 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition refusing every existing path (the first version's rule);tier_partition returning 0 rows moved +tier_partition tier_partition a rollback after the whole call leaves the rows local, exactly once 15;16;17;18;19 never - +tier_partition tier_partition a server of another wrapper is refused 15;16;17;18;19 2026-10-13 tier_partition without the wrapper check +tier_partition tier_partition a stored generated column moves and reads back the same 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition returning 0 rows moved;tier_partition's foreign table without generation expressions;tier_partition's foreign table without generation expressions (PG18, where the virtual arm runs) +tier_partition tier_partition a table that is not a partition is refused 15;16;17;18;19 2026-10-13 tier_partition without the is-a-partition check +tier_partition tier_partition a virtual generated column moves and reads back the same 15;16;17;18;19 2026-10-13 tier_partition's foreign table without generation expressions (PG18, where the virtual arm runs) +tier_partition tier_partition a write in flight when the move starts is in the exported rows 15;16;17;18;19 2026-10-13 tier_partition returning 0 rows moved;tier_partition without the SHARE lock on the partition +tier_partition tier_partition after the move the caller's lock_timeout is the caller's again 15;16;17;18;19 2026-10-13 tier_partition leaving lock_wait as the caller's lock_timeout +tier_partition tier_partition an existing path is refused, so no file a table reads is overwritten 15;16;17;18;19 2026-10-13 tier_partition without the path refusal +tier_partition tier_partition an insert racing the move leaves every row exactly once, whichever side gives way 15;16;17;18;19 never - +tier_partition tier_partition an insert routed to the tiered range is refused 15;16;17;18;19 never - +tier_partition tier_partition and no foreign table behind it after attach 15;16;17;18;19 never - +tier_partition tier_partition and no foreign table behind it after create 15;16;17;18;19 never - +tier_partition tier_partition and no foreign table behind it after detach 15;16;17;18;19 never - +tier_partition tier_partition and no foreign table behind it after drop 15;16;17;18;19 never - +tier_partition tier_partition and no foreign table behind it after export 15;16;17;18;19 never - +tier_partition tier_partition and the refusals moved nothing 15;16;17;18;19 2026-10-13 tier_partition without the path refusal +tier_partition tier_partition and the timed-out move leaves the rows local, with nothing attached 15;16;17;18;19 2026-10-13 tier_partition without the lock_wait bound +tier_partition tier_partition at read committed a row committed between the caller's snapshot and the lock survives 15;16;17;18;19 never - +tier_partition tier_partition at read committed the move goes ahead and exports that row 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition returning 0 rows moved +tier_partition tier_partition at repeatable read a row committed between the caller's snapshot and the lock survives 15;16;17;18;19 2026-10-13 tier_partition at any isolation level +tier_partition tier_partition at repeatable read the move is refused, and the rows stay local 15;16;17;18;19 2026-10-13 tier_partition at any isolation level +tier_partition tier_partition at serializable a row committed between the caller's snapshot and the lock survives 15;16;17;18;19 2026-10-13 tier_partition at any isolation level +tier_partition tier_partition at serializable the move is refused, and the rows stay local 15;16;17;18;19 2026-10-13 tier_partition at any isolation level +tier_partition tier_partition inside the transaction the call moved the cold partition 15;16;17;18;19 2026-10-13 tier_partition returning 0 rows moved +tier_partition tier_partition nor any row-group row 15;16;17;18;19 2026-10-13 tier_partition renaming the partition aside instead of dropping it +tier_partition tier_partition premise: at read committed the caller's snapshot was taken before the writer committed 15;16;17;18;19 never - +tier_partition tier_partition premise: at repeatable read the caller's snapshot was taken before the writer committed 15;16;17;18;19 never - +tier_partition tier_partition premise: at serializable the caller's snapshot was taken before the writer committed 15;16;17;18;19 never - +tier_partition tier_partition premise: one side of the race gave way or both went through, and nothing hung 15;16;17;18;19 never - +tier_partition tier_partition premise: the call fails with the fault injected after attach 15;16;17;18;19 never - +tier_partition tier_partition premise: the call fails with the fault injected after create 15;16;17;18;19 never - +tier_partition tier_partition premise: the call fails with the fault injected after detach 15;16;17;18;19 never - +tier_partition tier_partition premise: the call fails with the fault injected after drop 15;16;17;18;19 never - +tier_partition tier_partition premise: the call fails with the fault injected after export 15;16;17;18;19 never - +tier_partition tier_partition premise: the cold partition holds its share 15;16;17;18;19 never - +tier_partition tier_partition premise: the cold partition is local and columnar, with its bound 15;16;17;18;19 never - +tier_partition tier_partition premise: the first attempt failed after export and left its file 15;16;17;18;19 never - +tier_partition tier_partition premise: the move started while the insert was in flight, and waited for it 15;16;17;18;19 never - +tier_partition tier_partition premise: the parent holds every row 15;16;17;18;19 never - +tier_partition tier_partition premise: the storage id was read before the move 15;16;17;18;19 never - +tier_partition tier_partition the answers over the parent are the same after the move 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix +tier_partition tier_partition the call holds ACCESS EXCLUSIVE on the parent until commit, as the docs say 15;16;17;18;19 never - +tier_partition tier_partition the cold partition is now a foreign partition with its own name and bound 15;16;17;18;19 2026-10-13 tier_partition attaching with the upper bound shifted a minute;tier_partition naming the foreign table with a suffix +tier_partition tier_partition the cold partition was moved with its grant and comment set 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition returning 0 rows moved +tier_partition tier_partition the foreign partition keeps the partition's grants and comment 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition not carrying the grants +tier_partition tier_partition the foreign partition reads the exported file 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix +tier_partition tier_partition the local partition leaves no pgcolumnar storage row behind 15;16;17;18;19 2026-10-13 tier_partition renaming the partition aside instead of dropping it +tier_partition tier_partition the move ran inside the caller's transaction 15;16;17;18;19 2026-10-13 tier_partition returning 0 rows moved +tier_partition tier_partition the other partition is still local after the refusals 15;16;17;18;19 2026-10-13 tier_partition without the path refusal +tier_partition tier_partition tier_partition moves the cold partition and returns the rows it moved 15;16;17;18;19 2026-10-13 tier_partition returning 0 rows moved +tier_partition tier_partition with the dependents gone, the same partition moves 15;16;17;18;19 2026-10-13 tier_partition returning 0 rows moved;tier_partition without the dependents check timeseries_recipes timeseries_recipes premise: the readings table is columnar 15;16;17;18;19 never - timeseries_recipes timeseries_recipes premise: the reference has 24 hours, 6 of them empty and one all-NULL 15;16;17;18;19 never - timeseries_recipes timeseries_recipes the LOCF recipe carries the last value forward, and leaves a leading gap NULL 15;16;17;18;19 2026-10-13 the how-to's 'Carry the last value forward' heading renamed, so the recipe is no longer found;the how-to's LOCF recipe carries with lag(), one bucket only, instead of first_value over the running-count group;the how-to's recipes join the series to the aggregate with JOIN instead of LEFT JOIN diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index ae04a0431..1d76df248 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1457,4 +1457,23 @@ suites_not_covered 247 # never 1700 + 1 = 1701 # dated 369 + 4 = 373 # 1701 + 373 == 2074 -checks_never_observed_red 1701 +# +# #1295 / PR #1317 (OffgridwithJD): 1701 -> 1734. tier_partition enters the +# ledger with 71 checks from a five-major run on this tree. Thirty-eight are +# dated by nineteen mutations of tier_partition (and the generation mutation +# again on PG18, where the virtual-column arm runs), each merged one log at a +# time with --mutation and --target; no premise is among them (four checks a +# mutation reddened were renamed from premises first, #1236). Thirty-three +# arrive `never`: fifteen premises, and eighteen arms -- eleven that hold because +# plpgsql is transactional (the five fault arms, their five no-foreign-table +# companions, the rollback), six that test behaviour of core's the docs promise +# (two pruning arms, the refused insert, the parent's lock, the race, the READ +# COMMITTED row), and one whose refusal comes from export_parquet (a partition +# that is not columnar). Restated on main after #1322; the 71 rows are +# unchanged from 663bf84. +# +# ledger rows 2074 + 71 = 2145 +# never 1701 + 33 = 1734 +# dated 373 + 38 = 411 +# 1734 + 411 == 2145 +checks_never_observed_red 1734 diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index f855f4e52..5c8b49911 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -579,4 +579,7 @@ guard_tests 419 # # cluster_tests 511 -> 513 for #1296's test_timeseries_recipes.py, two tests, by collection # on this branch rebased onto 024e737. -cluster_tests 513 +# +# cluster_tests 513 -> 516 for #1295's test_tier_partition.py (three tests), by collection +# on this branch rebased onto 2e34f90 (#1322). +cluster_tests 516 From 145f8319786daf5c046e089c737105d0c474c332 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 01:16:49 +0000 Subject: [PATCH 8/9] test: TESTS.md section 96 describes the #1317 review's arms and the virtual-column test Co-Authored-By: Claude Opus 5.5 --- test/pytest/TESTS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index db5e0daca..406d7dd50 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -6673,6 +6673,8 @@ What the arms found, in order: - A foreign key REFERENCING a columnar table is refused by pgcolumnar itself, so it cannot arise. In the first draft of that arm the setup failed, rolled back the previous case's DROP with it (a multi-statement string is one transaction), and the arm passed on a leftover trigger. Each dependent is now made in statements of its own, with a premise that it exists, and a control arm shows the same partition moves once they are gone. - The deadlock @jdatcmd predicted is real: an insert through the parent racing the move ends with one side cancelled (`40P01`), and every row is still exactly once. +@jdatcmd's review of #1317 found four inputs that failed after the export, each leaving an orphan file, each an arm now: a schema, owner or grantee whose name needs quoting (`%I` over a reg* cast quoted it twice), a column with its own collation, and a stored generated column (both refused at ATTACH). On PostgreSQL 18 a virtual generated column is carried too; measured, ATTACH accepts a foreign table with either kind, and an identity column on the parent needs nothing. + On a private database, because foreign servers are per database. A local path stands in for object storage. The pytest half uses connections in threads where the shell half uses background sessions. ### Every test @@ -6681,3 +6683,4 @@ On a private database, because foreign servers are per database. A local path st | --- | --- | | `test_tier_partition_moves_a_partition_exactly_once` | the move returns the rows moved; the parent's answers are the same; the cold range is a foreign partition with the same name and bound, reading the exported file; no pgcolumnar catalog rows are left; pruning keeps the ranges apart; an insert into the tiered range is refused; five refusals move nothing; the parent is held ACCESS EXCLUSIVE until commit, and a rollback leaves the rows local; a write in flight is exported; a fault after each of the five steps leaves the rows exactly once, still local | | `test_tier_partition_refuses_what_it_cannot_move_and_survives_concurrency` | at each isolation level a row committed between the snapshot and the lock survives, and only READ COMMITTED moves; a UNIQUE parent and four kinds of dependent are refused with no file left, and the partition moves once they are gone; grants and comment carry over; a racing insert leaves every row once; lock_wait times a held parent out with nothing attached; the caller's lock_timeout survives; a retry over a failed attempt's file moves the partition | +| `test_tier_partition_moves_a_virtual_generated_column` | PostgreSQL 18+: a partition whose parent has a virtual generated column moves and reads back the same; below 18, `UNSUPPORTED_MAJOR`, listed in `expected_unrunnable.txt` | From 99680fb8b0e4b06871d7ecbd78384d647049c251 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 02:50:10 +0000 Subject: [PATCH 9/9] test: tier_partition joins the server-file privilege boundary (#1317 CI) CI's suites leg: server_file_privilege's coverage arm found tier_partition (it takes a path) without a boundary entry. It writes its file through export_parquet, so its gate is pg_write_server_files. The fixture is a columnar partition both test roles can SELECT and UPDATE (UPDATE for the SHARE lock taken first), so the role gate is the only thing between them; past it, the privileged role writes the file and is stopped by DETACH. Red first on PG17 (45 + 1, the coverage arm). Each gate arm is red under its own mutation: tier_partition SECURITY DEFINER (the unprivileged role is no longer refused) and tier_partition refusing on entry by naming the role (the privileged role no longer gets past). server_file_privilege 48/48 on PG15-19. Co-Authored-By: Claude Opus 5.5 --- test/server_file_privilege.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/server_file_privilege.sh b/test/server_file_privilege.sh index 26f4bb270..8edf8ed45 100755 --- a/test/server_file_privilege.sh +++ b/test/server_file_privilege.sh @@ -67,6 +67,16 @@ psql_run "CREATE SERVER pqsrv FOREIGN DATA WRAPPER pgcolumnar_parquet;" psql_run "CREATE FOREIGN TABLE ft (id int, v text) SERVER pqsrv OPTIONS (path '$nope');" psql_run "GRANT SELECT ON ft TO t_none, t_priv;" +# tier_partition (#1295) writes its file through export_parquet, so its gate is +# export_parquet's. To reach it a role needs SELECT and, for the SHARE lock the +# function takes first, UPDATE on the partition; both roles get them, so the +# role gate is the only thing between them. Past the gate, the privileged role +# writes the file and is then stopped by DETACH, which needs the parent's owner. +psql_run "CREATE TABLE sfp_parent (k int) PARTITION BY RANGE (k);" +psql_run "CREATE TABLE sfp_p1 PARTITION OF sfp_parent FOR VALUES FROM (0) TO (10) USING pgcolumnar;" +psql_run "INSERT INTO sfp_parent VALUES (1);" +psql_run "GRANT SELECT, UPDATE ON sfp_p1 TO t_none, t_priv;" + run_as() { # role sql -> combined output; SET ROLE drops to the test role in C env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ -d "$PGC_DB" -X -v ON_ERROR_STOP=0 -c "SET ROLE $1;" -c "$2" 2>&1 @@ -89,6 +99,7 @@ entry_points=( "export_parquet|write|SELECT pgcolumnar.export_parquet('imp_target'::regclass, '$out.parquet')" "export_arrow|write|SELECT pgcolumnar.export_arrow('imp_target'::regclass, '$out.arrow')" "parallel_export_parquet|write|SELECT pgcolumnar.parallel_export_parquet('imp_target'::regclass, '$outdir', 2)" + "tier_partition|write|SELECT pgcolumnar.tier_partition('sfp_p1'::regclass, 'pqsrv', '$out.parquet')" ) declare -A listed