From e360c12b6562daf9cc09ebac9b020baf7f06acd2 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 03:29:59 +0000 Subject: [PATCH 01/13] test: the per-column extent, written first (#1304 option 5), red Against the design on #1304. The oracle is the data's own min() and max(); equality only after deleted rows are retired or rewritten, containment for the one case that may legitimately be wider (a rolled-back savepoint that flushed a group). Red on PG17 (main at 024e737): 6 passed + 25 failed. The function does not exist (42883) and planning still scans zone_map once. The six that pass hold on main by design: the fixture premise, the fallback and its estimate, two concurrent writers committing, expire and compact_rewrite doing their jobs. Co-Authored-By: Claude Opus 5.5 --- test/column_extent.sh | 199 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 test/column_extent.sh diff --git a/test/column_extent.sh b/test/column_extent.sh new file mode 100644 index 000000000..28c332c3d --- /dev/null +++ b/test/column_extent.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# +# The per-column extent (#1304 option 5, for #1292 phase 2). +# +# WRITTEN BEFORE THE CODE, against the design on #1304: +# +# pgcolumnar.column_extent catalog: (storage_id, column_index, complete, +# minimum, maximum), no GRANT +# pgcolumnar.column_extent(rel regclass, col name) +# -> (complete bool, minimum text, maximum text), zero rows when the +# planner would fall back to the zone maps +# +# Each committing writer inserts one row of its own (never an in-place update), a +# complete row comes only from a recompute over every visible zone map, and every +# path that retires or rewrites groups recomputes in the same transaction. The +# planner trusts the extent only when a complete row is visible. +# +# THE ORACLE is the data: min() and max() of the column, printed by the same output +# function. A zone map can be wider than the live rows only by rows deleted but not +# yet retired, so every arm that asserts equality first retires or rewrites those; +# the one arm whose extent may legitimately be wider (a rolled-back savepoint that +# flushed a group) asserts containment, not equality. +# +# Usage: test/column_extent.sh [PG_CONFIG] + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +q "CREATE EXTENSION IF NOT EXISTS pgcolumnar;" >/dev/null +check_num "premise: the extent function exists" \ + "$(q "SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'pgcolumnar' AND p.proname = 'column_extent';")" "1" + +ext() { # ext TABLE COLUMN -> "complete|min|max", "none" when no complete extent, or the SQLSTATE + # Its own psql, not q: "none" must mean the function answered with no row, and a + # missing function or a refused call must not read as that. + local r + r="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -v ON_ERROR_STOP=1 -v VERBOSITY=sqlstate \ + -c "SELECT complete || '|' || minimum || '|' || maximum FROM pgcolumnar.column_extent('$1', '$2');" 2>&1)" + case "$r" in ERROR:*) echo "error $(sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p' <<<"$r" | head -1)";; "") echo none;; *) echo "$r";; esac +} +truth() { # truth TABLE COLUMN -> "true|min|max" from the data itself + q "SELECT 'true|' || min($2)::text || '|' || max($2)::text FROM $1;" +} +sid() { q "SELECT pgcolumnar.get_storage_id('$1');"; } +GB="SET pgcolumnar.enable_group_vectorization = on;" + +# ---- the fold is the column type's own ordering ------------------------------ +# +# Byte order is not value order for any of these: ints are stored little-endian +# (256 sorts before 1 as bytes), negatives and numerics have sign and scale in +# their bytes, and text under "C" puts 'Z' before 'a'. Several row groups, so the +# fold crosses groups as well as values. +q "CREATE TABLE ty (ts timestamptz, i int, n numeric, d date, s text COLLATE \"C\") USING pgcolumnar; + SELECT pgcolumnar.set_options('ty', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO ty SELECT '2026-02-01'::timestamptz + g * interval '1 min', (g * 37) % 3001 - 1500, + ((g * 7919) % 20001 - 10000) / 7.0, date '2026-01-01' + (g % 400), + chr(65 + (g % 26)) || chr(97 + (g % 26)) FROM generate_series(1, 6000) g;" >/dev/null +check "premise: the typed fixture spans several row groups" \ + "$([ "$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = $(sid ty);")" -ge 6 ] && echo yes || echo no)" "yes" +for c in ts i n d s; do + check "the extent of ty.$c is the data's min and max, in the type's order" "$(ext ty $c)" "$(truth ty $c)" +done + +# ---- the planner reads the extent, not the zone maps -------------------------- +# +# The #1287 instrument native_groupagg uses: the planning transaction's own scans of +# zone_map and its index, which no other session can move. 50 groups, so a fallback +# would show as a scan and the extent as none. +q "CREATE TABLE pl (ts timestamptz, v float8) USING pgcolumnar; + SELECT pgcolumnar.set_options('pl', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO pl SELECT '2026-01-01'::timestamptz + g * interval '1 second', g FROM generate_series(1, 50000) g; + ANALYZE pl;" >/dev/null +plan_scans() { # zone_map scans and the estimate, planning a bucketed GROUP BY on pl + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -At -q \ + -c "$GB" -c "BEGIN" -c "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM pl GROUP BY b" \ + -c "SELECT 'scans=' || (pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass) + + pg_stat_get_xact_numscans('pgcolumnar.zone_map_pkey'::regclass))" -c "ROLLBACK" 2>&1 +} +pl_with="$(plan_scans)" +check "planning a bucketed GROUP BY scans zone_map not at all when the extent is complete" \ + "$(sed -n 's/^scans=//p' <<<"$pl_with")" "0" +# The same estimate from the fallback: remove the extent rows (superuser, as a stale +# catalog would look) and plan again. +pl_est_with="$(grep -oE 'rows=[0-9]+' <<<"$pl_with" | head -1)" +q "DELETE FROM pgcolumnar.column_extent WHERE storage_id = $(sid pl);" >/dev/null +pl_without="$(plan_scans)" +check "without the extent rows the planner falls back to one zone_map scan" \ + "$(sed -n 's/^scans=//p' <<<"$pl_without")" "1" +check "and the estimate is the one the zone maps give" "$pl_est_with" "$(grep -oE 'rows=[0-9]+' <<<"$pl_without" | head -1)" +check "and with no extent rows the function reports no complete extent" "$(ext pl ts)" "none" +# The writer-side recompute: the next committing writer finds no complete row and +# makes one, from every visible zone map, so the old groups are in it. +q "INSERT INTO pl VALUES ('2025-12-31 00:00:00+00', 0);" >/dev/null +check "a writer that finds no complete extent recomputes one over every group" "$(ext pl ts)" "$(truth pl ts)" + +# ---- concurrent writers never share a row -------------------------------------- +# +# Two sessions flush groups into one table at once and commit; an in-place update +# from both would fail ("tuple concurrently updated") or lose one widening. +q "CREATE TABLE cw (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('cw', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO cw SELECT '2026-06-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 2000) g;" >/dev/null +writer() { # writer FROM-TS -> its psql output; flushes three groups, holds them, commits + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -At -q \ + -v ON_ERROR_STOP=1 -c "BEGIN; + INSERT INTO cw SELECT '$1'::timestamptz + g * interval '1 s', g FROM generate_series(1, 3000) g; + SELECT pg_sleep(2); COMMIT;" 2>&1 +} +writer "2020-01-01" > "$PGC_WORKDIR/cw1.out" & w1=$! +writer "2030-01-01" > "$PGC_WORKDIR/cw2.out" & w2=$! +wait "$w1"; r1=$?; wait "$w2"; r2=$? +check "two writers flushing into one table at once both commit" \ + "$r1 $r2 $(cat "$PGC_WORKDIR/cw1.out" "$PGC_WORKDIR/cw2.out" | grep -c ERROR)" "0 0 0" +check "and the extent covers both" "$(ext cw ts)" "$(truth cw ts)" + +# ---- a writer that aborts leaves nothing; a rolled-back savepoint only widens -- +q "BEGIN; INSERT INTO cw SELECT '1990-01-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 3000) g; ROLLBACK;" >/dev/null +check "a writer that rolls back leaves the extent as the data has it" "$(ext cw ts)" "$(truth cw ts)" +q "BEGIN; INSERT INTO cw VALUES ('2026-06-02', 1); SAVEPOINT s; + INSERT INTO cw SELECT '1980-01-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 3000) g; + ROLLBACK TO s; COMMIT;" >/dev/null +contains() { # contains TABLE COLUMN TYPE -> yes when the extent contains the data's range + q "SELECT CASE WHEN e.complete AND e.minimum::$3 <= (SELECT min($2) FROM $1) + AND e.maximum::$3 >= (SELECT max($2) FROM $1) THEN 'yes' ELSE 'no' END + FROM pgcolumnar.column_extent('$1', '$2') e;" +} +check "a savepoint rolled back after flushing may widen the extent, never narrow it" "$(contains cw ts timestamptz)" "yes" + +# ---- a shrink is recomputed, never guessed (jdatcmd, #1304) --------------------- +# +# Each path that retires or rewrites groups recomputes the extent in the same +# transaction, so it equals the data afterwards: not narrower (an under-estimate), +# and not left at the old width. +q "CREATE TABLE ex (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('ex', stripe_row_limit => 1000, chunk_group_row_limit => 1000, + ttl_column => 'ts', ttl_interval => '30 days'); + INSERT INTO ex SELECT now() - interval '400 days' + g * interval '1 min', g FROM generate_series(1, 3000) g; + INSERT INTO ex SELECT now() - interval '1 day' + g * interval '1 s', g FROM generate_series(1, 3000) g;" >/dev/null +ex_before="$(ext ex ts)" +retired="$(q "SELECT pgcolumnar.expire('ex');")" +check "expire retires the old row groups with the extent in place" "$([ "${retired:-0}" -ge 1 ] && echo yes || echo no)" "yes" +check "after expire the extent is recomputed to the data that is left" "$(ext ex ts)" "$(truth ex ts)" +check "and it did move: the retired groups held the old minimum" \ + "$([ "$ex_before" != "$(ext ex ts)" ] && echo yes || echo no)" "yes" + +q "CREATE TABLE cr (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('cr', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO cr SELECT '2026-01-01'::timestamptz + g * interval '1 min', g FROM generate_series(1, 4000) g; + DELETE FROM cr WHERE v > 3500 OR v <= 400;" >/dev/null +rewritten="$(q "SELECT pgcolumnar.compact_rewrite('cr', 0.1);")" +check "compact_rewrite rewrites the groups the deletes thinned" "$([ "${rewritten:-0}" -ge 1 ] && echo yes || echo no)" "yes" +check "after compact_rewrite the extent is the live rows'" "$(ext cr ts)" "$(truth cr ts)" + +q "CREATE TABLE vr (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('vr', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO vr SELECT '2026-01-01'::timestamptz + g * interval '1 min', g FROM generate_series(1, 3000) g; + DELETE FROM vr WHERE v > 2000;" >/dev/null +q "VACUUM vr;" >/dev/null +check "after VACUUM retires a fully deleted group the extent is the live rows'" "$(ext vr ts)" "$(truth vr ts)" + +# ---- no complete extent where the zone maps cannot give one ---------------------- +q "CREATE TABLE ac (ts timestamptz) USING pgcolumnar; + SELECT pgcolumnar.set_options('ac', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO ac SELECT '2026-01-01'::timestamptz + g * interval '1 s' FROM generate_series(1, 2000) g; + ALTER TABLE ac ADD COLUMN v int DEFAULT 5; + INSERT INTO ac SELECT '2026-02-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 2000) g;" >/dev/null +check "a column added after older groups has no complete extent (those groups have no zone map for it)" \ + "$(ext ac v)" "none" +check "while the column that was there from the start still has one" "$(ext ac ts)" "$(truth ac ts)" +q "CREATE TABLE no (p point) USING pgcolumnar; INSERT INTO no SELECT point(g, g) FROM generate_series(1, 2000) g;" >/dev/null +check "a type with no ordering gets no extent" "$(ext no p)" "none" + +# ---- housekeeping ----------------------------------------------------------------- +q "CREATE TABLE hk (ts timestamptz, v int) USING pgcolumnar;" >/dev/null +for k in $(seq 1 50); do q "INSERT INTO hk VALUES ('2026-01-01'::timestamptz + interval '$k min', $k);" >/dev/null; done +hk_rows="$(q "SELECT count(*) FROM pgcolumnar.column_extent WHERE storage_id = $(sid hk) AND column_index = 0;")" +check "fifty committing writers leave a bounded number of extent rows" \ + "$([ "${hk_rows:-99}" -le 9 ] && echo yes || echo "no (${hk_rows:-none})")" "yes" +check "and the extent still covers all fifty" "$(ext hk ts)" "$(truth hk ts)" +hk_old="$(sid hk)" +q "TRUNCATE hk;" >/dev/null +check "TRUNCATE removes the old storage's extent rows" \ + "$(q "SELECT count(*) FROM pgcolumnar.column_extent WHERE storage_id = $hk_old;")" "0" +cw_old="$(sid cw)" +q "DROP TABLE cw;" >/dev/null +check "DROP TABLE removes them" "$(q "SELECT count(*) FROM pgcolumnar.column_extent WHERE storage_id = $cw_old;")" "0" + +q "CREATE ROLE ce_r; GRANT USAGE ON SCHEMA pgcolumnar TO ce_r;" >/dev/null +as_role() { env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -v VERBOSITY=sqlstate -c "SET ROLE $1" -c "$2" 2>&1 | grep -v '^SET$' | tail -1; } +check "the extent catalog is not readable by an ordinary role (it holds column values)" \ + "$(as_role ce_r "SELECT count(*) FROM pgcolumnar.column_extent" | sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p')" "42501" +check "nor is a table's extent through the function, without SELECT on the table" \ + "$(as_role ce_r "SELECT * FROM pgcolumnar.column_extent('public.ty', 'ts')" | sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p')" "42501" + +pgc_summary From 1609430b4549455abe256767ee44fcf02e109f9b Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 03:39:57 +0000 Subject: [PATCH 02/13] feat: the per-column extent bounds the bucket estimate in one lookup (#1304 option 5) pgcolumnar.column_extent holds rows of (storage, column, complete, minimum, maximum), inserted and deleted, never updated in place: - each transaction folds the whole-chunk zone maps it flushes (the type's comparison, the column's collation) and inserts one row at pre-commit, or pre-prepare; its groups and its row commit together, and two writers never touch one tuple; - a complete row comes only from a recompute over every zone map one snapshot sees, which replaces the rows that snapshot sees; it refuses when a group has no zone map for the column or values without an ordering; - PgColumnarRetireGroup marks the storage and it is recomputed before commit, so expire, compact_rewrite, VACUUM and the reclusters shrink the extent by recomputing it; - a writer recomputes when it finds no complete row and merges the rows when there are more than eight (a fold of those rows only). The planner's time range reads the extent when a complete row is visible and otherwise the zone maps. pgcolumnar.column_extent(rel, col) shows it, to a role that can SELECT the table. Found while making the suite green: pre-commit has no active snapshot (an assert-build TRAP), and the catalog snapshot can predate a storage row this transaction inserted, which made the existence check skip every table created and written in one transaction. column_extent.sh 33/33 on PG17, with two arms added since the red commit (PREPARE TRANSACTION, and a table created in a rolled-back savepoint). Co-Authored-By: Claude Opus 5.5 --- Makefile | 1 + pgcolumnar--1.0-alpha4--1.0-alpha5.sql | 30 ++ pgcolumnar--1.0-alpha5.sql | 30 ++ src/columnar.h | 30 ++ src/columnar_extent.c | 551 +++++++++++++++++++++++++ src/columnar_metadata.c | 170 ++++++++ src/columnar_tableam.c | 16 + src/columnar_vector.c | 22 + src/columnar_write_state.c | 16 + test/column_extent.sh | 22 + 10 files changed, 888 insertions(+) create mode 100644 src/columnar_extent.c diff --git a/Makefile b/Makefile index 98c9a58ad..f2c531ff8 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ OBJS = \ src/columnar_tableam.o \ src/columnar_storage.o \ src/columnar_metadata.o \ + src/columnar_extent.o \ src/columnar_write_state.o \ src/columnar_compression.o \ src/columnar_encoding.o \ diff --git a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql index f805c3186..54fc7a518 100644 --- a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql @@ -983,6 +983,36 @@ CREATE TABLE pgcolumnar.retired_group ( CREATE UNIQUE INDEX retired_group_pkey ON pgcolumnar.retired_group USING btree (storage_id, group_number); +/* --------------------------------------------------------------------------- + * pgcolumnar.column_extent (#1304 option 5) + * + * A bound on every value of a column, for the planner's bucket estimate: one + * index lookup instead of every row group's zone map. Rows are inserted and + * deleted, never updated: each committing writer adds one row (complete = + * false) for what it flushed, and a recompute over every visible zone map + * replaces the rows it saw with one complete row. The planner trusts the rows + * only when a complete one is visible; then their fold is a true bound. + * minimum and maximum are encoded as the zone maps encode them; NULL when the + * row saw no value. No GRANT: the values are column data, like zone_map's. + * ------------------------------------------------------------------------- */ +CREATE TABLE pgcolumnar.column_extent ( + storage_id bigint NOT NULL, + column_index smallint NOT NULL, + complete bool NOT NULL, + minimum bytea, + maximum bytea +); +CREATE INDEX column_extent_pkey + ON pgcolumnar.column_extent USING btree (storage_id, column_index); + +CREATE FUNCTION pgcolumnar.column_extent(rel regclass, col name) + RETURNS TABLE (complete bool, minimum text, maximum text) + LANGUAGE C STRICT STABLE + AS 'MODULE_PATHNAME', 'pgcolumnar_column_extent'; + +COMMENT ON FUNCTION pgcolumnar.column_extent(regclass, name) + IS 'the bound on a column the planner reads from pgcolumnar.column_extent, printed by the column type; no row when it would read the zone maps instead (#1304); requires SELECT on the table'; + /* --------------------------------------------------------------------------- * Incremental rollups, phase 1: append-only (#1294). diff --git a/pgcolumnar--1.0-alpha5.sql b/pgcolumnar--1.0-alpha5.sql index 05e156e74..d742a4cdb 100644 --- a/pgcolumnar--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha5.sql @@ -341,6 +341,36 @@ CREATE TABLE pgcolumnar.retired_group ( CREATE UNIQUE INDEX retired_group_pkey ON pgcolumnar.retired_group USING btree (storage_id, group_number); +/* --------------------------------------------------------------------------- + * pgcolumnar.column_extent (#1304 option 5) + * + * A bound on every value of a column, for the planner's bucket estimate: one + * index lookup instead of every row group's zone map. Rows are inserted and + * deleted, never updated: each committing writer adds one row (complete = + * false) for what it flushed, and a recompute over every visible zone map + * replaces the rows it saw with one complete row. The planner trusts the rows + * only when a complete one is visible; then their fold is a true bound. + * minimum and maximum are encoded as the zone maps encode them; NULL when the + * row saw no value. No GRANT: the values are column data, like zone_map's. + * ------------------------------------------------------------------------- */ +CREATE TABLE pgcolumnar.column_extent ( + storage_id bigint NOT NULL, + column_index smallint NOT NULL, + complete bool NOT NULL, + minimum bytea, + maximum bytea +); +CREATE INDEX column_extent_pkey + ON pgcolumnar.column_extent USING btree (storage_id, column_index); + +CREATE FUNCTION pgcolumnar.column_extent(rel regclass, col name) + RETURNS TABLE (complete bool, minimum text, maximum text) + LANGUAGE C STRICT STABLE + AS 'MODULE_PATHNAME', 'pgcolumnar_column_extent'; + +COMMENT ON FUNCTION pgcolumnar.column_extent(regclass, name) + IS 'the bound on a column the planner reads from pgcolumnar.column_extent, printed by the column type; no row when it would read the zone maps instead (#1304); requires SELECT on the table'; + /* --------------------------------------------------------------------------- * Access method (spec 8.1) * ------------------------------------------------------------------------- */ diff --git a/src/columnar.h b/src/columnar.h index 562d4e1df..277ac85e1 100644 --- a/src/columnar.h +++ b/src/columnar.h @@ -612,6 +612,35 @@ extern List *PgColumnarReadWholeChunkZoneMapsForColumn(uint64 storageId, int col extern void PgColumnarDeleteMetadata(uint64 storageId); extern void PgColumnarRecordRetention(uint64 storageId, uint64 groupNumber); +/* pgcolumnar.column_extent (#1304 option 5): columnar_metadata.c */ +typedef struct NativeExtentRow +{ + ItemPointerData tid; + bool complete; + bool hasValues; /* false: the row's writer saw only NULLs */ + char *minimum; + int minimumLen; + char *maximum; + int maximumLen; +} NativeExtentRow; +extern bool PgColumnarExtentCatalogPresent(void); +extern void PgColumnarInsertExtentRow(uint64 storageId, int columnIndex, bool complete, + const char *minimum, int minimumLen, + const char *maximum, int maximumLen); +extern List *PgColumnarReadExtentRows(uint64 storageId, int columnIndex, Snapshot snapshot); +extern void PgColumnarDeleteExtentRow(ItemPointer tid); +extern Oid PgColumnarStorageRelationOid(uint64 storageId, Snapshot snapshot); + +/* the extent's lifecycle: columnar_extent.c */ +extern void PgColumnarExtentNote(Oid relid, uint64 storageId, int columnIndex, + const NativeZoneMapMetadata *z); +extern void PgColumnarExtentMarkRecompute(uint64 storageId); +extern void PgColumnarExtentForget(uint64 storageId); +extern void PgColumnarExtentPreCommit(void); +extern void PgColumnarExtentAtXactEnd(void); +extern bool PgColumnarExtentBound(Relation rel, int columnIndex, Snapshot snapshot, + Datum *minimum, Datum *maximum, bool *hasValues); + /* declared retention, read by pgcolumnar.expire (#403 item 5a) */ extern bool PgColumnarReadTtl(Oid relid, char **column, Interval **interval); @@ -1007,6 +1036,7 @@ extern char *PgColumnarDecompressValueStream(const char *comp, uint32 compLen, #define PGCOLUMNAR_LOCKCLASS_STORAGE_ROW 102 #define PGCOLUMNAR_LOCKCLASS_UNIQUE_KEY 103 #define PGCOLUMNAR_LOCKCLASS_ROW_IDENTITY 104 +#define PGCOLUMNAR_LOCKCLASS_EXTENT 105 /* ------------------------------------------------------------------------- * concurrent unique-key insert serialization (pgcolumnar_unique.c, issue #5) diff --git a/src/columnar_extent.c b/src/columnar_extent.c new file mode 100644 index 000000000..4552bca5b --- /dev/null +++ b/src/columnar_extent.c @@ -0,0 +1,551 @@ +/*------------------------------------------------------------------------- + * + * columnar_extent.c + * The per-column extent (#1304 option 5): a bound on every value of a + * column, read in one index lookup instead of every zone map. + * + * THE CATALOG, pgcolumnar.column_extent, holds rows of (storage, column, + * complete, minimum, maximum). Rows are inserted and deleted, never updated: + * + * - Each transaction folds the whole-chunk zone maps it flushes, in memory, + * with the column type's comparison and collation, and at pre-commit + * inserts ONE row per (storage, column), complete = false. Its groups and + * its row commit or abort together, so any snapshot sees both or neither, + * and two writers never touch the same tuple. + * - A COMPLETE row comes only from a recompute: the fold of every whole-chunk + * zone map visible to one snapshot, which then replaces the rows visible to + * that same snapshot. It refuses (and changes nothing) when a group lacks a + * zone map for the column or has values without a minimum and maximum. + * - Every path that retires a group marks its storage (PgColumnarRetireGroup) + * and the storage is recomputed before commit, so a shrink is computed, + * never guessed. + * + * THE READER trusts the rows only when a complete one is visible, and then the + * fold of every visible row is a true bound. A reader's snapshot S that sees a + * complete row R (computed at snapshot B, committed before S): the groups S sees + * were either visible at B, and so are in R, or committed between B and S, in + * which case their writer's own row committed with them and S sees it. A retire + * and its recompute are one transaction, so S sees both or neither. Otherwise + * the caller falls back to the zone maps. + * + * A ROLLED-BACK SUBTRANSACTION that flushed a group leaves its values in the + * fold: wider than the data, which only over-estimates. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "columnar.h" +#include "access/relation.h" +#include "access/table.h" +#include "access/xact.h" +#include "funcapi.h" +#include "lib/stringinfo.h" +#include "miscadmin.h" +#include "storage/lmgr.h" +#include "storage/lock.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/datum.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/rel.h" +#include "utils/snapmgr.h" +#include "utils/typcache.h" + +PG_FUNCTION_INFO_V1(pgcolumnar_column_extent); + +/* + * More rows than this for one column, with a complete one among them, and the + * committing writer folds them into one. The fold reads only these rows, never + * the zone maps, so it costs a few tuples however large the table. + */ +#define EXTENT_MERGE_ABOVE 8 + +typedef struct ExtentKey +{ + uint64 storageId; + int32 columnIndex; +} ExtentKey; + +typedef struct ExtentEntry +{ + ExtentKey key; + Oid relid; + FormData_pg_attribute att; + Datum minimum; + Datum maximum; +} ExtentEntry; + +/* both live in TopTransactionContext, and are forgotten at transaction end */ +static HTAB *extent_pending = NULL; +static HTAB *extent_recompute = NULL; /* uint64 storage ids */ + +static int +extent_cmp(Form_pg_attribute att, Datum a, Datum b) +{ + TypeCacheEntry *tce = lookup_type_cache(att->atttypid, TYPECACHE_CMP_PROC_FINFO); + + if (!OidIsValid(tce->cmp_proc)) + elog(ERROR, "column extent: type %u has no btree comparison", att->atttypid); + return DatumGetInt32(FunctionCall2Coll(&tce->cmp_proc_finfo, att->attcollation, a, b)); +} + +static Datum +extent_decode(Form_pg_attribute att, const char *bytes, int len) +{ + char *cur = (char *) bytes; + + return PgColumnarDecodeValue(att, &cur, (char *) bytes + len, CurrentMemoryContext); +} + +static HTAB * +extent_hash(const char *name, Size keysize, Size entrysize) +{ + HASHCTL ctl; + + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = keysize; + ctl.entrysize = entrysize; + ctl.hcxt = TopTransactionContext; + return hash_create(name, 16, &ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); +} + +/* + * PgColumnarExtentNote + * Fold one whole-chunk zone map, just flushed for the relation's base + * storage, into this transaction's extent for the column. + */ +void +PgColumnarExtentNote(Oid relid, uint64 storageId, int columnIndex, + const NativeZoneMapMetadata *z) +{ + ExtentKey key; + ExtentEntry *e; + bool found; + Relation rel; + Form_pg_attribute att; + MemoryContext old; + Datum mn, + mx; + + if (!z->hasMinMax) + return; /* all NULL, or a type with no ordering */ + if (extent_pending == NULL) + extent_pending = extent_hash("pgcolumnar column extents", sizeof(ExtentKey), + sizeof(ExtentEntry)); + memset(&key, 0, sizeof(key)); + key.storageId = storageId; + key.columnIndex = columnIndex; + e = (ExtentEntry *) hash_search(extent_pending, &key, HASH_ENTER, &found); + if (!found) + { + rel = relation_open(relid, NoLock); + att = TupleDescAttr(RelationGetDescr(rel), columnIndex); + memcpy(&e->att, att, sizeof(FormData_pg_attribute)); + relation_close(rel, NoLock); + e->relid = relid; + } + att = &e->att; + mn = extent_decode(att, z->minimum, z->minimumLen); + mx = extent_decode(att, z->maximum, z->maximumLen); + old = MemoryContextSwitchTo(TopTransactionContext); + if (!found || extent_cmp(att, mn, e->minimum) < 0) + e->minimum = datumCopy(mn, att->attbyval, att->attlen); + if (!found || extent_cmp(att, mx, e->maximum) > 0) + e->maximum = datumCopy(mx, att->attbyval, att->attlen); + MemoryContextSwitchTo(old); +} + +void +PgColumnarExtentMarkRecompute(uint64 storageId) +{ + if (extent_recompute == NULL) + extent_recompute = extent_hash("pgcolumnar extent recomputes", sizeof(uint64), + sizeof(uint64)); + (void) hash_search(extent_recompute, &storageId, HASH_ENTER, NULL); +} + +/* the storage's metadata is gone in this transaction: write nothing for it */ +void +PgColumnarExtentForget(uint64 storageId) +{ + if (extent_pending != NULL) + { + HASH_SEQ_STATUS st; + ExtentEntry *e; + + hash_seq_init(&st, extent_pending); + while ((e = (ExtentEntry *) hash_seq_search(&st)) != NULL) + if (e->key.storageId == storageId) + (void) hash_search(extent_pending, &e->key, HASH_REMOVE, NULL); + } + if (extent_recompute != NULL) + (void) hash_search(extent_recompute, &storageId, HASH_REMOVE, NULL); +} + +void +PgColumnarExtentAtXactEnd(void) +{ + /* the tables fall with TopTransactionContext */ + extent_pending = NULL; + extent_recompute = NULL; +} + +static void +extent_encode(Form_pg_attribute att, Datum v, StringInfo buf) +{ + resetStringInfo(buf); + PgColumnarEncodeValue(buf, att, v); +} + +/* + * One recompute or merge per storage at a time, held to transaction end: the + * rows a recompute deletes are the rows its snapshot sees, and a second one + * deleting the same rows would fail. Returns false only when !wait and taken. + */ +static bool +extent_lock(uint64 storageId, bool wait) +{ + LOCKTAG tag; + + SET_LOCKTAG_ADVISORY(tag, MyDatabaseId, + (uint32) (storageId >> 32), (uint32) (storageId & 0xFFFFFFFF), + PGCOLUMNAR_LOCKCLASS_EXTENT); + return LockAcquire(&tag, ExclusiveLock, false, !wait) != LOCKACQUIRE_NOT_AVAIL; +} + +/* replace every row the snapshot sees with one complete row */ +static void +extent_replace(uint64 storageId, int col, Form_pg_attribute att, Snapshot snap, + bool any, Datum mn, Datum mx) +{ + List *rows = PgColumnarReadExtentRows(storageId, col, snap); + ListCell *lc; + StringInfoData a, + b; + + foreach(lc, rows) + PgColumnarDeleteExtentRow(&((NativeExtentRow *) lfirst(lc))->tid); + if (!any) + { + PgColumnarInsertExtentRow(storageId, col, true, NULL, 0, NULL, 0); + return; + } + initStringInfo(&a); + initStringInfo(&b); + extent_encode(att, mn, &a); + extent_encode(att, mx, &b); + PgColumnarInsertExtentRow(storageId, col, true, a.data, a.len, b.data, b.len); +} + +/* + * The full recompute, from every whole-chunk zone map the snapshot sees. Gives + * up, writing nothing, when the zone maps cannot bound the column. + */ +static void +extent_recompute_column(uint64 storageId, int col, Form_pg_attribute att, Snapshot snap) +{ + List *groups = PgColumnarReadRowGroupList(storageId, snap); + List *zones = PgColumnarReadWholeChunkZoneMapsForColumn(storageId, col, snap); + ListCell *lc; + bool any = false; + Datum mn = (Datum) 0, + mx = (Datum) 0; + + if (list_length(zones) != list_length(groups)) + return; /* a group with no zone map: a column added later */ + foreach(lc, zones) + { + NativeZoneMapMetadata *z = (NativeZoneMapMetadata *) lfirst(lc); + Datum zmn, + zmx; + + if (!z->hasMinMax) + { + if (z->valueCount == 0) + continue; /* all NULL */ + return; /* values without an ordering */ + } + zmn = extent_decode(att, z->minimum, z->minimumLen); + zmx = extent_decode(att, z->maximum, z->maximumLen); + if (!any || extent_cmp(att, zmn, mn) < 0) + mn = zmn; + if (!any || extent_cmp(att, zmx, mx) > 0) + mx = zmx; + any = true; + } + extent_replace(storageId, col, att, snap, any, mn, mx); +} + +/* + * The merge: fold the rows the snapshot sees into one complete row. Valid only + * when a complete row is among them -- then their fold is a bound on every + * group the snapshot sees, by the reader's argument above. + */ +static void +extent_merge_column(uint64 storageId, int col, Form_pg_attribute att, Snapshot snap) +{ + List *rows = PgColumnarReadExtentRows(storageId, col, snap); + ListCell *lc; + bool complete = false; + bool any = false; + Datum mn = (Datum) 0, + mx = (Datum) 0; + + foreach(lc, rows) + { + NativeExtentRow *r = (NativeExtentRow *) lfirst(lc); + Datum rmn, + rmx; + + complete |= r->complete; + if (!r->hasValues) + continue; + rmn = extent_decode(att, r->minimum, r->minimumLen); + rmx = extent_decode(att, r->maximum, r->maximumLen); + if (!any || extent_cmp(att, rmn, mn) < 0) + mn = rmn; + if (!any || extent_cmp(att, rmx, mx) > 0) + mx = rmx; + any = true; + } + if (complete) + extent_replace(storageId, col, att, snap, any, mn, mx); + else + extent_recompute_column(storageId, col, att, snap); +} + +/* a fresh snapshot, taken after the lock, that sees this transaction's own rows */ +static Snapshot +extent_snapshot(void) +{ + CommandCounterIncrement(); + return RegisterSnapshot(GetLatestSnapshot()); +} + +static void +extent_recompute_storage(uint64 storageId) +{ + Snapshot snap = extent_snapshot(); + Oid relid = PgColumnarStorageRelationOid(storageId, snap); + Relation rel; + TupleDesc td; + int i; + + UnregisterSnapshot(snap); + if (!OidIsValid(relid)) + return; + rel = try_relation_open(relid, AccessShareLock); + if (rel == NULL) + return; + /* a projection's storage names its base relation: no extents for those */ + if (PgColumnarStorageId(rel) != storageId) + { + relation_close(rel, AccessShareLock); + return; + } + (void) extent_lock(storageId, true); + snap = extent_snapshot(); + td = RelationGetDescr(rel); + for (i = 0; i < td->natts; i++) + if (!TupleDescAttr(td, i)->attisdropped) + extent_recompute_column(storageId, i, TupleDescAttr(td, i), snap); + UnregisterSnapshot(snap); + relation_close(rel, AccessShareLock); +} + +/* + * PgColumnarExtentPreCommit + * Called at pre-commit and pre-prepare, after the pending writes are + * flushed: write this transaction's rows, then recompute what it marked. + */ +void +PgColumnarExtentPreCommit(void) +{ + HASH_SEQ_STATUS st; + ExtentEntry *e; + uint64 *sid; + List *written = NIL; + ListCell *lc; + StringInfoData a, + b; + bool pushed = false; + + if (extent_pending == NULL && extent_recompute == NULL) + return; + if (!PgColumnarExtentCatalogPresent()) + { + PgColumnarExtentAtXactEnd(); + return; + } + /* pre-commit runs with no active snapshot, and catalog writes need one */ + if (!ActiveSnapshotSet()) + { + PushActiveSnapshot(GetTransactionSnapshot()); + pushed = true; + } + initStringInfo(&a); + initStringInfo(&b); + if (extent_pending != NULL) + { + Snapshot seen = extent_snapshot(); + + hash_seq_init(&st, extent_pending); + while ((e = (ExtentEntry *) hash_seq_search(&st)) != NULL) + { + /* a recompute of the storage takes this transaction's groups anyway */ + if (extent_recompute != NULL && + hash_search(extent_recompute, &e->key.storageId, HASH_FIND, NULL) != NULL) + continue; + /* created in a savepoint that rolled back: nothing to bound */ + if (!OidIsValid(PgColumnarStorageRelationOid(e->key.storageId, seen))) + continue; + extent_encode(&e->att, e->minimum, &a); + extent_encode(&e->att, e->maximum, &b); + PgColumnarInsertExtentRow(e->key.storageId, e->key.columnIndex, false, + a.data, a.len, b.data, b.len); + written = lappend(written, e); + } + UnregisterSnapshot(seen); + } + if (extent_recompute != NULL) + { + hash_seq_init(&st, extent_recompute); + while ((sid = (uint64 *) hash_seq_search(&st)) != NULL) + extent_recompute_storage(*sid); + } + + /* + * Keep the rows few. A writer that finds no complete row recomputes one; one + * that finds too many merges them. Conditional: a writer never waits for + * another's recompute, it leaves the work to the next. + */ + foreach(lc, written) + { + List *rows; + ListCell *rc; + bool complete = false; + Snapshot snap; + + e = (ExtentEntry *) lfirst(lc); + snap = extent_snapshot(); + rows = PgColumnarReadExtentRows(e->key.storageId, e->key.columnIndex, snap); + UnregisterSnapshot(snap); + foreach(rc, rows) + complete |= ((NativeExtentRow *) lfirst(rc))->complete; + if (complete && list_length(rows) <= EXTENT_MERGE_ABOVE) + continue; + if (!extent_lock(e->key.storageId, false)) + continue; + snap = extent_snapshot(); + if (complete) + extent_merge_column(e->key.storageId, e->key.columnIndex, &e->att, snap); + else + extent_recompute_column(e->key.storageId, e->key.columnIndex, &e->att, snap); + UnregisterSnapshot(snap); + } + if (pushed) + PopActiveSnapshot(); + PgColumnarExtentAtXactEnd(); +} + +/* + * PgColumnarExtentBound + * The column's bound as the snapshot sees it, when a complete row is + * visible. false: no trustworthy extent, fall back to the zone maps. + * *hasValues false: the column holds no non-NULL value. + */ +bool +PgColumnarExtentBound(Relation rel, int columnIndex, Snapshot snapshot, + Datum *minimum, Datum *maximum, bool *hasValues) +{ + Form_pg_attribute att; + List *rows; + ListCell *lc; + bool complete = false; + + if (columnIndex < 0 || columnIndex >= RelationGetDescr(rel)->natts || + !PgColumnarExtentCatalogPresent()) + return false; + att = TupleDescAttr(RelationGetDescr(rel), columnIndex); + rows = PgColumnarReadExtentRows(PgColumnarStorageId(rel), columnIndex, snapshot); + *hasValues = false; + foreach(lc, rows) + { + NativeExtentRow *r = (NativeExtentRow *) lfirst(lc); + Datum rmn, + rmx; + + complete |= r->complete; + if (!r->hasValues) + continue; + rmn = extent_decode(att, r->minimum, r->minimumLen); + rmx = extent_decode(att, r->maximum, r->maximumLen); + if (!*hasValues || extent_cmp(att, rmn, *minimum) < 0) + *minimum = rmn; + if (!*hasValues || extent_cmp(att, rmx, *maximum) > 0) + *maximum = rmx; + *hasValues = true; + } + return complete; +} + +/* + * pgcolumnar.column_extent(rel regclass, col name) + * -> (complete bool, minimum text, maximum text) + * The bound the planner would use, printed by the column type's output + * function; no row when it would fall back to the zone maps. The values are + * column data, so the caller must be able to SELECT the table. + */ +Datum +pgcolumnar_column_extent(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + Oid relid = PG_GETARG_OID(0); + char *col = NameStr(*PG_GETARG_NAME(1)); + AclResult ac; + Relation rel; + AttrNumber attnum; + Datum mn = (Datum) 0, + mx = (Datum) 0; + bool hasValues; + + InitMaterializedSRF(fcinfo, 0); + ac = pg_class_aclcheck(relid, GetUserId(), ACL_SELECT); + if (ac != ACLCHECK_OK) + aclcheck_error(ac, OBJECT_TABLE, get_rel_name(relid)); + rel = relation_open(relid, AccessShareLock); + if (!PgColumnarIsColumnarRelation(relid)) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is not a pgcolumnar table", RelationGetRelationName(rel)))); + attnum = get_attnum(relid, col); + if (attnum <= 0) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + col, RelationGetRelationName(rel)))); + if (PgColumnarExtentBound(rel, attnum - 1, + PgColumnarCatalogSnapshot(GetActiveSnapshot()), + &mn, &mx, &hasValues)) + { + Form_pg_attribute att = TupleDescAttr(RelationGetDescr(rel), attnum - 1); + Datum values[3]; + bool nulls[3] = {false, !hasValues, !hasValues}; + Oid out; + bool isvarlena; + + values[0] = BoolGetDatum(true); + if (hasValues) + { + getTypeOutputInfo(att->atttypid, &out, &isvarlena); + values[1] = CStringGetTextDatum(OidOutputFunctionCall(out, mn)); + values[2] = CStringGetTextDatum(OidOutputFunctionCall(out, mx)); + } + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); + } + relation_close(rel, AccessShareLock); + return (Datum) 0; +} diff --git a/src/columnar_metadata.c b/src/columnar_metadata.c index 1f356f4f8..208b1438e 100644 --- a/src/columnar_metadata.c +++ b/src/columnar_metadata.c @@ -140,6 +140,14 @@ #define Anum_free_space_freed_xid 4 #define Natts_free_space 4 +/* attribute numbers for pgcolumnar.column_extent (#1304 option 5) */ +#define Anum_column_extent_storage_id 1 +#define Anum_column_extent_column_index 2 +#define Anum_column_extent_complete 3 +#define Anum_column_extent_minimum 4 +#define Anum_column_extent_maximum 5 +#define Natts_column_extent 5 + /* attribute numbers for pgcolumnar.retired_group (#1294) */ #define Anum_retired_group_storage_id 1 #define Anum_retired_group_group_number 2 @@ -1044,6 +1052,14 @@ PgColumnarRetireGroup(uint64 storageId, uint64 groupNumber) */ PgColumnarGroupMemoReset(true); + /* + * A group leaving the storage may take the column's minimum or maximum with + * it, so its extent is recomputed before this transaction commits (#1304 + * option 5): a shrink is recomputed, never guessed. Every path that retires a + * group comes through here. + */ + PgColumnarExtentMarkRecompute(storageId); + if (haveRange && byteLength > 0) record_free_space(storageId, fileOffset, byteLength); } @@ -2004,6 +2020,13 @@ PgColumnarDeleteMetadata(uint64 storageId) delete_rows_by_storage_id("bloom", Anum_bloom_storage_id, storageId); delete_rows_by_storage_id("free_space", Anum_free_space_storage_id, storageId); delete_rows_by_storage_id("retired_group", Anum_retired_group_storage_id, storageId); + /* + * #1304 option 5. Present only once the extension's SQL is at 1.0-alpha5; a + * library ahead of its SQL must still be able to drop and truncate. + */ + if (PgColumnarExtentCatalogPresent()) + delete_rows_by_storage_id("column_extent", Anum_column_extent_storage_id, storageId); + PgColumnarExtentForget(storageId); delete_rows_by_storage_id("row_group", Anum_row_group_storage_id, storageId); delete_rows_by_storage_id("storage", Anum_native_storage_storage_id, storageId); @@ -4626,3 +4649,150 @@ PgColumnarDeleteProjectionRow(uint64 storageId, int projectionId) table_close(rel, RowExclusiveLock); CommandCounterIncrement(); } + +/* ------------------------------------------------------------------------- + * pgcolumnar.column_extent (#1304 option 5) + * + * Rows are only inserted and deleted, never updated in place: each committing + * writer inserts its own row, and a recompute replaces the rows it folded with + * one complete row. So two backends never touch the same tuple. + * ------------------------------------------------------------------------- */ + +bool +PgColumnarExtentCatalogPresent(void) +{ + return OidIsValid(get_relname_relid("column_extent", pgcolumnar_schema_oid())); +} + +static bytea * +extent_bytea(const char *bytes, int len) +{ + bytea *b = (bytea *) palloc(VARHDRSZ + len); + + SET_VARSIZE(b, VARHDRSZ + len); + if (len > 0) + memcpy(VARDATA(b), bytes, len); + return b; +} + +void +PgColumnarInsertExtentRow(uint64 storageId, int columnIndex, bool complete, + const char *minimum, int minimumLen, + const char *maximum, int maximumLen) +{ + Relation rel = open_columnar_table("column_extent", RowExclusiveLock); + Datum values[Natts_column_extent]; + bool nulls[Natts_column_extent] = {false}; + HeapTuple tuple; + + values[Anum_column_extent_storage_id - 1] = Int64GetDatum((int64) storageId); + values[Anum_column_extent_column_index - 1] = Int16GetDatum((int16) columnIndex); + values[Anum_column_extent_complete - 1] = BoolGetDatum(complete); + if (minimum == NULL) + { + /* a column with no values yet: no bound, and nothing for a fold to take */ + nulls[Anum_column_extent_minimum - 1] = true; + nulls[Anum_column_extent_maximum - 1] = true; + } + else + { + values[Anum_column_extent_minimum - 1] = PointerGetDatum(extent_bytea(minimum, minimumLen)); + values[Anum_column_extent_maximum - 1] = PointerGetDatum(extent_bytea(maximum, maximumLen)); + } + tuple = heap_form_tuple(RelationGetDescr(rel), values, nulls); + CatalogTupleInsert(rel, tuple); + heap_freetuple(tuple); + table_close(rel, RowExclusiveLock); +} + +/* + * The extent rows of one column visible to the snapshot, as NativeExtentRow *. + * A NULL snapshot is the catalog snapshot. + */ +List * +PgColumnarReadExtentRows(uint64 storageId, int columnIndex, Snapshot snapshot) +{ + Relation rel = open_columnar_table("column_extent", AccessShareLock); + TupleDesc td = RelationGetDescr(rel); + ScanKeyData key[2]; + SysScanDesc scan; + HeapTuple tuple; + Oid idx; + List *rows = NIL; + + ScanKeyInit(&key[0], Anum_column_extent_storage_id, BTEqualStrategyNumber, + F_INT8EQ, Int64GetDatum((int64) storageId)); + ScanKeyInit(&key[1], Anum_column_extent_column_index, BTEqualStrategyNumber, + F_INT2EQ, Int16GetDatum((int16) columnIndex)); + idx = pgcolumnar_scan_index_oid(rel, "column_extent_pkey"); + scan = systable_beginscan(rel, idx, OidIsValid(idx), snapshot, 2, key); + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + NativeExtentRow *r = (NativeExtentRow *) palloc0(sizeof(NativeExtentRow)); + bool isnull; + Datum d; + + r->tid = tuple->t_self; + r->complete = DatumGetBool(heap_getattr(tuple, Anum_column_extent_complete, td, &isnull)); + d = heap_getattr(tuple, Anum_column_extent_minimum, td, &isnull); + if (!isnull) + { + bytea *mn = DatumGetByteaPCopy(d); + bytea *mx = DatumGetByteaPCopy(heap_getattr(tuple, Anum_column_extent_maximum, td, &isnull)); + + r->hasValues = true; + r->minimum = VARDATA(mn); + r->minimumLen = VARSIZE(mn) - VARHDRSZ; + r->maximum = VARDATA(mx); + r->maximumLen = VARSIZE(mx) - VARHDRSZ; + } + rows = lappend(rows, r); + } + systable_endscan(scan); + table_close(rel, AccessShareLock); + return rows; +} + +void +PgColumnarDeleteExtentRow(ItemPointer tid) +{ + Relation rel = open_columnar_table("column_extent", RowExclusiveLock); + + CatalogTupleDelete(rel, tid); + table_close(rel, RowExclusiveLock); +} + +/* + * The relation a storage id belongs to, from pgcolumnar.storage, or InvalidOid. + * A projection's storage names its base relation too; callers that need the + * base storage compare PgColumnarStorageId of the opened relation. Pass a + * snapshot taken after the caller's own writes: the catalog snapshot can + * predate a storage row this transaction inserted. + */ +Oid +PgColumnarStorageRelationOid(uint64 storageId, Snapshot snapshot) +{ + Relation rel = open_columnar_table("storage", AccessShareLock); + ScanKeyData key[1]; + SysScanDesc scan; + HeapTuple tuple; + Oid idx; + Oid relid = InvalidOid; + + ScanKeyInit(&key[0], Anum_native_storage_storage_id, BTEqualStrategyNumber, + F_INT8EQ, Int64GetDatum((int64) storageId)); + idx = pgcolumnar_scan_index_oid(rel, "storage_pkey"); + scan = systable_beginscan(rel, idx, OidIsValid(idx), snapshot, 1, key); + if (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + bool isnull; + Datum d = heap_getattr(tuple, Anum_native_storage_relation_oid, + RelationGetDescr(rel), &isnull); + + if (!isnull) + relid = DatumGetObjectId(d); + } + systable_endscan(scan); + table_close(rel, AccessShareLock); + return relid; +} diff --git a/src/columnar_tableam.c b/src/columnar_tableam.c index b90e90cc2..647485943 100644 --- a/src/columnar_tableam.c +++ b/src/columnar_tableam.c @@ -2663,11 +2663,26 @@ pgcolumnar_xact_callback(XactEvent event, void *arg) { switch (event) { + case XACT_EVENT_PRE_PREPARE: + /* + * #1304 option 5: a prepared transaction's extent rows are written + * here, where catalog writes are still allowed, so its groups are + * flushed here first. The PREPARE arm below then finds nothing. + */ + PgColumnarFlushAllPendingWrites(); + PgColumnarFlushAllDeleteVectors(); + PgColumnarExtentPreCommit(); + break; case XACT_EVENT_PRE_COMMIT: case XACT_EVENT_PARALLEL_PRE_COMMIT: case XACT_EVENT_PREPARE: PgColumnarFlushAllPendingWrites(); PgColumnarFlushAllDeleteVectors(); + /* after the flush, which is what gives the extent its last groups */ + if (event == XACT_EVENT_PRE_COMMIT) + PgColumnarExtentPreCommit(); + else + PgColumnarExtentAtXactEnd(); /* * PREPARE frees TopTransactionContext and never reaches the @@ -2700,6 +2715,7 @@ pgcolumnar_xact_callback(XactEvent event, void *arg) PgColumnarDiscardAllPendingWrites(); PgColumnarDiscardAllDeleteVectors(); PgColumnarDiscardFetchCache(); + PgColumnarExtentAtXactEnd(); /* * Descriptors only: the memo contexts are TopTransactionContext * children, still live at callback time but taken by the diff --git a/src/columnar_vector.c b/src/columnar_vector.c index 8363dc338..17c023e37 100644 --- a/src/columnar_vector.c +++ b/src/columnar_vector.c @@ -2078,6 +2078,28 @@ pgcolumnar_zonemap_time_range(PlannerInfo *root, RelOptInfo *input_rel, Var *tsv storageId = PgColumnarStorageId(r); snap = PgColumnarCatalogSnapshot(GetActiveSnapshot()); + /* + * THE EXTENT FIRST (#1304 option 5): one index lookup, constant in the table, + * when a complete extent is visible. A timestamp is byval, so the bound + * outlives the relation. Otherwise the zone maps below, which are correct and + * cost a scan of every group's. + */ + { + Datum emin, + emax; + bool hasValues; + + if (PgColumnarExtentBound(r, col, snap, &emin, &emax, &hasValues)) + { + table_close(r, AccessShareLock); + if (!hasValues) + return false; /* no value at all: nothing to bound */ + *lo = (double) DatumGetTimestamp(emin); + *hi = (double) DatumGetTimestamp(emax); + return true; + } + } + /* * TWO CATALOG SCANS, NOT ONE PER GROUP (#1304 review): the group list, for the * count, and every group's zone map for this column. A group with no zone-map diff --git a/src/columnar_write_state.c b/src/columnar_write_state.c index 94845de9d..e08fcb050 100644 --- a/src/columnar_write_state.c +++ b/src/columnar_write_state.c @@ -3246,6 +3246,22 @@ pgcolumnar_flush_row_group(PgColumnarWriteState *writeState) } PG_END_TRY(); + /* + * The column extent (#1304 option 5) folds this group's whole-chunk zone + * maps; the transaction writes its row at pre-commit. The base table only: + * a projection's writer has its own tuple descriptor, and nothing reads a + * projection's extent. + */ + if (writeState->tupdescIsRel) + foreach(lc, zoneRows) + { + NativeZoneMapMetadata *z = (NativeZoneMapMetadata *) lfirst(lc); + + if (z->vectorIndex == -1) + PgColumnarExtentNote(writeState->relid, writeState->storageId, + z->columnIndex, z); + } + table_close(rel, RowExclusiveLock); MemoryContextSwitchTo(oldContext); diff --git a/test/column_extent.sh b/test/column_extent.sh index 28c332c3d..c810ee35d 100644 --- a/test/column_extent.sh +++ b/test/column_extent.sh @@ -25,6 +25,7 @@ set -uo pipefail . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +export PGC_EXTRA_CONF='max_prepared_transactions=2' pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" q "CREATE EXTENSION IF NOT EXISTS pgcolumnar;" >/dev/null @@ -129,6 +130,27 @@ contains() { # contains TABLE COLUMN TYPE -> yes when the extent contains the da } check "a savepoint rolled back after flushing may widen the extent, never narrow it" "$(contains cw ts timestamptz)" "yes" +# A prepared transaction writes its extent row before PREPARE, where a catalog write +# is still allowed; its groups commit with COMMIT PREPARED and must be inside it. +# Its own table: cw's extent is already wider than its data, by the savepoint above. +q "CREATE TABLE tp (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('tp', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO tp SELECT '2026-06-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 2000) g;" >/dev/null +q "BEGIN; INSERT INTO tp SELECT '2040-01-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 3000) g; + PREPARE TRANSACTION 'ce_2pc';" >/dev/null +q "COMMIT PREPARED 'ce_2pc';" >/dev/null +check "a prepared transaction's groups are inside the extent after COMMIT PREPARED" "$(ext tp ts)" "$(truth tp ts)" + +# A table created and written inside a savepoint that rolls back leaves no extent +# row behind: its storage never existed for anyone, and an orphan row would be kept +# by nothing. +q "BEGIN; SAVEPOINT s; CREATE TABLE gone (ts timestamptz) USING pgcolumnar; + INSERT INTO gone SELECT now() FROM generate_series(1, 2000); + ROLLBACK TO s; COMMIT;" >/dev/null +check "a table created in a rolled-back savepoint leaves no extent rows" \ + "$(q "SELECT count(*) FROM pgcolumnar.column_extent e + WHERE NOT EXISTS (SELECT 1 FROM pgcolumnar.storage s WHERE s.storage_id = e.storage_id);")" "0" + # ---- a shrink is recomputed, never guessed (jdatcmd, #1304) --------------------- # # Each path that retires or rewrites groups recomputes the extent in the same From 9f3f6444e028e1a02a19b83e794bc50164e97bac Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 03:51:46 +0000 Subject: [PATCH 03/13] test: column_extent's pytest twin, the suite registered, and native_groupagg measures the fallback (#1304 option 5) - The concurrency arm now uses parallel_copy. Ordinary writers into one table are serialized by the storage-row lock (class 102) from first flush to commit, so the old two-writer arm never overlapped them. parallel_copy's loaders flush into one storage at once, each through PREPARE. - native_groupagg's one-scan arm measures the zone-map fallback, so it removes the fixture's extent rows first; with a complete extent the plan scans zone_map not at all. The pytest twin measures on a fresh connection: the xact scan counters are a backend's pending stats, flushed about once a second rather than at transaction end, and in the fixture's connection they still held the INSERT's own zone_map reads (the extent's first recompute, at its commit). - test_column_extent.py: the independent twin. Its COMMIT PREPARED runs in another session; in the preparing one it crashes the backend on main (to be filed). - column_extent.sh is executable (harness_selftest). PG17: column_extent 33/33, native_groupagg 81 + 1 skip, harness_selftest 1223. PG16: test_column_extent.py + test_native_groupagg.py 122 + 1 unrunnable (no ICU on this build). Co-Authored-By: Claude Opus 5.5 --- test/column_extent.sh | 29 ++-- test/native_groupagg.sh | 4 + test/pytest/test_column_extent.py | 258 ++++++++++++++++++++++++++++ test/pytest/test_native_groupagg.py | 25 ++- test/run_all_versions.sh | 1 + 5 files changed, 295 insertions(+), 22 deletions(-) mode change 100644 => 100755 test/column_extent.sh create mode 100644 test/pytest/test_column_extent.py diff --git a/test/column_extent.sh b/test/column_extent.sh old mode 100644 new mode 100755 index c810ee35d..955b71541 --- a/test/column_extent.sh +++ b/test/column_extent.sh @@ -25,7 +25,7 @@ set -uo pipefail . "$(dirname "${BASH_SOURCE[0]}")/lib.sh" -export PGC_EXTRA_CONF='max_prepared_transactions=2' +export PGC_EXTRA_CONF=$'max_prepared_transactions=8\nmax_worker_processes=16' pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" q "CREATE EXTENSION IF NOT EXISTS pgcolumnar;" >/dev/null @@ -99,23 +99,22 @@ check "a writer that finds no complete extent recomputes one over every group" " # ---- concurrent writers never share a row -------------------------------------- # -# Two sessions flush groups into one table at once and commit; an in-place update -# from both would fail ("tuple concurrently updated") or lose one widening. +# Ordinary writers into one table are serialized by the storage-row lock (class +# 102) from their first flush to their commit, so they never overlap. The writers +# that do are parallel_copy's loaders: separate sessions flushing into one storage +# at once, each preparing its transaction. An in-place update from two of them +# would fail ("tuple concurrently updated") or lose one's widening; and each +# writes its row at PRE_PREPARE, so one missing would leave the extent too narrow. +# The file's earliest rows and its latest are far apart in it, so different loaders +# hold them. q "CREATE TABLE cw (ts timestamptz, v int) USING pgcolumnar; SELECT pgcolumnar.set_options('cw', stripe_row_limit => 1000, chunk_group_row_limit => 1000); INSERT INTO cw SELECT '2026-06-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 2000) g;" >/dev/null -writer() { # writer FROM-TS -> its psql output; flushes three groups, holds them, commits - env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -At -q \ - -v ON_ERROR_STOP=1 -c "BEGIN; - INSERT INTO cw SELECT '$1'::timestamptz + g * interval '1 s', g FROM generate_series(1, 3000) g; - SELECT pg_sleep(2); COMMIT;" 2>&1 -} -writer "2020-01-01" > "$PGC_WORKDIR/cw1.out" & w1=$! -writer "2030-01-01" > "$PGC_WORKDIR/cw2.out" & w2=$! -wait "$w1"; r1=$?; wait "$w2"; r2=$? -check "two writers flushing into one table at once both commit" \ - "$r1 $r2 $(cat "$PGC_WORKDIR/cw1.out" "$PGC_WORKDIR/cw2.out" | grep -c ERROR)" "0 0 0" -check "and the extent covers both" "$(ext cw ts)" "$(truth cw ts)" +q "COPY (SELECT '2020-01-01'::timestamptz + g * interval '1 day', g FROM generate_series(1, 4000) g) + TO '$PGC_WORKDIR/cw.txt' WITH (FORMAT text);" >/dev/null +check "parallel_copy's loaders, flushing into one table at once, all commit" \ + "$(q "SELECT pgcolumnar.parallel_copy('cw', '$PGC_WORKDIR/cw.txt', 4);")" "4000" +check "and the extent covers what they loaded" "$(ext cw ts)" "$(truth cw ts)" # ---- a writer that aborts leaves nothing; a rolled-back savepoint only widens -- q "BEGIN; INSERT INTO cw SELECT '1990-01-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 3000) g; ROLLBACK;" >/dev/null diff --git a/test/native_groupagg.sh b/test/native_groupagg.sh index 8bbe41cf2..6dcf7c40e 100755 --- a/test/native_groupagg.sh +++ b/test/native_groupagg.sh @@ -494,6 +494,10 @@ psql_run "CREATE TABLE gbz (ts timestamptz, v float8) USING pgcolumnar; INSERT INTO gbz SELECT '2026-01-01'::timestamptz + g * interval '1 second', g FROM generate_series(1,50000) g; ANALYZE gbz;" >/dev/null 2>&1 gbz_groups="$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('gbz');")" +# THE FALLBACK IS WHAT THIS MEASURES. Since #1304 option 5 a complete column extent +# bounds the range with no zone_map scan at all (column_extent.sh asserts 0); without +# one the planner reads the zone maps, and that read must stay one scan. +q "DELETE FROM pgcolumnar.column_extent WHERE storage_id = pgcolumnar.get_storage_id('gbz');" >/dev/null gbz_scans="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -At -q \ -c "$GB" -c "BEGIN" -c "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM gbz GROUP BY b" \ -c "SELECT pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass) diff --git a/test/pytest/test_column_extent.py b/test/pytest/test_column_extent.py new file mode 100644 index 000000000..929873457 --- /dev/null +++ b/test/pytest/test_column_extent.py @@ -0,0 +1,258 @@ +"""The per-column extent (#1304 option 5, for #1292 phase 2). + +pgcolumnar.column_extent holds, per (storage, column), rows that are inserted and +deleted but never updated: each committing writer adds one for what it flushed, and +a recompute over every visible zone map replaces the rows it saw with one complete +row. The planner trusts the rows only when a complete one is visible; their fold is +then a true bound. pgcolumnar.column_extent(rel, col) shows that bound, and returns +no row when the planner would fall back to the zone maps. + +THE ORACLE is the data: min() and max() of the column, printed by the same output +function. Equality only once deleted rows are retired or rewritten; the one case +that may legitimately be wider (a rolled-back savepoint that flushed a group) is +asserted as containment. + +ON A PRIVATE DATABASE (`pgc_own_db`): the role, and the transaction-local scan +counters. Independent of `test/column_extent.sh`: its own row counts, values, times +and helpers; the check names match, so the pair is graded one for one. +""" + +import psycopg + +GB = "SET pgcolumnar.enable_group_vectorization = on" + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + row = cur.fetchone() + return row[0] if row else None + + +def _run(conn, *sqls): + with conn.cursor() as cur: + for sql in sqls: + cur.execute(sql) + + +def _ext(conn, table, col): + """'true|min|max', 'none' when no complete extent, or 'error '.""" + try: + with conn.cursor() as cur: + cur.execute("SELECT complete::text || '|' || minimum || '|' || maximum" + f" FROM pgcolumnar.column_extent('{table}', '{col}')") + row = cur.fetchone() + except psycopg.Error as e: + return f"error {e.sqlstate}" + return row[0] if row else "none" + + +def _truth(conn, table, col): + return _one(conn, f"SELECT 'true|' || min({col})::text || '|' || max({col})::text FROM {table}") + + +def _sid(conn, table): + return _one(conn, f"SELECT pgcolumnar.get_storage_id('{table}')") + + +def _small_groups(conn, table): + _run(conn, f"SELECT pgcolumnar.set_options('{table}', stripe_row_limit => 1000," + " chunk_group_row_limit => 1000)") + + +def test_the_extent_is_the_columns_own_order(pgc_own_db, expect): + conn = pgc_own_db + expect.num(_one(conn, "SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace" + " WHERE n.nspname = 'pgcolumnar' AND p.proname = 'column_extent'"), 1, + "premise: the extent function exists") + # Byte order is not value order for any of these: little-endian ints across 256, + # signed values, numeric scale, and "C" text where 'Q' sorts before 'b'. + _run(conn, 'CREATE TABLE ty (ts timestamptz, i int, n numeric, d date, s text COLLATE "C") USING pgcolumnar') + _small_groups(conn, "ty") + _run(conn, "INSERT INTO ty SELECT '2025-11-03 08:00:00+00'::timestamptz + g * interval '47 s'," + " (g * 53) % 4001 - 2000, ((g * 6301) % 30011 - 15005) / 11.0, date '2025-03-01' + (g % 523)," + " chr(81 + (g % 10)) || chr(98 + (g % 20)) FROM generate_series(1, 7300) g") + expect.at_least(_one(conn, f"SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = {_sid(conn, 'ty')}"), + 6, "premise: the typed fixture spans several row groups") + for c in ("ts", "i", "n", "d", "s"): + expect.text(_ext(conn, "ty", c), _truth(conn, "ty", c), + f"the extent of ty.{c} is the data's min and max, in the type's order") + + +def _plan(pgc_cluster, dbname): + """(zone_map scans, the estimate) planning a bucketed GROUP BY on pl, in one transaction.""" + with psycopg.connect(pgc_cluster.dsn(dbname=dbname)) as c: + with c.cursor() as cur: + cur.execute(GB) + cur.execute("EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM pl GROUP BY b") + plan = "\n".join(r[0] for r in cur.fetchall()) + cur.execute("SELECT pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass)" + " + pg_stat_get_xact_numscans('pgcolumnar.zone_map_pkey'::regclass)") + scans = cur.fetchone()[0] + c.rollback() + est = plan.split("rows=", 1)[1].split(" ", 1)[0] if "rows=" in plan else "" + return scans, est + + +def test_the_planner_reads_the_extent_not_the_zone_maps(pgc_own_db, pgc_cluster, expect): + conn = pgc_own_db + db = _one(conn, "SELECT current_database()") + _run(conn, "CREATE TABLE pl (ts timestamptz, v float8) USING pgcolumnar") + _small_groups(conn, "pl") + _run(conn, "INSERT INTO pl SELECT '2025-07-01'::timestamptz + g * interval '3 s', g" + " FROM generate_series(1, 60000) g", "ANALYZE pl") + scans_with, est_with = _plan(pgc_cluster, db) + expect.num(scans_with, 0, "planning a bucketed GROUP BY scans zone_map not at all when the extent is complete") + _run(conn, f"DELETE FROM pgcolumnar.column_extent WHERE storage_id = {_sid(conn, 'pl')}") + scans_without, est_without = _plan(pgc_cluster, db) + expect.num(scans_without, 1, "without the extent rows the planner falls back to one zone_map scan") + expect.text(f"rows={est_with}", f"rows={est_without}", "and the estimate is the one the zone maps give") + expect.text(_ext(conn, "pl", "ts"), "none", "and with no extent rows the function reports no complete extent") + _run(conn, "INSERT INTO pl VALUES ('2025-06-30 00:00:00+00', 0)") + expect.text(_ext(conn, "pl", "ts"), _truth(conn, "pl", "ts"), + "a writer that finds no complete extent recomputes one over every group") + + +def test_writers_concurrent_aborted_and_prepared(pgc_own_db, pgc_cluster, expect): + conn = pgc_own_db + dsn = pgc_cluster.dsn(dbname=_one(conn, "SELECT current_database()")) + _run(conn, "CREATE TABLE cw (ts timestamptz, v int) USING pgcolumnar") + _small_groups(conn, "cw") + _run(conn, "INSERT INTO cw SELECT '2025-09-09'::timestamptz + g * interval '2 s', g FROM generate_series(1, 1500) g") + + # Ordinary writers are serialized by the storage-row lock from first flush to + # commit; parallel_copy's loaders are the ones that flush into one storage at + # once, each through PREPARE. The first and last rows of the file are far apart. + path = _one(conn, "SELECT current_setting('data_directory')") + "/ce_py_cw.txt" + _run(conn, "COPY (SELECT '2018-03-03'::timestamptz + g * interval '13 hours', g" + f" FROM generate_series(1, 5200) g) TO '{path}' WITH (FORMAT text)") + expect.num(_one(conn, f"SELECT pgcolumnar.parallel_copy('cw', '{path}', 3)"), 5200, + "parallel_copy's loaders, flushing into one table at once, all commit") + expect.text(_ext(conn, "cw", "ts"), _truth(conn, "cw", "ts"), "and the extent covers what they loaded") + + with psycopg.connect(dsn) as c: + _run(c, "INSERT INTO cw SELECT '1995-01-01'::timestamptz + g * interval '2 s', g FROM generate_series(1, 2600) g") + c.rollback() + expect.text(_ext(conn, "cw", "ts"), _truth(conn, "cw", "ts"), + "a writer that rolls back leaves the extent as the data has it") + + with psycopg.connect(dsn) as c: + _run(c, "INSERT INTO cw VALUES ('2025-09-10', 1)", "SAVEPOINT s", + "INSERT INTO cw SELECT '1985-01-01'::timestamptz + g * interval '2 s', g FROM generate_series(1, 2600) g", + "ROLLBACK TO s") + c.commit() + expect.text(_one(conn, "SELECT CASE WHEN e.complete AND e.minimum::timestamptz <= (SELECT min(ts) FROM cw)" + " AND e.maximum::timestamptz >= (SELECT max(ts) FROM cw) THEN 'yes' ELSE 'no' END" + " FROM pgcolumnar.column_extent('cw', 'ts') e"), "yes", + "a savepoint rolled back after flushing may widen the extent, never narrow it") + + _run(conn, "CREATE TABLE tp (ts timestamptz, v int) USING pgcolumnar") + _small_groups(conn, "tp") + _run(conn, "INSERT INTO tp SELECT '2025-09-09'::timestamptz + g * interval '2 s', g FROM generate_series(1, 1500) g") + # COMMIT PREPARED from another session: in the preparing session it crashes the + # backend on main, a separate bug (the write states outlive PREPARE). + with psycopg.connect(dsn, autocommit=True) as c: + _run(c, "BEGIN", + "INSERT INTO tp SELECT '2044-02-02'::timestamptz + g * interval '2 s', g FROM generate_series(1, 2600) g", + "PREPARE TRANSACTION 'ce_py_2pc'") + _run(conn, "COMMIT PREPARED 'ce_py_2pc'") + expect.text(_ext(conn, "tp", "ts"), _truth(conn, "tp", "ts"), + "a prepared transaction's groups are inside the extent after COMMIT PREPARED") + + with psycopg.connect(dsn) as c: + _run(c, "SAVEPOINT s", "CREATE TABLE gone (ts timestamptz) USING pgcolumnar", + "INSERT INTO gone SELECT now() FROM generate_series(1, 2500)", "ROLLBACK TO s") + c.commit() + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.column_extent e WHERE NOT EXISTS" + " (SELECT 1 FROM pgcolumnar.storage s WHERE s.storage_id = e.storage_id)"), 0, + "a table created in a rolled-back savepoint leaves no extent rows") + + +def test_a_shrink_is_recomputed_never_guessed(pgc_own_db, expect): + conn = pgc_own_db + _run(conn, "CREATE TABLE ex (ts timestamptz, v int) USING pgcolumnar") + _run(conn, "SELECT pgcolumnar.set_options('ex', stripe_row_limit => 1000, chunk_group_row_limit => 1000," + " ttl_column => 'ts', ttl_interval => '60 days')") + _run(conn, "INSERT INTO ex SELECT now() - interval '700 days' + g * interval '90 s', g FROM generate_series(1, 2500) g", + "INSERT INTO ex SELECT now() - interval '2 days' + g * interval '7 s', g FROM generate_series(1, 2500) g") + before = _ext(conn, "ex", "ts") + retired = _one(conn, "SELECT pgcolumnar.expire('ex')") + expect.text("yes" if (retired or 0) >= 1 else f"no ({retired})", "yes", + "expire retires the old row groups with the extent in place") + after = _ext(conn, "ex", "ts") + expect.text(after, _truth(conn, "ex", "ts"), "after expire the extent is recomputed to the data that is left") + expect.text("yes" if before != after else "no", "yes", + "and it did move: the retired groups held the old minimum") + + _run(conn, "CREATE TABLE cr (ts timestamptz, v int) USING pgcolumnar") + _small_groups(conn, "cr") + _run(conn, "INSERT INTO cr SELECT '2025-05-05'::timestamptz + g * interval '5 min', g FROM generate_series(1, 4500) g", + "DELETE FROM cr WHERE v > 4200 OR v <= 650") + rewritten = _one(conn, "SELECT pgcolumnar.compact_rewrite('cr', 0.1)") + expect.text("yes" if (rewritten or 0) >= 1 else f"no ({rewritten})", "yes", + "compact_rewrite rewrites the groups the deletes thinned") + expect.text(_ext(conn, "cr", "ts"), _truth(conn, "cr", "ts"), "after compact_rewrite the extent is the live rows'") + + _run(conn, "CREATE TABLE vr (ts timestamptz, v int) USING pgcolumnar") + _small_groups(conn, "vr") + _run(conn, "INSERT INTO vr SELECT '2025-05-05'::timestamptz + g * interval '5 min', g FROM generate_series(1, 3400) g", + "DELETE FROM vr WHERE v > 2000", "VACUUM vr") + expect.text(_ext(conn, "vr", "ts"), _truth(conn, "vr", "ts"), + "after VACUUM retires a fully deleted group the extent is the live rows'") + + +def test_no_complete_extent_where_the_zone_maps_cannot_give_one(pgc_own_db, expect): + conn = pgc_own_db + _run(conn, "CREATE TABLE ac (ts timestamptz) USING pgcolumnar") + _small_groups(conn, "ac") + _run(conn, "INSERT INTO ac SELECT '2025-03-03'::timestamptz + g * interval '9 s' FROM generate_series(1, 2400) g", + "ALTER TABLE ac ADD COLUMN v int DEFAULT 11", + "INSERT INTO ac SELECT '2025-04-04'::timestamptz + g * interval '9 s', g FROM generate_series(1, 2400) g") + expect.text(_ext(conn, "ac", "v"), "none", + "a column added after older groups has no complete extent (those groups have no zone map for it)") + expect.text(_ext(conn, "ac", "ts"), _truth(conn, "ac", "ts"), + "while the column that was there from the start still has one") + _run(conn, "CREATE TABLE no (p point) USING pgcolumnar", + "INSERT INTO no SELECT point(g, -g) FROM generate_series(1, 2400) g") + expect.text(_ext(conn, "no", "p"), "none", "a type with no ordering gets no extent") + + +def test_housekeeping_and_privilege(pgc_own_db, pgc_cluster, expect): + conn = pgc_own_db + dsn = pgc_cluster.dsn(dbname=_one(conn, "SELECT current_database()")) + _run(conn, "CREATE TABLE hk (ts timestamptz, v int) USING pgcolumnar") + for k in range(1, 51): + _run(conn, f"INSERT INTO hk VALUES ('2025-01-01'::timestamptz + interval '{k} hours', {k})") + rows = _one(conn, f"SELECT count(*) FROM pgcolumnar.column_extent WHERE storage_id = {_sid(conn, 'hk')}" + " AND column_index = 0") + expect.text("yes" if rows <= 9 else f"no ({rows})", "yes", + "fifty committing writers leave a bounded number of extent rows") + expect.text(_ext(conn, "hk", "ts"), _truth(conn, "hk", "ts"), "and the extent still covers all fifty") + old = _sid(conn, "hk") + _run(conn, "TRUNCATE hk") + expect.num(_one(conn, f"SELECT count(*) FROM pgcolumnar.column_extent WHERE storage_id = {old}"), 0, + "TRUNCATE removes the old storage's extent rows") + _run(conn, "CREATE TABLE dr (ts timestamptz) USING pgcolumnar", + "INSERT INTO dr SELECT now() FROM generate_series(1, 10)") + old = _sid(conn, "dr") + _run(conn, "DROP TABLE dr") + expect.num(_one(conn, f"SELECT count(*) FROM pgcolumnar.column_extent WHERE storage_id = {old}"), 0, + "DROP TABLE removes them") + + _run(conn, "DO $$ BEGIN CREATE ROLE ce_py; EXCEPTION WHEN duplicate_object THEN NULL; END $$", + "GRANT USAGE ON SCHEMA pgcolumnar TO ce_py", + "CREATE TABLE IF NOT EXISTS priv (ts timestamptz) USING pgcolumnar", + "INSERT INTO priv VALUES (now())") + + def as_role(sql): + with psycopg.connect(dsn, autocommit=True) as c: + c.execute("SET ROLE ce_py") + try: + c.execute(sql).fetchall() + return "ok" + except psycopg.Error as e: + return e.sqlstate or "?" + expect.text(as_role("SELECT count(*) FROM pgcolumnar.column_extent"), "42501", + "the extent catalog is not readable by an ordinary role (it holds column values)") + expect.text(as_role("SELECT * FROM pgcolumnar.column_extent('public.priv', 'ts')"), "42501", + "nor is a table's extent through the function, without SELECT on the table") diff --git a/test/pytest/test_native_groupagg.py b/test/pytest/test_native_groupagg.py index 560f1df49..6973495c6 100644 --- a/test/pytest/test_native_groupagg.py +++ b/test/pytest/test_native_groupagg.py @@ -658,7 +658,7 @@ def within(sql): "a non-time expression key gets no bound either") -def test_the_bound_reads_the_zone_maps_in_one_catalog_scan(pgc_conn, expect): +def test_the_bound_reads_the_zone_maps_in_one_catalog_scan(pgc_conn, pgc_cluster, expect): """The range costs one catalog scan, not one per row group (#1304 review). The first version probed zone_map_pkey once per group, on every plan: 109 -> 313 @@ -677,14 +677,25 @@ def test_the_bound_reads_the_zone_maps_in_one_catalog_scan(pgc_conn, expect): groups = int(_one(pgc_conn, "SELECT count(*) FROM pgcolumnar.row_group " "WHERE storage_id = pgcolumnar.get_storage_id('gbz')")) - _exec(pgc_conn, "BEGIN") - try: - _exec(pgc_conn, "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM gbz GROUP BY b") - scans = int(_one(pgc_conn, + # The fallback is what this measures: since #1304 option 5 a complete column + # extent bounds the range with no zone_map scan (test_column_extent.py asserts + # 0), and without one the zone-map read must stay one scan. + _exec(pgc_conn, "DELETE FROM pgcolumnar.column_extent " + "WHERE storage_id = pgcolumnar.get_storage_id('gbz')") + # ON A FRESH CONNECTION. The "xact" counters are this backend's pending stats, + # which are flushed to shared memory at most about once a second, not at + # transaction end -- so in pgc_conn they still held the INSERT's own zone_map + # reads (the column extent's first recompute, at its commit), and read 3 for a + # plan that scanned once. The shell twin starts a new psql, so it never saw them. + with psycopg.connect(pgc_cluster.dsn(dbname=_one(pgc_conn, "SELECT current_database()"))) as m: + _exec(m, "SELECT set_config('search_path', %s, false)" % ( + "'" + _one(pgc_conn, "SELECT current_setting('search_path')").replace("'", "''") + "'")) + _gb_session(m) + _exec(m, "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM gbz GROUP BY b") + scans = int(_one(m, "SELECT pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass) " "+ pg_stat_get_xact_numscans('pgcolumnar.zone_map_pkey'::regclass)")) - finally: - _exec(pgc_conn, "ROLLBACK") + m.rollback() print(f"-- {groups} row groups, {scans} zone_map scans while planning") expect.at_least(groups, 40, "premise: the fixture has many row groups, so one scan and one per group differ") diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index fa6cf7d54..9f0c52540 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -67,6 +67,7 @@ SUITES=( catalog_natts catalog_plan_index catalog_scan_index + column_extent column_projection concurrency concurrent_diff From a3a0f367439426e69c29af32c0304d077a7c50a5 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 03:53:40 +0000 Subject: [PATCH 04/13] fix: drop PgColumnarExtentForget, and make the no-ordering arm reach a recompute (#1304 option 5) Forget was redundant: pre-commit already skips any entry whose storage row is gone, so no arm could tell it was there. The no-ordering arm could not fail: a point column never gets a writer row, so no recompute ever ran for it, and a complete row with NULL bounds printed as "none" anyway. It now runs VACUUM over a fully deleted group, and both twins print a NULL bound as '-'. Co-Authored-By: Claude Opus 5.5 --- src/columnar.h | 1 - src/columnar_extent.c | 18 ------------------ src/columnar_metadata.c | 1 - test/column_extent.sh | 11 +++++++++-- test/pytest/test_column_extent.py | 10 +++++++--- 5 files changed, 16 insertions(+), 25 deletions(-) diff --git a/src/columnar.h b/src/columnar.h index 277ac85e1..7ea050e4d 100644 --- a/src/columnar.h +++ b/src/columnar.h @@ -635,7 +635,6 @@ extern Oid PgColumnarStorageRelationOid(uint64 storageId, Snapshot snapshot); extern void PgColumnarExtentNote(Oid relid, uint64 storageId, int columnIndex, const NativeZoneMapMetadata *z); extern void PgColumnarExtentMarkRecompute(uint64 storageId); -extern void PgColumnarExtentForget(uint64 storageId); extern void PgColumnarExtentPreCommit(void); extern void PgColumnarExtentAtXactEnd(void); extern bool PgColumnarExtentBound(Relation rel, int columnIndex, Snapshot snapshot, diff --git a/src/columnar_extent.c b/src/columnar_extent.c index 4552bca5b..199be9776 100644 --- a/src/columnar_extent.c +++ b/src/columnar_extent.c @@ -167,24 +167,6 @@ PgColumnarExtentMarkRecompute(uint64 storageId) (void) hash_search(extent_recompute, &storageId, HASH_ENTER, NULL); } -/* the storage's metadata is gone in this transaction: write nothing for it */ -void -PgColumnarExtentForget(uint64 storageId) -{ - if (extent_pending != NULL) - { - HASH_SEQ_STATUS st; - ExtentEntry *e; - - hash_seq_init(&st, extent_pending); - while ((e = (ExtentEntry *) hash_seq_search(&st)) != NULL) - if (e->key.storageId == storageId) - (void) hash_search(extent_pending, &e->key, HASH_REMOVE, NULL); - } - if (extent_recompute != NULL) - (void) hash_search(extent_recompute, &storageId, HASH_REMOVE, NULL); -} - void PgColumnarExtentAtXactEnd(void) { diff --git a/src/columnar_metadata.c b/src/columnar_metadata.c index 208b1438e..c5277c037 100644 --- a/src/columnar_metadata.c +++ b/src/columnar_metadata.c @@ -2026,7 +2026,6 @@ PgColumnarDeleteMetadata(uint64 storageId) */ if (PgColumnarExtentCatalogPresent()) delete_rows_by_storage_id("column_extent", Anum_column_extent_storage_id, storageId); - PgColumnarExtentForget(storageId); delete_rows_by_storage_id("row_group", Anum_row_group_storage_id, storageId); delete_rows_by_storage_id("storage", Anum_native_storage_storage_id, storageId); diff --git a/test/column_extent.sh b/test/column_extent.sh index 955b71541..b42fba635 100755 --- a/test/column_extent.sh +++ b/test/column_extent.sh @@ -39,7 +39,7 @@ ext() { # ext TABLE COLUMN -> "complete|min|max", "none" when no complete exten local r r="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ -v ON_ERROR_STOP=1 -v VERBOSITY=sqlstate \ - -c "SELECT complete || '|' || minimum || '|' || maximum FROM pgcolumnar.column_extent('$1', '$2');" 2>&1)" + -c "SELECT complete || '|' || coalesce(minimum, '-') || '|' || coalesce(maximum, '-') FROM pgcolumnar.column_extent('$1', '$2');" 2>&1)" case "$r" in ERROR:*) echo "error $(sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p' <<<"$r" | head -1)";; "") echo none;; *) echo "$r";; esac } truth() { # truth TABLE COLUMN -> "true|min|max" from the data itself @@ -191,7 +191,14 @@ q "CREATE TABLE ac (ts timestamptz) USING pgcolumnar; check "a column added after older groups has no complete extent (those groups have no zone map for it)" \ "$(ext ac v)" "none" check "while the column that was there from the start still has one" "$(ext ac ts)" "$(truth ac ts)" -q "CREATE TABLE no (p point) USING pgcolumnar; INSERT INTO no SELECT point(g, g) FROM generate_series(1, 2000) g;" >/dev/null +# Even after a recompute: a type with no ordering has values its zone maps cannot +# bound, and a recompute must refuse rather than claim the column holds none. A +# fully deleted group, then VACUUM, makes one run. +q "CREATE TABLE no (p point, k int) USING pgcolumnar; + SELECT pgcolumnar.set_options('no', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO no SELECT point(g, g), g FROM generate_series(1, 3000) g; + DELETE FROM no WHERE k > 2000;" >/dev/null +q "VACUUM no;" >/dev/null check "a type with no ordering gets no extent" "$(ext no p)" "none" # ---- housekeeping ----------------------------------------------------------------- diff --git a/test/pytest/test_column_extent.py b/test/pytest/test_column_extent.py index 929873457..1529444a2 100644 --- a/test/pytest/test_column_extent.py +++ b/test/pytest/test_column_extent.py @@ -39,7 +39,7 @@ def _ext(conn, table, col): """'true|min|max', 'none' when no complete extent, or 'error '.""" try: with conn.cursor() as cur: - cur.execute("SELECT complete::text || '|' || minimum || '|' || maximum" + cur.execute("SELECT complete::text || '|' || coalesce(minimum, '-') || '|' || coalesce(maximum, '-')" f" FROM pgcolumnar.column_extent('{table}', '{col}')") row = cur.fetchone() except psycopg.Error as e: @@ -212,8 +212,12 @@ def test_no_complete_extent_where_the_zone_maps_cannot_give_one(pgc_own_db, expe "a column added after older groups has no complete extent (those groups have no zone map for it)") expect.text(_ext(conn, "ac", "ts"), _truth(conn, "ac", "ts"), "while the column that was there from the start still has one") - _run(conn, "CREATE TABLE no (p point) USING pgcolumnar", - "INSERT INTO no SELECT point(g, -g) FROM generate_series(1, 2400) g") + # A recompute (VACUUM after a fully deleted group) must refuse a column whose + # values have no ordering, not claim it holds none. + _run(conn, "CREATE TABLE no (p point, k int) USING pgcolumnar") + _small_groups(conn, "no") + _run(conn, "INSERT INTO no SELECT point(g, -g), g FROM generate_series(1, 2400) g", + "DELETE FROM no WHERE k > 2000", "VACUUM no") expect.text(_ext(conn, "no", "p"), "none", "a type with no ordering gets no extent") From 1326dfea652b0cbafde8761dcff865725a296855 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 03:55:23 +0000 Subject: [PATCH 05/13] docs, test: column_extent in the reference, CHANGELOG, TESTS.md 98, the pair, cluster_tests 517 (#1304 option 5) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 15 ++++++++++++++ docs/sql-reference.md | 31 +++++++++++++++++++++++++++++ test/pytest/TESTS.md | 28 ++++++++++++++++++++++++++ test/pytest/expected_tests.txt | 5 ++++- test/pytest/test_compare_to_bash.py | 1 + 5 files changed, 79 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb6f21865..657a731d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,21 @@ true until the next version shipped. ### Added +- The planner's estimate of how many time buckets a `GROUP BY date_trunc(...)` + or `date_bin(...)` produces now reads a per-column extent + (`pgcolumnar.column_extent`) in one index lookup, instead of every row group's + zone map (#1304, for #1292). Planning cost no longer grows with the table. + - **Writes:** a write adds its own extent row when it commits. + - **Recomputed:** every operation that drops or rewrites row groups (`expire`, + `compact_rewrite`, `VACUUM`, `recluster`) recomputes the extent before + committing. So it is never narrower than the data, and wider only by rows + deleted but still stored. + - **Fallback:** where the zone maps cannot give a bound, the planner reads them + as before: a column added after older groups, or a type with no ordering. + - **Inspection:** `pgcolumnar.column_extent(rel, col)` shows the bound. + - **Tests:** `test/column_extent.sh` and an independent + `test_column_extent.py`. + - Approximate distinct counts and percentiles over columnar tables are tested with the `hll` and `tdigest` extensions (#1296 phase 2), and a how-to section states the tested versions: hll 2.19 on PostgreSQL 15 to 18 (it does not build diff --git a/docs/sql-reference.md b/docs/sql-reference.md index acba89adc..e8e86221c 100644 --- a/docs/sql-reference.md +++ b/docs/sql-reference.md @@ -450,6 +450,37 @@ which counts as appended. The record is internal storage metadata and `pg_dump` does not carry it, so a restored table reports no sorted groups until you sort it again. +### pgcolumnar.column_extent(rel regclass, col name) + +Shows the bound the planner uses for a column's smallest and largest value, when +it estimates how many time buckets a `GROUP BY date_trunc(...)` or `date_bin(...)` +produces. Returns one row, or none: + +| Column | Type | Meaning | +| --- | --- | --- | +| `complete` | bool | Always true when a row is returned. | +| `minimum` | text | The smallest value, printed by the column's type. NULL if the column holds no non-NULL value. | +| `maximum` | text | The largest value, printed the same way. | + +The bound is kept in `pgcolumnar.column_extent`, and the planner reads it in one +index lookup however many row groups the table has. Writes widen it when they +commit. Every operation that drops or rewrites row groups recomputes it before +committing: `expire`, `compact_rewrite`, `VACUUM`, `recluster`. So it can be +wider than the data only by rows that are deleted but still stored. + +No row means the planner reads every row group's zone map instead, which gives +the same answer at a cost that grows with the table. That happens for: + +- a column added after some row groups were written; +- a type with no ordering; +- a table written by an earlier version, until its next recompute. + +The values are column data, so the caller needs `SELECT` on the table. + +```sql +SELECT minimum, maximum FROM pgcolumnar.column_extent('events', 'ts'); +``` + ### pgcolumnar.maintenance_due(rel regclass, compact_due_fraction float8 DEFAULT 0.2, recluster_due_fraction float8 DEFAULT 0.05) Reports whether an online maintenance verb is worth running on a table, from its diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index b0483aace..362823813 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -147,6 +147,7 @@ behaviour, the source of that number is named. - [99. test_suite_exit_trap.py: a suite's own EXIT trap must still stop its cluster](#99-test_suite_exit_trappy-a-suites-own-exit-trap-must-still-stop-its-cluster) - [100. test_tier_partition.py: tier_partition moves a cold partition to Parquet, exactly once](#100-test_tier_partitionpy-tier_partition-moves-a-cold-partition-to-parquet-exactly-once) - [101. test_prepared_writes.py: a write prepared and finished in one session leaves the backend alive](#101-test_prepared_writespy-a-write-prepared-and-finished-in-one-session-leaves-the-backend-alive) +- [102. test_column_extent.py: the per-column extent is a true bound, recomputed whenever groups go](#102-test_column_extentpy-the-per-column-extent-is-a-true-bound-recomputed-whenever-groups-go) ## 1. How to read a test in here @@ -6699,3 +6700,30 @@ Red on main, the first arm reading `lost`. Each of the two discards is red under | test | what it holds | | --- | --- | | `test_a_write_prepared_and_finished_in_one_session` | an insert, a delete and an update prepared and committed in one session, an insert prepared and rolled back, a write after COMMIT PREPARED, and an index fetch on each side of it, all answered by the same session; the server log gains no "terminated by signal"; no prepared transaction is left | + +## 102. test_column_extent.py: the per-column extent is a true bound, recomputed whenever groups go + +Port of `column_extent.sh` (#1304 option 5, for #1292 phase 2). `pgcolumnar.column_extent` keeps, per column, rows that are inserted and deleted but never updated. Each committing writer adds one row for the groups it flushed. A recompute over every zone map one snapshot sees replaces the rows that snapshot sees with one complete row. Every path that retires a group recomputes before commit. The planner uses the rows only when a complete one is visible, and their fold is then a true bound. `pgcolumnar.column_extent(rel, col)` shows that bound. + +THE ORACLE IS THE DATA: `min()` and `max()` of the column. Every equality arm first makes sure no group is left holding deleted rows, so the zone maps and the data agree. Twice, the first draft of an arm deleted rows that left a group partly live and so wider than the data, and the extent was right where the arm was wrong. The one case that may legitimately be wider, a rolled-back savepoint that flushed a group, is asserted as containment. + +Independent of the shell suite: its own row counts, values, times and helpers. The check names match, so `compare_to_bash` grades the pair one for one. + +WHAT WRITING IT FOUND: + +- **Ordinary writers never overlap.** Concurrent writers into one table are serialized by the storage-row lock (class 102) from their first flush to their commit. The concurrency arm therefore uses `parallel_copy`, whose loaders flush into one storage at once, each through PREPARE. +- **COMMIT PREPARED crashes in the preparing session.** On main, `COMMIT PREPARED` in the session that prepared a pgcolumnar write crashes the backend (#1323), so this file commits from another session. +- **The xact scan counters carry over.** `pg_stat_get_xact_numscans` reports a backend's pending stats, flushed about once a second rather than at transaction end. A measurement in the same connection as the writes counts their zone-map reads too, so the planning arm measures on a fresh connection. + +Written red first: the shell twin, against main, with the function absent (42883). + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_extent_is_the_columns_own_order` | `timestamptz`, `int` across 256 and negative, negative `numeric`, `date` and `text COLLATE "C"`: each column's extent is the data's min and max, over several row groups | +| `test_the_planner_reads_the_extent_not_the_zone_maps` | planning a bucketed GROUP BY scans `zone_map` 0 times with a complete extent and 1 without, with the same estimate; with no rows the function returns none; the next writer recomputes one over every group | +| `test_writers_concurrent_aborted_and_prepared` | `parallel_copy` with three loaders, an aborted writer, a rolled-back savepoint (containment), PREPARE and COMMIT PREPARED, and a table created in a rolled-back savepoint (no orphan rows) | +| `test_a_shrink_is_recomputed_never_guessed` | after `expire`, `compact_rewrite` and VACUUM retire or rewrite groups, the extent equals the data left, and after `expire` it moved | +| `test_no_complete_extent_where_the_zone_maps_cannot_give_one` | a column added after older groups, and a `point` column even after a recompute, get no complete extent | +| `test_housekeeping_and_privilege` | fifty committing writers leave at most nine rows; TRUNCATE and DROP remove them; an ordinary role cannot read the catalog, nor the function without SELECT | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 7357fd88e..1a767972f 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -585,4 +585,7 @@ guard_tests 419 # # cluster_tests 516 -> 517 for #1323's test_prepared_writes.py, one test, by collection # on this branch rebased onto 305e15e (#1317). -cluster_tests 517 +# +# cluster_tests 517 -> 523 for #1304 option 5's test_column_extent.py, six tests, +# by collection on this branch rebased onto 036d452 (#1324). +cluster_tests 523 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 6814e0def..d243c5d0f 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -95,6 +95,7 @@ "catalog_delete_index", "catalog_plan_index", "catalog_scan_index", + "column_extent", "differential", "encode_post_codec", "format_downgrade", From f3a1091375b82484dd781fcce82fa16eb11b55ad Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 03:56:26 +0000 Subject: [PATCH 06/13] fix: include utils/tuplestore.h for column_extent's result set (PG19) PG19 no longer reaches tuplestore_putvalues through the headers that 15-18 do; the PG19 build failed on an implicit declaration. Co-Authored-By: Claude Opus 5.5 --- src/columnar_extent.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/columnar_extent.c b/src/columnar_extent.c index 199be9776..c4197c4c8 100644 --- a/src/columnar_extent.c +++ b/src/columnar_extent.c @@ -52,6 +52,7 @@ #include "utils/memutils.h" #include "utils/rel.h" #include "utils/snapmgr.h" +#include "utils/tuplestore.h" #include "utils/typcache.h" PG_FUNCTION_INFO_V1(pgcolumnar_column_extent); From 8b9c542eb6a6ab646453844e37b76eaa85890aea Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 03:59:40 +0000 Subject: [PATCH 07/13] test: column_extent enters the ledger, 33 checks, 27 dated by 18 mutations (#1304 option 5) Clean: column_extent.sh and test_column_extent.py 33/33 on PG15-19. Each mutation log merged one at a time with --mutation and --target. A nineteenth (the existence check on the catalog snapshot) survived and is equivalent. Census 1698 -> 1704; rows 2092; the gate agrees. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 33 +++++++++++++++++++++++++++++++++ test/check_ledger_budget.txt | 21 ++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index cf690552c..d6541f5e4 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -204,6 +204,39 @@ catalog_scan_index catalog_scan_index premise: truncating a projected table reac catalog_scan_index catalog_scan_index premise: truncating a projected table took effect 15;16;17;18;19 never - catalog_scan_index catalog_scan_index renaming a projected column does not read pgcolumnar.projection_declaration whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole catalog_scan_index catalog_scan_index truncating a projected table does not read pgcolumnar.projection_declaration whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole +column_extent column_extent DROP TABLE removes them 15;16;17;18;19 2026-10-13 PgColumnarDeleteMetadata leaves the extent rows +column_extent column_extent TRUNCATE removes the old storage's extent rows 15;16;17;18;19 2026-10-13 PgColumnarDeleteMetadata leaves the extent rows +column_extent column_extent a column added after older groups has no complete extent (those groups have no zone map for it) 15;16;17;18;19 2026-10-13 a recompute that ignores a group with no zone map for the column;pgcolumnar.column_extent returns a row with no complete extent;the extent trusted without a complete row +column_extent column_extent a prepared transaction's groups are inside the extent after COMMIT PREPARED 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit;no extent rows written at PRE_PREPARE +column_extent column_extent a savepoint rolled back after flushing may widen the extent, never narrow it 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit;no extent rows written at PRE_PREPARE;the extent folded by comparing Datums as words, not with the type's comparison;the writer's fold keeps the largest minimum +column_extent column_extent a table created in a rolled-back savepoint leaves no extent rows 15;16;17;18;19 2026-10-13 pre-commit writes extent rows for a storage that does not exist +column_extent column_extent a type with no ordering gets no extent 15;16;17;18;19 2026-10-13 a recompute that skips values without an ordering;pgcolumnar.column_extent returns a row with no complete extent +column_extent column_extent a writer that finds no complete extent recomputes one over every group 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one +column_extent column_extent a writer that rolls back leaves the extent as the data has it 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit;no extent rows written at PRE_PREPARE;the writer's fold keeps the largest minimum +column_extent column_extent after VACUUM retires a fully deleted group the extent is the live rows' 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded +column_extent column_extent after compact_rewrite the extent is the live rows' 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded +column_extent column_extent after expire the extent is recomputed to the data that is left 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded +column_extent column_extent and it did move: the retired groups held the old minimum 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded +column_extent column_extent and the estimate is the one the zone maps give 15;16;17;18;19 2026-10-13 the planner takes the extent's minimum as its upper bound too +column_extent column_extent and the extent covers what they loaded 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit;no extent rows written at PRE_PREPARE;the writer's fold keeps the largest minimum +column_extent column_extent and the extent still covers all fifty 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit +column_extent column_extent and with no extent rows the function reports no complete extent 15;16;17;18;19 2026-10-13 pgcolumnar.column_extent returns a row with no complete extent +column_extent column_extent compact_rewrite rewrites the groups the deletes thinned 15;16;17;18;19 never - +column_extent column_extent expire retires the old row groups with the extent in place 15;16;17;18;19 never - +column_extent column_extent fifty committing writers leave a bounded number of extent rows 15;16;17;18;19 2026-10-13 a recompute that leaves the rows it folded;a writer that finds no complete extent does not recompute one;no merge however many extent rows a column has +column_extent column_extent nor is a table's extent through the function, without SELECT on the table 15;16;17;18;19 2026-10-13 pgcolumnar.column_extent without its SELECT check +column_extent column_extent parallel_copy's loaders, flushing into one table at once, all commit 15;16;17;18;19 never - +column_extent column_extent planning a bucketed GROUP BY scans zone_map not at all when the extent is complete 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;the planner's time range ignores the extent +column_extent column_extent premise: the extent function exists 15;16;17;18;19 never - +column_extent column_extent premise: the typed fixture spans several row groups 15;16;17;18;19 never - +column_extent column_extent the extent catalog is not readable by an ordinary role (it holds column values) 15;16;17;18;19 2026-10-13 the column_extent catalog granted to PUBLIC +column_extent column_extent the extent of ty.d is the data's min and max, in the type's order 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one +column_extent column_extent the extent of ty.i is the data's min and max, in the type's order 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one +column_extent column_extent the extent of ty.n is the data's min and max, in the type's order 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;the extent folded by comparing Datums as words, not with the type's comparison +column_extent column_extent the extent of ty.s is the data's min and max, in the type's order 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one +column_extent column_extent the extent of ty.ts is the data's min and max, in the type's order 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one +column_extent column_extent while the column that was there from the start still has one 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one +column_extent column_extent without the extent rows the planner falls back to one zone_map scan 15;16;17;18;19 never - differential differential agg avg 15;16;17;18;19 never - differential differential agg count 15;16;17;18;19 never - differential differential agg minmax 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 9a35a789c..f5ac2a0d2 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1493,4 +1493,23 @@ suites_not_covered 247 # never 1734 + 5 = 1739 # dated 411 + 6 = 417 # 1739 + 417 == 2156 -checks_never_observed_red 1739 +# +# #1304 option 5, the per-column extent (OffgridwithJD): 1739 -> 1745. +# column_extent enters the ledger with 33 checks from a five-major run +# (column_extent.sh 33/33 on each of 15-19). Twenty-seven are dated by eighteen +# mutations of the extent code, its SQL and the planner's read, each merged one +# log at a time with --mutation and --target; no premise is among them. A +# nineteenth mutation survived and is equivalent: the existence check reading +# the catalog snapshot instead of its own, after the CommandCounterIncrement +# that refreshes both. Six arrive `never`: two premises (the function exists, +# the fixture's row groups) and four arms held by construction -- expire and +# compact_rewrite still retire and rewrite with the extent in place, +# parallel_copy's loaders all commit (no row is ever updated in place, so there +# is no update to conflict), and with no extent the planner's zone-map read is +# still one scan (main's code, guarded by native_groupagg too). +# +# ledger rows 2156 + 33 = 2189 +# never 1739 + 6 = 1745 +# dated 417 + 27 = 444 +# 1745 + 444 == 2189 +checks_never_observed_red 1745 From 6c62013588f06fc06b6d05053907edc5fb4eccf0 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 12:07:29 +0000 Subject: [PATCH 08/13] test: a transaction's own flushed groups must be inside the extent it reads, red (#1304 option 5) Its extent row is written at pre-commit, but its groups are visible to it as soon as they flush. Seen red on PG16 in both twins: column_extent reads 'no', and the in-transaction estimate is 35 rows (61 in the twin) against the zone maps' 5000 (4500), an under-estimate. Co-Authored-By: Claude Opus 5.5 --- test/column_extent.sh | 28 +++++++++++++++++++ test/pytest/test_column_extent.py | 45 +++++++++++++++++++++++++++++++ 2 files changed, 73 insertions(+) diff --git a/test/column_extent.sh b/test/column_extent.sh index b42fba635..932c9d15c 100755 --- a/test/column_extent.sh +++ b/test/column_extent.sh @@ -150,6 +150,34 @@ check "a table created in a rolled-back savepoint leaves no extent rows" \ "$(q "SELECT count(*) FROM pgcolumnar.column_extent e WHERE NOT EXISTS (SELECT 1 FROM pgcolumnar.storage s WHERE s.storage_id = e.storage_id);")" "0" +# ---- a transaction reads its own flushed groups in the extent -------------------- +# +# Its extent row is written at pre-commit, but its groups are visible to it from the +# moment they flush. A plan made in between must not bound the column by the +# committed rows alone: that is an UNDER-estimate, the direction the gate on this +# bound must never take. The oracle is the same transaction with the extent rows +# deleted, so the planner falls back to the zone maps, which do include them. +q "CREATE TABLE ot (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('ot', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO ot SELECT '2026-06-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 2000) g; + ANALYZE ot;" >/dev/null +OWN_INS="INSERT INTO ot SELECT '2050-01-01'::timestamptz + g * interval '1 min', g FROM generate_series(1, 3000) g" +check "a transaction's own flushed groups are inside the extent it reads before commit" \ + "$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At -q \ + -c "BEGIN" -c "$OWN_INS" \ + -c "SELECT CASE WHEN e.complete AND e.minimum::timestamptz <= (SELECT min(ts) FROM ot) + AND e.maximum::timestamptz >= (SELECT max(ts) FROM ot) THEN 'yes' ELSE 'no' END + FROM pgcolumnar.column_extent('ot', 'ts') e" -c "ROLLBACK" 2>&1)" "yes" +own_plans="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At -q \ + -c "$GB" -c "BEGIN" -c "$OWN_INS" \ + -c "SELECT 'plan'" -c "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM ot GROUP BY b" \ + -c "DELETE FROM pgcolumnar.column_extent WHERE storage_id = $(sid ot)" \ + -c "SELECT 'plan'" -c "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM ot GROUP BY b" -c "ROLLBACK" 2>&1 | + awk '/^plan$/ { want = 1; next } want && match($0, /rows=[0-9]+/) { print substr($0, RSTART, RLENGTH); want = 0 }')" +check "and a plan made in that transaction estimates as the zone maps do" \ + "$(sed -n 1p <<<"$own_plans")" "$(sed -n 2p <<<"$own_plans")" +check "premise: the transaction's plans gave two estimates" "$(wc -l <<<"$own_plans")" "2" + # ---- a shrink is recomputed, never guessed (jdatcmd, #1304) --------------------- # # Each path that retires or rewrites groups recomputes the extent in the same diff --git a/test/pytest/test_column_extent.py b/test/pytest/test_column_extent.py index 1529444a2..4d74761d1 100644 --- a/test/pytest/test_column_extent.py +++ b/test/pytest/test_column_extent.py @@ -168,6 +168,51 @@ def test_writers_concurrent_aborted_and_prepared(pgc_own_db, pgc_cluster, expect "a table created in a rolled-back savepoint leaves no extent rows") +def test_a_transaction_reads_its_own_flushed_groups(pgc_own_db, pgc_cluster, expect): + """Its extent row is written at pre-commit, but its groups are visible to it as + soon as they flush; a plan made in between must not bound the column by the + committed rows alone. The oracle is the same transaction with the extent rows + deleted, which sends the planner to the zone maps.""" + conn = pgc_own_db + dsn = pgc_cluster.dsn(dbname=_one(conn, "SELECT current_database()")) + _run(conn, "CREATE TABLE ow (ts timestamptz, v int) USING pgcolumnar") + _small_groups(conn, "ow") + _run(conn, "INSERT INTO ow SELECT '2025-03-03'::timestamptz + g * interval '2 s', g FROM generate_series(1, 1800) g", + "ANALYZE ow") + sid = _sid(conn, "ow") + own = "INSERT INTO ow SELECT '1999-09-09'::timestamptz + g * interval '1 min', g FROM generate_series(1, 2700) g" + plan = "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM ow GROUP BY b" + + def estimate(cur): + cur.execute(plan) + text = "\n".join(r[0] for r in cur.fetchall()) + return text.split("rows=", 1)[1].split(" ", 1)[0] if "rows=" in text else "" + + with psycopg.connect(dsn) as c: + with c.cursor() as cur: + cur.execute(own) + cur.execute("SELECT CASE WHEN e.complete AND e.minimum::timestamptz <= (SELECT min(ts) FROM ow)" + " AND e.maximum::timestamptz >= (SELECT max(ts) FROM ow) THEN 'yes' ELSE 'no' END" + " FROM pgcolumnar.column_extent('ow', 'ts') e") + row = cur.fetchone() + inside = row[0] if row else "none" + c.rollback() + expect.text(inside, "yes", "a transaction's own flushed groups are inside the extent it reads before commit") + + with psycopg.connect(dsn) as c: + with c.cursor() as cur: + cur.execute(GB) + cur.execute(own) + with_extent = estimate(cur) + cur.execute(f"DELETE FROM pgcolumnar.column_extent WHERE storage_id = {sid}") + from_zone_maps = estimate(cur) + c.rollback() + expect.text("two" if with_extent and from_zone_maps else f"[{with_extent}] [{from_zone_maps}]", "two", + "premise: the transaction's plans gave two estimates") + expect.text(f"rows={with_extent}", f"rows={from_zone_maps}", + "and a plan made in that transaction estimates as the zone maps do") + + def test_a_shrink_is_recomputed_never_guessed(pgc_own_db, expect): conn = pgc_own_db _run(conn, "CREATE TABLE ex (ts timestamptz, v int) USING pgcolumnar") From abcd36976e06060d9024ad1d7ad03a74e4ed6533 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 12:07:29 +0000 Subject: [PATCH 09/13] fix: the extent a transaction reads includes its own pending fold (#1304 option 5) Co-Authored-By: Claude Opus 5.5 --- src/columnar_extent.c | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/src/columnar_extent.c b/src/columnar_extent.c index c4197c4c8..1a43395e3 100644 --- a/src/columnar_extent.c +++ b/src/columnar_extent.c @@ -25,8 +25,10 @@ * complete row R (computed at snapshot B, committed before S): the groups S sees * were either visible at B, and so are in R, or committed between B and S, in * which case their writer's own row committed with them and S sees it. A retire - * and its recompute are one transaction, so S sees both or neither. Otherwise - * the caller falls back to the zone maps. + * and its recompute are one transaction, so S sees both or neither. The + * reader's own transaction adds its pending fold, since its groups are visible + * to it before its row is written. Otherwise the caller falls back to the zone + * maps. * * A ROLLED-BACK SUBTRANSACTION that flushed a group leaves its values in the * fold: wider than the data, which only over-estimates. @@ -439,6 +441,12 @@ PgColumnarExtentPreCommit(void) * The column's bound as the snapshot sees it, when a complete row is * visible. false: no trustworthy extent, fall back to the zone maps. * *hasValues false: the column holds no non-NULL value. + * + * THE CALLER'S OWN TRANSACTION sees the groups it has flushed, but its + * extent row is written only at pre-commit, so the visible rows alone would + * leave those groups out and bound the column too narrowly. Its pending fold + * is added in. Groups it has retired are still in the visible rows until + * its recompute commits: wider than the data, which only over-estimates. */ bool PgColumnarExtentBound(Relation rel, int columnIndex, Snapshot snapshot, @@ -448,12 +456,14 @@ PgColumnarExtentBound(Relation rel, int columnIndex, Snapshot snapshot, List *rows; ListCell *lc; bool complete = false; + uint64 storageId; if (columnIndex < 0 || columnIndex >= RelationGetDescr(rel)->natts || !PgColumnarExtentCatalogPresent()) return false; att = TupleDescAttr(RelationGetDescr(rel), columnIndex); - rows = PgColumnarReadExtentRows(PgColumnarStorageId(rel), columnIndex, snapshot); + storageId = PgColumnarStorageId(rel); + rows = PgColumnarReadExtentRows(storageId, columnIndex, snapshot); *hasValues = false; foreach(lc, rows) { @@ -472,6 +482,24 @@ PgColumnarExtentBound(Relation rel, int columnIndex, Snapshot snapshot, *maximum = rmx; *hasValues = true; } + if (complete && extent_pending != NULL) + { + ExtentKey key; + ExtentEntry *e; + + memset(&key, 0, sizeof(key)); + key.storageId = storageId; + key.columnIndex = columnIndex; + e = (ExtentEntry *) hash_search(extent_pending, &key, HASH_FIND, NULL); + if (e != NULL) + { + if (!*hasValues || extent_cmp(att, e->minimum, *minimum) < 0) + *minimum = e->minimum; + if (!*hasValues || extent_cmp(att, e->maximum, *maximum) > 0) + *maximum = e->maximum; + *hasValues = true; + } + } return complete; } From 585cc17b9feafae22a03564a3bf9abb620894d6b Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 12:09:53 +0000 Subject: [PATCH 10/13] test: option 5's own-transaction checks enter the ledger and the twin's docs (#1304 option 5) column_extent.sh 36/36 and test_column_extent.py 7/7 on each of PG15-19. The two arms are dated by the run before the fix; the instrument premise arrives never. Ledger 2095 / 1705. cluster_tests 517 -> 518 by collection. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 3 +++ test/check_ledger_budget.txt | 15 ++++++++++++++- test/pytest/TESTS.md | 2 ++ test/pytest/expected_tests.txt | 5 ++++- 4 files changed, 23 insertions(+), 2 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index d6541f5e4..a9506a49f 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -210,12 +210,14 @@ column_extent column_extent a column added after older groups has no complete ex column_extent column_extent a prepared transaction's groups are inside the extent after COMMIT PREPARED 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit;no extent rows written at PRE_PREPARE column_extent column_extent a savepoint rolled back after flushing may widen the extent, never narrow it 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit;no extent rows written at PRE_PREPARE;the extent folded by comparing Datums as words, not with the type's comparison;the writer's fold keeps the largest minimum column_extent column_extent a table created in a rolled-back savepoint leaves no extent rows 15;16;17;18;19 2026-10-13 pre-commit writes extent rows for a storage that does not exist +column_extent column_extent a transaction's own flushed groups are inside the extent it reads before commit 15;16;17;18;19 2026-10-13 the extent a transaction reads leaves out its own pending fold (the tree before the fix) column_extent column_extent a type with no ordering gets no extent 15;16;17;18;19 2026-10-13 a recompute that skips values without an ordering;pgcolumnar.column_extent returns a row with no complete extent column_extent column_extent a writer that finds no complete extent recomputes one over every group 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one column_extent column_extent a writer that rolls back leaves the extent as the data has it 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit;no extent rows written at PRE_PREPARE;the writer's fold keeps the largest minimum column_extent column_extent after VACUUM retires a fully deleted group the extent is the live rows' 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded column_extent column_extent after compact_rewrite the extent is the live rows' 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded column_extent column_extent after expire the extent is recomputed to the data that is left 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded +column_extent column_extent and a plan made in that transaction estimates as the zone maps do 15;16;17;18;19 2026-10-13 the extent a transaction reads leaves out its own pending fold (the tree before the fix) column_extent column_extent and it did move: the retired groups held the old minimum 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded column_extent column_extent and the estimate is the one the zone maps give 15;16;17;18;19 2026-10-13 the planner takes the extent's minimum as its upper bound too column_extent column_extent and the extent covers what they loaded 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit;no extent rows written at PRE_PREPARE;the writer's fold keeps the largest minimum @@ -228,6 +230,7 @@ column_extent column_extent nor is a table's extent through the function, withou column_extent column_extent parallel_copy's loaders, flushing into one table at once, all commit 15;16;17;18;19 never - column_extent column_extent planning a bucketed GROUP BY scans zone_map not at all when the extent is complete 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;the planner's time range ignores the extent column_extent column_extent premise: the extent function exists 15;16;17;18;19 never - +column_extent column_extent premise: the transaction's plans gave two estimates 15;16;17;18;19 never - column_extent column_extent premise: the typed fixture spans several row groups 15;16;17;18;19 never - column_extent column_extent the extent catalog is not readable by an ordinary role (it holds column values) 15;16;17;18;19 2026-10-13 the column_extent catalog granted to PUBLIC column_extent column_extent the extent of ty.d is the data's min and max, in the type's order 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index f5ac2a0d2..58b158409 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1512,4 +1512,17 @@ suites_not_covered 247 # never 1739 + 6 = 1745 # dated 417 + 27 = 444 # 1745 + 444 == 2189 -checks_never_observed_red 1745 +# +# Option 5, a transaction's own flushed groups (OffgridwithJD): 1745 -> 1746. +# Three checks from a five-major run of the fixed tree (column_extent.sh 36/36 +# on each of 15-19). The two arms are dated by the run before the fix, the +# tree whose bound left out the transaction's own pending fold: column_extent +# read 'no', and the estimate was 35 rows against the zone maps' 5000. The +# premise that the plans gave two estimates is a fact about the instrument and +# arrives never. +# +# ledger rows 2189 + 3 = 2192 +# never 1745 + 1 = 1746 +# dated 444 + 2 = 446 +# 1746 + 446 == 2192 +checks_never_observed_red 1746 diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 362823813..8a9eb7113 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -6713,6 +6713,7 @@ WHAT WRITING IT FOUND: - **Ordinary writers never overlap.** Concurrent writers into one table are serialized by the storage-row lock (class 102) from their first flush to their commit. The concurrency arm therefore uses `parallel_copy`, whose loaders flush into one storage at once, each through PREPARE. - **COMMIT PREPARED crashes in the preparing session.** On main, `COMMIT PREPARED` in the session that prepared a pgcolumnar write crashes the backend (#1323), so this file commits from another session. +- **A transaction did not see its own groups in the extent.** Its extent row is written at pre-commit, but its groups are visible to it from the moment they flush. A plan made in between bounded the column by the committed rows alone, which is an under-estimate: 61 rows against the zone maps' 4500 in this file's fixture. The bound now adds the transaction's own pending fold. - **The xact scan counters carry over.** `pg_stat_get_xact_numscans` reports a backend's pending stats, flushed about once a second rather than at transaction end. A measurement in the same connection as the writes counts their zone-map reads too, so the planning arm measures on a fresh connection. Written red first: the shell twin, against main, with the function absent (42883). @@ -6724,6 +6725,7 @@ Written red first: the shell twin, against main, with the function absent (42883 | `test_the_extent_is_the_columns_own_order` | `timestamptz`, `int` across 256 and negative, negative `numeric`, `date` and `text COLLATE "C"`: each column's extent is the data's min and max, over several row groups | | `test_the_planner_reads_the_extent_not_the_zone_maps` | planning a bucketed GROUP BY scans `zone_map` 0 times with a complete extent and 1 without, with the same estimate; with no rows the function returns none; the next writer recomputes one over every group | | `test_writers_concurrent_aborted_and_prepared` | `parallel_copy` with three loaders, an aborted writer, a rolled-back savepoint (containment), PREPARE and COMMIT PREPARED, and a table created in a rolled-back savepoint (no orphan rows) | +| `test_a_transaction_reads_its_own_flushed_groups` | inside a transaction that has flushed groups far outside the committed extent, `column_extent` contains them, and a plan estimates as it does with the extent rows deleted (the zone maps) | | `test_a_shrink_is_recomputed_never_guessed` | after `expire`, `compact_rewrite` and VACUUM retire or rewrite groups, the extent equals the data left, and after `expire` it moved | | `test_no_complete_extent_where_the_zone_maps_cannot_give_one` | a column added after older groups, and a `point` column even after a recompute, get no complete extent | | `test_housekeeping_and_privilege` | fifty committing writers leave at most nine rows; TRUNCATE and DROP remove them; an ordinary role cannot read the catalog, nor the function without SELECT | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 1a767972f..168a8877e 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -588,4 +588,7 @@ guard_tests 419 # # cluster_tests 517 -> 523 for #1304 option 5's test_column_extent.py, six tests, # by collection on this branch rebased onto 036d452 (#1324). -cluster_tests 523 +# +# cluster_tests 523 -> 524 for option 5's test_a_transaction_reads_its_own_flushed_groups, +# by collection on the same tree. +cluster_tests 524 From 8b4da2af700e4f22ffbac2bad0d68191192e7042 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 13:03:54 +0000 Subject: [PATCH 11/13] fix: initialise the extent and zone-map bounds the planner reads (#1325 CI) gcc 13/14 with LTO warned that zlo and zhi may be used uninitialized, and CI treats warnings as failures. Every path that returns true sets both, and the extent's emin/emax are read only when the column has a value; gcc cannot see either through the out-parameters. Not reproduced locally: gcc 15 with the same flags and LTO is silent, so CI's build is the evidence. Co-Authored-By: Claude Opus 5.5 --- src/columnar_vector.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/columnar_vector.c b/src/columnar_vector.c index 17c023e37..c02e2accb 100644 --- a/src/columnar_vector.c +++ b/src/columnar_vector.c @@ -2085,9 +2085,14 @@ pgcolumnar_zonemap_time_range(PlannerInfo *root, RelOptInfo *input_rel, Var *tsv * cost a scan of every group's. */ { - Datum emin, - emax; - bool hasValues; + /* + * Set only when the column has a value, which the return below checks; + * initialised because gcc 13/14 with LTO cannot see that, and CI treats + * its maybe-uninitialized warning as a failure (#1325 review). + */ + Datum emin = (Datum) 0, + emax = (Datum) 0; + bool hasValues = false; if (PgColumnarExtentBound(r, col, snap, &emin, &emax, &hasValues)) { @@ -2323,8 +2328,8 @@ pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel, * UNDER-estimate, the direction a gate on this number must never take. */ { - double zlo, - zhi; + double zlo = 0, + zhi = 0; /* set whenever the call returns true */ if (pgcolumnar_zonemap_time_range(root, input_rel, tsvar, &zlo, &zhi)) { From 277ac414bdcd3ac626cf392d2e912efe8de497c2 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 13:03:54 +0000 Subject: [PATCH 12/13] test: three column_extent checks that hold by construction are premises (#1236) compact_rewrite and expire retiring with the extent in place, and parallel_copy's loaders all committing, are facts about the fixture that no mutation of this change reddens. Renamed in both twins, from jdatcmd's review of #1325. Co-Authored-By: Claude Opus 5.5 --- test/column_extent.sh | 6 +++--- test/pytest/test_column_extent.py | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/test/column_extent.sh b/test/column_extent.sh index 932c9d15c..b81c750f3 100755 --- a/test/column_extent.sh +++ b/test/column_extent.sh @@ -112,7 +112,7 @@ q "CREATE TABLE cw (ts timestamptz, v int) USING pgcolumnar; INSERT INTO cw SELECT '2026-06-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 2000) g;" >/dev/null q "COPY (SELECT '2020-01-01'::timestamptz + g * interval '1 day', g FROM generate_series(1, 4000) g) TO '$PGC_WORKDIR/cw.txt' WITH (FORMAT text);" >/dev/null -check "parallel_copy's loaders, flushing into one table at once, all commit" \ +check "premise: parallel_copy's loaders, flushing into one table at once, all commit" \ "$(q "SELECT pgcolumnar.parallel_copy('cw', '$PGC_WORKDIR/cw.txt', 4);")" "4000" check "and the extent covers what they loaded" "$(ext cw ts)" "$(truth cw ts)" @@ -190,7 +190,7 @@ q "CREATE TABLE ex (ts timestamptz, v int) USING pgcolumnar; INSERT INTO ex SELECT now() - interval '1 day' + g * interval '1 s', g FROM generate_series(1, 3000) g;" >/dev/null ex_before="$(ext ex ts)" retired="$(q "SELECT pgcolumnar.expire('ex');")" -check "expire retires the old row groups with the extent in place" "$([ "${retired:-0}" -ge 1 ] && echo yes || echo no)" "yes" +check "premise: expire retires the old row groups with the extent in place" "$([ "${retired:-0}" -ge 1 ] && echo yes || echo no)" "yes" check "after expire the extent is recomputed to the data that is left" "$(ext ex ts)" "$(truth ex ts)" check "and it did move: the retired groups held the old minimum" \ "$([ "$ex_before" != "$(ext ex ts)" ] && echo yes || echo no)" "yes" @@ -200,7 +200,7 @@ q "CREATE TABLE cr (ts timestamptz, v int) USING pgcolumnar; INSERT INTO cr SELECT '2026-01-01'::timestamptz + g * interval '1 min', g FROM generate_series(1, 4000) g; DELETE FROM cr WHERE v > 3500 OR v <= 400;" >/dev/null rewritten="$(q "SELECT pgcolumnar.compact_rewrite('cr', 0.1);")" -check "compact_rewrite rewrites the groups the deletes thinned" "$([ "${rewritten:-0}" -ge 1 ] && echo yes || echo no)" "yes" +check "premise: compact_rewrite rewrites the groups the deletes thinned" "$([ "${rewritten:-0}" -ge 1 ] && echo yes || echo no)" "yes" check "after compact_rewrite the extent is the live rows'" "$(ext cr ts)" "$(truth cr ts)" q "CREATE TABLE vr (ts timestamptz, v int) USING pgcolumnar; diff --git a/test/pytest/test_column_extent.py b/test/pytest/test_column_extent.py index 4d74761d1..302b22498 100644 --- a/test/pytest/test_column_extent.py +++ b/test/pytest/test_column_extent.py @@ -127,7 +127,7 @@ def test_writers_concurrent_aborted_and_prepared(pgc_own_db, pgc_cluster, expect _run(conn, "COPY (SELECT '2018-03-03'::timestamptz + g * interval '13 hours', g" f" FROM generate_series(1, 5200) g) TO '{path}' WITH (FORMAT text)") expect.num(_one(conn, f"SELECT pgcolumnar.parallel_copy('cw', '{path}', 3)"), 5200, - "parallel_copy's loaders, flushing into one table at once, all commit") + "premise: parallel_copy's loaders, flushing into one table at once, all commit") expect.text(_ext(conn, "cw", "ts"), _truth(conn, "cw", "ts"), "and the extent covers what they loaded") with psycopg.connect(dsn) as c: @@ -223,7 +223,7 @@ def test_a_shrink_is_recomputed_never_guessed(pgc_own_db, expect): before = _ext(conn, "ex", "ts") retired = _one(conn, "SELECT pgcolumnar.expire('ex')") expect.text("yes" if (retired or 0) >= 1 else f"no ({retired})", "yes", - "expire retires the old row groups with the extent in place") + "premise: expire retires the old row groups with the extent in place") after = _ext(conn, "ex", "ts") expect.text(after, _truth(conn, "ex", "ts"), "after expire the extent is recomputed to the data that is left") expect.text("yes" if before != after else "no", "yes", @@ -235,7 +235,7 @@ def test_a_shrink_is_recomputed_never_guessed(pgc_own_db, expect): "DELETE FROM cr WHERE v > 4200 OR v <= 650") rewritten = _one(conn, "SELECT pgcolumnar.compact_rewrite('cr', 0.1)") expect.text("yes" if (rewritten or 0) >= 1 else f"no ({rewritten})", "yes", - "compact_rewrite rewrites the groups the deletes thinned") + "premise: compact_rewrite rewrites the groups the deletes thinned") expect.text(_ext(conn, "cr", "ts"), _truth(conn, "cr", "ts"), "after compact_rewrite the extent is the live rows'") _run(conn, "CREATE TABLE vr (ts timestamptz, v int) USING pgcolumnar") From 3c71b6a6db7ba3bbb476dcd6103a6f58c19b5f87 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 13:06:53 +0000 Subject: [PATCH 13/13] test: the renamed premises and the fallback arm in the ledger (#1325 review) Three renamed rows replaced from a five-major run of the renamed suite; the fallback arm dated by a mutation that removes the zone-map fallback. Ledger 2192 / 1745, gate rc=0. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 10 +++++----- test/check_ledger_budget.txt | 17 ++++++++++++++++- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index a9506a49f..e5f4e4dca 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -219,16 +219,16 @@ column_extent column_extent after compact_rewrite the extent is the live rows' 1 column_extent column_extent after expire the extent is recomputed to the data that is left 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded column_extent column_extent and a plan made in that transaction estimates as the zone maps do 15;16;17;18;19 2026-10-13 the extent a transaction reads leaves out its own pending fold (the tree before the fix) column_extent column_extent and it did move: the retired groups held the old minimum 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded -column_extent column_extent and the estimate is the one the zone maps give 15;16;17;18;19 2026-10-13 the planner takes the extent's minimum as its upper bound too +column_extent column_extent and the estimate is the one the zone maps give 15;16;17;18;19 2026-10-13 the planner takes the extent's minimum as its upper bound too;the planner's zone-map fallback removed: with no complete extent it reads nothing and bounds nothing column_extent column_extent and the extent covers what they loaded 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit;no extent rows written at PRE_PREPARE;the writer's fold keeps the largest minimum column_extent column_extent and the extent still covers all fifty 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit column_extent column_extent and with no extent rows the function reports no complete extent 15;16;17;18;19 2026-10-13 pgcolumnar.column_extent returns a row with no complete extent -column_extent column_extent compact_rewrite rewrites the groups the deletes thinned 15;16;17;18;19 never - -column_extent column_extent expire retires the old row groups with the extent in place 15;16;17;18;19 never - column_extent column_extent fifty committing writers leave a bounded number of extent rows 15;16;17;18;19 2026-10-13 a recompute that leaves the rows it folded;a writer that finds no complete extent does not recompute one;no merge however many extent rows a column has column_extent column_extent nor is a table's extent through the function, without SELECT on the table 15;16;17;18;19 2026-10-13 pgcolumnar.column_extent without its SELECT check -column_extent column_extent parallel_copy's loaders, flushing into one table at once, all commit 15;16;17;18;19 never - column_extent column_extent planning a bucketed GROUP BY scans zone_map not at all when the extent is complete 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;the planner's time range ignores the extent +column_extent column_extent premise: compact_rewrite rewrites the groups the deletes thinned 15;16;17;18;19 never - +column_extent column_extent premise: expire retires the old row groups with the extent in place 15;16;17;18;19 never - +column_extent column_extent premise: parallel_copy's loaders, flushing into one table at once, all commit 15;16;17;18;19 never - column_extent column_extent premise: the extent function exists 15;16;17;18;19 never - column_extent column_extent premise: the transaction's plans gave two estimates 15;16;17;18;19 never - column_extent column_extent premise: the typed fixture spans several row groups 15;16;17;18;19 never - @@ -239,7 +239,7 @@ column_extent column_extent the extent of ty.n is the data's min and max, in the column_extent column_extent the extent of ty.s is the data's min and max, in the type's order 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one column_extent column_extent the extent of ty.ts is the data's min and max, in the type's order 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one column_extent column_extent while the column that was there from the start still has one 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one -column_extent column_extent without the extent rows the planner falls back to one zone_map scan 15;16;17;18;19 never - +column_extent column_extent without the extent rows the planner falls back to one zone_map scan 15;16;17;18;19 2026-10-13 the planner's zone-map fallback removed: with no complete extent it reads nothing and bounds nothing differential differential agg avg 15;16;17;18;19 never - differential differential agg count 15;16;17;18;19 never - differential differential agg minmax 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 58b158409..93b191383 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1525,4 +1525,19 @@ suites_not_covered 247 # never 1745 + 1 = 1746 # dated 444 + 2 = 446 # 1746 + 446 == 2192 -checks_never_observed_red 1746 +# +# #1325 review (jdatcmd): 1746 -> 1745. Of column_extent's seven never rows, +# four were not premises. Three hold by construction -- compact_rewrite and +# expire still retiring with the extent in place, and parallel_copy's loaders +# all committing -- and are renamed premise: in both twins; their old rows are +# removed and the renamed ones arrive never from a five-major run of the +# renamed suite (36/36 on each of 15-19). The fourth, the planner falling back +# to one zone_map scan without the extent rows, is dated by a mutation that +# removes the fallback; the estimate check beside it went red under the same +# mutation and was already dated. +# +# ledger rows 2192 - 3 + 3 = 2192 +# never 1746 - 1 = 1745 +# dated 446 + 1 = 447 +# 1745 + 447 == 2192 +checks_never_observed_red 1745