diff --git a/CHANGELOG.md b/CHANGELOG.md index bb6f21865..657a731d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,21 @@ true until the next version shipped. ### Added +- The planner's estimate of how many time buckets a `GROUP BY date_trunc(...)` + or `date_bin(...)` produces now reads a per-column extent + (`pgcolumnar.column_extent`) in one index lookup, instead of every row group's + zone map (#1304, for #1292). Planning cost no longer grows with the table. + - **Writes:** a write adds its own extent row when it commits. + - **Recomputed:** every operation that drops or rewrites row groups (`expire`, + `compact_rewrite`, `VACUUM`, `recluster`) recomputes the extent before + committing. So it is never narrower than the data, and wider only by rows + deleted but still stored. + - **Fallback:** where the zone maps cannot give a bound, the planner reads them + as before: a column added after older groups, or a type with no ordering. + - **Inspection:** `pgcolumnar.column_extent(rel, col)` shows the bound. + - **Tests:** `test/column_extent.sh` and an independent + `test_column_extent.py`. + - Approximate distinct counts and percentiles over columnar tables are tested with the `hll` and `tdigest` extensions (#1296 phase 2), and a how-to section states the tested versions: hll 2.19 on PostgreSQL 15 to 18 (it does not build diff --git a/Makefile b/Makefile index 98c9a58ad..f2c531ff8 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,7 @@ OBJS = \ src/columnar_tableam.o \ src/columnar_storage.o \ src/columnar_metadata.o \ + src/columnar_extent.o \ src/columnar_write_state.o \ src/columnar_compression.o \ src/columnar_encoding.o \ diff --git a/docs/sql-reference.md b/docs/sql-reference.md index acba89adc..e8e86221c 100644 --- a/docs/sql-reference.md +++ b/docs/sql-reference.md @@ -450,6 +450,37 @@ which counts as appended. The record is internal storage metadata and `pg_dump` does not carry it, so a restored table reports no sorted groups until you sort it again. +### pgcolumnar.column_extent(rel regclass, col name) + +Shows the bound the planner uses for a column's smallest and largest value, when +it estimates how many time buckets a `GROUP BY date_trunc(...)` or `date_bin(...)` +produces. Returns one row, or none: + +| Column | Type | Meaning | +| --- | --- | --- | +| `complete` | bool | Always true when a row is returned. | +| `minimum` | text | The smallest value, printed by the column's type. NULL if the column holds no non-NULL value. | +| `maximum` | text | The largest value, printed the same way. | + +The bound is kept in `pgcolumnar.column_extent`, and the planner reads it in one +index lookup however many row groups the table has. Writes widen it when they +commit. Every operation that drops or rewrites row groups recomputes it before +committing: `expire`, `compact_rewrite`, `VACUUM`, `recluster`. So it can be +wider than the data only by rows that are deleted but still stored. + +No row means the planner reads every row group's zone map instead, which gives +the same answer at a cost that grows with the table. That happens for: + +- a column added after some row groups were written; +- a type with no ordering; +- a table written by an earlier version, until its next recompute. + +The values are column data, so the caller needs `SELECT` on the table. + +```sql +SELECT minimum, maximum FROM pgcolumnar.column_extent('events', 'ts'); +``` + ### pgcolumnar.maintenance_due(rel regclass, compact_due_fraction float8 DEFAULT 0.2, recluster_due_fraction float8 DEFAULT 0.05) Reports whether an online maintenance verb is worth running on a table, from its diff --git a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql index f805c3186..54fc7a518 100644 --- a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql @@ -983,6 +983,36 @@ CREATE TABLE pgcolumnar.retired_group ( CREATE UNIQUE INDEX retired_group_pkey ON pgcolumnar.retired_group USING btree (storage_id, group_number); +/* --------------------------------------------------------------------------- + * pgcolumnar.column_extent (#1304 option 5) + * + * A bound on every value of a column, for the planner's bucket estimate: one + * index lookup instead of every row group's zone map. Rows are inserted and + * deleted, never updated: each committing writer adds one row (complete = + * false) for what it flushed, and a recompute over every visible zone map + * replaces the rows it saw with one complete row. The planner trusts the rows + * only when a complete one is visible; then their fold is a true bound. + * minimum and maximum are encoded as the zone maps encode them; NULL when the + * row saw no value. No GRANT: the values are column data, like zone_map's. + * ------------------------------------------------------------------------- */ +CREATE TABLE pgcolumnar.column_extent ( + storage_id bigint NOT NULL, + column_index smallint NOT NULL, + complete bool NOT NULL, + minimum bytea, + maximum bytea +); +CREATE INDEX column_extent_pkey + ON pgcolumnar.column_extent USING btree (storage_id, column_index); + +CREATE FUNCTION pgcolumnar.column_extent(rel regclass, col name) + RETURNS TABLE (complete bool, minimum text, maximum text) + LANGUAGE C STRICT STABLE + AS 'MODULE_PATHNAME', 'pgcolumnar_column_extent'; + +COMMENT ON FUNCTION pgcolumnar.column_extent(regclass, name) + IS 'the bound on a column the planner reads from pgcolumnar.column_extent, printed by the column type; no row when it would read the zone maps instead (#1304); requires SELECT on the table'; + /* --------------------------------------------------------------------------- * Incremental rollups, phase 1: append-only (#1294). diff --git a/pgcolumnar--1.0-alpha5.sql b/pgcolumnar--1.0-alpha5.sql index 05e156e74..d742a4cdb 100644 --- a/pgcolumnar--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha5.sql @@ -341,6 +341,36 @@ CREATE TABLE pgcolumnar.retired_group ( CREATE UNIQUE INDEX retired_group_pkey ON pgcolumnar.retired_group USING btree (storage_id, group_number); +/* --------------------------------------------------------------------------- + * pgcolumnar.column_extent (#1304 option 5) + * + * A bound on every value of a column, for the planner's bucket estimate: one + * index lookup instead of every row group's zone map. Rows are inserted and + * deleted, never updated: each committing writer adds one row (complete = + * false) for what it flushed, and a recompute over every visible zone map + * replaces the rows it saw with one complete row. The planner trusts the rows + * only when a complete one is visible; then their fold is a true bound. + * minimum and maximum are encoded as the zone maps encode them; NULL when the + * row saw no value. No GRANT: the values are column data, like zone_map's. + * ------------------------------------------------------------------------- */ +CREATE TABLE pgcolumnar.column_extent ( + storage_id bigint NOT NULL, + column_index smallint NOT NULL, + complete bool NOT NULL, + minimum bytea, + maximum bytea +); +CREATE INDEX column_extent_pkey + ON pgcolumnar.column_extent USING btree (storage_id, column_index); + +CREATE FUNCTION pgcolumnar.column_extent(rel regclass, col name) + RETURNS TABLE (complete bool, minimum text, maximum text) + LANGUAGE C STRICT STABLE + AS 'MODULE_PATHNAME', 'pgcolumnar_column_extent'; + +COMMENT ON FUNCTION pgcolumnar.column_extent(regclass, name) + IS 'the bound on a column the planner reads from pgcolumnar.column_extent, printed by the column type; no row when it would read the zone maps instead (#1304); requires SELECT on the table'; + /* --------------------------------------------------------------------------- * Access method (spec 8.1) * ------------------------------------------------------------------------- */ diff --git a/src/columnar.h b/src/columnar.h index 562d4e1df..7ea050e4d 100644 --- a/src/columnar.h +++ b/src/columnar.h @@ -612,6 +612,34 @@ extern List *PgColumnarReadWholeChunkZoneMapsForColumn(uint64 storageId, int col extern void PgColumnarDeleteMetadata(uint64 storageId); extern void PgColumnarRecordRetention(uint64 storageId, uint64 groupNumber); +/* pgcolumnar.column_extent (#1304 option 5): columnar_metadata.c */ +typedef struct NativeExtentRow +{ + ItemPointerData tid; + bool complete; + bool hasValues; /* false: the row's writer saw only NULLs */ + char *minimum; + int minimumLen; + char *maximum; + int maximumLen; +} NativeExtentRow; +extern bool PgColumnarExtentCatalogPresent(void); +extern void PgColumnarInsertExtentRow(uint64 storageId, int columnIndex, bool complete, + const char *minimum, int minimumLen, + const char *maximum, int maximumLen); +extern List *PgColumnarReadExtentRows(uint64 storageId, int columnIndex, Snapshot snapshot); +extern void PgColumnarDeleteExtentRow(ItemPointer tid); +extern Oid PgColumnarStorageRelationOid(uint64 storageId, Snapshot snapshot); + +/* the extent's lifecycle: columnar_extent.c */ +extern void PgColumnarExtentNote(Oid relid, uint64 storageId, int columnIndex, + const NativeZoneMapMetadata *z); +extern void PgColumnarExtentMarkRecompute(uint64 storageId); +extern void PgColumnarExtentPreCommit(void); +extern void PgColumnarExtentAtXactEnd(void); +extern bool PgColumnarExtentBound(Relation rel, int columnIndex, Snapshot snapshot, + Datum *minimum, Datum *maximum, bool *hasValues); + /* declared retention, read by pgcolumnar.expire (#403 item 5a) */ extern bool PgColumnarReadTtl(Oid relid, char **column, Interval **interval); @@ -1007,6 +1035,7 @@ extern char *PgColumnarDecompressValueStream(const char *comp, uint32 compLen, #define PGCOLUMNAR_LOCKCLASS_STORAGE_ROW 102 #define PGCOLUMNAR_LOCKCLASS_UNIQUE_KEY 103 #define PGCOLUMNAR_LOCKCLASS_ROW_IDENTITY 104 +#define PGCOLUMNAR_LOCKCLASS_EXTENT 105 /* ------------------------------------------------------------------------- * concurrent unique-key insert serialization (pgcolumnar_unique.c, issue #5) diff --git a/src/columnar_extent.c b/src/columnar_extent.c new file mode 100644 index 000000000..1a43395e3 --- /dev/null +++ b/src/columnar_extent.c @@ -0,0 +1,562 @@ +/*------------------------------------------------------------------------- + * + * columnar_extent.c + * The per-column extent (#1304 option 5): a bound on every value of a + * column, read in one index lookup instead of every zone map. + * + * THE CATALOG, pgcolumnar.column_extent, holds rows of (storage, column, + * complete, minimum, maximum). Rows are inserted and deleted, never updated: + * + * - Each transaction folds the whole-chunk zone maps it flushes, in memory, + * with the column type's comparison and collation, and at pre-commit + * inserts ONE row per (storage, column), complete = false. Its groups and + * its row commit or abort together, so any snapshot sees both or neither, + * and two writers never touch the same tuple. + * - A COMPLETE row comes only from a recompute: the fold of every whole-chunk + * zone map visible to one snapshot, which then replaces the rows visible to + * that same snapshot. It refuses (and changes nothing) when a group lacks a + * zone map for the column or has values without a minimum and maximum. + * - Every path that retires a group marks its storage (PgColumnarRetireGroup) + * and the storage is recomputed before commit, so a shrink is computed, + * never guessed. + * + * THE READER trusts the rows only when a complete one is visible, and then the + * fold of every visible row is a true bound. A reader's snapshot S that sees a + * complete row R (computed at snapshot B, committed before S): the groups S sees + * were either visible at B, and so are in R, or committed between B and S, in + * which case their writer's own row committed with them and S sees it. A retire + * and its recompute are one transaction, so S sees both or neither. The + * reader's own transaction adds its pending fold, since its groups are visible + * to it before its row is written. Otherwise the caller falls back to the zone + * maps. + * + * A ROLLED-BACK SUBTRANSACTION that flushed a group leaves its values in the + * fold: wider than the data, which only over-estimates. + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "columnar.h" +#include "access/relation.h" +#include "access/table.h" +#include "access/xact.h" +#include "funcapi.h" +#include "lib/stringinfo.h" +#include "miscadmin.h" +#include "storage/lmgr.h" +#include "storage/lock.h" +#include "utils/acl.h" +#include "utils/builtins.h" +#include "utils/datum.h" +#include "utils/hsearch.h" +#include "utils/lsyscache.h" +#include "utils/memutils.h" +#include "utils/rel.h" +#include "utils/snapmgr.h" +#include "utils/tuplestore.h" +#include "utils/typcache.h" + +PG_FUNCTION_INFO_V1(pgcolumnar_column_extent); + +/* + * More rows than this for one column, with a complete one among them, and the + * committing writer folds them into one. The fold reads only these rows, never + * the zone maps, so it costs a few tuples however large the table. + */ +#define EXTENT_MERGE_ABOVE 8 + +typedef struct ExtentKey +{ + uint64 storageId; + int32 columnIndex; +} ExtentKey; + +typedef struct ExtentEntry +{ + ExtentKey key; + Oid relid; + FormData_pg_attribute att; + Datum minimum; + Datum maximum; +} ExtentEntry; + +/* both live in TopTransactionContext, and are forgotten at transaction end */ +static HTAB *extent_pending = NULL; +static HTAB *extent_recompute = NULL; /* uint64 storage ids */ + +static int +extent_cmp(Form_pg_attribute att, Datum a, Datum b) +{ + TypeCacheEntry *tce = lookup_type_cache(att->atttypid, TYPECACHE_CMP_PROC_FINFO); + + if (!OidIsValid(tce->cmp_proc)) + elog(ERROR, "column extent: type %u has no btree comparison", att->atttypid); + return DatumGetInt32(FunctionCall2Coll(&tce->cmp_proc_finfo, att->attcollation, a, b)); +} + +static Datum +extent_decode(Form_pg_attribute att, const char *bytes, int len) +{ + char *cur = (char *) bytes; + + return PgColumnarDecodeValue(att, &cur, (char *) bytes + len, CurrentMemoryContext); +} + +static HTAB * +extent_hash(const char *name, Size keysize, Size entrysize) +{ + HASHCTL ctl; + + memset(&ctl, 0, sizeof(ctl)); + ctl.keysize = keysize; + ctl.entrysize = entrysize; + ctl.hcxt = TopTransactionContext; + return hash_create(name, 16, &ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); +} + +/* + * PgColumnarExtentNote + * Fold one whole-chunk zone map, just flushed for the relation's base + * storage, into this transaction's extent for the column. + */ +void +PgColumnarExtentNote(Oid relid, uint64 storageId, int columnIndex, + const NativeZoneMapMetadata *z) +{ + ExtentKey key; + ExtentEntry *e; + bool found; + Relation rel; + Form_pg_attribute att; + MemoryContext old; + Datum mn, + mx; + + if (!z->hasMinMax) + return; /* all NULL, or a type with no ordering */ + if (extent_pending == NULL) + extent_pending = extent_hash("pgcolumnar column extents", sizeof(ExtentKey), + sizeof(ExtentEntry)); + memset(&key, 0, sizeof(key)); + key.storageId = storageId; + key.columnIndex = columnIndex; + e = (ExtentEntry *) hash_search(extent_pending, &key, HASH_ENTER, &found); + if (!found) + { + rel = relation_open(relid, NoLock); + att = TupleDescAttr(RelationGetDescr(rel), columnIndex); + memcpy(&e->att, att, sizeof(FormData_pg_attribute)); + relation_close(rel, NoLock); + e->relid = relid; + } + att = &e->att; + mn = extent_decode(att, z->minimum, z->minimumLen); + mx = extent_decode(att, z->maximum, z->maximumLen); + old = MemoryContextSwitchTo(TopTransactionContext); + if (!found || extent_cmp(att, mn, e->minimum) < 0) + e->minimum = datumCopy(mn, att->attbyval, att->attlen); + if (!found || extent_cmp(att, mx, e->maximum) > 0) + e->maximum = datumCopy(mx, att->attbyval, att->attlen); + MemoryContextSwitchTo(old); +} + +void +PgColumnarExtentMarkRecompute(uint64 storageId) +{ + if (extent_recompute == NULL) + extent_recompute = extent_hash("pgcolumnar extent recomputes", sizeof(uint64), + sizeof(uint64)); + (void) hash_search(extent_recompute, &storageId, HASH_ENTER, NULL); +} + +void +PgColumnarExtentAtXactEnd(void) +{ + /* the tables fall with TopTransactionContext */ + extent_pending = NULL; + extent_recompute = NULL; +} + +static void +extent_encode(Form_pg_attribute att, Datum v, StringInfo buf) +{ + resetStringInfo(buf); + PgColumnarEncodeValue(buf, att, v); +} + +/* + * One recompute or merge per storage at a time, held to transaction end: the + * rows a recompute deletes are the rows its snapshot sees, and a second one + * deleting the same rows would fail. Returns false only when !wait and taken. + */ +static bool +extent_lock(uint64 storageId, bool wait) +{ + LOCKTAG tag; + + SET_LOCKTAG_ADVISORY(tag, MyDatabaseId, + (uint32) (storageId >> 32), (uint32) (storageId & 0xFFFFFFFF), + PGCOLUMNAR_LOCKCLASS_EXTENT); + return LockAcquire(&tag, ExclusiveLock, false, !wait) != LOCKACQUIRE_NOT_AVAIL; +} + +/* replace every row the snapshot sees with one complete row */ +static void +extent_replace(uint64 storageId, int col, Form_pg_attribute att, Snapshot snap, + bool any, Datum mn, Datum mx) +{ + List *rows = PgColumnarReadExtentRows(storageId, col, snap); + ListCell *lc; + StringInfoData a, + b; + + foreach(lc, rows) + PgColumnarDeleteExtentRow(&((NativeExtentRow *) lfirst(lc))->tid); + if (!any) + { + PgColumnarInsertExtentRow(storageId, col, true, NULL, 0, NULL, 0); + return; + } + initStringInfo(&a); + initStringInfo(&b); + extent_encode(att, mn, &a); + extent_encode(att, mx, &b); + PgColumnarInsertExtentRow(storageId, col, true, a.data, a.len, b.data, b.len); +} + +/* + * The full recompute, from every whole-chunk zone map the snapshot sees. Gives + * up, writing nothing, when the zone maps cannot bound the column. + */ +static void +extent_recompute_column(uint64 storageId, int col, Form_pg_attribute att, Snapshot snap) +{ + List *groups = PgColumnarReadRowGroupList(storageId, snap); + List *zones = PgColumnarReadWholeChunkZoneMapsForColumn(storageId, col, snap); + ListCell *lc; + bool any = false; + Datum mn = (Datum) 0, + mx = (Datum) 0; + + if (list_length(zones) != list_length(groups)) + return; /* a group with no zone map: a column added later */ + foreach(lc, zones) + { + NativeZoneMapMetadata *z = (NativeZoneMapMetadata *) lfirst(lc); + Datum zmn, + zmx; + + if (!z->hasMinMax) + { + if (z->valueCount == 0) + continue; /* all NULL */ + return; /* values without an ordering */ + } + zmn = extent_decode(att, z->minimum, z->minimumLen); + zmx = extent_decode(att, z->maximum, z->maximumLen); + if (!any || extent_cmp(att, zmn, mn) < 0) + mn = zmn; + if (!any || extent_cmp(att, zmx, mx) > 0) + mx = zmx; + any = true; + } + extent_replace(storageId, col, att, snap, any, mn, mx); +} + +/* + * The merge: fold the rows the snapshot sees into one complete row. Valid only + * when a complete row is among them -- then their fold is a bound on every + * group the snapshot sees, by the reader's argument above. + */ +static void +extent_merge_column(uint64 storageId, int col, Form_pg_attribute att, Snapshot snap) +{ + List *rows = PgColumnarReadExtentRows(storageId, col, snap); + ListCell *lc; + bool complete = false; + bool any = false; + Datum mn = (Datum) 0, + mx = (Datum) 0; + + foreach(lc, rows) + { + NativeExtentRow *r = (NativeExtentRow *) lfirst(lc); + Datum rmn, + rmx; + + complete |= r->complete; + if (!r->hasValues) + continue; + rmn = extent_decode(att, r->minimum, r->minimumLen); + rmx = extent_decode(att, r->maximum, r->maximumLen); + if (!any || extent_cmp(att, rmn, mn) < 0) + mn = rmn; + if (!any || extent_cmp(att, rmx, mx) > 0) + mx = rmx; + any = true; + } + if (complete) + extent_replace(storageId, col, att, snap, any, mn, mx); + else + extent_recompute_column(storageId, col, att, snap); +} + +/* a fresh snapshot, taken after the lock, that sees this transaction's own rows */ +static Snapshot +extent_snapshot(void) +{ + CommandCounterIncrement(); + return RegisterSnapshot(GetLatestSnapshot()); +} + +static void +extent_recompute_storage(uint64 storageId) +{ + Snapshot snap = extent_snapshot(); + Oid relid = PgColumnarStorageRelationOid(storageId, snap); + Relation rel; + TupleDesc td; + int i; + + UnregisterSnapshot(snap); + if (!OidIsValid(relid)) + return; + rel = try_relation_open(relid, AccessShareLock); + if (rel == NULL) + return; + /* a projection's storage names its base relation: no extents for those */ + if (PgColumnarStorageId(rel) != storageId) + { + relation_close(rel, AccessShareLock); + return; + } + (void) extent_lock(storageId, true); + snap = extent_snapshot(); + td = RelationGetDescr(rel); + for (i = 0; i < td->natts; i++) + if (!TupleDescAttr(td, i)->attisdropped) + extent_recompute_column(storageId, i, TupleDescAttr(td, i), snap); + UnregisterSnapshot(snap); + relation_close(rel, AccessShareLock); +} + +/* + * PgColumnarExtentPreCommit + * Called at pre-commit and pre-prepare, after the pending writes are + * flushed: write this transaction's rows, then recompute what it marked. + */ +void +PgColumnarExtentPreCommit(void) +{ + HASH_SEQ_STATUS st; + ExtentEntry *e; + uint64 *sid; + List *written = NIL; + ListCell *lc; + StringInfoData a, + b; + bool pushed = false; + + if (extent_pending == NULL && extent_recompute == NULL) + return; + if (!PgColumnarExtentCatalogPresent()) + { + PgColumnarExtentAtXactEnd(); + return; + } + /* pre-commit runs with no active snapshot, and catalog writes need one */ + if (!ActiveSnapshotSet()) + { + PushActiveSnapshot(GetTransactionSnapshot()); + pushed = true; + } + initStringInfo(&a); + initStringInfo(&b); + if (extent_pending != NULL) + { + Snapshot seen = extent_snapshot(); + + hash_seq_init(&st, extent_pending); + while ((e = (ExtentEntry *) hash_seq_search(&st)) != NULL) + { + /* a recompute of the storage takes this transaction's groups anyway */ + if (extent_recompute != NULL && + hash_search(extent_recompute, &e->key.storageId, HASH_FIND, NULL) != NULL) + continue; + /* created in a savepoint that rolled back: nothing to bound */ + if (!OidIsValid(PgColumnarStorageRelationOid(e->key.storageId, seen))) + continue; + extent_encode(&e->att, e->minimum, &a); + extent_encode(&e->att, e->maximum, &b); + PgColumnarInsertExtentRow(e->key.storageId, e->key.columnIndex, false, + a.data, a.len, b.data, b.len); + written = lappend(written, e); + } + UnregisterSnapshot(seen); + } + if (extent_recompute != NULL) + { + hash_seq_init(&st, extent_recompute); + while ((sid = (uint64 *) hash_seq_search(&st)) != NULL) + extent_recompute_storage(*sid); + } + + /* + * Keep the rows few. A writer that finds no complete row recomputes one; one + * that finds too many merges them. Conditional: a writer never waits for + * another's recompute, it leaves the work to the next. + */ + foreach(lc, written) + { + List *rows; + ListCell *rc; + bool complete = false; + Snapshot snap; + + e = (ExtentEntry *) lfirst(lc); + snap = extent_snapshot(); + rows = PgColumnarReadExtentRows(e->key.storageId, e->key.columnIndex, snap); + UnregisterSnapshot(snap); + foreach(rc, rows) + complete |= ((NativeExtentRow *) lfirst(rc))->complete; + if (complete && list_length(rows) <= EXTENT_MERGE_ABOVE) + continue; + if (!extent_lock(e->key.storageId, false)) + continue; + snap = extent_snapshot(); + if (complete) + extent_merge_column(e->key.storageId, e->key.columnIndex, &e->att, snap); + else + extent_recompute_column(e->key.storageId, e->key.columnIndex, &e->att, snap); + UnregisterSnapshot(snap); + } + if (pushed) + PopActiveSnapshot(); + PgColumnarExtentAtXactEnd(); +} + +/* + * PgColumnarExtentBound + * The column's bound as the snapshot sees it, when a complete row is + * visible. false: no trustworthy extent, fall back to the zone maps. + * *hasValues false: the column holds no non-NULL value. + * + * THE CALLER'S OWN TRANSACTION sees the groups it has flushed, but its + * extent row is written only at pre-commit, so the visible rows alone would + * leave those groups out and bound the column too narrowly. Its pending fold + * is added in. Groups it has retired are still in the visible rows until + * its recompute commits: wider than the data, which only over-estimates. + */ +bool +PgColumnarExtentBound(Relation rel, int columnIndex, Snapshot snapshot, + Datum *minimum, Datum *maximum, bool *hasValues) +{ + Form_pg_attribute att; + List *rows; + ListCell *lc; + bool complete = false; + uint64 storageId; + + if (columnIndex < 0 || columnIndex >= RelationGetDescr(rel)->natts || + !PgColumnarExtentCatalogPresent()) + return false; + att = TupleDescAttr(RelationGetDescr(rel), columnIndex); + storageId = PgColumnarStorageId(rel); + rows = PgColumnarReadExtentRows(storageId, columnIndex, snapshot); + *hasValues = false; + foreach(lc, rows) + { + NativeExtentRow *r = (NativeExtentRow *) lfirst(lc); + Datum rmn, + rmx; + + complete |= r->complete; + if (!r->hasValues) + continue; + rmn = extent_decode(att, r->minimum, r->minimumLen); + rmx = extent_decode(att, r->maximum, r->maximumLen); + if (!*hasValues || extent_cmp(att, rmn, *minimum) < 0) + *minimum = rmn; + if (!*hasValues || extent_cmp(att, rmx, *maximum) > 0) + *maximum = rmx; + *hasValues = true; + } + if (complete && extent_pending != NULL) + { + ExtentKey key; + ExtentEntry *e; + + memset(&key, 0, sizeof(key)); + key.storageId = storageId; + key.columnIndex = columnIndex; + e = (ExtentEntry *) hash_search(extent_pending, &key, HASH_FIND, NULL); + if (e != NULL) + { + if (!*hasValues || extent_cmp(att, e->minimum, *minimum) < 0) + *minimum = e->minimum; + if (!*hasValues || extent_cmp(att, e->maximum, *maximum) > 0) + *maximum = e->maximum; + *hasValues = true; + } + } + return complete; +} + +/* + * pgcolumnar.column_extent(rel regclass, col name) + * -> (complete bool, minimum text, maximum text) + * The bound the planner would use, printed by the column type's output + * function; no row when it would fall back to the zone maps. The values are + * column data, so the caller must be able to SELECT the table. + */ +Datum +pgcolumnar_column_extent(PG_FUNCTION_ARGS) +{ + ReturnSetInfo *rsinfo = (ReturnSetInfo *) fcinfo->resultinfo; + Oid relid = PG_GETARG_OID(0); + char *col = NameStr(*PG_GETARG_NAME(1)); + AclResult ac; + Relation rel; + AttrNumber attnum; + Datum mn = (Datum) 0, + mx = (Datum) 0; + bool hasValues; + + InitMaterializedSRF(fcinfo, 0); + ac = pg_class_aclcheck(relid, GetUserId(), ACL_SELECT); + if (ac != ACLCHECK_OK) + aclcheck_error(ac, OBJECT_TABLE, get_rel_name(relid)); + rel = relation_open(relid, AccessShareLock); + if (!PgColumnarIsColumnarRelation(relid)) + ereport(ERROR, + (errcode(ERRCODE_WRONG_OBJECT_TYPE), + errmsg("\"%s\" is not a pgcolumnar table", RelationGetRelationName(rel)))); + attnum = get_attnum(relid, col); + if (attnum <= 0) + ereport(ERROR, + (errcode(ERRCODE_UNDEFINED_COLUMN), + errmsg("column \"%s\" of relation \"%s\" does not exist", + col, RelationGetRelationName(rel)))); + if (PgColumnarExtentBound(rel, attnum - 1, + PgColumnarCatalogSnapshot(GetActiveSnapshot()), + &mn, &mx, &hasValues)) + { + Form_pg_attribute att = TupleDescAttr(RelationGetDescr(rel), attnum - 1); + Datum values[3]; + bool nulls[3] = {false, !hasValues, !hasValues}; + Oid out; + bool isvarlena; + + values[0] = BoolGetDatum(true); + if (hasValues) + { + getTypeOutputInfo(att->atttypid, &out, &isvarlena); + values[1] = CStringGetTextDatum(OidOutputFunctionCall(out, mn)); + values[2] = CStringGetTextDatum(OidOutputFunctionCall(out, mx)); + } + tuplestore_putvalues(rsinfo->setResult, rsinfo->setDesc, values, nulls); + } + relation_close(rel, AccessShareLock); + return (Datum) 0; +} diff --git a/src/columnar_metadata.c b/src/columnar_metadata.c index 1f356f4f8..c5277c037 100644 --- a/src/columnar_metadata.c +++ b/src/columnar_metadata.c @@ -140,6 +140,14 @@ #define Anum_free_space_freed_xid 4 #define Natts_free_space 4 +/* attribute numbers for pgcolumnar.column_extent (#1304 option 5) */ +#define Anum_column_extent_storage_id 1 +#define Anum_column_extent_column_index 2 +#define Anum_column_extent_complete 3 +#define Anum_column_extent_minimum 4 +#define Anum_column_extent_maximum 5 +#define Natts_column_extent 5 + /* attribute numbers for pgcolumnar.retired_group (#1294) */ #define Anum_retired_group_storage_id 1 #define Anum_retired_group_group_number 2 @@ -1044,6 +1052,14 @@ PgColumnarRetireGroup(uint64 storageId, uint64 groupNumber) */ PgColumnarGroupMemoReset(true); + /* + * A group leaving the storage may take the column's minimum or maximum with + * it, so its extent is recomputed before this transaction commits (#1304 + * option 5): a shrink is recomputed, never guessed. Every path that retires a + * group comes through here. + */ + PgColumnarExtentMarkRecompute(storageId); + if (haveRange && byteLength > 0) record_free_space(storageId, fileOffset, byteLength); } @@ -2004,6 +2020,12 @@ PgColumnarDeleteMetadata(uint64 storageId) delete_rows_by_storage_id("bloom", Anum_bloom_storage_id, storageId); delete_rows_by_storage_id("free_space", Anum_free_space_storage_id, storageId); delete_rows_by_storage_id("retired_group", Anum_retired_group_storage_id, storageId); + /* + * #1304 option 5. Present only once the extension's SQL is at 1.0-alpha5; a + * library ahead of its SQL must still be able to drop and truncate. + */ + if (PgColumnarExtentCatalogPresent()) + delete_rows_by_storage_id("column_extent", Anum_column_extent_storage_id, storageId); delete_rows_by_storage_id("row_group", Anum_row_group_storage_id, storageId); delete_rows_by_storage_id("storage", Anum_native_storage_storage_id, storageId); @@ -4626,3 +4648,150 @@ PgColumnarDeleteProjectionRow(uint64 storageId, int projectionId) table_close(rel, RowExclusiveLock); CommandCounterIncrement(); } + +/* ------------------------------------------------------------------------- + * pgcolumnar.column_extent (#1304 option 5) + * + * Rows are only inserted and deleted, never updated in place: each committing + * writer inserts its own row, and a recompute replaces the rows it folded with + * one complete row. So two backends never touch the same tuple. + * ------------------------------------------------------------------------- */ + +bool +PgColumnarExtentCatalogPresent(void) +{ + return OidIsValid(get_relname_relid("column_extent", pgcolumnar_schema_oid())); +} + +static bytea * +extent_bytea(const char *bytes, int len) +{ + bytea *b = (bytea *) palloc(VARHDRSZ + len); + + SET_VARSIZE(b, VARHDRSZ + len); + if (len > 0) + memcpy(VARDATA(b), bytes, len); + return b; +} + +void +PgColumnarInsertExtentRow(uint64 storageId, int columnIndex, bool complete, + const char *minimum, int minimumLen, + const char *maximum, int maximumLen) +{ + Relation rel = open_columnar_table("column_extent", RowExclusiveLock); + Datum values[Natts_column_extent]; + bool nulls[Natts_column_extent] = {false}; + HeapTuple tuple; + + values[Anum_column_extent_storage_id - 1] = Int64GetDatum((int64) storageId); + values[Anum_column_extent_column_index - 1] = Int16GetDatum((int16) columnIndex); + values[Anum_column_extent_complete - 1] = BoolGetDatum(complete); + if (minimum == NULL) + { + /* a column with no values yet: no bound, and nothing for a fold to take */ + nulls[Anum_column_extent_minimum - 1] = true; + nulls[Anum_column_extent_maximum - 1] = true; + } + else + { + values[Anum_column_extent_minimum - 1] = PointerGetDatum(extent_bytea(minimum, minimumLen)); + values[Anum_column_extent_maximum - 1] = PointerGetDatum(extent_bytea(maximum, maximumLen)); + } + tuple = heap_form_tuple(RelationGetDescr(rel), values, nulls); + CatalogTupleInsert(rel, tuple); + heap_freetuple(tuple); + table_close(rel, RowExclusiveLock); +} + +/* + * The extent rows of one column visible to the snapshot, as NativeExtentRow *. + * A NULL snapshot is the catalog snapshot. + */ +List * +PgColumnarReadExtentRows(uint64 storageId, int columnIndex, Snapshot snapshot) +{ + Relation rel = open_columnar_table("column_extent", AccessShareLock); + TupleDesc td = RelationGetDescr(rel); + ScanKeyData key[2]; + SysScanDesc scan; + HeapTuple tuple; + Oid idx; + List *rows = NIL; + + ScanKeyInit(&key[0], Anum_column_extent_storage_id, BTEqualStrategyNumber, + F_INT8EQ, Int64GetDatum((int64) storageId)); + ScanKeyInit(&key[1], Anum_column_extent_column_index, BTEqualStrategyNumber, + F_INT2EQ, Int16GetDatum((int16) columnIndex)); + idx = pgcolumnar_scan_index_oid(rel, "column_extent_pkey"); + scan = systable_beginscan(rel, idx, OidIsValid(idx), snapshot, 2, key); + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + NativeExtentRow *r = (NativeExtentRow *) palloc0(sizeof(NativeExtentRow)); + bool isnull; + Datum d; + + r->tid = tuple->t_self; + r->complete = DatumGetBool(heap_getattr(tuple, Anum_column_extent_complete, td, &isnull)); + d = heap_getattr(tuple, Anum_column_extent_minimum, td, &isnull); + if (!isnull) + { + bytea *mn = DatumGetByteaPCopy(d); + bytea *mx = DatumGetByteaPCopy(heap_getattr(tuple, Anum_column_extent_maximum, td, &isnull)); + + r->hasValues = true; + r->minimum = VARDATA(mn); + r->minimumLen = VARSIZE(mn) - VARHDRSZ; + r->maximum = VARDATA(mx); + r->maximumLen = VARSIZE(mx) - VARHDRSZ; + } + rows = lappend(rows, r); + } + systable_endscan(scan); + table_close(rel, AccessShareLock); + return rows; +} + +void +PgColumnarDeleteExtentRow(ItemPointer tid) +{ + Relation rel = open_columnar_table("column_extent", RowExclusiveLock); + + CatalogTupleDelete(rel, tid); + table_close(rel, RowExclusiveLock); +} + +/* + * The relation a storage id belongs to, from pgcolumnar.storage, or InvalidOid. + * A projection's storage names its base relation too; callers that need the + * base storage compare PgColumnarStorageId of the opened relation. Pass a + * snapshot taken after the caller's own writes: the catalog snapshot can + * predate a storage row this transaction inserted. + */ +Oid +PgColumnarStorageRelationOid(uint64 storageId, Snapshot snapshot) +{ + Relation rel = open_columnar_table("storage", AccessShareLock); + ScanKeyData key[1]; + SysScanDesc scan; + HeapTuple tuple; + Oid idx; + Oid relid = InvalidOid; + + ScanKeyInit(&key[0], Anum_native_storage_storage_id, BTEqualStrategyNumber, + F_INT8EQ, Int64GetDatum((int64) storageId)); + idx = pgcolumnar_scan_index_oid(rel, "storage_pkey"); + scan = systable_beginscan(rel, idx, OidIsValid(idx), snapshot, 1, key); + if (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + bool isnull; + Datum d = heap_getattr(tuple, Anum_native_storage_relation_oid, + RelationGetDescr(rel), &isnull); + + if (!isnull) + relid = DatumGetObjectId(d); + } + systable_endscan(scan); + table_close(rel, AccessShareLock); + return relid; +} diff --git a/src/columnar_tableam.c b/src/columnar_tableam.c index b90e90cc2..647485943 100644 --- a/src/columnar_tableam.c +++ b/src/columnar_tableam.c @@ -2663,11 +2663,26 @@ pgcolumnar_xact_callback(XactEvent event, void *arg) { switch (event) { + case XACT_EVENT_PRE_PREPARE: + /* + * #1304 option 5: a prepared transaction's extent rows are written + * here, where catalog writes are still allowed, so its groups are + * flushed here first. The PREPARE arm below then finds nothing. + */ + PgColumnarFlushAllPendingWrites(); + PgColumnarFlushAllDeleteVectors(); + PgColumnarExtentPreCommit(); + break; case XACT_EVENT_PRE_COMMIT: case XACT_EVENT_PARALLEL_PRE_COMMIT: case XACT_EVENT_PREPARE: PgColumnarFlushAllPendingWrites(); PgColumnarFlushAllDeleteVectors(); + /* after the flush, which is what gives the extent its last groups */ + if (event == XACT_EVENT_PRE_COMMIT) + PgColumnarExtentPreCommit(); + else + PgColumnarExtentAtXactEnd(); /* * PREPARE frees TopTransactionContext and never reaches the @@ -2700,6 +2715,7 @@ pgcolumnar_xact_callback(XactEvent event, void *arg) PgColumnarDiscardAllPendingWrites(); PgColumnarDiscardAllDeleteVectors(); PgColumnarDiscardFetchCache(); + PgColumnarExtentAtXactEnd(); /* * Descriptors only: the memo contexts are TopTransactionContext * children, still live at callback time but taken by the diff --git a/src/columnar_vector.c b/src/columnar_vector.c index 8363dc338..c02e2accb 100644 --- a/src/columnar_vector.c +++ b/src/columnar_vector.c @@ -2078,6 +2078,33 @@ pgcolumnar_zonemap_time_range(PlannerInfo *root, RelOptInfo *input_rel, Var *tsv storageId = PgColumnarStorageId(r); snap = PgColumnarCatalogSnapshot(GetActiveSnapshot()); + /* + * THE EXTENT FIRST (#1304 option 5): one index lookup, constant in the table, + * when a complete extent is visible. A timestamp is byval, so the bound + * outlives the relation. Otherwise the zone maps below, which are correct and + * cost a scan of every group's. + */ + { + /* + * Set only when the column has a value, which the return below checks; + * initialised because gcc 13/14 with LTO cannot see that, and CI treats + * its maybe-uninitialized warning as a failure (#1325 review). + */ + Datum emin = (Datum) 0, + emax = (Datum) 0; + bool hasValues = false; + + if (PgColumnarExtentBound(r, col, snap, &emin, &emax, &hasValues)) + { + table_close(r, AccessShareLock); + if (!hasValues) + return false; /* no value at all: nothing to bound */ + *lo = (double) DatumGetTimestamp(emin); + *hi = (double) DatumGetTimestamp(emax); + return true; + } + } + /* * TWO CATALOG SCANS, NOT ONE PER GROUP (#1304 review): the group list, for the * count, and every group's zone map for this column. A group with no zone-map @@ -2301,8 +2328,8 @@ pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel, * UNDER-estimate, the direction a gate on this number must never take. */ { - double zlo, - zhi; + double zlo = 0, + zhi = 0; /* set whenever the call returns true */ if (pgcolumnar_zonemap_time_range(root, input_rel, tsvar, &zlo, &zhi)) { diff --git a/src/columnar_write_state.c b/src/columnar_write_state.c index 94845de9d..e08fcb050 100644 --- a/src/columnar_write_state.c +++ b/src/columnar_write_state.c @@ -3246,6 +3246,22 @@ pgcolumnar_flush_row_group(PgColumnarWriteState *writeState) } PG_END_TRY(); + /* + * The column extent (#1304 option 5) folds this group's whole-chunk zone + * maps; the transaction writes its row at pre-commit. The base table only: + * a projection's writer has its own tuple descriptor, and nothing reads a + * projection's extent. + */ + if (writeState->tupdescIsRel) + foreach(lc, zoneRows) + { + NativeZoneMapMetadata *z = (NativeZoneMapMetadata *) lfirst(lc); + + if (z->vectorIndex == -1) + PgColumnarExtentNote(writeState->relid, writeState->storageId, + z->columnIndex, z); + } + table_close(rel, RowExclusiveLock); MemoryContextSwitchTo(oldContext); diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index cf690552c..e5f4e4dca 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -204,6 +204,42 @@ catalog_scan_index catalog_scan_index premise: truncating a projected table reac catalog_scan_index catalog_scan_index premise: truncating a projected table took effect 15;16;17;18;19 never - catalog_scan_index catalog_scan_index renaming a projected column does not read pgcolumnar.projection_declaration whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole catalog_scan_index catalog_scan_index truncating a projected table does not read pgcolumnar.projection_declaration whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole +column_extent column_extent DROP TABLE removes them 15;16;17;18;19 2026-10-13 PgColumnarDeleteMetadata leaves the extent rows +column_extent column_extent TRUNCATE removes the old storage's extent rows 15;16;17;18;19 2026-10-13 PgColumnarDeleteMetadata leaves the extent rows +column_extent column_extent a column added after older groups has no complete extent (those groups have no zone map for it) 15;16;17;18;19 2026-10-13 a recompute that ignores a group with no zone map for the column;pgcolumnar.column_extent returns a row with no complete extent;the extent trusted without a complete row +column_extent column_extent a prepared transaction's groups are inside the extent after COMMIT PREPARED 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit;no extent rows written at PRE_PREPARE +column_extent column_extent a savepoint rolled back after flushing may widen the extent, never narrow it 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit;no extent rows written at PRE_PREPARE;the extent folded by comparing Datums as words, not with the type's comparison;the writer's fold keeps the largest minimum +column_extent column_extent a table created in a rolled-back savepoint leaves no extent rows 15;16;17;18;19 2026-10-13 pre-commit writes extent rows for a storage that does not exist +column_extent column_extent a transaction's own flushed groups are inside the extent it reads before commit 15;16;17;18;19 2026-10-13 the extent a transaction reads leaves out its own pending fold (the tree before the fix) +column_extent column_extent a type with no ordering gets no extent 15;16;17;18;19 2026-10-13 a recompute that skips values without an ordering;pgcolumnar.column_extent returns a row with no complete extent +column_extent column_extent a writer that finds no complete extent recomputes one over every group 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one +column_extent column_extent a writer that rolls back leaves the extent as the data has it 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit;no extent rows written at PRE_PREPARE;the writer's fold keeps the largest minimum +column_extent column_extent after VACUUM retires a fully deleted group the extent is the live rows' 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded +column_extent column_extent after compact_rewrite the extent is the live rows' 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded +column_extent column_extent after expire the extent is recomputed to the data that is left 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded +column_extent column_extent and a plan made in that transaction estimates as the zone maps do 15;16;17;18;19 2026-10-13 the extent a transaction reads leaves out its own pending fold (the tree before the fix) +column_extent column_extent and it did move: the retired groups held the old minimum 15;16;17;18;19 2026-10-13 PgColumnarRetireGroup without marking the storage for a recompute;a recompute that leaves the rows it folded +column_extent column_extent and the estimate is the one the zone maps give 15;16;17;18;19 2026-10-13 the planner takes the extent's minimum as its upper bound too;the planner's zone-map fallback removed: with no complete extent it reads nothing and bounds nothing +column_extent column_extent and the extent covers what they loaded 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit;no extent rows written at PRE_PREPARE;the writer's fold keeps the largest minimum +column_extent column_extent and the extent still covers all fifty 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;no extent row written at pre-commit +column_extent column_extent and with no extent rows the function reports no complete extent 15;16;17;18;19 2026-10-13 pgcolumnar.column_extent returns a row with no complete extent +column_extent column_extent fifty committing writers leave a bounded number of extent rows 15;16;17;18;19 2026-10-13 a recompute that leaves the rows it folded;a writer that finds no complete extent does not recompute one;no merge however many extent rows a column has +column_extent column_extent nor is a table's extent through the function, without SELECT on the table 15;16;17;18;19 2026-10-13 pgcolumnar.column_extent without its SELECT check +column_extent column_extent planning a bucketed GROUP BY scans zone_map not at all when the extent is complete 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;the planner's time range ignores the extent +column_extent column_extent premise: compact_rewrite rewrites the groups the deletes thinned 15;16;17;18;19 never - +column_extent column_extent premise: expire retires the old row groups with the extent in place 15;16;17;18;19 never - +column_extent column_extent premise: parallel_copy's loaders, flushing into one table at once, all commit 15;16;17;18;19 never - +column_extent column_extent premise: the extent function exists 15;16;17;18;19 never - +column_extent column_extent premise: the transaction's plans gave two estimates 15;16;17;18;19 never - +column_extent column_extent premise: the typed fixture spans several row groups 15;16;17;18;19 never - +column_extent column_extent the extent catalog is not readable by an ordinary role (it holds column values) 15;16;17;18;19 2026-10-13 the column_extent catalog granted to PUBLIC +column_extent column_extent the extent of ty.d is the data's min and max, in the type's order 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one +column_extent column_extent the extent of ty.i is the data's min and max, in the type's order 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one +column_extent column_extent the extent of ty.n is the data's min and max, in the type's order 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one;the extent folded by comparing Datums as words, not with the type's comparison +column_extent column_extent the extent of ty.s is the data's min and max, in the type's order 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one +column_extent column_extent the extent of ty.ts is the data's min and max, in the type's order 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one +column_extent column_extent while the column that was there from the start still has one 15;16;17;18;19 2026-10-13 a writer that finds no complete extent does not recompute one +column_extent column_extent without the extent rows the planner falls back to one zone_map scan 15;16;17;18;19 2026-10-13 the planner's zone-map fallback removed: with no complete extent it reads nothing and bounds nothing differential differential agg avg 15;16;17;18;19 never - differential differential agg count 15;16;17;18;19 never - differential differential agg minmax 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 9a35a789c..93b191383 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1493,4 +1493,51 @@ suites_not_covered 247 # never 1734 + 5 = 1739 # dated 411 + 6 = 417 # 1739 + 417 == 2156 -checks_never_observed_red 1739 +# +# #1304 option 5, the per-column extent (OffgridwithJD): 1739 -> 1745. +# column_extent enters the ledger with 33 checks from a five-major run +# (column_extent.sh 33/33 on each of 15-19). Twenty-seven are dated by eighteen +# mutations of the extent code, its SQL and the planner's read, each merged one +# log at a time with --mutation and --target; no premise is among them. A +# nineteenth mutation survived and is equivalent: the existence check reading +# the catalog snapshot instead of its own, after the CommandCounterIncrement +# that refreshes both. Six arrive `never`: two premises (the function exists, +# the fixture's row groups) and four arms held by construction -- expire and +# compact_rewrite still retire and rewrite with the extent in place, +# parallel_copy's loaders all commit (no row is ever updated in place, so there +# is no update to conflict), and with no extent the planner's zone-map read is +# still one scan (main's code, guarded by native_groupagg too). +# +# ledger rows 2156 + 33 = 2189 +# never 1739 + 6 = 1745 +# dated 417 + 27 = 444 +# 1745 + 444 == 2189 +# +# Option 5, a transaction's own flushed groups (OffgridwithJD): 1745 -> 1746. +# Three checks from a five-major run of the fixed tree (column_extent.sh 36/36 +# on each of 15-19). The two arms are dated by the run before the fix, the +# tree whose bound left out the transaction's own pending fold: column_extent +# read 'no', and the estimate was 35 rows against the zone maps' 5000. The +# premise that the plans gave two estimates is a fact about the instrument and +# arrives never. +# +# ledger rows 2189 + 3 = 2192 +# never 1745 + 1 = 1746 +# dated 444 + 2 = 446 +# 1746 + 446 == 2192 +# +# #1325 review (jdatcmd): 1746 -> 1745. Of column_extent's seven never rows, +# four were not premises. Three hold by construction -- compact_rewrite and +# expire still retiring with the extent in place, and parallel_copy's loaders +# all committing -- and are renamed premise: in both twins; their old rows are +# removed and the renamed ones arrive never from a five-major run of the +# renamed suite (36/36 on each of 15-19). The fourth, the planner falling back +# to one zone_map scan without the extent rows, is dated by a mutation that +# removes the fallback; the estimate check beside it went red under the same +# mutation and was already dated. +# +# ledger rows 2192 - 3 + 3 = 2192 +# never 1746 - 1 = 1745 +# dated 446 + 1 = 447 +# 1745 + 447 == 2192 +checks_never_observed_red 1745 diff --git a/test/column_extent.sh b/test/column_extent.sh new file mode 100755 index 000000000..b81c750f3 --- /dev/null +++ b/test/column_extent.sh @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +# +# The per-column extent (#1304 option 5, for #1292 phase 2). +# +# WRITTEN BEFORE THE CODE, against the design on #1304: +# +# pgcolumnar.column_extent catalog: (storage_id, column_index, complete, +# minimum, maximum), no GRANT +# pgcolumnar.column_extent(rel regclass, col name) +# -> (complete bool, minimum text, maximum text), zero rows when the +# planner would fall back to the zone maps +# +# Each committing writer inserts one row of its own (never an in-place update), a +# complete row comes only from a recompute over every visible zone map, and every +# path that retires or rewrites groups recomputes in the same transaction. The +# planner trusts the extent only when a complete row is visible. +# +# THE ORACLE is the data: min() and max() of the column, printed by the same output +# function. A zone map can be wider than the live rows only by rows deleted but not +# yet retired, so every arm that asserts equality first retires or rewrites those; +# the one arm whose extent may legitimately be wider (a rolled-back savepoint that +# flushed a group) asserts containment, not equality. +# +# Usage: test/column_extent.sh [PG_CONFIG] + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +export PGC_EXTRA_CONF=$'max_prepared_transactions=8\nmax_worker_processes=16' +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +q "CREATE EXTENSION IF NOT EXISTS pgcolumnar;" >/dev/null +check_num "premise: the extent function exists" \ + "$(q "SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = 'pgcolumnar' AND p.proname = 'column_extent';")" "1" + +ext() { # ext TABLE COLUMN -> "complete|min|max", "none" when no complete extent, or the SQLSTATE + # Its own psql, not q: "none" must mean the function answered with no row, and a + # missing function or a refused call must not read as that. + local r + r="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -v ON_ERROR_STOP=1 -v VERBOSITY=sqlstate \ + -c "SELECT complete || '|' || coalesce(minimum, '-') || '|' || coalesce(maximum, '-') FROM pgcolumnar.column_extent('$1', '$2');" 2>&1)" + case "$r" in ERROR:*) echo "error $(sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p' <<<"$r" | head -1)";; "") echo none;; *) echo "$r";; esac +} +truth() { # truth TABLE COLUMN -> "true|min|max" from the data itself + q "SELECT 'true|' || min($2)::text || '|' || max($2)::text FROM $1;" +} +sid() { q "SELECT pgcolumnar.get_storage_id('$1');"; } +GB="SET pgcolumnar.enable_group_vectorization = on;" + +# ---- the fold is the column type's own ordering ------------------------------ +# +# Byte order is not value order for any of these: ints are stored little-endian +# (256 sorts before 1 as bytes), negatives and numerics have sign and scale in +# their bytes, and text under "C" puts 'Z' before 'a'. Several row groups, so the +# fold crosses groups as well as values. +q "CREATE TABLE ty (ts timestamptz, i int, n numeric, d date, s text COLLATE \"C\") USING pgcolumnar; + SELECT pgcolumnar.set_options('ty', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO ty SELECT '2026-02-01'::timestamptz + g * interval '1 min', (g * 37) % 3001 - 1500, + ((g * 7919) % 20001 - 10000) / 7.0, date '2026-01-01' + (g % 400), + chr(65 + (g % 26)) || chr(97 + (g % 26)) FROM generate_series(1, 6000) g;" >/dev/null +check "premise: the typed fixture spans several row groups" \ + "$([ "$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = $(sid ty);")" -ge 6 ] && echo yes || echo no)" "yes" +for c in ts i n d s; do + check "the extent of ty.$c is the data's min and max, in the type's order" "$(ext ty $c)" "$(truth ty $c)" +done + +# ---- the planner reads the extent, not the zone maps -------------------------- +# +# The #1287 instrument native_groupagg uses: the planning transaction's own scans of +# zone_map and its index, which no other session can move. 50 groups, so a fallback +# would show as a scan and the extent as none. +q "CREATE TABLE pl (ts timestamptz, v float8) USING pgcolumnar; + SELECT pgcolumnar.set_options('pl', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO pl SELECT '2026-01-01'::timestamptz + g * interval '1 second', g FROM generate_series(1, 50000) g; + ANALYZE pl;" >/dev/null +plan_scans() { # zone_map scans and the estimate, planning a bucketed GROUP BY on pl + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -At -q \ + -c "$GB" -c "BEGIN" -c "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM pl GROUP BY b" \ + -c "SELECT 'scans=' || (pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass) + + pg_stat_get_xact_numscans('pgcolumnar.zone_map_pkey'::regclass))" -c "ROLLBACK" 2>&1 +} +pl_with="$(plan_scans)" +check "planning a bucketed GROUP BY scans zone_map not at all when the extent is complete" \ + "$(sed -n 's/^scans=//p' <<<"$pl_with")" "0" +# The same estimate from the fallback: remove the extent rows (superuser, as a stale +# catalog would look) and plan again. +pl_est_with="$(grep -oE 'rows=[0-9]+' <<<"$pl_with" | head -1)" +q "DELETE FROM pgcolumnar.column_extent WHERE storage_id = $(sid pl);" >/dev/null +pl_without="$(plan_scans)" +check "without the extent rows the planner falls back to one zone_map scan" \ + "$(sed -n 's/^scans=//p' <<<"$pl_without")" "1" +check "and the estimate is the one the zone maps give" "$pl_est_with" "$(grep -oE 'rows=[0-9]+' <<<"$pl_without" | head -1)" +check "and with no extent rows the function reports no complete extent" "$(ext pl ts)" "none" +# The writer-side recompute: the next committing writer finds no complete row and +# makes one, from every visible zone map, so the old groups are in it. +q "INSERT INTO pl VALUES ('2025-12-31 00:00:00+00', 0);" >/dev/null +check "a writer that finds no complete extent recomputes one over every group" "$(ext pl ts)" "$(truth pl ts)" + +# ---- concurrent writers never share a row -------------------------------------- +# +# Ordinary writers into one table are serialized by the storage-row lock (class +# 102) from their first flush to their commit, so they never overlap. The writers +# that do are parallel_copy's loaders: separate sessions flushing into one storage +# at once, each preparing its transaction. An in-place update from two of them +# would fail ("tuple concurrently updated") or lose one's widening; and each +# writes its row at PRE_PREPARE, so one missing would leave the extent too narrow. +# The file's earliest rows and its latest are far apart in it, so different loaders +# hold them. +q "CREATE TABLE cw (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('cw', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO cw SELECT '2026-06-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 2000) g;" >/dev/null +q "COPY (SELECT '2020-01-01'::timestamptz + g * interval '1 day', g FROM generate_series(1, 4000) g) + TO '$PGC_WORKDIR/cw.txt' WITH (FORMAT text);" >/dev/null +check "premise: parallel_copy's loaders, flushing into one table at once, all commit" \ + "$(q "SELECT pgcolumnar.parallel_copy('cw', '$PGC_WORKDIR/cw.txt', 4);")" "4000" +check "and the extent covers what they loaded" "$(ext cw ts)" "$(truth cw ts)" + +# ---- a writer that aborts leaves nothing; a rolled-back savepoint only widens -- +q "BEGIN; INSERT INTO cw SELECT '1990-01-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 3000) g; ROLLBACK;" >/dev/null +check "a writer that rolls back leaves the extent as the data has it" "$(ext cw ts)" "$(truth cw ts)" +q "BEGIN; INSERT INTO cw VALUES ('2026-06-02', 1); SAVEPOINT s; + INSERT INTO cw SELECT '1980-01-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 3000) g; + ROLLBACK TO s; COMMIT;" >/dev/null +contains() { # contains TABLE COLUMN TYPE -> yes when the extent contains the data's range + q "SELECT CASE WHEN e.complete AND e.minimum::$3 <= (SELECT min($2) FROM $1) + AND e.maximum::$3 >= (SELECT max($2) FROM $1) THEN 'yes' ELSE 'no' END + FROM pgcolumnar.column_extent('$1', '$2') e;" +} +check "a savepoint rolled back after flushing may widen the extent, never narrow it" "$(contains cw ts timestamptz)" "yes" + +# A prepared transaction writes its extent row before PREPARE, where a catalog write +# is still allowed; its groups commit with COMMIT PREPARED and must be inside it. +# Its own table: cw's extent is already wider than its data, by the savepoint above. +q "CREATE TABLE tp (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('tp', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO tp SELECT '2026-06-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 2000) g;" >/dev/null +q "BEGIN; INSERT INTO tp SELECT '2040-01-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 3000) g; + PREPARE TRANSACTION 'ce_2pc';" >/dev/null +q "COMMIT PREPARED 'ce_2pc';" >/dev/null +check "a prepared transaction's groups are inside the extent after COMMIT PREPARED" "$(ext tp ts)" "$(truth tp ts)" + +# A table created and written inside a savepoint that rolls back leaves no extent +# row behind: its storage never existed for anyone, and an orphan row would be kept +# by nothing. +q "BEGIN; SAVEPOINT s; CREATE TABLE gone (ts timestamptz) USING pgcolumnar; + INSERT INTO gone SELECT now() FROM generate_series(1, 2000); + ROLLBACK TO s; COMMIT;" >/dev/null +check "a table created in a rolled-back savepoint leaves no extent rows" \ + "$(q "SELECT count(*) FROM pgcolumnar.column_extent e + WHERE NOT EXISTS (SELECT 1 FROM pgcolumnar.storage s WHERE s.storage_id = e.storage_id);")" "0" + +# ---- a transaction reads its own flushed groups in the extent -------------------- +# +# Its extent row is written at pre-commit, but its groups are visible to it from the +# moment they flush. A plan made in between must not bound the column by the +# committed rows alone: that is an UNDER-estimate, the direction the gate on this +# bound must never take. The oracle is the same transaction with the extent rows +# deleted, so the planner falls back to the zone maps, which do include them. +q "CREATE TABLE ot (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('ot', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO ot SELECT '2026-06-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 2000) g; + ANALYZE ot;" >/dev/null +OWN_INS="INSERT INTO ot SELECT '2050-01-01'::timestamptz + g * interval '1 min', g FROM generate_series(1, 3000) g" +check "a transaction's own flushed groups are inside the extent it reads before commit" \ + "$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At -q \ + -c "BEGIN" -c "$OWN_INS" \ + -c "SELECT CASE WHEN e.complete AND e.minimum::timestamptz <= (SELECT min(ts) FROM ot) + AND e.maximum::timestamptz >= (SELECT max(ts) FROM ot) THEN 'yes' ELSE 'no' END + FROM pgcolumnar.column_extent('ot', 'ts') e" -c "ROLLBACK" 2>&1)" "yes" +own_plans="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At -q \ + -c "$GB" -c "BEGIN" -c "$OWN_INS" \ + -c "SELECT 'plan'" -c "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM ot GROUP BY b" \ + -c "DELETE FROM pgcolumnar.column_extent WHERE storage_id = $(sid ot)" \ + -c "SELECT 'plan'" -c "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM ot GROUP BY b" -c "ROLLBACK" 2>&1 | + awk '/^plan$/ { want = 1; next } want && match($0, /rows=[0-9]+/) { print substr($0, RSTART, RLENGTH); want = 0 }')" +check "and a plan made in that transaction estimates as the zone maps do" \ + "$(sed -n 1p <<<"$own_plans")" "$(sed -n 2p <<<"$own_plans")" +check "premise: the transaction's plans gave two estimates" "$(wc -l <<<"$own_plans")" "2" + +# ---- a shrink is recomputed, never guessed (jdatcmd, #1304) --------------------- +# +# Each path that retires or rewrites groups recomputes the extent in the same +# transaction, so it equals the data afterwards: not narrower (an under-estimate), +# and not left at the old width. +q "CREATE TABLE ex (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('ex', stripe_row_limit => 1000, chunk_group_row_limit => 1000, + ttl_column => 'ts', ttl_interval => '30 days'); + INSERT INTO ex SELECT now() - interval '400 days' + g * interval '1 min', g FROM generate_series(1, 3000) g; + INSERT INTO ex SELECT now() - interval '1 day' + g * interval '1 s', g FROM generate_series(1, 3000) g;" >/dev/null +ex_before="$(ext ex ts)" +retired="$(q "SELECT pgcolumnar.expire('ex');")" +check "premise: expire retires the old row groups with the extent in place" "$([ "${retired:-0}" -ge 1 ] && echo yes || echo no)" "yes" +check "after expire the extent is recomputed to the data that is left" "$(ext ex ts)" "$(truth ex ts)" +check "and it did move: the retired groups held the old minimum" \ + "$([ "$ex_before" != "$(ext ex ts)" ] && echo yes || echo no)" "yes" + +q "CREATE TABLE cr (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('cr', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO cr SELECT '2026-01-01'::timestamptz + g * interval '1 min', g FROM generate_series(1, 4000) g; + DELETE FROM cr WHERE v > 3500 OR v <= 400;" >/dev/null +rewritten="$(q "SELECT pgcolumnar.compact_rewrite('cr', 0.1);")" +check "premise: compact_rewrite rewrites the groups the deletes thinned" "$([ "${rewritten:-0}" -ge 1 ] && echo yes || echo no)" "yes" +check "after compact_rewrite the extent is the live rows'" "$(ext cr ts)" "$(truth cr ts)" + +q "CREATE TABLE vr (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('vr', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO vr SELECT '2026-01-01'::timestamptz + g * interval '1 min', g FROM generate_series(1, 3000) g; + DELETE FROM vr WHERE v > 2000;" >/dev/null +q "VACUUM vr;" >/dev/null +check "after VACUUM retires a fully deleted group the extent is the live rows'" "$(ext vr ts)" "$(truth vr ts)" + +# ---- no complete extent where the zone maps cannot give one ---------------------- +q "CREATE TABLE ac (ts timestamptz) USING pgcolumnar; + SELECT pgcolumnar.set_options('ac', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO ac SELECT '2026-01-01'::timestamptz + g * interval '1 s' FROM generate_series(1, 2000) g; + ALTER TABLE ac ADD COLUMN v int DEFAULT 5; + INSERT INTO ac SELECT '2026-02-01'::timestamptz + g * interval '1 s', g FROM generate_series(1, 2000) g;" >/dev/null +check "a column added after older groups has no complete extent (those groups have no zone map for it)" \ + "$(ext ac v)" "none" +check "while the column that was there from the start still has one" "$(ext ac ts)" "$(truth ac ts)" +# Even after a recompute: a type with no ordering has values its zone maps cannot +# bound, and a recompute must refuse rather than claim the column holds none. A +# fully deleted group, then VACUUM, makes one run. +q "CREATE TABLE no (p point, k int) USING pgcolumnar; + SELECT pgcolumnar.set_options('no', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO no SELECT point(g, g), g FROM generate_series(1, 3000) g; + DELETE FROM no WHERE k > 2000;" >/dev/null +q "VACUUM no;" >/dev/null +check "a type with no ordering gets no extent" "$(ext no p)" "none" + +# ---- housekeeping ----------------------------------------------------------------- +q "CREATE TABLE hk (ts timestamptz, v int) USING pgcolumnar;" >/dev/null +for k in $(seq 1 50); do q "INSERT INTO hk VALUES ('2026-01-01'::timestamptz + interval '$k min', $k);" >/dev/null; done +hk_rows="$(q "SELECT count(*) FROM pgcolumnar.column_extent WHERE storage_id = $(sid hk) AND column_index = 0;")" +check "fifty committing writers leave a bounded number of extent rows" \ + "$([ "${hk_rows:-99}" -le 9 ] && echo yes || echo "no (${hk_rows:-none})")" "yes" +check "and the extent still covers all fifty" "$(ext hk ts)" "$(truth hk ts)" +hk_old="$(sid hk)" +q "TRUNCATE hk;" >/dev/null +check "TRUNCATE removes the old storage's extent rows" \ + "$(q "SELECT count(*) FROM pgcolumnar.column_extent WHERE storage_id = $hk_old;")" "0" +cw_old="$(sid cw)" +q "DROP TABLE cw;" >/dev/null +check "DROP TABLE removes them" "$(q "SELECT count(*) FROM pgcolumnar.column_extent WHERE storage_id = $cw_old;")" "0" + +q "CREATE ROLE ce_r; GRANT USAGE ON SCHEMA pgcolumnar TO ce_r;" >/dev/null +as_role() { env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -v VERBOSITY=sqlstate -c "SET ROLE $1" -c "$2" 2>&1 | grep -v '^SET$' | tail -1; } +check "the extent catalog is not readable by an ordinary role (it holds column values)" \ + "$(as_role ce_r "SELECT count(*) FROM pgcolumnar.column_extent" | sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p')" "42501" +check "nor is a table's extent through the function, without SELECT on the table" \ + "$(as_role ce_r "SELECT * FROM pgcolumnar.column_extent('public.ty', 'ts')" | sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p')" "42501" + +pgc_summary diff --git a/test/native_groupagg.sh b/test/native_groupagg.sh index 8bbe41cf2..6dcf7c40e 100755 --- a/test/native_groupagg.sh +++ b/test/native_groupagg.sh @@ -494,6 +494,10 @@ psql_run "CREATE TABLE gbz (ts timestamptz, v float8) USING pgcolumnar; INSERT INTO gbz SELECT '2026-01-01'::timestamptz + g * interval '1 second', g FROM generate_series(1,50000) g; ANALYZE gbz;" >/dev/null 2>&1 gbz_groups="$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('gbz');")" +# THE FALLBACK IS WHAT THIS MEASURES. Since #1304 option 5 a complete column extent +# bounds the range with no zone_map scan at all (column_extent.sh asserts 0); without +# one the planner reads the zone maps, and that read must stay one scan. +q "DELETE FROM pgcolumnar.column_extent WHERE storage_id = pgcolumnar.get_storage_id('gbz');" >/dev/null gbz_scans="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -At -q \ -c "$GB" -c "BEGIN" -c "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM gbz GROUP BY b" \ -c "SELECT pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass) diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index b0483aace..8a9eb7113 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -147,6 +147,7 @@ behaviour, the source of that number is named. - [99. test_suite_exit_trap.py: a suite's own EXIT trap must still stop its cluster](#99-test_suite_exit_trappy-a-suites-own-exit-trap-must-still-stop-its-cluster) - [100. test_tier_partition.py: tier_partition moves a cold partition to Parquet, exactly once](#100-test_tier_partitionpy-tier_partition-moves-a-cold-partition-to-parquet-exactly-once) - [101. test_prepared_writes.py: a write prepared and finished in one session leaves the backend alive](#101-test_prepared_writespy-a-write-prepared-and-finished-in-one-session-leaves-the-backend-alive) +- [102. test_column_extent.py: the per-column extent is a true bound, recomputed whenever groups go](#102-test_column_extentpy-the-per-column-extent-is-a-true-bound-recomputed-whenever-groups-go) ## 1. How to read a test in here @@ -6699,3 +6700,32 @@ Red on main, the first arm reading `lost`. Each of the two discards is red under | test | what it holds | | --- | --- | | `test_a_write_prepared_and_finished_in_one_session` | an insert, a delete and an update prepared and committed in one session, an insert prepared and rolled back, a write after COMMIT PREPARED, and an index fetch on each side of it, all answered by the same session; the server log gains no "terminated by signal"; no prepared transaction is left | + +## 102. test_column_extent.py: the per-column extent is a true bound, recomputed whenever groups go + +Port of `column_extent.sh` (#1304 option 5, for #1292 phase 2). `pgcolumnar.column_extent` keeps, per column, rows that are inserted and deleted but never updated. Each committing writer adds one row for the groups it flushed. A recompute over every zone map one snapshot sees replaces the rows that snapshot sees with one complete row. Every path that retires a group recomputes before commit. The planner uses the rows only when a complete one is visible, and their fold is then a true bound. `pgcolumnar.column_extent(rel, col)` shows that bound. + +THE ORACLE IS THE DATA: `min()` and `max()` of the column. Every equality arm first makes sure no group is left holding deleted rows, so the zone maps and the data agree. Twice, the first draft of an arm deleted rows that left a group partly live and so wider than the data, and the extent was right where the arm was wrong. The one case that may legitimately be wider, a rolled-back savepoint that flushed a group, is asserted as containment. + +Independent of the shell suite: its own row counts, values, times and helpers. The check names match, so `compare_to_bash` grades the pair one for one. + +WHAT WRITING IT FOUND: + +- **Ordinary writers never overlap.** Concurrent writers into one table are serialized by the storage-row lock (class 102) from their first flush to their commit. The concurrency arm therefore uses `parallel_copy`, whose loaders flush into one storage at once, each through PREPARE. +- **COMMIT PREPARED crashes in the preparing session.** On main, `COMMIT PREPARED` in the session that prepared a pgcolumnar write crashes the backend (#1323), so this file commits from another session. +- **A transaction did not see its own groups in the extent.** Its extent row is written at pre-commit, but its groups are visible to it from the moment they flush. A plan made in between bounded the column by the committed rows alone, which is an under-estimate: 61 rows against the zone maps' 4500 in this file's fixture. The bound now adds the transaction's own pending fold. +- **The xact scan counters carry over.** `pg_stat_get_xact_numscans` reports a backend's pending stats, flushed about once a second rather than at transaction end. A measurement in the same connection as the writes counts their zone-map reads too, so the planning arm measures on a fresh connection. + +Written red first: the shell twin, against main, with the function absent (42883). + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_extent_is_the_columns_own_order` | `timestamptz`, `int` across 256 and negative, negative `numeric`, `date` and `text COLLATE "C"`: each column's extent is the data's min and max, over several row groups | +| `test_the_planner_reads_the_extent_not_the_zone_maps` | planning a bucketed GROUP BY scans `zone_map` 0 times with a complete extent and 1 without, with the same estimate; with no rows the function returns none; the next writer recomputes one over every group | +| `test_writers_concurrent_aborted_and_prepared` | `parallel_copy` with three loaders, an aborted writer, a rolled-back savepoint (containment), PREPARE and COMMIT PREPARED, and a table created in a rolled-back savepoint (no orphan rows) | +| `test_a_transaction_reads_its_own_flushed_groups` | inside a transaction that has flushed groups far outside the committed extent, `column_extent` contains them, and a plan estimates as it does with the extent rows deleted (the zone maps) | +| `test_a_shrink_is_recomputed_never_guessed` | after `expire`, `compact_rewrite` and VACUUM retire or rewrite groups, the extent equals the data left, and after `expire` it moved | +| `test_no_complete_extent_where_the_zone_maps_cannot_give_one` | a column added after older groups, and a `point` column even after a recompute, get no complete extent | +| `test_housekeeping_and_privilege` | fifty committing writers leave at most nine rows; TRUNCATE and DROP remove them; an ordinary role cannot read the catalog, nor the function without SELECT | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 7357fd88e..168a8877e 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -585,4 +585,10 @@ guard_tests 419 # # cluster_tests 516 -> 517 for #1323's test_prepared_writes.py, one test, by collection # on this branch rebased onto 305e15e (#1317). -cluster_tests 517 +# +# cluster_tests 517 -> 523 for #1304 option 5's test_column_extent.py, six tests, +# by collection on this branch rebased onto 036d452 (#1324). +# +# cluster_tests 523 -> 524 for option 5's test_a_transaction_reads_its_own_flushed_groups, +# by collection on the same tree. +cluster_tests 524 diff --git a/test/pytest/test_column_extent.py b/test/pytest/test_column_extent.py new file mode 100644 index 000000000..302b22498 --- /dev/null +++ b/test/pytest/test_column_extent.py @@ -0,0 +1,307 @@ +"""The per-column extent (#1304 option 5, for #1292 phase 2). + +pgcolumnar.column_extent holds, per (storage, column), rows that are inserted and +deleted but never updated: each committing writer adds one for what it flushed, and +a recompute over every visible zone map replaces the rows it saw with one complete +row. The planner trusts the rows only when a complete one is visible; their fold is +then a true bound. pgcolumnar.column_extent(rel, col) shows that bound, and returns +no row when the planner would fall back to the zone maps. + +THE ORACLE is the data: min() and max() of the column, printed by the same output +function. Equality only once deleted rows are retired or rewritten; the one case +that may legitimately be wider (a rolled-back savepoint that flushed a group) is +asserted as containment. + +ON A PRIVATE DATABASE (`pgc_own_db`): the role, and the transaction-local scan +counters. Independent of `test/column_extent.sh`: its own row counts, values, times +and helpers; the check names match, so the pair is graded one for one. +""" + +import psycopg + +GB = "SET pgcolumnar.enable_group_vectorization = on" + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + row = cur.fetchone() + return row[0] if row else None + + +def _run(conn, *sqls): + with conn.cursor() as cur: + for sql in sqls: + cur.execute(sql) + + +def _ext(conn, table, col): + """'true|min|max', 'none' when no complete extent, or 'error '.""" + try: + with conn.cursor() as cur: + cur.execute("SELECT complete::text || '|' || coalesce(minimum, '-') || '|' || coalesce(maximum, '-')" + f" FROM pgcolumnar.column_extent('{table}', '{col}')") + row = cur.fetchone() + except psycopg.Error as e: + return f"error {e.sqlstate}" + return row[0] if row else "none" + + +def _truth(conn, table, col): + return _one(conn, f"SELECT 'true|' || min({col})::text || '|' || max({col})::text FROM {table}") + + +def _sid(conn, table): + return _one(conn, f"SELECT pgcolumnar.get_storage_id('{table}')") + + +def _small_groups(conn, table): + _run(conn, f"SELECT pgcolumnar.set_options('{table}', stripe_row_limit => 1000," + " chunk_group_row_limit => 1000)") + + +def test_the_extent_is_the_columns_own_order(pgc_own_db, expect): + conn = pgc_own_db + expect.num(_one(conn, "SELECT count(*) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace" + " WHERE n.nspname = 'pgcolumnar' AND p.proname = 'column_extent'"), 1, + "premise: the extent function exists") + # Byte order is not value order for any of these: little-endian ints across 256, + # signed values, numeric scale, and "C" text where 'Q' sorts before 'b'. + _run(conn, 'CREATE TABLE ty (ts timestamptz, i int, n numeric, d date, s text COLLATE "C") USING pgcolumnar') + _small_groups(conn, "ty") + _run(conn, "INSERT INTO ty SELECT '2025-11-03 08:00:00+00'::timestamptz + g * interval '47 s'," + " (g * 53) % 4001 - 2000, ((g * 6301) % 30011 - 15005) / 11.0, date '2025-03-01' + (g % 523)," + " chr(81 + (g % 10)) || chr(98 + (g % 20)) FROM generate_series(1, 7300) g") + expect.at_least(_one(conn, f"SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = {_sid(conn, 'ty')}"), + 6, "premise: the typed fixture spans several row groups") + for c in ("ts", "i", "n", "d", "s"): + expect.text(_ext(conn, "ty", c), _truth(conn, "ty", c), + f"the extent of ty.{c} is the data's min and max, in the type's order") + + +def _plan(pgc_cluster, dbname): + """(zone_map scans, the estimate) planning a bucketed GROUP BY on pl, in one transaction.""" + with psycopg.connect(pgc_cluster.dsn(dbname=dbname)) as c: + with c.cursor() as cur: + cur.execute(GB) + cur.execute("EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM pl GROUP BY b") + plan = "\n".join(r[0] for r in cur.fetchall()) + cur.execute("SELECT pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass)" + " + pg_stat_get_xact_numscans('pgcolumnar.zone_map_pkey'::regclass)") + scans = cur.fetchone()[0] + c.rollback() + est = plan.split("rows=", 1)[1].split(" ", 1)[0] if "rows=" in plan else "" + return scans, est + + +def test_the_planner_reads_the_extent_not_the_zone_maps(pgc_own_db, pgc_cluster, expect): + conn = pgc_own_db + db = _one(conn, "SELECT current_database()") + _run(conn, "CREATE TABLE pl (ts timestamptz, v float8) USING pgcolumnar") + _small_groups(conn, "pl") + _run(conn, "INSERT INTO pl SELECT '2025-07-01'::timestamptz + g * interval '3 s', g" + " FROM generate_series(1, 60000) g", "ANALYZE pl") + scans_with, est_with = _plan(pgc_cluster, db) + expect.num(scans_with, 0, "planning a bucketed GROUP BY scans zone_map not at all when the extent is complete") + _run(conn, f"DELETE FROM pgcolumnar.column_extent WHERE storage_id = {_sid(conn, 'pl')}") + scans_without, est_without = _plan(pgc_cluster, db) + expect.num(scans_without, 1, "without the extent rows the planner falls back to one zone_map scan") + expect.text(f"rows={est_with}", f"rows={est_without}", "and the estimate is the one the zone maps give") + expect.text(_ext(conn, "pl", "ts"), "none", "and with no extent rows the function reports no complete extent") + _run(conn, "INSERT INTO pl VALUES ('2025-06-30 00:00:00+00', 0)") + expect.text(_ext(conn, "pl", "ts"), _truth(conn, "pl", "ts"), + "a writer that finds no complete extent recomputes one over every group") + + +def test_writers_concurrent_aborted_and_prepared(pgc_own_db, pgc_cluster, expect): + conn = pgc_own_db + dsn = pgc_cluster.dsn(dbname=_one(conn, "SELECT current_database()")) + _run(conn, "CREATE TABLE cw (ts timestamptz, v int) USING pgcolumnar") + _small_groups(conn, "cw") + _run(conn, "INSERT INTO cw SELECT '2025-09-09'::timestamptz + g * interval '2 s', g FROM generate_series(1, 1500) g") + + # Ordinary writers are serialized by the storage-row lock from first flush to + # commit; parallel_copy's loaders are the ones that flush into one storage at + # once, each through PREPARE. The first and last rows of the file are far apart. + path = _one(conn, "SELECT current_setting('data_directory')") + "/ce_py_cw.txt" + _run(conn, "COPY (SELECT '2018-03-03'::timestamptz + g * interval '13 hours', g" + f" FROM generate_series(1, 5200) g) TO '{path}' WITH (FORMAT text)") + expect.num(_one(conn, f"SELECT pgcolumnar.parallel_copy('cw', '{path}', 3)"), 5200, + "premise: parallel_copy's loaders, flushing into one table at once, all commit") + expect.text(_ext(conn, "cw", "ts"), _truth(conn, "cw", "ts"), "and the extent covers what they loaded") + + with psycopg.connect(dsn) as c: + _run(c, "INSERT INTO cw SELECT '1995-01-01'::timestamptz + g * interval '2 s', g FROM generate_series(1, 2600) g") + c.rollback() + expect.text(_ext(conn, "cw", "ts"), _truth(conn, "cw", "ts"), + "a writer that rolls back leaves the extent as the data has it") + + with psycopg.connect(dsn) as c: + _run(c, "INSERT INTO cw VALUES ('2025-09-10', 1)", "SAVEPOINT s", + "INSERT INTO cw SELECT '1985-01-01'::timestamptz + g * interval '2 s', g FROM generate_series(1, 2600) g", + "ROLLBACK TO s") + c.commit() + expect.text(_one(conn, "SELECT CASE WHEN e.complete AND e.minimum::timestamptz <= (SELECT min(ts) FROM cw)" + " AND e.maximum::timestamptz >= (SELECT max(ts) FROM cw) THEN 'yes' ELSE 'no' END" + " FROM pgcolumnar.column_extent('cw', 'ts') e"), "yes", + "a savepoint rolled back after flushing may widen the extent, never narrow it") + + _run(conn, "CREATE TABLE tp (ts timestamptz, v int) USING pgcolumnar") + _small_groups(conn, "tp") + _run(conn, "INSERT INTO tp SELECT '2025-09-09'::timestamptz + g * interval '2 s', g FROM generate_series(1, 1500) g") + # COMMIT PREPARED from another session: in the preparing session it crashes the + # backend on main, a separate bug (the write states outlive PREPARE). + with psycopg.connect(dsn, autocommit=True) as c: + _run(c, "BEGIN", + "INSERT INTO tp SELECT '2044-02-02'::timestamptz + g * interval '2 s', g FROM generate_series(1, 2600) g", + "PREPARE TRANSACTION 'ce_py_2pc'") + _run(conn, "COMMIT PREPARED 'ce_py_2pc'") + expect.text(_ext(conn, "tp", "ts"), _truth(conn, "tp", "ts"), + "a prepared transaction's groups are inside the extent after COMMIT PREPARED") + + with psycopg.connect(dsn) as c: + _run(c, "SAVEPOINT s", "CREATE TABLE gone (ts timestamptz) USING pgcolumnar", + "INSERT INTO gone SELECT now() FROM generate_series(1, 2500)", "ROLLBACK TO s") + c.commit() + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.column_extent e WHERE NOT EXISTS" + " (SELECT 1 FROM pgcolumnar.storage s WHERE s.storage_id = e.storage_id)"), 0, + "a table created in a rolled-back savepoint leaves no extent rows") + + +def test_a_transaction_reads_its_own_flushed_groups(pgc_own_db, pgc_cluster, expect): + """Its extent row is written at pre-commit, but its groups are visible to it as + soon as they flush; a plan made in between must not bound the column by the + committed rows alone. The oracle is the same transaction with the extent rows + deleted, which sends the planner to the zone maps.""" + conn = pgc_own_db + dsn = pgc_cluster.dsn(dbname=_one(conn, "SELECT current_database()")) + _run(conn, "CREATE TABLE ow (ts timestamptz, v int) USING pgcolumnar") + _small_groups(conn, "ow") + _run(conn, "INSERT INTO ow SELECT '2025-03-03'::timestamptz + g * interval '2 s', g FROM generate_series(1, 1800) g", + "ANALYZE ow") + sid = _sid(conn, "ow") + own = "INSERT INTO ow SELECT '1999-09-09'::timestamptz + g * interval '1 min', g FROM generate_series(1, 2700) g" + plan = "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM ow GROUP BY b" + + def estimate(cur): + cur.execute(plan) + text = "\n".join(r[0] for r in cur.fetchall()) + return text.split("rows=", 1)[1].split(" ", 1)[0] if "rows=" in text else "" + + with psycopg.connect(dsn) as c: + with c.cursor() as cur: + cur.execute(own) + cur.execute("SELECT CASE WHEN e.complete AND e.minimum::timestamptz <= (SELECT min(ts) FROM ow)" + " AND e.maximum::timestamptz >= (SELECT max(ts) FROM ow) THEN 'yes' ELSE 'no' END" + " FROM pgcolumnar.column_extent('ow', 'ts') e") + row = cur.fetchone() + inside = row[0] if row else "none" + c.rollback() + expect.text(inside, "yes", "a transaction's own flushed groups are inside the extent it reads before commit") + + with psycopg.connect(dsn) as c: + with c.cursor() as cur: + cur.execute(GB) + cur.execute(own) + with_extent = estimate(cur) + cur.execute(f"DELETE FROM pgcolumnar.column_extent WHERE storage_id = {sid}") + from_zone_maps = estimate(cur) + c.rollback() + expect.text("two" if with_extent and from_zone_maps else f"[{with_extent}] [{from_zone_maps}]", "two", + "premise: the transaction's plans gave two estimates") + expect.text(f"rows={with_extent}", f"rows={from_zone_maps}", + "and a plan made in that transaction estimates as the zone maps do") + + +def test_a_shrink_is_recomputed_never_guessed(pgc_own_db, expect): + conn = pgc_own_db + _run(conn, "CREATE TABLE ex (ts timestamptz, v int) USING pgcolumnar") + _run(conn, "SELECT pgcolumnar.set_options('ex', stripe_row_limit => 1000, chunk_group_row_limit => 1000," + " ttl_column => 'ts', ttl_interval => '60 days')") + _run(conn, "INSERT INTO ex SELECT now() - interval '700 days' + g * interval '90 s', g FROM generate_series(1, 2500) g", + "INSERT INTO ex SELECT now() - interval '2 days' + g * interval '7 s', g FROM generate_series(1, 2500) g") + before = _ext(conn, "ex", "ts") + retired = _one(conn, "SELECT pgcolumnar.expire('ex')") + expect.text("yes" if (retired or 0) >= 1 else f"no ({retired})", "yes", + "premise: expire retires the old row groups with the extent in place") + after = _ext(conn, "ex", "ts") + expect.text(after, _truth(conn, "ex", "ts"), "after expire the extent is recomputed to the data that is left") + expect.text("yes" if before != after else "no", "yes", + "and it did move: the retired groups held the old minimum") + + _run(conn, "CREATE TABLE cr (ts timestamptz, v int) USING pgcolumnar") + _small_groups(conn, "cr") + _run(conn, "INSERT INTO cr SELECT '2025-05-05'::timestamptz + g * interval '5 min', g FROM generate_series(1, 4500) g", + "DELETE FROM cr WHERE v > 4200 OR v <= 650") + rewritten = _one(conn, "SELECT pgcolumnar.compact_rewrite('cr', 0.1)") + expect.text("yes" if (rewritten or 0) >= 1 else f"no ({rewritten})", "yes", + "premise: compact_rewrite rewrites the groups the deletes thinned") + expect.text(_ext(conn, "cr", "ts"), _truth(conn, "cr", "ts"), "after compact_rewrite the extent is the live rows'") + + _run(conn, "CREATE TABLE vr (ts timestamptz, v int) USING pgcolumnar") + _small_groups(conn, "vr") + _run(conn, "INSERT INTO vr SELECT '2025-05-05'::timestamptz + g * interval '5 min', g FROM generate_series(1, 3400) g", + "DELETE FROM vr WHERE v > 2000", "VACUUM vr") + expect.text(_ext(conn, "vr", "ts"), _truth(conn, "vr", "ts"), + "after VACUUM retires a fully deleted group the extent is the live rows'") + + +def test_no_complete_extent_where_the_zone_maps_cannot_give_one(pgc_own_db, expect): + conn = pgc_own_db + _run(conn, "CREATE TABLE ac (ts timestamptz) USING pgcolumnar") + _small_groups(conn, "ac") + _run(conn, "INSERT INTO ac SELECT '2025-03-03'::timestamptz + g * interval '9 s' FROM generate_series(1, 2400) g", + "ALTER TABLE ac ADD COLUMN v int DEFAULT 11", + "INSERT INTO ac SELECT '2025-04-04'::timestamptz + g * interval '9 s', g FROM generate_series(1, 2400) g") + expect.text(_ext(conn, "ac", "v"), "none", + "a column added after older groups has no complete extent (those groups have no zone map for it)") + expect.text(_ext(conn, "ac", "ts"), _truth(conn, "ac", "ts"), + "while the column that was there from the start still has one") + # A recompute (VACUUM after a fully deleted group) must refuse a column whose + # values have no ordering, not claim it holds none. + _run(conn, "CREATE TABLE no (p point, k int) USING pgcolumnar") + _small_groups(conn, "no") + _run(conn, "INSERT INTO no SELECT point(g, -g), g FROM generate_series(1, 2400) g", + "DELETE FROM no WHERE k > 2000", "VACUUM no") + expect.text(_ext(conn, "no", "p"), "none", "a type with no ordering gets no extent") + + +def test_housekeeping_and_privilege(pgc_own_db, pgc_cluster, expect): + conn = pgc_own_db + dsn = pgc_cluster.dsn(dbname=_one(conn, "SELECT current_database()")) + _run(conn, "CREATE TABLE hk (ts timestamptz, v int) USING pgcolumnar") + for k in range(1, 51): + _run(conn, f"INSERT INTO hk VALUES ('2025-01-01'::timestamptz + interval '{k} hours', {k})") + rows = _one(conn, f"SELECT count(*) FROM pgcolumnar.column_extent WHERE storage_id = {_sid(conn, 'hk')}" + " AND column_index = 0") + expect.text("yes" if rows <= 9 else f"no ({rows})", "yes", + "fifty committing writers leave a bounded number of extent rows") + expect.text(_ext(conn, "hk", "ts"), _truth(conn, "hk", "ts"), "and the extent still covers all fifty") + old = _sid(conn, "hk") + _run(conn, "TRUNCATE hk") + expect.num(_one(conn, f"SELECT count(*) FROM pgcolumnar.column_extent WHERE storage_id = {old}"), 0, + "TRUNCATE removes the old storage's extent rows") + _run(conn, "CREATE TABLE dr (ts timestamptz) USING pgcolumnar", + "INSERT INTO dr SELECT now() FROM generate_series(1, 10)") + old = _sid(conn, "dr") + _run(conn, "DROP TABLE dr") + expect.num(_one(conn, f"SELECT count(*) FROM pgcolumnar.column_extent WHERE storage_id = {old}"), 0, + "DROP TABLE removes them") + + _run(conn, "DO $$ BEGIN CREATE ROLE ce_py; EXCEPTION WHEN duplicate_object THEN NULL; END $$", + "GRANT USAGE ON SCHEMA pgcolumnar TO ce_py", + "CREATE TABLE IF NOT EXISTS priv (ts timestamptz) USING pgcolumnar", + "INSERT INTO priv VALUES (now())") + + def as_role(sql): + with psycopg.connect(dsn, autocommit=True) as c: + c.execute("SET ROLE ce_py") + try: + c.execute(sql).fetchall() + return "ok" + except psycopg.Error as e: + return e.sqlstate or "?" + expect.text(as_role("SELECT count(*) FROM pgcolumnar.column_extent"), "42501", + "the extent catalog is not readable by an ordinary role (it holds column values)") + expect.text(as_role("SELECT * FROM pgcolumnar.column_extent('public.priv', 'ts')"), "42501", + "nor is a table's extent through the function, without SELECT on the table") diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 6814e0def..d243c5d0f 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -95,6 +95,7 @@ "catalog_delete_index", "catalog_plan_index", "catalog_scan_index", + "column_extent", "differential", "encode_post_codec", "format_downgrade", diff --git a/test/pytest/test_native_groupagg.py b/test/pytest/test_native_groupagg.py index 560f1df49..6973495c6 100644 --- a/test/pytest/test_native_groupagg.py +++ b/test/pytest/test_native_groupagg.py @@ -658,7 +658,7 @@ def within(sql): "a non-time expression key gets no bound either") -def test_the_bound_reads_the_zone_maps_in_one_catalog_scan(pgc_conn, expect): +def test_the_bound_reads_the_zone_maps_in_one_catalog_scan(pgc_conn, pgc_cluster, expect): """The range costs one catalog scan, not one per row group (#1304 review). The first version probed zone_map_pkey once per group, on every plan: 109 -> 313 @@ -677,14 +677,25 @@ def test_the_bound_reads_the_zone_maps_in_one_catalog_scan(pgc_conn, expect): groups = int(_one(pgc_conn, "SELECT count(*) FROM pgcolumnar.row_group " "WHERE storage_id = pgcolumnar.get_storage_id('gbz')")) - _exec(pgc_conn, "BEGIN") - try: - _exec(pgc_conn, "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM gbz GROUP BY b") - scans = int(_one(pgc_conn, + # The fallback is what this measures: since #1304 option 5 a complete column + # extent bounds the range with no zone_map scan (test_column_extent.py asserts + # 0), and without one the zone-map read must stay one scan. + _exec(pgc_conn, "DELETE FROM pgcolumnar.column_extent " + "WHERE storage_id = pgcolumnar.get_storage_id('gbz')") + # ON A FRESH CONNECTION. The "xact" counters are this backend's pending stats, + # which are flushed to shared memory at most about once a second, not at + # transaction end -- so in pgc_conn they still held the INSERT's own zone_map + # reads (the column extent's first recompute, at its commit), and read 3 for a + # plan that scanned once. The shell twin starts a new psql, so it never saw them. + with psycopg.connect(pgc_cluster.dsn(dbname=_one(pgc_conn, "SELECT current_database()"))) as m: + _exec(m, "SELECT set_config('search_path', %s, false)" % ( + "'" + _one(pgc_conn, "SELECT current_setting('search_path')").replace("'", "''") + "'")) + _gb_session(m) + _exec(m, "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM gbz GROUP BY b") + scans = int(_one(m, "SELECT pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass) " "+ pg_stat_get_xact_numscans('pgcolumnar.zone_map_pkey'::regclass)")) - finally: - _exec(pgc_conn, "ROLLBACK") + m.rollback() print(f"-- {groups} row groups, {scans} zone_map scans while planning") expect.at_least(groups, 40, "premise: the fixture has many row groups, so one scan and one per group differ") diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index fa6cf7d54..9f0c52540 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -67,6 +67,7 @@ SUITES=( catalog_natts catalog_plan_index catalog_scan_index + column_extent column_projection concurrency concurrent_diff