diff --git a/CHANGELOG.md b/CHANGELOG.md index 47c50b9c..6282c609 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,49 @@ true until the next version shipped. ### Fixed +- Twelve metadata lookups no longer read their whole catalog to answer a + question about one table (#1207). Each passed `InvalidOid` to + `systable_beginscan` on a key an existing index starts with, and the + `pgcolumnar` catalogs are shared by every columnar table in the database, so + each paid for every other table's rows: + + free_space the compaction reuse, both end-truncation reads + row_group row_group_exists, reached by EVERY DELETE + storage three existence checks on every write, and the + sorted-extent update + projection_declaration list, rename a column, drop one, drop all + + Each now goes through `pgcolumnar_scan_index_oid()`, so it probes the index + once the catalog reaches `pgcolumnar.index_min_blocks` pages and reads the heap + below that, as the earlier conversions do (#1213, #1217). Measured on PG 17 + with 400 other projected tables in the database, catalog buffers per + operation, before and after: + + one-row DELETE row_group 45 -> 19 + INSERT storage 19 -> 13 + vacuum_sorted storage 128 -> 93 + compact_rewrite free_space 43 -> 37 + truncate() free_space 26 -> 17 + RENAME COLUMN projection_declaration 9 -> 4 + + `test/catalog_scan_index.sh` and its pytest twin drive eleven operations + that between them reach all twelve sites. Each operation runs after the + population has put its catalog above the threshold, and each arm asserts the + operation made no sequential scan of that catalog. That is a path assertion, + which `catalog_delete_index.sh` warns against, and it is used here only where + a premise shows that declining the probe cannot be correct. Two cost-based + versions were tried first and a revert passed both. The full revert reddens + all eleven arms on PG 15 through 19. Reverting each site on its own reddens + the arm for the operation that reaches it. + + Two side effects, neither a behaviour anyone relied on. + `PgColumnarAllocateFreeSpace` picks the smallest range that fits, and a tie + now goes to the lowest file offset instead of heap order. Projection + declarations are listed in name order instead of heap order. + + The `server_dir` pytest fixture moved to `conftest.py`, because a second file + now needs a directory the server can write. + - `CREATE MATERIALIZED VIEW ... USING pgcolumnar ... WITH DATA` no longer orphans `pgcolumnar.storage.relation_oid` (#1275). The statement builds a transient relation, fills it and swaps, exactly as a rewriting `ALTER` does, diff --git a/src/columnar_metadata.c b/src/columnar_metadata.c index da2c31db..83a855fe 100644 --- a/src/columnar_metadata.c +++ b/src/columnar_metadata.c @@ -1028,6 +1028,7 @@ bool PgColumnarAllocateFreeSpace(uint64 storageId, uint64 dataLength, TransactionId oldestXmin, uint64 *fileOffset) { + Oid scanIdx; Relation rel = open_columnar_table("free_space", RowExclusiveLock); TupleDesc td = RelationGetDescr(rel); ScanKeyData key[1]; @@ -1044,7 +1045,8 @@ PgColumnarAllocateFreeSpace(uint64 storageId, uint64 dataLength, snap = RegisterSnapshot(GetLatestSnapshot()); ScanKeyInit(&key[0], Anum_free_space_storage_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum((int64) storageId)); - scan = systable_beginscan(rel, InvalidOid, false, snap, 1, key); + scanIdx = pgcolumnar_scan_index_oid(rel, "free_space_pkey"); + scan = systable_beginscan(rel, scanIdx, OidIsValid(scanIdx), snap, 1, key); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { bool isnull; @@ -1245,6 +1247,7 @@ bool PgColumnarTrailingFreeSpaceSafe(uint64 storageId, uint64 liveEnd, TransactionId oldestXmin) { + Oid scanIdx; Relation rel = open_columnar_table("free_space", AccessShareLock); TupleDesc td = RelationGetDescr(rel); Snapshot snap = RegisterSnapshot(GetLatestSnapshot()); @@ -1255,7 +1258,8 @@ PgColumnarTrailingFreeSpaceSafe(uint64 storageId, uint64 liveEnd, ScanKeyInit(&key[0], Anum_free_space_storage_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum((int64) storageId)); - scan = systable_beginscan(rel, InvalidOid, false, snap, 1, key); + scanIdx = pgcolumnar_scan_index_oid(rel, "free_space_pkey"); + scan = systable_beginscan(rel, scanIdx, OidIsValid(scanIdx), snap, 1, key); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { bool isnull; @@ -1289,6 +1293,7 @@ PgColumnarTrailingFreeSpaceSafe(uint64 storageId, uint64 liveEnd, void PgColumnarDeleteFreeSpaceAtOrAbove(uint64 storageId, uint64 liveEnd) { + Oid scanIdx; Relation rel = open_columnar_table("free_space", RowExclusiveLock); TupleDesc td = RelationGetDescr(rel); Snapshot snap = RegisterSnapshot(GetLatestSnapshot()); @@ -1300,7 +1305,8 @@ PgColumnarDeleteFreeSpaceAtOrAbove(uint64 storageId, uint64 liveEnd) ScanKeyInit(&key[0], Anum_free_space_storage_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum((int64) storageId)); - scan = systable_beginscan(rel, InvalidOid, false, snap, 1, key); + scanIdx = pgcolumnar_scan_index_oid(rel, "free_space_pkey"); + scan = systable_beginscan(rel, scanIdx, OidIsValid(scanIdx), snap, 1, key); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { bool isnull; @@ -1748,6 +1754,7 @@ delete_vector_lock_chunk_group(uint64 storageId, uint64 stripeId, int chunkId) static bool row_group_exists(uint64 storageId, uint64 groupNumber, Snapshot snapshot) { + Oid scanIdx; Relation rel = open_columnar_table("row_group", AccessShareLock); ScanKeyData key[2]; SysScanDesc scan; @@ -1757,7 +1764,8 @@ row_group_exists(uint64 storageId, uint64 groupNumber, Snapshot snapshot) F_INT8EQ, Int64GetDatum((int64) storageId)); ScanKeyInit(&key[1], Anum_row_group_group_number, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum((int64) groupNumber)); - scan = systable_beginscan(rel, InvalidOid, false, snapshot, 2, key); + scanIdx = pgcolumnar_scan_index_oid(rel, "row_group_pkey"); + scan = systable_beginscan(rel, scanIdx, OidIsValid(scanIdx), snapshot, 2, key); found = HeapTupleIsValid(systable_getnext(scan)); systable_endscan(scan); table_close(rel, AccessShareLock); @@ -2031,6 +2039,7 @@ columnar_relation_is_new_in_xact(Oid relid) void PgColumnarInsertNativeStorageRow(const NativeStorageMetadata *s) { + Oid scanIdx; Relation rel = open_columnar_table("storage", RowExclusiveLock); TupleDesc tupdesc = RelationGetDescr(rel); Datum values[Natts_native_storage]; @@ -2074,7 +2083,8 @@ PgColumnarInsertNativeStorageRow(const NativeStorageMetadata *s) snapshot = PgColumnarCatalogSnapshot(GetActiveSnapshot()); ScanKeyInit(&key[0], Anum_native_storage_storage_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum((int64) s->storageId)); - scan = systable_beginscan(rel, InvalidOid, false, snapshot, 1, key); + scanIdx = pgcolumnar_scan_index_oid(rel, "storage_pkey"); + scan = systable_beginscan(rel, scanIdx, OidIsValid(scanIdx), snapshot, 1, key); exists = HeapTupleIsValid(systable_getnext(scan)); systable_endscan(scan); PopActiveSnapshot(); @@ -2112,7 +2122,8 @@ PgColumnarInsertNativeStorageRow(const NativeStorageMetadata *s) snapshot = PgColumnarCatalogSnapshot(GetActiveSnapshot()); ScanKeyInit(&key[0], Anum_native_storage_storage_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum((int64) s->storageId)); - scan = systable_beginscan(rel, InvalidOid, false, snapshot, 1, key); + scanIdx = pgcolumnar_scan_index_oid(rel, "storage_pkey"); + scan = systable_beginscan(rel, scanIdx, OidIsValid(scanIdx), snapshot, 1, key); exists = HeapTupleIsValid(systable_getnext(scan)); systable_endscan(scan); if (exists) @@ -2144,7 +2155,8 @@ PgColumnarInsertNativeStorageRow(const NativeStorageMetadata *s) snapshot = PgColumnarCatalogSnapshot(GetActiveSnapshot()); ScanKeyInit(&key[0], Anum_native_storage_storage_id, BTEqualStrategyNumber, F_INT8EQ, Int64GetDatum((int64) s->storageId)); - scan = systable_beginscan(rel, InvalidOid, false, snapshot, 1, key); + scanIdx = pgcolumnar_scan_index_oid(rel, "storage_pkey"); + scan = systable_beginscan(rel, scanIdx, OidIsValid(scanIdx), snapshot, 1, key); exists = HeapTupleIsValid(systable_getnext(scan)); systable_endscan(scan); PopActiveSnapshot(); @@ -2474,6 +2486,7 @@ void PgColumnarSetSortedExtent(uint64 storageId, int64 firstGroup, int64 lastGroup, List *sortByNames, const char *sortedKind) { + Oid scanIdx; Relation rel; TupleDesc tupdesc; ScanKeyData key[1]; @@ -2513,7 +2526,8 @@ PgColumnarSetSortedExtent(uint64 storageId, int64 firstGroup, int64 lastGroup, F_INT8EQ, Int64GetDatum((int64) storageId)); /* NULL snapshot -> catalog snapshot, as the other read-side scans use. */ - scan = systable_beginscan(rel, InvalidOid, false, NULL, 1, key); + scanIdx = pgcolumnar_scan_index_oid(rel, "storage_pkey"); + scan = systable_beginscan(rel, scanIdx, OidIsValid(scanIdx), NULL, 1, key); tuple = systable_getnext(scan); if (HeapTupleIsValid(tuple)) { @@ -4071,6 +4085,7 @@ PgColumnarInsertProjectionRow(const PgColumnarProjection *proj) List * PgColumnarListProjectionDeclarations(Oid relid) { + Oid scanIdx; Relation rel = open_columnar_table("projection_declaration", AccessShareLock); TupleDesc tupdesc = RelationGetDescr(rel); @@ -4082,7 +4097,8 @@ PgColumnarListProjectionDeclarations(Oid relid) ScanKeyInit(&key[0], Anum_projection_declaration_rel, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); - scan = systable_beginscan(rel, InvalidOid, false, NULL, 1, key); + scanIdx = pgcolumnar_scan_index_oid(rel, "projection_declaration_pkey"); + scan = systable_beginscan(rel, scanIdx, OidIsValid(scanIdx), NULL, 1, key); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { PgColumnarProjectionDeclaration *d = @@ -4213,6 +4229,7 @@ void PgColumnarRenameProjectionDeclarationColumn(Oid relid, const char *oldName, const char *newName) { + Oid scanIdx; Relation rel; TupleDesc tupdesc; ScanKeyData key[1]; @@ -4223,7 +4240,8 @@ PgColumnarRenameProjectionDeclarationColumn(Oid relid, const char *oldName, tupdesc = RelationGetDescr(rel); ScanKeyInit(&key[0], Anum_projection_declaration_rel, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); - scan = systable_beginscan(rel, InvalidOid, false, NULL, 1, key); + scanIdx = pgcolumnar_scan_index_oid(rel, "projection_declaration_pkey"); + scan = systable_beginscan(rel, scanIdx, OidIsValid(scanIdx), NULL, 1, key); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { @@ -4288,6 +4306,7 @@ PgColumnarRenameProjectionDeclarationColumn(Oid relid, const char *oldName, void PgColumnarDeleteProjectionDeclaration(Oid relid, const char *name) { + Oid scanIdx; Relation rel = open_columnar_table("projection_declaration", RowExclusiveLock); ScanKeyData key[1]; @@ -4298,7 +4317,8 @@ PgColumnarDeleteProjectionDeclaration(Oid relid, const char *name) ScanKeyInit(&key[0], Anum_projection_declaration_rel, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); - scan = systable_beginscan(rel, InvalidOid, false, NULL, 1, key); + scanIdx = pgcolumnar_scan_index_oid(rel, "projection_declaration_pkey"); + scan = systable_beginscan(rel, scanIdx, OidIsValid(scanIdx), NULL, 1, key); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { bool isnull; @@ -4329,6 +4349,7 @@ PgColumnarDeleteProjectionDeclaration(Oid relid, const char *name) void PgColumnarDeleteProjectionDeclarationsForRel(Oid relid) { + Oid scanIdx; Relation rel = open_columnar_table("projection_declaration", RowExclusiveLock); ScanKeyData key[1]; @@ -4338,7 +4359,8 @@ PgColumnarDeleteProjectionDeclarationsForRel(Oid relid) ScanKeyInit(&key[0], Anum_projection_declaration_rel, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); - scan = systable_beginscan(rel, InvalidOid, false, NULL, 1, key); + scanIdx = pgcolumnar_scan_index_oid(rel, "projection_declaration_pkey"); + scan = systable_beginscan(rel, scanIdx, OidIsValid(scanIdx), NULL, 1, key); while (HeapTupleIsValid(tuple = systable_getnext(scan))) CatalogTupleDelete(rel, &tuple->t_self); systable_endscan(scan); diff --git a/test/catalog_scan_index.sh b/test/catalog_scan_index.sh new file mode 100755 index 00000000..e61ead60 --- /dev/null +++ b/test/catalog_scan_index.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash +# +# Twelve metadata scans must not cost more because the database holds other +# columnar tables (#1207). +# +# THE BUG. Twelve systable_beginscan calls in columnar_metadata.c passed +# InvalidOid on a key that an existing index starts with, so each one read its +# whole catalog to answer a question about one table: +# +# free_space storage_id :AllocateFreeSpace, :TrailingFreeSpaceSafe, +# :DeleteFreeSpaceAtOrAbove +# row_group storage_id, row_group_exists -- EVERY DELETE +# group_number +# storage storage_id three in InsertNativeStorageRow, +# one in SetSortedExtent +# projection_declaration rel list, rename column, drop one, drop all +# +# The pgcolumnar catalogs are SHARED by every columnar table in the database, +# so each of those reads was charged for every other table's rows. Measured on +# a database of 300 projected columnar tables, a one-row DELETE read 35 row_group +# buffers; through the index, 11. +# +# THE FIX ROUTES EACH ONE THROUGH pgcolumnar_scan_index_oid(), which probes only +# when the catalog is at least pgcolumnar.index_min_blocks pages (#1213, #1217). +# +# WHAT IS ASSERTED, AND WHY IT IS A PATH AND NOT A COST. catalog_delete_index.sh +# explains why that suite asserts buffers and never seq_scan: a build that +# DECLINES a probe on a small catalog is correct, and a path assertion fires on +# it. That objection holds only where declining can be right. Here every arm +# runs after 400 other projected tables have put each target catalog AT OR ABOVE +# the threshold, and a premise asserts it against the live setting. On such a +# catalog the resolver never declines, so a sequential scan of it during the +# operation is one of these sites still reading whole -- the defect, and +# nothing else. +# +# TWO COST MEASURES WERE TRIED FIRST, AND A REVERT SURVIVED BOTH. +# +# default against index_min_blocks = huge. A revert passed four arms of +# nine. The setting also governs the scans converted BEFORE this change +# (#1213, #1237), so forcing it made those read whole too, and the forced +# reading was worse than the unfixed code's. It measured the setting. +# +# growth from an empty database to the populated one. A probe costs about +# three buffers where a one-page catalog costs one, and vacuum_sorted makes +# 28 storage scans, so the FIXED code grew by more than the pages added. +# Separating the two needs catalogs near a hundred pages. +# +# The buffers are printed beside each arm, as diagnosis. +# +# Usage: test/catalog_scan_index.sh [PG_CONFIG] + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +# pgcolumnar.parallel_copy commits its loaders by two-phase commit and runs them +# as background workers; both settings are PGC_POSTMASTER (parallel_copy.sh). +export PGC_EXTRA_CONF=$'max_prepared_transactions=8\nmax_worker_processes=16' +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + + + +# at_least GOT FLOOR -- GOT when it falls short of FLOOR, else FLOOR. An arm +# comparing this against FLOOR passes exactly when GOT >= FLOOR, and a failing +# one reports the number it measured rather than a bare 0 (#1164, selftest +# part 540). Defined above the first recorder call; catalog_plan_index.sh says +# why the placement matters. +at_least() { [ "$1" -lt "$2" ] && echo "$1" || echo "$2"; } + +q "CREATE EXTENSION IF NOT EXISTS pgcolumnar;" >/dev/null +pages_of() { q "SELECT pg_relation_size('pgcolumnar.$1') / 8192;"; } +min_blocks="$(q "SHOW pgcolumnar.index_min_blocks;")" +echo "-- pgcolumnar.index_min_blocks=$(q "SHOW pgcolumnar.index_min_blocks;") (the shipped default this run measures)" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +# make_target TABLE NAME -- one measured table, for the arm called NAME: seven 1000-row groups and a +# projection over both columns, shaped so every operation below has work to do: +# +# group 0 retired a free range, so the rewrite can reuse one +# group 5 every other row dead a group compact_rewrite will write +# group 6 retired a TRAILING free range, so truncate() has a +# tail to remove +# +# With TARGET_PROJ=0 the projection is left off, for the end-truncation arm +# alone: a projection is its own storage IN THE SAME FILE, written after the +# base's groups, so its live data sits behind group 6 and there is no tail to +# reclaim. Measured: with the projection truncate() returns 0 and both free +# ranges stay; without it, 3 blocks and one. +TARGET_PROJ=1 +make_target() { + q "CREATE TABLE $1 (a int, b text) USING pgcolumnar; + SELECT pgcolumnar.set_options('$1', stripe_row_limit => 1000); + INSERT INTO $1 SELECT g, md5(g::text) FROM generate_series(1,7000) g; + DELETE FROM $1 WHERE a <= 1000 OR a > 6000;" >/dev/null + [ "$TARGET_PROJ" = 1 ] && + q "SELECT pgcolumnar.add_projection('$1', 'p', '{a,b}', '{a}');" >/dev/null + # ITS OWN STATEMENT, AND THAT IS LOAD-BEARING. One q is one psql -c, which + # runs as one transaction, and compact() cannot retire a group whose rows + # were deleted by a transaction that has not committed. In one q with the + # DELETE it retired nothing and every free_space arm measured an empty + # table's free list: the premises below are what caught it. + q "SELECT pgcolumnar.compact('$1');" >/dev/null + q "DELETE FROM $1 WHERE a BETWEEN 5001 AND 6000 AND a % 2 = 0;" >/dev/null + check_num "premise: the $2 target holds two free ranges, one of them trailing" \ + "$(q "SELECT count(*) FROM pgcolumnar.free_space WHERE storage_id = pgcolumnar.get_storage_id('$1');")" "2" +} + +# work CATALOG MIN_BLOCKS SQL -- "buffers seq_scan idx_scan" for CATALOG while +# SQL ran with index_min_blocks at MIN_BLOCKS. Buffers are heap AND index. SQL may hold several statements +# separated by ";;". +# +# ONE SESSION, ONE -c PER STATEMENT, and both halves matter. The SET has to be +# in the session that runs the statement. But one -c holding several statements +# is ONE implicit transaction, and pgcolumnar.truncate() refuses to run inside a +# transaction block -- it raised, q() discarded the error, and the arm read +# zero buffers for an operation that never ran. Separate -c arguments run in +# one session as separate transactions. +work() { + local -a cmds=(-c "SET pgcolumnar.index_min_blocks = $2") + local rest="$3" stmt + while [ -n "$rest" ]; do + stmt="${rest%%;;*}" + cmds+=(-c "$stmt") + [ "$stmt" = "$rest" ] && rest="" || rest="${rest#*;;}" + done + q "SELECT pg_stat_reset();" >/dev/null + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -At "${cmds[@]}" >/dev/null 2>&1 + q "SELECT pg_stat_force_next_flush();" >/dev/null + q "SELECT (coalesce(io.heap_blks_read + io.heap_blks_hit + + coalesce(io.idx_blks_read,0) + coalesce(io.idx_blks_hit,0), 0))::text + || ' ' || coalesce(st.seq_scan,0)::text || ' ' || coalesce(st.idx_scan,0)::text + FROM pg_statio_all_tables io JOIN pg_stat_all_tables st USING (relid) + WHERE io.schemaname = 'pgcolumnar' AND io.relname = '$1';" +} + +# arm NAME CATALOG SQL EFFECT WANT -- run SQL (@T is the table) on a fresh +# target and assert it did not read CATALOG whole. +# +# EFFECT IS A PREMISE, AND IT IS NOT OPTIONAL. q() discards errors, so a +# statement that failed outright still leaves a reading behind, and "no +# sequential scan" is exactly what a statement that never ran reports. Both +# kinds of silent failure happened while this file was written (a compaction +# that could not retire, a truncate() that refused a transaction block), and +# this premise and the next are what caught each. +N_ARM=0 +arm() { + local name="$1" cat="$2" tmpl="$3" effect="$4" want="$5" + N_ARM=$((N_ARM + 1)) + local t="arm${N_ARM}" r bufs seq idx + make_target "$t" "$name" + r="$(work "$cat" "$min_blocks" "${tmpl//@T/$t}")" + read -r bufs seq idx <<<"$r" + echo "-- $name: pgcolumnar.$cat buffers=$bufs seq_scan=$seq idx_scan=$idx" + check_text "premise: $name took effect" "$(q "${effect//@T/$t}")" "$want" + check_num "premise: $name reached pgcolumnar.$cat" \ + "$(at_least "$((${seq:-0} + ${idx:-0}))" 1)" "1" + check_num "$name does not read pgcolumnar.$cat whole" "${seq:-x}" "0" +} + +# --------------------------------------------------------------------------- +# The population: other columnar tables in the same catalogs +# --------------------------------------------------------------------------- +# +# Each noise table holds four 1000-row groups, a projection, and two retired +# groups that are NOT adjacent (0 and 2), so each leaves two free_space rows +# rather than one coalesced one. That is what puts every target catalog above +# the threshold: a contiguous delete coalesces, and a scattered one retires +# nothing and leaves free_space empty -- the fixture that hid this catalog on +# #1207. +NOISE=400 +G="$PGC_WORKDIR/noise.sql" +: > "$G" +for i in $(seq 1 $NOISE); do + cat >> "$G" < 1000); +INSERT INTO noise_$i SELECT g, md5(g::text) FROM generate_series(1,4000) g; +SELECT pgcolumnar.add_projection('noise_$i', 'p', '{a}', '{a}'); +DELETE FROM noise_$i WHERE (a - 1) / 1000 IN (0, 2); +SELECT pgcolumnar.compact('noise_$i'); +SQL +done +psql_file "$G" >/dev/null + +# EACH PREMISE IS THE CONDITION ITS ARMS NEED. A catalog still below the +# threshold in the big database is read whole by design (#1213), it grows by +# its pages, and the arm would fail for a reason that says nothing about the +# code. +for c in storage row_group free_space projection_declaration; do + p="$(pages_of "$c")" + echo "-- pgcolumnar.$c: $p pages" + check_num "premise: pgcolumnar.$c is at least index_min_blocks pages" \ + "$(at_least "$p" "$min_blocks")" "$min_blocks" +done + +# --------------------------------------------------------------------------- +# The arms, one per operation that reaches a converted site +# --------------------------------------------------------------------------- + +# 7000 rows, minus groups 0 and 6, minus half of group 5: 4500 live. +arm "an insert into an existing table" storage \ + "INSERT INTO @T SELECT g, 'x' FROM generate_series(7001,7100) g;" \ + "SELECT count(*) FROM @T;" "4600" +# THE SAME FUNCTION, TWO MORE OF ITS THREE SCANS. A table created in the same +# transaction takes the unlocked existence check (:2115 on be31874); a +# parallel_copy loader, which sets pgcolumnar_bulk_parallel_writer, takes the +# one before it (:2077). Each is its own arm because each is its own scan. +arm "an insert into a table created in the same transaction" storage \ + "BEGIN; CREATE TABLE @T_new (a int, b text) USING pgcolumnar; INSERT INTO @T_new SELECT g, 'x' FROM generate_series(1,100) g; COMMIT;" \ + "SELECT count(*) FROM @T_new;" "100" +PCDIR="$PGC_WORKDIR/pcopy" +mkdir -p "$PCDIR"; chmod 777 "$PCDIR" +if [ "$(id -u)" = "0" ]; then chown postgres "$PCDIR"; fi +PCF="$PCDIR/load.txt" +q "COPY (SELECT g, 'p' || g FROM generate_series(10001,12000) g) TO '$PCF' WITH (FORMAT text);" >/dev/null +arm "a parallel bulk load" storage \ + "SELECT pgcolumnar.parallel_copy('@T', '$PCF', 2);" \ + "SELECT count(*) FROM @T;" "6500" +arm "a one-row delete" row_group \ + "DELETE FROM @T WHERE a = 1500;" \ + "SELECT count(*) FROM @T;" "4499" +arm "an online rewrite" free_space \ + "SELECT pgcolumnar.compact_rewrite('@T', 0.2, 0);" \ + "SELECT sum(deletedrows) FROM pgcolumnar.stats('@T');" "0" +# Two free ranges before (groups 0 and 6); the trailing one is removed. +TARGET_PROJ=0 +arm "an end truncation" free_space \ + "SET pgcolumnar.enable_end_truncation = on;;SELECT pgcolumnar.truncate('@T');" \ + "SELECT count(*) FROM pgcolumnar.free_space WHERE storage_id = pgcolumnar.get_storage_id('@T');" "1" +TARGET_PROJ=1 +arm "a sorted rewrite" storage \ + "SELECT pgcolumnar.vacuum_sorted('@T', 'a');" \ + "SELECT sorted_through IS NOT NULL FROM pgcolumnar.storage WHERE storage_id = pgcolumnar.get_storage_id('@T');" "t" +arm "renaming a projected column" projection_declaration \ + "ALTER TABLE @T RENAME COLUMN b TO bb;" \ + "SELECT count(*) FROM pgcolumnar.projection_declaration WHERE rel = '@T'::regclass AND columns::text LIKE '%bb%';" "1" +arm "truncating a projected table" projection_declaration \ + "TRUNCATE @T;" \ + "SELECT count(*) FROM @T;" "0" +arm "dropping a projection" projection_declaration \ + "SELECT pgcolumnar.drop_projection('@T', 'p');" \ + "SELECT count(*) FROM pgcolumnar.projection_declaration WHERE rel = '@T'::regclass;" "0" +arm "dropping a projected table" projection_declaration \ + "DROP TABLE @T;" \ + "SELECT to_regclass('@T') IS NULL;" "t" + +# The arms are eleven calls of one function; a refactor that dropped one would +# leave every remaining arm green. +check_num "premise: eleven operations are measured" "$N_ARM" "11" + +pgc_summary diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 7975663e..09b6d615 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -122,6 +122,55 @@ catalog_plan_index catalog_plan_index premise: this query shape is priced differ catalog_plan_index catalog_plan_index the newest columnar table costs no more catalog work than the oldest 15;16;17;18;19 never - catalog_plan_index catalog_plan_index the planned cost does not depend on which storage row the heap returns first 15;16;17;18;19 never - catalog_plan_index catalog_plan_index with the catalog populated, planning costs less than reading it whole 15;16;17;18;19 2026-09-24 pgcolumnar_scan_index_oid stops asking the relation its size;the seven planner sites back to the unconditional pgcolumnar_index_oid +catalog_scan_index catalog_scan_index a one-row delete does not read pgcolumnar.row_group whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole +catalog_scan_index catalog_scan_index a parallel bulk load does not read pgcolumnar.storage whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole +catalog_scan_index catalog_scan_index a sorted rewrite does not read pgcolumnar.storage whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole +catalog_scan_index catalog_scan_index an end truncation does not read pgcolumnar.free_space whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole +catalog_scan_index catalog_scan_index an insert into a table created in the same transaction does not read pgcolumnar.storage whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole +catalog_scan_index catalog_scan_index an insert into an existing table does not read pgcolumnar.storage whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole +catalog_scan_index catalog_scan_index an online rewrite does not read pgcolumnar.free_space whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole +catalog_scan_index catalog_scan_index dropping a projected table does not read pgcolumnar.projection_declaration whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole +catalog_scan_index catalog_scan_index dropping a projection does not read pgcolumnar.projection_declaration whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole +catalog_scan_index catalog_scan_index premise: a one-row delete reached pgcolumnar.row_group 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: a one-row delete took effect 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: a parallel bulk load reached pgcolumnar.storage 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: a parallel bulk load took effect 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: a sorted rewrite reached pgcolumnar.storage 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: a sorted rewrite took effect 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: an end truncation reached pgcolumnar.free_space 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: an end truncation took effect 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: an insert into a table created in the same transaction reached pgcolumnar.storage 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: an insert into a table created in the same transaction took effect 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: an insert into an existing table reached pgcolumnar.storage 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: an insert into an existing table took effect 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: an online rewrite reached pgcolumnar.free_space 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: an online rewrite took effect 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: dropping a projected table reached pgcolumnar.projection_declaration 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: dropping a projected table took effect 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: dropping a projection reached pgcolumnar.projection_declaration 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: dropping a projection took effect 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: eleven operations are measured 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: pgcolumnar.free_space is at least index_min_blocks pages 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: pgcolumnar.projection_declaration is at least index_min_blocks pages 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: pgcolumnar.row_group is at least index_min_blocks pages 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: pgcolumnar.storage is at least index_min_blocks pages 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: renaming a projected column reached pgcolumnar.projection_declaration 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: renaming a projected column took effect 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: the a one-row delete target holds two free ranges, one of them trailing 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: the a parallel bulk load target holds two free ranges, one of them trailing 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: the a sorted rewrite target holds two free ranges, one of them trailing 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: the an end truncation target holds two free ranges, one of them trailing 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: the an insert into a table created in the same transaction target holds two free ranges, one of them trailing 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: the an insert into an existing table target holds two free ranges, one of them trailing 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: the an online rewrite target holds two free ranges, one of them trailing 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: the dropping a projected table target holds two free ranges, one of them trailing 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: the dropping a projection target holds two free ranges, one of them trailing 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: the renaming a projected column target holds two free ranges, one of them trailing 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: the truncating a projected table target holds two free ranges, one of them trailing 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: truncating a projected table reached pgcolumnar.projection_declaration 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index premise: truncating a projected table took effect 15;16;17;18;19 never - +catalog_scan_index catalog_scan_index renaming a projected column does not read pgcolumnar.projection_declaration whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole +catalog_scan_index catalog_scan_index truncating a projected table does not read pgcolumnar.projection_declaration whole 15;16;17;18;19 2026-10-12 the twelve converted sites reverted to InvalidOid, so each reads its catalog whole differential differential agg avg 15;16;17;18;19 never - differential differential agg count 15;16;17;18;19 never - differential differential agg minmax 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index c187d40c..2a338303 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1177,4 +1177,16 @@ suites_not_covered 247 # REBASED FROM 1bcfb92b ONTO 81ec3ddc, 32 commits, by the maintainer rather than # the author. The branch predates #1261, and three of its five CI reds were # fixed by the rebase alone rather than by any change here -- see the PR. -checks_never_observed_red 1633 +# #1207 (OffgridwithJD): 1633 -> 1671. `catalog_scan_index` enters the ledger +# with 49 checks. The 11 path arms are dated by one mutation, the twelve +# converted sites reverted to InvalidOid, merged one major at a time from a +# five-major run. The 38 premises arrive `never`. +# +# ledger rows 1810 + 49 = 1859 +# never 1633 + 38 = 1671 +# dated 177 + 11 = 188 +# 1671 + 188 == 1859 +# +# suites_not_covered does not move: the suite is registered and covered in the +# same change. +checks_never_observed_red 1671 diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index a309e637..0b92e951 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -128,6 +128,7 @@ behaviour, the source of that number is named. - [80. test_catalog_plan_index.py: planning pays for these catalogs only when they hold something](#80-test_catalog_plan_indexpy-planning-pays-for-these-catalogs-only-when-they-hold-something) - [81. test_catalog_delete_index.py: retiring a row group costs no more for a bigger database](#81-test_catalog_delete_indexpy-retiring-a-row-group-costs-no-more-for-a-bigger-database) - [82. test_rewrite_storage_oid.py: a type rewrite keeps the storage row on the live table](#82-test_rewrite_storage_oidpy-a-type-rewrite-keeps-the-storage-row-on-the-live-table) +- [83. test_catalog_scan_index.py: twelve metadata scans do not read their catalog whole](#83-test_catalog_scan_indexpy-twelve-metadata-scans-do-not-read-their-catalog-whole) ## 1. How to read a test in here @@ -6297,3 +6298,25 @@ THE `REFRESH` ARM GUARDS THE PATH THAT ALREADY WORKED. It repaired this before t | --- | --- | | `test_rewrite_storage_oid` | the storage row matches the live regclass before the rewrite, the row count survives, an unrewritten table still matches itself, and the rewritten table's storage row matches the live regclass afterwards | | `test_a_matview_created_with_data_points_at_itself` | that the matview and the `CREATE TABLE AS` table both hold their rows, that `CREATE TABLE ... AS` points at itself (the control that narrows the fix), that a matview created `WITH DATA` points at itself, and that `REFRESH` still does | + +## 83. test_catalog_scan_index.py: twelve metadata scans do not read their catalog whole + +Port of `catalog_scan_index.sh` (#1207). Twelve `systable_beginscan` calls in `columnar_metadata.c` passed `InvalidOid` on a key that an existing index starts with, so each read its whole catalog to answer a question about one table. There are three on `free_space` (the compaction reuse and both end-truncation reads), one on `row_group` (`row_group_exists`, reached by EVERY `DELETE`), four on `storage` (three existence checks in `PgColumnarInsertNativeStorageRow` and the sorted-extent update), and four on `projection_declaration` (list, rename a column, drop one, drop all). The fix routes each through `pgcolumnar_scan_index_oid()`. + +IT ASSERTS THE PATH, AND SECTION 81 SAYS NOT TO, so the reason has to be stated. A path assertion is wrong where declining a probe can be right, which is on a catalog below `pgcolumnar.index_min_blocks`. Every arm here runs after 350 other projected tables have put all four target catalogs above that threshold, and a premise asserts it against the live setting. The resolver never declines on such a catalog, so a sequential scan of it during the operation can only be one of these sites still reading whole. Buffers and `idx_scan` are printed beside each arm as diagnosis. + +TWO COST MEASURES WERE TRIED FIRST, AND A REVERT SURVIVED BOTH. Comparing the default against `index_min_blocks` forced high let a full revert through on four arms of nine. The setting also governs the scans converted earlier (#1213, #1237), so forcing it made those read whole too, and the forced reading came out worse than the unfixed code's. Growth from an empty database to the populated one failed on correct code instead. A probe costs about three buffers where a one-page catalog costs one, and `vacuum_sorted` makes 28 `storage` scans, so the fixed build grew by more than the pages added. + +EVERY ARM HAS AN EFFECT PREMISE, AND BOTH SILENT FAILURES IT GUARDS AGAINST HAPPENED. `compact()` cannot retire rows deleted by a transaction that has not committed, so a fixture that deleted and compacted in one transaction left no free range to measure. `pgcolumnar.truncate()` refuses a transaction block, so a multi-statement call raised and read zero. "No sequential scan" is exactly what an operation that never ran reports. Each arm therefore asserts what the operation changed, and that it reached the catalog at all. + +THE END-TRUNCATION TARGET HAS NO PROJECTION. A projection is its own storage in the same file, written after the base's groups, so its live data sits behind the base's last group and there is no tail to reclaim. Measured: with the projection, `truncate()` returns 0 and both free ranges stay; without it, 3 blocks are reclaimed and one range is left. + +Removal proof, on the shell twin at PG 17: the full revert reddens all eleven path arms, and reverting each of the twelve sites ALONE reddens at least one arm named for the operation that reaches it. This twin aborts at its first failure, so under the revert it reports the first arm (`storage seq_scan=2`). + +The two halves share no code and no fixture. The shell suite adds 400 tables with `int` keys and retires groups 0 and 2; this one adds 350 with `bigint` keys and retires groups 1 and 3, and its effect values differ throughout. This half flushes statistics before each reset, for the reason section 81 gives. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_catalog_scans_do_not_read_their_catalog_whole` | each of the four catalogs at or above `index_min_blocks`; then, for eleven operations (an insert into an existing table, into a table created in the same transaction, a `parallel_copy` load, a one-row delete, `compact_rewrite`, `truncate()`, `vacuum_sorted`, renaming a projected column, `TRUNCATE`, `drop_projection`, `DROP TABLE`), that the target held two free ranges, that the operation took effect, that it reached its catalog, and that it made no sequential scan of it | diff --git a/test/pytest/conftest.py b/test/pytest/conftest.py index 2f352156..e113fc55 100644 --- a/test/pytest/conftest.py +++ b/test/pytest/conftest.py @@ -188,6 +188,24 @@ def _serve_cluster(cluster, root, verdict, worker_id): shutil.rmtree(root, ignore_errors=True) +@pytest.fixture(scope="module") +def server_dir(): + """A directory the SERVER can write, which `tmp_path` is not. + + `COPY ... TO` and `parallel_copy` are executed by the backend, and the backend runs + as a different user: pytest's `tmp_path` lives under `/tmp/pytest-of-root/` at mode + 700, so the server cannot reach it and the error names a permission rather than the + real cause. Opening the parents matters as much as the leaf -- one closed directory + above makes the subtree unreachable however open the leaf is -- so this makes its + own directory instead of trying to prise `tmp_path` open. + """ + import tempfile + d = pathlib.Path(tempfile.mkdtemp(prefix="pgc_server_dir_")) + os.chmod(d, 0o777) + yield d + shutil.rmtree(d, ignore_errors=True) + + @pytest.fixture def pgc_own_db(pgc_cluster, request): """A private DATABASE, where `pgc_conn` gives a private schema. diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index f17473f5..19d65eaf 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -543,4 +543,4 @@ guard_tests 404 # own test is not in it. Git conflicted again -- loudly, because the key moved # on both sides -- and the answer is neither branch's number. Collected on the # rebased tree, not added up. -cluster_tests 488 +cluster_tests 489 diff --git a/test/pytest/test_catalog_scan_index.py b/test/pytest/test_catalog_scan_index.py new file mode 100644 index 00000000..af72ff74 --- /dev/null +++ b/test/pytest/test_catalog_scan_index.py @@ -0,0 +1,197 @@ +"""Twelve metadata scans must not read their catalog whole (#1207). + +Twelve `systable_beginscan` calls in `columnar_metadata.c` passed `InvalidOid` on +a key an existing index starts with, so each read its whole catalog to answer a +question about one table: three on `free_space`, `row_group_exists` (reached by +EVERY DELETE), four on `storage`, four on `projection_declaration`. The +catalogs are shared by every columnar table in the database, so each read was +charged for every other table's rows. The fix routes each through +`pgcolumnar_scan_index_oid()`, which probes at or above +`pgcolumnar.index_min_blocks` pages. + +WHAT IS ASSERTED, AND WHY IT IS A PATH. `test_catalog_delete_index.py` asserts +buffers and never `seq_scan`, because a build that DECLINES a probe on a small +catalog is correct and a path assertion fires on it. That holds only where +declining can be right. Every arm here runs after 350 other projected tables +have put each target catalog AT OR ABOVE the threshold, and a premise asserts it +against the live setting; there the resolver never declines, so a sequential +scan during the operation is one of these sites still reading whole. Two cost +measures were tried first and a revert survived both; `catalog_scan_index.sh` +records how. + +Independent of `test/catalog_scan_index.sh`: its own population (bigint keys, +350 tables, groups 1 and 3 retired rather than 0 and 2), its own row counts, and +its own effect values. + +ON A PRIVATE DATABASE (`pgc_own_db`): the premises are about catalog size, and +must not depend on what other files left behind. `pg_stat_reset()` is per +database; this file must not run concurrently with another that reads +statistics. +""" + +NOISE = 350 + + +def _q(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchone()[0] if cur.description else None + + +def _make_target(conn, expect, t, name, projection=True): + """Seven 1000-row groups: + + group 0 retired a free range, so the rewrite can reuse one + group 5 every other row dead a group compact_rewrite will write + group 6 retired a TRAILING free range for truncate() + + and a projection over both columns, except for the end-truncation arm: a + projection is its own storage in the same file, written after the base's + groups, so it leaves no tail to reclaim. + + The compaction is its own statement on an autocommit connection: compact() + cannot retire rows deleted by a transaction that has not committed. + """ + with conn.cursor() as cur: + cur.execute(f"CREATE TABLE {t} (a bigint, b text) USING pgcolumnar") + cur.execute("SELECT pgcolumnar.set_options(%s, stripe_row_limit => 1000)", (t,)) + cur.execute(f"INSERT INTO {t} SELECT g, 'r' || g FROM generate_series(1,7000) g") + cur.execute(f"DELETE FROM {t} WHERE a <= 1000 OR a > 6000") + if projection: + cur.execute("SELECT pgcolumnar.add_projection(%s, 'p', '{a,b}', '{a}')", (t,)) + cur.execute("SELECT pgcolumnar.compact(%s)", (t,)) + cur.execute(f"DELETE FROM {t} WHERE a BETWEEN 5001 AND 6000 AND a % 2 = 1") + expect.num( + _q(conn, "SELECT count(*) FROM pgcolumnar.free_space " + f"WHERE storage_id = pgcolumnar.get_storage_id('{t}')"), + 2, + f"premise: the {name} target holds two free ranges, one of them trailing", + ) + + +def _work(conn, catalog, statements): + """(buffers, seq_scan, idx_scan) for `catalog` while `statements` ran. + + FLUSH BEFORE THE RESET: one connection holds this whole file, so the + fixture's own pending statistics would otherwise land on the reading + (measured on test_catalog_delete_index.py). Each statement is its own + transaction on the autocommit connection, which pgcolumnar.truncate() + requires. + """ + with conn.cursor() as cur: + cur.execute("SELECT pg_stat_force_next_flush()") + cur.execute("SELECT pg_stat_reset()") + for stmt in statements: + cur.execute(stmt) + cur.execute("RESET pgcolumnar.enable_end_truncation") + cur.execute("SELECT pg_stat_force_next_flush()") + cur.execute( + "SELECT coalesce(io.heap_blks_read,0) + coalesce(io.heap_blks_hit,0) " + " + coalesce(io.idx_blks_read,0) + coalesce(io.idx_blks_hit,0), " + " coalesce(st.seq_scan,0), coalesce(st.idx_scan,0) " + "FROM pg_statio_all_tables io JOIN pg_stat_all_tables st USING (relid) " + "WHERE io.schemaname = 'pgcolumnar' AND io.relname = %s", + (catalog,), + ) + row = cur.fetchone() + return tuple(int(v) for v in row) if row else (0, 0, 0) + + +def test_catalog_scans_do_not_read_their_catalog_whole(pgc_own_db, expect, server_dir): + conn = pgc_own_db + with conn.cursor() as cur: + for i in range(NOISE): + t = f"other_{i}" + cur.execute(f"CREATE TABLE {t} (a bigint, b text) USING pgcolumnar") + cur.execute("SELECT pgcolumnar.set_options(%s, stripe_row_limit => 1000)", (t,)) + cur.execute(f"INSERT INTO {t} SELECT g, 'n' || g FROM generate_series(1,4000) g") + cur.execute("SELECT pgcolumnar.add_projection(%s, 'p', '{a}', '{a}')", (t,)) + # Groups 1 and 3: NOT adjacent, so two free ranges rather than one. + cur.execute(f"DELETE FROM {t} WHERE (a - 1) / 1000 IN (1, 3)") + cur.execute("SELECT pgcolumnar.compact(%s)", (t,)) + for c in ("storage", "row_group", "free_space", "projection_declaration"): + cur.execute(f"VACUUM pgcolumnar.{c}") + + min_blocks = int(_q(conn, "SHOW pgcolumnar.index_min_blocks")) + for c in ("storage", "row_group", "free_space", "projection_declaration"): + pages = _q(conn, f"SELECT pg_relation_size('pgcolumnar.{c}') / 8192") + print(f"-- pgcolumnar.{c}: {pages} pages") + expect.at_least( + pages, + min_blocks, + f"premise: pgcolumnar.{c} is at least index_min_blocks pages", + ) + + load = str(server_dir / "scan_index_load.txt") + with conn.cursor() as cur: + cur.execute( + "COPY (SELECT g, 'q' || g FROM generate_series(20001,21500) g) " + "TO '%s' WITH (FORMAT text)" % load + ) + + # (name, catalog, statements with {t}, effect with {t}, want, projection) + # 7000 rows less groups 0 and 6 and half of group 5: 4500 live. + arms = [ + ("an insert into an existing table", "storage", + ["INSERT INTO {t} SELECT g, 'y' FROM generate_series(8001,8050) g"], + "SELECT count(*)::text FROM {t}", "4550", True), + ("an insert into a table created in the same transaction", "storage", + ["BEGIN", + "CREATE TABLE {t}_new (a bigint, b text) USING pgcolumnar", + "INSERT INTO {t}_new SELECT g, 'z' FROM generate_series(1,75) g", + "COMMIT"], + "SELECT count(*)::text FROM {t}_new", "75", True), + ("a parallel bulk load", "storage", + ["SELECT pgcolumnar.parallel_copy('{t}', '%s', 2)" % load], + "SELECT count(*)::text FROM {t}", "6000", True), + ("a one-row delete", "row_group", + ["DELETE FROM {t} WHERE a = 2500"], + "SELECT count(*)::text FROM {t}", "4499", True), + ("an online rewrite", "free_space", + ["SELECT pgcolumnar.compact_rewrite('{t}', 0.2, 0)"], + "SELECT sum(deletedrows)::text FROM pgcolumnar.stats('{t}')", "0", True), + ("an end truncation", "free_space", + ["SET pgcolumnar.enable_end_truncation = on", + "SELECT pgcolumnar.truncate('{t}')"], + "SELECT count(*)::text FROM pgcolumnar.free_space " + "WHERE storage_id = pgcolumnar.get_storage_id('{t}')", "1", False), + ("a sorted rewrite", "storage", + ["SELECT pgcolumnar.vacuum_sorted('{t}', 'a')"], + "SELECT (sorted_through IS NOT NULL)::text FROM pgcolumnar.storage " + "WHERE storage_id = pgcolumnar.get_storage_id('{t}')", "true", True), + ("renaming a projected column", "projection_declaration", + ["ALTER TABLE {t} RENAME COLUMN b TO b2"], + "SELECT count(*)::text FROM pgcolumnar.projection_declaration " + "WHERE rel = '{t}'::regclass AND 'b2' = ANY(columns)", "1", True), + ("truncating a projected table", "projection_declaration", + ["TRUNCATE {t}"], + "SELECT count(*)::text FROM {t}", "0", True), + ("dropping a projection", "projection_declaration", + ["SELECT pgcolumnar.drop_projection('{t}', 'p')"], + "SELECT count(*)::text FROM pgcolumnar.projection_declaration " + "WHERE rel = '{t}'::regclass", "0", True), + ("dropping a projected table", "projection_declaration", + ["DROP TABLE {t}"], + "SELECT (to_regclass('{t}') IS NULL)::text", "true", True), + ] + + # The arms below are a loop over a list, and a loop that never ran asserts + # nothing while the premises above still pass. + expect.num(len(arms), 11, "premise: eleven operations are measured") + + for n, (name, cat, stmts, effect, want, projection) in enumerate(arms, 1): + t = f"target_{n}" + _make_target(conn, expect, t, name, projection) + bufs, seq, idx = _work(conn, cat, [s.format(t=t) for s in stmts]) + print(f"-- {name}: pgcolumnar.{cat} buffers={bufs} seq_scan={seq} idx_scan={idx}") + expect.text( + _q(conn, effect.format(t=t)), + want, + f"premise: {name} took effect", + ) + expect.at_least( + seq + idx, + 1, + f"premise: {name} reached pgcolumnar.{cat}", + ) + expect.num(seq, 0, f"{name} does not read pgcolumnar.{cat} whole") diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 1d5681f2..98cab219 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -92,6 +92,7 @@ "base_scan_io", "catalog_delete_index", "catalog_plan_index", + "catalog_scan_index", "differential", "encode_post_codec", "hilbert_cluster", diff --git a/test/pytest/test_sorted_pathkeys.py b/test/pytest/test_sorted_pathkeys.py index c1c9a3f2..9b0e4898 100644 --- a/test/pytest/test_sorted_pathkeys.py +++ b/test/pytest/test_sorted_pathkeys.py @@ -720,22 +720,8 @@ def test_an_enum_add_value_before_does_not_renumber(fx, expect): # The appended rows carry k = -600..-1, all below the run, so a plan still claiming the # old order answers 0,0,0,0,0 where the truth is -600,-599,-598,-597,-596. -@pytest.fixture(scope="module") -def server_dir(): - """A directory the SERVER can write, which `tmp_path` is not. - - `COPY ... TO` and `parallel_copy` are executed by the backend, and the backend runs - as a different user: pytest's `tmp_path` lives under `/tmp/pytest-of-root/` at mode - 700, so the server cannot reach it and the error names a permission rather than the - real cause. Opening the parents matters as much as the leaf -- one closed directory - above makes the subtree unreachable however open the leaf is -- so this makes its - own directory instead of trying to prise `tmp_path` open. - """ - import os, shutil, tempfile - d = pathlib.Path(tempfile.mkdtemp(prefix="pgc_sorted_pathkeys_")) - os.chmod(d, 0o777) - yield d - shutil.rmtree(d, ignore_errors=True) +# `server_dir` moved to conftest.py when a second file needed it (#1207), as +# `pgc_own_db` did before it. WANT_FIRST5 = [-600, -599, -598, -597, -596] diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index a0f4485a..7ca80c94 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -64,6 +64,7 @@ SUITES=( catalog_delete_index catalog_natts catalog_plan_index + catalog_scan_index column_projection concurrency concurrent_diff