From ffaa3b0c948f232db84a4997aee2b9e00367db5b Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 21:50:38 +0000 Subject: [PATCH 1/5] test: a ctid range reads only the row groups that hold it, written first (#1294) --- test/tid_range_scan.sh | 68 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100755 test/tid_range_scan.sh diff --git a/test/tid_range_scan.sh b/test/tid_range_scan.sh new file mode 100755 index 000000000..ffe30e7cd --- /dev/null +++ b/test/tid_range_scan.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# +# A ctid range reads only the row groups that hold it (TID range scans). +# +# WRITTEN BEFORE THE CODE. Measured on #1294: `WHERE ctid >= '(600000,0)'` on a +# 10M-row table matched 0 rows and still read all 67 row groups, because the +# access method has no scan_set_tidrange and the range is a per-row filter. A +# rollup that processes "rows added since the last refresh" (#1294) needs to read +# the newest row groups only, and so does anything else that bounds ctid. +# +# THE ORACLE is the same query with TID scans disabled (enable_tidscan = off), +# which is the per-row filter every build applies today: a TID range scan must +# return exactly what that returns, for every shape of range. THE COST is the +# table's own blocks from pg_statio, flushed by the querying session -- an +# instrument that does not depend on which plan node answers. +# +# Usage: test/tid_range_scan.sh [PG_CONFIG] + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +at_least() { [ -z "$1" ] && return; [ "$1" -lt "$2" ] && echo "$1" || echo "$2"; } + +q "CREATE EXTENSION IF NOT EXISTS pgcolumnar;" >/dev/null +q "CREATE TABLE tr (id bigint, v text) USING pgcolumnar; + SELECT pgcolumnar.set_options('tr', stripe_row_limit => 10000); + INSERT INTO tr SELECT g, md5(g::text) FROM generate_series(1, 400000) g;" >/dev/null +check_num "premise: the table holds every row" "$(q "SELECT count(*) FROM tr;")" "400000" +groups="$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('tr');")" +check_num "premise: the table has many row groups, so skipping can show" "$(at_least "$groups" 30)" "30" + +# ctid of the row with a given id, as the AM numbers it +tid_of() { q "SELECT ctid FROM tr WHERE id = $1;"; } +LAST="$(tid_of 390001)"; MID_A="$(tid_of 150001)"; MID_B="$(tid_of 170000)"; END="$(tid_of 400000)" +echo "-- ctids: 390001=$LAST 150001=$MID_A 170000=$MID_B 400000=$END" + +hash() { q "SET enable_tidscan = ${2:-on}; SELECT md5(string_agg(id::text || v, ',' ORDER BY id)) || '|' || count(*) FROM tr WHERE $1;" | tail -1; } +for pred in "ctid >= '$LAST'" "ctid >= '$MID_A' AND ctid <= '$MID_B'" "ctid < '$MID_A'" \ + "ctid > '$END'" "ctid >= '(0,0)'"; do + check "a ctid range returns what the per-row filter returns: ${pred//$LAST/LAST}" \ + "$(hash "$pred" on)" "$(hash "$pred" off)" +done + +# max(v || 'x'), not max(v): an unfiltered count/max is answered from zone maps +# (#1293) and read 5 blocks of this table in the first draft, so the "whole +# table" baseline measured metadata, not data. +blocks() { # blocks PRED -> tr blocks read, in the querying session + q "SELECT pg_stat_reset();" >/dev/null + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -At \ + -c "SELECT count(*), max(v || 'x') FROM tr WHERE $1" -c "SELECT pg_stat_force_next_flush()" >/dev/null 2>&1 + q "SELECT coalesce(heap_blks_read, 0) + coalesce(heap_blks_hit, 0) FROM pg_statio_user_tables WHERE relname = 'tr';" +} +full="$(blocks "true")" +tail_b="$(blocks "ctid >= '$LAST'")" +echo "-- blocks: whole table $full, last 2.5% by ctid $tail_b" +check_num "premise: reading the whole table reads its blocks" "$(at_least "$full" 100)" "100" +check_num "a ctid range over the last row groups reads under a fifth of the table" \ + "$( [ -n "$tail_b" ] && [ "$tail_b" -ge 1 ] && at_least $((full - 5 * tail_b)) 1 || echo "read ${tail_b:-nothing}")" "1" + +# Deleted rows inside the range stay invisible. +q "DELETE FROM tr WHERE id BETWEEN 395000 AND 395999;" >/dev/null +check "and rows deleted inside the range stay deleted" \ + "$(hash "ctid >= '$LAST'" on)" "$(hash "ctid >= '$LAST'" off)" +check_num "premise: the delete removed rows from the range" \ + "$(q "SET enable_tidscan = off; SELECT count(*) FROM tr WHERE ctid >= '$LAST';" | tail -1)" "9000" + +pgc_summary From 92c649ebaba1dd55b06b874d356b2b2db005099b Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 22:02:28 +0000 Subject: [PATCH 2/5] feat: a ctid range reads only the row groups that hold it (TID range scans) scan_set_tidrange maps the bounds to row numbers, clamping offsets no row uses (0, and past the 291 a block holds: the executor turns ctid < '(B,0)' into (B-1,65535)), and restricts the reader to the intersecting row groups. getnextslot_tidrange filters the boundary groups' rows. A range over the last 2.5% of a 400k-row table reads 28 blocks where the table is 923. PostgreSQL 19's parallel TID range scan shares the reader's group claim. Co-Authored-By: Claude Opus 5.5 --- src/columnar_tableam.c | 144 +++++++++++++++++++++++++++++++++++++++++ test/tid_range_scan.sh | 64 +++++++++++++++++- 2 files changed, 207 insertions(+), 1 deletion(-) diff --git a/src/columnar_tableam.c b/src/columnar_tableam.c index 0aae63190..0c5912b9e 100644 --- a/src/columnar_tableam.c +++ b/src/columnar_tableam.c @@ -289,6 +289,18 @@ typedef struct PgColumnarScanDescData */ MemoryContext scanContext; PgColumnarAnalyzeState *analyzeState; + + /* + * A TID range scan: the inclusive row-number bounds scan_set_tidrange + * derived from the requested item pointers, and the row groups that + * intersect them. The reader is restricted to those groups when it is + * built, so a range over the tail of a table reads the tail only. + */ + bool hasRowRange; + uint64 minRow; + uint64 maxRow; + uint64 *rangeGroups; + int numRangeGroups; } PgColumnarScanDescData; typedef struct PgColumnarScanDescData *PgColumnarScanDesc; @@ -658,6 +670,9 @@ pgcolumnar_scan_read_state(PgColumnarScanDesc scan, TupleDesc tupdesc) scan->rs_base.rs_parallel, NULL, scan->rs_base.rs_nkeys, scan->rs_base.rs_key); + if (scan->hasRowRange) + PgColumnarReadRestrictToGroups(scan->readState, scan->rangeGroups, + scan->numRangeGroups); MemoryContextSwitchTo(oldContext); } @@ -749,6 +764,133 @@ pgcolumnar_scan_getnextslot(TableScanDesc sscan, ScanDirection direction, return true; } +/* + * pgcolumnar_scan_set_tidrange + * Bound the scan to the item pointers [mintid, maxtid], inclusive. + * + * Item pointers here are synthetic row addresses (columnar.h), so the bounds + * become row numbers and the row groups that hold any of them are found from + * the row-group catalog at the scan's snapshot. Rows outside the range in a + * boundary group are filtered by getnextslot_tidrange. + * + * Called before the first getnextslot_tidrange and again after each rescan, + * so a reader built for an earlier range is ended here; the next row builds a + * fresh one restricted to the new groups. + */ +static void +pgcolumnar_scan_set_tidrange(TableScanDesc sscan, ItemPointer mintid, + ItemPointer maxtid) +{ + PgColumnarScanDesc scan = (PgColumnarScanDesc) sscan; + uint64 lo; + uint64 hi; + List *groups; + ListCell *lc; + MemoryContext oldContext; + + if (scan->readState != NULL) + { + PgColumnarEndRead(scan->readState); + scan->readState = NULL; + } + + /* + * A bound may name an offset no row uses: 0, or anything past the + * COLUMNAR_VALID_ITEMPOINTER_OFFSETS a block holds -- the executor turns + * "ctid < '(B,0)'" into an upper bound of (B-1, 65535). Rows of block B + * are B*K + (offset - 1) for offset 1..K, so a lower bound below offset 1 + * means the block's first row and one past K the next block's first row; + * an upper bound past K means the block's last row and one below offset 1 + * the row before the block, which is no row at all for block 0. + */ + { + uint64 k = COLUMNAR_VALID_ITEMPOINTER_OFFSETS; + uint64 blk = ItemPointerGetBlockNumberNoCheck(mintid); + uint64 off = ItemPointerGetOffsetNumberNoCheck(mintid); + + if (off < FirstOffsetNumber) + lo = blk * k; + else if (off > k) + lo = (blk + 1) * k; + else + lo = blk * k + (off - FirstOffsetNumber); + + blk = ItemPointerGetBlockNumberNoCheck(maxtid); + off = ItemPointerGetOffsetNumberNoCheck(maxtid); + if (off > k) + hi = blk * k + (k - 1); + else if (off >= FirstOffsetNumber) + hi = blk * k + (off - FirstOffsetNumber); + else if (blk > 0) + hi = blk * k - 1; + else + { + /* below row 0, which no row uses: an empty range */ + lo = 1; + hi = 0; + } + } + + scan->hasRowRange = true; + scan->minRow = lo; + scan->maxRow = hi; + + oldContext = MemoryContextSwitchTo(scan->scanContext); + if (scan->rangeGroups != NULL) + pfree(scan->rangeGroups); + scan->numRangeGroups = 0; + groups = (lo <= hi) ? + PgColumnarReadRowGroupList(PgColumnarStorageId(sscan->rs_rd), + sscan->rs_snapshot) : NIL; + scan->rangeGroups = (uint64 *) palloc(sizeof(uint64) * + Max(list_length(groups), 1)); + foreach(lc, groups) + { + NativeRowGroupMetadata *rg = (NativeRowGroupMetadata *) lfirst(lc); + + if (rg->rowCount == 0 || + rg->firstRowNumber > hi || + rg->firstRowNumber + rg->rowCount - 1 < lo) + continue; + scan->rangeGroups[scan->numRangeGroups++] = rg->groupNumber; + } + list_free_deep(groups); + MemoryContextSwitchTo(oldContext); +} + +/* + * pgcolumnar_scan_getnextslot_tidrange + * The next row whose item pointer is inside the range set above. + * + * The reader visits only the groups that intersect the range; rows of those + * groups outside it are skipped. A row past the upper bound does not end the + * scan: group order is not row-number order once compaction has moved rows + * into later groups, so stopping there could drop rows that are in range. + */ +static bool +pgcolumnar_scan_getnextslot_tidrange(TableScanDesc sscan, + ScanDirection direction, + TupleTableSlot *slot) +{ + PgColumnarScanDesc scan = (PgColumnarScanDesc) sscan; + + Assert(scan->hasRowRange); + if (scan->minRow > scan->maxRow || scan->numRangeGroups == 0) + { + ExecClearTuple(slot); + return false; + } + + while (pgcolumnar_scan_getnextslot(sscan, direction, slot)) + { + uint64 row = PgColumnarItemPointerToRowNumber(&slot->tts_tid); + + if (row >= scan->minRow && row <= scan->maxRow) + return true; + } + return false; +} + /* ------------------------------------------------------------------------- * parallel scan: shared group claim via phs_nallocated (see pgcolumnar_reader.c) * ------------------------------------------------------------------------- */ @@ -2252,6 +2394,8 @@ static const TableAmRoutine pgcolumnar_am_methods = { .scan_begin = pgcolumnar_scan_begin, .scan_end = pgcolumnar_scan_end, .scan_rescan = pgcolumnar_scan_rescan, + .scan_set_tidrange = pgcolumnar_scan_set_tidrange, + .scan_getnextslot_tidrange = pgcolumnar_scan_getnextslot_tidrange, .scan_getnextslot = pgcolumnar_scan_getnextslot, .parallelscan_estimate = pgcolumnar_parallelscan_estimate, diff --git a/test/tid_range_scan.sh b/test/tid_range_scan.sh index ffe30e7cd..12413b850 100755 --- a/test/tid_range_scan.sh +++ b/test/tid_range_scan.sh @@ -33,15 +33,64 @@ check_num "premise: the table has many row groups, so skipping can show" "$(at_l # ctid of the row with a given id, as the AM numbers it tid_of() { q "SELECT ctid FROM tr WHERE id = $1;"; } LAST="$(tid_of 390001)"; MID_A="$(tid_of 150001)"; MID_B="$(tid_of 170000)"; END="$(tid_of 400000)" +# Offset 0 is no row's offset, and neither is any offset past the 291 a block +# holds (MaxHeapTuplesPerPage), though a bound may name either: the executor +# turns `ctid < '(B,0)'` into an upper bound of (B-1,65535). Those are the edges +# the AM must translate; read unclamped, offset 400 named a row in block B+1. +BLK="${MID_A#(}"; BLK="${BLK%%,*}" echo "-- ctids: 390001=$LAST 150001=$MID_A 170000=$MID_B 400000=$END" +check "premise: a ctid range plans a TID range scan, so the arms below exercise it" \ + "$(q "EXPLAIN SELECT id FROM tr WHERE ctid >= '$LAST';" | grep -c 'Tid Range Scan')" "1" + hash() { q "SET enable_tidscan = ${2:-on}; SELECT md5(string_agg(id::text || v, ',' ORDER BY id)) || '|' || count(*) FROM tr WHERE $1;" | tail -1; } for pred in "ctid >= '$LAST'" "ctid >= '$MID_A' AND ctid <= '$MID_B'" "ctid < '$MID_A'" \ - "ctid > '$END'" "ctid >= '(0,0)'"; do + "ctid > '$END'" "ctid >= '(0,0)'" \ + "ctid <= '($BLK,0)'" "ctid > '($BLK,0)'" "ctid < '($BLK,1)'" "ctid <= '(0,0)'" \ + "ctid >= '($BLK,7)' AND ctid < '($BLK,7)'" \ + "ctid < '($BLK,0)'" "ctid >= '($BLK,400)'" "ctid <= '($BLK,400)'" \ + "ctid > '($BLK,65535)'" "ctid >= '($BLK,291)' AND ctid <= '($BLK,292)'"; do check "a ctid range returns what the per-row filter returns: ${pred//$LAST/LAST}" \ "$(hash "$pred" on)" "$(hash "$pred" off)" done +# Rescan: an outer row per range, so the scan is rescanned with new bounds and +# a reader built for the previous range must not survive into the next one. +LATQ="SELECT string_agg(r.n || ':' || x.c || ':' || x.h, ' ' ORDER BY r.n) + FROM (VALUES (1, '$MID_A'::tid, '$MID_B'::tid), (2, '$LAST'::tid, '$END'::tid), + (3, '(0,1)'::tid, '$MID_A'::tid), (4, '$END'::tid, '$LAST'::tid)) r(n, lo, hi), + LATERAL (SELECT count(*) c, md5(string_agg(id::text, ',' ORDER BY id)) h FROM tr + WHERE ctid >= r.lo AND ctid <= r.hi) x" +lat() { q "SET enable_tidscan = $1; $LATQ;" | tail -1; } +# The query the arm runs, not a stand-in: a draft that left the lateral's +# output unreferenced planned no scan of tr at all. +check "premise: the per-range join rescans a TID range scan" \ + "$(q "EXPLAIN $LATQ;" | grep -c 'Tid Range Scan')" "1" +check "a TID range scan rescanned with new bounds returns what the filter returns" "$(lat on)" "$(lat off)" + +# PostgreSQL 19 runs a TID range scan in parallel: each worker sets the range +# and the reader's shared group claim hands out only the restricted groups. The +# custom scan is turned off because, with parallelism cheap, the planner prefers +# a parallel custom scan that filters ctid per row, and this arm is about the +# AM's path. +if [ "$PGC_MAJOR" -ge 19 ]; then + PAR="SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; SET min_parallel_table_scan_size = 0; + SET max_parallel_workers_per_gather = 2; SET parallel_leader_participation = off; + SET pgcolumnar.enable_custom_scan = off;" + PQ="SELECT md5(string_agg(id::text || v, ',' ORDER BY id)) || '|' || count(*) FROM tr + WHERE ctid >= '$MID_A' AND ctid <= '$LAST'" + plan="$(q "$PAR EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) $PQ;")" + check "premise: the range runs as a parallel TID range scan with two workers" \ + "$(grep -c 'Parallel Tid Range Scan' <<<"$plan")|$(grep -o 'Workers Launched: [0-9]*' <<<"$plan")" \ + "1|Workers Launched: 2" + check "a parallel TID range scan returns what the filter returns" \ + "$(q "$PAR $PQ;" | tail -1)" "$(hash "ctid >= '$MID_A' AND ctid <= '$LAST'" off)" +else + check_skip "a parallel TID range scan returns what the filter returns" \ + "SKIP a parallel TID range scan returns what the filter returns (PostgreSQL $PGC_MAJOR has none; 19 added it)" \ + "PostgreSQL $PGC_MAJOR has no parallel TID range scan" +fi + # max(v || 'x'), not max(v): an unfiltered count/max is answered from zone maps # (#1293) and read 5 blocks of this table in the first draft, so the "whole # table" baseline measured metadata, not data. @@ -65,4 +114,17 @@ check "and rows deleted inside the range stay deleted" \ check_num "premise: the delete removed rows from the range" \ "$(q "SET enable_tidscan = off; SELECT count(*) FROM tr WHERE ctid >= '$LAST';" | tail -1)" "9000" +# Compaction moves a group's surviving rows into a new group with new, higher +# row numbers. The range over the old addresses now names rows nowhere, and the +# range over the new ones must find them all. +check_num "premise: compaction rewrote the group the delete thinned" \ + "$(q "SELECT pgcolumnar.compact_rewrite('tr', 0.05, 0);")" "1" +MOVED="$(tid_of 390001)" +check "premise: compaction gave the surviving rows new addresses" \ + "$([ "$MOVED" != "$LAST" ] && echo moved || echo "kept $MOVED")" "moved" +for pred in "ctid >= '$LAST'" "ctid >= '$MOVED'" "ctid >= '$MID_A' AND ctid <= '$MOVED'"; do + check "after compaction a ctid range returns what the filter returns: ${pred//$LAST/LAST}" \ + "$(hash "$pred" on)" "$(hash "$pred" off)" +done + pgc_summary From cadd1c59418196f858948641e9c9556697dd08cd Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 22:38:55 +0000 Subject: [PATCH 3/5] test: the TID range scan pytest twin, registries and ledger test_tid_range_scan.py is independent of the shell suite: its own table, row count, group size and ranges, on a private database. Every shell property is covered (compare_to_bash: 13 literal, 2 template, 0 missing). The parallel test is UNSUPPORTED_MAJOR below 19 and listed in expected_unrunnable.txt. Ledger: 31 rows from a five-major run, replayed onto #1286's ledger; 15 dated by eight mutations merged one log at a time with --mutation and --target; census 1674 -> 1690. TESTS.md section 92; cluster_tests 499 by collection. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 13 ++ test/pytest/TESTS.md | 28 +++++ test/pytest/expected_tests.txt | 4 +- test/pytest/expected_unrunnable.txt | 8 +- test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_tid_range_scan.py | 182 ++++++++++++++++++++++++++++ test/run_all_versions.sh | 1 + 7 files changed, 235 insertions(+), 2 deletions(-) create mode 100644 test/pytest/test_tid_range_scan.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c522a1a7..21e2979c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,19 @@ true until the next version shipped. FAILS the suite unless `PGC_ALLOW_MISSING_PG_PARTMAN=1` is set. The CI suites and pytest jobs now install `postgresql-N-partman` from PGDG, in the cache key as well as the install. +- A ctid range reads only the row groups that hold it. The table access method + now implements TID range scans (`scan_set_tidrange`), so a query like + `WHERE ctid >= '(1340,62)'` restricts the reader to the row groups that + intersect the range. Before, it read every group and filtered per row: on + #1294, a range matching 0 rows of a 10-million-row table read all 67 groups. + Measured with 400,000 rows: the last 2.5% by ctid read 28 blocks, against 923 + for the whole table. PostgreSQL 19's parallel TID range scan shares the + reader's group claim. The custom scan still treats ctid as a per-row filter, + so with parallelism cheap the planner may prefer it. `test/tid_range_scan.sh` + and its pytest twin hold the bounds no row has (offset 0, and offsets past the + 291 a block holds, which the executor produces for `ctid < '(B,0)'`), rescans, + deletes and compaction. The first build read those offsets unclamped and + returned rows of the next block; the twin's arms for them were red against it. ### Changed diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index e2433eb6d..014250448 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -137,6 +137,7 @@ behaviour, the source of that number is named. - [89. test_docs_cross_database_query.py: no documented query asks another database for its extensions](#89-test_docs_cross_database_querypy-no-documented-query-asks-another-database-for-its-extensions) - [90. test_partman_columnar.py: pg_partman makes columnar partitions with one setting](#90-test_partman_columnarpy-pg_partman-makes-columnar-partitions-with-one-setting) - [91. test_capability_decline.py: a missing pyarrow declines by name, and nothing declines silently](#91-test_capability_declinepy-a-missing-pyarrow-declines-by-name-and-nothing-declines-silently) +- [92. test_tid_range_scan.py: a ctid range reads only the row groups that hold it](#92-test_tid_range_scanpy-a-ctid-range-reads-only-the-row-groups-that-hold-it) ## 1. How to read a test in here @@ -6484,3 +6485,30 @@ Written red first: all seven failed before `pgc_vacuity` had the methods. The fo | `test_a_broken_install_fails_rather_than_declines` | a `pyarrow` that raises `ImportError` from inside fails with its own error | | `test_a_missing_dependency_of_pyarrow_fails_rather_than_declines` | a `pyarrow` that cannot import its own dependency raises `ModuleNotFoundError` naming that dependency, and fails rather than declines; shown red with `require_module`'s `.name` check deleted (OffgridwithJD, review of #1286) | | `test_every_unguarded_import_spelling_is_refused` | each of six spellings, `__import__` among them, is named and refused; a file using `require_module` is not | + +## 92. test_tid_range_scan.py: a ctid range reads only the row groups that hold it + +Port of `tid_range_scan.sh`. Before this, the access method had no `scan_set_tidrange`, so a ctid range was a per-row filter: on #1294, `WHERE ctid >= '(600000,0)'` on a 10-million-row table matched 0 rows and still read all 67 row groups. The rollup that #1294 proposes needs "the newest row groups only", and so does anything else that bounds ctid. + +The oracle is the same query with `enable_tidscan = off`, which is that per-row filter. The cost is the table's own blocks from pg_statio, read under `max(body || 'y')`, because an unfiltered max is answered from the zone maps and reads no data. The tail range must read under a fifth of what a decode of the whole table reads, and at least one block, because a zero is not a measurement. + +THE BOUNDS THAT MATTER ARE THE ONES NO ROW HAS. A block holds 291 rows, at offsets 1 to 291, but a bound may name offset 0 or anything up to 65535. The executor turns `ctid < '(B,0)'` into an upper bound of `(B-1,65535)`. The first build read those offsets unclamped, so offset 400 named a row in the next block. The shell twin's arms for those bounds went red against it, which is how it was found. + +Also held: a rescan with new bounds per outer row (the plan premise is taken on the very query compared, because a draft whose lateral output went unreferenced planned no scan at all); deletes inside the range; and compaction, which gives a thinned group's surviving rows new, higher row numbers. On PostgreSQL 19, a parallel TID range scan with two workers is held too. Below 19 that test is `UNSUPPORTED_MAJOR` and listed in `expected_unrunnable.txt`. + +On a private database, because `pg_stat_reset()` resets the whole database's counters. + +Removal proofs, each red on PostgreSQL 19 on the arm named: +- without the group restriction: the tail-range cost arm; +- with an upper offset past 291 unclamped: `ctid < '(B,0)'`; +- with a lower offset past 291 unclamped: `ctid >= '(B,500)' AND ctid <= '(B+1,700)'`; +- without the boundary-row filter: the tail range and the parallel arm; +- with the reader kept across a rescan: the rescan arm; +- without the shared parallel claim: the parallel arm, which returned every row twice. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_a_ctid_range_reads_only_its_row_groups` | the table has its rows and many groups; a ctid range plans a TID range scan; eleven ranges, including offsets 0, 291, 500 and 65535, return what the per-row filter returns; a rescanned scan does too; the tail range reads at least one block and under a fifth of the table; deleted rows stay deleted; after compaction moves rows, ranges over the old and new addresses return what the filter returns | +| `test_a_parallel_tid_range_scan_returns_what_the_filter_returns` | PostgreSQL 19+: the range runs as a parallel TID range scan with two workers launched, and returns what the filter returns. Below 19, `UNSUPPORTED_MAJOR` | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index df9bd7c6e..415781ffa 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -554,4 +554,6 @@ guard_tests 413 # # cluster_tests 497 -> 498 for #1304's test_the_bound_reads_the_zone_maps_in_one_catalog_scan, # collected over the cluster files on the tree rebased onto 503f080 (#1286). -cluster_tests 498 +# 498 -> 500 for the TID range scan pair: test_tid_range_scan.py's two tests. +# Collected on this tree, after #1304. +cluster_tests 500 diff --git a/test/pytest/expected_unrunnable.txt b/test/pytest/expected_unrunnable.txt index 2cf87421c..1890b3a53 100644 --- a/test/pytest/expected_unrunnable.txt +++ b/test/pytest/expected_unrunnable.txt @@ -18,7 +18,7 @@ # land without it. # # FORMAT: majornodeid. Comments and blank lines are ignored. A major with -# no rows expects nothing to decline, which is what 18 and 19 claim. +# no rows expects nothing to decline, which is what 19 claims. # PostgreSQL 15, 16 and 17: pgcolumnar.analyze() writes through # pg_restore_attribute_stats, which core added in PostgreSQL 18. @@ -46,3 +46,9 @@ 15 test_temporal.py::test_temporal 16 test_temporal.py::test_temporal 17 test_temporal.py::test_temporal + +# PostgreSQL 15 to 18: parallel TID range scans arrived in PostgreSQL 19. +15 test_tid_range_scan.py::test_a_parallel_tid_range_scan_returns_what_the_filter_returns +16 test_tid_range_scan.py::test_a_parallel_tid_range_scan_returns_what_the_filter_returns +17 test_tid_range_scan.py::test_a_parallel_tid_range_scan_returns_what_the_filter_returns +18 test_tid_range_scan.py::test_a_parallel_tid_range_scan_returns_what_the_filter_returns diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index cd584275b..6262d9442 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -131,6 +131,7 @@ "stats_privilege", "storage_row_repair", "temporal", + "tid_range_scan", "ttl_expire", "validity_elision", "zonemap_boundaries"] diff --git a/test/pytest/test_tid_range_scan.py b/test/pytest/test_tid_range_scan.py new file mode 100644 index 000000000..05fe20ea7 --- /dev/null +++ b/test/pytest/test_tid_range_scan.py @@ -0,0 +1,182 @@ +"""A ctid range reads only the row groups that hold it (TID range scans). + +Before this, `WHERE ctid >= '(600000,0)'` on a 10M-row table matched 0 rows and +still read all 67 row groups (#1294): the access method had no scan_set_tidrange, +so the range was a per-row filter over the whole table. + +THE ORACLE is the same query with `enable_tidscan = off`, which is that per-row +filter. THE COST is the table's own blocks from pg_statio, read under a decode the +zone maps cannot answer (`max(body || 'y')`: an unfiltered max is metadata-only). + +The bounds that matter are the ones no row has. A block holds 291 rows +(MaxHeapTuplesPerPage), offsets 1..291, but a bound may say offset 0 or anything up +to 65535, and the executor makes `ctid < '(B,0)'` into an upper bound of +(B-1, 65535). Read without clamping, offset 400 named a row in the next block. + +ON A PRIVATE DATABASE (`pgc_own_db`): `pg_stat_reset()` resets the database's +counters, and nothing else may be counting there. Independent of +`test/tid_range_scan.sh`: its own columns, row count, group size, and ranges. +""" + +ROWS = 260000 +GROUP = 7000 + + +def _one(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + row = cur.fetchone() + return row[0] if row else None + + +def _plan(conn, sql): + with conn.cursor() as cur: + cur.execute("EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) " + sql) + return "\n".join(r[0] for r in cur.fetchall()) + + +def _digest(conn, where, tidscan): + with conn.cursor() as cur: + cur.execute(f"SET enable_tidscan = {tidscan}") + cur.execute("SELECT count(*), md5(string_agg(k::text || body, ',' ORDER BY k))" + f" FROM tids WHERE {where}") + n, h = cur.fetchone() + cur.execute("RESET enable_tidscan") + return f"{n}|{h}" + + +def _blocks(conn, where): + """Blocks of `tids` read while a forced decode over `where` ran.""" + with conn.cursor() as cur: + cur.execute("SELECT pg_stat_force_next_flush()") + cur.execute("SELECT pg_stat_reset()") + cur.execute(f"SELECT count(*), max(body || 'y') FROM tids WHERE {where}") + cur.fetchall() + cur.execute("SELECT pg_stat_force_next_flush()") + cur.execute("SELECT coalesce(heap_blks_read,0) + coalesce(heap_blks_hit,0)" + " FROM pg_statio_user_tables WHERE relname = 'tids'") + row = cur.fetchone() + return int(row[0]) if row else None + + +def _tid(conn, k): + return _one(conn, f"SELECT ctid FROM tids WHERE k = {k}") + + +def _fixture(conn, expect): + with conn.cursor() as cur: + cur.execute("CREATE TABLE tids (k bigint, body text) USING pgcolumnar") + cur.execute(f"SELECT pgcolumnar.set_options('tids', stripe_row_limit => {GROUP})") + cur.execute(f"INSERT INTO tids SELECT g, repeat(md5((g * 7)::text), 2)" + f" FROM generate_series(1, {ROWS}) g") + cur.execute("ANALYZE tids") + expect.num(_one(conn, "SELECT count(*) FROM tids"), ROWS, "premise: the table holds every row") + groups = _one(conn, "SELECT count(*) FROM pgcolumnar.row_group" + " WHERE storage_id = pgcolumnar.get_storage_id('tids')") + expect.at_least(groups, 30, "premise: the table has many row groups, so skipping can show") + + +def test_a_ctid_range_reads_only_its_row_groups(pgc_own_db, expect): + conn = pgc_own_db + _fixture(conn, expect) + + tail, a, b, last = _tid(conn, 246001), _tid(conn, 61234), _tid(conn, 98765), _tid(conn, ROWS) + blk = int(a.strip("()").split(",")[0]) + print(f"-- ctids: 246001={tail} 61234={a} 98765={b} {ROWS}={last}") + + expect.text("yes" if "Tid Range Scan" in _plan(conn, f"SELECT k FROM tids WHERE ctid >= '{tail}'") + else "no", "yes", "premise: a ctid range plans a TID range scan, so the arms below exercise it") + + ranges = [ + f"ctid >= '{tail}'", + f"ctid > '{a}' AND ctid < '{b}'", + f"ctid <= '{a}'", + f"ctid >= '{last}'", + f"ctid > '{last}'", + "ctid < '(0,1)'", + f"ctid < '({blk},0)'", + f"ctid >= '({blk},0)' AND ctid <= '({blk + 3},0)'", + f"ctid > '({blk},291)'", + f"ctid >= '({blk},500)' AND ctid <= '({blk + 1},700)'", + f"ctid <= '({blk},65535)'", + ] + for where in ranges: + expect.text(_digest(conn, where, "on"), _digest(conn, where, "off"), + f"a ctid range returns what the per-row filter returns: {where}") + + # Rescanned with new bounds per outer row: a reader built for one range must + # not answer the next. The plan premise is taken on the very query compared. + lat = ("SELECT string_agg(r.i || ':' || x.n || ':' || x.h, ' ' ORDER BY r.i)" + f" FROM (VALUES (1, '{a}'::tid, '{b}'::tid), (2, '{tail}'::tid, '{last}'::tid)," + f" (3, '{b}'::tid, '{a}'::tid), (4, '(0,0)'::tid, '{a}'::tid)) r(i, lo, hi)," + " LATERAL (SELECT count(*) n, md5(string_agg(k::text, ',' ORDER BY k)) h FROM tids" + " WHERE ctid >= r.lo AND ctid <= r.hi) x") + expect.text("yes" if "Tid Range Scan" in _plan(conn, lat) else "no", "yes", + "premise: the per-range join rescans a TID range scan") + with conn.cursor() as cur: + on = _one(conn, lat) + cur.execute("SET enable_tidscan = off") + off = _one(conn, lat) + cur.execute("RESET enable_tidscan") + expect.text(on, off, "a TID range scan rescanned with new bounds returns what the filter returns") + + whole = _blocks(conn, "true") + part = _blocks(conn, f"ctid >= '{tail}'") + print(f"-- blocks: whole {whole}, tail {part}") + expect.at_least(whole, 100, "premise: reading the whole table reads its blocks") + expect.at_least(part, 1, "premise: the tail range read the table at all") + # The margin is whole - 5 * part: positive passes, and a failure reports by how much. + expect.at_least(whole - 5 * part, 1, + "a ctid range over the last row groups reads under a fifth of the table") + + # Deletes inside the range, then compaction, which gives the surviving rows of + # the thinned group new, higher row numbers. + before = _tid(conn, 249000) + with conn.cursor() as cur: + cur.execute("DELETE FROM tids WHERE k BETWEEN 250500 AND 251900") + expect.text(_digest(conn, f"ctid >= '{tail}'", "on"), _digest(conn, f"ctid >= '{tail}'", "off"), + "and rows deleted inside the range stay deleted") + with conn.cursor() as cur: + cur.execute("SET enable_tidscan = off") + left = _one(conn, f"SELECT count(*) FROM tids WHERE ctid >= '{tail}'") + cur.execute("RESET enable_tidscan") + # 246001..260000 is 14,000 rows; the delete took 1,401 of them. + expect.num(left, 14000 - 1401, "premise: the delete removed rows from the range") + expect.num(_one(conn, "SELECT pgcolumnar.compact_rewrite('tids', 0.1, 0)"), 1, + "premise: compaction rewrote the group the delete thinned") + moved = _tid(conn, 249000) + expect.text("moved" if moved != before else f"kept {moved}", "moved", + "premise: compaction gave the surviving rows new addresses") + for where in (f"ctid >= '{tail}'", f"ctid >= '{moved}'", f"ctid >= '{a}' AND ctid <= '{moved}'"): + expect.text(_digest(conn, where, "on"), _digest(conn, where, "off"), + f"after compaction a ctid range returns what the filter returns: {where}") + + +def test_a_parallel_tid_range_scan_returns_what_the_filter_returns(pgc_own_db, expect): + conn = pgc_own_db + major = int(_one(conn, "SHOW server_version_num")) // 10000 + if major < 19: + expect.cannot_run( + "UNSUPPORTED_MAJOR", + f"parallel TID range scans arrived in PostgreSQL 19; this server is {major}", + name="a parallel TID range scan returns what the filter returns") + return + _fixture(conn, expect) + lo, hi = _tid(conn, 40001), _tid(conn, 222222) + where = f"ctid >= '{lo}' AND ctid <= '{hi}'" + want = _digest(conn, where, "off") + with conn.cursor() as cur: + # The custom scan off: with parallelism cheap the planner prefers a parallel + # custom scan that filters ctid per row, and this is about the AM's path. + for s in ("parallel_setup_cost = 0", "parallel_tuple_cost = 0", + "min_parallel_table_scan_size = 0", "max_parallel_workers_per_gather = 2", + "parallel_leader_participation = off", "pgcolumnar.enable_custom_scan = off"): + cur.execute("SET " + s) + q = ("SELECT count(*) || '|' || md5(string_agg(k::text || body, ',' ORDER BY k))" + f" FROM tids WHERE {where}") + plan = _plan(conn, q) + launched = [ln.strip() for ln in plan.splitlines() if "Workers Launched" in ln] + expect.text(f"{'Parallel Tid Range Scan' in plan}|{launched}", + "True|['Workers Launched: 2']", + "premise: the range runs as a parallel TID range scan with two workers") + expect.text(_one(conn, q), want, "a parallel TID range scan returns what the filter returns") diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index a537c3b2f..4241aed4a 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -293,6 +293,7 @@ SUITES=( storage_row_repair tablesample temporal + tid_range_scan truncate_cleanup ttl_expire ungrouped_vector_agg From fb2f2aa3b7b410a02f2145ac8f21f74ded899eea Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 23:00:10 +0000 Subject: [PATCH 4/5] fix: a cursor over a TID range scan gets a Material, so a backward fetch never reaches the AM (#1306 review) PostgreSQL counts a TID range scan as able to scan backward, so it puts no Material above one under a cursor that may scroll, and the access method, which reads forward only, refused the backward fetch (0A000). Found by @jdatcmd: a SCROLL cursor's FETCH LAST errored where main's plan answered. A plain DECLARE hit it too, because the portal makes a cursor scrollable when its plan claims backward support. A planner hook wraps a cursor's plan (SCROLL, or a DECLARE without NO SCROLL) in a Material when it holds a TID range scan of a columnar table, as standard_planner does for a SCROLL cursor over a plan that cannot run backward. PostgreSQL 19's hook takes an ExplainState as well. Arms, red on e568782 in both twins: a SCROLL cursor's FETCH LAST, BACKWARD and ABSOLUTE match the filter's plan; a plain cursor answers a backward fetch correctly. Plus jdatcmd's verified case as an arm: rows inserted earlier in the same transaction are in the range. Removal proofs: no hook (both cursor arms), SCROLL only (the plain-cursor arm). Co-Authored-By: Claude Opus 5.5 --- src/columnar_tableam.c | 122 +++++++++++++++++++++++++++++ test/pytest/test_tid_range_scan.py | 56 +++++++++++++ test/tid_range_scan.sh | 40 ++++++++++ 3 files changed, 218 insertions(+) diff --git a/src/columnar_tableam.c b/src/columnar_tableam.c index 0c5912b9e..daebcc57c 100644 --- a/src/columnar_tableam.c +++ b/src/columnar_tableam.c @@ -51,6 +51,11 @@ #include "storage/smgr.h" #include "storage/spin.h" #include "utils/builtins.h" +#include "optimizer/planner.h" +#include "optimizer/planmain.h" +#include "nodes/makefuncs.h" +#include "nodes/nodeFuncs.h" +#include "parser/parsetree.h" #include "utils/guc.h" #include "catalog/indexing.h" #include "catalog/namespace.h" @@ -891,6 +896,120 @@ pgcolumnar_scan_getnextslot_tidrange(TableScanDesc sscan, return false; } +/* + * Cursors over a TID range scan (#1306 review). + * + * PostgreSQL counts a TID range scan as able to scan backward + * (ExecSupportsBackwardScan), so it puts no Material above one under a cursor + * that may scroll, and a backward fetch reaches the access method, which reads + * forward only and refuses (pgcolumnar_scan_getnextslot). That covers a SCROLL + * cursor, and a plain DECLARE too: the portal makes a cursor scrollable when + * its plan claims backward support. So when a cursor's plan holds a TID range + * scan of a columnar table, a Material goes on top, as standard_planner does + * for a SCROLL cursor over a plan that cannot run backward. + */ +static planner_hook_type pgcolumnar_prev_planner_hook = NULL; + +static bool +pgcolumnar_plan_has_columnar_tidrange(Plan *plan, List *rtable, int *maxid) +{ + bool found = false; + ListCell *lc; + + if (plan == NULL) + return false; + *maxid = Max(*maxid, plan->plan_node_id); + if (IsA(plan, TidRangeScan)) + { + RangeTblEntry *rte = rt_fetch(((Scan *) plan)->scanrelid, rtable); + + if (rte->rtekind == RTE_RELATION && PgColumnarIsColumnarRelation(rte->relid)) + found = true; + } + found |= pgcolumnar_plan_has_columnar_tidrange(plan->lefttree, rtable, maxid); + found |= pgcolumnar_plan_has_columnar_tidrange(plan->righttree, rtable, maxid); + switch (nodeTag(plan)) + { + case T_Append: + foreach(lc, ((Append *) plan)->appendplans) + found |= pgcolumnar_plan_has_columnar_tidrange(lfirst(lc), rtable, maxid); + break; + case T_MergeAppend: + foreach(lc, ((MergeAppend *) plan)->mergeplans) + found |= pgcolumnar_plan_has_columnar_tidrange(lfirst(lc), rtable, maxid); + break; + case T_SubqueryScan: + found |= pgcolumnar_plan_has_columnar_tidrange(((SubqueryScan *) plan)->subplan, + rtable, maxid); + break; + case T_CustomScan: + foreach(lc, ((CustomScan *) plan)->custom_plans) + found |= pgcolumnar_plan_has_columnar_tidrange(lfirst(lc), rtable, maxid); + break; + default: + break; + } + return found; +} + +/* PostgreSQL 19 passes the planner hook an ExplainState as well */ +#if PG_VERSION_NUM >= 190000 +#define PGC_PLANNER_PARAMS Query *parse, const char *query_string, int cursorOptions, \ + ParamListInfo boundParams, ExplainState *es +#define PGC_PLANNER_ARGS parse, query_string, cursorOptions, boundParams, es +#else +#define PGC_PLANNER_PARAMS Query *parse, const char *query_string, int cursorOptions, \ + ParamListInfo boundParams +#define PGC_PLANNER_ARGS parse, query_string, cursorOptions, boundParams +#endif + +static PlannedStmt * +pgcolumnar_planner(PGC_PLANNER_PARAMS) +{ + PlannedStmt *stmt; + int maxid = 0; + + stmt = pgcolumnar_prev_planner_hook + ? pgcolumnar_prev_planner_hook(PGC_PLANNER_ARGS) + : standard_planner(PGC_PLANNER_ARGS); + + /* + * A cursor that may scroll: SCROLL, or a DECLARE (CURSOR_OPT_FAST_PLAN) + * that did not say NO SCROLL. Nothing else fetches backward. + */ + if ((cursorOptions & CURSOR_OPT_NO_SCROLL) || + !(cursorOptions & (CURSOR_OPT_SCROLL | CURSOR_OPT_FAST_PLAN)) || + stmt->planTree == NULL || IsA(stmt->planTree, Material)) + return stmt; + if (pgcolumnar_plan_has_columnar_tidrange(stmt->planTree, stmt->rtable, &maxid)) + { + Plan *sub = stmt->planTree; + Plan *mat = materialize_finished_plan(sub); + List *tlist = NIL; + ListCell *lc; + + /* + * The plan is already finished, so the Material's target list is built + * as set_plan_references would build it: one OUTER_VAR per column of + * the plan below. It gets a node id of its own. + */ + foreach(lc, sub->targetlist) + { + TargetEntry *tle = (TargetEntry *) lfirst(lc); + Var *v = makeVar(OUTER_VAR, tle->resno, exprType((Node *) tle->expr), + exprTypmod((Node *) tle->expr), + exprCollation((Node *) tle->expr), 0); + + tlist = lappend(tlist, makeTargetEntry((Expr *) v, tle->resno, + tle->resname, tle->resjunk)); + } + mat->targetlist = tlist; + mat->plan_node_id = maxid + 1; + stmt->planTree = mat; + } + return stmt; +} + /* ------------------------------------------------------------------------- * parallel scan: shared group claim via phs_nallocated (see pgcolumnar_reader.c) * ------------------------------------------------------------------------- */ @@ -4064,6 +4183,9 @@ _PG_init(void) GUC_UNIT_MS, NULL, NULL, NULL); + pgcolumnar_prev_planner_hook = planner_hook; + planner_hook = pgcolumnar_planner; + MarkGUCPrefixReserved("pgcolumnar"); /* diff --git a/test/pytest/test_tid_range_scan.py b/test/pytest/test_tid_range_scan.py index 05fe20ea7..17315f394 100644 --- a/test/pytest/test_tid_range_scan.py +++ b/test/pytest/test_tid_range_scan.py @@ -18,6 +18,8 @@ `test/tid_range_scan.sh`: its own columns, row count, group size, and ranges. """ +import psycopg + ROWS = 260000 GROUP = 7000 @@ -120,6 +122,60 @@ def test_a_ctid_range_reads_only_its_row_groups(pgc_own_db, expect): cur.execute("RESET enable_tidscan") expect.text(on, off, "a TID range scan rescanned with new bounds returns what the filter returns") + # Cursors (#1306 review). A TID range scan counts as able to scan backward, so + # no Material goes above it: a SCROLL cursor, and a plain DECLARE the portal + # makes scrollable, send a backward fetch to the access method. Compared, + # fetch for fetch, with the same session under enable_tidscan = off. + cq = f"SELECT k FROM tids WHERE ctid > '{a}' AND ctid < '{b}'" + expect.text("yes" if "Tid Range Scan" in _plan(conn, cq) else "no", "yes", + "premise: the cursor's query plans a TID range scan") + + def cursor_run(tidscan, kind): + out = [] + with conn.cursor() as cur: + cur.execute(f"SET enable_tidscan = {tidscan}") + cur.execute("BEGIN") + cur.execute(f"DECLARE c {kind} CURSOR FOR {cq}") + for f in ("FETCH 3 FROM c", "FETCH LAST FROM c", "FETCH BACKWARD 4 FROM c", + "FETCH ABSOLUTE 2 FROM c"): + try: + cur.execute(f) + out.append(",".join(str(r[0]) for r in cur.fetchall())) + except psycopg.Error as e: + out.append(f"error {e.sqlstate}") + break + cur.execute("ROLLBACK") + cur.execute("RESET enable_tidscan") + return " | ".join(out) + expect.text(cursor_run("on", "SCROLL"), cursor_run("off", "SCROLL"), + "a SCROLL cursor over a ctid range fetches backward as the filter's plan does") + # A plain DECLARE over the filter's plan is made forward-only; over a TID + # range scan it is made scrollable, so it must answer as a SCROLL cursor + # does. What it may never do is reach the access method backward. + expect.text(cursor_run("on", ""), cursor_run("off", "SCROLL"), + "a plain cursor over a ctid range answers a backward fetch correctly") + + # Rows written earlier in the same transaction are in the range. + def same_xact(tidscan): + with conn.cursor() as cur: + cur.execute(f"SET enable_tidscan = {tidscan}") + cur.execute("BEGIN") + cur.execute(f"INSERT INTO tids SELECT g, 'pending' FROM generate_series({ROWS + 1}, {ROWS + 650}) g") + cur.execute("EXPLAIN SELECT k FROM tids WHERE ctid >= '" + tail + "'") + plan = "\n".join(r[0] for r in cur.fetchall()) + cur.execute("SELECT count(*) || '|' || md5(string_agg(k::text, ',' ORDER BY k))" + f" FROM tids WHERE ctid >= '{tail}'") + got = cur.fetchone()[0] + cur.execute("ROLLBACK") + cur.execute("RESET enable_tidscan") + return plan, got + plan_on, got_on = same_xact("on") + _, got_off = same_xact("off") + expect.text("yes" if "Tid Range Scan" in plan_on else "no", "yes", + "premise: inside a transaction the range still plans a TID range scan") + expect.text(got_on, got_off, + "rows inserted earlier in the same transaction are in the range, as the filter sees them") + whole = _blocks(conn, "true") part = _blocks(conn, f"ctid >= '{tail}'") print(f"-- blocks: whole {whole}, tail {part}") diff --git a/test/tid_range_scan.sh b/test/tid_range_scan.sh index 12413b850..bb6671bf1 100755 --- a/test/tid_range_scan.sh +++ b/test/tid_range_scan.sh @@ -68,6 +68,46 @@ check "premise: the per-range join rescans a TID range scan" \ "$(q "EXPLAIN $LATQ;" | grep -c 'Tid Range Scan')" "1" check "a TID range scan rescanned with new bounds returns what the filter returns" "$(lat on)" "$(lat off)" +# Cursors. PostgreSQL counts a TID range scan as able to scan backward +# (ExecSupportsBackwardScan), so no Material goes above it: a SCROLL cursor, and a +# plain DECLARE that the portal makes scrollable for the same reason, send a +# backward fetch straight to the access method. Found by @jdatcmd reviewing +# #1306: FETCH LAST errored where main's plan answered. Compared, outcome for +# outcome, with the same session under enable_tidscan = off. +CQ="SELECT id FROM tr WHERE ctid >= '$MID_A' AND ctid <= '$MID_B'" +cursor_run() { # cursor_run on|off SCROLL|plain -> every fetch's rows or SQLSTATE, on one line + local kind=""; [ "$2" = SCROLL ] && kind="SCROLL" + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -v VERBOSITY=sqlstate -c "SET enable_tidscan = $1" -c "BEGIN" \ + -c "DECLARE c $kind CURSOR FOR $CQ" -c "FETCH 2 FROM c" -c "FETCH LAST FROM c" \ + -c "FETCH BACKWARD 2 FROM c" -c "FETCH ABSOLUTE 3 FROM c" -c "COMMIT" 2>&1 | tr '\n' ' ' +} +check "premise: the cursor's query plans a TID range scan" \ + "$(q "EXPLAIN $CQ;" | grep -c 'Tid Range Scan')" "1" +check "a SCROLL cursor over a ctid range fetches backward as the filter's plan does" \ + "$(cursor_run on SCROLL)" "$(cursor_run off SCROLL)" +# A plain DECLARE over the filter's plan is made forward-only (55000 on a backward +# fetch). Over a TID range scan it is made scrollable, so it must answer as a +# SCROLL cursor does, correctly; what it may never do is reach the access method. +check "a plain cursor over a ctid range answers a backward fetch correctly" \ + "$(cursor_run on plain)" "$(cursor_run off SCROLL)" + +# Rows written earlier in the same transaction: set_tidrange reads the row-group +# catalog, and the pending writes must be in it by then (#1306 review). +same_xact() { # same_xact on|off -> count|digest of the range after an insert in the same transaction + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -c "SET enable_tidscan = $1" -c "BEGIN" \ + -c "INSERT INTO tr SELECT g, md5(g::text) FROM generate_series(400001, 400700) g" \ + -c "SELECT count(*) || '|' || md5(string_agg(id::text, ',' ORDER BY id)) FROM tr WHERE ctid >= '$LAST'" \ + -c "ROLLBACK" 2>&1 | grep '|' +} +check "premise: inside a transaction the range still plans a TID range scan" \ + "$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -c "BEGIN" -c "INSERT INTO tr VALUES (0, 'x')" -c "EXPLAIN SELECT id FROM tr WHERE ctid >= '$LAST'" -c "ROLLBACK" | + grep -c 'Tid Range Scan')" "1" +check "rows inserted earlier in the same transaction are in the range, as the filter sees them" \ + "$(same_xact on)" "$(same_xact off)" + # PostgreSQL 19 runs a TID range scan in parallel: each worker sets the range # and the reader's shared group claim hands out only the restricted groups. The # custom scan is turned off because, with parallelism cheap, the planner prefers From 23a25abff19973fd39ac618cddcdf20cd0d4d6d6 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 23:04:31 +0000 Subject: [PATCH 5/5] test: the TID range scan ledger replayed after #1304, with the cursor arms (#1306) 36 rows from a five-major run on this tree; 17 dated by ten mutations of columnar_tableam.c (no planner hook and SCROLL-only among them); census 1674 -> 1693. TESTS.md and CHANGELOG describe the cursor fix. cluster_tests 500 by collection. harness_selftest PG17 1223 passed. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +++ test/check_ledger.tsv | 36 ++++++++++++++++++++++++++++++++++++ test/check_ledger_budget.txt | 17 ++++++++++++++++- test/pytest/TESTS.md | 8 ++++++-- 4 files changed, 61 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21e2979c6..13fa9e4e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -84,6 +84,9 @@ true until the next version shipped. 291 a block holds, which the executor produces for `ctid < '(B,0)'`), rescans, deletes and compaction. The first build read those offsets unclamped and returned rows of the next block; the twin's arms for them were red against it. + A cursor over a ctid range gets a Material above the scan, so `FETCH + BACKWARD` and `FETCH LAST` work: PostgreSQL treats a TID range scan as able to + scan backward, and without it the fetch reached the access method and failed. ### Changed diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 7e1c6e20b..5d60165b8 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -1880,6 +1880,42 @@ storage_row_repair storage_row_repair premise: no row names a missing relation b storage_row_repair storage_row_repair premise: the damaged table still reads every row 15;16;17;18;19 never - storage_row_repair storage_row_repair premise: the release notes give the query that finds wrong rows 15;16;17;18;19 never - storage_row_repair storage_row_repair the notes' query finds both rows alpha4 left wrong 15;16;17;18;19 2026-10-12 the release notes' query is edited from NOT EXISTS to EXISTS +tid_range_scan tid_range_scan a SCROLL cursor over a ctid range fetches backward as the filter's plan does 15;16;17;18;19 2026-10-12 no planner hook: a cursor over a TID range scan gets no Material +tid_range_scan tid_range_scan a TID range scan rescanned with new bounds returns what the filter returns 15;16;17;18;19 2026-10-12 getnextslot_tidrange without the boundary-row filter;scan_set_tidrange keeping the reader of the previous range +tid_range_scan tid_range_scan a ctid range over the last row groups reads under a fifth of the table 15;16;17;18;19 2026-10-12 scan_set_tidrange without the group restriction +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid < '(515,0)' 15;16;17;18;19 2026-10-12 a lower bound at offset 0 read unclamped;an upper bound's offset past 291 read unclamped;getnextslot_tidrange without the boundary-row filter +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid < '(515,1)' 15;16;17;18;19 2026-10-12 a lower bound at offset 0 read unclamped;getnextslot_tidrange without the boundary-row filter +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid < '(515,137)' 15;16;17;18;19 2026-10-12 a lower bound at offset 0 read unclamped +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid <= '(0,0)' 15;16;17;18;19 2026-10-12 an upper bound below row 0 read as the last row +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid <= '(515,0)' 15;16;17;18;19 2026-10-12 a lower bound at offset 0 read unclamped;getnextslot_tidrange without the boundary-row filter +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid <= '(515,400)' 15;16;17;18;19 2026-10-12 a lower bound at offset 0 read unclamped;an upper bound's offset past 291 read unclamped;getnextslot_tidrange without the boundary-row filter +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid > '(1374,167)' 15;16;17;18;19 never - +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid > '(515,0)' 15;16;17;18;19 2026-10-12 getnextslot_tidrange without the boundary-row filter +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid > '(515,65535)' 15;16;17;18;19 2026-10-12 a lower bound at offset 0 read unclamped;getnextslot_tidrange without the boundary-row filter +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid >= '(0,0)' 15;16;17;18;19 2026-10-12 a lower bound at offset 0 read unclamped +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid >= '(515,137)' AND ctid <= '(584,57)' 15;16;17;18;19 never - +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid >= '(515,291)' AND ctid <= '(515,292)' 15;16;17;18;19 2026-10-12 an upper bound's offset past 291 read unclamped;getnextslot_tidrange without the boundary-row filter +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid >= '(515,400)' 15;16;17;18;19 2026-10-12 a lower bound's offset past 291 read unclamped;getnextslot_tidrange without the boundary-row filter +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid >= '(515,7)' AND ctid < '(515,7)' 15;16;17;18;19 never - +tid_range_scan tid_range_scan a ctid range returns what the per-row filter returns: ctid >= 'LAST' 15;16;17;18;19 never - +tid_range_scan tid_range_scan a parallel TID range scan returns what the filter returns 15;16;17;18;19 2026-10-12 scan_set_tidrange dropping the parallel scan's shared group claim +tid_range_scan tid_range_scan a plain cursor over a ctid range answers a backward fetch correctly 15;16;17;18;19 2026-10-12 no planner hook: a cursor over a TID range scan gets no Material;the planner hook covering SCROLL cursors only, not a plain DECLARE +tid_range_scan tid_range_scan after compaction a ctid range returns what the filter returns: ctid >= '(1477,195)' 15;16;17;18;19 never - +tid_range_scan tid_range_scan after compaction a ctid range returns what the filter returns: ctid >= '(515,137)' AND ctid <= '(1477,195)' 15;16;17;18;19 2026-10-12 getnextslot_tidrange without the boundary-row filter +tid_range_scan tid_range_scan after compaction a ctid range returns what the filter returns: ctid >= 'LAST' 15;16;17;18;19 never - +tid_range_scan tid_range_scan and rows deleted inside the range stay deleted 15;16;17;18;19 never - +tid_range_scan tid_range_scan premise: a ctid range plans a TID range scan, so the arms below exercise it 15;16;17;18;19 never - +tid_range_scan tid_range_scan premise: compaction gave the surviving rows new addresses 15;16;17;18;19 never - +tid_range_scan tid_range_scan premise: compaction rewrote the group the delete thinned 15;16;17;18;19 never - +tid_range_scan tid_range_scan premise: inside a transaction the range still plans a TID range scan 15;16;17;18;19 never - +tid_range_scan tid_range_scan premise: reading the whole table reads its blocks 15;16;17;18;19 never - +tid_range_scan tid_range_scan premise: the cursor's query plans a TID range scan 15;16;17;18;19 never - +tid_range_scan tid_range_scan premise: the delete removed rows from the range 15;16;17;18;19 never - +tid_range_scan tid_range_scan premise: the per-range join rescans a TID range scan 15;16;17;18;19 never - +tid_range_scan tid_range_scan premise: the range runs as a parallel TID range scan with two workers 19 never - +tid_range_scan tid_range_scan premise: the table has many row groups, so skipping can show 15;16;17;18;19 never - +tid_range_scan tid_range_scan premise: the table holds every row 15;16;17;18;19 never - +tid_range_scan tid_range_scan rows inserted earlier in the same transaction are in the range, as the filter sees them 15;16;17;18;19 never - validity_elision validity_elision a chunk claiming no bitmap while it holds fewer values than rows is refused 15;16;17;18;19 never - validity_elision validity_elision a column with no nulls stores no validity bitmap 15;16;17;18;19 2026-09-18 writer: flush_one_column's presentCount == rowCount forced false, so the bitmap is written even when the chunk holds no null validity_elision validity_elision a null-free column elides its bitmap beside a null-bearing one in the same row group 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 90d0a683b..9001004e8 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1296,4 +1296,19 @@ suites_not_covered 247 # never 1671 + 3 = 1674 # dated 224 + 9 = 233 # 1674 + 233 == 1907 -checks_never_observed_red 1674 +# +# TID range scans, #1306 (OffgridwithJD): 1674 -> 1693. tid_range_scan enters the +# ledger with 36 checks from a five-major run on this tree (after #1304); the +# parallel plan premise runs on 19 only. Seventeen are dated by ten mutations of +# src/columnar_tableam.c, each merged one log at a time with --mutation and +# --target (the parallel claim on PG19, the rest on PG17; #1304 did not touch +# that file). Nineteen arrive `never`: eleven premises, and eight arms no mutant +# of this change reddens -- ranges whose bounds are real rows, the empty range, +# the deleted rows, and rows inserted earlier in the same transaction -- kept +# because they hold the access method against the per-row filter. +# +# ledger rows 1907 + 36 = 1943 +# never 1674 + 19 = 1693 +# dated 233 + 17 = 250 +# 1693 + 250 == 1943 +checks_never_observed_red 1693 diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 014250448..d2d3d9c20 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -6494,7 +6494,9 @@ The oracle is the same query with `enable_tidscan = off`, which is that per-row THE BOUNDS THAT MATTER ARE THE ONES NO ROW HAS. A block holds 291 rows, at offsets 1 to 291, but a bound may name offset 0 or anything up to 65535. The executor turns `ctid < '(B,0)'` into an upper bound of `(B-1,65535)`. The first build read those offsets unclamped, so offset 400 named a row in the next block. The shell twin's arms for those bounds went red against it, which is how it was found. -Also held: a rescan with new bounds per outer row (the plan premise is taken on the very query compared, because a draft whose lateral output went unreferenced planned no scan at all); deletes inside the range; and compaction, which gives a thinned group's surviving rows new, higher row numbers. On PostgreSQL 19, a parallel TID range scan with two workers is held too. Below 19 that test is `UNSUPPORTED_MAJOR` and listed in `expected_unrunnable.txt`. +CURSORS. PostgreSQL counts a TID range scan as able to scan backward, so it puts no Material above one under a cursor that may scroll, and a backward fetch reached the access method, which refused it (`0A000`). Found by @jdatcmd reviewing #1306. A planner hook now puts a Material above a cursor's plan that holds a TID range scan of a columnar table. The arms: a SCROLL cursor's `FETCH LAST`, `BACKWARD` and `ABSOLUTE` match the filter's plan, and a plain DECLARE (which the portal makes scrollable) answers a backward fetch correctly. Both were red before the hook. + +Also held: rows inserted earlier in the same transaction are in the range; a rescan with new bounds per outer row (the plan premise is taken on the very query compared, because a draft whose lateral output went unreferenced planned no scan at all); deletes inside the range; and compaction, which gives a thinned group's surviving rows new, higher row numbers. On PostgreSQL 19, a parallel TID range scan with two workers is held too. Below 19 that test is `UNSUPPORTED_MAJOR` and listed in `expected_unrunnable.txt`. On a private database, because `pg_stat_reset()` resets the whole database's counters. @@ -6505,10 +6507,12 @@ Removal proofs, each red on PostgreSQL 19 on the arm named: - without the boundary-row filter: the tail range and the parallel arm; - with the reader kept across a rescan: the rescan arm; - without the shared parallel claim: the parallel arm, which returned every row twice. +- without the planner hook: both cursor arms; +- with the hook for SCROLL cursors only: the plain-cursor arm. ### Every test | test | what it holds | | --- | --- | -| `test_a_ctid_range_reads_only_its_row_groups` | the table has its rows and many groups; a ctid range plans a TID range scan; eleven ranges, including offsets 0, 291, 500 and 65535, return what the per-row filter returns; a rescanned scan does too; the tail range reads at least one block and under a fifth of the table; deleted rows stay deleted; after compaction moves rows, ranges over the old and new addresses return what the filter returns | +| `test_a_ctid_range_reads_only_its_row_groups` | the table has its rows and many groups; a ctid range plans a TID range scan; eleven ranges, including offsets 0, 291, 500 and 65535, return what the per-row filter returns; a rescanned scan does too; a SCROLL cursor and a plain cursor fetch backward correctly; rows inserted earlier in the transaction are in the range; the tail range reads at least one block and under a fifth of the table; deleted rows stay deleted; after compaction moves rows, ranges over the old and new addresses return what the filter returns | | `test_a_parallel_tid_range_scan_returns_what_the_filter_returns` | PostgreSQL 19+: the range runs as a parallel TID range scan with two workers launched, and returns what the filter returns. Below 19, `UNSUPPORTED_MAJOR` |