test: audit.sh's two index arms read the index they name (#1339)

test/audit.sh

@@ -213,9 +213,26 @@ check "add-column aggregate" \
 	"$(q "SELECT sum(c) || '/' || count(d) || '/' || count(b) FROM ac;")" "218/6/1"
 
 # index scan over an added column
+#
+# FORCED THROUGH THE INDEX, AND ASSERTED (#1339). enable_seqscan alone leaves the
+# columnar custom scan on, so this and "create index rows indexed once" below could
+# be answered without reading the index they name. Both lookups now turn the
+# custom scan and bitmap scans off (a bitmap scan would also collapse a duplicate
+# index entry), and assert an Index Scan or Index Only Scan through their index.
+IDXONLY="SET enable_seqscan=off; SET enable_bitmapscan=off; SET pgcolumnar.enable_custom_scan=off;"
+reads_index() {	# reads_index INDEX QUERY -> yes, or the plan's scan line
+	local plan
+	plan="$(q "$IDXONLY EXPLAIN (COSTS OFF) $2")"
+	case "$plan" in
+		*"Index Scan using $1 "*|*"Index Only Scan using $1 "*) echo yes ;;
+		*) echo "no: $(grep -m1 'Scan' <<<"$plan" | sed 's/^ *//')" ;;
+	esac
+}
 q "CREATE INDEX ac_c_idx ON ac(c);" >/dev/null
+check "premise: the add-column lookup reads ac_c_idx" \
+	"$(reads_index ac_c_idx "SELECT count(*) FROM ac WHERE c=42")" "yes"
 check "add-column index scan" \
-	"$(q "SET enable_seqscan=off; SELECT count(*) FROM ac WHERE c=42;")" "5"
+	"$(q "$IDXONLY SELECT count(*) FROM ac WHERE c=42;" | tail -1)" "5"
 q "DROP TABLE ac;" >/dev/null
 
 # ---------------------------------------------------------------------------
@@ -369,8 +386,10 @@ LEAKS="$(run_pg "grep -c 'resource was not closed' '$LOGFILE'" || true)"
 check "create index no relation leak" "$LEAKS" "0"
 
 # and every row is indexed exactly once (no duplicate TIDs from a per-worker rescan)
+check "premise: the indexed-once count reads idxleak_a_idx" \
+	"$(reads_index idxleak_a_idx "SELECT count(*) FROM idxleak WHERE a>0")" "yes"
 check "create index rows indexed once" \
-	"$(q "SET enable_seqscan=off; SELECT count(*) FROM idxleak WHERE a>0;")" "50000"
+	"$(q "$IDXONLY SELECT count(*) FROM idxleak WHERE a>0;" | tail -1)" "50000"
 q "DROP TABLE idxleak;" >/dev/null
 
 # ---------------------------------------------------------------------------

test/check_ledger.tsv

@@ -2479,6 +2479,21 @@ native_encdesc_golden	native_encdesc_golden	int column a rawLen parses as 1200 (
 native_encdesc_golden	native_encdesc_golden	the table reads back all rows	15;16;17;18;19	never	-
 native_encdesc_golden	native_encdesc_golden	trailing sharedTableLen is 0 (u32 at offset 6+13=19)	15;16;17;18;19	never	-
 native_encdesc_golden	native_encdesc_golden	vectorCount parses as 1 (at offset 2)	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	a narrow projection over a wide group is cached too (#433)	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	columns that fit are cached even when their group does not (#433)	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	every payload survived the round trip at full length	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: fetching one row touched a measurable number of buffers	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: fetching ten rows touched a measurable number of buffers	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: one row is fetched through an index scan	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: ten rows are fetched through an index scan	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the fixture really has a row group above the cap	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the medium projection is fetched through an index scan	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the medium projection touched a measurable number of buffers	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the narrow projection is fetched through an index scan	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the narrow projection touched a measurable number of buffers	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the table loaded every row	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the wide column is the one over the cap, so this arm is #359's	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	the fetched rows are the right rows	15;16;17;18;19	2026-10-16	columnar_tableam.c: the deferred slot decode reads tag as NULL on every 7th row number
 native_fetch_cache	native_fetch_cache	a hit re-checks the group's fileOffset against the row group	15;16;17;18;19	2026-10-16	columnar_reader.c: a fetch-cache hit no longer re-checks fileOffset
 native_fetch_cache	native_fetch_cache	a hit re-checks the group's firstRowNumber against the row group	15;16;17;18;19	2026-10-16	columnar_reader.c: a fetch-cache hit no longer re-checks firstRowNumber
 native_fetch_cache	native_fetch_cache	a hit re-checks the group's natts against the scan's column count	15;16;17;18;19	2026-10-16	columnar_reader.c: a fetch-cache hit no longer re-checks natts
@@ -3243,21 +3258,22 @@ partition_lifecycle	partition_lifecycle	running the plan leaves only the partiti
 partition_lifecycle	partition_lifecycle	the parent now has the partitions it reported	15;16;17;18;19	2026-10-14	partition_create_ahead creating its last partition without attaching it
 partition_lifecycle	partition_lifecycle	the plan retires the partitions wholly before the cutoff, in range order	15;16;17;18;19	2026-10-13	a plain DROP, not refusing while the partition is still attached;the cutoff filter removed: every partition planned;the lower bound read where the upper is meant;the plan detaches but drops nothing;the plan detaches without CONCURRENTLY;the plan in catalog order, not range order
 partition_lifecycle	partition_lifecycle	they are contiguous, each one width wide	15;16;17;18;19	2026-10-12	partition_create_ahead leaving a minute between partitions
-partition_rows	partition_rows	a duplicate key inserted through the parent is refused (23505), as on heap	15;16;17;18;19	never	-
-partition_rows	partition_rows	a duplicate of a moved row's new key is refused in its new partition (23505), as on heap	15;16;17;18;19	never	-
+partition_rows	partition_rows	a duplicate key inserted through the parent is refused (23505), as on heap	15;16;17;18;19	2026-10-16	columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row
+partition_rows	partition_rows	a duplicate of a moved row's new key is refused in its new partition (23505), as on heap	15;16;17;18;19	2026-10-16	columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row
 partition_rows	partition_rows	a key a moved row gave up can be used again, as on heap	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete)
-partition_rows	partition_rows	a second insert of the same new key through the parent waits, then is refused (23505), and one row is left, as on heap	15;16;17;18;19	never	-
+partition_rows	partition_rows	a second insert of the same new key through the parent waits, then is refused (23505), and one row is left, as on heap	15;16;17;18;19	2026-10-16	columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row
 partition_rows	partition_rows	a second session moving a row the first is moving waits, then fails with 40001, and the row exists once, as on heap	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete);the tree before #1421: a row moved between partitions is serialized only at flush, after its new version's unique check
 partition_rows	partition_rows	an UPDATE in place of a row another session is moving waits, then fails with 40001, as on heap	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete)
 partition_rows	partition_rows	an UPDATE moving 25,000 keyed rows to the other partition keeps under 32 MB likewise	15;16;17;18;19	2026-10-15	PgColumnarSerializeRowNow reads the delete vector into the caller's context, keeping a bitmap copy per row (the tree before the #1422 review fix)
 partition_rows	partition_rows	an UPDATE moving 49 rows to the other partition succeeds, as on heap	15;16;17;18;19	never	-
 partition_rows	partition_rows	an UPDATE of 25,000 keyed rows keeps under 32 MB of memory between its first and last row	15;16;17;18;19	2026-10-15	PgColumnarSerializeRowNow reads the delete vector into the caller's context, keeping a bitmap copy per row (the tree before the #1422 review fix)
-partition_rows	partition_rows	and after the races the parents hold the same rows	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete)
+partition_rows	partition_rows	and after the races the parents hold the same rows	15;16;17;18;19	2026-10-16	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete);columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row
 partition_rows	partition_rows	and each UPDATE reached every row once	15;16;17;18;19	never	-
 partition_rows	partition_rows	and each partition holds what heap's does	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete)
 partition_rows	partition_rows	and the parent holds the same rows as heap's	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete)
 partition_rows	partition_rows	each keyed table's group carries delete marks before its UPDATE	15;16;17;18;19	never	-
-partition_rows	partition_rows	the primary key finds a moved row at its new key, and not at its old one, as on heap	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete)
+partition_rows	partition_rows	premise: the columnar key lookup reads the primary key	15;16;17;18;19	never	-
+partition_rows	partition_rows	the primary key finds a moved row at its new key, and not at its old one, as on heap	15;16;17;18;19	2026-10-16	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete);columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row
 partman_columnar	partman_columnar	and no row-group row behind	15;16;17;18;19	2026-10-12	the drop hook stops deleting row_group rows, so they outlive their deleted storage row
 partman_columnar	partman_columnar	and read back as heap does	15;16;17;18;19	2026-10-12	a sequential scan drops the last row group from its list, so every one-group partition reads empty
 partman_columnar	partman_columnar	and the rows land in columnar partitions	15;16;17;18;19	2026-10-12	a sequential scan drops the last row group from its list, so every one-group partition reads empty
@@ -3883,6 +3899,35 @@ settings_reference	settings_reference	each documented default is the server's	15
 settings_reference	settings_reference	every documented setting exists on the server	15;16;17;18;19	2026-10-15	the compression_level row renamed compression_levl in docs/configuration.md
 settings_reference	settings_reference	every setting the server defines is documented, the developer hooks aside	15;16;17;18;19	2026-10-15	the backticks around every setting name removed from docs/configuration.md;the compression_level row deleted from docs/configuration.md;the compression_level row renamed compression_levl in docs/configuration.md
 settings_reference	settings_reference	the reference documents at least thirty settings, so the comparisons below read something	15;16;17;18;19	2026-10-15	the backticks around every setting name removed from docs/configuration.md
+sorted_projection	sorted_projection	control: the set oracle is order-blind by design	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	count matches heap	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	idempotent sorted rewrite	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	index point lookup after sort	15;16;17;18;19	2026-10-16	columnar_vacuum.c: pgcolumnar_compact_relation no longer reindexes after its rewrite
+sorted_projection	sorted_projection	index sees deletion after sort	15;16;17;18;19	2026-10-16	columnar_vacuum.c: pgcolumnar_compact_relation no longer reindexes after its rewrite
+sorted_projection	sorted_projection	live count after delete+sort	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	multi-key range	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	post aggregate	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	post equality	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	post order-by	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	post range	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	post star	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	post-sort prunes more chunk groups	15;16;17;18;19	2026-10-16	columnar_vacuum.c: pgcolumnar_compact_relation never sorts the rows it rewrites
+sorted_projection	sorted_projection	pre aggregate	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	pre equality	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	pre order-by	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	pre range	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	pre star	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	pre-sort prunes little	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	premise: the deletion lookup reads t_col_id_idx	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	premise: the ordered oracle agrees with itself	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	premise: the ordered oracle is order-sensitive	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	premise: the point lookup reads t_col_id_idx	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	reject no sort columns	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	reject non-columnar table	15;16;17;18;19	2026-10-16	columnar_vacuum.c: pgcolumnar_vacuum_sorted no longer checks for a columnar relation
+sorted_projection	sorted_projection	reject non-sortable type	15;16;17;18;19	2026-10-16	columnar_vacuum.c: pgcolumnar_compact_relation never sorts the rows it rewrites
+sorted_projection	sorted_projection	reject unknown column	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	sum matches heap	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	the plan under test is a columnar custom scan	15;16;17;18;19	never	-
 statement_self_visibility	statement_self_visibility	INSERT INTO t SELECT FROM t runs to completion	15;16;17;18;19	2026-10-15	a serial scan re-reads its row-group list when it reaches the end, so it picks up groups its own statement flushed (src/columnar_reader.c pgcolumnar_next_group_index)
 statement_self_visibility	statement_self_visibility	UPDATE of every row runs to completion	15;16;17;18;19	2026-10-15	a serial scan re-reads its row-group list when it reaches the end, so it picks up groups its own statement flushed (src/columnar_reader.c pgcolumnar_next_group_index)
 statement_self_visibility	statement_self_visibility	an INSERT in a WITH clause reading its own table returns the rows it inserted, as heap does	15;16;17;18;19	2026-10-15	a serial scan re-reads its row-group list when it reaches the end, so it picks up groups its own statement flushed (src/columnar_reader.c pgcolumnar_next_group_index)

test/check_ledger_budget.txt

@@ -15,7 +15,7 @@
 #
 #   Adding a check to a suite that is already covered does not move it, which is
 #   what makes it safe to bound.
-suites_not_covered 193
+suites_not_covered 191
 #
 # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become
 #   one.
@@ -3467,4 +3467,42 @@ suites_not_covered 193
 #     never         3019 + 2 - 4 = 3017
 #     dated         1311 + 4 = 1315
 #     3017 + 1315 == 4332
-checks_never_observed_red 3017
+#
+# #1341, native_fetch_bigcap seeded, with its twin (OffgridwithJD): 3017 -> 3031,
+# and suites_not_covered 193 -> 192. The suite had no rows. It enters with all 15
+# checks from five-major runs (15/15 on each of 15-19). One is dated: the right-rows
+# arm, by the deferred decode reading tag as NULL on every 7th row number. Fourteen
+# arrive never: eleven premises; the payload length; and the two #433 ratios, which
+# the cache missing every lookup and the cap dropping every column cannot move,
+# because each re-reads only the projected columns.
+#
+#     ledger rows   4332 + 15 = 4347
+#     never         3017 + 14 = 3031
+#     dated         1315 +  1 = 1316
+#     3031 + 1316 == 4347
+#
+# #1339, partition_rows' primary-key arm reads the key it names (OffgridwithJD):
+# 3031 -> 3029. One premise is new, never. With pgcolumnar_index_fetch_tuple
+# finding no row, five arms are red: the three duplicate-key refusals entered never
+# and are now dated, and the primary-key and races arms gain it in their history.
+#
+#     ledger rows   4347 + 1 = 4348
+#     never         3031 + 1 - 3 = 3029
+#     dated         1316 + 3 = 1319
+#     3029 + 1319 == 4348
+#
+# #1341, sorted_projection seeded, with its twin (OffgridwithJD): 3029 -> 3053, and
+# suites_not_covered 192 -> 191. The suite had no rows. It enters with all 29 checks
+# from five-major runs (29/29 on each of 15-19). 5 are dated by three mutations:
+# the post-sort reindex removed (both index lookups), the compaction not sorting
+# (the pruning arm and the json refusal, which the sort raises), and vacuum_sorted's
+# columnar check removed. Twenty-four arrive never: the ten heap-oracle comparisons
+# and the multi-key and post-delete counts, which no rewrite mutation here moves;
+# the oracle and index premises; the pre-sort control; idempotence; and the
+# unknown-column and no-sort-key refusals.
+#
+#     ledger rows   4348 + 29 = 4377
+#     never         3029 + 24 = 3053
+#     dated         1319 +  5 = 1324
+#     3053 + 1324 == 4377
+checks_never_observed_red 3053

test/native_fetch_bigcap.sh

@@ -33,8 +33,13 @@ CAP=$((32 * 1024 * 1024))	# COLUMNAR_FETCH_CACHE_MAX_BYTES, src/columnar.h
 # by its own model, and then nothing measures the fetch path at all. So the index
 # path is forced, and every measured query asserts it got one. Whether the
 # planner should choose it is #434, and that is a separate suite.
+#
+# pgcolumnar.enable_custom_scan=off as well: enable_seqscan alone does not turn
+# the columnar custom scan off (#1339). Every measured query asserts its plan, so
+# a planner change cannot quietly take the arms off the fetch path.
 FORCE_INDEX="SET max_parallel_workers_per_gather=0; SET enable_seqscan=off;
-             SET enable_bitmapscan=off; SET pgcolumnar.enable_index_fetch_penalty=off;"
+             SET enable_bitmapscan=off; SET pgcolumnar.enable_index_fetch_penalty=off;
+             SET pgcolumnar.enable_custom_scan=off;"
 
 # ---------------------------------------------------------------------------
 # The fixture, whose premise is its ON-DISK SIZE.
@@ -144,6 +149,11 @@ nbufs() {	# $1 = number of rows, projecting one narrow column
 		} } END { print t + 0 }'
 }
 
+# The narrow and medium projections had no plan premise of their own; only the
+# payload query's plan was asserted (#1339). They plan as Index Scans today, and
+# these keep a planner change from turning their ratios into scan-against-scan.
+check_num "premise: the narrow projection is fetched through an index scan" \
+	"$(pgc_uses_row_fetch "$FORCE_INDEX" "SELECT sum(tag) FROM bigcap WHERE id BETWEEN 1 AND 10" | grep -c yes)" "1"
 N1=$(nbufs 1)
 N10=$(nbufs 10)
 echo "-- narrow projection buffers: 1 row = ${N1}, 10 rows = ${N10}"
@@ -172,6 +182,8 @@ mbufs() {	# $1 = number of rows, projecting three medium columns
 		} } END { print t + 0 }'
 }
 
+check_num "premise: the medium projection is fetched through an index scan" \
+	"$(pgc_uses_row_fetch "$FORCE_INDEX" "SELECT count(m1 || m2 || m3) FROM bigcap WHERE id BETWEEN 1 AND 10" | grep -c yes)" "1"
 M1=$(mbufs 1)
 M10=$(mbufs 10)
 echo "-- three medium columns from an oversized group: 1 row = ${M1}, 10 rows = ${M10}"