test: native_index_projection gets its pytest twin, and every oracle reads the index it names (#1341)

test/check_ledger.tsv

@@ -2570,6 +2570,47 @@ native_groupagg	native_groupagg	toggle sum signed-zero (-0 preserved) [node fire
 native_groupagg	native_groupagg	while a matching-type predicate still gets one	15;16;17;18;19	never	-
 native_groupagg	native_groupagg	with no time bound at all, a truncating key is bounded from the zone maps	15;16;17;18;19	never	-
 native_groupagg	native_groupagg	with only a lower time bound, a truncating key is still bounded	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	a parallel build projects one column of twenty, not all twenty	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	a parallel expression build projects its expression's columns	15;16;17;18;19	2026-10-16	columnar_tableam.c: the index build's projection omits ii_Expressions' columns
+native_index_projection	native_index_projection	a parallel partial build projects the predicate's columns too	15;16;17;18;19	2026-10-16	columnar_tableam.c: the index build's projection omits ii_Predicate's columns
+native_index_projection	native_index_projection	a partial index projects the predicate's columns too	15;16;17;18;19	2026-10-16	columnar_tableam.c: the index build's projection omits ii_Predicate's columns
+native_index_projection	native_index_projection	an expression index projects its expression's columns	15;16;17;18;19	2026-10-16	columnar_tableam.c: the index build's projection omits ii_Expressions' columns
+native_index_projection	native_index_projection	an expression over a text column projects that column	15;16;17;18;19	2026-10-16	columnar_tableam.c: the index build's projection omits ii_Expressions' columns
+native_index_projection	native_index_projection	an index on a late column projects one, not everything before it	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	bt_index_check(w_c18)	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	bt_index_check(w_expr)	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	bt_index_check(w_k)	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	bt_index_check(w_k12)	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	bt_index_check(w_len)	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	bt_index_check(w_par)	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	bt_index_check(w_par2)	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	bt_index_check(w_pare)	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	bt_index_check(w_parp)	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	bt_index_check(w_part)	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	bt_index_check(w_ser)	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	expression index agrees	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	expression index finds the row	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	fixture is wide	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	fixture rows	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	forcing parallel maintenance workers does reach the parallel branch	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	late-column index agrees	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	parallel-built index agrees with a sequential scan	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	parallel-built index returns each row once	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	partial index agrees on its subset	15;16;17;18;19	2026-10-16	columnar_tableam.c: the index build's projection omits ii_Predicate's columns
+native_index_projection	native_index_projection	partial index finds the row inside its predicate	15;16;17;18;19	2026-10-16	columnar_tableam.c: the index build's projection omits ii_Predicate's columns
+native_index_projection	native_index_projection	plain index agrees with a sequential scan	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	plain index finds the row	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	plain index projects one column of twenty	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	premise: expression index agrees -- the index side reads w_expr	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	premise: late-column index agrees -- the index side reads w_c18	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	premise: parallel-built index agrees with a sequential scan -- the index side reads w_par2	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	premise: partial index agrees on its subset -- the index side reads w_part	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	premise: plain index agrees with a sequential scan -- the index side reads w_k(12)?	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	premise: text expression index agrees -- the index side reads w_len	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	text expression index agrees	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	the oracle below really reads the parallel-built index	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	the serial branch is still reached when workers are refused	15;16;17;18;19	never	-
+native_index_projection	native_index_projection	two-column index projects two	15;16;17;18;19	never	-
 native_join_runtime_filter	native_join_runtime_filter	3-table answer equals filter-off	15;16;17;18;19	2026-09-23	the runtime bloom probe's verdict inverted, so the filter rejects what matches
 native_join_runtime_filter	native_join_runtime_filter	3-table join order matches filter-off	15;16;17;18;19	never	-
 native_join_runtime_filter	native_join_runtime_filter	3-table plan has coordinator	15;16;17;18;19	never	-

test/check_ledger_budget.txt

@@ -15,7 +15,7 @@
 #
 #   Adding a check to a suite that is already covered does not move it, which is
 #   what makes it safe to bound.
-suites_not_covered 203
+suites_not_covered 202
 #
 # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become
 #   one.
@@ -3241,4 +3241,19 @@ suites_not_covered 203
 #     never         2894 - 5 = 2889
 #     dated         1113 + 5 = 1118
 #     2889 + 1118 == 4007
-checks_never_observed_red 2889
+#
+# #1341, native_index_projection seeded, with its twin (OffgridwithJD): 2889 -> 2923,
+# and suites_not_covered 203 -> 202. The suite had no rows. It enters with all 41
+# checks from five-major runs (41/41 on each of 15-19). 7 are dated by two mutations of
+# the index build's projection: ii_Expressions' columns omitted (the three expression
+# projection arms) and ii_Predicate's omitted (the partial projections, its lookup and
+# its oracle). Thirty-four arrive never: the six new oracle premises, which a harness
+# change reddens (an index dropped before the oracles), the fixture and the serial and
+# parallel branch arms, the plain and late-column projections, the oracles a projection
+# mutation cannot reach, and the eleven amcheck arms.
+#
+#     ledger rows   4007 + 41 = 4048
+#     never         2889 + 34 = 2923
+#     dated         1118 +  7 = 1125
+#     2923 + 1125 == 4048
+checks_never_observed_red 2923

test/native_index_projection.sh

@@ -78,12 +78,13 @@ check "an expression over a text column projects that column" \
 
 # Correctness. An index built from under-projected data indexes unset slot values, so
 # each of these must find what a sequential scan finds.
+# The custom scan off as well: measured, the expression lookup ran as a Custom Scan.
 check "plain index finds the row" \
-	"$(qset 'SET enable_seqscan=off' 'SELECT k2 FROM w WHERE k = 12345')" "24690"
+	"$(qset 'SET enable_seqscan=off; SET pgcolumnar.enable_custom_scan=off' 'SELECT k2 FROM w WHERE k = 12345')" "24690"
 check "expression index finds the row" \
-	"$(qset 'SET enable_seqscan=off' 'SELECT count(*) FROM w WHERE (k + k2) = 30000')" "1"
+	"$(qset 'SET enable_seqscan=off; SET pgcolumnar.enable_custom_scan=off' 'SELECT count(*) FROM w WHERE (k + k2) = 30000')" "1"
 check "partial index finds the row inside its predicate" \
-	"$(qset 'SET enable_seqscan=off' 'SELECT count(*) FROM w WHERE k = 50000 AND k2 > 100')" "1"
+	"$(qset 'SET enable_seqscan=off; SET pgcolumnar.enable_custom_scan=off' 'SELECT count(*) FROM w WHERE k = 50000 AND k2 > 100')" "1"
 
 # The oracle, depending on nothing but PostgreSQL. Compare the FULL ordered result of a
 # forced index scan against a forced seq scan.
@@ -96,26 +97,38 @@ check "partial index finds the row inside its predicate" \
 # The emptiness half is `check_text` in `test/lib.sh` now (#418, #422), so this file no
 # longer carries its own version of it. The shape half stays here: an md5 is 32 hex
 # characters, and this file knows that where the shared helper only knows "not empty".
-agree() {  # $1 label, $2 predicate, $3 selected expression
-	local viaix viaseq
-	viaix=$(qset "SET enable_seqscan=off; SET enable_bitmapscan=off" \
-	              "SELECT md5(string_agg(t::text, ',' ORDER BY t))
-	                 FROM (SELECT $3 AS t FROM w WHERE $2) s")
-	viaseq=$(qset "SET enable_indexscan=off; SET enable_bitmapscan=off; SET enable_indexonlyscan=off" \
-	              "SELECT md5(string_agg(t::text, ',' ORDER BY t))
-	                 FROM (SELECT $3 AS t FROM w WHERE $2) s")
+# THE INDEX SIDE MUST READ THE INDEX (#1341). enable_seqscan=off does not disable the
+# columnar custom scan, and measured, the late-column, expression and text-expression
+# oracles ran as Custom Scan on BOTH sides: a sequential scan compared with itself,
+# whatever the index held. So the custom scan is off too, and each oracle names the
+# index it must read and asserts the plan does.
+IXOFF="SET enable_seqscan=off; SET enable_bitmapscan=off; SET pgcolumnar.enable_custom_scan=off"
+#
+# The premise reads the plan of the oracle query ITSELF, fenced with OFFSET 0. Without
+# the fence the aggregate's ORDER BY asked for rows in key order, and the planner took
+# a full scan of w_k12 with the late-column predicate as a filter: every row fetched
+# through the index, two minutes at 60,000 rows and hours at this size.
+agree() {  # $1 label, $2 predicate, $3 selected expression, $4 the index it must read
+	local viaix viaseq ixplan oq
+	oq="SELECT md5(string_agg(t::text, ',' ORDER BY t)) FROM (SELECT $3 AS t FROM w WHERE $2 OFFSET 0) s"
+	ixplan="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \
+		-At -q -c "$IXOFF" -c "EXPLAIN (COSTS OFF) $oq" 2>/dev/null)"
+	check "premise: $1 -- the index side reads $4" \
+		"$(grep -cE "Index (Only )?Scan using $4 on w" <<<"$ixplan")" "1"
+	viaix=$(qset "$IXOFF" "$oq")
+	viaseq=$(qset "SET enable_indexscan=off; SET enable_bitmapscan=off; SET enable_indexonlyscan=off" "$oq")
 	if ! grep -qE '^[0-9a-f]{32}$' <<<"$viaix" || ! grep -qE '^[0-9a-f]{32}$' <<<"$viaseq"; then
-		check "$1 (both sides must be a real md5, not empty or an error)" \
-			"index=[$viaix] seq=[$viaseq]" "two md5 hashes"
+		# under the oracle's own name, so a red here is a red of that check
+		check "$1" "index=[$viaix] seq=[$viaseq] (both sides must be a real md5)" "two md5 hashes"
 		return
 	fi
 	check_text "$1" "$viaix" "$viaseq"
 }
-agree "plain index agrees with a sequential scan"   "k BETWEEN 1000 AND 9999"        "k"
-agree "late-column index agrees"                    "c18 > repeat('x',80)||'99000'"  "k"
-agree "expression index agrees"                     "(k + k2) BETWEEN 300 AND 30000" "k"
-agree "partial index agrees on its subset"          "k2 > 100 AND k < 5000"          "k"
-agree "text expression index agrees"                "length(c1) = 83"                "k"
+agree "plain index agrees with a sequential scan"   "k BETWEEN 1000 AND 9999"        "k" "w_k(12)?"
+agree "late-column index agrees"                    "c18 > repeat('x',80)||'99000'"  "k" "w_c18"
+agree "expression index agrees"                     "(k + k2) BETWEEN 300 AND 30000" "k" "w_expr"
+agree "partial index agrees on its subset"          "k2 > 100 AND k < 5000"          "k" "w_part"
+agree "text expression index agrees"                "length(c1) = 83"                "k" "w_len"
 
 # The PARALLEL build, which is the default path for any table of consequential size and
 # was the one left unprojected.
@@ -178,7 +191,7 @@ usesix() {  # $1 = query -> the index name the plan scans
 }
 check "the oracle below really reads the parallel-built index" \
 	"$(usesix 'SELECT k2 FROM w WHERE k2 BETWEEN 2000 AND 19998')" "w_par2"
-agree "parallel-built index agrees with a sequential scan" "k2 BETWEEN 2000 AND 19998" "k2"
+agree "parallel-built index agrees with a sequential scan" "k2 BETWEEN 2000 AND 19998" "k2" "w_par2"
 check "parallel-built index returns each row once" \
 	"$(qset 'SET enable_seqscan=off; SET enable_bitmapscan=off' \
 		'SELECT count(*) FROM w WHERE k2 BETWEEN 2 AND 2000')" "1000"

test/pytest/TESTS.md

@@ -208,6 +208,7 @@ behaviour, the source of that number is named.
 - [160. test_native_encdesc_golden.py: the encoding descriptor's wire layout](#160-test_native_encdesc_goldenpy-the-encoding-descriptors-wire-layout)
 - [161. test_native_parquet_schema.py: parquet_schema round trip](#161-test_native_parquet_schemapy-parquet_schema-round-trip)
 - [162. test_native_vecskip.py: per-vector skipping by the zone maps](#162-test_native_vecskippy-per-vector-skipping-by-the-zone-maps)
+- [163. test_native_index_projection.py: CREATE INDEX projects only what it needs](#163-test_native_index_projectionpy-create-index-projects-only-what-it-needs)
 
 ## 1. How to read a test in here
 
@@ -7627,3 +7628,16 @@ The pytest twin of `test/native_vecskip.sh` (ported for #1341), every check by t
 | test | what it holds |
 | --- | --- |
 | `test_per_vector_skipping` | the zone maps skip exactly the seven vectors outside the range without decoding them, on the scalar scan and the vectorized aggregate, and every range returns the heap's rows |
+
+## 163. test_native_index_projection.py: CREATE INDEX projects only what it needs
+
+The pytest twin of `test/native_index_projection.sh` (ported for #1341), every check by the shell's name. CREATE INDEX projects only the columns the index needs (#413), serial and parallel: each build's own DEBUG1 line says which branch ran and how many columns its reader decodes. The built indexes must then answer as a sequential scan does: point lookups through the index, and the full ordered result of an index scan against a sequential scan. The index side runs with the columnar scan off, and each oracle asserts the index its own query reads. Before that, three of the oracles ran as the columnar scan on both sides.
+
+### Every test
+
+| test | what it holds |
+| --- | --- |
+| `test_the_built_indexes_answer_as_a_sequential_scan_does` | each serial index finds its rows, and its full ordered result equals a sequential scan's, read through that index |
+| `test_each_build_projects_only_what_it_needs` | every serial and parallel build projects only its key, expression and predicate columns |
+| `test_the_parallel_built_index_agrees_and_holds_each_row_once` | the parallel-built index agrees with a sequential scan and holds each row once |
+| `test_amcheck_finds_every_index_sound` | `bt_index_check` finds all eleven indexes sound |

test/pytest/expected_tests.txt

@@ -787,4 +787,6 @@ guard_tests 438
 # two tests, by collection over the cluster files on this tree.
 # cluster_tests 697 -> 698 for #1341's test_native_vecskip.py, the twin of native_vecskip.sh:
 # one test, by collection over the cluster files on this tree.
-cluster_tests 698
+# cluster_tests 698 -> 702 for #1341's test_native_index_projection.py, the twin of native_index_projection.sh:
+# four tests, by collection over the cluster files on this tree.
+cluster_tests 702

test/pytest/test_compare_to_bash.py

@@ -132,6 +132,7 @@
             "native_fetch_coalesce",
             "native_groupagg",
             "native_index_fetch_stripe_cost",
+            "native_index_projection",
             "native_join_vector_agg",
             "native_ownership",
             "native_parquet_dict_oob",
@@ -409,8 +410,8 @@ def test_the_loop_reader_invents_nothing_in_this_corpus(expect):
                 "came from")
     expect.text(", ".join(files),
                 "build_refusal, differential, join_runtime_filter, "
-                "native_reclaim, native_repack, native_vacuum_race, native_vecskip, "
-                "sorted_pathkeys",
+                "native_index_projection, native_reclaim, native_repack, "
+                "native_vacuum_race, native_vecskip, sorted_pathkeys",
                 "and it is these files, so another appearing is a diff a reviewer "
                 "sees rather than a number that moved")