From 6bc09e078a6b81b66c70f0cb5af1f591170a1965 Mon Sep 17 00:00:00 2001
From: "Joshua D. Drake" <jd@commandprompt.com>
Date: Fri, 16 Oct 2026 18:09:29 +0000
Subject: [PATCH 1/7] test: native_fetch_bigcap gets its pytest twin, and its
 narrow and medium projections assert their plans (#1341, #1339)

test_native_fetch_bigcap.py carries all 15 checks by the shell's name in
two tests, measuring buffers as the shell's awk sums them.

Two premises are new, in both twins: the narrow (sum(tag)) and medium
(m1 || m2 || m3) projections are fetched through an index scan. Only the
payload query's plan was asserted. Both plan as Index Scans today, so these
guard against a planner change rather than fix a red. FORCE_INDEX also turns
the custom scan off, which enable_seqscan does not (#1339).

Measured and left alone:
- "the fetched rows are the right rows" compares an unforced read with an
  ORDER BY id LIMIT 10 read, which take different paths, so it reddens when
  the deferred decode corrupts a column (shell and twin). Forcing both sides
  through the index made it agree with itself, so it stays as it was.
- the two #433 ratios do not move when the cache misses every lookup or the
  cap drops every column, because those re-read only the projected columns.
  #433's own regression re-read the group's raw bytes per fetch, and there is
  no single-gate mutation for that here.

COMPLETE gains native_fetch_bigcap. TESTS.md section 173, cluster_tests
742 -> 744.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 test/native_fetch_bigcap.sh             |  14 ++-
 test/pytest/TESTS.md                    |  12 +++
 test/pytest/expected_tests.txt          |   4 +-
 test/pytest/test_compare_to_bash.py     |   1 +
 test/pytest/test_native_fetch_bigcap.py | 138 ++++++++++++++++++++++++
 5 files changed, 167 insertions(+), 2 deletions(-)
 create mode 100644 test/pytest/test_native_fetch_bigcap.py

diff --git a/test/native_fetch_bigcap.sh b/test/native_fetch_bigcap.sh
index d91c52091..9e26b7228 100755
--- a/test/native_fetch_bigcap.sh
+++ b/test/native_fetch_bigcap.sh
@@ -33,8 +33,13 @@ CAP=$((32 * 1024 * 1024))	# COLUMNAR_FETCH_CACHE_MAX_BYTES, src/columnar.h
 # by its own model, and then nothing measures the fetch path at all. So the index
 # path is forced, and every measured query asserts it got one. Whether the
 # planner should choose it is #434, and that is a separate suite.
+#
+# pgcolumnar.enable_custom_scan=off as well: enable_seqscan alone does not turn
+# the columnar custom scan off (#1339). Every measured query asserts its plan, so
+# a planner change cannot quietly take the arms off the fetch path.
 FORCE_INDEX="SET max_parallel_workers_per_gather=0; SET enable_seqscan=off;
-             SET enable_bitmapscan=off; SET pgcolumnar.enable_index_fetch_penalty=off;"
+             SET enable_bitmapscan=off; SET pgcolumnar.enable_index_fetch_penalty=off;
+             SET pgcolumnar.enable_custom_scan=off;"
 
 # ---------------------------------------------------------------------------
 # The fixture, whose premise is its ON-DISK SIZE.
@@ -144,6 +149,11 @@ nbufs() {	# $1 = number of rows, projecting one narrow column
 		} } END { print t + 0 }'
 }
 
+# The narrow and medium projections had no plan premise of their own; only the
+# payload query's plan was asserted (#1339). They plan as Index Scans today, and
+# these keep a planner change from turning their ratios into scan-against-scan.
+check_num "premise: the narrow projection is fetched through an index scan" \
+	"$(pgc_uses_row_fetch "$FORCE_INDEX" "SELECT sum(tag) FROM bigcap WHERE id BETWEEN 1 AND 10" | grep -c yes)" "1"
 N1=$(nbufs 1)
 N10=$(nbufs 10)
 echo "-- narrow projection buffers: 1 row = ${N1}, 10 rows = ${N10}"
@@ -172,6 +182,8 @@ mbufs() {	# $1 = number of rows, projecting three medium columns
 		} } END { print t + 0 }'
 }
 
+check_num "premise: the medium projection is fetched through an index scan" \
+	"$(pgc_uses_row_fetch "$FORCE_INDEX" "SELECT count(m1 || m2 || m3) FROM bigcap WHERE id BETWEEN 1 AND 10" | grep -c yes)" "1"
 M1=$(mbufs 1)
 M10=$(mbufs 10)
 echo "-- three medium columns from an oversized group: 1 row = ${M1}, 10 rows = ${M10}"
diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md
index a82be591b..b20401af7 100644
--- a/test/pytest/TESTS.md
+++ b/test/pytest/TESTS.md
@@ -218,6 +218,7 @@ behaviour, the source of that number is named.
 - [170. test_native_parquet_flba.py: uuid and DECIMAL read from fixed-length byte arrays](#170-test_native_parquet_flbapy-uuid-and-decimal-read-from-fixed-length-byte-arrays)
 - [171. test_native_parquet_multifile.py: directories and globs read as one relation](#171-test_native_parquet_multifilepy-directories-and-globs-read-as-one-relation)
 - [172. test_native_index.py: index lookups and unique enforcement fetch by row number](#172-test_native_indexpy-index-lookups-and-unique-enforcement-fetch-by-row-number)
+- [173. test_native_fetch_bigcap.py: fetching from a row group above the cache cap](#173-test_native_fetch_bigcappy-fetching-from-a-row-group-above-the-cache-cap)
 
 ## 1. How to read a test in here
 
@@ -7773,3 +7774,14 @@ The pytest twin of `test/native_index.sh` (ported for #1341), every check by the
 | --- | --- |
 | `test_index_lookups_match_the_heap` | four lookups plan as Index Scans and match the heap, and a duplicate primary key is refused with the row count unchanged |
 | `test_a_standalone_unique_index_enforces` | a standalone unique index refuses a duplicate and admits a new key |
+
+## 173. test_native_fetch_bigcap.py: fetching from a row group above the cache cap
+
+The pytest twin of `test/native_fetch_bigcap.sh` (ported for #1341), every check by the shell's name. Above the 32 MB fetch-cache cap, the entry used to be dropped whole after every fetch, so each fetched row re-read the whole group (#433). The measure is buffers, a count rather than a time. The fixture's premise is its on-disk size, read back from `pgcolumnar.row_group`. Every measured query is forced onto the index path with the custom scan off, and asserts its plan. A payload wider than the cap re-decodes per fetch by design (#359). A narrow projection, and columns that fit, must still be cached.
+
+### Every test
+
+| test | what it holds |
+| --- | --- |
+| `test_a_group_above_the_cap_is_read_once_for_what_fits` | the fixture has a group above the cap, every measured query is an index fetch, and ten rows of a narrow or a fitting projection cost under three times one row |
+| `test_caching_changes_no_row` | the first ten rows equal the same rows read in key order, and every payload is full length |
diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt
index fdbaa4406..cbafe1c5c 100644
--- a/test/pytest/expected_tests.txt
+++ b/test/pytest/expected_tests.txt
@@ -807,4 +807,6 @@ guard_tests 438
 # three tests, by collection over the cluster files on this tree.
 # cluster_tests 740 -> 742 for #1341's test_native_index.py, the twin of native_index.sh:
 # two tests, by collection over the cluster files on this tree.
-cluster_tests 742
+# cluster_tests 742 -> 744 for #1341's test_native_fetch_bigcap.py, the twin of native_fetch_bigcap.sh:
+# two tests, by collection over the cluster files on this tree.
+cluster_tests 744
diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py
index 6708c3f05..f2de2f5bc 100644
--- a/test/pytest/test_compare_to_bash.py
+++ b/test/pytest/test_compare_to_bash.py
@@ -129,6 +129,7 @@
             "native_dict_underfill",
             "native_dml",
             "native_encdesc_golden",
+            "native_fetch_bigcap",
             "native_fetch_cache",
             "native_fetch_coalesce",
             "native_fetch_position",
diff --git a/test/pytest/test_native_fetch_bigcap.py b/test/pytest/test_native_fetch_bigcap.py
new file mode 100644
index 000000000..1ee132908
--- /dev/null
+++ b/test/pytest/test_native_fetch_bigcap.py
@@ -0,0 +1,138 @@
+"""Fetching from a row group larger than the fetch-cache cap (#433, #1341). The pytest
+twin of `test/native_fetch_bigcap.sh`, every check by the shell's name.
+
+Above the 32 MB cap the hit rate used to be zero by construction: the entry was
+populated, used once and dropped whole, so every fetched row re-read the whole group.
+The measure is BUFFERS, a count rather than a time. The fixture's premise is its
+on-disk size, read back from pgcolumnar.row_group. Every measured query is forced
+onto the index path, with the custom scan off, and asserts its plan. A payload wider
+than the cap re-decodes per fetch by design (#359); a narrow projection, and columns
+that fit, must be cached anyway. Correctness last.
+"""
+import psycopg
+import pytest
+
+CAP = 32 * 1024 * 1024
+FORCE_INDEX = ("max_parallel_workers_per_gather=0", "enable_seqscan=off", "enable_bitmapscan=off",
+               "pgcolumnar.enable_index_fetch_penalty=off", "pgcolumnar.enable_custom_scan=off")
+
+
+class _Sets:
+    def __init__(self, c, sets):
+        self.c, self.sets = c, sets
+
+    def __enter__(self):
+        for s in self.sets:
+            self.c.execute(f"SET {s}")
+
+    def __exit__(self, *exc):
+        for s in self.sets:
+            self.c.execute(f"RESET {s.split('=')[0]}")
+
+
+def _one(c, sql):
+    try:
+        return c.execute(sql).fetchone()[0]
+    except psycopg.Error as exc:
+        return f"error: {exc.diag.message_primary}"
+
+
+def _index_scan(c, sql):
+    """-> 1 when `sql` plans an Index Scan under FORCE_INDEX, as the shell counts it."""
+    with _Sets(c, FORCE_INDEX):
+        return int(any("Index Scan" in r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {sql}")))
+
+
+def _bufs(c, sql):
+    """-> the buffers EXPLAIN (ANALYZE, BUFFERS) reports for `sql`, after one warming run,
+    summed as the shell's awk sums them: every shared/read/hit token's digits."""
+    total = 0
+    with _Sets(c, FORCE_INDEX):
+        c.execute(f"EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING OFF, FORMAT TEXT) {sql}").fetchall()
+        for (line,) in c.execute(f"EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING OFF, FORMAT TEXT) {sql}"):
+            if "Buffers:" not in line:
+                continue
+            for tok in line.split():
+                if tok.startswith(("shared", "read", "hit")):
+                    digits = "".join(ch for ch in tok if ch.isdigit())
+                    if digits:
+                        total += int(digits)
+    return total
+
+
+def _ratio(a, b, bound):
+    """-> 'within' when a/b, rounded to two places, is at most `bound`, as check_ratio."""
+    if a == 0 or b == 0:
+        return f"a side of the ratio is zero, so nothing was measured: a=[{a}] b=[{b}]"
+    r = round(a / b, 2)
+    return "within" if r <= bound else f"{r:.2f}x exceeds the {bound}x bound (a={a} b={b})"
+
+
+@pytest.fixture(scope="module")
+def bigcap(pgc_cluster):
+    db = "pgc_bigcap"
+    with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a:
+        a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)")
+        a.execute(f"CREATE DATABASE {db}")
+    try:
+        with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c:
+            c.execute("CREATE EXTENSION pgcolumnar")
+            c.execute("CREATE TABLE bigcap (id int, tag int, m1 text, m2 text, m3 text, payload text) USING pgcolumnar")
+            # per-row distinct hex from a subquery correlated on g, the codec off: this
+            # fixture compressed 17x and 25x in earlier attempts, so its size is asserted
+            c.execute("SELECT pgcolumnar.set_options('bigcap', stripe_row_limit => 10000, compression => 'none')")
+            c.execute("""INSERT INTO bigcap
+                SELECT g, g % 97, md5((g * 7)::text), md5((g * 11)::text), md5((g * 13)::text),
+                       (SELECT string_agg(md5((g * 300 + s)::text), '') FROM generate_series(1, 256) s)
+                  FROM generate_series(1, 20000) g""")
+            c.execute("CREATE INDEX bigcap_id ON bigcap (id)")
+            c.execute("ANALYZE bigcap")
+            yield c
+    finally:
+        with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a:
+            a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)")
+
+
+def test_a_group_above_the_cap_is_read_once_for_what_fits(bigcap, expect):
+    c = bigcap
+    maxg = _one(c, "SELECT max(byte_length) FROM pgcolumnar.row_group "
+                   "WHERE storage_id = pgcolumnar.get_storage_id('bigcap')")
+    print(f"-- largest row group: {maxg} bytes, cap {CAP}")
+    expect.at_least(maxg if isinstance(maxg, int) else 0, CAP + 1,
+                    "premise: the fixture really has a row group above the cap")
+    expect.num(_one(c, "SELECT count(*) FROM bigcap"), 20000, "premise: the table loaded every row")
+
+    payload = "SELECT sum(length(payload)) FROM bigcap WHERE id BETWEEN 1 AND {}"
+    expect.num(_index_scan(c, payload.format(1)), 1, "premise: one row is fetched through an index scan")
+    expect.num(_index_scan(c, payload.format(10)), 1, "premise: ten rows are fetched through an index scan")
+    b1, b10 = _bufs(c, payload.format(1)), _bufs(c, payload.format(10))
+    print(f"-- buffers: 1 row = {b1}, 10 rows = {b10}")
+    # both sides a measurement before any comparison (#418): a failed EXPLAIN reads as 0
+    expect.at_least(b1, 101, "premise: fetching one row touched a measurable number of buffers")
+    expect.at_least(b10, 101, "premise: fetching ten rows touched a measurable number of buffers")
+    # the payload alone exceeds the cap, so it re-decodes per fetch: #359's design, not #433
+    expect.at_least(b10, b1 + 1, "premise: the wide column is the one over the cap, so this arm is #359's")
+
+    narrow = "SELECT sum(tag) FROM bigcap WHERE id BETWEEN 1 AND {}"
+    expect.num(_index_scan(c, narrow.format(10)), 1, "premise: the narrow projection is fetched through an index scan")
+    n1, n10 = _bufs(c, narrow.format(1)), _bufs(c, narrow.format(10))
+    print(f"-- narrow projection buffers: 1 row = {n1}, 10 rows = {n10}")
+    expect.at_least(n1, 1, "premise: the narrow projection touched a measurable number of buffers")
+    expect.text(_ratio(n10, n1, 3), "within", "a narrow projection over a wide group is cached too (#433)")
+
+    medium = "SELECT count(m1 || m2 || m3) FROM bigcap WHERE id BETWEEN 1 AND {}"
+    expect.num(_index_scan(c, medium.format(10)), 1, "premise: the medium projection is fetched through an index scan")
+    m1, m10 = _bufs(c, medium.format(1)), _bufs(c, medium.format(10))
+    print(f"-- three medium columns from an oversized group: 1 row = {m1}, 10 rows = {m10}")
+    expect.at_least(m1, 1, "premise: the medium projection touched a measurable number of buffers")
+    expect.text(_ratio(m10, m1, 3), "within", "columns that fit are cached even when their group does not (#433)")
+
+
+def test_caching_changes_no_row(bigcap, expect):
+    c = bigcap
+    row = "md5(string_agg(id || ':' || tag || ':' || m1 || m2 || m3 || ':' || md5(payload), ',' ORDER BY id))"
+    expect.text(str(_one(c, f"SELECT {row} FROM bigcap WHERE id BETWEEN 1 AND 10")),
+                str(_one(c, f"SELECT {row} FROM (SELECT * FROM bigcap ORDER BY id LIMIT 10) s")),
+                "the fetched rows are the right rows")
+    expect.num(_one(c, "SELECT count(*) FROM bigcap WHERE length(payload) <> 8192"), 0,
+               "every payload survived the round trip at full length")

From 45a8bfa55f496e1062b796a5b81ec28e10f836e8 Mon Sep 17 00:00:00 2001
From: "Joshua D. Drake" <jd@commandprompt.com>
Date: Fri, 16 Oct 2026 18:11:17 +0000
Subject: [PATCH 2/7] test: native_fetch_bigcap seeds the ledger, its
 right-rows arm dated by a deferred-decode mutation (#1341)

The suite had no rows. suites_not_covered 193 -> 192; census 4347 rows /
3031 never / 1316 dated.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 test/check_ledger.tsv        | 15 +++++++++++++++
 test/check_ledger_budget.txt | 17 +++++++++++++++--
 2 files changed, 30 insertions(+), 2 deletions(-)

diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv
index 2a9ea5c9b..2b17555a8 100644
--- a/test/check_ledger.tsv
+++ b/test/check_ledger.tsv
@@ -2479,6 +2479,21 @@ native_encdesc_golden	native_encdesc_golden	int column a rawLen parses as 1200 (
 native_encdesc_golden	native_encdesc_golden	the table reads back all rows	15;16;17;18;19	never	-
 native_encdesc_golden	native_encdesc_golden	trailing sharedTableLen is 0 (u32 at offset 6+13=19)	15;16;17;18;19	never	-
 native_encdesc_golden	native_encdesc_golden	vectorCount parses as 1 (at offset 2)	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	a narrow projection over a wide group is cached too (#433)	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	columns that fit are cached even when their group does not (#433)	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	every payload survived the round trip at full length	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: fetching one row touched a measurable number of buffers	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: fetching ten rows touched a measurable number of buffers	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: one row is fetched through an index scan	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: ten rows are fetched through an index scan	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the fixture really has a row group above the cap	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the medium projection is fetched through an index scan	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the medium projection touched a measurable number of buffers	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the narrow projection is fetched through an index scan	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the narrow projection touched a measurable number of buffers	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the table loaded every row	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	premise: the wide column is the one over the cap, so this arm is #359's	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	the fetched rows are the right rows	15;16;17;18;19	2026-10-16	columnar_tableam.c: the deferred slot decode reads tag as NULL on every 7th row number
 native_fetch_cache	native_fetch_cache	a hit re-checks the group's fileOffset against the row group	15;16;17;18;19	2026-10-16	columnar_reader.c: a fetch-cache hit no longer re-checks fileOffset
 native_fetch_cache	native_fetch_cache	a hit re-checks the group's firstRowNumber against the row group	15;16;17;18;19	2026-10-16	columnar_reader.c: a fetch-cache hit no longer re-checks firstRowNumber
 native_fetch_cache	native_fetch_cache	a hit re-checks the group's natts against the scan's column count	15;16;17;18;19	2026-10-16	columnar_reader.c: a fetch-cache hit no longer re-checks natts
diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt
index affa347e8..78a46d49a 100644
--- a/test/check_ledger_budget.txt
+++ b/test/check_ledger_budget.txt
@@ -15,7 +15,7 @@
 #
 #   Adding a check to a suite that is already covered does not move it, which is
 #   what makes it safe to bound.
-suites_not_covered 193
+suites_not_covered 192
 #
 # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become
 #   one.
@@ -3467,4 +3467,17 @@ suites_not_covered 193
 #     never         3019 + 2 - 4 = 3017
 #     dated         1311 + 4 = 1315
 #     3017 + 1315 == 4332
-checks_never_observed_red 3017
+#
+# #1341, native_fetch_bigcap seeded, with its twin (OffgridwithJD): 3017 -> 3031,
+# and suites_not_covered 193 -> 192. The suite had no rows. It enters with all 15
+# checks from five-major runs (15/15 on each of 15-19). One is dated: the right-rows
+# arm, by the deferred decode reading tag as NULL on every 7th row number. Fourteen
+# arrive never: eleven premises; the payload length; and the two #433 ratios, which
+# the cache missing every lookup and the cap dropping every column cannot move,
+# because each re-reads only the projected columns.
+#
+#     ledger rows   4332 + 15 = 4347
+#     never         3017 + 14 = 3031
+#     dated         1315 +  1 = 1316
+#     3031 + 1316 == 4347
+checks_never_observed_red 3031

From 2006aff33dc764fd89e2c72d4e52a5ffcdfcec48 Mon Sep 17 00:00:00 2001
From: "Joshua D. Drake" <jd@commandprompt.com>
Date: Fri, 16 Oct 2026 18:18:20 +0000
Subject: [PATCH 3/7] test: partition_rows' primary-key arm reads the key it
 names (#1339)

"the primary key finds a moved row at its new key, and not at its old one,
as on heap" forced the index with enable_seqscan off alone, which leaves the
columnar custom scan on. Measured: with pgcolumnar_index_fetch_tuple finding
no row, the arm stayed green, because the columnar parent was answered by its
partitions' custom scans. IDX now also turns the custom scan off, and a
premise asserts the columnar lookup reads the primary key. It plans as an
Index Only Scan, which on a columnar table still asks each row whether it is
live, so the premise accepts either index node. With the same mutation the
arm is now red, in both twins.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 test/partition_rows.sh             | 14 ++++++++++++--
 test/pytest/test_partition_rows.py | 15 ++++++++++++++-
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/test/partition_rows.sh b/test/partition_rows.sh
index 9e6be2d15..b282ee991 100755
--- a/test/partition_rows.sh
+++ b/test/partition_rows.sh
@@ -52,9 +52,19 @@ check "an UPDATE moving 49 rows to the other partition succeeds, as on heap" \
 	"$(both "UPDATE %p SET id = id + 1000 WHERE id < 50")" "ok|ok"
 check "and each partition holds what heap's does" "$(answers "$SPLIT")" "950/49|950/49"
 check "and the parent holds the same rows as heap's" "$(same "$CONTENT")" "same"
-IDX="SET enable_seqscan = off; SET enable_bitmapscan = off;"
+# The custom scan off too, and a premise that the columnar lookup reads the key:
+# enable_seqscan alone leaves the columnar custom scan on. Measured (#1339): with
+# the index fetch finding no row, this arm stayed green, because the columnar
+# parent was answered by its partitions' custom scans, not by the primary key.
+IDX="SET enable_seqscan = off; SET enable_bitmapscan = off; SET pgcolumnar.enable_custom_scan = off;"
+# It plans as an Index Only Scan (it reads only id), which on a columnar table still
+# asks each row whether it is live; the premise accepts either index node.
+KEY_SQL="SELECT count(*) FILTER (WHERE id = 1001) || '/' || count(*) FILTER (WHERE id = 1) FROM %p WHERE id IN (1, 1001)"
+check "premise: the columnar key lookup reads the primary key" \
+	"$(case "$(q "$IDX EXPLAIN (COSTS OFF) ${KEY_SQL//%p/pr_pgcolumnar}")" in
+		*"Index Scan using"*|*"Index Only Scan using"*) echo yes ;; *) echo no ;; esac)" yes
 check "the primary key finds a moved row at its new key, and not at its old one, as on heap" \
-	"$(answers "$IDX SELECT count(*) FILTER (WHERE id = 1001) || '/' || count(*) FILTER (WHERE id = 1) FROM %p WHERE id IN (1, 1001)")" \
+	"$(answers "$IDX $KEY_SQL")" \
 	"1/0|1/0"
 
 # ---- the key on the parent ----------------------------------------------------------
diff --git a/test/pytest/test_partition_rows.py b/test/pytest/test_partition_rows.py
index 533d95257..d5e321ce8 100644
--- a/test/pytest/test_partition_rows.py
+++ b/test/pytest/test_partition_rows.py
@@ -21,7 +21,9 @@
 CONTENT = ("SELECT count(*) || '/' || md5(string_agg(id || ':' || k || ':' || v, ',' ORDER BY id)) "
            "FROM %p")
 SPLIT = "SELECT (SELECT count(*) FROM %p_1) || '/' || (SELECT count(*) FROM %p_2)"
-IDX = ("SET enable_seqscan = off", "SET enable_bitmapscan = off")
+# the custom scan off too: enable_seqscan alone leaves it on, and the columnar parent
+# was answered by its partitions' custom scans rather than the primary key (#1339)
+IDX = ("SET enable_seqscan = off", "SET enable_bitmapscan = off", "SET pgcolumnar.enable_custom_scan = off")
 
 
 def _connect(cluster, schema, name):
@@ -118,6 +120,17 @@ def test_rows_cross_columnar_partitions_and_keys_hold_through_the_parent(pgc_clu
     expect.text(_answers(pgc_conn, SPLIT), "950/49|950/49", "and each partition holds what heap's does")
     c, h = _answers(pgc_conn, CONTENT).split("|")
     expect.text("same" if c == h else f"{c}|{h}", "same", "and the parent holds the same rows as heap's")
+    with pgc_conn.cursor() as cur:
+        for s in IDX:
+            cur.execute(s)
+        cur.execute("EXPLAIN (COSTS OFF) SELECT count(*) FILTER (WHERE id = 1001) || '/' || "
+                    "count(*) FILTER (WHERE id = 1) FROM pr_pgcolumnar WHERE id IN (1, 1001)")
+        # an Index Only Scan here (only id is read), which still asks each row whether it is live
+        lookup = "yes" if any("Index Scan using" in r[0] or "Index Only Scan using" in r[0]
+                              for r in cur.fetchall()) else "no"
+        for s in IDX:
+            cur.execute(s.replace("SET ", "RESET ").split(" =")[0])
+    expect.text(lookup, "yes", "premise: the columnar key lookup reads the primary key")
     expect.text(_answers(pgc_conn, "SELECT count(*) FILTER (WHERE id = 1001) || '/' || "
                                    "count(*) FILTER (WHERE id = 1) FROM %p WHERE id IN (1, 1001)", IDX),
                 "1/0|1/0", "the primary key finds a moved row at its new key, and not at its old one, as on heap")

From 5aed207aaf91daf36d46741d87dafa32c76e76b5 Mon Sep 17 00:00:00 2001
From: "Joshua D. Drake" <jd@commandprompt.com>
Date: Fri, 16 Oct 2026 18:18:35 +0000
Subject: [PATCH 4/7] test: partition_rows' key premise joins the ledger, and
 three duplicate-key arms are dated (#1339)

Census 4348 rows / 3029 never / 1319 dated.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 test/check_ledger.tsv        | 11 ++++++-----
 test/check_ledger_budget.txt | 12 +++++++++++-
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv
index 2b17555a8..0a7902e8a 100644
--- a/test/check_ledger.tsv
+++ b/test/check_ledger.tsv
@@ -3258,21 +3258,22 @@ partition_lifecycle	partition_lifecycle	running the plan leaves only the partiti
 partition_lifecycle	partition_lifecycle	the parent now has the partitions it reported	15;16;17;18;19	2026-10-14	partition_create_ahead creating its last partition without attaching it
 partition_lifecycle	partition_lifecycle	the plan retires the partitions wholly before the cutoff, in range order	15;16;17;18;19	2026-10-13	a plain DROP, not refusing while the partition is still attached;the cutoff filter removed: every partition planned;the lower bound read where the upper is meant;the plan detaches but drops nothing;the plan detaches without CONCURRENTLY;the plan in catalog order, not range order
 partition_lifecycle	partition_lifecycle	they are contiguous, each one width wide	15;16;17;18;19	2026-10-12	partition_create_ahead leaving a minute between partitions
-partition_rows	partition_rows	a duplicate key inserted through the parent is refused (23505), as on heap	15;16;17;18;19	never	-
-partition_rows	partition_rows	a duplicate of a moved row's new key is refused in its new partition (23505), as on heap	15;16;17;18;19	never	-
+partition_rows	partition_rows	a duplicate key inserted through the parent is refused (23505), as on heap	15;16;17;18;19	2026-10-16	columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row
+partition_rows	partition_rows	a duplicate of a moved row's new key is refused in its new partition (23505), as on heap	15;16;17;18;19	2026-10-16	columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row
 partition_rows	partition_rows	a key a moved row gave up can be used again, as on heap	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete)
-partition_rows	partition_rows	a second insert of the same new key through the parent waits, then is refused (23505), and one row is left, as on heap	15;16;17;18;19	never	-
+partition_rows	partition_rows	a second insert of the same new key through the parent waits, then is refused (23505), and one row is left, as on heap	15;16;17;18;19	2026-10-16	columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row
 partition_rows	partition_rows	a second session moving a row the first is moving waits, then fails with 40001, and the row exists once, as on heap	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete);the tree before #1421: a row moved between partitions is serialized only at flush, after its new version's unique check
 partition_rows	partition_rows	an UPDATE in place of a row another session is moving waits, then fails with 40001, as on heap	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete)
 partition_rows	partition_rows	an UPDATE moving 25,000 keyed rows to the other partition keeps under 32 MB likewise	15;16;17;18;19	2026-10-15	PgColumnarSerializeRowNow reads the delete vector into the caller's context, keeping a bitmap copy per row (the tree before the #1422 review fix)
 partition_rows	partition_rows	an UPDATE moving 49 rows to the other partition succeeds, as on heap	15;16;17;18;19	never	-
 partition_rows	partition_rows	an UPDATE of 25,000 keyed rows keeps under 32 MB of memory between its first and last row	15;16;17;18;19	2026-10-15	PgColumnarSerializeRowNow reads the delete vector into the caller's context, keeping a bitmap copy per row (the tree before the #1422 review fix)
-partition_rows	partition_rows	and after the races the parents hold the same rows	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete)
+partition_rows	partition_rows	and after the races the parents hold the same rows	15;16;17;18;19	2026-10-16	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete);columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row
 partition_rows	partition_rows	and each UPDATE reached every row once	15;16;17;18;19	never	-
 partition_rows	partition_rows	and each partition holds what heap's does	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete)
 partition_rows	partition_rows	and the parent holds the same rows as heap's	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete)
 partition_rows	partition_rows	each keyed table's group carries delete marks before its UPDATE	15;16;17;18;19	never	-
-partition_rows	partition_rows	the primary key finds a moved row at its new key, and not at its old one, as on heap	15;16;17;18;19	2026-10-15	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete)
+partition_rows	partition_rows	premise: the columnar key lookup reads the primary key	15;16;17;18;19	never	-
+partition_rows	partition_rows	the primary key finds a moved row at its new key, and not at its old one, as on heap	15;16;17;18;19	2026-10-16	a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete);columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row
 partman_columnar	partman_columnar	and no row-group row behind	15;16;17;18;19	2026-10-12	the drop hook stops deleting row_group rows, so they outlive their deleted storage row
 partman_columnar	partman_columnar	and read back as heap does	15;16;17;18;19	2026-10-12	a sequential scan drops the last row group from its list, so every one-group partition reads empty
 partman_columnar	partman_columnar	and the rows land in columnar partitions	15;16;17;18;19	2026-10-12	a sequential scan drops the last row group from its list, so every one-group partition reads empty
diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt
index 78a46d49a..f20909e8f 100644
--- a/test/check_ledger_budget.txt
+++ b/test/check_ledger_budget.txt
@@ -3480,4 +3480,14 @@ suites_not_covered 192
 #     never         3017 + 14 = 3031
 #     dated         1315 +  1 = 1316
 #     3031 + 1316 == 4347
-checks_never_observed_red 3031
+#
+# #1339, partition_rows' primary-key arm reads the key it names (OffgridwithJD):
+# 3031 -> 3029. One premise is new, never. With pgcolumnar_index_fetch_tuple
+# finding no row, five arms are red: the three duplicate-key refusals entered never
+# and are now dated, and the primary-key and races arms gain it in their history.
+#
+#     ledger rows   4347 + 1 = 4348
+#     never         3031 + 1 - 3 = 3029
+#     dated         1316 + 3 = 1319
+#     3029 + 1319 == 4348
+checks_never_observed_red 3029

From 30d725193e3b80305536ba7f7bca7e04ad82b0f7 Mon Sep 17 00:00:00 2001
From: "Joshua D. Drake" <jd@commandprompt.com>
Date: Fri, 16 Oct 2026 18:27:03 +0000
Subject: [PATCH 5/7] test: sorted_projection gets its pytest twin, its index
 lookups read the index, and each refusal must be its own (#1341, #1339)

test_sorted_projection.py carries all 29 checks by the shell's name in
three tests. Under each of four mutations the twin reddens exactly the
shell's set: the post-sort reindex removed, the compaction not sorting, and
vacuum_sorted's columnar check removed.

Three shell changes, each measured red first:
- "index point lookup after sort" set only enable_seqscan off. With the
  reindex removed, so the index points at the old row numbers, it stayed
  green: it never read the index. The lookups now turn the custom scan off,
  with a premise each that they read t_col_id_idx (an Index Scan or, for the
  count, an Index Only Scan).
- "reject non-columnar table" counted any error. With vacuum_sorted's
  columnar check removed, the heap table still raised -- XX001, reading the
  heap's file as columnar storage -- and the arm stayed green. Each refusal
  now requires its own SQLSTATE (42809, 42703, 22023, 42883).
- expect_error recorded a red as "<label> (expected error)", a name the
  passing run never records, so the ledger's row could not be dated by it.
  It records under the label.

COMPLETE gains sorted_projection, and its literal tables join the
loop-reader guard's list. TESTS.md section 174, cluster_tests 744 -> 747.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 test/pytest/TESTS.md                  |  13 ++
 test/pytest/expected_tests.txt        |   4 +-
 test/pytest/test_compare_to_bash.py   |   4 +-
 test/pytest/test_sorted_projection.py | 172 ++++++++++++++++++++++++++
 test/sorted_projection.sh             |  49 ++++++--
 5 files changed, 228 insertions(+), 14 deletions(-)
 create mode 100644 test/pytest/test_sorted_projection.py

diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md
index b20401af7..6d3efc7fd 100644
--- a/test/pytest/TESTS.md
+++ b/test/pytest/TESTS.md
@@ -219,6 +219,7 @@ behaviour, the source of that number is named.
 - [171. test_native_parquet_multifile.py: directories and globs read as one relation](#171-test_native_parquet_multifilepy-directories-and-globs-read-as-one-relation)
 - [172. test_native_index.py: index lookups and unique enforcement fetch by row number](#172-test_native_indexpy-index-lookups-and-unique-enforcement-fetch-by-row-number)
 - [173. test_native_fetch_bigcap.py: fetching from a row group above the cache cap](#173-test_native_fetch_bigcappy-fetching-from-a-row-group-above-the-cache-cap)
+- [174. test_sorted_projection.py: vacuum_sorted reorders storage and changes no answer](#174-test_sorted_projectionpy-vacuum_sorted-reorders-storage-and-changes-no-answer)
 
 ## 1. How to read a test in here
 
@@ -7785,3 +7786,15 @@ The pytest twin of `test/native_fetch_bigcap.sh` (ported for #1341), every check
 | --- | --- |
 | `test_a_group_above_the_cap_is_read_once_for_what_fits` | the fixture has a group above the cap, every measured query is an index fetch, and ten rows of a narrow or a fitting projection cost under three times one row |
 | `test_caching_changes_no_row` | the first ten rows equal the same rows read in key order, and every payload is full length |
+
+## 174. test_sorted_projection.py: vacuum_sorted reorders storage and changes no answer
+
+The pytest twin of `test/sorted_projection.sh` (ported for #1341), every check by the shell's name. `pgcolumnar.vacuum_sorted()` rewrites a columnar table physically sorted on a key. It must never change an answer: every query shape agrees with a heap oracle before and after, order-independently, and in order where the query orders. It must also reorder storage for real, so a narrow range on the sort key skips vectors it could not skip before. The rest covers a multi-column key, the reclaim of deleted rows with the index rebuilt, idempotence, and argument validation. Each index lookup is forced with the custom scan off and asserts that it reads the index. Each refusal must be the function's own, identified by its SQLSTATE.
+
+### Every test
+
+| test | what it holds |
+| --- | --- |
+| `test_the_sorted_rewrite_changes_no_answer_and_really_reorders` | five query shapes match the heap before and after the sort, the ordered oracle is order-sensitive, and the sort makes the range skip vectors |
+| `test_a_multi_key_sort_reclaims_and_rebuilds_the_index` | a two-column sort matches, deletes are reclaimed, the rebuilt index finds and hides the right rows, and a second sort changes nothing |
+| `test_vacuum_sorted_refuses_what_it_cannot_sort` | a heap table, an unknown column, no sort key and a json key are each refused with their own SQLSTATE |
diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt
index cbafe1c5c..294633d71 100644
--- a/test/pytest/expected_tests.txt
+++ b/test/pytest/expected_tests.txt
@@ -809,4 +809,6 @@ guard_tests 438
 # two tests, by collection over the cluster files on this tree.
 # cluster_tests 742 -> 744 for #1341's test_native_fetch_bigcap.py, the twin of native_fetch_bigcap.sh:
 # two tests, by collection over the cluster files on this tree.
-cluster_tests 744
+# cluster_tests 744 -> 747 for #1341's test_sorted_projection.py, the twin of sorted_projection.sh:
+# three tests, by collection over the cluster files on this tree.
+cluster_tests 747
diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py
index f2de2f5bc..cd5850245 100644
--- a/test/pytest/test_compare_to_bash.py
+++ b/test/pytest/test_compare_to_bash.py
@@ -188,6 +188,7 @@
             "serializable",
             "settings_reference",
             "sorted_pathkeys",
+            "sorted_projection",
             "statement_self_visibility",
             "stats_privilege",
             "storage_row_repair",
@@ -421,7 +422,8 @@ def test_the_loop_reader_invents_nothing_in_this_corpus(expect):
     expect.text(", ".join(files),
                 "build_refusal, differential, join_runtime_filter, native_groupagg_batch, "
                 "native_index_projection, native_parquet_pushdown, native_reclaim, native_repack, "
-                "native_saop_pushdown, native_vacuum_race, native_vecskip, sorted_pathkeys",
+                "native_saop_pushdown, native_vacuum_race, native_vecskip, sorted_pathkeys, "
+                "sorted_projection",
                 "and it is these files, so another appearing is a diff a reviewer "
                 "sees rather than a number that moved")
 
diff --git a/test/pytest/test_sorted_projection.py b/test/pytest/test_sorted_projection.py
new file mode 100644
index 000000000..41ddc9c40
--- /dev/null
+++ b/test/pytest/test_sorted_projection.py
@@ -0,0 +1,172 @@
+"""vacuum_sorted rewrites a columnar table physically sorted on a key (gap 26, #1341). The
+pytest twin of `test/sorted_projection.sh`, every check by the shell's name.
+
+The rewrite must never change an answer: every query shape agrees with a heap oracle
+before and after it, order-independently, and in order where the query orders. It
+must really reorder storage: a narrow range on the sort key skips vectors it could not
+skip before. Then a multi-column key, deleted rows reclaimed with the index rebuilt
+(each index lookup forced with the custom scan off, and asserting it reads the index),
+idempotence, and argument validation.
+"""
+import psycopg
+import pytest
+
+SET_HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) "
+            "FROM (SELECT _row::text AS t FROM ({}) _row) _s")
+SEQ_HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY n)), $e$EMPTY$e$) "
+            "FROM (SELECT row_number() OVER () AS n, _row::text AS t FROM ({}) _row) _s")
+IDXON = ("SET enable_seqscan=off", "SET enable_bitmapscan=off", "SET pgcolumnar.enable_custom_scan=off")
+
+
+def _hash(c, tmpl, sql):
+    try:
+        return c.execute(tmpl.format(sql)).fetchone()[0]
+    except psycopg.Error as exc:
+        return f"error: {exc.diag.message_primary}"
+
+
+def _one(c, sql):
+    try:
+        return str(c.execute(sql).fetchone()[0])
+    except psycopg.Error as exc:
+        return f"error: {exc.diag.message_primary}"
+
+
+def _pair(c, tmpl, hashing=SET_HASH):
+    """-> (columnar, heap) digests of `tmpl` with %T for the table, as diff_query."""
+    return _hash(c, hashing, tmpl.replace("%T", "t_col")), _hash(c, hashing, tmpl.replace("%T", "t_heap"))
+
+
+def _vectors_skipped(c, sql):
+    for (line,) in c.execute(f"EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) {sql}"):
+        if "Vectors Skipped" in line:
+            return int(line.rsplit(" ", 1)[-1])
+    return 0
+
+
+def _forced(c, sql, explain=False):
+    for s in IDXON:
+        c.execute(s)
+    try:
+        if explain:
+            plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {sql}"))
+            return "yes" if ("Index Scan using t_col_id_idx" in plan or "Index Only Scan using t_col_id_idx" in plan) \
+                else f"no: {plan.splitlines()[0] if plan else ''}"
+        return _one(c, sql)
+    finally:
+        for g in ("enable_seqscan", "enable_bitmapscan", "pgcolumnar.enable_custom_scan"):
+            c.execute(f"RESET {g}")
+
+
+def _refused(c, sql):
+    """-> the SQLSTATE `sql` is refused with, or 'succeeded': the refusal must be the
+    function's own, since with its columnar check gone a heap table still raised XX001."""
+    try:
+        c.execute(sql)
+        return "succeeded"
+    except psycopg.Error as exc:
+        return exc.sqlstate or "?"
+
+
+@pytest.fixture(scope="module")
+def sp(pgc_cluster):
+    db = "pgc_sortproj"
+    with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a:
+        a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)")
+        a.execute(f"CREATE DATABASE {db}")
+    try:
+        with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c:
+            c.execute("CREATE EXTENSION pgcolumnar")
+            # k scrambled against insert order, so before sorting every group spans the domain
+            c.execute("CREATE TABLE t_heap (id int, k int, v bigint, t text) USING heap")
+            c.execute("CREATE TABLE t_col (id int, k int, v bigint, t text) USING pgcolumnar")
+            c.execute("SELECT pgcolumnar.set_options('t_col', chunk_group_row_limit => 1000)")
+            c.execute("INSERT INTO t_heap SELECT g, ((g*7919)%1000), g::bigint*2, 'r'||g FROM generate_series(1,50000) g")
+            c.execute("INSERT INTO t_col SELECT * FROM t_heap")
+            yield c
+    finally:
+        with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a:
+            a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)")
+
+
+def test_the_sorted_rewrite_changes_no_answer_and_really_reorders(sp, expect):
+    c = sp
+    for label, tmpl in (("pre range", "SELECT id, v, t FROM %T WHERE k BETWEEN 100 AND 120"),
+                        ("pre equality", "SELECT id FROM %T WHERE k = 500")):
+        col, heap = _pair(c, tmpl)
+        expect.text(col, heap, label)
+    # the premise behind the ordered comparisons, as lib.sh's pgc_check_ordered_oracle
+    fwd = _hash(c, SEQ_HASH, "SELECT g FROM generate_series(1,5) g ORDER BY g")
+    rev = _hash(c, SEQ_HASH, "SELECT g FROM generate_series(1,5) g ORDER BY g DESC")
+    again = _hash(c, SEQ_HASH, "SELECT g FROM generate_series(1,5) g ORDER BY g")
+    sfwd = _hash(c, SET_HASH, "SELECT g FROM generate_series(1,5) g ORDER BY g")
+    srev = _hash(c, SET_HASH, "SELECT g FROM generate_series(1,5) g ORDER BY g DESC")
+    expect.text("yes" if fwd != rev else "no", "yes", "premise: the ordered oracle is order-sensitive")
+    expect.text("yes" if fwd == again else "no", "yes", "premise: the ordered oracle agrees with itself")
+    expect.text("yes" if sfwd == srev else "no", "yes", "control: the set oracle is order-blind by design")
+    col, heap = _pair(c, "SELECT k, v FROM %T ORDER BY k, v", SEQ_HASH)
+    expect.text(col, heap, "pre order-by")
+    for label, tmpl in (("pre aggregate", "SELECT k, count(*), sum(v) FROM %T GROUP BY k"),
+                        ("pre star", "SELECT * FROM %T WHERE id % 997 = 0")):
+        col, heap = _pair(c, tmpl)
+        expect.text(col, heap, label)
+
+    narrow = "SELECT sum(v) FROM t_col WHERE k BETWEEN 100 AND 120"
+    plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {narrow}"))
+    expect.text("yes" if "Columnar Projected Columns" in plan else "no", "yes",
+                "the plan under test is a columnar custom scan")
+    before = _vectors_skipped(c, narrow)
+
+    c.execute("SELECT pgcolumnar.vacuum_sorted('t_col', 'k')")
+    for label, tmpl in (("post range", "SELECT id, v, t FROM %T WHERE k BETWEEN 100 AND 120"),
+                        ("post equality", "SELECT id FROM %T WHERE k = 500")):
+        col, heap = _pair(c, tmpl)
+        expect.text(col, heap, label)
+    col, heap = _pair(c, "SELECT k, v FROM %T ORDER BY k, v", SEQ_HASH)
+    expect.text(col, heap, "post order-by")
+    for label, tmpl in (("post aggregate", "SELECT k, count(*), sum(v) FROM %T GROUP BY k"),
+                        ("post star", "SELECT * FROM %T WHERE id % 997 = 0")):
+        col, heap = _pair(c, tmpl)
+        expect.text(col, heap, label)
+
+    after = _vectors_skipped(c, narrow)
+    expect.num(before, 0, "pre-sort prunes little")
+    expect.text("more" if after > before else f"removed_before={before} removed_after={after}", "more",
+                "post-sort prunes more chunk groups")
+
+
+def test_a_multi_key_sort_reclaims_and_rebuilds_the_index(sp, expect):
+    c = sp
+    c.execute("SELECT pgcolumnar.vacuum_sorted('t_col', 'k', 'v')")
+    col, heap = _pair(c, "SELECT id, v FROM %T WHERE k BETWEEN 100 AND 120")
+    expect.text(col, heap, "multi-key range")
+
+    c.execute("CREATE INDEX t_col_id_idx ON t_col (id)")
+    c.execute("DELETE FROM t_heap WHERE id % 2 = 0")
+    c.execute("DELETE FROM t_col WHERE id % 2 = 0")
+    c.execute("SELECT pgcolumnar.vacuum_sorted('t_col', 'k')")
+    expect.text(_one(c, "SELECT count(*) FROM t_col"), "25000", "live count after delete+sort")
+    for label, tmpl in (("count matches heap", "SELECT count(*) FROM %T"), ("sum matches heap", "SELECT sum(v) FROM %T")):
+        col, heap = _pair(c, tmpl)
+        expect.text(col, heap, label)
+    # forced and asserted: with seqscan alone off the point lookup never read the index
+    expect.text(_forced(c, "SELECT t FROM t_col WHERE id = 4999", explain=True), "yes",
+                "premise: the point lookup reads t_col_id_idx")
+    expect.text(_forced(c, "SELECT t FROM t_col WHERE id = 4999"), "r4999", "index point lookup after sort")
+    expect.text(_forced(c, "SELECT count(*) FROM t_col WHERE id = 5000", explain=True), "yes",
+                "premise: the deletion lookup reads t_col_id_idx")
+    expect.text(_forced(c, "SELECT count(*) FROM t_col WHERE id = 5000"), "0", "index sees deletion after sort")
+
+    h1 = _hash(c, SET_HASH, "SELECT * FROM t_col")
+    c.execute("SELECT pgcolumnar.vacuum_sorted('t_col', 'k')")
+    expect.text(_hash(c, SET_HASH, "SELECT * FROM t_col"), h1, "idempotent sorted rewrite")
+
+
+def test_vacuum_sorted_refuses_what_it_cannot_sort(sp, expect):
+    c = sp
+    expect.text(_refused(c, "SELECT pgcolumnar.vacuum_sorted('t_heap', 'k')"), "42809", "reject non-columnar table")
+    expect.text(_refused(c, "SELECT pgcolumnar.vacuum_sorted('t_col', 'nope')"), "42703", "reject unknown column")
+    expect.text(_refused(c, "SELECT pgcolumnar.vacuum_sorted('t_col')"), "22023", "reject no sort columns")
+    c.execute("CREATE TABLE t_js (a int, j json) USING pgcolumnar")
+    c.execute("INSERT INTO t_js SELECT g, '{}'::json FROM generate_series(1,10) g")
+    expect.text(_refused(c, "SELECT pgcolumnar.vacuum_sorted('t_js', 'j')"), "42883", "reject non-sortable type")
diff --git a/test/sorted_projection.sh b/test/sorted_projection.sh
index f2d8367a0..3209611bd 100755
--- a/test/sorted_projection.sh
+++ b/test/sorted_projection.sh
@@ -29,13 +29,20 @@ groups_removed() {
 		| grep -F "Vectors Skipped" | grep -oE '[0-9]+$' | head -1
 }
 
+# expect_error LABEL SQL SQLSTATE: the refusal must be THIS one, by its SQLSTATE.
+#
+# Any error used to count. Measured (#1341): with vacuum_sorted's columnar check
+# removed, the heap table still raised -- XX001, reading the heap's file as
+# columnar storage -- and "reject non-columnar table" stayed green on a refusal
+# that was no longer there. And a red was recorded as "$label (expected error)",
+# a name the passing run never records, so the ledger's row could not be dated;
+# it is recorded under the label now.
 expect_error() {
-	local label="$1" sql="$2"
-	if psql_run "$sql" >/dev/null 2>&1; then
-		check "$label (expected error)" "succeeded" "error"
-	else
-		check "$label" "error" "error"
-	fi
+	local label="$1" sql="$2" want="$3" out state
+	out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \
+		-X -At -v ON_ERROR_STOP=1 -v VERBOSITY=sqlstate -c "$sql" 2>&1 >/dev/null)"
+	state="$(sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p' <<<"$out" | head -1)"
+	check "$label" "${state:-succeeded}" "$want"
 }
 
 # Many small chunk groups; the sort key k is scrambled relative to insert order
@@ -105,10 +112,28 @@ check "live count after delete+sort" "$(q "SELECT count(*) FROM t_col;")" "25000
 diff_query "count matches heap"  "SELECT count(*) FROM %T"
 diff_query "sum matches heap"    "SELECT sum(v) FROM %T"
 # SET prints its own command tag; the scalar value is the last line.
+#
+# The custom scan off too, and a premise per arm that it reads t_col_id_idx:
+# enable_seqscan alone leaves the columnar custom scan on. Measured (#1339): with
+# vacuum_sorted's reindex removed, so the index points at the old row numbers,
+# the point lookup stayed green, because it never read the index. The deletion
+# lookup plans as an Index Only Scan (it reads only id), which still asks each
+# row whether it is live, so the premise accepts either index node.
+IDXON="SET enable_seqscan=off; SET enable_bitmapscan=off; SET pgcolumnar.enable_custom_scan=off;"
+reads_index() {	# reads_index QUERY -> yes, or the plan's scan line
+	local plan
+	plan="$(q "$IDXON EXPLAIN (COSTS OFF) $1")"
+	case "$plan" in
+		*"Index Scan using t_col_id_idx"*|*"Index Only Scan using t_col_id_idx"*) echo yes ;;
+		*) echo "no: $(grep -m1 'Scan' <<<"$plan" | sed 's/^ *//')" ;;
+	esac
+}
+check "premise: the point lookup reads t_col_id_idx" "$(reads_index "SELECT t FROM t_col WHERE id = 4999")" "yes"
 check "index point lookup after sort" \
-	"$(q "SET enable_seqscan=off; SELECT t FROM t_col WHERE id = 4999;" | tail -1)" "r4999"
+	"$(q "$IDXON SELECT t FROM t_col WHERE id = 4999;" | tail -1)" "r4999"
+check "premise: the deletion lookup reads t_col_id_idx" "$(reads_index "SELECT count(*) FROM t_col WHERE id = 5000")" "yes"
 check "index sees deletion after sort" \
-	"$(q "SET enable_seqscan=off; SELECT count(*) FROM t_col WHERE id = 5000;" | tail -1)" "0"
+	"$(q "$IDXON SELECT count(*) FROM t_col WHERE id = 5000;" | tail -1)" "0"
 
 # ---------------------------------------------------------------------------
 # Idempotence: sorting an already-sorted table changes nothing observable.
@@ -123,12 +148,12 @@ check "idempotent sorted rewrite" "$h2" "$h1"
 # Argument validation.
 # ---------------------------------------------------------------------------
 echo "-- argument validation"
-expect_error "reject non-columnar table"   "SELECT pgcolumnar.vacuum_sorted('t_heap', 'k');"
-expect_error "reject unknown column"        "SELECT pgcolumnar.vacuum_sorted('t_col', 'nope');"
-expect_error "reject no sort columns"       "SELECT pgcolumnar.vacuum_sorted('t_col');"
+expect_error "reject non-columnar table"   "SELECT pgcolumnar.vacuum_sorted('t_heap', 'k');" 42809
+expect_error "reject unknown column"        "SELECT pgcolumnar.vacuum_sorted('t_col', 'nope');" 42703
+expect_error "reject no sort columns"       "SELECT pgcolumnar.vacuum_sorted('t_col');" 22023
 q "DROP TABLE IF EXISTS t_js;" >/dev/null
 q "CREATE TABLE t_js (a int, j json) USING pgcolumnar;" >/dev/null
 q "INSERT INTO t_js SELECT g, '{}'::json FROM generate_series(1,10) g;" >/dev/null
-expect_error "reject non-sortable type"     "SELECT pgcolumnar.vacuum_sorted('t_js', 'j');"
+expect_error "reject non-sortable type"     "SELECT pgcolumnar.vacuum_sorted('t_js', 'j');" 42883
 
 pgc_summary

From 332435ef62def55b89ba9482011c4c05428770e1 Mon Sep 17 00:00:00 2001
From: "Joshua D. Drake" <jd@commandprompt.com>
Date: Fri, 16 Oct 2026 18:27:20 +0000
Subject: [PATCH 6/7] test: sorted_projection seeds the ledger, 5 of 29 arms
 dated by three vacuum_sorted mutations (#1341)

The suite had no rows. suites_not_covered 192 -> 191; census 4377 rows /
3053 never / 1324 dated.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 test/check_ledger.tsv        | 29 +++++++++++++++++++++++++++++
 test/check_ledger_budget.txt | 19 +++++++++++++++++--
 2 files changed, 46 insertions(+), 2 deletions(-)

diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv
index 0a7902e8a..de7c53af7 100644
--- a/test/check_ledger.tsv
+++ b/test/check_ledger.tsv
@@ -3899,6 +3899,35 @@ settings_reference	settings_reference	each documented default is the server's	15
 settings_reference	settings_reference	every documented setting exists on the server	15;16;17;18;19	2026-10-15	the compression_level row renamed compression_levl in docs/configuration.md
 settings_reference	settings_reference	every setting the server defines is documented, the developer hooks aside	15;16;17;18;19	2026-10-15	the backticks around every setting name removed from docs/configuration.md;the compression_level row deleted from docs/configuration.md;the compression_level row renamed compression_levl in docs/configuration.md
 settings_reference	settings_reference	the reference documents at least thirty settings, so the comparisons below read something	15;16;17;18;19	2026-10-15	the backticks around every setting name removed from docs/configuration.md
+sorted_projection	sorted_projection	control: the set oracle is order-blind by design	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	count matches heap	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	idempotent sorted rewrite	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	index point lookup after sort	15;16;17;18;19	2026-10-16	columnar_vacuum.c: pgcolumnar_compact_relation no longer reindexes after its rewrite
+sorted_projection	sorted_projection	index sees deletion after sort	15;16;17;18;19	2026-10-16	columnar_vacuum.c: pgcolumnar_compact_relation no longer reindexes after its rewrite
+sorted_projection	sorted_projection	live count after delete+sort	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	multi-key range	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	post aggregate	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	post equality	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	post order-by	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	post range	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	post star	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	post-sort prunes more chunk groups	15;16;17;18;19	2026-10-16	columnar_vacuum.c: pgcolumnar_compact_relation never sorts the rows it rewrites
+sorted_projection	sorted_projection	pre aggregate	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	pre equality	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	pre order-by	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	pre range	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	pre star	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	pre-sort prunes little	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	premise: the deletion lookup reads t_col_id_idx	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	premise: the ordered oracle agrees with itself	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	premise: the ordered oracle is order-sensitive	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	premise: the point lookup reads t_col_id_idx	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	reject no sort columns	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	reject non-columnar table	15;16;17;18;19	2026-10-16	columnar_vacuum.c: pgcolumnar_vacuum_sorted no longer checks for a columnar relation
+sorted_projection	sorted_projection	reject non-sortable type	15;16;17;18;19	2026-10-16	columnar_vacuum.c: pgcolumnar_compact_relation never sorts the rows it rewrites
+sorted_projection	sorted_projection	reject unknown column	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	sum matches heap	15;16;17;18;19	never	-
+sorted_projection	sorted_projection	the plan under test is a columnar custom scan	15;16;17;18;19	never	-
 statement_self_visibility	statement_self_visibility	INSERT INTO t SELECT FROM t runs to completion	15;16;17;18;19	2026-10-15	a serial scan re-reads its row-group list when it reaches the end, so it picks up groups its own statement flushed (src/columnar_reader.c pgcolumnar_next_group_index)
 statement_self_visibility	statement_self_visibility	UPDATE of every row runs to completion	15;16;17;18;19	2026-10-15	a serial scan re-reads its row-group list when it reaches the end, so it picks up groups its own statement flushed (src/columnar_reader.c pgcolumnar_next_group_index)
 statement_self_visibility	statement_self_visibility	an INSERT in a WITH clause reading its own table returns the rows it inserted, as heap does	15;16;17;18;19	2026-10-15	a serial scan re-reads its row-group list when it reaches the end, so it picks up groups its own statement flushed (src/columnar_reader.c pgcolumnar_next_group_index)
diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt
index f20909e8f..3a65a959b 100644
--- a/test/check_ledger_budget.txt
+++ b/test/check_ledger_budget.txt
@@ -15,7 +15,7 @@
 #
 #   Adding a check to a suite that is already covered does not move it, which is
 #   what makes it safe to bound.
-suites_not_covered 192
+suites_not_covered 191
 #
 # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become
 #   one.
@@ -3490,4 +3490,19 @@ suites_not_covered 192
 #     never         3031 + 1 - 3 = 3029
 #     dated         1316 + 3 = 1319
 #     3029 + 1319 == 4348
-checks_never_observed_red 3029
+#
+# #1341, sorted_projection seeded, with its twin (OffgridwithJD): 3029 -> 3053, and
+# suites_not_covered 192 -> 191. The suite had no rows. It enters with all 29 checks
+# from five-major runs (29/29 on each of 15-19). 5 are dated by three mutations:
+# the post-sort reindex removed (both index lookups), the compaction not sorting
+# (the pruning arm and the json refusal, which the sort raises), and vacuum_sorted's
+# columnar check removed. Twenty-four arrive never: the ten heap-oracle comparisons
+# and the multi-key and post-delete counts, which no rewrite mutation here moves;
+# the oracle and index premises; the pre-sort control; idempotence; and the
+# unknown-column and no-sort-key refusals.
+#
+#     ledger rows   4348 + 29 = 4377
+#     never         3029 + 24 = 3053
+#     dated         1319 +  5 = 1324
+#     3053 + 1324 == 4377
+checks_never_observed_red 3053

From 481becfe02cd19264cd5a3da7fdbd2c9c7bf3603 Mon Sep 17 00:00:00 2001
From: "Joshua D. Drake" <jd@commandprompt.com>
Date: Fri, 16 Oct 2026 18:30:19 +0000
Subject: [PATCH 7/7] test: audit.sh's two index arms read the index they name
 (#1339)

"add-column index scan" and "create index rows indexed once" forced their
index with enable_seqscan off alone, which leaves the columnar custom scan
on, and left bitmap scans on (which would also collapse a duplicate index
entry). Both lookups now turn off the custom scan and bitmap scans, and each
asserts an Index Scan or Index Only Scan through its index.

Red first, with the index build handing every row to its callback twice:
the old suite's "add-column index scan" stayed green, because it was not
reading ac_c_idx. It is red now. "create index rows indexed once" was
already red under it (its plan happened to read the index), and now its
plan is asserted.

audit has no ledger rows and no pytest twin.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 test/audit.sh | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/test/audit.sh b/test/audit.sh
index 613e21c61..a75405a26 100755
--- a/test/audit.sh
+++ b/test/audit.sh
@@ -213,9 +213,26 @@ check "add-column aggregate" \
 	"$(q "SELECT sum(c) || '/' || count(d) || '/' || count(b) FROM ac;")" "218/6/1"
 
 # index scan over an added column
+#
+# FORCED THROUGH THE INDEX, AND ASSERTED (#1339). enable_seqscan alone leaves the
+# columnar custom scan on, so this and "create index rows indexed once" below could
+# be answered without reading the index they name. Both lookups now turn the
+# custom scan and bitmap scans off (a bitmap scan would also collapse a duplicate
+# index entry), and assert an Index Scan or Index Only Scan through their index.
+IDXONLY="SET enable_seqscan=off; SET enable_bitmapscan=off; SET pgcolumnar.enable_custom_scan=off;"
+reads_index() {	# reads_index INDEX QUERY -> yes, or the plan's scan line
+	local plan
+	plan="$(q "$IDXONLY EXPLAIN (COSTS OFF) $2")"
+	case "$plan" in
+		*"Index Scan using $1 "*|*"Index Only Scan using $1 "*) echo yes ;;
+		*) echo "no: $(grep -m1 'Scan' <<<"$plan" | sed 's/^ *//')" ;;
+	esac
+}
 q "CREATE INDEX ac_c_idx ON ac(c);" >/dev/null
+check "premise: the add-column lookup reads ac_c_idx" \
+	"$(reads_index ac_c_idx "SELECT count(*) FROM ac WHERE c=42")" "yes"
 check "add-column index scan" \
-	"$(q "SET enable_seqscan=off; SELECT count(*) FROM ac WHERE c=42;")" "5"
+	"$(q "$IDXONLY SELECT count(*) FROM ac WHERE c=42;" | tail -1)" "5"
 q "DROP TABLE ac;" >/dev/null
 
 # ---------------------------------------------------------------------------
@@ -369,8 +386,10 @@ LEAKS="$(run_pg "grep -c 'resource was not closed' '$LOGFILE'" || true)"
 check "create index no relation leak" "$LEAKS" "0"
 
 # and every row is indexed exactly once (no duplicate TIDs from a per-worker rescan)
+check "premise: the indexed-once count reads idxleak_a_idx" \
+	"$(reads_index idxleak_a_idx "SELECT count(*) FROM idxleak WHERE a>0")" "yes"
 check "create index rows indexed once" \
-	"$(q "SET enable_seqscan=off; SELECT count(*) FROM idxleak WHERE a>0;")" "50000"
+	"$(q "$IDXONLY SELECT count(*) FROM idxleak WHERE a>0;" | tail -1)" "50000"
 q "DROP TABLE idxleak;" >/dev/null
 
 # ---------------------------------------------------------------------------