test: date the #359 and #433 cost arms, and hold maintenance_crash's index to a scan (#1341)

test/check_ledger.tsv

@@ -2245,13 +2245,21 @@ latest_reading	latest_reading	the GROUP BY form returns what DISTINCT ON returns
 latest_reading	latest_reading	the LATERAL form returns what DISTINCT ON returns	15;16;17;18;19	2026-10-12	the document's LATERAL recipe orders by ts ascending, so it returns each device's FIRST reading
 latest_reading	latest_reading	with the btree the LATERAL form reads under a tenth of the table	15;16;17;18;19	never	-
 latest_reading	latest_reading	without the btree the LATERAL form reads the table once per device	15;16;17;18;19	never	-
+maintenance_crash	maintenance_crash	after a crash with compact committed, the index reads the rows the scan reads	15;16;17;18;19	2026-10-16	pgcolumnar_index_fetch_tuple finds no row (columnar_tableam.c)
 maintenance_crash	maintenance_crash	after a crash with compact committed, the table is as compact left it, with the same rows	15;16;17;18;19	never	-
+maintenance_crash	maintenance_crash	after a crash with compact uncommitted, the index reads the rows the scan reads	15;16;17;18;19	2026-10-16	pgcolumnar_index_fetch_tuple finds no row (columnar_tableam.c)
 maintenance_crash	maintenance_crash	after a crash with compact uncommitted, the table is as it was, by scan and by index	15;16;17;18;19	never	-
+maintenance_crash	maintenance_crash	after a crash with compact_rewrite committed, the index reads the rows the scan reads	15;16;17;18;19	2026-10-16	pgcolumnar_index_fetch_tuple finds no row (columnar_tableam.c)
 maintenance_crash	maintenance_crash	after a crash with compact_rewrite committed, the table is as compact_rewrite left it, with the same rows	15;16;17;18;19	2026-10-15	PgColumnarWriteLogicalData writes data pages without WAL, so a committed rewrite's pages are lost in a crash
+maintenance_crash	maintenance_crash	after a crash with compact_rewrite uncommitted, the index reads the rows the scan reads	15;16;17;18;19	2026-10-16	pgcolumnar_index_fetch_tuple finds no row (columnar_tableam.c)
 maintenance_crash	maintenance_crash	after a crash with compact_rewrite uncommitted, the table is as it was, by scan and by index	15;16;17;18;19	never	-
+maintenance_crash	maintenance_crash	after a crash with recluster committed, the index reads the rows the scan reads	15;16;17;18;19	2026-10-16	pgcolumnar_index_fetch_tuple finds no row (columnar_tableam.c)
 maintenance_crash	maintenance_crash	after a crash with recluster committed, the table is as recluster left it, with the same rows	15;16;17;18;19	2026-10-15	PgColumnarWriteLogicalData writes data pages without WAL, so a committed rewrite's pages are lost in a crash
+maintenance_crash	maintenance_crash	after a crash with recluster uncommitted, the index reads the rows the scan reads	15;16;17;18;19	2026-10-16	pgcolumnar_index_fetch_tuple finds no row (columnar_tableam.c)
 maintenance_crash	maintenance_crash	after a crash with recluster uncommitted, the table is as it was, by scan and by index	15;16;17;18;19	never	-
+maintenance_crash	maintenance_crash	after a crash with vacuum committed, the index reads the rows the scan reads	15;16;17;18;19	2026-10-16	pgcolumnar_index_fetch_tuple finds no row (columnar_tableam.c)
 maintenance_crash	maintenance_crash	after a crash with vacuum committed, the table is as vacuum left it, with the same rows	15;16;17;18;19	2026-10-15	PgColumnarWriteLogicalData writes data pages without WAL, so a committed rewrite's pages are lost in a crash
+maintenance_crash	maintenance_crash	after a crash with vacuum uncommitted, the index reads the rows the scan reads	15;16;17;18;19	2026-10-16	pgcolumnar_index_fetch_tuple finds no row (columnar_tableam.c)
 maintenance_crash	maintenance_crash	after a crash with vacuum uncommitted, the table is as it was, by scan and by index	15;16;17;18;19	never	-
 maintenance_crash	maintenance_crash	compact changed the row groups before the crash	15;16;17;18;19	never	-
 maintenance_crash	maintenance_crash	compact changed the row groups in its open transaction	15;16;17;18;19	never	-
@@ -2479,8 +2487,8 @@ native_encdesc_golden	native_encdesc_golden	int column a rawLen parses as 1200 (
 native_encdesc_golden	native_encdesc_golden	the table reads back all rows	15;16;17;18;19	never	-
 native_encdesc_golden	native_encdesc_golden	trailing sharedTableLen is 0 (u32 at offset 6+13=19)	15;16;17;18;19	never	-
 native_encdesc_golden	native_encdesc_golden	vectorCount parses as 1 (at offset 2)	15;16;17;18;19	never	-
-native_fetch_bigcap	native_fetch_bigcap	a narrow projection over a wide group is cached too (#433)	15;16;17;18;19	never	-
-native_fetch_bigcap	native_fetch_bigcap	columns that fit are cached even when their group does not (#433)	15;16;17;18;19	never	-
+native_fetch_bigcap	native_fetch_bigcap	a narrow projection over a wide group is cached too (#433)	15;16;17;18;19	2026-10-16	the fetch cache reads its row group whole and drops an entry whose group exceeds the cap after every fetch, the #433 defect (columnar_reader.c)
+native_fetch_bigcap	native_fetch_bigcap	columns that fit are cached even when their group does not (#433)	15;16;17;18;19	2026-10-16	the fetch cache reads its row group whole and drops an entry whose group exceeds the cap after every fetch, the #433 defect (columnar_reader.c)
 native_fetch_bigcap	native_fetch_bigcap	every payload survived the round trip at full length	15;16;17;18;19	never	-
 native_fetch_bigcap	native_fetch_bigcap	premise: fetching one row touched a measurable number of buffers	15;16;17;18;19	never	-
 native_fetch_bigcap	native_fetch_bigcap	premise: fetching ten rows touched a measurable number of buffers	15;16;17;18;19	never	-
@@ -2500,7 +2508,7 @@ native_fetch_cache	native_fetch_cache	a hit re-checks the group's natts against
 native_fetch_cache	native_fetch_cache	a hit re-checks the group's rowCount against the row group	15;16;17;18;19	2026-10-16	columnar_reader.c: a fetch-cache hit no longer re-checks rowCount
 native_fetch_cache	native_fetch_cache	a wide group's fetch is not far dearer than a small group's (#353)	15;16;17;18;19	2026-10-16	columnar_reader.c: crossing the fetch-cache cap drops every column, not the one that crossed it;columnar_reader.c: every fetch-cache lookup misses, so each fetch decodes its group again
 native_fetch_cache	native_fetch_cache	an entry from an earlier command is rejected	15;16;17;18;19	2026-10-16	columnar_reader.c: a fetch-cache entry from an earlier command is not rejected
-native_fetch_cache	native_fetch_cache	crossing the fetch cache cap costs proportionally, not totally (#359)	15;16;17;18;19	never	-
+native_fetch_cache	native_fetch_cache	crossing the fetch cache cap costs proportionally, not totally (#359)	15;16;17;18;19	2026-10-16	the fetch cache releases no column over the cap and drops the whole entry after a fetch that crossed it, the #359 defect (columnar_reader.c)
 native_fetch_cache	native_fetch_cache	every fc_many reading measured ten row groups (#801)	15;16;17;18;19	never	-
 native_fetch_cache	native_fetch_cache	every fc_one reading measured one row group (#801)	15;16;17;18;19	never	-
 native_fetch_cache	native_fetch_cache	every group-number comparison is keyed by the storage id too	15;16;17;18;19	2026-10-16	columnar_reader.c: the fetch-cache key compares the group number without the storage id

test/check_ledger_budget.txt

@@ -3518,4 +3518,20 @@ suites_not_covered 190
 #     never         3053 +  5 = 3058
 #     dated         1324 +  6 = 1330
 #     3058 + 1330 == 4388
-checks_never_observed_red 3058
+#
+# #1341 gaps, three arms that no mutation had reddened, and eight new (OffgridwithJD):
+# 3058 -> 3055. maintenance_crash gains eight arms, one per crash and function: after
+# recovery the index reads the rows a scan reads. Its as-before arms compare two
+# by-index readings, so an index fetch that finds nothing left all 25 green; the new
+# eight go red under it and are dated by it. Two #433 arms in native_fetch_bigcap
+# are dated by restoring the #433 defect (each entry reads its group whole, and a
+# group over the cap is dropped after every fetch), and native_fetch_cache's #359
+# ratio by restoring the #359 one (no column released, the whole entry dropped once
+# over the cap): 1956 ms against 68. All from five-major runs, 33, 15 and 26 checks
+# green on each of 15-19.
+#
+#     ledger rows   4388 +  8 = 4396
+#     never         3058 -  3 = 3055
+#     dated         1330 + 11 = 1341
+#     3055 + 1341 == 4396
+checks_never_observed_red 3055

test/maintenance_crash.sh

@@ -12,7 +12,8 @@
 #
 # The table has row groups of 1000 rows, two rows in five deleted, one group deleted
 # whole, and an index. After each recovery the rows must match by content, the row
-# groups by count and rows, and an index scan must agree with them.
+# groups by count and rows, and an index scan must agree with them: with the state
+# before, and with a scan of the table after.
 #
 # Usage:  test/maintenance_crash.sh [PG_CONFIG]
 # Written fresh for pgColumnar.
@@ -116,6 +117,11 @@ for f in "${FUNCS[@]}"; do
 		"yes|ok|recovered"
 	check "after a crash with $name uncommitted, the table is as it was, by scan and by index" \
 		"$(state)" "$before"
+	# The arm above compares after with before, both through by_index, so an index
+	# that was already wrong before the crash agrees with itself there. This one
+	# holds the index to the rows a scan reads, now.
+	check "after a crash with $name uncommitted, the index reads the rows the scan reads" \
+		"$(by_index)" "$(rows)"
 done
 
 # ---- committed, not checkpointed ----------------------------------------------------
@@ -134,6 +140,8 @@ for f in "${FUNCS[@]}"; do
 		"$(restart_cluster)|$(r1="$(redo_count)"; [ "$r1" -gt "$redo0" ] && echo recovered || echo "no recovery logged ($redo0 before, $r1 after)")" "ok|recovered"
 	check "after a crash with $name committed, the table is as $name left it, with the same rows" \
 		"$(state)|$(rows)" "$after|$before_rows"
+	check "after a crash with $name committed, the index reads the rows the scan reads" \
+		"$(by_index)" "$(rows)"
 done
 
 pgc_summary

test/pytest/TESTS.md

@@ -7162,13 +7162,13 @@ The pytest twin of `test/prepared_across_sessions.sh` (#1341), every check by th
 
 ## 123. test_maintenance_crash.py: a crash part-way through maintenance loses nothing
 
-The pytest twin of `test/maintenance_crash.sh` (#1341), every check by the shell's name. A private cluster, because each arm crashes the postmaster: the shell kills its process group, and this side stops it with `-m immediate`, which leaves the same redo for the next start. `compact_rewrite`, `recluster`, `compact` and `vacuum` each run twice on a fresh table with row groups of 1000 rows, two rows in five deleted, one group deleted whole, and an index. First the function runs in an open transaction, another session checkpoints, and the server crashes: after recovery the table is as it was. Then the function commits and the server crashes before any checkpoint: after recovery the table is as the function left it. Each arm also checks that the server logged a crash recovery.
+The pytest twin of `test/maintenance_crash.sh` (#1341), every check by the shell's name. A private cluster, because each arm crashes the postmaster: the shell kills its process group, and this side stops it with `-m immediate`, which leaves the same redo for the next start. `compact_rewrite`, `recluster`, `compact` and `vacuum` each run twice on a fresh table with row groups of 1000 rows, two rows in five deleted, one group deleted whole, and an index. First the function runs in an open transaction, another session checkpoints, and the server crashes: after recovery the table is as it was. Then the function commits and the server crashes before any checkpoint: after recovery the table is as the function left it. Each arm also checks that the server logged a crash recovery. After each recovery an index scan through `mc_id` must read the rows a scan reads: the as-before arms compare two by-index readings with each other, so an index already wrong before the crash agrees with itself there.
 
 ### Every test
 
 | test | what it holds |
 | --- | --- |
-| `test_a_crash_part_way_through_maintenance_loses_nothing` | for each function, the uncommitted crash and the committed crash, each with its recovery logged |
+| `test_a_crash_part_way_through_maintenance_loses_nothing` | for each function, the uncommitted crash and the committed crash, each with its recovery logged and the index agreeing with a scan after it |
 
 ## 124. test_serializable.py: SERIALIZABLE refuses a write skew on a columnar table
 

test/pytest/test_maintenance_crash.py

@@ -9,6 +9,8 @@
   - the function has committed, and nothing has checkpointed since, so its work exists
     only in WAL. Recovery must leave the table as the function left it.
 
+After each recovery an index scan must agree with the state before, and with a scan.
+
 A PRIVATE CLUSTER, because each crash takes the postmaster down. The shell kills the
 postmaster's process group; this side stops it with `-m immediate`, which leaves the
 same work for the next start: "database system was not properly shut down", and redo.
@@ -37,11 +39,14 @@ def _one(dsn, sql, setup=()):
 BY_INDEX = ("SET enable_seqscan = off", "SET enable_bitmapscan = off", "SET pgcolumnar.enable_custom_scan = off")
 
 
-def _state(dsn):
+def _by_index(dsn):
     # the custom scan off too, and asserted by a premise: with only seqscan off this
     # whole-table range planned as Custom Scan (PgColumnarScan) and read no index
-    by_index = _one(dsn, ROWS + " WHERE id BETWEEN 1 AND 10000", BY_INDEX)
-    return f"{_one(dsn, ROWS)}|{_one(dsn, GROUPS)}|{by_index}"
+    return _one(dsn, ROWS + " WHERE id BETWEEN 1 AND 10000", BY_INDEX)
+
+
+def _state(dsn):
+    return f"{_one(dsn, ROWS)}|{_one(dsn, GROUPS)}|{_by_index(dsn)}"
 
 
 def _fresh_table(dsn):
@@ -116,6 +121,10 @@ def test_a_crash_part_way_through_maintenance_loses_nothing(pgc_cluster, expect)
                         f"{name} ran in its open transaction, and the cluster came back after a crash")
             expect.text(_state(dsn), before,
                         f"after a crash with {name} uncommitted, the table is as it was, by scan and by index")
+            # the arm above compares with before, both by index: an index already wrong
+            # before the crash agrees with itself there, so hold it to a scan now
+            expect.text(_by_index(dsn), _one(dsn, ROWS),
+                        f"after a crash with {name} uncommitted, the index reads the rows the scan reads")
 
         for f in ("compact_rewrite('mc', 0.0)", "recluster('mc', 'v')", "compact('mc')", "vacuum('mc')"):
             name = f.split("(")[0]
@@ -139,6 +148,8 @@ def test_a_crash_part_way_through_maintenance_loses_nothing(pgc_cluster, expect)
                         f"{name} committed, and the cluster came back after a crash before any checkpoint")
             expect.text(f"{_state(dsn)}|{_one(dsn, ROWS)}", f"{after}|{before_rows}",
                         f"after a crash with {name} committed, the table is as {name} left it, with the same rows")
+            expect.text(_by_index(dsn), _one(dsn, ROWS),
+                        f"after a crash with {name} committed, the index reads the rows the scan reads")
     finally:
         cluster.stop()
         shutil.rmtree(work, ignore_errors=True)