test: parquet_count_bounds gains its pytest twin, and a crashed backend no longer passes as survived (#1341)

test/check_ledger.tsv

@@ -3902,6 +3902,43 @@ parallel_scan_cost	parallel_scan_cost	premise: the serial plan has no Gather	15;
 parallel_scan_cost	parallel_scan_cost	premise: the serial plan is a columnar scan	15;16;17;18;19	never	-
 parallel_scan_cost	parallel_scan_cost	premise: the table holds every inserted row	15;16;17;18;19	never	-
 parallel_scan_cost	parallel_scan_cost	the leader-participation branch changes the divisor	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	a huge num_rows is rejected rather than walking off the allocation	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	a valid three-record list imports three rows (not over-rejected)	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	an honest file imports its rows	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	an honest one-record list imports one row	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	and it did not silently return more rows than the file holds	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	and it did not silently return more rows than the file's records	15;16;17;18;19	2026-10-21	columnar_parquet_reader.c: pq_require_entry never refuses a read past the decoded entries
+parquet_count_bounds	parquet_count_bounds	and the backend is still up (it did not wrap-allocate and crash)	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	and the backend survived it	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	and the backend survived the list over-read	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	num_rows past a list's record count is rejected, not emitted as phantom rows	15;16;17;18;19	2026-10-21	columnar_parquet_reader.c: pq_require_entry never refuses a read past the decoded entries
+parquet_count_bounds	parquet_count_bounds	num_rows past the chunk's values is rejected, not emitted as phantom rows	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	num_values 2^62 is rejected, not imported	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	premise: the crafter produced the seven files	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	premise: the server is up before any malformed file is read	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	the num_rows rejection names num_rows in its detail	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	the num_values rejection names num_values in its detail	15;16;17;18;19	never	-
+parquet_count_bounds	parquet_count_bounds	the server is still up after the honest import	15;16;17;18;19	never	-
+parquet_level_width	parquet_level_width	a non-positive maximum returns before shifting at all	15;16;17;18;19	2026-10-21	columnar_parquet_format.h: pq_bits_for's non-positive guard is commented out
+parquet_level_width	parquet_level_width	and both callers reach the shared one	15;16;17;18;19	never	-
+parquet_level_width	parquet_level_width	and no signed 1 << remains in it	15;16;17;18;19	2026-10-21	columnar_parquet_format.h: pq_bits_for shifts a signed 1
+parquet_level_width	parquet_level_width	no local copy of the helper survives in the writer or the reader	15;16;17;18;19	2026-10-21	columnar_parquet.c: a local bits_for copy reappears in the writer
+parquet_level_width	parquet_level_width	the loop is bounded below 32 shifts	15;16;17;18;19	2026-10-21	columnar_parquet_format.h: pq_bits_for's loop loses its b < 31 bound
+parquet_level_width	parquet_level_width	the shared level-width helper exists	15;16;17;18;19	never	-
+parquet_level_width	parquet_level_width	the shift is unsigned	15;16;17;18;19	2026-10-21	columnar_parquet_format.h: pq_bits_for shifts a signed 1
+parquet_nested	parquet_nested	nested Parquet export rows written	15;16;17;18;19	never	-
+parquet_nested	parquet_nested	nested Parquet field types	15;16;17;18;19	never	-
+parquet_nested	parquet_nested	nested Parquet values match expectation	15;16;17;18;19	2026-10-21	columnar_parquet.c: an empty array is written with definition level 0, so it reads back NULL
+parquet_nested	parquet_nested	np2 export rows	15;16;17;18;19	never	-
+parquet_nested	parquet_nested	np2 values match	15;16;17;18;19	never	-
+parquet_nested	parquet_nested	reject multi-dim array	15;16;17;18;19	2026-10-21	columnar_parquet.c: a multi-dimensional array is no longer refused
+parquet_nested_import	parquet_nested_import	export rows	15;16;17;18;19	never	-
+parquet_nested_import	parquet_nested_import	import count matches	15;16;17;18;19	never	-
+parquet_nested_import	parquet_nested_import	import rows	15;16;17;18;19	never	-
+parquet_nested_import	parquet_nested_import	nested import round-trips (arrays + composite)	15;16;17;18;19	2026-10-21	columnar_parquet_reader.c: an empty Parquet list is read back as a NULL array
+parquet_nested_import	parquet_nested_import	pyarrow import rows	15;16;17;18;19	never	-
+parquet_nested_import	parquet_nested_import	pyarrow nested file matches oracle	15;16;17;18;19	2026-10-21	columnar_parquet_reader.c: an empty Parquet list is read back as a NULL array
+parquet_nested_import	parquet_nested_import	round-trips after delete	15;16;17;18;19	2026-10-21	columnar_parquet_reader.c: an empty Parquet list is read back as a NULL array
 partition_lifecycle	partition_lifecycle	a NULL cutoff is refused, in both overloads	15;16;17;18;19	2026-10-13	partition_drop_plan without its NULL-cutoff check, in both overloads
 partition_lifecycle	partition_lifecycle	a call that would create more than 1000 partitions is refused before any DDL	15;16;17;18;19	2026-10-12	partition_create_ahead without the 1000-partition bound
 partition_lifecycle	partition_lifecycle	a cutoff before every partition plans nothing	15;16;17;18;19	2026-10-13	the cutoff filter removed: every partition planned
@@ -4497,6 +4534,19 @@ range_pruning	range_pruning	premise: the range type collates differently from it
 range_pruning	range_pruning	so does a containment probe	15;16;17;18;19	never	-
 range_pruning	range_pruning	the overlap qual is pushed down, where it used to be dropped	15;16;17;18;19	never	-
 range_pruning	range_pruning	the same for a containment probe far to its right	15;16;17;18;19	2026-10-17	columnar_write_state.c: the writer ignores an unbounded range, so its unit records only the largest finite upper bound
+read_stream	read_stream	[off] a fresh session reads enable_read_stream as off	15;16;17;18;19	2026-10-21	columnar_tableam.c: pgcolumnar.enable_read_stream registered as pgcolumnar.enable_read_streamz, so the product's setting is gone
+read_stream	read_stream	[off] aggregate	15;16;17;18;19	2026-10-20	the synchronous block-read path copies each page's bytes one byte late
+read_stream	read_stream	[off] filtered	15;16;17;18;19	2026-10-20	the synchronous block-read path copies each page's bytes one byte late
+read_stream	read_stream	[off] full scan	15;16;17;18;19	2026-10-20	the synchronous block-read path copies each page's bytes one byte late
+read_stream	read_stream	[off] range	15;16;17;18;19	2026-10-20	the synchronous block-read path copies each page's bytes one byte late
+read_stream	read_stream	[off] wide sample	15;16;17;18;19	2026-10-20	the synchronous block-read path copies each page's bytes one byte late
+read_stream	read_stream	[on] a fresh session reads enable_read_stream as on	15;16;17;18;19	2026-10-21	columnar_tableam.c: pgcolumnar.enable_read_stream registered as pgcolumnar.enable_read_streamz, so the product's setting is gone
+read_stream	read_stream	[on] aggregate	15;16;17;18;19	2026-10-20	the read-stream path copies each page's bytes one byte late (PG17+)
+read_stream	read_stream	[on] filtered	15;16;17;18;19	2026-10-20	the read-stream path copies each page's bytes one byte late (PG17+)
+read_stream	read_stream	[on] full scan	15;16;17;18;19	2026-10-20	the read-stream path copies each page's bytes one byte late (PG17+)
+read_stream	read_stream	[on] range	15;16;17;18;19	2026-10-20	the read-stream path copies each page's bytes one byte late (PG17+)
+read_stream	read_stream	[on] wide sample	15;16;17;18;19	2026-10-20	the read-stream path copies each page's bytes one byte late (PG17+)
+read_stream	read_stream	multiple stripes present	15;16;17;18;19	never	-
 reader_buffer_reuse	reader_buffer_reuse	a PROJECTED scan (one column of two) matches too	15;16;17;18;19	never	-
 reader_buffer_reuse	reader_buffer_reuse	a column ADDED after the groups exist reads back NULL, as on heap	15;16;17;18;19	2026-10-21	columnar_reader.c: when the group buffer is reused rather than grown, a projected read does not refresh it, so it decodes the previous group's bytes
 reader_buffer_reuse	reader_buffer_reuse	a rescan (nested loop) returns the same rows	15;16;17;18;19	never	-

test/check_ledger_budget.txt

@@ -15,7 +15,7 @@
 #
 #   Adding a check to a suite that is already covered does not move it, which is
 #   what makes it safe to bound.
-suites_not_covered 113
+suites_not_covered 108
 #
 # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become
 #   one.
@@ -5211,4 +5211,88 @@ suites_not_covered 113
 #     never         3790 + 3 = 3793
 #     dated         2063 + 10 = 2073
 #     3793 + 2073 == 5866
-checks_never_observed_red 3793
+#
+# #1341, read_stream seeded, with its twin (jdatcmd): 3503 -> 3504, and
+# suites_not_covered 149 -> 148. The suite had no rows. It enters with 13 checks from
+# five-major runs (13/13 on each of 15-19, shell and twin alike), two of them new.
+# The setting arrives by ALTER DATABASE SET, whose error the shell's q drops, so a
+# setting that never took ran both modes down one path: with it misspelled and the
+# synchronous path broken, the shell passed 11 of 11. Each mode now reads the setting
+# back from a fresh session first. Those two reads name a pgcolumnar setting, and the
+# setting registered under another name reddens them, so they are not premises. The
+# twin's fixture returns the failed setting as their value, so they red by name there
+# too. The ten comparisons' names are literal, and the stripe premise carries its
+# count and leaves lossy_arms.tsv. 12 are dated on PG17, each in both twins with the
+# same names: each mode's five comparisons by its path copying a page one byte late,
+# and the two read-backs by the renamed setting. 1 arrives never. Nothing observes
+# which path a read took: with the stream path taken whatever the setting says and the
+# synchronous path broken, all 13 pass, so the [off] rows hold only while the setting
+# routes reads. On 15 and 16 the stream path is compiled out and [on] repeats [off].
+#
+#     ledger rows   5866 + 13 = 5879
+#     never         3793 + 1 = 3794
+#     dated         2073 + 12 = 2085
+#     3794 + 2085 == 5879
+#
+# #1341, parquet_level_width seeded, with its twin (jdatcmd): 3543 -> 3545, and
+# suites_not_covered 144 -> 143. The suite had no rows. It enters with 7 checks from
+# five-major runs (7/7 on each of 15-19; its twin needs no cluster). Three fixes: "no local
+# copy" summed grep -c over two files, so with the writer's path wrong it read 0 and
+# passed; it now needs both files. "no signed 1 << remains" counted over the extracted
+# body, so with the header unreadable it read 0 and passed; it now needs a body. And the
+# body checks read the comments inside the body, so a guard or bound commented out of the
+# code passed; the body now has its comments blanked. 5 are dated by four source
+# mutations, each reddening the same names in both twins: a signed shift, the b < 31
+# bound removed, a local copy back in the writer, and the non-positive guard commented
+# out. 2 arrive never.
+#
+#     ledger rows   5879 + 7 = 5886
+#     never         3794 + 2 = 3796
+#     dated         2085 + 5 = 2090
+#     3796 + 2090 == 5886
+#
+# #1341, parquet_nested seeded, with its twin (jdatcmd): 3543 -> 3547, and
+# suites_not_covered 144 -> 143. The suite had no rows. It enters with 6 checks from
+# five-major runs (6/6 on each of 15-19, shell and twin alike), after two fixes. The
+# value comparison visited only the rows the file held, so a zero-row file of the same
+# schema printed MATCH; it now requires every id 1..5000. And any error counted as the
+# multi-dimensional refusal, a missing table's included; it now reads that refusal. 2 are
+# dated by two mutations, each reddening the same name in both twins: an empty array
+# written as NULL, and the multi-dimensional refusal removed. 4 arrive never.
+#
+#     ledger rows   5886 + 6 = 5892
+#     never         3796 + 4 = 3800
+#     dated         2090 + 2 = 2092
+#     3800 + 2092 == 5892
+#
+# #1341, parquet_nested_import seeded, with its twin (jdatcmd): 3600 -> 3604, and
+# suites_not_covered 139 -> 138. The suite had no rows. It enters with 7 checks from
+# five-major runs (7/7 on each of 15-19, shell and twin alike), after one fix: "import
+# count matches" compared two counts with check, so with both tables missing two empty
+# readings agreed; it is check_num. 3 are dated by one mutation, reddening the same
+# names in both twins: the reader taking an empty Parquet list for a NULL array, which
+# every round-trip and the pyarrow oracle catch. 4 arrive never.
+#
+#     ledger rows   5892 + 7 = 5899
+#     never         3800 + 4 = 3804
+#     dated         2092 + 3 = 2095
+#     3804 + 2095 == 5899
+#
+# #1341, parquet_count_bounds seeded, with its twin (jdatcmd): 3650 -> 3665, and
+# suites_not_covered 125 -> 124. The suite had no rows. It enters with 17 checks from
+# five-major runs (17/17 on each of 15-19, shell and twin alike), after three fixes. The
+# survival checks read SELECT 1, which answers after crash recovery: with the num_values
+# bound removed the 2^62 file crashed the backend and "the backend survived it" passed;
+# they now also need the log's count of killed backends unmoved. The phantom-row checks
+# caught only the crafter's exact count; they now catch any count above the file's. And
+# the crafter premise covers all seven files, not four. 2 are dated, by pq_require_entry
+# refusing nothing, which reddens the two list checks in both twins. The num_values and
+# num_rows mutations crash the backend, and a crash reddens checks it does not target
+# (every read during recovery fails), so the ledger cannot attribute them and those rows
+# stay never; the PR records what they reddened. 15 arrive never.
+#
+#     ledger rows   5899 + 17 = 5916
+#     never         3804 + 15 = 3819
+#     dated         2095 + 2 = 2097
+#     3819 + 2097 == 5916
+checks_never_observed_red 3819

test/lossy_arms.tsv

@@ -147,7 +147,6 @@ projection_scan_cost	a tight covering projection is cheaper relative to the base
 projection_scan_cost	premise: the prunable range is priced above the one-stripe floor, so the arms differ
 projection_scan_cost	while a plain range on the sort key still earns its discount
 projections	fp2 spans multiple projection row groups
-read_stream	multiple stripes present
 scan_decode_cost	a wide projection costs materially more than a narrow one (decode is priced)
 scan_decode_cost	four wide text columns cost more than eight narrow int ones (#768)
 truncate_cleanup	PREMISE the projection has its own storage, distinct from the base

test/parquet_count_bounds.sh

@@ -69,16 +69,32 @@ detail_of() {
 		-c "SELECT pgcolumnar.import_parquet('pcb','$W/$1');" 2>&1
 }
 
-# The server must still be up between calls. If a malformed file crashed the
-# backend, this probe fails and every arm below is meaningless, so it gates them.
-alive() {
-	env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -At \
-		-c 'SELECT 1;' 2>&1 | head -1
+# The server must still be up between calls, AND no backend may have died since setup.
+# A fresh connection answers again after crash recovery, so "SELECT 1" alone passed with
+# the num_values bound removed: the 2^62 file crashed the backend, recovery finished,
+# and "the backend survived it" read 1 (#1341, as avro_manifest found in #1576). The
+# count of killed backends in the server log must not have moved either.
+crashes() {
+	local _n
+	_n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)"
+	case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac
+}
+alive() {	# -> 1 when the server answers and no backend died since setup
+	local _up _now
+	_up="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -At \
+		-c 'SELECT 1;' 2>&1 | head -1)"
+	_now="$(crashes)"
+	if ! [[ "$CRASHES_BEFORE" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then echo "no (crash count unreadable)"
+	elif [ "$_up" = 1 ] && [ "$_now" = "$CRASHES_BEFORE" ]; then echo 1
+	else echo "no ([$_up], $(( _now - CRASHES_BEFORE )) backend crashes)"; fi
 }
+CRASHES_BEFORE="$(crashes)"
 
 # ---- premises ---------------------------------------------------------------
-check "premise: the crafter produced the four files" \
-	"$([ -s "$W/honest.parquet" ] && [ -s "$W/bignv.parquet" ] && [ -s "$W/bignr.parquet" ] && [ -s "$W/hugenr.parquet" ] && echo yes || echo no)" "yes"
+# All seven: the three list files were never checked, so a crafter that skipped them
+# left the list arms reading "could not open file" (#1341).
+check "premise: the crafter produced the seven files" \
+	"$(m=""; for f in honest bignv bignr hugenr list_honest list_valid3 list_bad; do [ -s "$W/$f.parquet" ] || m="$m $f"; done; echo "${m:-yes}" | sed 's/^ /missing:/')" "yes"
 check "premise: the server is up before any malformed file is read" "$(alive)" "1"
 
 # ---- the honest file is unaffected -----------------------------------------
@@ -96,8 +112,10 @@ r_bignr="$(import_of bignr.parquet)"
 check "num_rows past the chunk's values is rejected, not emitted as phantom rows" \
 	"$(grep -qi 'malformed row group' <<<"$r_bignr" && echo rejected || echo "$r_bignr")" "rejected"
 # The phantom-row harm specifically: it must NOT return 40 rows for an 8-value file.
+# ANY count above the file's 8 rows is a leak, not only the 40 the crafter declares:
+# with the bound removed, a different phantom count read "ok" (#1341).
 check "and it did not silently return more rows than the file holds" \
-	"$([ "$r_bignr" = "40" ] && echo leaked || echo ok)" "ok"
+	"$(if [[ "$r_bignr" =~ ^[0-9]+$ ]] && [ "$r_bignr" -gt 8 ]; then echo "leaked ($r_bignr rows)"; else echo ok; fi)" "ok"
 
 r_hugenr="$(import_of hugenr.parquet)"
 check "a huge num_rows is rejected rather than walking off the allocation" \
@@ -134,7 +152,7 @@ r_listbad="$(import_list_of list_bad.parquet)"
 check "num_rows past a list's record count is rejected, not emitted as phantom rows" \
 	"$(grep -qi 'malformed row group' <<<"$r_listbad" && echo rejected || echo "$r_listbad")" "rejected"
 check "and it did not silently return more rows than the file's records" \
-	"$([ "$r_listbad" = "8" ] && echo leaked || echo ok)" "ok"
+	"$(if [[ "$r_listbad" =~ ^[0-9]+$ ]] && [ "$r_listbad" -gt 1 ]; then echo "leaked ($r_listbad rows)"; else echo ok; fi)" "ok"
 check "and the backend survived the list over-read" "$(alive)" "1"
 
 pgc_summary