test: native_varlena_bound gains its pytest twin, and its checks reach the bound they name (#1341)

test/check_ledger.tsv

@@ -3172,6 +3172,17 @@ native_truncate	native_truncate	second truncate is a no-op	15;16;17;18;19	never
 native_truncate	native_truncate	truncate before any delete is a no-op	15;16;17;18;19	never	-
 native_truncate	native_truncate	truncate refused inside a transaction block	15;16;17;18;19	2026-10-16	pgcolumnar.truncate() runs inside a transaction block, its IsTransactionBlock refusal removed (columnar_vacuum.c)
 native_truncate	native_truncate	truncate returned blocks	15;16;17;18;19	2026-10-16	pgcolumnar.truncate() returns 0 without truncating (columnar_vacuum.c)
+native_varlena_bound	native_varlena_bound	a 4-byte header cut off by the buffer's end is refused (XX001)	15;16;17;18;19	2026-10-17	columnar.h: PgColumnarVarSizeAnyUnalignedBounded reads a 4-byte header with fewer than 4 bytes left
+native_varlena_bound	native_varlena_bound	a max with an over-long varlena length is refused, not an OOB read (XX001)	15;16;17;18;19	2026-10-17	columnar.h: PgColumnarVarSizeAnyUnalignedBounded trusts the stored length (no check that it fits the stream);columnar_reader.c: native_zone_excludes decodes the zone map's maximum with no bound (its end 2 GB away)
+native_varlena_bound	native_varlena_bound	a min whose length exactly fits its buffer reads cleanly	15;16;17;18;19	never	-
+native_varlena_bound	native_varlena_bound	a min whose length runs one byte past its buffer is refused (XX001)	15;16;17;18;19	2026-10-17	columnar.h: PgColumnarVarSizeAnyUnalignedBounded lets the stored length run one byte past the stream end;columnar.h: PgColumnarVarSizeAnyUnalignedBounded trusts the stored length (no check that it fits the stream)
+native_varlena_bound	native_varlena_bound	a min with an over-long varlena length is refused, not an OOB read (XX001)	15;16;17;18;19	2026-10-17	columnar.h: PgColumnarVarSizeAnyUnalignedBounded trusts the stored length (no check that it fits the stream)
+native_varlena_bound	native_varlena_bound	an external-TOAST-tagged min value is refused (XX001)	15;16;17;18;19	2026-10-17	columnar.h: PgColumnarVarSizeAnyUnalignedBounded accepts an external-TOAST tag
+native_varlena_bound	native_varlena_bound	backend survived the corrupt maximum	15;16;17;18;19	2026-10-17	columnar.h: PgColumnarVarSizeAnyUnalignedBounded trusts the stored length (no check that it fits the stream);columnar_reader.c: native_zone_excludes decodes the zone map's maximum with no bound (its end 2 GB away)
+native_varlena_bound	native_varlena_bound	backend survived the corrupt minimum	15;16;17;18;19	2026-10-17	columnar.h: PgColumnarVarSizeAnyUnalignedBounded trusts the stored length (no check that it fits the stream)
+native_varlena_bound	native_varlena_bound	backend survived the external-tagged minimum	15;16;17;18;19	never	-
+native_varlena_bound	native_varlena_bound	the txt column has a whole-chunk zone map to corrupt	15;16;17;18;19	never	-
+native_varlena_bound	native_varlena_bound	with the minimum restored, the range read is clean	15;16;17;18;19	never	-
 native_vecskip	native_vecskip	and the aggregate skips the same seven vectors	15;16;17;18;19	2026-10-16	columnar_reader.c: no vector skip at all (zone-map rule off, refine_skipvec and qual_skipvec return at once)
 native_vecskip	native_vecskip	cross-vector range parity	15;16;17;18;19	never	-
 native_vecskip	native_vecskip	empty range parity	15;16;17;18;19	never	-

test/check_ledger_budget.txt

@@ -15,7 +15,7 @@
 #
 #   Adding a check to a suite that is already covered does not move it, which is
 #   what makes it safe to bound.
-suites_not_covered 185
+suites_not_covered 184
 #
 # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become
 #   one.
@@ -3665,4 +3665,29 @@ suites_not_covered 185
 #     never         3080 - 3 = 3077
 #     dated         1375 + 3 = 1378
 #     3077 + 1378 == 4455
-checks_never_observed_red 3077
+#
+# #1341, native_varlena_bound seeded, with its twin (OffgridwithJD): 3077 -> 3081,
+# and suites_not_covered 185 -> 184. The suite had no rows. It enters with 11 checks
+# from five-major runs (11/11 on each of 15-19), after two fixes and three new arms.
+# It never restored the corrupt minimum before corrupting the maximum, and the zone
+# map decodes the minimum first, so the maximum's check was answered by the minimum
+# (the old suite is 7/7 with the maximum alone decoded unbounded); and "backend
+# survived" was a liveness probe, which passes once crash recovery finishes, so it
+# now also counts crashes in the server log and waits recovery out. A ~1 GB length
+# cannot tell a correct bound from an off-by-one (#1524 review), so the boundary is
+# planted too: a length that fits exactly, one that runs one byte past, and a 4-byte
+# header with fewer than 4 bytes left, read by its message, since with the header
+# bound gone the length check after it can raise the same XX001 (#1524 review: it
+# did on @jdatcmd's build; on these the mis-read length is accepted and no error
+# comes, red either way). 7 are dated by five mutations: the bounded
+# reader trusting the stored length, letting it run one byte past, reading a cut-off
+# 4-byte header, accepting an external-TOAST tag, and the maximum alone decoded
+# unbounded. Four arrive never: the zone-map premise, the restore, the exact fit,
+# and survival of the external tag (under the tag's mutation the read returns with
+# no error and no backend dies).
+#
+#     ledger rows   4455 + 11 = 4466
+#     never         3077 +  4 = 3081
+#     dated         1378 +  7 = 1385
+#     3081 + 1385 == 4466
+checks_never_observed_red 3081

test/native_varlena_bound.sh

@@ -58,29 +58,90 @@ errcode() { # SQL -> the 5-char SQLSTATE it raises, or NOERR
 		-qtA -v VERBOSITY=sqlstate -c "$1" 2>&1 \
 		| sed -n 's/^ERROR:  \([0-9A-Z]\{5\}\).*/\1/p' | head -1
 }
-alive() { [ "$(q 'SELECT 1;')" = "1" ] && echo yes || echo no; }
+refusal() { # SQL -> "SQLSTATE: message" of the error it raises, or NOERR's empty
+	# the SQLSTATE alone cannot tell which bound refused: with the header bound gone,
+	# the length check after it raises the same XX001 (#1524 review)
+	env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \
+		-qtA -v VERBOSITY=verbose -c "$1" 2>&1 \
+		| sed -n 's/^ERROR:  \([0-9A-Z]\{5\}\): \(.*\)/\1: \2/p' | head -1
+}
+# A liveness probe alone passes once crash recovery finishes, so survival also
+# requires that no backend has died since its scenario started, read from the server
+# log rather than from timing (corruption.sh measured the same hole).
+crashes() {
+	# grep -c prints 0 itself when nothing matches, and then exits 1
+	pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true
+}
+alive() {	# since this scenario's corruption, which notes the count
+	# a crash puts the server through recovery: wait it out, so the next scenario
+	# starts on a server that answers and one crash cannot fail the checks after it
+	local _i
+	for _i in $(seq 60); do [ "$(q 'SELECT 1;' 2>/dev/null)" = "1" ] && break; sleep 1; done
+	[ "$(q 'SELECT 1;')" = "1" ] && [ "$(crashes)" = "$CRASHES_BEFORE" ] && echo yes \
+		|| echo "no ($(( $(crashes) - CRASHES_BEFORE )) backend crashes)"
+}
+# the zone map's untouched bounds, to put back: native_zone_excludes decodes the
+# minimum BEFORE the maximum, so a corrupt minimum left in place answers for the
+# maximum's check, which then never reaches the maximum
+ORIG_MIN="$(q "SELECT encode(minimum, 'hex') FROM pgcolumnar.zone_map
+   WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;")"
+ORIG_MAX="$(q "SELECT encode(maximum, 'hex') FROM pgcolumnar.zone_map
+   WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;")"
 
 # --- corrupt the MINIMUM ------------------------------------------------------
 q "UPDATE pgcolumnar.zone_map SET minimum = E'\\\\xfcffffff00'::bytea
    WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null
+CRASHES_BEFORE="$(crashes)"
 check "a min with an over-long varlena length is refused, not an OOB read (XX001)" \
 	"$(errcode "SELECT count(*) FROM t WHERE txt > 'a';")" "XX001"
 check "backend survived the corrupt minimum" "$(alive)" "yes"
 
 # --- restore, then corrupt the MAXIMUM ---------------------------------------
+q "UPDATE pgcolumnar.zone_map SET minimum = decode('$ORIG_MIN', 'hex')
+   WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null
+# put back from a saved hex: an empty or wrong restore would leave the minimum
+# corrupt, and its XX001 would answer for the maximum
+check "with the minimum restored, the range read is clean" \
+	"$(q "SELECT count(*) FROM t WHERE txt < 'zzz';")" "2000"
 q "UPDATE pgcolumnar.zone_map SET maximum = E'\\\\xfcffffff00'::bytea
    WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null
+CRASHES_BEFORE="$(crashes)"
 check "a max with an over-long varlena length is refused, not an OOB read (XX001)" \
 	"$(errcode "SELECT count(*) FROM t WHERE txt < 'zzz';")" "XX001"
 check "backend survived the corrupt maximum" "$(alive)" "yes"
 
 # --- an external-TOAST tag in the raw value stream is refused -----------------
 # 0x01 is the 1-byte external-TOAST tag; native storage never holds one, and
 # detoasting it would deref a bogus pointer. VARSIZE alone would accept it.
+# The maximum is restored first, so only the tagged minimum is wrong.
+q "UPDATE pgcolumnar.zone_map SET maximum = decode('$ORIG_MAX', 'hex')
+   WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null
 q "UPDATE pgcolumnar.zone_map SET minimum = E'\\\\x0102030405060708090a0b0c0d0e0f1011'::bytea
    WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null
+CRASHES_BEFORE="$(crashes)"
 check "an external-TOAST-tagged min value is refused (XX001)" \
 	"$(errcode "SELECT count(*) FROM t WHERE txt > 'a';")" "XX001"
 check "backend survived the external-tagged minimum" "$(alive)" "yes"
 
+# --- the boundary: a length that fits exactly, and one that runs one byte past --
+# A length far past the stream cannot tell a correct bound from an off-by-one,
+# and a one-byte overread is what the bound exists to stop (#1524 review). 0x07 is
+# a 1-byte header saying 3 bytes, header included: 'v1', exactly the stream. 0x09
+# says 4, one past it. Then 3 bytes whose first is a 4-byte header: the header
+# itself runs past the stream end, read by its message: the length check after the
+# header bound raises the same XX001. (The header bytes are little-endian's layout.)
+q "UPDATE pgcolumnar.zone_map SET minimum = E'\\\\x077631'::bytea
+   WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null
+check "a min whose length exactly fits its buffer reads cleanly" \
+	"$(q "SELECT count(*) FROM t WHERE txt > 'a';")" "2000"
+q "UPDATE pgcolumnar.zone_map SET minimum = E'\\\\x097631'::bytea
+   WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null
+check "a min whose length runs one byte past its buffer is refused (XX001)" \
+	"$(errcode "SELECT count(*) FROM t WHERE txt > 'a';")" "XX001"
+q "UPDATE pgcolumnar.zone_map SET minimum = E'\\\\x0c0000'::bytea
+   WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null
+check "a 4-byte header cut off by the buffer's end is refused (XX001)" \
+	"$(refusal "SELECT count(*) FROM t WHERE txt > 'a';")" \
+	"XX001: pgcolumnar: varlena header runs past the value stream end"
+
 pgc_summary

test/pytest/TESTS.md

@@ -226,6 +226,7 @@ behaviour, the source of that number is named.
 - [178. test_native_roundtrip.py: a native table reads back what the heap holds](#178-test_native_roundtrippy-a-native-table-reads-back-what-the-heap-holds)
 - [179. test_native_bloom.py: a bloom filter prunes the equality probe min/max cannot](#179-test_native_bloompy-a-bloom-filter-prunes-the-equality-probe-minmax-cannot)
 - [180. test_native_toasted_write.py: a toasted value is written once and read back whole](#180-test_native_toasted_writepy-a-toasted-value-is-written-once-and-read-back-whole)
+- [181. test_native_varlena_bound.py: a corrupt varlena length is refused, not read past its buffer](#181-test_native_varlena_boundpy-a-corrupt-varlena-length-is-refused-not-read-past-its-buffer)
 
 ## 1. How to read a test in here
 
@@ -7978,3 +7979,25 @@ Each check is its own test.
 | `test_the_equality_probe_reaches_the_bloom` | the probe is a usable skip predicate |
 | `test_equality_on_a_toasted_value_matches_heap` | and its count matches the heap |
 | `test_the_non_toasted_column_is_byte_identical` | the short column's digest matches the heap |
+
+## 181. test_native_varlena_bound.py: a corrupt varlena length is refused, not read past its buffer
+
+The pytest twin of `test/native_varlena_bound.sh` (ported for #1341), every check by the shell's name. Every decode site reads a varlena's stored length through `PgColumnarVarSizeAnyUnalignedBounded`, which refuses a length, or an external-TOAST tag, that would read past the value stream. The zone map's min/max is a catalog bytea, so a plain UPDATE reaches it: a 5-byte value whose header claims ~1 GB, then a predicate that consults it. The text column is corrupted, since the int id's min/max is no varlena. A length far past the stream cannot tell a correct bound from an off-by-one, so the boundary is planted too: a length that fits exactly, one that runs one byte past, and a 4-byte header with fewer than 4 bytes left. The minimum is restored before the maximum is corrupted, because the zone map decodes the minimum first and a corrupt minimum left in place would answer for the maximum. A crash is read from the server log, counted from each corruption, since a liveness probe alone passes once crash recovery finishes. It runs on a private cluster, because a crash drops every session on the server.
+
+### Every test
+
+Each check is its own test.
+
+| test | what it holds |
+| --- | --- |
+| `test_the_txt_column_has_a_whole_chunk_zone_map_to_corrupt` | there is one zone-map row to corrupt |
+| `test_a_min_with_an_over_long_varlena_length_is_refused` | a ~1 GB minimum raises XX001 |
+| `test_backend_survived_the_corrupt_minimum` | and no backend died |
+| `test_with_the_minimum_restored_the_range_read_is_clean` | the restored minimum reads back all 2000 rows |
+| `test_a_max_with_an_over_long_varlena_length_is_refused` | a ~1 GB maximum raises XX001 |
+| `test_backend_survived_the_corrupt_maximum` | and no backend died |
+| `test_an_external_toast_tagged_min_value_is_refused` | an external-TOAST tag raises XX001 |
+| `test_backend_survived_the_external_tagged_minimum` | and no backend died |
+| `test_a_min_whose_length_exactly_fits_its_buffer_reads_cleanly` | an exact-fit length reads all 2000 rows |
+| `test_a_min_whose_length_runs_one_byte_past_its_buffer_is_refused` | one byte past raises XX001 |
+| `test_a_4_byte_header_cut_off_by_the_buffers_end_is_refused` | a header with fewer than 4 bytes left raises XX001 |

test/pytest/expected_tests.txt

@@ -842,4 +842,6 @@ guard_tests 442
 # tests), by collection over the cluster files.
 # cluster_tests 900 -> 932 for #1341, test_range_pruning.py ported by name, one check per
 # test (2 -> 34 tests), by collection over the cluster files.
-cluster_tests 932
+# cluster_tests 932 -> 943 for #1341's test_native_varlena_bound.py, the twin of native_varlena_bound.sh:
+# eleven tests, one per check, by collection over the cluster files on this tree.
+cluster_tests 943

test/pytest/test_compare_to_bash.py

@@ -164,6 +164,7 @@
             "native_toasted_write",
             "native_truncate",
             "native_vacuum_race",
+            "native_varlena_bound",
             "native_vecskip",
             "native_writer",
             "objstore_allowlist",