From a2e8f90e54b39740452ea3a4a314a9e04bd745ed Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Sat, 17 Oct 2026 13:01:24 +0000 Subject: [PATCH] test: native_varlena_bound gains its pytest twin, and its checks reach the bound they name (#1341) Two of the suite's arms could not see what they name, and its one planted length could not tell a correct bound from an off-by-one. - It corrupted the minimum, then the maximum, without restoring the minimum. native_zone_excludes decodes the minimum first, so the maximum's XX001 came from the minimum. With the maximum alone decoded unbounded, the old suite is 7/7. Both bounds are now saved and each is restored before the next corruption, and a new check asserts the restored minimum reads all 2000 rows. - "backend survived" was a liveness probe, which passes once crash recovery finishes, and a probe that landed during recovery failed the next scenario's checks as well. It now counts "terminated by signal" in the server log from each corruption, as corruption.sh does, and waits recovery out. - The boundary (#1524 review): a length that fits exactly reads cleanly, one that runs one byte past is refused, and a 4-byte header with fewer than 4 bytes left is refused. With the bound off by one, or the header bound removed, the suite was 8/8. The twin runs on a private cluster, since a crash drops every session on the server, one check per test. Seeded: 11 rows, 7 dated by five mutations, 4 never. suites_not_covered 185 -> 184, census 3077 -> 3081, cluster_tests 932 -> 943. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 11 ++ test/check_ledger_budget.txt | 29 +++- test/native_varlena_bound.sh | 63 ++++++- test/pytest/TESTS.md | 23 +++ test/pytest/expected_tests.txt | 4 +- test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_native_varlena_bound.py | 200 +++++++++++++++++++++++ test/pytest/unported_suites.txt | 1 - 8 files changed, 327 insertions(+), 5 deletions(-) create mode 100644 test/pytest/test_native_varlena_bound.py diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index e4f40e129..c9f66ac96 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -3172,6 +3172,17 @@ native_truncate native_truncate second truncate is a no-op 15;16;17;18;19 never native_truncate native_truncate truncate before any delete is a no-op 15;16;17;18;19 never - native_truncate native_truncate truncate refused inside a transaction block 15;16;17;18;19 2026-10-16 pgcolumnar.truncate() runs inside a transaction block, its IsTransactionBlock refusal removed (columnar_vacuum.c) native_truncate native_truncate truncate returned blocks 15;16;17;18;19 2026-10-16 pgcolumnar.truncate() returns 0 without truncating (columnar_vacuum.c) +native_varlena_bound native_varlena_bound a 4-byte header cut off by the buffer's end is refused (XX001) 15;16;17;18;19 2026-10-17 columnar.h: PgColumnarVarSizeAnyUnalignedBounded reads a 4-byte header with fewer than 4 bytes left +native_varlena_bound native_varlena_bound a max with an over-long varlena length is refused, not an OOB read (XX001) 15;16;17;18;19 2026-10-17 columnar.h: PgColumnarVarSizeAnyUnalignedBounded trusts the stored length (no check that it fits the stream);columnar_reader.c: native_zone_excludes decodes the zone map's maximum with no bound (its end 2 GB away) +native_varlena_bound native_varlena_bound a min whose length exactly fits its buffer reads cleanly 15;16;17;18;19 never - +native_varlena_bound native_varlena_bound a min whose length runs one byte past its buffer is refused (XX001) 15;16;17;18;19 2026-10-17 columnar.h: PgColumnarVarSizeAnyUnalignedBounded lets the stored length run one byte past the stream end;columnar.h: PgColumnarVarSizeAnyUnalignedBounded trusts the stored length (no check that it fits the stream) +native_varlena_bound native_varlena_bound a min with an over-long varlena length is refused, not an OOB read (XX001) 15;16;17;18;19 2026-10-17 columnar.h: PgColumnarVarSizeAnyUnalignedBounded trusts the stored length (no check that it fits the stream) +native_varlena_bound native_varlena_bound an external-TOAST-tagged min value is refused (XX001) 15;16;17;18;19 2026-10-17 columnar.h: PgColumnarVarSizeAnyUnalignedBounded accepts an external-TOAST tag +native_varlena_bound native_varlena_bound backend survived the corrupt maximum 15;16;17;18;19 2026-10-17 columnar.h: PgColumnarVarSizeAnyUnalignedBounded trusts the stored length (no check that it fits the stream);columnar_reader.c: native_zone_excludes decodes the zone map's maximum with no bound (its end 2 GB away) +native_varlena_bound native_varlena_bound backend survived the corrupt minimum 15;16;17;18;19 2026-10-17 columnar.h: PgColumnarVarSizeAnyUnalignedBounded trusts the stored length (no check that it fits the stream) +native_varlena_bound native_varlena_bound backend survived the external-tagged minimum 15;16;17;18;19 never - +native_varlena_bound native_varlena_bound the txt column has a whole-chunk zone map to corrupt 15;16;17;18;19 never - +native_varlena_bound native_varlena_bound with the minimum restored, the range read is clean 15;16;17;18;19 never - native_vecskip native_vecskip and the aggregate skips the same seven vectors 15;16;17;18;19 2026-10-16 columnar_reader.c: no vector skip at all (zone-map rule off, refine_skipvec and qual_skipvec return at once) native_vecskip native_vecskip cross-vector range parity 15;16;17;18;19 never - native_vecskip native_vecskip empty range parity 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 12ffeb587..1107206cf 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 185 +suites_not_covered 184 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3665,4 +3665,29 @@ suites_not_covered 185 # never 3080 - 3 = 3077 # dated 1375 + 3 = 1378 # 3077 + 1378 == 4455 -checks_never_observed_red 3077 +# +# #1341, native_varlena_bound seeded, with its twin (OffgridwithJD): 3077 -> 3081, +# and suites_not_covered 185 -> 184. The suite had no rows. It enters with 11 checks +# from five-major runs (11/11 on each of 15-19), after two fixes and three new arms. +# It never restored the corrupt minimum before corrupting the maximum, and the zone +# map decodes the minimum first, so the maximum's check was answered by the minimum +# (the old suite is 7/7 with the maximum alone decoded unbounded); and "backend +# survived" was a liveness probe, which passes once crash recovery finishes, so it +# now also counts crashes in the server log and waits recovery out. A ~1 GB length +# cannot tell a correct bound from an off-by-one (#1524 review), so the boundary is +# planted too: a length that fits exactly, one that runs one byte past, and a 4-byte +# header with fewer than 4 bytes left, read by its message, since with the header +# bound gone the length check after it can raise the same XX001 (#1524 review: it +# did on @jdatcmd's build; on these the mis-read length is accepted and no error +# comes, red either way). 7 are dated by five mutations: the bounded +# reader trusting the stored length, letting it run one byte past, reading a cut-off +# 4-byte header, accepting an external-TOAST tag, and the maximum alone decoded +# unbounded. Four arrive never: the zone-map premise, the restore, the exact fit, +# and survival of the external tag (under the tag's mutation the read returns with +# no error and no backend dies). +# +# ledger rows 4455 + 11 = 4466 +# never 3077 + 4 = 3081 +# dated 1378 + 7 = 1385 +# 3081 + 1385 == 4466 +checks_never_observed_red 3081 diff --git a/test/native_varlena_bound.sh b/test/native_varlena_bound.sh index 797092b6d..5315b3bcd 100755 --- a/test/native_varlena_bound.sh +++ b/test/native_varlena_bound.sh @@ -58,18 +58,54 @@ errcode() { # SQL -> the 5-char SQLSTATE it raises, or NOERR -qtA -v VERBOSITY=sqlstate -c "$1" 2>&1 \ | sed -n 's/^ERROR: \([0-9A-Z]\{5\}\).*/\1/p' | head -1 } -alive() { [ "$(q 'SELECT 1;')" = "1" ] && echo yes || echo no; } +refusal() { # SQL -> "SQLSTATE: message" of the error it raises, or NOERR's empty + # the SQLSTATE alone cannot tell which bound refused: with the header bound gone, + # the length check after it raises the same XX001 (#1524 review) + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -qtA -v VERBOSITY=verbose -c "$1" 2>&1 \ + | sed -n 's/^ERROR: \([0-9A-Z]\{5\}\): \(.*\)/\1: \2/p' | head -1 +} +# A liveness probe alone passes once crash recovery finishes, so survival also +# requires that no backend has died since its scenario started, read from the server +# log rather than from timing (corruption.sh measured the same hole). +crashes() { + # grep -c prints 0 itself when nothing matches, and then exits 1 + pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true +} +alive() { # since this scenario's corruption, which notes the count + # a crash puts the server through recovery: wait it out, so the next scenario + # starts on a server that answers and one crash cannot fail the checks after it + local _i + for _i in $(seq 60); do [ "$(q 'SELECT 1;' 2>/dev/null)" = "1" ] && break; sleep 1; done + [ "$(q 'SELECT 1;')" = "1" ] && [ "$(crashes)" = "$CRASHES_BEFORE" ] && echo yes \ + || echo "no ($(( $(crashes) - CRASHES_BEFORE )) backend crashes)" +} +# the zone map's untouched bounds, to put back: native_zone_excludes decodes the +# minimum BEFORE the maximum, so a corrupt minimum left in place answers for the +# maximum's check, which then never reaches the maximum +ORIG_MIN="$(q "SELECT encode(minimum, 'hex') FROM pgcolumnar.zone_map + WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;")" +ORIG_MAX="$(q "SELECT encode(maximum, 'hex') FROM pgcolumnar.zone_map + WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;")" # --- corrupt the MINIMUM ------------------------------------------------------ q "UPDATE pgcolumnar.zone_map SET minimum = E'\\\\xfcffffff00'::bytea WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null +CRASHES_BEFORE="$(crashes)" check "a min with an over-long varlena length is refused, not an OOB read (XX001)" \ "$(errcode "SELECT count(*) FROM t WHERE txt > 'a';")" "XX001" check "backend survived the corrupt minimum" "$(alive)" "yes" # --- restore, then corrupt the MAXIMUM --------------------------------------- +q "UPDATE pgcolumnar.zone_map SET minimum = decode('$ORIG_MIN', 'hex') + WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null +# put back from a saved hex: an empty or wrong restore would leave the minimum +# corrupt, and its XX001 would answer for the maximum +check "with the minimum restored, the range read is clean" \ + "$(q "SELECT count(*) FROM t WHERE txt < 'zzz';")" "2000" q "UPDATE pgcolumnar.zone_map SET maximum = E'\\\\xfcffffff00'::bytea WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null +CRASHES_BEFORE="$(crashes)" check "a max with an over-long varlena length is refused, not an OOB read (XX001)" \ "$(errcode "SELECT count(*) FROM t WHERE txt < 'zzz';")" "XX001" check "backend survived the corrupt maximum" "$(alive)" "yes" @@ -77,10 +113,35 @@ check "backend survived the corrupt maximum" "$(alive)" "yes" # --- an external-TOAST tag in the raw value stream is refused ----------------- # 0x01 is the 1-byte external-TOAST tag; native storage never holds one, and # detoasting it would deref a bogus pointer. VARSIZE alone would accept it. +# The maximum is restored first, so only the tagged minimum is wrong. +q "UPDATE pgcolumnar.zone_map SET maximum = decode('$ORIG_MAX', 'hex') + WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null q "UPDATE pgcolumnar.zone_map SET minimum = E'\\\\x0102030405060708090a0b0c0d0e0f1011'::bytea WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null +CRASHES_BEFORE="$(crashes)" check "an external-TOAST-tagged min value is refused (XX001)" \ "$(errcode "SELECT count(*) FROM t WHERE txt > 'a';")" "XX001" check "backend survived the external-tagged minimum" "$(alive)" "yes" +# --- the boundary: a length that fits exactly, and one that runs one byte past -- +# A length far past the stream cannot tell a correct bound from an off-by-one, +# and a one-byte overread is what the bound exists to stop (#1524 review). 0x07 is +# a 1-byte header saying 3 bytes, header included: 'v1', exactly the stream. 0x09 +# says 4, one past it. Then 3 bytes whose first is a 4-byte header: the header +# itself runs past the stream end, read by its message: the length check after the +# header bound raises the same XX001. (The header bytes are little-endian's layout.) +q "UPDATE pgcolumnar.zone_map SET minimum = E'\\\\x077631'::bytea + WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null +check "a min whose length exactly fits its buffer reads cleanly" \ + "$(q "SELECT count(*) FROM t WHERE txt > 'a';")" "2000" +q "UPDATE pgcolumnar.zone_map SET minimum = E'\\\\x097631'::bytea + WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null +check "a min whose length runs one byte past its buffer is refused (XX001)" \ + "$(errcode "SELECT count(*) FROM t WHERE txt > 'a';")" "XX001" +q "UPDATE pgcolumnar.zone_map SET minimum = E'\\\\x0c0000'::bytea + WHERE storage_id=$SID AND column_index=1 AND vector_index=-1;" >/dev/null +check "a 4-byte header cut off by the buffer's end is refused (XX001)" \ + "$(refusal "SELECT count(*) FROM t WHERE txt > 'a';")" \ + "XX001: pgcolumnar: varlena header runs past the value stream end" + pgc_summary diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index fe85d4f83..3f8a6f7ed 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -226,6 +226,7 @@ behaviour, the source of that number is named. - [178. test_native_roundtrip.py: a native table reads back what the heap holds](#178-test_native_roundtrippy-a-native-table-reads-back-what-the-heap-holds) - [179. test_native_bloom.py: a bloom filter prunes the equality probe min/max cannot](#179-test_native_bloompy-a-bloom-filter-prunes-the-equality-probe-minmax-cannot) - [180. test_native_toasted_write.py: a toasted value is written once and read back whole](#180-test_native_toasted_writepy-a-toasted-value-is-written-once-and-read-back-whole) +- [181. test_native_varlena_bound.py: a corrupt varlena length is refused, not read past its buffer](#181-test_native_varlena_boundpy-a-corrupt-varlena-length-is-refused-not-read-past-its-buffer) ## 1. How to read a test in here @@ -7978,3 +7979,25 @@ Each check is its own test. | `test_the_equality_probe_reaches_the_bloom` | the probe is a usable skip predicate | | `test_equality_on_a_toasted_value_matches_heap` | and its count matches the heap | | `test_the_non_toasted_column_is_byte_identical` | the short column's digest matches the heap | + +## 181. test_native_varlena_bound.py: a corrupt varlena length is refused, not read past its buffer + +The pytest twin of `test/native_varlena_bound.sh` (ported for #1341), every check by the shell's name. Every decode site reads a varlena's stored length through `PgColumnarVarSizeAnyUnalignedBounded`, which refuses a length, or an external-TOAST tag, that would read past the value stream. The zone map's min/max is a catalog bytea, so a plain UPDATE reaches it: a 5-byte value whose header claims ~1 GB, then a predicate that consults it. The text column is corrupted, since the int id's min/max is no varlena. A length far past the stream cannot tell a correct bound from an off-by-one, so the boundary is planted too: a length that fits exactly, one that runs one byte past, and a 4-byte header with fewer than 4 bytes left. The minimum is restored before the maximum is corrupted, because the zone map decodes the minimum first and a corrupt minimum left in place would answer for the maximum. A crash is read from the server log, counted from each corruption, since a liveness probe alone passes once crash recovery finishes. It runs on a private cluster, because a crash drops every session on the server. + +### Every test + +Each check is its own test. + +| test | what it holds | +| --- | --- | +| `test_the_txt_column_has_a_whole_chunk_zone_map_to_corrupt` | there is one zone-map row to corrupt | +| `test_a_min_with_an_over_long_varlena_length_is_refused` | a ~1 GB minimum raises XX001 | +| `test_backend_survived_the_corrupt_minimum` | and no backend died | +| `test_with_the_minimum_restored_the_range_read_is_clean` | the restored minimum reads back all 2000 rows | +| `test_a_max_with_an_over_long_varlena_length_is_refused` | a ~1 GB maximum raises XX001 | +| `test_backend_survived_the_corrupt_maximum` | and no backend died | +| `test_an_external_toast_tagged_min_value_is_refused` | an external-TOAST tag raises XX001 | +| `test_backend_survived_the_external_tagged_minimum` | and no backend died | +| `test_a_min_whose_length_exactly_fits_its_buffer_reads_cleanly` | an exact-fit length reads all 2000 rows | +| `test_a_min_whose_length_runs_one_byte_past_its_buffer_is_refused` | one byte past raises XX001 | +| `test_a_4_byte_header_cut_off_by_the_buffers_end_is_refused` | a header with fewer than 4 bytes left raises XX001 | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index fe5243f4b..0a77aff30 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -842,4 +842,6 @@ guard_tests 442 # tests), by collection over the cluster files. # cluster_tests 900 -> 932 for #1341, test_range_pruning.py ported by name, one check per # test (2 -> 34 tests), by collection over the cluster files. -cluster_tests 932 +# cluster_tests 932 -> 943 for #1341's test_native_varlena_bound.py, the twin of native_varlena_bound.sh: +# eleven tests, one per check, by collection over the cluster files on this tree. +cluster_tests 943 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index e713319be..b91144b13 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -164,6 +164,7 @@ "native_toasted_write", "native_truncate", "native_vacuum_race", + "native_varlena_bound", "native_vecskip", "native_writer", "objstore_allowlist", diff --git a/test/pytest/test_native_varlena_bound.py b/test/pytest/test_native_varlena_bound.py new file mode 100644 index 000000000..31d3eeb8c --- /dev/null +++ b/test/pytest/test_native_varlena_bound.py @@ -0,0 +1,200 @@ +"""A varlena's stored length is bounded by its buffer. The pytest twin of +`test/native_varlena_bound.sh`, every check by the shell's name. + +A varying-length value in a native chunk or a zone map's min/max carries its own +length prefix, and every decode site reads it through +PgColumnarVarSizeAnyUnalignedBounded, which refuses a length (or an external-TOAST +tag) that would read past the buffer. The zone map's min/max is a catalog bytea, so +a plain UPDATE reaches it: a 5-byte value whose 4-byte header claims ~1 GB, then a +predicate that consults it. Unbounded, that is a ~1 GB memcpy from 5 bytes and a +SIGSEGV; bounded, a clean XX001 and a surviving backend. A crash is read from the +server log, counted from the corruption: a liveness probe alone passes once crash +recovery has finished. + +A PRIVATE CLUSTER, because a crash would drop every session on the server. +""" +import pathlib +import shutil +import tempfile +import time + +import psycopg +import pytest + +from pgc_vacuity import query_error + +# the text column's whole-chunk zone map: column 0 is the int id, whose min/max is no varlena +WHERE = "storage_id = {sid} AND column_index = 1 AND vector_index = -1" +# 0xfcffffff is a 4-byte varlena header claiming 0x3fffffff bytes; the value is 5 +OVERLONG = r"'\xfcffffff00'::bytea" +# 0x01 is the 1-byte external-TOAST tag, which native storage never holds +EXTERNAL = r"'\x0102030405060708090a0b0c0d0e0f1011'::bytea" +# the header bytes are little-endian's layout +EXACT, ONE_PAST, CUT_HEADER = r"'\x077631'::bytea", r"'\x097631'::bytea", r"'\x0c0000'::bytea" + + +def _connect(cluster): + """A connection, after waiting out any crash recovery: each scenario starts on + a server that accepts it, so one crash cannot fail the scenarios after it.""" + deadline = time.monotonic() + 60 + while True: + try: + return psycopg.connect(cluster.dsn(), autocommit=True, connect_timeout=5) + except psycopg.OperationalError: + if time.monotonic() > deadline: + raise + time.sleep(0.5) + + +def _one(cluster, sql): + try: + with _connect(cluster) as c: + return str(c.execute(sql).fetchone()[0]) + except psycopg.Error as exc: + return query_error(str(exc).splitlines()[0]) + + +def _refused(cluster, sql): + """-> the SQLSTATE sql raised, or what happened instead: a crash ends the + connection with no SQLSTATE at all.""" + try: + with _connect(cluster) as c: + row = c.execute(sql).fetchone() + return f"no error ({row})" + except psycopg.Error as exc: + return exc.sqlstate or f"no error ({str(exc).splitlines()[0]})" + + +def _refusal(cluster, sql): + """-> "SQLSTATE: message" of the error sql raised: the SQLSTATE alone cannot tell + which bound refused, since with the header bound gone the length check after it + raises the same XX001 (#1524 review).""" + try: + with _connect(cluster) as c: + row = c.execute(sql).fetchone() + return f"no error ({row})" + except psycopg.Error as exc: + if exc.sqlstate is None: + return f"no error ({str(exc).splitlines()[0]})" + return f"{exc.sqlstate}: {exc.diag.message_primary}" + + +def _crashes(cluster): + try: + return (cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal") + except OSError: + return 0 + + +def _scenario(cluster, sql): + """-> (the SQLSTATE sql raised, whether the backend survived it).""" + before = _crashes(cluster) + state = _refused(cluster, sql) + up = _one(cluster, "SELECT 1") == "1" + died = _crashes(cluster) - before + return state, "yes" if up and died == 0 else f"no ({died} backend crashes)" + + +@pytest.fixture(scope="module") +def vb(pgc_cluster): + """Every step of the shell, in its order, each reading recorded as a value so each + check is its own test and one red cannot stop the rest.""" + from pgc_cluster import Cluster, pick_port + + work = pathlib.Path(tempfile.mkdtemp(prefix="pgc-varlena-")) + work.chmod(0o777) + datadir = work / "data" + datadir.mkdir() + cluster = Cluster(pgc_cluster.pg_config, "varlena", datadir, pick_port(24)) + try: + cluster.initdb() + cluster.start() + with _connect(cluster) as c: + c.execute("CREATE EXTENSION IF NOT EXISTS pgcolumnar") + c.execute("CREATE TABLE t (id int, txt text) USING pgcolumnar") + c.execute("INSERT INTO t SELECT g, 'v' || g FROM generate_series(1, 2000) g") + sid = c.execute("SELECT pgcolumnar.get_storage_id('t')").fetchone()[0] + # the untouched bounds, to put back: native_zone_excludes decodes the minimum + # BEFORE the maximum, so a corrupt minimum left in place would answer for + # the maximum's check, which would then never reach the maximum + where = WHERE.format(sid=sid) + orig_min, orig_max = c.execute(f"SELECT minimum, maximum FROM pgcolumnar.zone_map WHERE {where}").fetchone() + + def put(col, value, params=()): + # a write that fails is printed and left for the checks to see + try: + with _connect(cluster) as c: + c.execute(f"UPDATE pgcolumnar.zone_map SET {col} = {value} WHERE {where}", params) + except psycopg.Error as exc: + print(f"-- zone map write failed: {str(exc).splitlines()[0]}") + + r = {"zone maps": _one(cluster, f"SELECT count(*) FROM pgcolumnar.zone_map WHERE {where}")} + put("minimum", OVERLONG) + r["min"] = _scenario(cluster, "SELECT count(*) FROM t WHERE txt > 'a'") + put("minimum", "%s", (orig_min,)) + r["restored"] = _one(cluster, "SELECT count(*) FROM t WHERE txt < 'zzz'") + put("maximum", OVERLONG) + r["max"] = _scenario(cluster, "SELECT count(*) FROM t WHERE txt < 'zzz'") + put("maximum", "%s", (orig_max,)) + put("minimum", EXTERNAL) + r["external"] = _scenario(cluster, "SELECT count(*) FROM t WHERE txt > 'a'") + # the boundary (#1524 review): a length far past the stream cannot tell a + # correct bound from an off-by-one. 0x07 is a 1-byte header saying 3 bytes, + # header included: 'v1', exactly the stream; 0x09 says 4, one past it. Then + # 3 bytes whose first is a 4-byte header, which itself runs past the end, read + # by its message: the length check after the header bound raises the same XX001. + put("minimum", EXACT) + r["exact"] = _one(cluster, "SELECT count(*) FROM t WHERE txt > 'a'") + put("minimum", ONE_PAST) + r["one past"] = _refused(cluster, "SELECT count(*) FROM t WHERE txt > 'a'") + put("minimum", CUT_HEADER) + r["cut header"] = _refusal(cluster, "SELECT count(*) FROM t WHERE txt > 'a'") + yield r + finally: + cluster.stop() + shutil.rmtree(work, ignore_errors=True) + + +def test_the_txt_column_has_a_whole_chunk_zone_map_to_corrupt(vb, expect): + expect.text(vb["zone maps"], "1", "the txt column has a whole-chunk zone map to corrupt") + + +def test_a_min_with_an_over_long_varlena_length_is_refused(vb, expect): + expect.text(vb["min"][0], "XX001", "a min with an over-long varlena length is refused, not an OOB read (XX001)") + + +def test_backend_survived_the_corrupt_minimum(vb, expect): + expect.text(vb["min"][1], "yes", "backend survived the corrupt minimum") + + +def test_with_the_minimum_restored_the_range_read_is_clean(vb, expect): + expect.text(vb["restored"], "2000", "with the minimum restored, the range read is clean") + + +def test_a_max_with_an_over_long_varlena_length_is_refused(vb, expect): + expect.text(vb["max"][0], "XX001", "a max with an over-long varlena length is refused, not an OOB read (XX001)") + + +def test_backend_survived_the_corrupt_maximum(vb, expect): + expect.text(vb["max"][1], "yes", "backend survived the corrupt maximum") + + +def test_an_external_toast_tagged_min_value_is_refused(vb, expect): + expect.text(vb["external"][0], "XX001", "an external-TOAST-tagged min value is refused (XX001)") + + +def test_backend_survived_the_external_tagged_minimum(vb, expect): + expect.text(vb["external"][1], "yes", "backend survived the external-tagged minimum") + + +def test_a_min_whose_length_exactly_fits_its_buffer_reads_cleanly(vb, expect): + expect.text(vb["exact"], "2000", "a min whose length exactly fits its buffer reads cleanly") + + +def test_a_min_whose_length_runs_one_byte_past_its_buffer_is_refused(vb, expect): + expect.text(vb["one past"], "XX001", "a min whose length runs one byte past its buffer is refused (XX001)") + + +def test_a_4_byte_header_cut_off_by_the_buffers_end_is_refused(vb, expect): + expect.text(vb["cut header"], "XX001: pgcolumnar: varlena header runs past the value stream end", + "a 4-byte header cut off by the buffer's end is refused (XX001)") diff --git a/test/pytest/unported_suites.txt b/test/pytest/unported_suites.txt index 460a70e18..31266df8f 100644 --- a/test/pytest/unported_suites.txt +++ b/test/pytest/unported_suites.txt @@ -107,7 +107,6 @@ native_reclaim_reconcile native_rewrite native_rewrite_conc native_sort_by -native_varlena_bound native_vecdecode native_zonemap native_zonemap_narrow