diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index dbcc327ba..70390f554 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -497,6 +497,26 @@ column_extent column_extent the extent of ty.s is the data's min and max, in the column_extent column_extent the extent of ty.ts is the data's min and max, in the type's order 15;16;17;18;19 2026-10-14 a writer that finds no complete extent does not recompute one;no extent row is ever written (PgColumnarExtentPreCommit returns at once) column_extent column_extent while the column that was there from the start still has one 15;16;17;18;19 2026-10-14 a writer that finds no complete extent does not recompute one;no extent row is ever written (PgColumnarExtentPreCommit returns at once) column_extent column_extent without the extent rows the planner falls back to one zone_map scan 15;16;17;18;19 2026-10-13 the planner's zone-map fallback removed: with no complete extent it reads nothing and bounds nothing +corruption corruption alive after bad descriptor 15;16;17;18;19 2026-10-16 columnar_reader.c: the unrecognized-descriptor refusal replaced by a backend crash +corruption corruption alive after bloom corruption 15;16;17;18;19 2026-10-16 columnar_bloom.c: the bloom length check removed +corruption corruption alive after byte_length 15;16;17;18;19 2026-10-16 columnar_reader.c: the row group span refusal replaced by a backend crash +corruption corruption alive after column_index=-1 15;16;17;18;19 never - +corruption corruption alive after page_length 15;16;17;18;19 2026-10-16 columnar_reader.c: the chunk-inside-its-group refusal replaced by a backend crash +corruption corruption alive after row_count 15;16;17;18;19 2026-10-16 columnar_reader.c: the no-nulls row accounting and the value-stream bound both removed +corruption corruption alive after value_count 15;16;17;18;19 2026-10-16 columnar_reader.c: the chunk-inside-its-group refusal replaced by a backend crash +corruption corruption bad descriptor errors 15;16;17;18;19 2026-10-16 columnar_reader.c: the unrecognized-descriptor refusal replaced by a backend crash +corruption corruption byte_length corruption errors 15;16;17;18;19 2026-10-16 columnar_reader.c: the row group span refusal replaced by a backend crash;columnar_reader.c: the row group's span checked against its chunks no longer +corruption corruption corrupt bloom equality still correct 15;16;17;18;19 never - +corruption corruption corrupt bloom: a value the table holds is still found 15;16;17;18;19 2026-10-16 columnar_bloom.c: a malformed bloom header read as absent rather than as no filter;columnar_bloom.c: the bloom length check removed +corruption corruption page_length corruption errors 15;16;17;18;19 2026-10-16 columnar_reader.c: the chunk-inside-its-group refusal replaced by a backend crash +corruption corruption premise: the bloom filter corruption changed a catalog row 15;16;17;18;19 never - +corruption corruption premise: the byte_length corruption changed a catalog row 15;16;17;18;19 never - +corruption corruption premise: the column_index corruption changed a catalog row 15;16;17;18;19 never - +corruption corruption premise: the encoding_descriptor corruption changed a catalog row 15;16;17;18;19 never - +corruption corruption premise: the page_length corruption changed a catalog row 15;16;17;18;19 never - +corruption corruption premise: the row_count corruption changed a catalog row 15;16;17;18;19 never - +corruption corruption premise: the value_count corruption changed a catalog row 15;16;17;18;19 never - +corruption corruption row_count corruption errors 15;16;17;18;19 2026-10-16 columnar_reader.c: the no-nulls row accounting and the value-stream bound both removed debug_hook_privilege debug_hook_privilege and that refusal is the owner gate too 15;16;17;18;19 2026-10-14 the debug set-metapage-version hook without its owner check debug_hook_privilege debug_hook_privilege and the refusal is the owner gate, not an EXECUTE denial 15;16;17;18;19 2026-10-14 pgcolumnar_debug_advance_reserved_offset without its owner check debug_hook_privilege debug_hook_privilege backend alive 15;16;17;18;19 never - @@ -2226,6 +2246,34 @@ native_agg native_agg sum(int4) 15;16;17;18;19 never - native_agg native_agg sum(int4,nulls) 15;16;17;18;19 never - native_agg native_agg sum/min/max uses the metadata agg node 15;16;17;18;19 never - native_agg native_agg the deletes removed rows at both ends, the same rows from each 15;16;17;18;19 never - +native_backend_crash native_backend_crash a second concurrent reader was also dropped by the crash 15;16;17;18;19 never - +native_backend_crash native_backend_crash crashA: kill lands mid-write (backend not yet in pg_sleep) 15;16;17;18;19 never - +native_backend_crash native_backend_crash crashA: the INSERT had written pages before the kill (the relation grew) 15;16;17;18;19 never - +native_backend_crash native_backend_crash crashB: kill lands after the stripes flushed (backend in pg_sleep) 15;16;17;18;19 never - +native_backend_crash native_backend_crash crashB: the INSERT had written pages before the kill (the relation grew) 15;16;17;18;19 never - +native_backend_crash native_backend_crash crashC: kill lands mid-write (backend not yet in pg_sleep) 15;16;17;18;19 never - +native_backend_crash native_backend_crash crashC: the INSERT had written pages before the kill (the relation grew) 15;16;17;18;19 never - +native_backend_crash native_backend_crash mid-write: cluster accepts new connections 15;16;17;18;19 never - +native_backend_crash native_backend_crash mid-write: committed content in c matches the heap mirror 15;16;17;18;19 2026-10-16 columnar_storage.c: a stripe data page written with no WAL record +native_backend_crash native_backend_crash mid-write: postmaster survived, same pid 15;16;17;18;19 never - +native_backend_crash native_backend_crash mid-write: the crash reinitialized the cluster (a live session was dropped) 15;16;17;18;19 never - +native_backend_crash native_backend_crash mid-write: the killed insert left no partial row group 15;16;17;18;19 never - +native_backend_crash native_backend_crash mid-write: the killed insert left no visible rows 15;16;17;18;19 never - +native_backend_crash native_backend_crash mid-write: the mirror itself is unchanged 15;16;17;18;19 never - +native_backend_crash native_backend_crash post-flush: cluster accepts new connections 15;16;17;18;19 never - +native_backend_crash native_backend_crash post-flush: committed content in c matches the heap mirror 15;16;17;18;19 2026-10-16 columnar_storage.c: a stripe data page written with no WAL record +native_backend_crash native_backend_crash post-flush: postmaster survived, same pid 15;16;17;18;19 never - +native_backend_crash native_backend_crash post-flush: the crash reinitialized the cluster (a live session was dropped) 15;16;17;18;19 never - +native_backend_crash native_backend_crash post-flush: the killed insert left no partial row group 15;16;17;18;19 never - +native_backend_crash native_backend_crash post-flush: the killed insert left no visible rows 15;16;17;18;19 never - +native_backend_crash native_backend_crash post-flush: the mirror itself is unchanged 15;16;17;18;19 never - +native_backend_crash native_backend_crash with-reader: cluster accepts new connections 15;16;17;18;19 never - +native_backend_crash native_backend_crash with-reader: committed content in c matches the heap mirror 15;16;17;18;19 2026-10-16 columnar_storage.c: a stripe data page written with no WAL record +native_backend_crash native_backend_crash with-reader: postmaster survived, same pid 15;16;17;18;19 never - +native_backend_crash native_backend_crash with-reader: the crash reinitialized the cluster (a live session was dropped) 15;16;17;18;19 never - +native_backend_crash native_backend_crash with-reader: the killed insert left no partial row group 15;16;17;18;19 never - +native_backend_crash native_backend_crash with-reader: the killed insert left no visible rows 15;16;17;18;19 never - +native_backend_crash native_backend_crash with-reader: the mirror itself is unchanged 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound a sequential scan of the same poisoned chunk is refused (XX001) 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound an index fetch of a chunk whose page_length is 2^32 too large is refused (XX001) 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound backend survived the sequential refusal 15;16;17;18;19 never - @@ -2352,6 +2400,9 @@ native_join_runtime_filter native_join_runtime_filter a (SELECT false) gate over native_join_runtime_filter native_join_runtime_filter a (SELECT false) gate over a heap dimension equals heap 15;16;17;18;19 2026-10-15 the join runtime filter took a join carrying a gating qual and failed on the Result over it (the tree before #1437) native_join_runtime_filter native_join_runtime_filter a (SELECT true) gate over a columnar dimension equals heap 15;16;17;18;19 2026-10-15 the join runtime filter took a join carrying a gating qual and failed on the Result over it (the tree before #1437) native_join_runtime_filter native_join_runtime_filter a (SELECT true) gate over a heap dimension equals heap 15;16;17;18;19 2026-10-15 the join runtime filter took a join carrying a gating qual and failed on the Result over it (the tree before #1437) +native_join_runtime_filter native_join_runtime_filter an empty build from a large dimension answers as heap 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter an empty build on one rescan does not empty the next 15;16;17;18;19 2026-10-16 the rescan kept the first execution's runtime filter state: prepared not reset on rescan (mutation, #1447) +native_join_runtime_filter native_join_runtime_filter an empty build reads no fact chunk group 15;16;17;18;19 2026-10-16 an inner join whose build was empty ran on, and the fact scan rejected every row one at a time (the tree before #1447) native_join_runtime_filter native_join_runtime_filter baseline core Hash Join 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter baseline reads all groups 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter build rows omit NULL 15;16;17;18;19 never - @@ -2377,8 +2428,12 @@ native_join_runtime_filter native_join_runtime_filter join runtime filter defaul native_join_runtime_filter native_join_runtime_filter plan has build tap 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter plan has runtime coordinator 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter plan retains core Hash Join 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: and its build is empty 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter premise: heap's answer under a true gate over a columnar dimension holds 200 rows 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter premise: heap's answer under a true gate over a heap dimension holds 200 rows 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: heap's empty-then-full rescan answer holds the 200 keys 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: the empty-then-full rescan runs under the coordinator 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: the large-dimension empty build runs under the coordinator 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter premise: with no gate, the columnar-dimension join has the coordinator 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter premise: with no gate, the heap-dimension join has the coordinator 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter projection join equals heap 15;16;17;18;19 never - @@ -3562,10 +3617,3 @@ vm_privilege vm_privilege vm entry point is covered by this boundary test: vm_se vm_privilege vm_privilege vm_is_visible refuses a heap relation even for its owner (#748) 15;16;17;18;19 never - vm_privilege vm_privilege vm_selftest did not report success to the unprivileged role 15;16;17;18;19 2026-10-14 vm_selftest without its owner check vm_privilege vm_privilege vm_selftest refuses a heap relation even for its owner (#748) 15;16;17;18;19 never - -native_join_runtime_filter native_join_runtime_filter premise: the large-dimension empty build runs under the coordinator 15;16;17;18;19 never - -native_join_runtime_filter native_join_runtime_filter premise: and its build is empty 15;16;17;18;19 never - -native_join_runtime_filter native_join_runtime_filter an empty build reads no fact chunk group 15;16;17;18;19 2026-10-16 an inner join whose build was empty ran on, and the fact scan rejected every row one at a time (the tree before #1447) -native_join_runtime_filter native_join_runtime_filter an empty build from a large dimension answers as heap 15;16;17;18;19 never - -native_join_runtime_filter native_join_runtime_filter premise: the empty-then-full rescan runs under the coordinator 15;16;17;18;19 never - -native_join_runtime_filter native_join_runtime_filter premise: heap's empty-then-full rescan answer holds the 200 keys 15;16;17;18;19 never - -native_join_runtime_filter native_join_runtime_filter an empty build on one rescan does not empty the next 15;16;17;18;19 2026-10-16 the rescan kept the first execution's runtime filter state: prepared not reset on rescan (mutation, #1447) diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index e4c69dd22..49a01d3b0 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 221 +suites_not_covered 219 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -2886,4 +2886,39 @@ suites_not_covered 221 # never 2639 + 5 = 2644 # dated 925 + 2 = 927 # 2644 + 927 == 3571 -checks_never_observed_red 2644 +# +# #1341, native_backend_crash seeded, with its twin (OffgridwithJD): 2644 -> 2669, +# and suites_not_covered 221 -> 220. The suite had no rows. It enters with all 28 +# checks from five-major runs (28/28 on each of 15-19). 3 are dated by a stripe data +# page written with no WAL record: each scenario's "committed content in c matches +# the heap mirror", which the reinit's crash recovery cannot rebuild. Twenty-five +# arrive never. The no-visible-rows and no-partial-row-group arms are MVCC's: no +# mutation of the extension leaves an aborted transaction's catalog rows visible. +# The sentinel, second-reader, kill-phase and relation-grew arms go red only under +# harness mutations (a signal that kills nothing, an INSERT whose stripes buffer +# whole), which observe the suite rather than the code. The rest hold of the heap +# mirror, the postmaster pid and a connection. +# +# ledger rows 3571 + 28 = 3599 +# never 2644 + 25 = 2669 +# dated 927 + 3 = 930 +# 2669 + 930 == 3599 +# +# #1341, corruption seeded, with its twin (OffgridwithJD): 2669 -> 2678, and +# suites_not_covered 220 -> 219. The suite had no rows. It enters with all 20 checks +# from five-major runs (20/20 on each of 15-19). 11 are dated by seven mutations of +# the reader and the bloom probe: the group-span check removed, the span, the +# chunk-bound and the descriptor refusals each replaced by a backend crash, the +# row-count accounting and the value-stream bound removed together, a malformed +# bloom header read as "absent", and the bloom length check removed. Each corruption +# has a second guard, so removing one refusal alone mostly leaves its arm green; a +# refusal turned into a crash is what the new XX001 and crash-count arms catch. +# Nine arrive never: the seven touched-a-row premises, which redden only when the +# harness's UPDATEs match nothing, column_index's liveness, which no single bound +# reddens, and the absent-value bloom lookup, which cannot see a false negative. +# +# ledger rows 3599 + 20 = 3619 +# never 2669 + 9 = 2678 +# dated 930 + 11 = 941 +# 2678 + 941 == 3619 +checks_never_observed_red 2678 diff --git a/test/corruption.sh b/test/corruption.sh index 99d8a1c06..f34c1614d 100755 --- a/test/corruption.sh +++ b/test/corruption.sh @@ -21,11 +21,39 @@ set -uo pipefail pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" -alive() { [ "$(q 'SELECT 1;')" = "1" ] && echo yes || echo no; } -errors() { if psql_run "$1" >/dev/null 2>&1; then echo no; else echo yes; fi; } +# A BACKEND CRASH PASSED THESE CHECKS (#1341). `errors` said yes to any failure of +# psql, and a crash is one. `alive` then connected after the postmaster's crash +# recovery, so it passed whenever recovery finished first: with the bad-descriptor +# refusal replaced by abort(), the suite was 12/12 once `alive` waited 3s. So a +# refusal must now be a clean XX001, which a crash never prints, and `alive` also +# requires that no backend has died since the scenario started, read from the +# server log rather than from timing. +crashes() { + # grep -c prints 0 itself when nothing matches, and then exits 1 + pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true +} +alive() { # since this scenario's mkc, which notes the count + [ "$(q 'SELECT 1;')" = "1" ] && [ "$(crashes)" = "$CRASHES_BEFORE" ] && echo yes \ + || echo "no ($(( $(crashes) - CRASHES_BEFORE )) backend crashes)" +} +refused() { # refused SQL -> the SQLSTATE of the ERROR it raised, or what it printed instead + local out + out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -qAtX -v VERBOSITY=verbose -c "$1" 2>&1)" + sed -nE 's/^ERROR: ([0-9A-Z]{5}): .*/\1/p' <<<"$out" | head -1 | grep . \ + || echo "no error ($(head -1 <<<"$out"))" +} sid() { q "SELECT pgcolumnar.get_storage_id('c');"; } +# corrupt FIELD "UPDATE ..." : run the corrupting UPDATE and assert it changed a +# catalog row. An UPDATE that matched nothing would leave the "must not crash" +# scenarios passing on a table nobody corrupted. +corrupt() { + check "premise: the $1 corruption changed a catalog row" \ + "$(q "WITH u AS ($2 RETURNING 1) SELECT count(*) > 0 FROM u;")" "t" +} mkc() { + CRASHES_BEFORE="$(crashes)" psql_run "DROP TABLE IF EXISTS c;" >/dev/null 2>&1 psql_run "CREATE TABLE c (id int, r bigint, t text) USING pgcolumnar;" # r random -> poorly compressible; id monotonic; t low-cardinality -> dictionary @@ -38,51 +66,57 @@ mkc() { # backend surviving. echo "-- row_group.byte_length corruption must error, not crash" mkc -psql_run "UPDATE pgcolumnar.row_group SET byte_length = byte_length + 123456 - WHERE storage_id = $(sid);" -check "byte_length corruption errors" "$(errors 'SELECT count(*), sum(r) FROM c;')" "yes" +corrupt "byte_length" "UPDATE pgcolumnar.row_group SET byte_length = byte_length + 123456 + WHERE storage_id = $(sid)" +check "byte_length corruption errors" "$(refused 'SELECT count(*), sum(r) FROM c;')" "XX001" check "alive after byte_length" "$(alive)" "yes" echo "-- row_group.row_count corruption must error, not crash" mkc -psql_run "UPDATE pgcolumnar.row_group SET row_count = row_count + 100000 - WHERE storage_id = $(sid);" -check "row_count corruption errors" "$(errors 'SELECT sum(r) FROM c;')" "yes" +corrupt "row_count" "UPDATE pgcolumnar.row_group SET row_count = row_count + 100000 + WHERE storage_id = $(sid)" +check "row_count corruption errors" "$(refused 'SELECT sum(r) FROM c;')" "XX001" check "alive after row_count" "$(alive)" "yes" echo "-- column_chunk.page_length corruption must error, not crash" mkc -psql_run "UPDATE pgcolumnar.column_chunk SET page_length = page_length + 123456 - WHERE storage_id = $(sid);" -check "page_length corruption errors" "$(errors 'SELECT sum(r) FROM c;')" "yes" +corrupt "page_length" "UPDATE pgcolumnar.column_chunk SET page_length = page_length + 123456 + WHERE storage_id = $(sid)" +check "page_length corruption errors" "$(refused 'SELECT sum(r) FROM c;')" "XX001" check "alive after page_length" "$(alive)" "yes" echo "-- column_chunk.value_count corruption must not crash (degrades safely)" mkc -psql_run "UPDATE pgcolumnar.column_chunk SET value_count = value_count + 100000 - WHERE storage_id = $(sid);" +corrupt "value_count" "UPDATE pgcolumnar.column_chunk SET value_count = value_count + 100000 + WHERE storage_id = $(sid)" q "SELECT sum(r) FROM c;" >/dev/null 2>&1 check "alive after value_count" "$(alive)" "yes" echo "-- corrupt encoding descriptor must error, not crash" mkc -psql_run "UPDATE pgcolumnar.column_chunk SET encoding_descriptor = '\\xffffffff'::bytea - WHERE storage_id = $(sid) AND column_index = 1;" -check "bad descriptor errors" "$(errors 'SELECT * FROM c;')" "yes" +corrupt "encoding_descriptor" "UPDATE pgcolumnar.column_chunk SET encoding_descriptor = '\\xffffffff'::bytea + WHERE storage_id = $(sid) AND column_index = 1" +check "bad descriptor errors" "$(refused 'SELECT * FROM c;')" "XX001" check "alive after bad descriptor" "$(alive)" "yes" echo "-- column_index = -1 (negative index guard) must not crash" mkc -psql_run "UPDATE pgcolumnar.column_chunk SET column_index = -1 - WHERE storage_id = $(sid) AND column_index = 1;" +corrupt "column_index" "UPDATE pgcolumnar.column_chunk SET column_index = -1 + WHERE storage_id = $(sid) AND column_index = 1" q "SELECT count(*) FROM c;" >/dev/null 2>&1 check "alive after column_index=-1" "$(alive)" "yes" echo "-- corrupt bloom header (huge nbits, short buffer) is treated as no filter" mkc -psql_run "UPDATE pgcolumnar.bloom SET filter = '\\x0000004006'::bytea - WHERE storage_id = $(sid);" +BLOOM_HIT="$(q 'SELECT r FROM c WHERE id = 7777;')" +corrupt "bloom filter" "UPDATE pgcolumnar.bloom SET filter = '\\x0000004006'::bytea + WHERE storage_id = $(sid)" check "corrupt bloom equality still correct" "$(q 'SELECT count(*) FROM c WHERE r = -999;')" "0" +# An absent value is 0 rows whether the corrupt filter is ignored or wrongly says +# "absent", so the arm above cannot see a false negative, which is the bloom's one +# way to lose rows (#1341). A value the table holds must still be found. +check "corrupt bloom: a value the table holds is still found" \ + "$(q "SELECT count(*) FROM c WHERE r = $BLOOM_HIT;")" "1" check "alive after bloom corruption" "$(alive)" "yes" pgc_summary diff --git a/test/native_backend_crash.sh b/test/native_backend_crash.sh index 9f46f430f..07769259d 100755 --- a/test/native_backend_crash.sh +++ b/test/native_backend_crash.sh @@ -103,8 +103,9 @@ open_sentinel() { # backend is parked in the trailing pg_sleep, between the last flush and the # commit that never comes. The phase is asserted, not assumed, from wait_event. crash_one_backend() { - local rows="$1" marker="$2" phase="$3" bg pid i we src + local rows="$1" marker="$2" phase="$3" bg pid i we src size0 open_sentinel || echo "-- sentinel did not connect" + size0="$(q "SELECT pg_relation_size('c');")" env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ -d "$PGC_DB" -c "BEGIN; INSERT INTO c /* $marker */ SELECT g, 'k'||g, g FROM generate_series(1, $rows) g; SELECT pg_sleep(60);" >/dev/null 2>&1 & @@ -131,8 +132,18 @@ crash_one_backend() { sleep 0.1 done else - sleep 0.5 + # Until the write has reached the relation. A fixed 0.5s stood here, and + # nothing showed the INSERT had written a page by then: with its stripes + # buffered whole (stripe_row_limit above the row count) the kill landed on + # a write that had produced nothing, and every mid-write arm still passed + # (#1341). So the kill now waits for the relation to grow, and asserts it. + for i in $(seq 1 600); do + [ "$(q "SELECT pg_relation_size('c') > $size0;")" = t ] && break + sleep 0.1 + done fi + check "$marker: the INSERT had written pages before the kill (the relation grew)" \ + "$(q "SELECT pg_relation_size('c') > $size0;")" "t" we="$(q "SELECT coalesce(wait_event,'') FROM pg_stat_activity WHERE pid = $pid;")" if [ "$phase" = sleep ]; then check "$marker: kill lands after the stripes flushed (backend in pg_sleep)" "$we" "PgSleep" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 0c8cd1f85..7048ce237 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -182,6 +182,8 @@ behaviour, the source of that number is named. - [134. test_unique_conc.py: concurrent inserters of one unique key serialize](#134-test_unique_concpy-concurrent-inserters-of-one-unique-key-serialize) - [135. test_update_conc.py: two writers of one row serialize](#135-test_update_concpy-two-writers-of-one-row-serialize) - [136. test_recovery.py: committed writes survive a crash, uncommitted ones leave nothing](#136-test_recoverypy-committed-writes-survive-a-crash-uncommitted-ones-leave-nothing) +- [137. test_native_backend_crash.py: one backend dies and committed data is intact](#137-test_native_backend_crashpy-one-backend-dies-and-committed-data-is-intact) +- [138. test_corruption.py: a corrupt catalog row is refused cleanly and no backend dies](#138-test_corruptionpy-a-corrupt-catalog-row-is-refused-cleanly-and-no-backend-dies) ## 1. How to read a test in here @@ -7296,3 +7298,35 @@ A private cluster, stopped with `-m immediate` for each crash. Each crash is pro | `test_the_pair_through_three_crashes` | rows committed after a checkpoint survive; an open transaction's checkpointed INSERT leaves no row; committed deletes survive; each against the heap pair | | `test_a_cleared_visibility_bit_stays_cleared` | a bit set and checkpointed, then cleared by a DELETE, stays cleared after the crash; a clean block's bit stays set; the rows match heap | | `test_a_projection_survives_a_crash` | a projection copy committed after the checkpoint replays, and reproduces the base's rows and a scan through it | + +## 137. test_native_backend_crash.py: one backend dies and committed data is intact + +The pytest twin of `test/native_backend_crash.sh` (#217, ported for #1341), every check by the shell's name. A SIGSEGV in one backend makes the postmaster drop every session and reinitialize, with crash recovery, while it stays up itself. Committed columnar rows must then match a heap mirror, and an INSERT killed before its commit must leave no row and no row group. + +The kill lands mid-write, once the INSERT has written pages to the relation and before it reaches its trailing `pg_sleep`, and after every stripe has flushed, while it sleeps; both are asserted. A sentinel session opened before each kill must be dropped by the reinit, which proves the crash happened. A private cluster, since the reinit drops every session on the server. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_a_kill_mid_write` | a 3,000,000-row INSERT killed after it has grown the relation: the cluster reinitialized, the postmaster kept its pid, and c holds exactly its committed rows and row groups | +| `test_a_kill_after_the_stripes_flushed` | the same for a 100,000-row INSERT killed in its trailing `pg_sleep` | +| `test_a_kill_with_a_second_reader` | the mid-write kill with a second parked reader, which the reinit drops too | + +## 138. test_corruption.py: a corrupt catalog row is refused cleanly and no backend dies + +The pytest twin of `test/corruption.sh` (ported for #1341), every check by the shell's name. The reader trusts the lengths, counts, offsets and descriptors the catalog stores. Each test corrupts one field on a fresh table and asserts that a read raises XX001, or, for a field the reader tolerates, that it does not crash. + +A crash is read from the server log, counted from the test's start, since a client error cannot tell a refusal from a crash and a liveness probe passes once crash recovery has finished. Each corrupting UPDATE must change a catalog row. The bloom test also looks up a value the table holds, since an absent value answers 0 even when a corrupt filter wrongly says "absent". A private cluster, since a crash drops every session. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_a_corrupt_byte_length` | `row_group.byte_length` inflated: the read raises XX001, and no backend died | +| `test_a_corrupt_row_count` | `row_group.row_count` inflated: XX001, no crash | +| `test_a_corrupt_page_length` | `column_chunk.page_length` inflated: XX001, no crash | +| `test_a_corrupt_value_count_degrades_safely` | `column_chunk.value_count` inflated: the read may answer or refuse, and no backend died | +| `test_a_corrupt_encoding_descriptor` | a descriptor of `\xffffffff`: XX001, no crash | +| `test_a_negative_column_index` | `column_index = -1`: no crash | +| `test_a_corrupt_bloom_header_is_no_filter` | a huge nbits over a short buffer: an absent value is 0 rows, a held value is still found, no crash | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 76ea8a89f..045205f9a 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -741,4 +741,8 @@ guard_tests 438 # tests, by collection over the cluster files on this tree. # cluster_tests 641 -> 643 for #1447's two tests in test_join_runtime_filter.py, by # collection over the cluster files on this tree. -cluster_tests 643 +# cluster_tests 643 -> 646 for #1341's test_native_backend_crash.py, the twin of +# native_backend_crash.sh: three tests, by collection over the cluster files on this tree. +# cluster_tests 646 -> 653 for #1341's test_corruption.py, the twin of corruption.sh: +# seven tests, by collection over the cluster files on this tree. +cluster_tests 653 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 15b5b7cc7..48ebbfdde 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -101,6 +101,7 @@ "catalog_scan_index", "collation_change", "column_extent", + "corruption", "debug_hook_privilege", "differential", "encode_post_codec", @@ -116,6 +117,7 @@ "latest_reading", "maintenance_crash", "maintenance_rollback", + "native_backend_crash", "native_chunk_length_bound", "native_delete_vector_index", "native_delete_visibility_paths", diff --git a/test/pytest/test_corruption.py b/test/pytest/test_corruption.py new file mode 100644 index 000000000..63cae400c --- /dev/null +++ b/test/pytest/test_corruption.py @@ -0,0 +1,179 @@ +"""A corrupt pgcolumnar catalog row is refused cleanly, or degrades safely, and no +backend dies (#1341). The pytest twin of `test/corruption.sh`, every check by the +shell's name. + +The reader trusts the lengths, counts, offsets and descriptors the catalog stores. +Each scenario corrupts one of those fields on a fresh table and asserts that a read +raises XX001 (data_corrupted), or for the fields the reader tolerates, that it does +not crash. A crash is read from the server log ("terminated by signal"), counted +from the scenario's start. A client error alone cannot tell a refusal from a crash, +and a liveness probe passes once crash recovery has finished. Each corrupting UPDATE +must change a catalog row, or its scenario would pass on a table nobody corrupted. + +A PRIVATE CLUSTER, because a crash would drop every session on the server. +""" +import pathlib +import shutil +import tempfile + +import psycopg +import pytest + +from pgc_vacuity import query_error + +NO_SUCH_STORAGE = False # True points every corruption at a storage id that does not exist + + +@pytest.fixture(scope="module") +def cor(pgc_cluster): + """-> the private cluster, started, with the extension created.""" + from pgc_cluster import Cluster, pick_port + + work = pathlib.Path(tempfile.mkdtemp(prefix="pgc-corrupt-")) + work.chmod(0o777) + datadir = work / "data" + datadir.mkdir() + cluster = Cluster(pgc_cluster.pg_config, "corrupt", datadir, pick_port(16)) + try: + cluster.initdb() + cluster.start() + with psycopg.connect(cluster.dsn(), autocommit=True) as c: + c.execute("CREATE EXTENSION IF NOT EXISTS pgcolumnar") + yield cluster + finally: + cluster.stop() + shutil.rmtree(work, ignore_errors=True) + + +def _one(cluster, sql): + """-> the first column of the first row, or a distinct failure sentinel.""" + try: + with psycopg.connect(cluster.dsn(), autocommit=True, connect_timeout=5) as c: + return c.execute(sql).fetchone()[0] + except psycopg.Error as exc: + return query_error(str(exc).splitlines()[0]) + + +def _crashes(cluster): + try: + return (cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal") + except OSError: + return 0 + + +class _Scenario: + """A fresh table c, the crash count at its start, and the probes the shell runs.""" + + def __init__(self, cluster): + self.cluster = cluster + self.crashes = _crashes(cluster) + with psycopg.connect(cluster.dsn(), autocommit=True) as c: + c.execute("DROP TABLE IF EXISTS c") + c.execute("CREATE TABLE c (id int, r bigint, t text) USING pgcolumnar") + # r random: poorly compressible; id monotonic; t low-cardinality: dictionary + c.execute("INSERT INTO c SELECT g, (random() * 9e18)::bigint, 'v' || (g % 5) " + "FROM generate_series(1, 20000) g") + self.sid = 0 if NO_SUCH_STORAGE else _one(cluster, "SELECT pgcolumnar.get_storage_id('c')") + + def corrupt(self, update): + """`update`, with {sid} filled. -> whether it changed a catalog row, as text.""" + return str(_one(self.cluster, f"WITH u AS ({update.format(sid=self.sid)} RETURNING 1) " + "SELECT count(*) > 0 FROM u")) + + def refused(self, sql): + """-> the SQLSTATE sql raised, or what happened instead. A crash ends the + connection with no SQLSTATE at all.""" + try: + with psycopg.connect(self.cluster.dsn(), autocommit=True) as c: + row = c.execute(sql).fetchone() + return f"no error ({row})" + except psycopg.Error as exc: + return exc.sqlstate or f"no error ({str(exc).splitlines()[0]})" + + def read(self, sql): + """Run sql for its effect on the backend; its answer is not the subject.""" + try: + with psycopg.connect(self.cluster.dsn(), autocommit=True) as c: + c.execute(sql).fetchall() + except psycopg.Error: + pass + + def alive(self): + crashed = _crashes(self.cluster) - self.crashes + return "yes" if _one(self.cluster, "SELECT 1") == 1 and crashed == 0 else f"no ({crashed} backend crashes)" + + +def test_a_corrupt_byte_length(cor, expect): + s = _Scenario(cor) + field = "byte_length" + expect.text(s.corrupt("UPDATE pgcolumnar.row_group SET byte_length = byte_length + 123456 " + "WHERE storage_id = {sid}"), "True", + f"premise: the {field} corruption changed a catalog row") + expect.text(s.refused("SELECT count(*), sum(r) FROM c"), "XX001", "byte_length corruption errors") + expect.text(s.alive(), "yes", "alive after byte_length") + + +def test_a_corrupt_row_count(cor, expect): + s = _Scenario(cor) + field = "row_count" + expect.text(s.corrupt("UPDATE pgcolumnar.row_group SET row_count = row_count + 100000 " + "WHERE storage_id = {sid}"), "True", + f"premise: the {field} corruption changed a catalog row") + expect.text(s.refused("SELECT sum(r) FROM c"), "XX001", "row_count corruption errors") + expect.text(s.alive(), "yes", "alive after row_count") + + +def test_a_corrupt_page_length(cor, expect): + s = _Scenario(cor) + field = "page_length" + expect.text(s.corrupt("UPDATE pgcolumnar.column_chunk SET page_length = page_length + 123456 " + "WHERE storage_id = {sid}"), "True", + f"premise: the {field} corruption changed a catalog row") + expect.text(s.refused("SELECT sum(r) FROM c"), "XX001", "page_length corruption errors") + expect.text(s.alive(), "yes", "alive after page_length") + + +def test_a_corrupt_value_count_degrades_safely(cor, expect): + s = _Scenario(cor) + field = "value_count" + expect.text(s.corrupt("UPDATE pgcolumnar.column_chunk SET value_count = value_count + 100000 " + "WHERE storage_id = {sid}"), "True", + f"premise: the {field} corruption changed a catalog row") + s.read("SELECT sum(r) FROM c") + expect.text(s.alive(), "yes", "alive after value_count") + + +def test_a_corrupt_encoding_descriptor(cor, expect): + s = _Scenario(cor) + field = "encoding_descriptor" + expect.text(s.corrupt("UPDATE pgcolumnar.column_chunk SET encoding_descriptor = '\\xffffffff'::bytea " + "WHERE storage_id = {sid} AND column_index = 1"), "True", + f"premise: the {field} corruption changed a catalog row") + expect.text(s.refused("SELECT * FROM c"), "XX001", "bad descriptor errors") + expect.text(s.alive(), "yes", "alive after bad descriptor") + + +def test_a_negative_column_index(cor, expect): + s = _Scenario(cor) + field = "column_index" + expect.text(s.corrupt("UPDATE pgcolumnar.column_chunk SET column_index = -1 " + "WHERE storage_id = {sid} AND column_index = 1"), "True", + f"premise: the {field} corruption changed a catalog row") + s.read("SELECT count(*) FROM c") + expect.text(s.alive(), "yes", "alive after column_index=-1") + + +def test_a_corrupt_bloom_header_is_no_filter(cor, expect): + """A huge nbits over a short buffer. Looking up an absent value answers 0 whether + the filter is ignored or wrongly says "absent", so a value the table holds is + looked up too: a false negative is the bloom's one way to lose rows.""" + s = _Scenario(cor) + hit = _one(cor, "SELECT r FROM c WHERE id = 7777") + field = "bloom filter" + expect.text(s.corrupt("UPDATE pgcolumnar.bloom SET filter = '\\x0000004006'::bytea " + "WHERE storage_id = {sid}"), "True", + f"premise: the {field} corruption changed a catalog row") + expect.num(_one(cor, "SELECT count(*) FROM c WHERE r = -999"), 0, "corrupt bloom equality still correct") + expect.num(_one(cor, f"SELECT count(*) FROM c WHERE r = {hit}"), 1, + "corrupt bloom: a value the table holds is still found") + expect.text(s.alive(), "yes", "alive after bloom corruption") diff --git a/test/pytest/test_native_backend_crash.py b/test/pytest/test_native_backend_crash.py new file mode 100644 index 000000000..1c7b8aed4 --- /dev/null +++ b/test/pytest/test_native_backend_crash.py @@ -0,0 +1,235 @@ +"""One backend dies, the postmaster lives, and committed columnar data is intact +(#217, #1341). The pytest twin of `test/native_backend_crash.sh`, every check by the +shell's name. + +A SIGSEGV in one backend makes the postmaster drop every session and reinitialize +shared memory, with crash recovery, while the postmaster itself stays up. Columnar +data goes through the buffer manager and WAL, so committed rows must match a heap +mirror afterwards, and an INSERT killed before its commit must leave no row and no +row group. The kill lands at two points: mid-write, once the INSERT has written +pages to the relation and before it reaches its trailing pg_sleep, and after every +stripe has flushed, while the backend sleeps. Both are asserted, not assumed. + +The write is uncommitted either way, so "no rows survived" cannot tell a crash from +a clean abort. A sentinel session, opened before each kill, must be dropped by the +reinit: that is what proves the crash happened. + +A PRIVATE CLUSTER, because the reinit drops every session on the server. +""" +import os +import pathlib +import shutil +import signal +import tempfile +import threading +import time + +import psycopg +import pytest + +from pgc_vacuity import query_error + +BIG_STRIPE = None # a stripe_row_limit for the inserting session; None keeps the default + + +@pytest.fixture(scope="module") +def nbc(pgc_cluster): + """-> the private cluster, started, with the extension and the h/c pair at 5000 rows.""" + from pgc_cluster import Cluster, pick_port + + work = pathlib.Path(tempfile.mkdtemp(prefix="pgc-nbcrash-")) + work.chmod(0o777) + datadir = work / "data" + datadir.mkdir() + cluster = Cluster(pgc_cluster.pg_config, "nbcrash", datadir, pick_port(14)) + try: + cluster.initdb() + cluster.start() + with psycopg.connect(cluster.dsn(), autocommit=True) as c: + c.execute("CREATE EXTENSION IF NOT EXISTS pgcolumnar") + # the postmaster must reinitialize after a backend crash, not shut down + c.execute("ALTER SYSTEM SET restart_after_crash = on") + c.execute("SELECT pg_reload_conf()") + c.execute("CREATE TABLE h (id int, v text, x float8)") + c.execute("CREATE TABLE c (id int, v text, x float8) USING pgcolumnar") + c.execute("INSERT INTO h SELECT g, 'b' || g, g * 2.5 FROM generate_series(1, 5000) g") + c.execute("INSERT INTO c SELECT g, 'b' || g, g * 2.5 FROM generate_series(1, 5000) g") + yield cluster + finally: + cluster.stop() + shutil.rmtree(work, ignore_errors=True) + + +def _one(cluster, sql): + """-> the first column of the first row, or a distinct failure sentinel.""" + try: + with psycopg.connect(cluster.dsn(), autocommit=True, connect_timeout=5) as c: + return c.execute(sql).fetchone()[0] + except psycopg.Error as exc: + return query_error(str(exc).splitlines()[0]) + + +def _rows(cluster, sql): + try: + with psycopg.connect(cluster.dsn(), autocommit=True, connect_timeout=5) as c: + return c.execute(sql).fetchall() + except psycopg.Error as exc: + return query_error(str(exc).splitlines()[0]) + + +def _row_groups(cluster): + return _one(cluster, "SELECT count(*) FROM pgcolumnar.row_group r JOIN pgcolumnar.storage s " + "ON s.storage_id = r.storage_id WHERE s.relation_oid = 'c'::regclass") + + +def _postmaster_pid(cluster): + return (cluster.datadir / "postmaster.pid").read_text().splitlines()[0] + + +class _Bystander: + """A session parked in pg_sleep on its own thread. -> .dropped once the server + ended it, which a plain exit of another backend never does.""" + + def __init__(self, cluster, seconds): + self.conn = psycopg.connect(cluster.dsn(), autocommit=True) + self.pid = self.conn.info.backend_pid + self.error = None + self.thread = threading.Thread(target=self._run, args=(seconds,), daemon=True) + self.thread.start() + + def _run(self, seconds): + try: + self.conn.execute(f"SELECT pg_sleep({seconds})") + except psycopg.Error as exc: + self.error = exc + + def dropped(self, within): + self.thread.join(within) + return not self.thread.is_alive() and self.error is not None + + def close(self, cluster): + _end(cluster, self.conn, self.thread, self.pid) + + +def _end(cluster, conn, thread, pid): + """Close a session whose statement may still run on its thread. A live one is + terminated first: closing a connection under a running statement crashes the + driver, and a session a failed check left behind holds its locks into the next + test.""" + if thread.is_alive(): + _one(cluster, f"SELECT pg_terminate_backend({pid})") + thread.join(10) + try: + conn.close() + except psycopg.Error: + pass + + +def _wait_up(cluster): + """Twelve successful connections in a row, 0.25s apart: a 3s window a reinit's + brief accepting blip cannot fake. -> True once it holds.""" + ok = 0 + for _ in range(1200): + if _one(cluster, "SELECT 1") == 1: + ok += 1 + if ok >= 12: + return True + else: + ok = 0 + time.sleep(0.25) + return False + + +def _crash_one_backend(cluster, expect, rows, marker, phase): + """An INSERT into c, held open by a trailing pg_sleep so it never commits, SIGSEGVed + in `phase` ('insert': mid-write, 'sleep': after every stripe flushed). + -> whether the sentinel was dropped by the reinit.""" + sentinel = _Bystander(cluster, 120) + size0 = _one(cluster, "SELECT pg_relation_size('c')") + writer = psycopg.connect(cluster.dsn(), autocommit=True) + pid = writer.info.backend_pid + + def write(): + try: + if BIG_STRIPE: + writer.execute(f"SET pgcolumnar.stripe_row_limit = {BIG_STRIPE}") + writer.execute("BEGIN") + writer.execute(f"INSERT INTO c /* {marker} */ SELECT g, 'k' || g, g FROM generate_series(1, {rows}) g") + writer.execute("SELECT pg_sleep(60)") + except psycopg.Error: + pass + thread = threading.Thread(target=write, daemon=True) + thread.start() + try: + return _kill_when_ready(cluster, expect, marker, phase, pid, size0, thread, sentinel) + finally: + _end(cluster, writer, thread, pid) + sentinel.close(cluster) + + +def _kill_when_ready(cluster, expect, marker, phase, pid, size0, thread, sentinel): + def wait_event(): + return _one(cluster, f"SELECT coalesce(wait_event, '') FROM pg_stat_activity WHERE pid = {pid}") + + for _ in range(600): + if phase == "sleep": + if wait_event() == "PgSleep": + break + elif _one(cluster, f"SELECT pg_relation_size('c') > {size0}") is True: + break + time.sleep(0.1) + expect.text(str(_one(cluster, f"SELECT pg_relation_size('c') > {size0}")), "True", + f"{marker}: the INSERT had written pages before the kill (the relation grew)") + we = wait_event() + if phase == "sleep": + expect.text(we, "PgSleep", f"{marker}: kill lands after the stripes flushed (backend in pg_sleep)") + else: + expect.text("yes" if we != "PgSleep" else "no", "yes", + f"{marker}: kill lands mid-write (backend not yet in pg_sleep)") + + os.kill(pid, signal.SIGSEGV) + thread.join(30) + _wait_up(cluster) + return sentinel.dropped(5) + + +def _assert_intact(cluster, expect, label, dropped, base): + expect.text("yes" if dropped else "no", "yes", + f"{label}: the crash reinitialized the cluster (a live session was dropped)") + expect.text(_postmaster_pid(cluster), base["pm"], f"{label}: postmaster survived, same pid") + expect.num(_one(cluster, "SELECT 1"), 1, f"{label}: cluster accepts new connections") + expect.row_set(_rows(cluster, "SELECT id, v, x FROM c"), base["content"], + f"{label}: committed content in c matches the heap mirror") + expect.row_set(_rows(cluster, "SELECT id, v, x FROM h"), base["mirror"], + f"{label}: the mirror itself is unchanged") + expect.num(_one(cluster, "SELECT count(*) FROM c"), 5000, f"{label}: the killed insert left no visible rows") + expect.num(_row_groups(cluster), base["groups"], f"{label}: the killed insert left no partial row group") + + +@pytest.fixture(scope="module") +def base(nbc): + """The committed baseline, read once before any crash.""" + return {"groups": _row_groups(nbc), "content": _rows(nbc, "SELECT id, v, x FROM c"), + "mirror": _rows(nbc, "SELECT id, v, x FROM h"), "pm": _postmaster_pid(nbc)} + + +def test_a_kill_mid_write(nbc, base, expect): + dropped = _crash_one_backend(nbc, expect, 3000000, "crashA", "insert") + _assert_intact(nbc, expect, "mid-write", dropped, base) + + +def test_a_kill_after_the_stripes_flushed(nbc, base, expect): + dropped = _crash_one_backend(nbc, expect, 100000, "crashB", "sleep") + _assert_intact(nbc, expect, "post-flush", dropped, base) + + +def test_a_kill_with_a_second_reader(nbc, base, expect): + """The reinit drops every session, so a second parked reader goes too.""" + reader = _Bystander(nbc, 90) + try: + dropped = _crash_one_backend(nbc, expect, 3000000, "crashC", "insert") + expect.text("yes" if reader.dropped(5) else "no", "yes", + "a second concurrent reader was also dropped by the crash") + finally: + reader.close(nbc) + _assert_intact(nbc, expect, "with-reader", dropped, base)