From 3b59ff38d11f805318ab24ae0358e08e9c8722ec Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 19 Oct 2026 19:14:54 +0000 Subject: [PATCH] test: four more suites' crash counts refuse when the log can't be read (#1341) corruption, inheritance, native_dict_underfill and parquet_import read "no backend died" from the server log the way #1535 found hardening and native_varlena_bound doing, and failed open the same way: the shell's crashes() ended `|| true`, so an unreadable log printed nothing both times ("" = "", and $(( - )) is 0), and the twins' _crashes returned 0 on OSError. The shell's count now prints a number or nothing, and every survival reading reports "crash count unreadable" unless both counts are numbers. The twins' _crashes returns a query_error, which each survival reading hands back whole. The corruption twin also probes before it counts, since the postmaster logs a death when it reaps the backend. Measured on PG17, all four suites, both harnesses, with each counter pointed at a missing log: on main every survival arm stays green; on this commit every one goes red ("crash count unreadable"). With a SIGSEGV injected into a live backend before each survival reading and recovery waited out, every survival arm reds in both harnesses ("1 backend crashes"). Both are harness mutations and date nothing. Clean on each of 15-19 in both harnesses; no check, ledger row or count changes. Co-Authored-By: Claude Opus 5.5 --- test/corruption.sh | 19 +++++++++++++++---- test/inheritance.sh | 19 +++++++++++++++---- test/native_dict_underfill.sh | 20 ++++++++++++++++---- test/parquet_import.sh | 18 +++++++++++++++--- test/pytest/test_corruption.py | 16 ++++++++++++---- test/pytest/test_inheritance.py | 22 +++++++++++++++++++--- test/pytest/test_native_dict_underfill.py | 21 ++++++++++++++++++--- test/pytest/test_parquet_import.py | 21 ++++++++++++++++++--- 8 files changed, 128 insertions(+), 28 deletions(-) diff --git a/test/corruption.sh b/test/corruption.sh index f34c1614d..a777f9afb 100755 --- a/test/corruption.sh +++ b/test/corruption.sh @@ -29,12 +29,23 @@ pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" # requires that no backend has died since the scenario started, read from the # server log rather than from timing. crashes() { - # grep -c prints 0 itself when nothing matches, and then exits 1 - pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true + # a number, or nothing when the log can't be read: grep -c prints 0 itself when + # nothing matches (and exits 1), but nothing at all for a file it cannot open, + # and a blind count must not read as "no crashes" + local _n + _n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)" + case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac } alive() { # since this scenario's mkc, which notes the count - [ "$(q 'SELECT 1;')" = "1" ] && [ "$(crashes)" = "$CRASHES_BEFORE" ] && echo yes \ - || echo "no ($(( $(crashes) - CRASHES_BEFORE )) backend crashes)" + local _up _now + _up="$(q 'SELECT 1;')"; _now="$(crashes)" + if ! [[ "$CRASHES_BEFORE" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then + echo "no (crash count unreadable)" + elif [ "$_up" = "1" ] && [ "$_now" = "$CRASHES_BEFORE" ]; then + echo yes + else + echo "no ($(( _now - CRASHES_BEFORE )) backend crashes)" + fi } refused() { # refused SQL -> the SQLSTATE of the ERROR it raised, or what it printed instead local out diff --git a/test/inheritance.sh b/test/inheritance.sh index a9998f2d2..d1dc10445 100755 --- a/test/inheritance.sh +++ b/test/inheritance.sh @@ -153,13 +153,24 @@ check "premise: while its leaf is an ordinary relation" \ # fails with 58P01 "could not open file .../0/0", which is the bug, and the arms # counted it as refused. inheritance stayed 26/26. CI runs release builds. inh_crashes() { - # grep -c prints 0 itself when nothing matches, and then exits 1 - pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true + # a number, or nothing when the log can't be read: grep -c prints 0 itself when + # nothing matches (and exits 1), but nothing at all for a file it cannot open, + # and a blind count must not read as "no crashes" + local _n + _n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)" + case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac } INH_CRASHES="$(inh_crashes)" inh_up() { - [ "$(q 'SELECT 1;')" = 1 ] && [ "$(inh_crashes)" = "$INH_CRASHES" ] && echo up \ - || echo "DOWN ($(( $(inh_crashes) - INH_CRASHES )) backend crashes)" + local _up _now + _up="$(q 'SELECT 1;')"; _now="$(inh_crashes)" + if ! [[ "$INH_CRASHES" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then + echo "DOWN (crash count unreadable)" + elif [ "$_up" = 1 ] && [ "$_now" = "$INH_CRASHES" ]; then + echo up + else + echo "DOWN ($(( _now - INH_CRASHES )) backend crashes)" + fi } inh_sqlstate() { # inh_sqlstate SQL -> the SQLSTATE of the ERROR it raised, or "accepted" local out diff --git a/test/native_dict_underfill.sh b/test/native_dict_underfill.sh index b5cf5bf8e..48ccfdb6d 100755 --- a/test/native_dict_underfill.sh +++ b/test/native_dict_underfill.sh @@ -28,8 +28,20 @@ check "the text column is DICT-encoded (precondition)" "$ENC" "6" # read old rawLen (uint32 LE at descriptor offset 11), inflate by 64, write back crashes() { - # grep -c prints 0 itself when nothing matches, and then exits 1 - pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true + # a number, or nothing when the log can't be read: grep -c prints 0 itself when + # nothing matches (and exits 1), but nothing at all for a file it cannot open, + # and a blind count must not read as "no crashes" + local _n + _n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)" + case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac +} +crashes_since() { # crashes_since BEFORE -> "N backend crashes", or why it can't count + local _now; _now="$(crashes)" + if [[ "$1" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then + echo "$(( _now - $1 )) backend crashes" + else + echo "crash count unreadable" + fi } crashes0="$(crashes)" q "WITH u AS (UPDATE pgcolumnar.column_chunk c SET encoding_descriptor = @@ -50,8 +62,8 @@ echo "-- select stderr:"; sed 's/^/ /' "$PGC_WORKDIR/uf.err" | head -3 # crashed backend passes once crash recovery has finished. The crash flag below was # computed and never checked. rejected="$(sed -nE 's/^ERROR: ([0-9A-Z]{5}): .*/\1/p' "$PGC_WORKDIR/uf.err" | head -1 | grep . || echo "no SQLSTATE ($(head -1 "$PGC_WORKDIR/uf.err"))")" -echo "-- rejected=$rejected crashes=$(( $(crashes) - crashes0 ))" +echo "-- rejected=$rejected crashes=$(crashes_since "$crashes0")" check "under-filled DICT chunk is rejected with a clean error" "$rejected" "XX001" check "backend survives the under-filled DICT chunk" \ - "$(q 'SELECT 1')|$(( $(crashes) - crashes0 )) backend crashes" "1|0 backend crashes" + "$(q 'SELECT 1')|$(crashes_since "$crashes0")" "1|0 backend crashes" pgc_summary diff --git a/test/parquet_import.sh b/test/parquet_import.sh index 11e4c6716..a6c4182f2 100755 --- a/test/parquet_import.sh +++ b/test/parquet_import.sh @@ -287,8 +287,20 @@ else # and only because it reached the server before crash recovery finished. A # check that connects after recovery reads a live cluster. pq_crashes() { - # grep -c prints 0 itself when nothing matches, and then exits 1 - pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true + # a number, or nothing when the log can't be read: grep -c prints 0 itself when + # nothing matches (and exits 1), but nothing at all for a file it cannot open, + # and a blind count must not read as "no crashes" + local _n + _n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)" + case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac + } + pq_crashes_since() { # pq_crashes_since BEFORE -> "N backend crashes", or why it can't count + local _now; _now="$(pq_crashes)" + if [[ "$1" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then + echo "$(( _now - $1 )) backend crashes" + else + echo "crash count unreadable" + fi } pq_crashes0="$(pq_crashes)" pq_out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ @@ -302,7 +314,7 @@ else # correct red against the unfixed build and would have stayed red against the # fix -- a discriminator that discriminates nothing. check "and the cluster is still up, so the refusal was not an abort" \ - "$(q "SELECT 1;")|$(( $(pq_crashes) - pq_crashes0 )) backend crashes" "1|0 backend crashes" + "$(q "SELECT 1;")|$(pq_crashes_since "$pq_crashes0")" "1|0 backend crashes" fi pgc_summary diff --git a/test/pytest/test_corruption.py b/test/pytest/test_corruption.py index 63cae400c..2ebcd2718 100644 --- a/test/pytest/test_corruption.py +++ b/test/pytest/test_corruption.py @@ -55,10 +55,12 @@ def _one(cluster, sql): def _crashes(cluster): + """-> how many backends the server log says died, or a failed reading when the + log can't be read: a blind count must not read as "no crashes".""" try: return (cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal") - except OSError: - return 0 + except OSError as exc: + return query_error(f"crash count unreadable: {exc}") class _Scenario: @@ -99,8 +101,14 @@ def read(self, sql): pass def alive(self): - crashed = _crashes(self.cluster) - self.crashes - return "yes" if _one(self.cluster, "SELECT 1") == 1 and crashed == 0 else f"no ({crashed} backend crashes)" + # probe first: the postmaster logs a backend's death when it reaps it, so a + # count read before the server answers again can miss the crash it is for + up = _one(self.cluster, "SELECT 1") == 1 + now = _crashes(self.cluster) + if not (isinstance(self.crashes, int) and isinstance(now, int)): + return now if isinstance(self.crashes, int) else self.crashes + crashed = now - self.crashes + return "yes" if up and crashed == 0 else f"no ({crashed} backend crashes)" def test_a_corrupt_byte_length(cor, expect): diff --git a/test/pytest/test_inheritance.py b/test/pytest/test_inheritance.py index 2f2379f3b..bc07d4225 100644 --- a/test/pytest/test_inheritance.py +++ b/test/pytest/test_inheritance.py @@ -13,12 +13,26 @@ """ import psycopg +from pgc_vacuity import query_error + def _crashes(pgc_cluster): + """-> how many backends the server log says died, or a failed reading when the + log can't be read: a blind count must not read as "no crashes".""" try: return (pgc_cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal") - except OSError: - return 0 + except OSError as exc: + return query_error(f"crash count unreadable: {exc}") + + +def _since(pgc_cluster, before): + """-> backends died since `before`, or the failed reading when either count is + one: a blind count must not read as "no crashes".""" + now = _crashes(pgc_cluster) + for v in (before, now): + if not isinstance(v, int): + return v + return now - before def _sqlstate(conn, sql): @@ -65,7 +79,9 @@ def test_a_partitioned_parent_is_refused_and_its_leaf_is_not(pgc_conn, pgc_clust def up(): alive = _sqlstate(conn, "SELECT 1") == "accepted" - crashed = _crashes(pgc_cluster) - crashes + crashed = _since(pgc_cluster, crashes) + if not isinstance(crashed, int): + return crashed return "up" if alive and crashed == 0 else f"DOWN ({crashed} backend crashes)" expect.text(_sqlstate(conn, "SELECT pgcolumnar.get_storage_id('inh_pp'::regclass)"), "42809", diff --git a/test/pytest/test_native_dict_underfill.py b/test/pytest/test_native_dict_underfill.py index 98c3f2c4f..36cff1270 100644 --- a/test/pytest/test_native_dict_underfill.py +++ b/test/pytest/test_native_dict_underfill.py @@ -12,12 +12,26 @@ """ import psycopg +from pgc_vacuity import query_error + def _crashes(pgc_cluster): + """-> how many backends the server log says died, or a failed reading when the + log can't be read: a blind count must not read as "no crashes".""" try: return (pgc_cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal") - except OSError: - return 0 + except OSError as exc: + return query_error(f"crash count unreadable: {exc}") + + +def _since(pgc_cluster, before): + """-> backends died since `before`, or the failed reading when either count is + one: a blind count must not read as "no crashes".""" + now = _crashes(pgc_cluster) + for v in (before, now): + if not isinstance(v, int): + return v + return now - before def _one(conn, sql): @@ -66,5 +80,6 @@ def test_an_under_filled_dict_chunk_is_refused(pgc_conn, pgc_cluster, expect): alive = c.execute("SELECT 1").fetchone()[0] == 1 except psycopg.Error: alive = False - expect.text(f"{'1' if alive else ''}|{_crashes(pgc_cluster) - crashes} backend crashes", "1|0 backend crashes", + died = _since(pgc_cluster, crashes) + expect.text(f"{'1' if alive else ''}|{died} backend crashes" if isinstance(died, int) else died, "1|0 backend crashes", "backend survives the under-filled DICT chunk") diff --git a/test/pytest/test_parquet_import.py b/test/pytest/test_parquet_import.py index 4e3787916..a78867840 100644 --- a/test/pytest/test_parquet_import.py +++ b/test/pytest/test_parquet_import.py @@ -11,12 +11,26 @@ """ import psycopg +from pgc_vacuity import query_error + def _crashes(pgc_cluster): + """-> how many backends the server log says died, or a failed reading when the + log can't be read: a blind count must not read as "no crashes".""" try: return (pgc_cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal") - except OSError: - return 0 + except OSError as exc: + return query_error(f"crash count unreadable: {exc}") + + +def _since(pgc_cluster, before): + """-> backends died since `before`, or the failed reading when either count is + one: a blind count must not read as "no crashes".""" + now = _crashes(pgc_cluster) + for v in (before, now): + if not isinstance(v, int): + return v + return now - before def _one(conn, sql, params=None): @@ -71,5 +85,6 @@ def test_a_partitioned_parent_is_refused(pgc_conn, pgc_cluster, server_dir, expe expect.text(_sqlstate(conn, "SELECT pgcolumnar.import_parquet('pq_parent', %s)", (path,)), "42809", "importing into the partitioned parent is refused") alive = _sqlstate(conn, "SELECT 1") == "accepted" - expect.text(f"{'1' if alive else ''}|{_crashes(pgc_cluster) - crashes} backend crashes", "1|0 backend crashes", + died = _since(pgc_cluster, crashes) + expect.text(f"{'1' if alive else ''}|{died} backend crashes" if isinstance(died, int) else died, "1|0 backend crashes", "and the cluster is still up, so the refusal was not an abort")