test: four more suites' crash counts refuse when the log can't be read (#1341)

test/corruption.sh

@@ -29,12 +29,23 @@ pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 # requires that no backend has died since the scenario started, read from the
 # server log rather than from timing.
 crashes() {
-	# grep -c prints 0 itself when nothing matches, and then exits 1
-	pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true
+	# a number, or nothing when the log can't be read: grep -c prints 0 itself when
+	# nothing matches (and exits 1), but nothing at all for a file it cannot open,
+	# and a blind count must not read as "no crashes"
+	local _n
+	_n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)"
+	case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac
 }
 alive() {	# since this scenario's mkc, which notes the count
-	[ "$(q 'SELECT 1;')" = "1" ] && [ "$(crashes)" = "$CRASHES_BEFORE" ] && echo yes \
-		|| echo "no ($(( $(crashes) - CRASHES_BEFORE )) backend crashes)"
+	local _up _now
+	_up="$(q 'SELECT 1;')"; _now="$(crashes)"
+	if ! [[ "$CRASHES_BEFORE" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then
+		echo "no (crash count unreadable)"
+	elif [ "$_up" = "1" ] && [ "$_now" = "$CRASHES_BEFORE" ]; then
+		echo yes
+	else
+		echo "no ($(( _now - CRASHES_BEFORE )) backend crashes)"
+	fi
 }
 refused() {	# refused SQL -> the SQLSTATE of the ERROR it raised, or what it printed instead
 	local out

test/inheritance.sh

@@ -153,13 +153,24 @@ check "premise: while its leaf is an ordinary relation" \
 # fails with 58P01 "could not open file .../0/0", which is the bug, and the arms
 # counted it as refused. inheritance stayed 26/26. CI runs release builds.
 inh_crashes() {
-	# grep -c prints 0 itself when nothing matches, and then exits 1
-	pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true
+	# a number, or nothing when the log can't be read: grep -c prints 0 itself when
+	# nothing matches (and exits 1), but nothing at all for a file it cannot open,
+	# and a blind count must not read as "no crashes"
+	local _n
+	_n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)"
+	case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac
 }
 INH_CRASHES="$(inh_crashes)"
 inh_up() {
-	[ "$(q 'SELECT 1;')" = 1 ] && [ "$(inh_crashes)" = "$INH_CRASHES" ] && echo up \
-		|| echo "DOWN ($(( $(inh_crashes) - INH_CRASHES )) backend crashes)"
+	local _up _now
+	_up="$(q 'SELECT 1;')"; _now="$(inh_crashes)"
+	if ! [[ "$INH_CRASHES" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then
+		echo "DOWN (crash count unreadable)"
+	elif [ "$_up" = 1 ] && [ "$_now" = "$INH_CRASHES" ]; then
+		echo up
+	else
+		echo "DOWN ($(( _now - INH_CRASHES )) backend crashes)"
+	fi
 }
 inh_sqlstate() {	# inh_sqlstate SQL -> the SQLSTATE of the ERROR it raised, or "accepted"
 	local out

test/native_dict_underfill.sh

@@ -28,8 +28,20 @@ check "the text column is DICT-encoded (precondition)" "$ENC" "6"
 
 # read old rawLen (uint32 LE at descriptor offset 11), inflate by 64, write back
 crashes() {
-	# grep -c prints 0 itself when nothing matches, and then exits 1
-	pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true
+	# a number, or nothing when the log can't be read: grep -c prints 0 itself when
+	# nothing matches (and exits 1), but nothing at all for a file it cannot open,
+	# and a blind count must not read as "no crashes"
+	local _n
+	_n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)"
+	case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac
+}
+crashes_since() {	# crashes_since BEFORE -> "N backend crashes", or why it can't count
+	local _now; _now="$(crashes)"
+	if [[ "$1" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then
+		echo "$(( _now - $1 )) backend crashes"
+	else
+		echo "crash count unreadable"
+	fi
 }
 crashes0="$(crashes)"
 q "WITH u AS (UPDATE pgcolumnar.column_chunk c SET encoding_descriptor =
@@ -50,8 +62,8 @@ echo "-- select stderr:"; sed 's/^/     /' "$PGC_WORKDIR/uf.err" | head -3
 # crashed backend passes once crash recovery has finished. The crash flag below was
 # computed and never checked.
 rejected="$(sed -nE 's/^ERROR:  ([0-9A-Z]{5}): .*/\1/p' "$PGC_WORKDIR/uf.err" | head -1 | grep . || echo "no SQLSTATE ($(head -1 "$PGC_WORKDIR/uf.err"))")"
-echo "-- rejected=$rejected crashes=$(( $(crashes) - crashes0 ))"
+echo "-- rejected=$rejected crashes=$(crashes_since "$crashes0")"
 check "under-filled DICT chunk is rejected with a clean error" "$rejected" "XX001"
 check "backend survives the under-filled DICT chunk" \
-	"$(q 'SELECT 1')|$(( $(crashes) - crashes0 )) backend crashes" "1|0 backend crashes"
+	"$(q 'SELECT 1')|$(crashes_since "$crashes0")" "1|0 backend crashes"
 pgc_summary

test/parquet_import.sh

@@ -287,8 +287,20 @@ else
 	# and only because it reached the server before crash recovery finished. A
 	# check that connects after recovery reads a live cluster.
 	pq_crashes() {
-		# grep -c prints 0 itself when nothing matches, and then exits 1
-		pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true
+		# a number, or nothing when the log can't be read: grep -c prints 0 itself when
+		# nothing matches (and exits 1), but nothing at all for a file it cannot open,
+		# and a blind count must not read as "no crashes"
+		local _n
+		_n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)"
+		case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac
+	}
+	pq_crashes_since() {	# pq_crashes_since BEFORE -> "N backend crashes", or why it can't count
+		local _now; _now="$(pq_crashes)"
+		if [[ "$1" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then
+			echo "$(( _now - $1 )) backend crashes"
+		else
+			echo "crash count unreadable"
+		fi
 	}
 	pq_crashes0="$(pq_crashes)"
 	pq_out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \
@@ -302,7 +314,7 @@ else
 	# correct red against the unfixed build and would have stayed red against the
 	# fix -- a discriminator that discriminates nothing.
 	check "and the cluster is still up, so the refusal was not an abort" \
-		"$(q "SELECT 1;")|$(( $(pq_crashes) - pq_crashes0 )) backend crashes" "1|0 backend crashes"
+		"$(q "SELECT 1;")|$(pq_crashes_since "$pq_crashes0")" "1|0 backend crashes"
 fi
 
 pgc_summary

test/pytest/test_corruption.py

@@ -55,10 +55,12 @@ def _one(cluster, sql):
 
 
 def _crashes(cluster):
+    """-> how many backends the server log says died, or a failed reading when the
+    log can't be read: a blind count must not read as "no crashes"."""
     try:
         return (cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal")
-    except OSError:
-        return 0
+    except OSError as exc:
+        return query_error(f"crash count unreadable: {exc}")
 
 
 class _Scenario:
@@ -99,8 +101,14 @@ def read(self, sql):
             pass
 
     def alive(self):
-        crashed = _crashes(self.cluster) - self.crashes
-        return "yes" if _one(self.cluster, "SELECT 1") == 1 and crashed == 0 else f"no ({crashed} backend crashes)"
+        # probe first: the postmaster logs a backend's death when it reaps it, so a
+        # count read before the server answers again can miss the crash it is for
+        up = _one(self.cluster, "SELECT 1") == 1
+        now = _crashes(self.cluster)
+        if not (isinstance(self.crashes, int) and isinstance(now, int)):
+            return now if isinstance(self.crashes, int) else self.crashes
+        crashed = now - self.crashes
+        return "yes" if up and crashed == 0 else f"no ({crashed} backend crashes)"
 
 
 def test_a_corrupt_byte_length(cor, expect):

test/pytest/test_inheritance.py

@@ -13,12 +13,26 @@
 """
 import psycopg
 
+from pgc_vacuity import query_error
+
 
 def _crashes(pgc_cluster):
+    """-> how many backends the server log says died, or a failed reading when the
+    log can't be read: a blind count must not read as "no crashes"."""
     try:
         return (pgc_cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal")
-    except OSError:
-        return 0
+    except OSError as exc:
+        return query_error(f"crash count unreadable: {exc}")
+
+
+def _since(pgc_cluster, before):
+    """-> backends died since `before`, or the failed reading when either count is
+    one: a blind count must not read as "no crashes"."""
+    now = _crashes(pgc_cluster)
+    for v in (before, now):
+        if not isinstance(v, int):
+            return v
+    return now - before
 
 
 def _sqlstate(conn, sql):
@@ -65,7 +79,9 @@ def test_a_partitioned_parent_is_refused_and_its_leaf_is_not(pgc_conn, pgc_clust
 
     def up():
         alive = _sqlstate(conn, "SELECT 1") == "accepted"
-        crashed = _crashes(pgc_cluster) - crashes
+        crashed = _since(pgc_cluster, crashes)
+        if not isinstance(crashed, int):
+            return crashed
         return "up" if alive and crashed == 0 else f"DOWN ({crashed} backend crashes)"
 
     expect.text(_sqlstate(conn, "SELECT pgcolumnar.get_storage_id('inh_pp'::regclass)"), "42809",

test/pytest/test_native_dict_underfill.py

@@ -12,12 +12,26 @@
 """
 import psycopg
 
+from pgc_vacuity import query_error
+
 
 def _crashes(pgc_cluster):
+    """-> how many backends the server log says died, or a failed reading when the
+    log can't be read: a blind count must not read as "no crashes"."""
     try:
         return (pgc_cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal")
-    except OSError:
-        return 0
+    except OSError as exc:
+        return query_error(f"crash count unreadable: {exc}")
+
+
+def _since(pgc_cluster, before):
+    """-> backends died since `before`, or the failed reading when either count is
+    one: a blind count must not read as "no crashes"."""
+    now = _crashes(pgc_cluster)
+    for v in (before, now):
+        if not isinstance(v, int):
+            return v
+    return now - before
 
 
 def _one(conn, sql):
@@ -66,5 +80,6 @@ def test_an_under_filled_dict_chunk_is_refused(pgc_conn, pgc_cluster, expect):
             alive = c.execute("SELECT 1").fetchone()[0] == 1
     except psycopg.Error:
         alive = False
-    expect.text(f"{'1' if alive else ''}|{_crashes(pgc_cluster) - crashes} backend crashes", "1|0 backend crashes",
+    died = _since(pgc_cluster, crashes)
+    expect.text(f"{'1' if alive else ''}|{died} backend crashes" if isinstance(died, int) else died, "1|0 backend crashes",
                 "backend survives the under-filled DICT chunk")

test/pytest/test_parquet_import.py

@@ -11,12 +11,26 @@
 """
 import psycopg
 
+from pgc_vacuity import query_error
+
 
 def _crashes(pgc_cluster):
+    """-> how many backends the server log says died, or a failed reading when the
+    log can't be read: a blind count must not read as "no crashes"."""
     try:
         return (pgc_cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal")
-    except OSError:
-        return 0
+    except OSError as exc:
+        return query_error(f"crash count unreadable: {exc}")
+
+
+def _since(pgc_cluster, before):
+    """-> backends died since `before`, or the failed reading when either count is
+    one: a blind count must not read as "no crashes"."""
+    now = _crashes(pgc_cluster)
+    for v in (before, now):
+        if not isinstance(v, int):
+            return v
+    return now - before
 
 
 def _one(conn, sql, params=None):
@@ -71,5 +85,6 @@ def test_a_partitioned_parent_is_refused(pgc_conn, pgc_cluster, server_dir, expe
     expect.text(_sqlstate(conn, "SELECT pgcolumnar.import_parquet('pq_parent', %s)", (path,)), "42809",
                 "importing into the partitioned parent is refused")
     alive = _sqlstate(conn, "SELECT 1") == "accepted"
-    expect.text(f"{'1' if alive else ''}|{_crashes(pgc_cluster) - crashes} backend crashes", "1|0 backend crashes",
+    died = _since(pgc_cluster, crashes)
+    expect.text(f"{'1' if alive else ''}|{died} backend crashes" if isinstance(died, int) else died, "1|0 backend crashes",
                 "and the cluster is still up, so the refusal was not an abort")