diff --git a/test/corruption.sh b/test/corruption.sh index f34c1614d..a777f9afb 100755 --- a/test/corruption.sh +++ b/test/corruption.sh @@ -29,12 +29,23 @@ pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" # requires that no backend has died since the scenario started, read from the # server log rather than from timing. crashes() { - # grep -c prints 0 itself when nothing matches, and then exits 1 - pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true + # a number, or nothing when the log can't be read: grep -c prints 0 itself when + # nothing matches (and exits 1), but nothing at all for a file it cannot open, + # and a blind count must not read as "no crashes" + local _n + _n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)" + case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac } alive() { # since this scenario's mkc, which notes the count - [ "$(q 'SELECT 1;')" = "1" ] && [ "$(crashes)" = "$CRASHES_BEFORE" ] && echo yes \ - || echo "no ($(( $(crashes) - CRASHES_BEFORE )) backend crashes)" + local _up _now + _up="$(q 'SELECT 1;')"; _now="$(crashes)" + if ! [[ "$CRASHES_BEFORE" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then + echo "no (crash count unreadable)" + elif [ "$_up" = "1" ] && [ "$_now" = "$CRASHES_BEFORE" ]; then + echo yes + else + echo "no ($(( _now - CRASHES_BEFORE )) backend crashes)" + fi } refused() { # refused SQL -> the SQLSTATE of the ERROR it raised, or what it printed instead local out diff --git a/test/inheritance.sh b/test/inheritance.sh index a9998f2d2..d1dc10445 100755 --- a/test/inheritance.sh +++ b/test/inheritance.sh @@ -153,13 +153,24 @@ check "premise: while its leaf is an ordinary relation" \ # fails with 58P01 "could not open file .../0/0", which is the bug, and the arms # counted it as refused. inheritance stayed 26/26. CI runs release builds. inh_crashes() { - # grep -c prints 0 itself when nothing matches, and then exits 1 - pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true + # a number, or nothing when the log can't be read: grep -c prints 0 itself when + # nothing matches (and exits 1), but nothing at all for a file it cannot open, + # and a blind count must not read as "no crashes" + local _n + _n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)" + case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac } INH_CRASHES="$(inh_crashes)" inh_up() { - [ "$(q 'SELECT 1;')" = 1 ] && [ "$(inh_crashes)" = "$INH_CRASHES" ] && echo up \ - || echo "DOWN ($(( $(inh_crashes) - INH_CRASHES )) backend crashes)" + local _up _now + _up="$(q 'SELECT 1;')"; _now="$(inh_crashes)" + if ! [[ "$INH_CRASHES" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then + echo "DOWN (crash count unreadable)" + elif [ "$_up" = 1 ] && [ "$_now" = "$INH_CRASHES" ]; then + echo up + else + echo "DOWN ($(( _now - INH_CRASHES )) backend crashes)" + fi } inh_sqlstate() { # inh_sqlstate SQL -> the SQLSTATE of the ERROR it raised, or "accepted" local out diff --git a/test/native_dict_underfill.sh b/test/native_dict_underfill.sh index b5cf5bf8e..48ccfdb6d 100755 --- a/test/native_dict_underfill.sh +++ b/test/native_dict_underfill.sh @@ -28,8 +28,20 @@ check "the text column is DICT-encoded (precondition)" "$ENC" "6" # read old rawLen (uint32 LE at descriptor offset 11), inflate by 64, write back crashes() { - # grep -c prints 0 itself when nothing matches, and then exits 1 - pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true + # a number, or nothing when the log can't be read: grep -c prints 0 itself when + # nothing matches (and exits 1), but nothing at all for a file it cannot open, + # and a blind count must not read as "no crashes" + local _n + _n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)" + case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac +} +crashes_since() { # crashes_since BEFORE -> "N backend crashes", or why it can't count + local _now; _now="$(crashes)" + if [[ "$1" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then + echo "$(( _now - $1 )) backend crashes" + else + echo "crash count unreadable" + fi } crashes0="$(crashes)" q "WITH u AS (UPDATE pgcolumnar.column_chunk c SET encoding_descriptor = @@ -50,8 +62,8 @@ echo "-- select stderr:"; sed 's/^/ /' "$PGC_WORKDIR/uf.err" | head -3 # crashed backend passes once crash recovery has finished. The crash flag below was # computed and never checked. rejected="$(sed -nE 's/^ERROR: ([0-9A-Z]{5}): .*/\1/p' "$PGC_WORKDIR/uf.err" | head -1 | grep . || echo "no SQLSTATE ($(head -1 "$PGC_WORKDIR/uf.err"))")" -echo "-- rejected=$rejected crashes=$(( $(crashes) - crashes0 ))" +echo "-- rejected=$rejected crashes=$(crashes_since "$crashes0")" check "under-filled DICT chunk is rejected with a clean error" "$rejected" "XX001" check "backend survives the under-filled DICT chunk" \ - "$(q 'SELECT 1')|$(( $(crashes) - crashes0 )) backend crashes" "1|0 backend crashes" + "$(q 'SELECT 1')|$(crashes_since "$crashes0")" "1|0 backend crashes" pgc_summary diff --git a/test/parquet_import.sh b/test/parquet_import.sh index 11e4c6716..a6c4182f2 100755 --- a/test/parquet_import.sh +++ b/test/parquet_import.sh @@ -287,8 +287,20 @@ else # and only because it reached the server before crash recovery finished. A # check that connects after recovery reads a live cluster. pq_crashes() { - # grep -c prints 0 itself when nothing matches, and then exits 1 - pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null || true + # a number, or nothing when the log can't be read: grep -c prints 0 itself when + # nothing matches (and exits 1), but nothing at all for a file it cannot open, + # and a blind count must not read as "no crashes" + local _n + _n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)" + case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac + } + pq_crashes_since() { # pq_crashes_since BEFORE -> "N backend crashes", or why it can't count + local _now; _now="$(pq_crashes)" + if [[ "$1" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then + echo "$(( _now - $1 )) backend crashes" + else + echo "crash count unreadable" + fi } pq_crashes0="$(pq_crashes)" pq_out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ @@ -302,7 +314,7 @@ else # correct red against the unfixed build and would have stayed red against the # fix -- a discriminator that discriminates nothing. check "and the cluster is still up, so the refusal was not an abort" \ - "$(q "SELECT 1;")|$(( $(pq_crashes) - pq_crashes0 )) backend crashes" "1|0 backend crashes" + "$(q "SELECT 1;")|$(pq_crashes_since "$pq_crashes0")" "1|0 backend crashes" fi pgc_summary diff --git a/test/pytest/test_corruption.py b/test/pytest/test_corruption.py index 63cae400c..2ebcd2718 100644 --- a/test/pytest/test_corruption.py +++ b/test/pytest/test_corruption.py @@ -55,10 +55,12 @@ def _one(cluster, sql): def _crashes(cluster): + """-> how many backends the server log says died, or a failed reading when the + log can't be read: a blind count must not read as "no crashes".""" try: return (cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal") - except OSError: - return 0 + except OSError as exc: + return query_error(f"crash count unreadable: {exc}") class _Scenario: @@ -99,8 +101,14 @@ def read(self, sql): pass def alive(self): - crashed = _crashes(self.cluster) - self.crashes - return "yes" if _one(self.cluster, "SELECT 1") == 1 and crashed == 0 else f"no ({crashed} backend crashes)" + # probe first: the postmaster logs a backend's death when it reaps it, so a + # count read before the server answers again can miss the crash it is for + up = _one(self.cluster, "SELECT 1") == 1 + now = _crashes(self.cluster) + if not (isinstance(self.crashes, int) and isinstance(now, int)): + return now if isinstance(self.crashes, int) else self.crashes + crashed = now - self.crashes + return "yes" if up and crashed == 0 else f"no ({crashed} backend crashes)" def test_a_corrupt_byte_length(cor, expect): diff --git a/test/pytest/test_inheritance.py b/test/pytest/test_inheritance.py index 2f2379f3b..bc07d4225 100644 --- a/test/pytest/test_inheritance.py +++ b/test/pytest/test_inheritance.py @@ -13,12 +13,26 @@ """ import psycopg +from pgc_vacuity import query_error + def _crashes(pgc_cluster): + """-> how many backends the server log says died, or a failed reading when the + log can't be read: a blind count must not read as "no crashes".""" try: return (pgc_cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal") - except OSError: - return 0 + except OSError as exc: + return query_error(f"crash count unreadable: {exc}") + + +def _since(pgc_cluster, before): + """-> backends died since `before`, or the failed reading when either count is + one: a blind count must not read as "no crashes".""" + now = _crashes(pgc_cluster) + for v in (before, now): + if not isinstance(v, int): + return v + return now - before def _sqlstate(conn, sql): @@ -65,7 +79,9 @@ def test_a_partitioned_parent_is_refused_and_its_leaf_is_not(pgc_conn, pgc_clust def up(): alive = _sqlstate(conn, "SELECT 1") == "accepted" - crashed = _crashes(pgc_cluster) - crashes + crashed = _since(pgc_cluster, crashes) + if not isinstance(crashed, int): + return crashed return "up" if alive and crashed == 0 else f"DOWN ({crashed} backend crashes)" expect.text(_sqlstate(conn, "SELECT pgcolumnar.get_storage_id('inh_pp'::regclass)"), "42809", diff --git a/test/pytest/test_native_dict_underfill.py b/test/pytest/test_native_dict_underfill.py index 98c3f2c4f..36cff1270 100644 --- a/test/pytest/test_native_dict_underfill.py +++ b/test/pytest/test_native_dict_underfill.py @@ -12,12 +12,26 @@ """ import psycopg +from pgc_vacuity import query_error + def _crashes(pgc_cluster): + """-> how many backends the server log says died, or a failed reading when the + log can't be read: a blind count must not read as "no crashes".""" try: return (pgc_cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal") - except OSError: - return 0 + except OSError as exc: + return query_error(f"crash count unreadable: {exc}") + + +def _since(pgc_cluster, before): + """-> backends died since `before`, or the failed reading when either count is + one: a blind count must not read as "no crashes".""" + now = _crashes(pgc_cluster) + for v in (before, now): + if not isinstance(v, int): + return v + return now - before def _one(conn, sql): @@ -66,5 +80,6 @@ def test_an_under_filled_dict_chunk_is_refused(pgc_conn, pgc_cluster, expect): alive = c.execute("SELECT 1").fetchone()[0] == 1 except psycopg.Error: alive = False - expect.text(f"{'1' if alive else ''}|{_crashes(pgc_cluster) - crashes} backend crashes", "1|0 backend crashes", + died = _since(pgc_cluster, crashes) + expect.text(f"{'1' if alive else ''}|{died} backend crashes" if isinstance(died, int) else died, "1|0 backend crashes", "backend survives the under-filled DICT chunk") diff --git a/test/pytest/test_parquet_import.py b/test/pytest/test_parquet_import.py index 4e3787916..a78867840 100644 --- a/test/pytest/test_parquet_import.py +++ b/test/pytest/test_parquet_import.py @@ -11,12 +11,26 @@ """ import psycopg +from pgc_vacuity import query_error + def _crashes(pgc_cluster): + """-> how many backends the server log says died, or a failed reading when the + log can't be read: a blind count must not read as "no crashes".""" try: return (pgc_cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal") - except OSError: - return 0 + except OSError as exc: + return query_error(f"crash count unreadable: {exc}") + + +def _since(pgc_cluster, before): + """-> backends died since `before`, or the failed reading when either count is + one: a blind count must not read as "no crashes".""" + now = _crashes(pgc_cluster) + for v in (before, now): + if not isinstance(v, int): + return v + return now - before def _one(conn, sql, params=None): @@ -71,5 +85,6 @@ def test_a_partitioned_parent_is_refused(pgc_conn, pgc_cluster, server_dir, expe expect.text(_sqlstate(conn, "SELECT pgcolumnar.import_parquet('pq_parent', %s)", (path,)), "42809", "importing into the partitioned parent is refused") alive = _sqlstate(conn, "SELECT 1") == "accepted" - expect.text(f"{'1' if alive else ''}|{_crashes(pgc_cluster) - crashes} backend crashes", "1|0 backend crashes", + died = _since(pgc_cluster, crashes) + expect.text(f"{'1' if alive else ''}|{died} backend crashes" if isinstance(died, int) else died, "1|0 backend crashes", "and the cluster is still up, so the refusal was not an abort")