diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index dbcc327ba..01eafab8d 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2226,6 +2226,34 @@ native_agg native_agg sum(int4) 15;16;17;18;19 never - native_agg native_agg sum(int4,nulls) 15;16;17;18;19 never - native_agg native_agg sum/min/max uses the metadata agg node 15;16;17;18;19 never - native_agg native_agg the deletes removed rows at both ends, the same rows from each 15;16;17;18;19 never - +native_backend_crash native_backend_crash a second concurrent reader was also dropped by the crash 15;16;17;18;19 never - +native_backend_crash native_backend_crash crashA: kill lands mid-write (backend not yet in pg_sleep) 15;16;17;18;19 never - +native_backend_crash native_backend_crash crashA: the INSERT had written pages before the kill (the relation grew) 15;16;17;18;19 never - +native_backend_crash native_backend_crash crashB: kill lands after the stripes flushed (backend in pg_sleep) 15;16;17;18;19 never - +native_backend_crash native_backend_crash crashB: the INSERT had written pages before the kill (the relation grew) 15;16;17;18;19 never - +native_backend_crash native_backend_crash crashC: kill lands mid-write (backend not yet in pg_sleep) 15;16;17;18;19 never - +native_backend_crash native_backend_crash crashC: the INSERT had written pages before the kill (the relation grew) 15;16;17;18;19 never - +native_backend_crash native_backend_crash mid-write: cluster accepts new connections 15;16;17;18;19 never - +native_backend_crash native_backend_crash mid-write: committed content in c matches the heap mirror 15;16;17;18;19 2026-10-16 columnar_storage.c: a stripe data page written with no WAL record +native_backend_crash native_backend_crash mid-write: postmaster survived, same pid 15;16;17;18;19 never - +native_backend_crash native_backend_crash mid-write: the crash reinitialized the cluster (a live session was dropped) 15;16;17;18;19 never - +native_backend_crash native_backend_crash mid-write: the killed insert left no partial row group 15;16;17;18;19 never - +native_backend_crash native_backend_crash mid-write: the killed insert left no visible rows 15;16;17;18;19 never - +native_backend_crash native_backend_crash mid-write: the mirror itself is unchanged 15;16;17;18;19 never - +native_backend_crash native_backend_crash post-flush: cluster accepts new connections 15;16;17;18;19 never - +native_backend_crash native_backend_crash post-flush: committed content in c matches the heap mirror 15;16;17;18;19 2026-10-16 columnar_storage.c: a stripe data page written with no WAL record +native_backend_crash native_backend_crash post-flush: postmaster survived, same pid 15;16;17;18;19 never - +native_backend_crash native_backend_crash post-flush: the crash reinitialized the cluster (a live session was dropped) 15;16;17;18;19 never - +native_backend_crash native_backend_crash post-flush: the killed insert left no partial row group 15;16;17;18;19 never - +native_backend_crash native_backend_crash post-flush: the killed insert left no visible rows 15;16;17;18;19 never - +native_backend_crash native_backend_crash post-flush: the mirror itself is unchanged 15;16;17;18;19 never - +native_backend_crash native_backend_crash with-reader: cluster accepts new connections 15;16;17;18;19 never - +native_backend_crash native_backend_crash with-reader: committed content in c matches the heap mirror 15;16;17;18;19 2026-10-16 columnar_storage.c: a stripe data page written with no WAL record +native_backend_crash native_backend_crash with-reader: postmaster survived, same pid 15;16;17;18;19 never - +native_backend_crash native_backend_crash with-reader: the crash reinitialized the cluster (a live session was dropped) 15;16;17;18;19 never - +native_backend_crash native_backend_crash with-reader: the killed insert left no partial row group 15;16;17;18;19 never - +native_backend_crash native_backend_crash with-reader: the killed insert left no visible rows 15;16;17;18;19 never - +native_backend_crash native_backend_crash with-reader: the mirror itself is unchanged 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound a sequential scan of the same poisoned chunk is refused (XX001) 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound an index fetch of a chunk whose page_length is 2^32 too large is refused (XX001) 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound backend survived the sequential refusal 15;16;17;18;19 never - @@ -2352,6 +2380,9 @@ native_join_runtime_filter native_join_runtime_filter a (SELECT false) gate over native_join_runtime_filter native_join_runtime_filter a (SELECT false) gate over a heap dimension equals heap 15;16;17;18;19 2026-10-15 the join runtime filter took a join carrying a gating qual and failed on the Result over it (the tree before #1437) native_join_runtime_filter native_join_runtime_filter a (SELECT true) gate over a columnar dimension equals heap 15;16;17;18;19 2026-10-15 the join runtime filter took a join carrying a gating qual and failed on the Result over it (the tree before #1437) native_join_runtime_filter native_join_runtime_filter a (SELECT true) gate over a heap dimension equals heap 15;16;17;18;19 2026-10-15 the join runtime filter took a join carrying a gating qual and failed on the Result over it (the tree before #1437) +native_join_runtime_filter native_join_runtime_filter an empty build from a large dimension answers as heap 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter an empty build on one rescan does not empty the next 15;16;17;18;19 2026-10-16 the rescan kept the first execution's runtime filter state: prepared not reset on rescan (mutation, #1447) +native_join_runtime_filter native_join_runtime_filter an empty build reads no fact chunk group 15;16;17;18;19 2026-10-16 an inner join whose build was empty ran on, and the fact scan rejected every row one at a time (the tree before #1447) native_join_runtime_filter native_join_runtime_filter baseline core Hash Join 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter baseline reads all groups 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter build rows omit NULL 15;16;17;18;19 never - @@ -2377,8 +2408,12 @@ native_join_runtime_filter native_join_runtime_filter join runtime filter defaul native_join_runtime_filter native_join_runtime_filter plan has build tap 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter plan has runtime coordinator 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter plan retains core Hash Join 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: and its build is empty 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter premise: heap's answer under a true gate over a columnar dimension holds 200 rows 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter premise: heap's answer under a true gate over a heap dimension holds 200 rows 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: heap's empty-then-full rescan answer holds the 200 keys 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: the empty-then-full rescan runs under the coordinator 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: the large-dimension empty build runs under the coordinator 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter premise: with no gate, the columnar-dimension join has the coordinator 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter premise: with no gate, the heap-dimension join has the coordinator 15;16;17;18;19 never - native_join_runtime_filter native_join_runtime_filter projection join equals heap 15;16;17;18;19 never - @@ -3562,10 +3597,3 @@ vm_privilege vm_privilege vm entry point is covered by this boundary test: vm_se vm_privilege vm_privilege vm_is_visible refuses a heap relation even for its owner (#748) 15;16;17;18;19 never - vm_privilege vm_privilege vm_selftest did not report success to the unprivileged role 15;16;17;18;19 2026-10-14 vm_selftest without its owner check vm_privilege vm_privilege vm_selftest refuses a heap relation even for its owner (#748) 15;16;17;18;19 never - -native_join_runtime_filter native_join_runtime_filter premise: the large-dimension empty build runs under the coordinator 15;16;17;18;19 never - -native_join_runtime_filter native_join_runtime_filter premise: and its build is empty 15;16;17;18;19 never - -native_join_runtime_filter native_join_runtime_filter an empty build reads no fact chunk group 15;16;17;18;19 2026-10-16 an inner join whose build was empty ran on, and the fact scan rejected every row one at a time (the tree before #1447) -native_join_runtime_filter native_join_runtime_filter an empty build from a large dimension answers as heap 15;16;17;18;19 never - -native_join_runtime_filter native_join_runtime_filter premise: the empty-then-full rescan runs under the coordinator 15;16;17;18;19 never - -native_join_runtime_filter native_join_runtime_filter premise: heap's empty-then-full rescan answer holds the 200 keys 15;16;17;18;19 never - -native_join_runtime_filter native_join_runtime_filter an empty build on one rescan does not empty the next 15;16;17;18;19 2026-10-16 the rescan kept the first execution's runtime filter state: prepared not reset on rescan (mutation, #1447) diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index e4c69dd22..2b4799909 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 221 +suites_not_covered 220 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -2886,4 +2886,21 @@ suites_not_covered 221 # never 2639 + 5 = 2644 # dated 925 + 2 = 927 # 2644 + 927 == 3571 -checks_never_observed_red 2644 +# +# #1341, native_backend_crash seeded, with its twin (OffgridwithJD): 2644 -> 2669, +# and suites_not_covered 221 -> 220. The suite had no rows. It enters with all 28 +# checks from five-major runs (28/28 on each of 15-19). 3 are dated by a stripe data +# page written with no WAL record: each scenario's "committed content in c matches +# the heap mirror", which the reinit's crash recovery cannot rebuild. Twenty-five +# arrive never. The no-visible-rows and no-partial-row-group arms are MVCC's: no +# mutation of the extension leaves an aborted transaction's catalog rows visible. +# The sentinel, second-reader, kill-phase and relation-grew arms go red only under +# harness mutations (a signal that kills nothing, an INSERT whose stripes buffer +# whole), which observe the suite rather than the code. The rest hold of the heap +# mirror, the postmaster pid and a connection. +# +# ledger rows 3571 + 28 = 3599 +# never 2644 + 25 = 2669 +# dated 927 + 3 = 930 +# 2669 + 930 == 3599 +checks_never_observed_red 2669 diff --git a/test/native_backend_crash.sh b/test/native_backend_crash.sh index 9f46f430f..07769259d 100755 --- a/test/native_backend_crash.sh +++ b/test/native_backend_crash.sh @@ -103,8 +103,9 @@ open_sentinel() { # backend is parked in the trailing pg_sleep, between the last flush and the # commit that never comes. The phase is asserted, not assumed, from wait_event. crash_one_backend() { - local rows="$1" marker="$2" phase="$3" bg pid i we src + local rows="$1" marker="$2" phase="$3" bg pid i we src size0 open_sentinel || echo "-- sentinel did not connect" + size0="$(q "SELECT pg_relation_size('c');")" env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ -d "$PGC_DB" -c "BEGIN; INSERT INTO c /* $marker */ SELECT g, 'k'||g, g FROM generate_series(1, $rows) g; SELECT pg_sleep(60);" >/dev/null 2>&1 & @@ -131,8 +132,18 @@ crash_one_backend() { sleep 0.1 done else - sleep 0.5 + # Until the write has reached the relation. A fixed 0.5s stood here, and + # nothing showed the INSERT had written a page by then: with its stripes + # buffered whole (stripe_row_limit above the row count) the kill landed on + # a write that had produced nothing, and every mid-write arm still passed + # (#1341). So the kill now waits for the relation to grow, and asserts it. + for i in $(seq 1 600); do + [ "$(q "SELECT pg_relation_size('c') > $size0;")" = t ] && break + sleep 0.1 + done fi + check "$marker: the INSERT had written pages before the kill (the relation grew)" \ + "$(q "SELECT pg_relation_size('c') > $size0;")" "t" we="$(q "SELECT coalesce(wait_event,'') FROM pg_stat_activity WHERE pid = $pid;")" if [ "$phase" = sleep ]; then check "$marker: kill lands after the stripes flushed (backend in pg_sleep)" "$we" "PgSleep" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 0c8cd1f85..67938e153 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -182,6 +182,7 @@ behaviour, the source of that number is named. - [134. test_unique_conc.py: concurrent inserters of one unique key serialize](#134-test_unique_concpy-concurrent-inserters-of-one-unique-key-serialize) - [135. test_update_conc.py: two writers of one row serialize](#135-test_update_concpy-two-writers-of-one-row-serialize) - [136. test_recovery.py: committed writes survive a crash, uncommitted ones leave nothing](#136-test_recoverypy-committed-writes-survive-a-crash-uncommitted-ones-leave-nothing) +- [137. test_native_backend_crash.py: one backend dies and committed data is intact](#137-test_native_backend_crashpy-one-backend-dies-and-committed-data-is-intact) ## 1. How to read a test in here @@ -7296,3 +7297,17 @@ A private cluster, stopped with `-m immediate` for each crash. Each crash is pro | `test_the_pair_through_three_crashes` | rows committed after a checkpoint survive; an open transaction's checkpointed INSERT leaves no row; committed deletes survive; each against the heap pair | | `test_a_cleared_visibility_bit_stays_cleared` | a bit set and checkpointed, then cleared by a DELETE, stays cleared after the crash; a clean block's bit stays set; the rows match heap | | `test_a_projection_survives_a_crash` | a projection copy committed after the checkpoint replays, and reproduces the base's rows and a scan through it | + +## 137. test_native_backend_crash.py: one backend dies and committed data is intact + +The pytest twin of `test/native_backend_crash.sh` (#217, ported for #1341), every check by the shell's name. A SIGSEGV in one backend makes the postmaster drop every session and reinitialize, with crash recovery, while it stays up itself. Committed columnar rows must then match a heap mirror, and an INSERT killed before its commit must leave no row and no row group. + +The kill lands mid-write, once the INSERT has written pages to the relation and before it reaches its trailing `pg_sleep`, and after every stripe has flushed, while it sleeps; both are asserted. A sentinel session opened before each kill must be dropped by the reinit, which proves the crash happened. A private cluster, since the reinit drops every session on the server. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_a_kill_mid_write` | a 3,000,000-row INSERT killed after it has grown the relation: the cluster reinitialized, the postmaster kept its pid, and c holds exactly its committed rows and row groups | +| `test_a_kill_after_the_stripes_flushed` | the same for a 100,000-row INSERT killed in its trailing `pg_sleep` | +| `test_a_kill_with_a_second_reader` | the mid-write kill with a second parked reader, which the reinit drops too | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 76ea8a89f..9dc22c869 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -741,4 +741,6 @@ guard_tests 438 # tests, by collection over the cluster files on this tree. # cluster_tests 641 -> 643 for #1447's two tests in test_join_runtime_filter.py, by # collection over the cluster files on this tree. -cluster_tests 643 +# cluster_tests 643 -> 646 for #1341's test_native_backend_crash.py, the twin of +# native_backend_crash.sh: three tests, by collection over the cluster files on this tree. +cluster_tests 646 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 15b5b7cc7..80b45aa7d 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -116,6 +116,7 @@ "latest_reading", "maintenance_crash", "maintenance_rollback", + "native_backend_crash", "native_chunk_length_bound", "native_delete_vector_index", "native_delete_visibility_paths", diff --git a/test/pytest/test_native_backend_crash.py b/test/pytest/test_native_backend_crash.py new file mode 100644 index 000000000..1c7b8aed4 --- /dev/null +++ b/test/pytest/test_native_backend_crash.py @@ -0,0 +1,235 @@ +"""One backend dies, the postmaster lives, and committed columnar data is intact +(#217, #1341). The pytest twin of `test/native_backend_crash.sh`, every check by the +shell's name. + +A SIGSEGV in one backend makes the postmaster drop every session and reinitialize +shared memory, with crash recovery, while the postmaster itself stays up. Columnar +data goes through the buffer manager and WAL, so committed rows must match a heap +mirror afterwards, and an INSERT killed before its commit must leave no row and no +row group. The kill lands at two points: mid-write, once the INSERT has written +pages to the relation and before it reaches its trailing pg_sleep, and after every +stripe has flushed, while the backend sleeps. Both are asserted, not assumed. + +The write is uncommitted either way, so "no rows survived" cannot tell a crash from +a clean abort. A sentinel session, opened before each kill, must be dropped by the +reinit: that is what proves the crash happened. + +A PRIVATE CLUSTER, because the reinit drops every session on the server. +""" +import os +import pathlib +import shutil +import signal +import tempfile +import threading +import time + +import psycopg +import pytest + +from pgc_vacuity import query_error + +BIG_STRIPE = None # a stripe_row_limit for the inserting session; None keeps the default + + +@pytest.fixture(scope="module") +def nbc(pgc_cluster): + """-> the private cluster, started, with the extension and the h/c pair at 5000 rows.""" + from pgc_cluster import Cluster, pick_port + + work = pathlib.Path(tempfile.mkdtemp(prefix="pgc-nbcrash-")) + work.chmod(0o777) + datadir = work / "data" + datadir.mkdir() + cluster = Cluster(pgc_cluster.pg_config, "nbcrash", datadir, pick_port(14)) + try: + cluster.initdb() + cluster.start() + with psycopg.connect(cluster.dsn(), autocommit=True) as c: + c.execute("CREATE EXTENSION IF NOT EXISTS pgcolumnar") + # the postmaster must reinitialize after a backend crash, not shut down + c.execute("ALTER SYSTEM SET restart_after_crash = on") + c.execute("SELECT pg_reload_conf()") + c.execute("CREATE TABLE h (id int, v text, x float8)") + c.execute("CREATE TABLE c (id int, v text, x float8) USING pgcolumnar") + c.execute("INSERT INTO h SELECT g, 'b' || g, g * 2.5 FROM generate_series(1, 5000) g") + c.execute("INSERT INTO c SELECT g, 'b' || g, g * 2.5 FROM generate_series(1, 5000) g") + yield cluster + finally: + cluster.stop() + shutil.rmtree(work, ignore_errors=True) + + +def _one(cluster, sql): + """-> the first column of the first row, or a distinct failure sentinel.""" + try: + with psycopg.connect(cluster.dsn(), autocommit=True, connect_timeout=5) as c: + return c.execute(sql).fetchone()[0] + except psycopg.Error as exc: + return query_error(str(exc).splitlines()[0]) + + +def _rows(cluster, sql): + try: + with psycopg.connect(cluster.dsn(), autocommit=True, connect_timeout=5) as c: + return c.execute(sql).fetchall() + except psycopg.Error as exc: + return query_error(str(exc).splitlines()[0]) + + +def _row_groups(cluster): + return _one(cluster, "SELECT count(*) FROM pgcolumnar.row_group r JOIN pgcolumnar.storage s " + "ON s.storage_id = r.storage_id WHERE s.relation_oid = 'c'::regclass") + + +def _postmaster_pid(cluster): + return (cluster.datadir / "postmaster.pid").read_text().splitlines()[0] + + +class _Bystander: + """A session parked in pg_sleep on its own thread. -> .dropped once the server + ended it, which a plain exit of another backend never does.""" + + def __init__(self, cluster, seconds): + self.conn = psycopg.connect(cluster.dsn(), autocommit=True) + self.pid = self.conn.info.backend_pid + self.error = None + self.thread = threading.Thread(target=self._run, args=(seconds,), daemon=True) + self.thread.start() + + def _run(self, seconds): + try: + self.conn.execute(f"SELECT pg_sleep({seconds})") + except psycopg.Error as exc: + self.error = exc + + def dropped(self, within): + self.thread.join(within) + return not self.thread.is_alive() and self.error is not None + + def close(self, cluster): + _end(cluster, self.conn, self.thread, self.pid) + + +def _end(cluster, conn, thread, pid): + """Close a session whose statement may still run on its thread. A live one is + terminated first: closing a connection under a running statement crashes the + driver, and a session a failed check left behind holds its locks into the next + test.""" + if thread.is_alive(): + _one(cluster, f"SELECT pg_terminate_backend({pid})") + thread.join(10) + try: + conn.close() + except psycopg.Error: + pass + + +def _wait_up(cluster): + """Twelve successful connections in a row, 0.25s apart: a 3s window a reinit's + brief accepting blip cannot fake. -> True once it holds.""" + ok = 0 + for _ in range(1200): + if _one(cluster, "SELECT 1") == 1: + ok += 1 + if ok >= 12: + return True + else: + ok = 0 + time.sleep(0.25) + return False + + +def _crash_one_backend(cluster, expect, rows, marker, phase): + """An INSERT into c, held open by a trailing pg_sleep so it never commits, SIGSEGVed + in `phase` ('insert': mid-write, 'sleep': after every stripe flushed). + -> whether the sentinel was dropped by the reinit.""" + sentinel = _Bystander(cluster, 120) + size0 = _one(cluster, "SELECT pg_relation_size('c')") + writer = psycopg.connect(cluster.dsn(), autocommit=True) + pid = writer.info.backend_pid + + def write(): + try: + if BIG_STRIPE: + writer.execute(f"SET pgcolumnar.stripe_row_limit = {BIG_STRIPE}") + writer.execute("BEGIN") + writer.execute(f"INSERT INTO c /* {marker} */ SELECT g, 'k' || g, g FROM generate_series(1, {rows}) g") + writer.execute("SELECT pg_sleep(60)") + except psycopg.Error: + pass + thread = threading.Thread(target=write, daemon=True) + thread.start() + try: + return _kill_when_ready(cluster, expect, marker, phase, pid, size0, thread, sentinel) + finally: + _end(cluster, writer, thread, pid) + sentinel.close(cluster) + + +def _kill_when_ready(cluster, expect, marker, phase, pid, size0, thread, sentinel): + def wait_event(): + return _one(cluster, f"SELECT coalesce(wait_event, '') FROM pg_stat_activity WHERE pid = {pid}") + + for _ in range(600): + if phase == "sleep": + if wait_event() == "PgSleep": + break + elif _one(cluster, f"SELECT pg_relation_size('c') > {size0}") is True: + break + time.sleep(0.1) + expect.text(str(_one(cluster, f"SELECT pg_relation_size('c') > {size0}")), "True", + f"{marker}: the INSERT had written pages before the kill (the relation grew)") + we = wait_event() + if phase == "sleep": + expect.text(we, "PgSleep", f"{marker}: kill lands after the stripes flushed (backend in pg_sleep)") + else: + expect.text("yes" if we != "PgSleep" else "no", "yes", + f"{marker}: kill lands mid-write (backend not yet in pg_sleep)") + + os.kill(pid, signal.SIGSEGV) + thread.join(30) + _wait_up(cluster) + return sentinel.dropped(5) + + +def _assert_intact(cluster, expect, label, dropped, base): + expect.text("yes" if dropped else "no", "yes", + f"{label}: the crash reinitialized the cluster (a live session was dropped)") + expect.text(_postmaster_pid(cluster), base["pm"], f"{label}: postmaster survived, same pid") + expect.num(_one(cluster, "SELECT 1"), 1, f"{label}: cluster accepts new connections") + expect.row_set(_rows(cluster, "SELECT id, v, x FROM c"), base["content"], + f"{label}: committed content in c matches the heap mirror") + expect.row_set(_rows(cluster, "SELECT id, v, x FROM h"), base["mirror"], + f"{label}: the mirror itself is unchanged") + expect.num(_one(cluster, "SELECT count(*) FROM c"), 5000, f"{label}: the killed insert left no visible rows") + expect.num(_row_groups(cluster), base["groups"], f"{label}: the killed insert left no partial row group") + + +@pytest.fixture(scope="module") +def base(nbc): + """The committed baseline, read once before any crash.""" + return {"groups": _row_groups(nbc), "content": _rows(nbc, "SELECT id, v, x FROM c"), + "mirror": _rows(nbc, "SELECT id, v, x FROM h"), "pm": _postmaster_pid(nbc)} + + +def test_a_kill_mid_write(nbc, base, expect): + dropped = _crash_one_backend(nbc, expect, 3000000, "crashA", "insert") + _assert_intact(nbc, expect, "mid-write", dropped, base) + + +def test_a_kill_after_the_stripes_flushed(nbc, base, expect): + dropped = _crash_one_backend(nbc, expect, 100000, "crashB", "sleep") + _assert_intact(nbc, expect, "post-flush", dropped, base) + + +def test_a_kill_with_a_second_reader(nbc, base, expect): + """The reinit drops every session, so a second parked reader goes too.""" + reader = _Bystander(nbc, 90) + try: + dropped = _crash_one_backend(nbc, expect, 3000000, "crashC", "insert") + expect.text("yes" if reader.dropped(5) else "no", "yes", + "a second concurrent reader was also dropped by the crash") + finally: + reader.close(nbc) + _assert_intact(nbc, expect, "with-reader", dropped, base)