diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index f39ed7f04..738978f03 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -3798,11 +3798,11 @@ tid_range_scan tid_range_scan premise: the table holds every row 15;16;17;18;19 tid_range_scan tid_range_scan rows inserted earlier in the same transaction are in the range, as the filter sees them 15;16;17;18;19 never - tier_partition tier_partition a DEFAULT partition is refused 15;16;17;18;19 2026-10-13 tier_partition without the DEFAULT refusal tier_partition tier_partition a column with its own collation moves and reads back the same 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition returning 0 rows moved;tier_partition's foreign table without the columns' collations -tier_partition tier_partition a failure after attach leaves the rows exactly once, still local 15;16;17;18;19 never - -tier_partition tier_partition a failure after create leaves the rows exactly once, still local 15;16;17;18;19 never - -tier_partition tier_partition a failure after detach leaves the rows exactly once, still local 15;16;17;18;19 never - -tier_partition tier_partition a failure after drop leaves the rows exactly once, still local 15;16;17;18;19 never - -tier_partition tier_partition a failure after export leaves the rows exactly once, still local 15;16;17;18;19 never - +tier_partition tier_partition a failure after attach leaves the rows exactly once, still local 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) +tier_partition tier_partition a failure after create leaves the rows exactly once, still local 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) +tier_partition tier_partition a failure after detach leaves the rows exactly once, still local 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) +tier_partition tier_partition a failure after drop leaves the rows exactly once, still local 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) +tier_partition tier_partition a failure after export leaves the rows exactly once, still local 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) tier_partition tier_partition a fault step that is not one of tier_partition's is refused 15;16;17;18;19 2026-10-13 the tier_fault_after check hook accepting any value tier_partition tier_partition a grant to a role whose name needs quoting is carried to the foreign partition 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition not carrying the grants;tier_partition returning 0 rows moved tier_partition tier_partition a non-superuser owner tiers its own partition 15;16;17;18;19 never - @@ -3817,7 +3817,7 @@ tier_partition tier_partition a partition with a row-level policy is refused bef tier_partition tier_partition a partition with a trigger of its own is refused before anything moves 15;16;17;18;19 2026-10-13 tier_partition without the dependents check tier_partition tier_partition a query on the other range scans only the local partition, beside a foreign one 15;16;17;18;19 never - tier_partition tier_partition a query on the tiered range scans only the foreign partition 15;16;17;18;19 never - -tier_partition tier_partition a retry over the failed attempt's file moves the partition, exactly once 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition refusing every existing path (the first version's rule);tier_partition returning 0 rows moved +tier_partition tier_partition a retry over the failed attempt's file moves the partition, exactly once 15;16;17;18;19 2026-10-16 tier_partition naming the foreign table with a suffix;tier_partition refusing every existing path (the first version's rule);tier_partition returning 0 rows moved;tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) tier_partition tier_partition a role that does not own the table is refused by its own privileges (42501) 15;16;17;18;19 2026-10-14 tier_partition made SECURITY DEFINER tier_partition tier_partition a rollback after the whole call leaves the rows local, exactly once 15;16;17;18;19 never - tier_partition tier_partition a server of another wrapper is refused 15;16;17;18;19 2026-10-14 tier_partition without its wrong-wrapper branch, so the iceberg wrapper refuses only after the export;tier_partition without the wrapper check @@ -3830,11 +3830,11 @@ tier_partition tier_partition after the move the caller's lock_timeout is the ca tier_partition tier_partition an existing path is refused, so no file a table reads is overwritten 15;16;17;18;19 2026-10-13 tier_partition without the path refusal tier_partition tier_partition an insert racing the move leaves every row exactly once, whichever side gives way 15;16;17;18;19 2026-10-13 tier_partition without the SHARE lock on the partition tier_partition tier_partition an insert routed to the tiered range is refused 15;16;17;18;19 never - -tier_partition tier_partition and no foreign table behind it after attach 15;16;17;18;19 never - -tier_partition tier_partition and no foreign table behind it after create 15;16;17;18;19 never - -tier_partition tier_partition and no foreign table behind it after detach 15;16;17;18;19 never - -tier_partition tier_partition and no foreign table behind it after drop 15;16;17;18;19 never - -tier_partition tier_partition and no foreign table behind it after export 15;16;17;18;19 never - +tier_partition tier_partition and no foreign table behind it after attach 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) +tier_partition tier_partition and no foreign table behind it after create 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) +tier_partition tier_partition and no foreign table behind it after detach 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) +tier_partition tier_partition and no foreign table behind it after drop 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) +tier_partition tier_partition and no foreign table behind it after export 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) tier_partition tier_partition and the refusals moved nothing 15;16;17;18;19 2026-10-13 tier_partition without the path refusal tier_partition tier_partition and the refused call leaves the partition local and attached, every row once 15;16;17;18;19 2026-10-14 tier_partition made SECURITY DEFINER tier_partition tier_partition and the refused call wrote no export file 15;16;17;18;19 2026-10-14 tier_partition made SECURITY DEFINER @@ -3851,23 +3851,23 @@ tier_partition tier_partition premise: at read committed the caller's snapshot w tier_partition tier_partition premise: at repeatable read the caller's snapshot was taken before the writer committed 15;16;17;18;19 never - tier_partition tier_partition premise: at serializable the caller's snapshot was taken before the writer committed 15;16;17;18;19 never - tier_partition tier_partition premise: one side of the race gave way or both went through, and nothing hung 15;16;17;18;19 never - -tier_partition tier_partition premise: the call fails with the fault injected after attach 15;16;17;18;19 never - -tier_partition tier_partition premise: the call fails with the fault injected after create 15;16;17;18;19 never - -tier_partition tier_partition premise: the call fails with the fault injected after detach 15;16;17;18;19 never - -tier_partition tier_partition premise: the call fails with the fault injected after drop 15;16;17;18;19 never - -tier_partition tier_partition premise: the call fails with the fault injected after export 15;16;17;18;19 never - tier_partition tier_partition premise: the cold partition holds its share 15;16;17;18;19 never - tier_partition tier_partition premise: the cold partition is local and columnar, with its bound 15;16;17;18;19 never - -tier_partition tier_partition premise: the first attempt failed after export and left its file 15;16;17;18;19 never - tier_partition tier_partition premise: the move started while the insert was in flight, and waited for it 15;16;17;18;19 never - tier_partition tier_partition premise: the parent holds every row 15;16;17;18;19 never - tier_partition tier_partition premise: the storage id was read before the move 15;16;17;18;19 never - tier_partition tier_partition premise: tier_own owns the parent and its partitions, and neither role is a superuser 15;16;17;18;19 never - tier_partition tier_partition premise: tier_peer can open a session and read the partition 15;16;17;18;19 never - tier_partition tier_partition the answers over the parent are the same after the move 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix +tier_partition tier_partition the call fails with the fault injected after attach 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) +tier_partition tier_partition the call fails with the fault injected after create 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) +tier_partition tier_partition the call fails with the fault injected after detach 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) +tier_partition tier_partition the call fails with the fault injected after drop 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) +tier_partition tier_partition the call fails with the fault injected after export 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) tier_partition tier_partition the call holds ACCESS EXCLUSIVE on the parent until commit, as the docs say 15;16;17;18;19 never - tier_partition tier_partition the cold partition is now a foreign partition with its own name and bound 15;16;17;18;19 2026-10-13 tier_partition attaching with the upper bound shifted a minute;tier_partition naming the foreign table with a suffix tier_partition tier_partition the cold partition was moved with its grant and comment set 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition returning 0 rows moved +tier_partition tier_partition the first attempt failed after export and left its file 15;16;17;18;19 2026-10-16 tier_partition's five injected-fault RAISEs removed, so the call succeeds (mutation, #1339) tier_partition tier_partition the foreign partition keeps the partition's grants and comment 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix;tier_partition not carrying the grants tier_partition tier_partition the foreign partition reads the exported file 15;16;17;18;19 2026-10-13 tier_partition naming the foreign table with a suffix tier_partition tier_partition the local partition leaves no pgcolumnar storage row behind 15;16;17;18;19 2026-10-13 tier_partition renaming the partition aside instead of dropping it @@ -4132,10 +4132,10 @@ vacuum_lock_privilege vacuum_lock_privilege truncate refuses a non-owner before vacuum_lock_privilege vacuum_lock_privilege truncate tells a non-owner nothing about the relation type (#1384) 15;16;17;18;19 2026-10-14 truncate checking ownership after its lock and its type check (main before #1384's fix) vacuum_lock_privilege vacuum_lock_privilege vacuum refuses a non-owner before taking the exclusive lock 15;16;17;18;19 2026-10-14 every maintenance entry point checks ownership after taking its lock;pgcolumnar.vacuum checks ownership after taking its AccessExclusiveLock vacuum_lock_privilege vacuum_lock_privilege vacuum_sorted refuses a non-owner before taking the exclusive lock 15;16;17;18;19 2026-10-14 every maintenance entry point checks ownership after taking its lock -validity_elision validity_elision a chunk claiming no bitmap while it holds fewer values than rows is refused 15;16;17;18;19 never - +validity_elision validity_elision a chunk claiming no bitmap while it holds fewer values than rows is refused 15;16;17;18;19 2026-10-16 the no-nulls guard (claims no nulls but does not account for every row) disabled, so a later guard raised the XX001 (mutation, #1339) validity_elision validity_elision a column with no nulls stores no validity bitmap 15;16;17;18;19 2026-09-18 writer: flush_one_column's presentCount == rowCount forced false, so the bitmap is written even when the chunk holds no null validity_elision validity_elision a null-free column elides its bitmap beside a null-bearing one in the same row group 15;16;17;18;19 never - -validity_elision validity_elision an implausible row count on an elided chunk is refused before anything is allocated 15;16;17;18;19 never - +validity_elision validity_elision an implausible row count on an elided chunk is refused before anything is allocated 15;16;17;18;19 2026-10-16 the no-nulls guard (claims no nulls but does not account for every row) disabled, so a later guard raised the XX001 (mutation, #1339) validity_elision validity_elision and the backend survives that refusal 15;16;17;18;19 never - validity_elision validity_elision and the backend survives that refusal too 15;16;17;18;19 never - validity_elision validity_elision premise: each fixture is a single row group, which is what the expected size assumes 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 4d97cc201..bc4355952 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -3320,4 +3320,27 @@ suites_not_covered 199 # never 2980 + 10 = 2990 # dated 1191 + 8 = 1199 # 2990 + 1199 == 4189 -checks_never_observed_red 2990 +# #1339, validity_elision's two guard arms ask which guard refused (jdatcmd): 2990 -> +# 2988. "a chunk claiming no bitmap while it holds fewer values than rows is refused" +# and "an implausible row count on an elided chunk is refused before anything is +# allocated" are dated by disabling the no-nulls guard, after which a later guard +# raised the same XX001. No row is added, removed or renamed. +# +# ledger rows 4189 + 0 = 4189 +# never 2990 - 2 = 2988 +# dated 1199 + 2 = 1201 +# 2988 + 1201 == 4189 +# #1339, tier_partition's injected-fault checks are arms, not premises (jdatcmd): +# 2988 -> 2972. Removing the five injected-fault RAISEs from tier_partition reddens +# "the call fails with the fault injected after " for all five steps, so a +# repository change reddens them; with "the first attempt failed after export and +# left its file" they lose the prefix, renamed with their history. The same +# mutation dates the ten "a failure after leaves the rows exactly once, still +# local" and "and no foreign table behind it after " arms, and adds itself to +# the already-dated retry arm. No row is added or removed. +# +# ledger rows 4189 + 0 = 4189 +# never 2988 - 16 = 2972 +# dated 1201 + 16 = 1217 +# 2972 + 1217 == 4189 +checks_never_observed_red 2972 diff --git a/test/pytest/test_tier_partition.py b/test/pytest/test_tier_partition.py index 929b99a17..7374c3da7 100644 --- a/test/pytest/test_tier_partition.py +++ b/test/pytest/test_tier_partition.py @@ -213,7 +213,7 @@ def mover(): cur.execute("RESET pgcolumnar.tier_fault_after") expect.text(f"injected fault after {step}" if f"injected fault after {step}" in err else err, f"injected fault after {step}", - f"premise: the call fails with the fault injected after {step}") + f"the call fails with the fault injected after {step}") expect.text(f"{_digest(conn, 'f')}|{_mar_shape(conn, 'f')}", f"{fb}|r:pgcolumnar:{BOUND}", f"a failure after {step} leaves the rows exactly once, still local") expect.num(_one(conn, "SELECT count(*) FROM pg_foreign_table t JOIN pg_class c ON c.oid = t.ftrelid" @@ -496,7 +496,7 @@ def moved_reads(t, cols): cur.execute("RESET pgcolumnar.tier_fault_after") expect.text(f"{'injected fault after export' if 'injected fault after export' in err else err}|" f"{'file' if p.exists() else 'nofile'}", "injected fault after export|file", - "premise: the first attempt failed after export and left its file") + "the first attempt failed after export and left its file") try: retried = str(_one(conn, _tier("again", p))) except psycopg.Error as e: diff --git a/test/pytest/test_validity_elision.py b/test/pytest/test_validity_elision.py index fbcea6970..d1a22d724 100644 --- a/test/pytest/test_validity_elision.py +++ b/test/pytest/test_validity_elision.py @@ -250,16 +250,27 @@ def test_the_layout_change_returns_the_same_rows(pgc_conn, expect): ) +def _guard(exc, phrase, label): + """`SQLSTATE|label` when the error is the guard whose message holds `phrase`; + otherwise `SQLSTATE|another guard: `, so a red names what fired.""" + if exc is None: + return "NOERR" + msg = getattr(getattr(exc, "diag", None), "message_primary", None) or str(exc) + state = getattr(exc, "sqlstate", None) or "none" + return f"{state}|{label}" if phrase in msg else f"{state}|another guard: {msg}" + + def test_the_guards_the_elision_added_refuse_a_lying_catalog(pgc_conn, expect): """The two guards, each corrupted into firing. Neither is reachable from a table written correctly, so the catalog is poisoned the way corruption.sh poisons the fields that predate this change. - THE SQLSTATE IS THE ASSERTION, not the message. `XX001` is - ERRCODE_DATA_CORRUPTED and comes only from a guard that ran; a message grep - is equally satisfied by a missing function, a bad argument or a login - failure. + THE SQLSTATE AND THE GUARD'S OWN MESSAGE. `XX001` is ERRCODE_DATA_CORRUPTED + and comes only from a guard that ran; a message grep alone is equally + satisfied by a missing function, a bad argument or a login failure. But + sixteen guards in the reader raise XX001, so the SQLSTATE alone cannot say + WHICH ran (#1339): `_guard` asks for both. """ import psycopg @@ -285,16 +296,26 @@ def test_the_guards_the_elision_added_refuse_a_lying_catalog(pgc_conn, expect): raised = None except psycopg.Error as exc: raised = exc - expect.sqlstate( - raised, "XX001", - "a chunk claiming no bitmap while it holds fewer values than rows is refused", - ) + # WHICH GUARD, not only that one ran (#1339). XX001 comes from sixteen guards in + # the reader: with this one disabled, the scan still raised XX001 and the arm + # stayed green. So the message must be this guard's own. + expect.text(_guard(raised, "claims no nulls but does not account for every row", + "the no-nulls guard"), + "XX001|the no-nulls guard", + "a chunk claiming no bitmap while it holds fewer values than rows is refused") expect.num(_count(c, "SELECT 1"), 1, "and the backend survives that refusal") # GUARD 2: a row count the chunk cannot hold. The descriptor still # accounts for its own rows, so the claim is false in the other - # direction and the reader refuses BEFORE synthesizing half a gigabyte - # of bits. + # direction and the reader refuses BEFORE synthesizing any bits. + # + # NINE BILLION, NOT FOUR, SO THE ORDER IS VISIBLE (#1339 review): 1.125 GB + # of bits is over MaxAllocSize, so the "implausible row count" backstop sits + # between the no-nulls guard and the allocation, and a no-nulls guard moved + # below the allocation is met by the backstop first. At four billion (500 MB) + # that move stayed green. A guard placed between the backstop and the + # allocation is still unseen. A reworded guard message reddens both arms in + # both twins, which must then be updated together. # # BOUNDING THE SYNTHESIZED BITMAP BY THE ROW GROUP'S BYTE LENGTH DOES # NOT WORK: an elided group of two well-encoded bigint columns measured @@ -303,7 +324,7 @@ def test_the_guards_the_elision_added_refuse_a_lying_catalog(pgc_conn, expect): c.execute( """ UPDATE pgcolumnar.row_group r - SET row_count = 4000000000 + SET row_count = 9000000000 FROM pgcolumnar.storage s WHERE s.storage_id = r.storage_id AND s.relation_oid = 'vb_full'::regclass @@ -314,10 +335,14 @@ def test_the_guards_the_elision_added_refuse_a_lying_catalog(pgc_conn, expect): raised = None except psycopg.Error as exc: raised = exc - expect.sqlstate( - raised, "XX001", - "an implausible row count on an elided chunk is refused before anything is allocated", - ) + # Refused BEFORE anything is allocated means by the value-count guard, which + # runs first; the MaxAllocSize check after it is only a backstop. With the + # value-count guard disabled at four billion rows, the bitmap was allocated, + # a later guard raised XX001, and this arm stayed green (#1339). + expect.text(_guard(raised, "claims no nulls but does not account for every row", + "the no-nulls guard"), + "XX001|the no-nulls guard", + "an implausible row count on an elided chunk is refused before anything is allocated") expect.num(_count(c, "SELECT 1"), 1, "and the backend survives that refusal too") diff --git a/test/tier_partition.sh b/test/tier_partition.sh index 186370e20..cb0b1bf7e 100755 --- a/test/tier_partition.sh +++ b/test/tier_partition.sh @@ -336,7 +336,7 @@ build o ob="$(digest o)" o_err="$(errtext "SET pgcolumnar.tier_fault_after = 'export'" \ "SELECT pgcolumnar.tier_partition('o_jan', 'tier_pq', '$W/o.parquet')")" -check "premise: the first attempt failed after export and left its file" \ +check "the first attempt failed after export and left its file" \ "$(grep -o 'injected fault after export' <<<"$o_err")|$([ -e "$W/o.parquet" ] && echo file || echo nofile)" \ "injected fault after export|file" check "a retry over the failed attempt's file moves the partition, exactly once" \ @@ -455,7 +455,7 @@ for step in export detach drop create attach; do fb="$(digest f)" err="$(errtext "SET pgcolumnar.tier_fault_after = '$step'" \ "SELECT pgcolumnar.tier_partition('f_jan', 'tier_pq', '$W/f_$step.parquet')")" - check "premise: the call fails with the fault injected after $step" \ + check "the call fails with the fault injected after $step" \ "$(grep -o "injected fault after $step" <<<"$err" || echo "$err")" "injected fault after $step" check "a failure after $step leaves the rows exactly once, still local" \ "$(digest f)|$(jan_shape f)" "$fb|r:pgcolumnar:$JAN_BOUND" diff --git a/test/validity_elision.sh b/test/validity_elision.sh index adb710f4f..de2a91a85 100755 --- a/test/validity_elision.sh +++ b/test/validity_elision.sh @@ -319,18 +319,31 @@ check "single-row fetches of the null-bearing column return its nulls as nulls" # catalog -- which is what a bit flip or a crafted file does, and what # corruption.sh does for the fields that predate this change. # -# ASSERT THE SQLSTATE, NOT THE TEXT. `42501`, `22023`, a login FATAL and a +# ASSERT THE SQLSTATE, NOT ONLY THE TEXT. `42501`, `22023`, a login FATAL and a # missing function all satisfy a grep for "ERROR"; XX001 is ERRCODE_DATA_CORRUPTED # and comes only from a guard that ran. -sqlstate() { # SQL -> the 5-char SQLSTATE it raises, or NOERR - local out +# +# AND WHICH GUARD (#1339). XX001 comes from sixteen guards in the reader: with the +# no-nulls guard disabled, both arms below still saw XX001 from a later one and +# stayed green. So each reads the message too, and a red names what fired. +# SQL -> `SQLSTATE|the no-nulls guard`, `SQLSTATE|another guard: ` or NOERR. +guard_of() { + local out state msg out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" \ -U postgres -d "$PGC_DB" -qtA 2>&1 <