From df720734275ecf5d4fed9d9d7ef4d28d798a1ee7 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 02:28:16 +0000 Subject: [PATCH 01/20] test: a projection sorted on a type with no ordering is refused (0A000) and leaves nothing declared, in both twins (#1341) Co-Authored-By: Claude Opus 5.5 --- test/projections.sh | 14 ++++++++++++++ test/pytest/TESTS.md | 1 + test/pytest/expected_tests.txt | 4 +++- test/pytest/test_projections.py | 21 +++++++++++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/test/projections.sh b/test/projections.sh index bf126fd5a..e4e5470a1 100755 --- a/test/projections.sh +++ b/test/projections.sh @@ -448,4 +448,18 @@ psql_run "INSERT INTO dt2 SELECT g, g FROM generate_series(200, 300) g;" check "control: a drop in its own transaction never left one" \ "$(proj_orphans dt2)" "0" +# ---- a sort key with no ordering is refused (#1341) -------------------------- +# +# add_projection does not check the sort column's type. The writer does, when it +# is built: json has no btree comparison, so the back-fill over existing rows +# must refuse with 0A000 rather than sort by a comparison function that does not +# exist. Last in the suite, because without the check that sort is not a clean +# error. +psql_run "CREATE TABLE pjs (id int, j json) USING pgcolumnar; + INSERT INTO pjs SELECT g, json_build_object('k', g) FROM generate_series(1, 500) g;" +check "a projection sorted on a type with no ordering (json) is refused (0A000)" \ + "$(proj_sqlstate "SELECT pgcolumnar.add_projection('pjs', 'pjp', ARRAY['id','j'], ARRAY['j'])")" "0A000" +check "and nothing of it is declared" \ + "$(q "SELECT count(*) FROM pgcolumnar.projection_declaration WHERE name = 'pjp';")" "0" + pgc_summary diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index d52a68b34..f19ca19cc 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -4434,6 +4434,7 @@ failing for a reason that has nothing to do with projections. | `test_the_first_add_records_the_base_and_the_new_projection` | both rows appear at once, with the base's columns, empty sort key, name and shared storage id | | `test_a_second_projection_may_have_no_sort_key` | a projection without a sort key, and three distinct storage ids | | `test_a_bad_projection_is_refused_by_its_own_code` | seven refusals, each by its measured SQLSTATE rather than by "it errored" | +| `test_a_sort_key_with_no_ordering_is_refused` | #1341: a projection sorted on `json`, which has no btree ordering, is refused `0A000` when the back-fill builds its writer, and nothing of it is declared | | `test_a_projection_on_a_heap_table_is_refused` | the refusal that is about the ACCESS METHOD, not the arguments | | `test_drop_removes_one_projection_and_leaves_the_rest` | drop is surgical, and the table is still readable after the DDL | | `test_a_projection_added_late_is_back_filled_from_the_existing_rows` | a projection added after the rows exist is populated from them, not left empty | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index e8dd7d78b..3c595766c 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -715,4 +715,6 @@ guard_tests 438 # objstore_allowlist.sh: one test, by collection over the cluster files on this tree. # cluster_tests 593 -> 596 for #1341's test_objstore_userinfo.py, the twin of # objstore_userinfo.sh: three tests, by collection over the cluster files on this tree. -cluster_tests 596 +# cluster_tests 596 -> 597 for #1341's test_a_sort_key_with_no_ordering_is_refused in +# test_projections.py, by collection over the cluster files on this tree. +cluster_tests 597 diff --git a/test/pytest/test_projections.py b/test/pytest/test_projections.py index b284143bf..1aad7783b 100644 --- a/test/pytest/test_projections.py +++ b/test/pytest/test_projections.py @@ -253,6 +253,27 @@ def test_a_bad_projection_is_refused_by_its_own_code(cat, expect, name, code, sq expect.sqlstate(exc.value, code, name) +def test_a_sort_key_with_no_ordering_is_refused(pgc_conn, expect): + """add_projection does not check the sort column's type; the writer does, when + the back-fill builds it (#1341). json has no btree comparison, so the refusal is + 0A000 with the type named, rather than a sort by a comparison that does not + exist, and nothing of the projection is left declared.""" + import psycopg + with pgc_conn.cursor() as cur: + cur.execute("CREATE TABLE pjs (id int, j json) USING pgcolumnar") + cur.execute("INSERT INTO pjs SELECT g, json_build_object('k', g) FROM generate_series(1, 500) g") + try: + with pgc_conn.cursor() as cur: + cur.execute("SELECT pgcolumnar.add_projection('pjs', 'pjp', ARRAY['id','j'], ARRAY['j'])") + got = "ok" + except psycopg.Error as e: + got = e.sqlstate or "?" + expect.text(got, "0A000", "a projection sorted on a type with no ordering (json) is refused (0A000)") + with pgc_conn.cursor() as cur: + expect.num(_one(cur, "SELECT count(*) FROM pgcolumnar.projection_declaration WHERE name = 'pjp'"), 0, + "and nothing of it is declared") + + def test_a_projection_on_a_heap_table_is_refused(cat, expect): """SEPARATE, because it needs a heap table the other refusals do not, and because the refusal is about the ACCESS METHOD rather than about the arguments.""" From 9dcdf628b93355c1e370331af615bd1d9385b78b Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 02:31:51 +0000 Subject: [PATCH 02/20] test: the sort-type arm's catalog check waits for a server that accepts connections (#1341) Co-Authored-By: Claude Opus 5.5 --- test/projections.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/projections.sh b/test/projections.sh index e4e5470a1..d6d8d68bd 100755 --- a/test/projections.sh +++ b/test/projections.sh @@ -453,12 +453,15 @@ check "control: a drop in its own transaction never left one" \ # add_projection does not check the sort column's type. The writer does, when it # is built: json has no btree comparison, so the back-fill over existing rows # must refuse with 0A000 rather than sort by a comparison function that does not -# exist. Last in the suite, because without the check that sort is not a clean -# error. +# exist. Last in the suite, because without the check that sort crashes the +# backend (measured). psql_run "CREATE TABLE pjs (id int, j json) USING pgcolumnar; INSERT INTO pjs SELECT g, json_build_object('k', g) FROM generate_series(1, 500) g;" check "a projection sorted on a type with no ordering (json) is refused (0A000)" \ "$(proj_sqlstate "SELECT pgcolumnar.add_projection('pjs', 'pjp', ARRAY['id','j'], ARRAY['j'])")" "0A000" +# On a server that accepts connections again: without the check the sort crashes +# its backend (measured), and this arm must answer for the catalog, not for that. +for _i in $(seq 1 60); do [ "$(q 'SELECT 1' 2>/dev/null)" = 1 ] && break; sleep 0.5; done check "and nothing of it is declared" \ "$(q "SELECT count(*) FROM pgcolumnar.projection_declaration WHERE name = 'pjp';")" "0" From 326a2c18bfab42a1c8c7bcb48eaa0e33ff237946 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 02:34:57 +0000 Subject: [PATCH 03/20] test: projections seeds the ledger, with the sort-type arm dated (#1341) Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 76 ++++++++++++++++++++++++++++++++++++ test/check_ledger_budget.txt | 15 ++++++- 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 6c5ca89a4..d0bb3388d 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2604,6 +2604,82 @@ projection_update projection_update projection matches the heap before any updat projection_update projection_update read_projection matches the live base after that update 15;16;17;18;19 2026-10-14 projection rows storing the first projected column's value in every projected column projection_update projection_update read_projection still matches the live base 15;16;17;18;19 2026-10-14 projection rows storing the first projected column's value in every projected column projection_update projection_update reconstruct via the projection still rebuilds every live row 15;16;17;18;19 2026-10-14 projection rows storing the first projected column's value in every projected column +projections projections DROP TABLE removes its declaration (#304) 15;16;17;18;19 never - +projections projections GUC off: no projection scan 15;16;17;18;19 never - +projections projections a mid-transaction drop leaves no orphan projection storage 15;16;17;18;19 never - +projections projections a projection added after a write in the same transaction gets the later rows 15;16;17;18;19 never - +projections projections a projection dropped mid-transaction is gone, not still being written 15;16;17;18;19 never - +projections projections a projection sorted on a type with no ordering (json) is refused (0A000) 15;16;17;18;19 2026-10-14 the projection writer without its sort-type ordering check (the back-fill sort then crashes the backend) +projections projections add on heap table rejected 15;16;17;18;19 never - +projections projections aggregate over projection scan matches oracle 15;16;17;18;19 never - +projections projections an orphan left by an older build is present 15;16;17;18;19 never - +projections projections an unrestricted base scan matches oracle after delete 15;16;17;18;19 never - +projections projections an unrestricted query reads the base, not the projection 15;16;17;18;19 never - +projections projections and it removed the orphan 15;16;17;18;19 never - +projections projections and leaves the other table's declaration alone 15;16;17;18;19 never - +projections projections and nothing of it is declared 15;16;17;18;19 never - +projections projections and the base table still took every row 15;16;17;18;19 never - +projections projections and they are the right rows, not merely the right number 15;16;17;18;19 never - +projections projections back-fill: p2 matches base (b column) 15;16;17;18;19 never - +projections projections back-fill: p2 populated from existing rows 15;16;17;18;19 never - +projections projections base + p2 remain 15;16;17;18;19 never - +projections projections base id 0 columns are all attrs 15;16;17;18;19 never - +projections projections base id 0 name 15;16;17;18;19 never - +projections projections base id 0 sort_key empty 15;16;17;18;19 never - +projections projections base proj_storage_id == base 15;16;17;18;19 never - +projections projections control: a drop in its own transaction never left one 15;16;17;18;19 never - +projections projections control: with no write before add_projection the projection was always right 15;16;17;18;19 never - +projections projections distinct storage ids 15;16;17;18;19 never - +projections projections drop base rejected 15;16;17;18;19 never - +projections projections drop unknown rejected 15;16;17;18;19 never - +projections projections duplicate column rejected 15;16;17;18;19 never - +projections projections duplicate name rejected 15;16;17;18;19 never - +projections projections empty columns rejected 15;16;17;18;19 never - +projections projections fp chunks carry min/max skip metadata 15;16;17;18;19 never - +projections projections fp count after delete matches base 15;16;17;18;19 never - +projections projections fp fan-out matches base (a,c) 15;16;17;18;19 never - +projections projections fp reflects deletes (a,c) 15;16;17;18;19 never - +projections projections fp row count matches base 15;16;17;18;19 never - +projections projections fp2 multi-stripe fan-out matches base 15;16;17;18;19 never - +projections projections fp2 spans multiple projection row groups 15;16;17;18;19 never - +projections projections fq fan-out matches base (b) 15;16;17;18;19 never - +projections projections new snapshot projection scan sees both batches 15;16;17;18;19 never - +projections projections no projection rows before first add 15;16;17;18;19 never - +projections projections non-covering query (references b) uses the base 15;16;17;18;19 never - +projections projections old snapshot projection scan does not see post-snapshot rows 15;16;17;18;19 never - +projections projections p1 columns 15;16;17;18;19 never - +projections projections p1 gone after drop 15;16;17;18;19 never - +projections projections p1 has its own storage id 15;16;17;18;19 never - +projections projections p1 id is 1 15;16;17;18;19 never - +projections projections p1 sort_key 15;16;17;18;19 never - +projections projections p2 columns 15;16;17;18;19 never - +projections projections p2 id is 2 15;16;17;18;19 never - +projections projections p2 sort_key empty 15;16;17;18;19 never - +projections projections planner still uses projection after vacuum 15;16;17;18;19 never - +projections projections premise: the base table holds every committed row 15;16;17;18;19 never - +projections projections premise: the drop really removed the projection 15;16;17;18;19 never - +projections projections projection chosen for covering + sort-key query 15;16;17;18;19 never - +projections projections projection matches base after second vacuum 15;16;17;18;19 never - +projections projections projection scan matches oracle after delete 15;16;17;18;19 never - +projections projections projection survives vacuum 15;16;17;18;19 never - +projections projections projection-scan matches oracle after vacuum 15;16;17;18;19 never - +projections projections projection-scan results match heap oracle 15;16;17;18;19 never - +projections projections read_projection base rejected 15;16;17;18;19 never - +projections projections read_projection matches base after vacuum 15;16;17;18;19 never - +projections projections reconstruct (a,b,c) matches base after vacuum 15;16;17;18;19 never - +projections projections reconstruct full row matches base 15;16;17;18;19 never - +projections projections reconstruct matches base after delete + NULLs 15;16;17;18;19 never - +projections projections reconstruct row count matches base 15;16;17;18;19 never - +projections projections session A baseline via projection sees batch 1 15;16;17;18;19 never - +projections projections so a rebuild still works for the rest of the database 15;16;17;18;19 never - +projections projections sort key not in columns 15;16;17;18;19 never - +projections projections storage id resolves 15;16;17;18;19 never - +projections projections table populated 15;16;17;18;19 never - +projections projections table still readable after DDL 15;16;17;18;19 never - +projections projections the rebuild does not abort on it 15;16;17;18;19 never - +projections projections two declared projections to start 15;16;17;18;19 never - +projections projections two rows after first add (base + p1) 15;16;17;18;19 never - +projections projections unknown column rejected 15;16;17;18;19 never - range_pruning range_pruning a clustered overlap near the end skips the four vectors only their upper bounds rule out 15;16;17;18;19 2026-10-13 native_range_excludes never using max_upper (haveUpper forced false) range_pruning range_pruning a clustered overlap skips vectors instead of decoding them 15;16;17;18;19 never - range_pruning range_pruning a scattered column skips nothing, which is what the documentation says 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 0cb4293c1..0e9fac106 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 227 +suites_not_covered 226 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -2632,4 +2632,15 @@ suites_not_covered 227 # never 2374 - 3 = 2371 # dated 802 + 3 = 805 # 2371 + 805 == 3176 -checks_never_observed_red 2371 +# +# #1341, projections seeded, with the sort-type arms (OffgridwithJD): 2371 -> 2446, and +# suites_not_covered 227 -> 226. The suite had no rows. It enters with all 76 checks +# from five-major runs (76/76 on each of 15-19). One is dated by the projection +# writer without its sort-type ordering check: a json sort key's back-fill then +# crashed the backend instead of refusing with 0A000. The other 75 arrive never. +# +# ledger rows 3176 + 76 = 3252 +# never 2371 + 75 = 2446 +# dated 805 + 1 = 806 +# 2446 + 806 == 3252 +checks_never_observed_red 2446 From 875429d8d2ce3db28cbef043ed2f736e39164f3e Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 22:00:44 +0000 Subject: [PATCH 04/20] test: the projections drop row takes the check's name, without the premise: prefix (#1341) Review of #1375 by jdatcmd: CI's suites (PG 17) ledger gate found the row seeded as 'premise: the drop really removed the projection' while projections.sh:410 and test_projections.py:795 name the check without the prefix, so the gate saw one check not in the ledger and one orphan row. The row is renamed and keeps its other fields. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index d0bb3388d..3f9562f27 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2657,7 +2657,6 @@ projections projections p2 id is 2 15;16;17;18;19 never - projections projections p2 sort_key empty 15;16;17;18;19 never - projections projections planner still uses projection after vacuum 15;16;17;18;19 never - projections projections premise: the base table holds every committed row 15;16;17;18;19 never - -projections projections premise: the drop really removed the projection 15;16;17;18;19 never - projections projections projection chosen for covering + sort-key query 15;16;17;18;19 never - projections projections projection matches base after second vacuum 15;16;17;18;19 never - projections projections projection scan matches oracle after delete 15;16;17;18;19 never - @@ -2676,6 +2675,7 @@ projections projections sort key not in columns 15;16;17;18;19 never - projections projections storage id resolves 15;16;17;18;19 never - projections projections table populated 15;16;17;18;19 never - projections projections table still readable after DDL 15;16;17;18;19 never - +projections projections the drop really removed the projection 15;16;17;18;19 never - projections projections the rebuild does not abort on it 15;16;17;18;19 never - projections projections two declared projections to start 15;16;17;18;19 never - projections projections two rows after first add (base + p1) 15;16;17;18;19 never - From 9cd207fb31e7f643ebd9bf3ff5442157ed66b1e9 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 02:15:00 +0000 Subject: [PATCH 05/20] test: analyze() refuses a named column the table does not have, before writing any statistics, in both twins (#1341) Co-Authored-By: Claude Opus 5.5 --- test/analyze_function.sh | 18 ++++++++++++++++++ test/pytest/TESTS.md | 1 + test/pytest/expected_tests.txt | 4 +++- test/pytest/test_analyze_function.py | 26 ++++++++++++++++++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/test/analyze_function.sh b/test/analyze_function.sh index 5f3c3f883..d1e3d4c3a 100755 --- a/test/analyze_function.sh +++ b/test/analyze_function.sh @@ -979,5 +979,23 @@ check_text "the two routes agree which storage a WITH DATA matview owns" \ check_text "pgcolumnar.analyze() reaches a matview created WITH DATA" \ "$mv_state" "succeeded" +# ---- a named column that does not exist is refused (#1341) -------------------- +# +# Silently collecting nothing for it is the failure that looks like success, and +# the refusal comes before any work: a real column named beside it gets no +# statistics either. A fresh table, so pg_stats starts empty for it. +q "DROP TABLE IF EXISTS af_unk; + CREATE TABLE af_unk (a int, b text) USING pgcolumnar; + INSERT INTO af_unk SELECT g, 'v'||g FROM generate_series(1, 5000) g;" >/dev/null +unk_out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At \ + -v ON_ERROR_STOP=1 -v VERBOSITY=verbose -c "SELECT pgcolumnar.analyze('af_unk'::regclass, ARRAY['a', 'nosuch'])" 2>&1)" +check_text "analyze() refuses a named column the table does not have, and names it" \ + "$(sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\): \(.*\)$/\1 \2/p' <<<"$unk_out" | head -1)" \ + "P0001 pgcolumnar.analyze(): column \"nosuch\" does not exist in af_unk" +check_num "and writes no statistics, not even for the real column named beside it" \ + "$(q "SELECT count(*) FROM pg_stats WHERE tablename = 'af_unk'")" "0" +q "SELECT pgcolumnar.analyze('af_unk'::regclass, ARRAY['a'])" >/dev/null +check_num "control: the same call naming only the real column writes its statistics" \ + "$(q "SELECT count(*) FROM pg_stats WHERE tablename = 'af_unk' AND attname = 'a'")" "1" pgc_summary diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index f19ca19cc..3643b6594 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -5849,6 +5849,7 @@ THE FIX IS NOT ONE LINE, AND A GREP IS WHY IT LOOKED LIKE ONE. `pgcolumnar."anal | test | what it holds | | --- | --- | | `test_analyze_function` | the 500,000-row fixture: null_frac exact on a fraction core's sample cannot express, n_distinct exact from one column, the statistics it does not compute left in place, the histogram's true maximum and its smallest non-most-common bound, no most-common value inside the histogram, every bound a value the column holds, the three MCV frequencies over total rows rather than non-null rows, the per-column statistics target honoured, and a column at `SET STATISTICS 0` left alone | +| `test_a_named_column_that_does_not_exist_is_refused` | #1341: `analyze(rel, ARRAY['a','nosuch'])` is refused, naming `nosuch`, and writes no statistics, not even for `a`; naming `a` alone does | | `test_histogram_bounds_are_a_positional_stride` | eleven distinct rows at target 3, where core's `values[floor(i*(nv-1)/(nhist-1))]` and `percentile_disc`'s `ceil(p*nv)-1` land on different values; the expectation comes from an independent oracle AND a hand-worked figure that must agree first | | `test_null_frac_counts_live_rows_not_ones_a_delete_left` | #485: null_frac came from the zone maps, which count what was WRITTEN, so one pg_stats row carried two statistics normalised against different populations; both must imply the same table | | `test_the_documented_statistics_are_the_ones_written` | the list in `docs/sql-reference.md` parsed and compared against what the function populates, plus the two negatives the doc states in prose | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 3c595766c..74101bb4b 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -717,4 +717,6 @@ guard_tests 438 # objstore_userinfo.sh: three tests, by collection over the cluster files on this tree. # cluster_tests 596 -> 597 for #1341's test_a_sort_key_with_no_ordering_is_refused in # test_projections.py, by collection over the cluster files on this tree. -cluster_tests 597 +# cluster_tests 597 -> 598 for #1341's test_a_named_column_that_does_not_exist_is_refused in +# test_analyze_function.py, by collection over the cluster files on this tree. +cluster_tests 598 diff --git a/test/pytest/test_analyze_function.py b/test/pytest/test_analyze_function.py index 31b1c6258..68849cde2 100644 --- a/test/pytest/test_analyze_function.py +++ b/test/pytest/test_analyze_function.py @@ -497,6 +497,32 @@ def _truth(count): "a column at SET STATISTICS 0 is left alone, because that is what zero means") +def test_a_named_column_that_does_not_exist_is_refused(pgc_conn, expect): + """Silently collecting nothing for a column that does not exist is the failure + that looks like success (#1341). The refusal comes before any work: a real + column named beside it gets no statistics either. The control names only the + real column, so "no statistics" is not a table that never gets any.""" + import psycopg + with pgc_conn.cursor() as cur: + cur.execute("CREATE TABLE af_unk (a int, b text) USING pgcolumnar") + cur.execute("INSERT INTO af_unk SELECT g, 'v'||g FROM generate_series(1, 5000) g") + try: + with pgc_conn.cursor() as cur: + cur.execute("SELECT pgcolumnar.analyze('af_unk'::regclass, ARRAY['a', 'nosuch'])") + got = "ok" + except psycopg.Error as e: + got = f"{e.sqlstate} {e.diag.message_primary}" + expect.text(got, 'P0001 pgcolumnar.analyze(): column "nosuch" does not exist in af_unk', + "analyze() refuses a named column the table does not have, and names it") + stats = "SELECT count(*) FROM pg_stats WHERE tablename = 'af_unk'" + with pgc_conn.cursor() as cur: + cur.execute(stats) + expect.num(cur.fetchone()[0], 0, "and writes no statistics, not even for the real column named beside it") + cur.execute("SELECT pgcolumnar.analyze('af_unk'::regclass, ARRAY['a'])") + cur.execute(stats + " AND attname = 'a'") + expect.num(cur.fetchone()[0], 1, "control: the same call naming only the real column writes its statistics") + + def test_histogram_bounds_are_a_positional_stride(pgc_conn, expect): """Core's bounds are POSITIONS, not quantiles, and the difference is observable. From ce3f5a8b288f7f54e2ca5e5b8032a33c9245d408 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 02:26:03 +0000 Subject: [PATCH 06/20] test: the unknown-column test declines below PostgreSQL 18, as its siblings do (#1341) Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_analyze_function.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/pytest/test_analyze_function.py b/test/pytest/test_analyze_function.py index 68849cde2..8b73f334f 100644 --- a/test/pytest/test_analyze_function.py +++ b/test/pytest/test_analyze_function.py @@ -503,6 +503,10 @@ def test_a_named_column_that_does_not_exist_is_refused(pgc_conn, expect): column named beside it gets no statistics either. The control names only the real column, so "no statistics" is not a table that never gets any.""" import psycopg + readable, _raw, major = _major(pgc_conn) + if not readable or major < 18: + _decline_unnamed(expect, major) + return with pgc_conn.cursor() as cur: cur.execute("CREATE TABLE af_unk (a int, b text) USING pgcolumnar") cur.execute("INSERT INTO af_unk SELECT g, 'v'||g FROM generate_series(1, 5000) g") From be7d28ef5b2ac687dd3767c68811b10e90da97c6 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 02:30:42 +0000 Subject: [PATCH 07/20] test: analyze_function seeds the ledger, with the unknown-column arms dated (#1341) Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 72 ++++++++++++++++++++++++++++++++++++ test/check_ledger_budget.txt | 17 ++++++++- 2 files changed, 87 insertions(+), 2 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 3f9562f27..80b62cf66 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -72,6 +72,78 @@ am_contracts am_contracts premise: the racing sessions contended for keys 15;16; am_contracts am_contracts premise: the truncate table holds 5000 rows 15;16;17;18;19 never - am_contracts am_contracts rows the parent buffered before a savepoint survive a TRUNCATE rolled back to it 15;16;17;18;19 2026-10-14 ExecutorEnd flushes no pending writes;TRUNCATE empties the old storage in place (RelationTruncate before the swap) am_contracts am_contracts sessions racing DO NOTHING on the same keys leave each key once, with no error 15;16;17;18;19 2026-10-14 a speculative row that lost the race is never withdrawn;speculative insert refused as unsupported, as before #1341 +analyze_function analyze_function a column at SET STATISTICS 0 is left alone, because that is what zero means 18;19 never - +analyze_function analyze_function analyze() refuses a named column the table does not have, and names it 18;19 2026-10-14 analyze() without its unknown-column check +analyze_function analyze_function and 42's 18;19 never - +analyze_function analyze_function and 7's frequency exactly, over total rows rather than non-null rows 18;19 never - +analyze_function analyze_function and 99's 18;19 never - +analyze_function analyze_function and core's own number for it is not the truth, as the arithmetic requires 18;19 never - +analyze_function analyze_function and correlation is absent, as the doc says 18;19 never - +analyze_function analyze_function and exactly for a fraction core's sample cannot express 18;19 never - +analyze_function analyze_function and it is an ordered ladder rather than two extremes 18;19 never - +analyze_function analyze_function and it is sorted ascending, which a histogram must be to be usable 18;19 never - +analyze_function analyze_function and no most-common value is inside skew's histogram either 18;19 never - +analyze_function analyze_function and so does the most-common list, which is capped by the same target 18;19 never - +analyze_function analyze_function and the smallest non-most-common value at the bottom 18;19 never - +analyze_function analyze_function and without a WARNING, which is how a mistyped argument is dropped 18;19 never - +analyze_function analyze_function and writes no statistics, not even for the real column named beside it 18;19 2026-10-14 analyze() without its unknown-column check +analyze_function analyze_function control: the same call naming only the real column writes its statistics 18;19 never - +analyze_function analyze_function every bound is a value the column actually holds 18;19 never - +analyze_function analyze_function every statistic the doc lists is one the function writes 18;19 never - +analyze_function analyze_function histogram_bounds are core's positional stride, not evenly spaced quantiles 18;19 never - +analyze_function analyze_function no most-common value appears in histogram_bounds, which would double-count it 18;19 never - +analyze_function analyze_function null_frac and the most-common frequencies agree on how many rows there are 18;19 never - +analyze_function analyze_function null_frac counts live rows, not rows a DELETE left behind 18;19 never - +analyze_function analyze_function pgcolumnar.analyze() 15;16;17 never - +analyze_function analyze_function pgcolumnar.analyze() covers every column whichever storage row the heap returns first 18;19 never - +analyze_function analyze_function pgcolumnar.analyze() leaves the statistics it does not compute in place 18;19 never - +analyze_function analyze_function pgcolumnar.analyze() puts the true maximum at the top of histogram_bounds 18;19 never - +analyze_function analyze_function pgcolumnar.analyze() ran for the zero-target column without raising 18;19 never - +analyze_function analyze_function pgcolumnar.analyze() ran without raising for the MCV column 18;19 never - +analyze_function analyze_function pgcolumnar.analyze() ran without raising, so the statistics below are its own 18;19 never - +analyze_function analyze_function pgcolumnar.analyze() reaches a matview created WITH DATA 18;19 never - +analyze_function analyze_function pgcolumnar.analyze() reports n_distinct exactly, from one column 18;19 never - +analyze_function analyze_function pgcolumnar.analyze() reports null_frac exactly, from reading the column 18;19 never - +analyze_function analyze_function pgcolumnar.analyze() still refuses a relation that has never been written 18;19 never - +analyze_function analyze_function pgcolumnar.analyze() writes the three repeated values as most_common_vals 18;19 never - +analyze_function analyze_function premise: 7 survived the delete and is still a most-common value 18;19 never - +analyze_function analyze_function premise: a covering projection gave the table a second storage row 18;19 never - +analyze_function analyze_function premise: a histogram exists for cat, so the exclusion below is not vacuous 18;19 never - +analyze_function analyze_function premise: a histogram was written, so the comparison below is not vacuous 18;19 never - +analyze_function analyze_function premise: a never-written columnar table has no storage row to find 18;19 never - +analyze_function analyze_function premise: and it really is one row in 500000 18;19 never - +analyze_function analyze_function premise: and no whole number of sampled rows gives that fraction, so core cannot report it 18;19 never - +analyze_function analyze_function premise: and nothing else in the column repeats, so the list is exactly three 18;19 never - +analyze_function analyze_function premise: and the global default differs from it, so the two are distinguishable 18;19 never - +analyze_function analyze_function premise: core ANALYZE produced a null_frac at all, so the numbers below exist 18;19 never - +analyze_function analyze_function premise: core samples fewer rows than the table holds, so its number is an estimate 18;19 never - +analyze_function analyze_function premise: every column has statistics when the base row is resolved 18;19 never - +analyze_function analyze_function premise: skew really is at a non-default statistics target 18;19 never - +analyze_function analyze_function premise: the MCV fixture has the row count the frequencies divide by 18;19 never - +analyze_function analyze_function premise: the MCV list really is gone before we write, so what follows is ours 18;19 never - +analyze_function analyze_function premise: the base storage covers every column index 18;19 never - +analyze_function analyze_function premise: the base storage row was first 18;19 never - +analyze_function analyze_function premise: the capture helper reports a success when there is one 18;19 never - +analyze_function analyze_function premise: the delete left fewer live rows than the zone maps describe 18;19 never - +analyze_function analyze_function premise: the doc lists the statistics it claims to write 18;19 never - +analyze_function analyze_function premise: the fixture has the cardinality this check compares against 18;19 never - +analyze_function analyze_function premise: the fixture is exactly one-in-ten NULL 18;19 never - +analyze_function analyze_function premise: the fixture stays on the absolute-count side of core's 10% rule 18;19 never - +analyze_function analyze_function premise: the fixture's column count has not changed 18;19 never - +analyze_function analyze_function premise: the independent oracle agrees with the hand-worked bounds 18;19 never - +analyze_function analyze_function premise: the k7 fixture is exactly one row in seven NULL 18;19 never - +analyze_function analyze_function premise: the matview's relation_oid finds a storage row 18;19 never - +analyze_function analyze_function premise: the outlier really is in the table 18;19 never - +analyze_function analyze_function premise: the projection's storage covers fewer of them 18;19 never - +analyze_function analyze_function premise: the stride fixture has no repeated value, so no MCV is excluded 18;19 never - +analyze_function analyze_function premise: the update put the projection's row first 18;19 never - +analyze_function analyze_function premise: value 42 appears exactly 54,000 times 18;19 never - +analyze_function analyze_function premise: value 7 appears exactly 90,000 times 18;19 never - +analyze_function analyze_function premise: value 99 appears exactly 27,000 times 18;19 never - +analyze_function analyze_function reltuples is untouched, as the doc says 18;19 never - +analyze_function analyze_function so the bottom bound is the smallest non-most-common value, not the column minimum 18;19 never - +analyze_function analyze_function the histogram honours the column's statistics target, not the global default 18;19 never - +analyze_function analyze_function the two routes agree which storage a WITH DATA matview owns 18;19 never - analyze_stats analyze_stats ANALYZE now collects statistics at all 15;16;17;18;19 never - analyze_stats analyze_stats ANALYZE on a wide table completes well inside its derived cap 15;16;17;18;19 never - analyze_stats analyze_stats a fast machine still gets the floor, not a bound that shrinks with it 15;16;17;18;19 2026-09-24 the cap loses its absolute floor, so the bound shrinks with the machine it runs on;the floor moved by one, so the boundary where floor and derived bound meet is off by one diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 0e9fac106..1e7a3c2a6 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 226 +suites_not_covered 225 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -2643,4 +2643,17 @@ suites_not_covered 226 # never 2371 + 75 = 2446 # dated 805 + 1 = 806 # 2446 + 806 == 3252 -checks_never_observed_red 2446 +# +# #1341, analyze_function seeded, with the unknown-column arms (OffgridwithJD): 2446 -> +# 2516, and suites_not_covered 226 -> 225. The suite had no rows. It enters with all 72 +# checks from five-major runs (71/71 on 18-19, and the one named version SKIP on +# 15-17, where pgcolumnar.analyze() needs PostgreSQL 18). Two are dated, on PG18, by +# analyze() without its unknown-column check: the call then succeeded and wrote +# statistics for the real column named beside the missing one. The other 70 arrive +# never. +# +# ledger rows 3252 + 72 = 3324 +# never 2446 + 70 = 2516 +# dated 806 + 2 = 808 +# 2516 + 808 == 3324 +checks_never_observed_red 2516 From e56de60d9fa187d094cb494bbf2c895bee8b1bfd Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 19:53:04 +0000 Subject: [PATCH 08/20] test: the unknown-column twin declines on PG15-17, so the registry says so (#1341) test_a_named_column_that_does_not_exist_is_refused calls pgcolumnar.analyze(), which needs PostgreSQL 18, so it declines on 15, 16 and 17 like the other analyze_function twins. It was missing from expected_unrunnable.txt, so CI's PG17 cluster job collected and passed 578 and still exited 67 on the unexpected decline. Measured on PG17: test_analyze_function.py exits 67 without these rows and 0 with them. Co-Authored-By: Claude Opus 5.5 --- test/pytest/expected_unrunnable.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/pytest/expected_unrunnable.txt b/test/pytest/expected_unrunnable.txt index b967980e0..731240aae 100644 --- a/test/pytest/expected_unrunnable.txt +++ b/test/pytest/expected_unrunnable.txt @@ -24,18 +24,21 @@ # pg_restore_attribute_stats, which core added in PostgreSQL 18. 15 test_analyze_differential.py::test_analyze_differential 15 test_analyze_function.py::test_analyze_function +15 test_analyze_function.py::test_a_named_column_that_does_not_exist_is_refused 15 test_analyze_function.py::test_histogram_bounds_are_a_positional_stride 15 test_analyze_function.py::test_null_frac_counts_live_rows_not_ones_a_delete_left 15 test_analyze_function.py::test_the_documented_statistics_are_the_ones_written 15 test_analyze_function.py::test_a_covering_projection_does_not_decide_which_columns_get_statistics 16 test_analyze_differential.py::test_analyze_differential 16 test_analyze_function.py::test_analyze_function +16 test_analyze_function.py::test_a_named_column_that_does_not_exist_is_refused 16 test_analyze_function.py::test_histogram_bounds_are_a_positional_stride 16 test_analyze_function.py::test_null_frac_counts_live_rows_not_ones_a_delete_left 16 test_analyze_function.py::test_the_documented_statistics_are_the_ones_written 16 test_analyze_function.py::test_a_covering_projection_does_not_decide_which_columns_get_statistics 17 test_analyze_differential.py::test_analyze_differential 17 test_analyze_function.py::test_analyze_function +17 test_analyze_function.py::test_a_named_column_that_does_not_exist_is_refused 17 test_analyze_function.py::test_histogram_bounds_are_a_positional_stride 17 test_analyze_function.py::test_null_frac_counts_live_rows_not_ones_a_delete_left 17 test_analyze_function.py::test_the_documented_statistics_are_the_ones_written From 193ef09e102313292996a42eb02fbf108c2ebbce Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 02:14:01 +0000 Subject: [PATCH 09/20] test: set_options refuses an unknown compression name and keeps the stored one (#1341) Co-Authored-By: Claude Opus 5.5 --- test/encode_effort.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/encode_effort.sh b/test/encode_effort.sh index 7299f5c4a..f86cbac12 100755 --- a/test/encode_effort.sh +++ b/test/encode_effort.sh @@ -75,6 +75,18 @@ check "a valid encode_effort is stored" \ "$(q "SELECT encode_effort FROM pgcolumnar.options WHERE regclass = 'ef_a'::regclass;" | tail -1)" \ "fast" +# The compression option's own guard (#1341), its sibling. Without it an unknown +# name is stored, and the C reader maps a name it does not know to -1 and falls +# back to the instance default: the table silently compresses as something else. +psql_run "SELECT pgcolumnar.set_options('ef_a', compression => 'zstd');" >/dev/null +badc="$(psql_run "SELECT pgcolumnar.set_options('ef_a', compression => 'brotli');" 2>&1 || true)" +check "an unknown compression name is refused" \ + "$(case "$badc" in *'unknown columnar compression "brotli"'*) echo yes ;; *) echo "no ($badc)" ;; esac)" \ + "yes" +check "and the compression set before it is still the one stored" \ + "$(q "SELECT compression FROM pgcolumnar.options WHERE regclass = 'ef_a'::regclass;" | tail -1)" \ + "zstd" + # --- 2. it changes what the writer does --------------------------------------- psql_run "DROP TABLE IF EXISTS ef_full; DROP TABLE IF EXISTS ef_fast; From f41f38bc5b54d0590761af197eda0e03b14b3a53 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 02:26:37 +0000 Subject: [PATCH 10/20] test: encode_effort seeds the ledger, with the compression-name arms dated (#1341) Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 12 ++++++++++++ test/check_ledger_budget.txt | 15 +++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 80b62cf66..5411d8719 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -711,6 +711,18 @@ differential differential wide point 15;16;17;18;19 never - differential differential wide range 15;16;17;18;19 never - differential differential wide row proj 15;16;17;18;19 never - differential differential wide row scan 15;16;17;18;19 never - +encode_effort encode_effort a table with no option set matches full 15;16;17;18;19 never - +encode_effort encode_effort a valid encode_effort is stored 15;16;17;18;19 never - +encode_effort encode_effort an unknown compression name is refused 15;16;17;18;19 2026-10-14 set_options without its compression-name check +encode_effort encode_effort an unknown encode_effort is refused 15;16;17;18;19 never - +encode_effort encode_effort and a filtered read agrees 15;16;17;18;19 never - +encode_effort encode_effort and an index fetch finds a fast-written row 15;16;17;18;19 never - +encode_effort encode_effort and is identical where FSST does not win 15;16;17;18;19 never - +encode_effort encode_effort and the compression set before it is still the one stored 15;16;17;18;19 2026-10-14 set_options without its compression-name check +encode_effort encode_effort and the refused value was not stored 15;16;17;18;19 never - +encode_effort encode_effort fast stores more than full on a shape where FSST wins 15;16;17;18;19 never - +encode_effort encode_effort reset_options clears it 15;16;17;18;19 never - +encode_effort encode_effort the rows are identical either way 15;16;17;18;19 never - encode_post_codec encode_post_codec a chunk is not stored larger than it would be with no encoding at all 15;16;17;18;19 never - encode_post_codec encode_post_codec and that column stays far below the no-encoding size, so the win is real 15;16;17;18;19 never - encode_post_codec encode_post_codec premise: both fixtures hold every row 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 1e7a3c2a6..b4449e49a 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 225 +suites_not_covered 224 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -2656,4 +2656,15 @@ suites_not_covered 225 # never 2446 + 70 = 2516 # dated 806 + 2 = 808 # 2516 + 808 == 3324 -checks_never_observed_red 2516 +# +# #1341, encode_effort seeded, with the compression-name arms (OffgridwithJD): 2516 -> +# 2526, and suites_not_covered 225 -> 224. The suite had no rows. It enters with all +# 12 checks from five-major runs (12/12 on each of 15-19). Two are dated by +# set_options without its compression-name check: "brotli" was stored over the +# zstd set before it. The other 10 arrive never. +# +# ledger rows 3324 + 12 = 3336 +# never 2516 + 10 = 2526 +# dated 808 + 2 = 810 +# 2526 + 810 == 3336 +checks_never_observed_red 2526 From e688e0fd659bd3a84bbfcc339ee5c1abe556ef13 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 21:18:02 +0000 Subject: [PATCH 11/20] test: every relation-taking entry point names the suite that asserts its bar, and four that had none get arms here, in both twins (#1386) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VjTxfQJj5pnii9wzmPP9Ho --- test/entry_point_privilege.sh | 71 ++++++++++++++++++++++- test/pytest/test_entry_point_privilege.py | 54 ++++++++++++++++- 2 files changed, 119 insertions(+), 6 deletions(-) diff --git a/test/entry_point_privilege.sh b/test/entry_point_privilege.sh index b7a28ea29..b5857ab3d 100755 --- a/test/entry_point_privilege.sh +++ b/test/entry_point_privilege.sh @@ -136,7 +136,10 @@ CATALOG_GOVERNED="$(echo $CATALOG_GOVERNED)" # Relation-taking entry points that are supposed to REFUSE an unprivileged # caller, asserted behaviourally rather than believed. -GUARDED_BEHAVIOURAL="read_projection reconstruct_via_projection vm_is_visible vm_selftest get_storage_id" +# #1386 added the four after get_storage_id: each had no arm anywhere that calls it as +# a caller its bar should stop (cluster_hilbert and recluster_hilbert share cluster's +# owner check in code, which is a reason to expect a refusal, not an assertion of one). +GUARDED_BEHAVIOURAL="read_projection reconstruct_via_projection vm_is_visible vm_selftest get_storage_id cluster_hilbert recluster_hilbert truncate require_caller_select" GUARDED_BEHAVIOURAL="$(echo $GUARDED_BEHAVIOURAL)" # ---- the enumeration, from the catalog ------------------------------------- @@ -277,6 +280,66 @@ check_num "the catalog-governed class is the six from #560" "$catalog_n" "6" # engineer who closes one must come here and remove the name. check_num "no relation-taking C entry point is still ungated" "$pinned_n" "0" +# ---- every claimed-guarded entry point names where its bar is asserted (#1386) ---- +# +# "Claimed-guarded" was the else branch: every relation-taking name that is neither +# catalog-governed nor pinned. A new relation-taking function with no check at all +# landed there and nothing reddened (measured: zz_mutant_leak(regclass) left this +# suite 31/31). So the claim is now written down, name by name, with the suite whose +# arms call it as a caller the bar should stop. A name with no entry is red, and so +# is an entry whose suite does not exist or never calls the function, or that names +# a function this population no longer has. +BAR_ASSERTED_IN=" +add_projection:vacuum_lock_privilege +drop_projection:vacuum_lock_privilege +vacuum:vacuum_lock_privilege +vacuum_sorted:vacuum_lock_privilege +cluster:vacuum_lock_privilege +recluster:vacuum_lock_privilege +compact:vacuum_lock_privilege +compact_rewrite:vacuum_lock_privilege +export_arrow:import_export_privilege +export_parquet:import_export_privilege +import_arrow:import_export_privilege +import_parquet:import_export_privilege +parallel_copy:import_export_privilege +parallel_export_parquet:import_export_privilege +expire:ttl_expire +column_extent:column_extent +maintenance_due:maintenance_due +refresh_rollup:rollup +rebase_rollup:rollup +rollup_source_groups:rollup +rollup_source_retired:rollup +tier_partition:tier_partition +partition_create_ahead:partition_lifecycle +" +# Relation-taking, with no bar to assert, each for its reason: +# partition_drop_candidates, partition_drop_plan return what pg_inherits and the +# partition bounds already tell any role, and the plan is text: the statements +# it lists run as whoever runs them, under core's own privilege checks. +NO_BAR_NEEDED="partition_drop_candidates partition_drop_plan" +bar_home() { awk -v f="$1" -F: '$1 == f { print $2 }' <<<"$BAR_ASSERTED_IN"; } +unhomed="" badhome="" stalehome="" +for fn in $relfns; do + case " $CATALOG_GOVERNED $KNOWN_UNGUARDED $NO_BAR_NEEDED " in *" $fn "*) continue ;; esac + # The behavioural arms below are this suite's own home for the names they call. + case " $GUARDED_BEHAVIOURAL " in *" $fn "*) continue ;; esac + home="$(bar_home "$fn")" + if [ -z "$home" ]; then unhomed="$unhomed $fn"; continue; fi + f="$(dirname "${BASH_SOURCE[0]}")/$home.sh" + [ -r "$f" ] && grep -q "pgcolumnar\.$fn(" "$f" || badhome="$badhome $fn:$home" +done +for fn in $(awk -F: 'NF == 2 { print $1 }' <<<"$BAR_ASSERTED_IN") $NO_BAR_NEEDED; do + case " $relfns " in *" $fn "*) ;; *) stalehome="$stalehome $fn" ;; esac +done +check "every claimed-guarded entry point names the suite that asserts its bar (#1386)" \ + "$([ -z "${unhomed// /}" ] && echo none || echo "$unhomed")" "none" +check "and each named suite exists and calls that entry point" \ + "$([ -z "${badhome// /}" ] && echo none || echo "$badhome")" "none" +check "and no entry names a function that is not a relation-taking entry point" \ + "$([ -z "${stalehome// /}" ] && echo none || echo "$stalehome")" "none" + # THE COUNT ABOVE IS NOT ENOUGH, AND THIS SUITE PROVED IT ON ITSELF. # # When #558 and #562 merged, four of the nine names here became genuinely @@ -353,7 +416,8 @@ call_for() { # call_for -> a valid argument vector for that function case "$1" in read_projection|reconstruct_via_projection) echo "SELECT pgcolumnar.$1('ep_target','p1');" ;; vm_is_visible|vm_selftest) echo "SELECT pgcolumnar.$1('ep_target',0);" ;; - get_storage_id) echo "SELECT pgcolumnar.$1('ep_target');" ;; + get_storage_id|truncate|require_caller_select) echo "SELECT pgcolumnar.$1('ep_target');" ;; + cluster_hilbert|recluster_hilbert) echo "SELECT pgcolumnar.$1('ep_target', 'id');" ;; export_arrow|export_parquet|import_arrow|import_parquet) echo "SELECT pgcolumnar.$1('ep_target','/nonexistent/dir/x');" ;; *) echo "" ;; @@ -386,7 +450,8 @@ done check "premise: t_ep now holds EXECUTE, so a refusal below can only be the C check" \ "$(q "SELECT bool_and(has_function_privilege('t_ep', p.oid, 'EXECUTE')) FROM pg_proc p JOIN pg_namespace n ON n.oid = p.pronamespace - WHERE n.nspname='pgcolumnar' AND p.proname IN ('read_projection','reconstruct_via_projection','vm_is_visible','vm_selftest');")" \ + WHERE n.nspname='pgcolumnar' AND p.proname IN ('read_projection','reconstruct_via_projection','vm_is_visible','vm_selftest', + 'cluster_hilbert','recluster_hilbert','truncate','require_caller_select');")" \ "t" for fn in $GUARDED_BEHAVIOURAL; do diff --git a/test/pytest/test_entry_point_privilege.py b/test/pytest/test_entry_point_privilege.py index 19c1210bb..52f251b8d 100644 --- a/test/pytest/test_entry_point_privilege.py +++ b/test/pytest/test_entry_point_privilege.py @@ -49,7 +49,34 @@ "vm_is_visible": "SELECT pgcolumnar.vm_is_visible('ep_target',0)", "vm_selftest": "SELECT pgcolumnar.vm_selftest('ep_target',0)", "get_storage_id": "SELECT pgcolumnar.get_storage_id('ep_target')", + # #1386: no arm anywhere called these as a caller their bar should stop. + "cluster_hilbert": "SELECT pgcolumnar.cluster_hilbert('ep_target', 'id')", + "recluster_hilbert": "SELECT pgcolumnar.recluster_hilbert('ep_target', 'id')", + "truncate": "SELECT pgcolumnar.truncate('ep_target')", + "require_caller_select": "SELECT pgcolumnar.require_caller_select('ep_target')", } +# Where each other claimed-guarded entry point's bar is asserted (#1386): the pytest +# file whose tests call it as a caller its bar should stop. A suite with no pytest twin +# yet is named as such, so the name is still accounted for and a new one is not. +NO_TWIN_YET = "no pytest twin yet: " +BAR_ASSERTED_IN = { + **{f: "test_vacuum_lock_privilege.py" for f in ("add_projection", "drop_projection", "vacuum", + "vacuum_sorted", "cluster", "recluster", + "compact", "compact_rewrite")}, + **{f: "test_import_export_privilege.py" for f in ("export_arrow", "export_parquet", "import_arrow", + "import_parquet", "parallel_copy", + "parallel_export_parquet")}, + "expire": "test_ttl_expire.py", + "column_extent": "test_column_extent.py", + "maintenance_due": NO_TWIN_YET + "maintenance_due.sh", + **{f: "test_rollup.py" for f in ("refresh_rollup", "rebase_rollup", "rollup_source_groups", + "rollup_source_retired")}, + "tier_partition": "test_tier_partition.py", + "partition_create_ahead": "test_partition_lifecycle.py", +} +# Relation-taking, with no bar to assert: they return what pg_inherits and the +# partition bounds already tell any role, and the plan's statements run as their caller. +NO_BAR_NEEDED = {"partition_drop_candidates", "partition_drop_plan"} FILE_SIBLINGS = ("export_arrow", "export_parquet", "import_arrow", "import_parquet") @@ -142,6 +169,24 @@ def test_every_relation_taking_entry_point_has_a_decided_bar(pgc_conn, pgc_clust expect.num(catalog_n, 6, "the catalog-governed class is the six from #560") expect.num(pinned_n, 0, "no relation-taking C entry point is still ungated") + # ---- every claimed-guarded entry point names where its bar is asserted (#1386) + claimed = sorted(relfns - CATALOG_GOVERNED - KNOWN_UNGUARDED - NO_BAR_NEEDED - set(GUARDED_BEHAVIOURAL)) + unhomed = [f for f in claimed if f not in BAR_ASSERTED_IN] + badhome = [] + for f in claimed: + home = BAR_ASSERTED_IN.get(f) + if home is None or home.startswith(NO_TWIN_YET): + continue + path = pathlib.Path(__file__).resolve().parent / home + if not (path.is_file() and f"pgcolumnar.{f}(" in path.read_text()): + badhome.append(f"{f}:{home}") + stale = sorted((set(BAR_ASSERTED_IN) | NO_BAR_NEEDED) - relfns) + expect.text(" ".join(unhomed) or "none", "none", + "every claimed-guarded entry point names the suite that asserts its bar (#1386)") + expect.text(" ".join(badhome) or "none", "none", "and each named suite exists and calls that entry point") + expect.text(" ".join(stale) or "none", "none", + "and no entry names a function that is not a relation-taking entry point") + # ---- the behavioural arms with conn.cursor() as cur: cur.execute("CREATE TABLE ep_target (id int, v text) USING pgcolumnar") @@ -166,12 +211,15 @@ def test_every_relation_taking_entry_point_has_a_decided_bar(pgc_conn, pgc_clust cur.execute(f"GRANT EXECUTE ON FUNCTION {sig} TO {ROLE}") expect.text(str(_one(conn, f"SELECT bool_and(has_function_privilege('{ROLE}', p.oid, 'EXECUTE')) {in_ext}" " AND p.proname IN ('read_projection','reconstruct_via_projection'," - "'vm_is_visible','vm_selftest')")), "True", + "'vm_is_visible','vm_selftest','cluster_hilbert','recluster_hilbert'," + "'truncate','require_caller_select')")), "True", "premise: t_ep now holds EXECUTE, so a refusal below can only be the C check") import psycopg - expect.num(len(GUARDED_BEHAVIOURAL), 5, "premise: the behavioural loop covers the five guarded names") - for fn, sql in sorted(GUARDED_BEHAVIOURAL.items()): + expect.num(len(GUARDED_BEHAVIOURAL), 9, "premise: the behavioural loop covers the nine guarded names") + # In declaration order, as the shell's loop runs: the projection readers before the + # Hilbert rewrites. + for fn, sql in GUARDED_BEHAVIOURAL.items(): try: _one(conn, sql) owner = "ok" From ab57d12766ac0b28990ed848c30de4c73897deda Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 21:21:16 +0000 Subject: [PATCH 12/20] test: the bar map and the four new arms enter the ledger, dated by their mutations (#1386) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VjTxfQJj5pnii9wzmPP9Ho --- test/check_ledger.tsv | 11 +++++++++++ test/check_ledger_budget.txt | 16 +++++++++++++++- test/pytest/TESTS.md | 2 +- 3 files changed, 27 insertions(+), 2 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 5411d8719..e3fb491be 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -734,10 +734,14 @@ encode_post_codec encode_post_codec the tail column holds no row the heap does n encode_post_codec encode_post_codec the tail column preserves its checksum 15;16;17;18;19 2026-10-13 the reader returning row 5 of every row group as NULL encode_post_codec encode_post_codec the tail column reads back exactly what the heap holds 15;16;17;18;19 2026-10-13 the reader returning row 5 of every row group as NULL encode_post_codec encode_post_codec while a column where encoding genuinely wins still encodes 15;16;17;18;19 never - +entry_point_privilege entry_point_privilege and each named suite exists and calls that entry point 15;16;17;18;19 never - +entry_point_privilege entry_point_privilege and no entry names a function that is not a relation-taking entry point 15;16;17;18;19 never - +entry_point_privilege entry_point_privilege cluster_hilbert refuses an unprivileged caller with 42501 15;16;17;18;19 2026-10-14 cluster_verb_validate without its owner check entry_point_privilege entry_point_privilege coverage: inputs == sum(buckets) 15;16;17;18;19 never - entry_point_privilege entry_point_privilege coverage: no entry point is unbucketed 15;16;17;18;19 2026-10-14 the install script gains a pgcolumnar function no bucket names entry_point_privilege entry_point_privilege coverage: relation-taking == pinned + claimed-guarded + catalog-governed 15;16;17;18;19 never - entry_point_privilege entry_point_privilege every catalog-governed name is still a relation-taking entry point 15;16;17;18;19 never - +entry_point_privilege entry_point_privilege every claimed-guarded entry point names the suite that asserts its bar (#1386) 15;16;17;18;19 2026-10-14 the install script gains a relation-taking function with no privilege check entry_point_privilege entry_point_privilege export_arrow refuses a caller with no privilege before touching the file 15;16;17;18;19 never - entry_point_privilege entry_point_privilege export_parquet refuses a caller with no privilege before touching the file 15;16;17;18;19 2026-10-14 export_parquet without its SELECT check entry_point_privilege entry_point_privilege get_storage_id refuses an unprivileged caller with 42501 15;16;17;18;19 2026-10-14 get_storage_id without its SELECT check @@ -754,15 +758,22 @@ entry_point_privilege entry_point_privilege premise: t_ep now holds EXECUTE, so entry_point_privilege entry_point_privilege premise: the extension is installed and C entry points were found 15;16;17;18;19 never - entry_point_privilege entry_point_privilege premise: the install script declares MODULE_PATHNAME symbols at all 15;16;17;18;19 never - entry_point_privilege entry_point_privilege read_projection refuses an unprivileged caller with 42501 15;16;17;18;19 2026-10-14 read_projection without its SELECT check +entry_point_privilege entry_point_privilege recluster_hilbert refuses an unprivileged caller with 42501 15;16;17;18;19 2026-10-14 cluster_verb_validate without its owner check entry_point_privilege entry_point_privilege reconstruct_via_projection refuses an unprivileged caller with 42501 15;16;17;18;19 never - +entry_point_privilege entry_point_privilege require_caller_select refuses an unprivileged caller with 42501 15;16;17;18;19 2026-10-14 require_caller_select checks nothing entry_point_privilege entry_point_privilege the catalog and the install script agree on the symbol set 15;16;17;18;19 never - entry_point_privilege entry_point_privilege the catalog-governed class is the six from #560 15;16;17;18;19 never - +entry_point_privilege entry_point_privilege the owner can call cluster_hilbert, so a refusal below is a refusal 15;16;17;18;19 never - entry_point_privilege entry_point_privilege the owner can call get_storage_id, so a refusal below is a refusal 15;16;17;18;19 never - entry_point_privilege entry_point_privilege the owner can call read_projection, so a refusal below is a refusal 15;16;17;18;19 never - +entry_point_privilege entry_point_privilege the owner can call recluster_hilbert, so a refusal below is a refusal 15;16;17;18;19 never - entry_point_privilege entry_point_privilege the owner can call reconstruct_via_projection, so a refusal below is a refusal 15;16;17;18;19 2026-10-14 reconstruct_via_projection refuses even a superuser owner +entry_point_privilege entry_point_privilege the owner can call require_caller_select, so a refusal below is a refusal 15;16;17;18;19 never - +entry_point_privilege entry_point_privilege the owner can call truncate, so a refusal below is a refusal 15;16;17;18;19 never - entry_point_privilege entry_point_privilege the owner can call vm_is_visible, so a refusal below is a refusal 15;16;17;18;19 never - entry_point_privilege entry_point_privilege the owner can call vm_selftest, so a refusal below is a refusal 15;16;17;18;19 never - entry_point_privilege entry_point_privilege the relation-by-other-means class is the three that are known 15;16;17;18;19 never - +entry_point_privilege entry_point_privilege truncate refuses an unprivileged caller with 42501 15;16;17;18;19 2026-10-14 truncate without its owner check entry_point_privilege entry_point_privilege vm_is_visible refuses an unprivileged caller with 42501 15;16;17;18;19 2026-10-14 vm_is_visible without its owner check entry_point_privilege entry_point_privilege vm_selftest refuses an unprivileged caller with 42501 15;16;17;18;19 never - fdw_plan_privilege fdw_plan_privilege a privileged planner estimates the Iceberg table from its manifests 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index b4449e49a..1054948d7 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -2667,4 +2667,18 @@ suites_not_covered 224 # never 2516 + 10 = 2526 # dated 808 + 2 = 810 # 2526 + 810 == 3336 -checks_never_observed_red 2526 +# +# #1386, every relation-taking entry point names the suite that asserts its bar +# (OffgridwithJD): 2526 -> 2532. 11 new entry_point_privilege checks: the three map arms, and +# an owner call and a 42501 arm for each of cluster_hilbert, recluster_hilbert, truncate +# and require_caller_select, which no arm called as a refused caller. Dated: the map arm +# by a relation-taking function with no check (it left the suite 31/31 before), and +# each new 42501 arm by its check removed (truncate's owner check, require_caller_select +# checking nothing, and cluster_verb_validate's owner check for both Hilbert verbs). +# Five-major runs: 42/42 on 15-19. +# +# ledger rows 3336 + 11 = 3347 +# never 2526 + 6 = 2532 +# dated 810 + 5 = 815 +# 2532 + 815 == 3347 +checks_never_observed_red 2532 diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 3643b6594..beeab22f1 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -6940,7 +6940,7 @@ The pytest twin of `test/entry_point_privilege.sh` (#569, ported for #1341), eve | test | what it holds | | --- | --- | -| `test_every_relation_taking_entry_point_has_a_decided_bar` | the enumeration premises, the symbol-set agreement, the bucket and class counts, the five 42501 refusals with their owner premises, and the four file siblings refused at the table | +| `test_every_relation_taking_entry_point_has_a_decided_bar` | the enumeration premises, the symbol-set agreement, the bucket and class counts, every claimed-guarded entry point named with the pytest file that asserts its bar (#1386), the nine 42501 refusals with their owner calls, and the four file siblings refused at the table | ## 110. test_am_contracts.py: contracts the access method owes core DML and transactions From eec3f3c24949e2d4b0530d22761f872c3427e518 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 20:26:12 +0000 Subject: [PATCH 13/20] test: six rollup checks that read the result of vacuum, compact_rewrite, expire or a refresh lose the premise prefix, in both twins (#1339) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VjTxfQJj5pnii9wzmPP9Ho --- test/check_ledger.tsv | 12 ++++++------ test/pytest/test_rollup.py | 12 ++++++------ test/rollup.sh | 12 ++++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index e3fb491be..e47738929 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2932,17 +2932,16 @@ rollup rollup another role cannot add a group row to the owner's rollup (42501) rollup rollup another role cannot read the source's groups through rollup_source_groups (42501) 15;16;17;18;19 2026-10-14 rollup_source_groups without its require_caller_select guard rollup rollup another role sees none of its rollup catalog rows 15;16;17;18;19 2026-10-13 the rollup catalogs without row-level security;the rollup catalogs without their grants rollup rollup both rollups over the source refreshed, so each has a view, partials and catalog rows 15;16;17;18;19 never - +rollup rollup compact_rewrite rewrote a row group 15;16;17;18;19 never - rollup rollup each row group's TID range holds exactly that group's rows 15;16;17;18;19 2026-10-13 row_tid mapping 290 rows a block instead of the AM's +rollup rollup expire retired the group in bucket D alone and kept the one that shares it 15;16;17;18;19 never - +rollup rollup expire retired the group the delete thinned 15;16;17;18;19 never - +rollup rollup expire retired the old row groups 15;16;17;18;19 never - rollup rollup nor its retired groups through rollup_source_retired (42501) 15;16;17;18;19 2026-10-14 rollup_source_retired without its require_caller_select guard +rollup rollup pgcolumnar.vacuum gave the source's row groups new identities 15;16;17;18;19 never - rollup rollup premise: and the fixture has NULL-key and NULL-bucket groups 15;16;17;18;19 never - -rollup rollup premise: compact_rewrite rewrote a row group 15;16;17;18;19 never - -rollup rollup premise: expire retired the group in bucket D alone and kept the one that shares it 15;16;17;18;19 never - -rollup rollup premise: expire retired the group the delete thinned 15;16;17;18;19 never - -rollup rollup premise: expire retired the old row groups 15;16;17;18;19 never - -rollup rollup premise: pgcolumnar.vacuum gave the source's row groups new identities 15;16;17;18;19 never - rollup rollup premise: the owner's rollup has group rows another role could reach 15;16;17;18;19 never - rollup rollup premise: the rollup functions exist 15;16;17;18;19 never - -rollup rollup premise: the rollup holds partials the delete below cannot reach 15;16;17;18;19 never - rollup rollup premise: the source holds every row 15;16;17;18;19 never - rollup rollup rebase_rollup of a table that is not a rollup is refused (42704) 15;16;17;18;19 2026-10-14 rebase_rollup without its not-a-rollup check rollup rollup rebase_rollup records the source's current row groups 15;16;17;18;19 2026-10-13 rebase_rollup returning 0 @@ -2960,6 +2959,7 @@ rollup rollup the paused refresh folded what was there before it began 15;16;17; rollup rollup the refresh after a small append read the source at all 15;16;17;18;19 never - rollup rollup the restored rollup reads as a recompute of the restored source 15;16;17;18;19 never - rollup rollup the restored source holds the dumped rows under the storage id recorded for the old one 15;16;17;18;19 never - +rollup rollup the rollup holds partials the delete below cannot reach 15;16;17;18;19 never - rollup rollup the rollup holds rows before the rewrite 15;16;17;18;19 never - rollup rollup the rollup table exists with the documented columns 15;16;17;18;19 never - rollup rollup the rollup to be dumped folds its source 15;16;17;18;19 never - diff --git a/test/pytest/test_rollup.py b/test/pytest/test_rollup.py index d97783416..aedc36fb6 100644 --- a/test/pytest/test_rollup.py +++ b/test/pytest/test_rollup.py @@ -194,7 +194,7 @@ def identity(): with conn.cursor() as cur: cur.execute("SELECT pgcolumnar.vacuum('feed')") expect.text("renumbered" if identity() != before else "same", "renumbered", - "premise: pgcolumnar.vacuum gave the source's row groups new identities") + "pgcolumnar.vacuum gave the source's row groups new identities") vb = _matches(conn) expect.at_least(_one(conn, "SELECT count(*) FROM gauge_15m"), 1, "the rollup holds rows before the rewrite") expect.text(f"{_refresh(conn)}|{_matches(conn)}", f"55000|{vb}", @@ -219,7 +219,7 @@ def identity(): f" FROM gauge_15m__partials WHERE {far}") far_before = _one(conn, at_far) expect.at_least(_one(conn, f"SELECT count(*) FROM gauge_15m__partials WHERE {far}"), 1, - "premise: the rollup holds partials the delete below cannot reach") + "the rollup holds partials the delete below cannot reach") with conn.cursor() as cur: cur.execute("DELETE FROM feed WHERE sensor = 4 AND at < '2026-02-01 01:00:00+00'") expect.text(_folds(conn), "folded", @@ -253,7 +253,7 @@ def identity(): rewrote = _one(conn, "SELECT pgcolumnar.compact_rewrite('feed', 0.02)") except psycopg.Error: rewrote = -1 - expect.at_least(rewrote, 1, "premise: compact_rewrite rewrote a row group") + expect.at_least(rewrote, 1, "compact_rewrite rewrote a row group") _refresh(conn) expect.text(_matches(conn), _oracle(conn), "after compact_rewrite the rollup equals a full recompute") expect.text(_refresh(conn), "0", "and the refresh after the rewrite recomputes nothing") @@ -316,7 +316,7 @@ def identity(): expect.at_least(old_buckets, 1, "the history rollup holds old buckets") with conn.cursor() as cur: cur.execute("SELECT pgcolumnar.set_options('aged', ttl_column => 'at', ttl_interval => '14 days')") - expect.at_least(_one(conn, "SELECT pgcolumnar.expire('aged')"), 1, "premise: expire retired the old row groups") + expect.at_least(_one(conn, "SELECT pgcolumnar.expire('aged')"), 1, "expire retired the old row groups") expect.text(f"{_refresh(conn, 'aged_hourly')}|{_one(conn, hist)}", f"0|{before_hist}", "a refresh after retention folds nothing, refuses nothing, and keeps every bucket") expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.rollup_group rg JOIN pgcolumnar.retired_group t" @@ -361,7 +361,7 @@ def layer(table): expect.text(_one(conn, f"SELECT count(*) FILTER (WHERE at < {h} + interval '40 min') || '|'" f" || count(*) FILTER (WHERE at >= {h} + interval '40 min' AND at < {h} + interval '1 hour')" " FROM layered"), "0|1", - "premise: expire retired the group in bucket D alone and kept the one that shares it") + "expire retired the group in bucket D alone and kept the one that shares it") lq = "SELECT md5(string_agg(bucket || ':' || count || ':' || max_reading, ',' ORDER BY bucket)) FROM layered_hourly" lr = (f"SELECT md5(string_agg(b || ':' || n || ':' || m, ',' ORDER BY b)) FROM (SELECT date_bin('1 hour', at," f" '{ORIGIN}') b, count(*) n, max(reading) m FROM layered GROUP BY 1) x") @@ -403,7 +403,7 @@ def layer(table): cur.execute("DELETE FROM thinned WHERE at < now() - interval '14 days' AND reading % 5 = 0") cur.execute("SELECT pgcolumnar.set_options('thinned', ttl_column => 'at', ttl_interval => '14 days')") expect.num(_one(conn, "SELECT pgcolumnar.expire('thinned')"), 1, - "premise: expire retired the group the delete thinned") + "expire retired the group the delete thinned") tq = "SELECT string_agg(bucket || ':' || count || ':' || max_reading, ',' ORDER BY bucket) FROM thinned_hourly" before_thin = _one(conn, tq) expect.text(f"{_refresh(conn, 'thinned_hourly')}|{_one(conn, tq)}", f"0A000|{before_thin}", diff --git a/test/rollup.sh b/test/rollup.sh index da9db2510..2404a6108 100755 --- a/test/rollup.sh +++ b/test/rollup.sh @@ -167,7 +167,7 @@ groups_before="$(q "SELECT pgcolumnar.get_storage_id('src') || ':' || string_agg q "SELECT pgcolumnar.vacuum('src');" >/dev/null groups_after="$(q "SELECT pgcolumnar.get_storage_id('src') || ':' || string_agg(group_number::text, ',' ORDER BY group_number) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('src');")" -check "premise: pgcolumnar.vacuum gave the source's row groups new identities" \ +check "pgcolumnar.vacuum gave the source's row groups new identities" \ "$([ -n "$groups_after" ] && [ "$groups_after" != "$groups_before" ] && echo renumbered || echo "same: $groups_after")" "renumbered" vb="$(exact "SELECT * FROM r_hourly")" check_num "the rollup holds rows before the rewrite" "$(at_least "$(q "SELECT count(*) FROM r_hourly;")" 1)" "1" @@ -191,7 +191,7 @@ check "and after the rebase the rollup equals a full recompute" "$(matches)" "$( partials_at() { q "SELECT count(*) || '|' || md5(string_agg(ctid::text, ',' ORDER BY ctid)) FROM r_hourly__partials WHERE $1;"; } FAR="bucket >= '2026-03-05 00:00:00+00'" # beyond the time range of every group the delete below reaches far_before="$(partials_at "$FAR")" -check_num "premise: the rollup holds partials the delete below cannot reach" \ +check_num "the rollup holds partials the delete below cannot reach" \ "$(at_least "$(q "SELECT count(*) FROM r_hourly__partials WHERE $FAR;")" 1)" "1" q "DELETE FROM src WHERE dev = 3 AND ts < '2026-03-01 06:00:00+00';" >/dev/null check "a refresh after a delete is not refused" \ @@ -218,7 +218,7 @@ check "after a delete and an append that straddles its bucket, a refresh counts # compact_rewrite drops a group's dead rows by rewriting the group. q "DELETE FROM src WHERE ts >= '2026-03-06 00:00:00+00' AND ts < '2026-03-06 12:00:00+00';" >/dev/null -check_num "premise: compact_rewrite rewrote a row group" \ +check_num "compact_rewrite rewrote a row group" \ "$(at_least "$(q "SELECT pgcolumnar.compact_rewrite('src', 0.01);")" 1)" "1" q "SELECT pgcolumnar.refresh_rollup('r_hourly');" >/dev/null check "after compact_rewrite the rollup equals a full recompute" "$(matches)" "$(oracle)" @@ -278,7 +278,7 @@ ob="$(q "SELECT count(*) FROM r_hist WHERE bucket < now() - interval '100 days'; check_num "the history rollup holds old buckets" "$(at_least "${ob:-0}" 1)" "1" q "SELECT pgcolumnar.set_options('hist', ttl_column => 'ts', ttl_interval => '30 days');" >/dev/null retired="$(q "SELECT pgcolumnar.expire('hist');")" -check_num "premise: expire retired the old row groups" "$(at_least "${retired:-0}" 1)" "1" +check_num "expire retired the old row groups" "$(at_least "${retired:-0}" 1)" "1" check "a refresh after retention folds nothing, refuses nothing, and keeps every bucket" \ "$(sqlstate_or "SELECT pgcolumnar.refresh_rollup('r_hist');")|$(q "SELECT md5(string_agg(bucket || ':' || count || ':' || sum_v, ',' ORDER BY bucket)) FROM r_hist;")" \ "0|$hb" @@ -310,7 +310,7 @@ layered() { # layered TABLE: the three groups, a daily rollup r_TABLE refreshed q "SELECT pgcolumnar.expire('$1');" >/dev/null } layered hd -check "premise: expire retired the group in bucket D alone and kept the one that shares it" \ +check "expire retired the group in bucket D alone and kept the one that shares it" \ "$(q "SELECT count(*) FILTER (WHERE ts < $D + interval '12 hours') || '|' || count(*) FILTER (WHERE ts >= $D + interval '12 hours' AND ts < $D + interval '1 day') FROM hd;")" "0|47" HDQ="SELECT md5(string_agg(bucket || ':' || count || ':' || sum_v, ',' ORDER BY bucket)) FROM r_hd" HDR="SELECT md5(string_agg(b || ':' || n || ':' || s, ',' ORDER BY b)) FROM (SELECT date_bin('1 day', ts, '$ORIGIN') b, count(*) n, sum(v)::numeric s FROM hd GROUP BY 1) x" @@ -343,7 +343,7 @@ q "CREATE TABLE dx (ts timestamptz, v int) USING pgcolumnar; SELECT pgcolumnar.refresh_rollup('r_dx'); DELETE FROM dx WHERE ts < now() - interval '30 days' AND v <= 400; SELECT pgcolumnar.set_options('dx', ttl_column => 'ts', ttl_interval => '30 days');" >/dev/null -check_num "premise: expire retired the group the delete thinned" "$(q "SELECT pgcolumnar.expire('dx');")" "1" +check_num "expire retired the group the delete thinned" "$(q "SELECT pgcolumnar.expire('dx');")" "1" dx_before="$(q "SELECT string_agg(bucket || ':' || count, ',' ORDER BY bucket) FROM r_dx;")" check "a delete in a group that expire then retires is refused, and the rollup is unchanged" \ "$(sqlstate "SELECT pgcolumnar.refresh_rollup('r_dx');")|$(q "SELECT string_agg(bucket || ':' || count, ',' ORDER BY bucket) FROM r_dx;")" \ From 1526d8ebaeda0044f2b0d9aea1b919e5e1e9de4a Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 20:33:03 +0000 Subject: [PATCH 14/20] test: five of the six rollup checks are dated by vacuum, compact_rewrite and expire mutations (#1339) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VjTxfQJj5pnii9wzmPP9Ho --- test/check_ledger.tsv | 18 +++++++++--------- test/check_ledger_budget.txt | 16 +++++++++++++++- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index e47738929..11961a204 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2864,16 +2864,16 @@ rls_direct_storage rls_direct_storage without FORCE the owner is allowed, becaus rollup rollup * with anything but count is refused 15;16;17;18;19 2026-10-13 create_rollup allowing * with any function rollup rollup DROP TABLE of a rollup's source is refused while the rollup exists (2BP01, #1376) 15;16;17;18;19 2026-10-14 a rollup's view, partials and source with no dependency between them, and no catalog cleanup (main before #1376's fix) rollup rollup DROP VIEW of a rollup takes its partials and its catalog rows with it (#1376) 15;16;17;18;19 2026-10-14 a rollup's view, partials and source with no dependency between them, and no catalog cleanup (main before #1376's fix) -rollup rollup a delete in a bucket that holds retired history is refused, and the rollup is unchanged 15;16;17;18;19 2026-10-13 refresh_rollup not storing the retired groups' ranges as history;refresh_rollup recording new groups without their time range;refresh_rollup without the history refusal -rollup rollup a delete in a bucket whose history the same refresh retires is refused, and the rollup is unchanged 15;16;17;18;19 2026-10-13 refresh_rollup blocking on stored history only, not on groups the same refresh retires;refresh_rollup recording new groups without their time range;refresh_rollup without the history refusal -rollup rollup a delete in a group that expire then retires is refused, and the rollup is unchanged 15;16;17;18;19 2026-10-13 expire recording a retired group's deleted count as 0;refresh_rollup not refusing a retiring group whose deleted count moved +rollup rollup a delete in a bucket that holds retired history is refused, and the rollup is unchanged 15;16;17;18;19 2026-10-14 expire retires nothing;refresh_rollup not storing the retired groups' ranges as history;refresh_rollup recording new groups without their time range;refresh_rollup without the history refusal +rollup rollup a delete in a bucket whose history the same refresh retires is refused, and the rollup is unchanged 15;16;17;18;19 2026-10-14 expire retires nothing;refresh_rollup blocking on stored history only, not on groups the same refresh retires;refresh_rollup recording new groups without their time range;refresh_rollup without the history refusal +rollup rollup a delete in a group that expire then retires is refused, and the rollup is unchanged 15;16;17;18;19 2026-10-14 expire recording a retired group's deleted count as 0;expire retires nothing;refresh_rollup not refusing a retiring group whose deleted count moved rollup rollup a delete in buckets with no history, next to retired ones, folds: those buckets equal a recompute and D keeps its history 15;16;17;18;19 2026-10-13 refresh_rollup recomputing every bucket once any group changed;refresh_rollup recording new groups without their time range;refresh_rollup refusing any change but appends (phase 1, main before this branch) rollup rollup a group_by column the source does not have is refused 15;16;17;18;19 2026-10-14 create_rollup without its group_by column check rollup rollup a refresh after a delete is not refused 15;16;17;18;19 2026-10-13 refresh_rollup refusing any change but appends (phase 1, main before this branch) rollup rollup a refresh after a small append reads under a fifth of the source 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup's per-group subquery without OFFSET 0, so it becomes a join and reads every group rollup rollup a refresh after appends folds exactly the appended rows, late ones included 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's rollup rollup a refresh after retention folds nothing, refuses nothing, and keeps every bucket 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup not dropping groups retention retired -rollup rollup a refresh after the source was rewritten is refused, naming rebase_rollup 15;16;17;18;19 2026-10-13 refresh_rollup without the source identity check +rollup rollup a refresh after the source was rewritten is refused, naming rebase_rollup 15;16;17;18;19 2026-10-14 pgcolumnar.vacuum rewrites nothing;refresh_rollup without the source identity check rollup rollup a refresh at REPEATABLE READ is refused 15;16;17;18;19 2026-10-13 refresh_rollup at any isolation level rollup rollup a refresh folds a bulk append 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's rollup rollup a refresh in a zone with a DST change recomputes a whole day bucket 15;16;17;18;19 2026-10-13 refresh_rollup adding the width in the session's zone;refresh_rollup recomputing every bucket once any group changed;refresh_rollup recording new groups without their time range;refresh_rollup refusing any change but appends (phase 1, main before this branch) @@ -2932,13 +2932,13 @@ rollup rollup another role cannot add a group row to the owner's rollup (42501) rollup rollup another role cannot read the source's groups through rollup_source_groups (42501) 15;16;17;18;19 2026-10-14 rollup_source_groups without its require_caller_select guard rollup rollup another role sees none of its rollup catalog rows 15;16;17;18;19 2026-10-13 the rollup catalogs without row-level security;the rollup catalogs without their grants rollup rollup both rollups over the source refreshed, so each has a view, partials and catalog rows 15;16;17;18;19 never - -rollup rollup compact_rewrite rewrote a row group 15;16;17;18;19 never - +rollup rollup compact_rewrite rewrote a row group 15;16;17;18;19 2026-10-14 compact_rewrite reports 0 whatever it rewrote rollup rollup each row group's TID range holds exactly that group's rows 15;16;17;18;19 2026-10-13 row_tid mapping 290 rows a block instead of the AM's -rollup rollup expire retired the group in bucket D alone and kept the one that shares it 15;16;17;18;19 never - -rollup rollup expire retired the group the delete thinned 15;16;17;18;19 never - -rollup rollup expire retired the old row groups 15;16;17;18;19 never - +rollup rollup expire retired the group in bucket D alone and kept the one that shares it 15;16;17;18;19 2026-10-14 expire retires nothing +rollup rollup expire retired the group the delete thinned 15;16;17;18;19 2026-10-14 expire reports 0 whatever it retired;expire retires nothing +rollup rollup expire retired the old row groups 15;16;17;18;19 2026-10-14 expire reports 0 whatever it retired;expire retires nothing rollup rollup nor its retired groups through rollup_source_retired (42501) 15;16;17;18;19 2026-10-14 rollup_source_retired without its require_caller_select guard -rollup rollup pgcolumnar.vacuum gave the source's row groups new identities 15;16;17;18;19 never - +rollup rollup pgcolumnar.vacuum gave the source's row groups new identities 15;16;17;18;19 2026-10-14 pgcolumnar.vacuum rewrites nothing rollup rollup premise: and the fixture has NULL-key and NULL-bucket groups 15;16;17;18;19 never - rollup rollup premise: the owner's rollup has group rows another role could reach 15;16;17;18;19 never - rollup rollup premise: the rollup functions exist 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 1054948d7..d5de2922e 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -2681,4 +2681,18 @@ suites_not_covered 224 # never 2526 + 6 = 2532 # dated 810 + 5 = 815 # 2532 + 815 == 3347 -checks_never_observed_red 2532 +# +# #1339, six rollup premises that read a product operation's result (OffgridwithJD, from +# jdatcmd's #1372 review): 2532 -> 2527. They lose the prefix, renamed with their rows. +# Five are dated: vacuum rewriting nothing (the renumbering check), compact_rewrite and +# expire reporting 0 (their checks), and expire retiring nothing (the bucket-D check, +# with the two expire counts and three refusal arms beside it). "the rollup holds +# partials the delete below cannot reach" arrives never: only a refresh that writes no +# partials reddens it, and that reddens most of the suite. Five-major runs: 82/82, and +# 105/105 on PG18 after the rebase onto d5c0a5e, which brought in 23 checks. +# +# ledger rows 3347 = 3347 +# never 2532 - 5 = 2527 +# dated 815 + 5 = 820 +# 2527 + 820 == 3347 +checks_never_observed_red 2527 From eaf10204bc82eb538235a5b5fad037c954d5763c Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 02:00:55 +0000 Subject: [PATCH 15/20] test: a rollup's min and max keep each source column's collation across a refresh, in both twins (#1341) Co-Authored-By: Claude Opus 5.5 --- test/pytest/TESTS.md | 1 + test/pytest/expected_tests.txt | 4 +++- test/pytest/test_rollup.py | 34 ++++++++++++++++++++++++++++++++++ test/rollup.sh | 32 ++++++++++++++++++++++++++++++++ 4 files changed, 70 insertions(+), 1 deletion(-) diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index beeab22f1..52480759f 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -6735,6 +6735,7 @@ Phase 1 removal proofs, each red on its arm: no `OFFSET 0` (the cost arm), no is | `test_rollup_phase_one` | the rollup equals a full recompute after the first refresh, appends with late rows, a bulk append, and a rebase; a refresh with nothing new folds nothing; rows committed during a refresh are folded by the next, once; two concurrent refreshes fold each row once; zones agree; a refresh after a small append reads under a fifth of the source; a rewritten source is refused until `rebase_rollup`; after a delete, an update, a straddling append, `compact_rewrite` or a deleted NULL-ts row the rollup equals a full recompute, rewriting no partial outside the changed groups' range; a recompute that would reach retained history is refused with `0A000` and changes nothing; a day bucket is recomputed whole across a DST switch; a heap rollup behaves the same; REPEATABLE READ is refused; an owner who is not a superuser creates and refreshes a rollup, and another role sees none of it; bad definitions are refused with `22023`; each group's TID range holds exactly its rows | | `test_a_rollup_and_its_source_are_dropped_together` | the view depends on its source, as a view over it would (#1376): `DROP TABLE` of the source is refused (`2BP01`), `CASCADE` takes the view, its partials table and its catalog rows, and `DROP VIEW` takes the partials and the rows. A role that cannot read a table cannot tie a rollup row to it (`42501`), so it cannot block that table's drop. Nor can it name another role's table as its partials (`42501`, and that table survives its view's drop), or a non-view as its rollup (`42809`); and a written row's view, partials and source cannot be changed (`0A000`) | | `test_rollup_dump_and_restore` | a dump carries a rollup's definition; after a restore into another database, and into the same one after DROP EXTENSION, the restored source has the recorded storage id, and the owner's refresh is refused with `55000` naming `rebase_rollup`, the rollup untouched; after the rebase a refresh folds nothing, then exactly the rows appended since, and the rollup equals a full recompute (#1377) | +| `test_min_and_max_keep_each_columns_collation` | #1341: min and max over a `"C"` column and an en_US column equal a recompute after a refresh that folds new rows into an existing bucket, where the two collations disagree; the partials columns carry the default collation, so the fold's own comparison is what is tested. No en_US collation: `cannot_run` | | `test_row_tid_refuses_a_row_number_below_one` | `row_tid(0)` and `row_tid(-1)` are refused with `22003` and `row_tid(1)` is accepted. A test of its own, last in the file: without the floor `row_tid(0)` crashed the backend | ## 98. test_timeseries_recipes.py: the gap-fill, LOCF and interpolation recipes answer what they claim diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 74101bb4b..786ac5aa1 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -719,4 +719,6 @@ guard_tests 438 # test_projections.py, by collection over the cluster files on this tree. # cluster_tests 597 -> 598 for #1341's test_a_named_column_that_does_not_exist_is_refused in # test_analyze_function.py, by collection over the cluster files on this tree. -cluster_tests 598 +# cluster_tests 598 -> 599 for #1341's test_min_and_max_keep_each_columns_collation in +# test_rollup.py, by collection over the cluster files on this tree. +cluster_tests 599 diff --git a/test/pytest/test_rollup.py b/test/pytest/test_rollup.py index aedc36fb6..791972564 100644 --- a/test/pytest/test_rollup.py +++ b/test/pytest/test_rollup.py @@ -781,6 +781,40 @@ def restore(db): admin("postgres", f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") +def test_min_and_max_keep_each_columns_collation(pgc_own_db, expect): + """The partials columns carry the database's default collation, and a refresh + that updates a bucket folds with least()/greatest(), which must compare in the + SOURCE column's collation (#1341). One "C" column and one en_US column, so + whichever is the default, the other differs from it. Lower-case rows first, + then upper-case into the same bucket: under C 'B' sorts before 'a', under en_US + after it, and only the second refresh's fold can get that wrong.""" + conn = pgc_own_db + coll = _one(conn, "SELECT collname FROM pg_collation WHERE collname IN ('en_US.utf8', 'en_US.UTF-8', 'en_US')" + " ORDER BY collname LIMIT 1") + if coll is None: + expect.cannot_run("MISSING_DEPENDENCY", "no en_US collation", + name="the server has no en_US collation") + return + with conn.cursor() as cur: + cur.execute(f'CREATE TABLE ct (ts timestamptz, sc text COLLATE "C", su text COLLATE "{coll}") USING pgcolumnar') + cur.execute("SELECT pgcolumnar.set_options('ct', stripe_row_limit => 1000)") + cur.execute(f"INSERT INTO ct SELECT '{ORIGIN}'::timestamptz + (g % 24) * interval '1 hour', 'a'||g, 'a'||g" + " FROM generate_series(1, 2000) g") + cur.execute(f"SELECT pgcolumnar.create_rollup('r_ct', 'ct', 'ts', '1 day', '{ORIGIN}', '{{}}'," + " '{min,max,min,max}', '{sc,sc,su,su}')") + cur.execute("SELECT pgcolumnar.refresh_rollup('r_ct')") + cur.execute(f"INSERT INTO ct SELECT '{ORIGIN}'::timestamptz + (g % 24) * interval '1 hour', 'B'||g, 'B'||g" + " FROM generate_series(1, 2000) g") + folded = _one(conn, "SELECT pgcolumnar.refresh_rollup('r_ct')") + expect.text(f"{folded}|{_one(conn, 'SELECT count(*) FROM r_ct')}", "2000|1", + "premise: the second refresh folds new rows into the existing bucket") + expect.text(_one(conn, "SELECT min_sc || '|' || max_sc || '|' || min_su || '|' || max_su FROM r_ct"), + _one(conn, "SELECT min(sc) || '|' || max(sc) || '|' || min(su) || '|' || max(su) FROM ct"), + "min and max over a C column and an en_US column keep each column's own order across a refresh") + expect.text(str(_one(conn, "SELECT min(sc) <> min(su) FROM ct")), "True", + "premise: the two collations disagree on these rows, so the arm can tell them apart") + + def test_row_tid_refuses_a_row_number_below_one(pgc_conn, expect): """Row numbers start at 1 (#1341). LAST IN THE FILE, AND A TEST OF ITS OWN: without the floor, row_tid(0) crashed the backend (measured on PG17), which would take the diff --git a/test/rollup.sh b/test/rollup.sh index 2404a6108..ec7f70acf 100755 --- a/test/rollup.sh +++ b/test/rollup.sh @@ -622,6 +622,38 @@ psql_run "CREATE TABLE lr_p5 (bucket timestamptz) USING pgcolumnar; CREATE VIEW check "a rollup row's view, partials and source cannot be changed once written (0A000, #1376)" \ "$(sqlstate "UPDATE pgcolumnar.rollup SET partials = 'lr_p6' WHERE rollup = 'lr_v5'::regclass")" "0A000" +# ---- min and max keep the source column's collation (#1341) ------------------ +# +# The partials columns are created with the type alone, so they carry the +# database's default collation. A refresh that updates a bucket folds the new +# partial into the stored one with least()/greatest(), and that comparison must be +# the source column's. Two text columns, "C" and en_US, so whichever is the +# database default, the other differs from it. Lower-case rows first, then +# upper-case rows into the SAME buckets: under C 'B' sorts before 'a', under en_US +# after it, and only the second refresh's fold can get that wrong. +CT_COLL="$(q "SELECT collname FROM pg_collation WHERE collname IN ('en_US.utf8', 'en_US.UTF-8', 'en_US') ORDER BY collname LIMIT 1;")" +if [ -n "$CT_COLL" ]; then + q "CREATE TABLE ct (ts timestamptz, sc text COLLATE \"C\", su text COLLATE \"$CT_COLL\") USING pgcolumnar; + SELECT pgcolumnar.set_options('ct', stripe_row_limit => 1000); + INSERT INTO ct SELECT '$ORIGIN'::timestamptz + (g % 24) * interval '1 hour', 'a'||g, 'a'||g FROM generate_series(1, 2000) g;" >/dev/null + q "SELECT pgcolumnar.create_rollup('r_ct', 'ct', 'ts', '1 day', '$ORIGIN', '{}', '{min,max,min,max}', '{sc,sc,su,su}');" >/dev/null + q "SELECT pgcolumnar.refresh_rollup('r_ct');" >/dev/null + q "INSERT INTO ct SELECT '$ORIGIN'::timestamptz + (g % 24) * interval '1 hour', 'B'||g, 'B'||g FROM generate_series(1, 2000) g;" >/dev/null + check "premise: the second refresh folds new rows into the existing bucket" \ + "$(q "SELECT pgcolumnar.refresh_rollup('r_ct');")|$(q "SELECT count(*) FROM r_ct;")" "2000|1" + check "min and max over a C column and an en_US column keep each column's own order across a refresh" \ + "$(q "SELECT min_sc || '|' || max_sc || '|' || min_su || '|' || max_su FROM r_ct;")" \ + "$(q "SELECT min(sc) || '|' || max(sc) || '|' || min(su) || '|' || max(su) FROM ct;")" + check "premise: the two collations disagree on these rows, so the arm can tell them apart" \ + "$(q "SELECT (min(sc) <> min(su))::text FROM ct;")" "true" +else + for _n in "premise: the second refresh folds new rows into the existing bucket" \ + "min and max over a C column and an en_US column keep each column's own order across a refresh" \ + "premise: the two collations disagree on these rows, so the arm can tell them apart"; do + check_skip "$_n" "SKIP $_n (no en_US collation)" "the server has no en_US collation" + done +fi + # Row numbers start at 1 (#1341). LAST IN THE SUITE ON PURPOSE: without the floor, # row_tid(0) crashed the backend (measured on PG17), and a crash mid-suite takes the # checks after it down with it, premises included. From 0aa42e7075736ecf0f47c67e411a8975d1675ca6 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 02:02:24 +0000 Subject: [PATCH 16/20] test: the collation premise compares both minimums in C, since the columns' collations differ (#1341) Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_rollup.py | 4 +++- test/rollup.sh | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/pytest/test_rollup.py b/test/pytest/test_rollup.py index 791972564..fcd8828a7 100644 --- a/test/pytest/test_rollup.py +++ b/test/pytest/test_rollup.py @@ -811,7 +811,9 @@ def test_min_and_max_keep_each_columns_collation(pgc_own_db, expect): expect.text(_one(conn, "SELECT min_sc || '|' || max_sc || '|' || min_su || '|' || max_su FROM r_ct"), _one(conn, "SELECT min(sc) || '|' || max(sc) || '|' || min(su) || '|' || max(su) FROM ct"), "min and max over a C column and an en_US column keep each column's own order across a refresh") - expect.text(str(_one(conn, "SELECT min(sc) <> min(su) FROM ct")), "True", + # Both sides in "C": the two columns' collations differ, so a bare comparison + # has no collation to use. Equality does not depend on which one is chosen. + expect.text(str(_one(conn, 'SELECT min(sc) COLLATE "C" <> min(su) COLLATE "C" FROM ct')), "True", "premise: the two collations disagree on these rows, so the arm can tell them apart") diff --git a/test/rollup.sh b/test/rollup.sh index ec7f70acf..e220a7cf2 100755 --- a/test/rollup.sh +++ b/test/rollup.sh @@ -645,7 +645,7 @@ if [ -n "$CT_COLL" ]; then "$(q "SELECT min_sc || '|' || max_sc || '|' || min_su || '|' || max_su FROM r_ct;")" \ "$(q "SELECT min(sc) || '|' || max(sc) || '|' || min(su) || '|' || max(su) FROM ct;")" check "premise: the two collations disagree on these rows, so the arm can tell them apart" \ - "$(q "SELECT (min(sc) <> min(su))::text FROM ct;")" "true" + "$(q "SELECT (min(sc) COLLATE \"C\" <> min(su) COLLATE \"C\")::text FROM ct;")" "true" else for _n in "premise: the second refresh folds new rows into the existing bucket" \ "min and max over a C column and an en_US column keep each column's own order across a refresh" \ From 66e96a1e2915e0af852482e94d02451ea0878c51 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 02:29:24 +0000 Subject: [PATCH 17/20] test: ledger rows for the rollup collation arms (#1341) Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 3 +++ test/check_ledger_budget.txt | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 11961a204..f141d707d 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2937,12 +2937,15 @@ rollup rollup each row group's TID range holds exactly that group's rows 15;16;1 rollup rollup expire retired the group in bucket D alone and kept the one that shares it 15;16;17;18;19 2026-10-14 expire retires nothing rollup rollup expire retired the group the delete thinned 15;16;17;18;19 2026-10-14 expire reports 0 whatever it retired;expire retires nothing rollup rollup expire retired the old row groups 15;16;17;18;19 2026-10-14 expire reports 0 whatever it retired;expire retires nothing +rollup rollup min and max over a C column and an en_US column keep each column's own order across a refresh 15;16;17;18;19 2026-10-14 the rollup fold of min and max compares in the partials' default collation rollup rollup nor its retired groups through rollup_source_retired (42501) 15;16;17;18;19 2026-10-14 rollup_source_retired without its require_caller_select guard rollup rollup pgcolumnar.vacuum gave the source's row groups new identities 15;16;17;18;19 2026-10-14 pgcolumnar.vacuum rewrites nothing rollup rollup premise: and the fixture has NULL-key and NULL-bucket groups 15;16;17;18;19 never - rollup rollup premise: the owner's rollup has group rows another role could reach 15;16;17;18;19 never - rollup rollup premise: the rollup functions exist 15;16;17;18;19 never - +rollup rollup premise: the second refresh folds new rows into the existing bucket 15;16;17;18;19 never - rollup rollup premise: the source holds every row 15;16;17;18;19 never - +rollup rollup premise: the two collations disagree on these rows, so the arm can tell them apart 15;16;17;18;19 never - rollup rollup rebase_rollup of a table that is not a rollup is refused (42704) 15;16;17;18;19 2026-10-14 rebase_rollup without its not-a-rollup check rollup rollup rebase_rollup records the source's current row groups 15;16;17;18;19 2026-10-13 rebase_rollup returning 0 rollup rollup refreshes from sessions in different time zones build the same rollup 15;16;17;18;19 2026-10-13 refresh_rollup bucketing with date_trunc, which follows the session's zone diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index d5de2922e..6ee71cac9 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -2695,4 +2695,15 @@ suites_not_covered 224 # never 2532 - 5 = 2527 # dated 815 + 5 = 820 # 2527 + 820 == 3347 -checks_never_observed_red 2527 +# +# #1341, a rollup's min and max keep each source column's collation (OffgridwithJD): +# 2527 -> 2529. Three new rollup checks. The arm is dated by the fold comparing in +# the partials' default collation (on an en_US database the C column's min became +# a1 where the recompute gives B1). Its two premises arrive never. Five-major runs +# on the files merged with #1359: rollup 85/85 on 15-19. +# +# ledger rows 3347 + 3 = 3350 +# never 2527 + 2 = 2529 +# dated 820 + 1 = 821 +# 2529 + 821 == 3350 +checks_never_observed_red 2529 From a22277caa220e6c1270eda3082512bea879fc6ff Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 19:58:27 +0000 Subject: [PATCH 18/20] test: the rollup refresh-count check loses the premise prefix, in both twins and the skip list (#1372 review) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VjTxfQJj5pnii9wzmPP9Ho --- test/check_ledger.tsv | 2 +- test/pytest/test_rollup.py | 2 +- test/rollup.sh | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index f141d707d..381d12c9a 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2943,7 +2943,6 @@ rollup rollup pgcolumnar.vacuum gave the source's row groups new identities 15;1 rollup rollup premise: and the fixture has NULL-key and NULL-bucket groups 15;16;17;18;19 never - rollup rollup premise: the owner's rollup has group rows another role could reach 15;16;17;18;19 never - rollup rollup premise: the rollup functions exist 15;16;17;18;19 never - -rollup rollup premise: the second refresh folds new rows into the existing bucket 15;16;17;18;19 never - rollup rollup premise: the source holds every row 15;16;17;18;19 never - rollup rollup premise: the two collations disagree on these rows, so the arm can tell them apart 15;16;17;18;19 never - rollup rollup rebase_rollup of a table that is not a rollup is refused (42704) 15;16;17;18;19 2026-10-14 rebase_rollup without its not-a-rollup check @@ -2966,6 +2965,7 @@ rollup rollup the rollup holds partials the delete below cannot reach 15;16;17;1 rollup rollup the rollup holds rows before the rewrite 15;16;17;18;19 never - rollup rollup the rollup table exists with the documented columns 15;16;17;18;19 never - rollup rollup the rollup to be dumped folds its source 15;16;17;18;19 never - +rollup rollup the second refresh folds new rows into the existing bucket 15;16;17;18;19 never - rollup rollup the second refresh of the NULL-key rollup folded the append 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's rollup rollup the zone rollups hold a row per key of the recompute 15;16;17;18;19 2026-10-13 refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's rollup rollup two refreshes started together fold the new rows once between them 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup without the self-conflicting lock;row_tid mapping 290 rows a block instead of the AM's diff --git a/test/pytest/test_rollup.py b/test/pytest/test_rollup.py index fcd8828a7..7ad7858ec 100644 --- a/test/pytest/test_rollup.py +++ b/test/pytest/test_rollup.py @@ -807,7 +807,7 @@ def test_min_and_max_keep_each_columns_collation(pgc_own_db, expect): " FROM generate_series(1, 2000) g") folded = _one(conn, "SELECT pgcolumnar.refresh_rollup('r_ct')") expect.text(f"{folded}|{_one(conn, 'SELECT count(*) FROM r_ct')}", "2000|1", - "premise: the second refresh folds new rows into the existing bucket") + "the second refresh folds new rows into the existing bucket") expect.text(_one(conn, "SELECT min_sc || '|' || max_sc || '|' || min_su || '|' || max_su FROM r_ct"), _one(conn, "SELECT min(sc) || '|' || max(sc) || '|' || min(su) || '|' || max(su) FROM ct"), "min and max over a C column and an en_US column keep each column's own order across a refresh") diff --git a/test/rollup.sh b/test/rollup.sh index e220a7cf2..02fb163e7 100755 --- a/test/rollup.sh +++ b/test/rollup.sh @@ -639,7 +639,9 @@ if [ -n "$CT_COLL" ]; then q "SELECT pgcolumnar.create_rollup('r_ct', 'ct', 'ts', '1 day', '$ORIGIN', '{}', '{min,max,min,max}', '{sc,sc,su,su}');" >/dev/null q "SELECT pgcolumnar.refresh_rollup('r_ct');" >/dev/null q "INSERT INTO ct SELECT '$ORIGIN'::timestamptz + (g % 24) * interval '1 hour', 'B'||g, 'B'||g FROM generate_series(1, 2000) g;" >/dev/null - check "premise: the second refresh folds new rows into the existing bucket" \ + # A check, not a premise (#1236; #1372 review): refresh_rollup's return value and + # the bucket count are both product output. + check "the second refresh folds new rows into the existing bucket" \ "$(q "SELECT pgcolumnar.refresh_rollup('r_ct');")|$(q "SELECT count(*) FROM r_ct;")" "2000|1" check "min and max over a C column and an en_US column keep each column's own order across a refresh" \ "$(q "SELECT min_sc || '|' || max_sc || '|' || min_su || '|' || max_su FROM r_ct;")" \ @@ -647,7 +649,7 @@ if [ -n "$CT_COLL" ]; then check "premise: the two collations disagree on these rows, so the arm can tell them apart" \ "$(q "SELECT (min(sc) COLLATE \"C\" <> min(su) COLLATE \"C\")::text FROM ct;")" "true" else - for _n in "premise: the second refresh folds new rows into the existing bucket" \ + for _n in "the second refresh folds new rows into the existing bucket" \ "min and max over a C column and an en_US column keep each column's own order across a refresh" \ "premise: the two collations disagree on these rows, so the arm can tell them apart"; do check_skip "$_n" "SKIP $_n (no en_US collation)" "the server has no en_US collation" From f07760fa667a5329fecfb32fc769500dfaef1f79 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 20:03:04 +0000 Subject: [PATCH 19/20] test: the refresh-count check is dated by refresh_rollup returning one more than it folded (#1372 review) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VjTxfQJj5pnii9wzmPP9Ho --- test/check_ledger.tsv | 32 ++++++++++++++++---------------- test/check_ledger_budget.txt | 14 +++++++++++++- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 381d12c9a..a543b0391 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2871,13 +2871,13 @@ rollup rollup a delete in buckets with no history, next to retired ones, folds: rollup rollup a group_by column the source does not have is refused 15;16;17;18;19 2026-10-14 create_rollup without its group_by column check rollup rollup a refresh after a delete is not refused 15;16;17;18;19 2026-10-13 refresh_rollup refusing any change but appends (phase 1, main before this branch) rollup rollup a refresh after a small append reads under a fifth of the source 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup's per-group subquery without OFFSET 0, so it becomes a join and reads every group -rollup rollup a refresh after appends folds exactly the appended rows, late ones included 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's -rollup rollup a refresh after retention folds nothing, refuses nothing, and keeps every bucket 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup not dropping groups retention retired +rollup rollup a refresh after appends folds exactly the appended rows, late ones included 15;16;17;18;19 2026-10-14 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup returns one more than it folded;row_tid mapping 290 rows a block instead of the AM's +rollup rollup a refresh after retention folds nothing, refuses nothing, and keeps every bucket 15;16;17;18;19 2026-10-14 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup not dropping groups retention retired;refresh_rollup returns one more than it folded rollup rollup a refresh after the source was rewritten is refused, naming rebase_rollup 15;16;17;18;19 2026-10-14 pgcolumnar.vacuum rewrites nothing;refresh_rollup without the source identity check rollup rollup a refresh at REPEATABLE READ is refused 15;16;17;18;19 2026-10-13 refresh_rollup at any isolation level -rollup rollup a refresh folds a bulk append 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's +rollup rollup a refresh folds a bulk append 15;16;17;18;19 2026-10-14 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup returns one more than it folded;row_tid mapping 290 rows a block instead of the AM's rollup rollup a refresh in a zone with a DST change recomputes a whole day bucket 15;16;17;18;19 2026-10-13 refresh_rollup adding the width in the session's zone;refresh_rollup recomputing every bucket once any group changed;refresh_rollup recording new groups without their time range;refresh_rollup refusing any change but appends (phase 1, main before this branch) -rollup rollup a refresh with nothing new folds nothing 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group +rollup rollup a refresh with nothing new folds nothing 15;16;17;18;19 2026-10-14 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup returns one more than it folded rollup rollup a restore keeps the rollup's definition 15;16;17;18;19 2026-10-14 the tree before #1377: pgcolumnar.rollup not dumped, no source_oid rollup rollup a role cannot name another role's table as its rollup's partials (42501), and that table survives its view's drop (#1376) 15;16;17;18;19 2026-10-14 the rollup trigger with no ownership, kind or no-change checks (8a7fab2, #1406 review) rollup rollup a role cannot tie a rollup row to a table it cannot read (42501, #1376) 15;16;17;18;19 2026-10-14 a rollup's view, partials and source with no dependency between them, and no catalog cleanup (main before #1376's fix) @@ -2892,7 +2892,7 @@ rollup rollup a ts_column that is not timestamptz is refused 15;16;17;18;19 2026 rollup rollup a width with months is refused 15;16;17;18;19 2026-10-13 create_rollup without the months refusal rollup rollup after a delete and an append that straddles its bucket, a refresh counts the new rows once 15;16;17;18;19 2026-10-13 rebase_rollup recording no time ranges;refresh_rollup folding new groups' rows into dirty buckets too;refresh_rollup recomputing every bucket once any group changed;refresh_rollup refusing any change but appends (phase 1, main before this branch) rollup rollup after a delete of rows whose ts is NULL the NULL-bucket rollup equals a full recompute 15;16;17;18;19 2026-10-13 refresh_rollup never dirtying the NULL bucket;refresh_rollup recomputing every bucket once any group changed;refresh_rollup refusing any change but appends (phase 1, main before this branch) -rollup rollup after a rebase the next refresh folds exactly the appended rows 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;row_tid mapping 290 rows a block instead of the AM's +rollup rollup after a rebase the next refresh folds exactly the appended rows 15;16;17;18;19 2026-10-14 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup returns one more than it folded;row_tid mapping 290 rows a block instead of the AM's rollup rollup after a restore into another database its owner's refresh is refused (55000), the rollup untouched 15;16;17;18;19 2026-10-14 refresh_rollup without the source_oid check: a restored source under the recorded storage id is folded twice;the tree before #1377: pgcolumnar.rollup not dumped, no source_oid rollup rollup after a restore into the same database its refresh is refused (55000), the rollup untouched 15;16;17;18;19 2026-10-14 refresh_rollup without the source_oid check: a restored source under the recorded storage id is folded twice;the tree before #1377: pgcolumnar.rollup not dumped, no source_oid rollup rollup after a restore the rollup's source cannot be dropped while the rollup exists (2BP01) 15;16;17;18;19 2026-10-14 the tree without #1406's dependency trigger: a restored rollup does not depend on its source @@ -2904,7 +2904,7 @@ rollup rollup an aggregate column the source does not have is refused 15;16;17;1 rollup rollup an aggregate that does not combine is refused 15;16;17;18;19 2026-10-13 create_rollup without the five-function allow-list rollup rollup and DROP TABLE ... CASCADE takes the rollup's view, partials and catalog rows with it (#1376) 15;16;17;18;19 2026-10-14 a rollup's view, partials and source with no dependency between them, and no catalog cleanup (main before #1376's fix) rollup rollup and after a delete the rollup equals a full recompute 15;16;17;18;19 2026-10-13 rebase_rollup recording no time ranges;refresh_rollup recomputing every bucket once any group changed;refresh_rollup refusing any change but appends (phase 1, main before this branch) -rollup rollup and after retention an append still folds 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup not dropping groups retention retired;row_tid mapping 290 rows a block instead of the AM's +rollup rollup and after retention an append still folds 15;16;17;18;19 2026-10-14 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup not dropping groups retention retired;refresh_rollup returns one more than it folded;row_tid mapping 290 rows a block instead of the AM's rollup rollup and after the bulk append the rollup equals a full recompute 15;16;17;18;19 2026-10-13 refresh_rollup combining min with greatest;refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup recording the folded groups from a fresh read after the fold;row_tid mapping 290 rows a block instead of the AM's rollup rollup and after the first refresh the rollup equals a full recompute 15;16;17;18;19 2026-10-13 refresh_rollup folding only the first new group rollup rollup and after the rebase the rollup equals a full recompute 15;16;17;18;19 2026-10-13 rebase_rollup clearing the rollup's rows;refresh_rollup combining min with greatest;refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup recording the folded groups from a fresh read after the fold;row_tid mapping 290 rows a block instead of the AM's @@ -2917,11 +2917,11 @@ rollup rollup and leaves the rollup as it was 15;16;17;18;19 2026-10-13 refresh_ rollup rollup and leaves the rollup's rows alone 15;16;17;18;19 2026-10-13 rebase_rollup clearing the rollup's rows rollup rollup and no refused rollup left a table behind 15;16;17;18;19 2026-10-14 create_rollup allowing * with any function;create_rollup without its timestamptz check on ts_column;create_rollup without the equal-length check;create_rollup without the months refusal rollup rollup and reads what a recompute gives 15;16;17;18;19 2026-10-13 refresh_rollup bucketing with date_trunc, which follows the session's zone;rollup_source_groups without SECURITY DEFINER;the rollup catalogs without their grants -rollup rollup and refreshes it 15;16;17;18;19 2026-10-13 rollup_source_groups without SECURITY DEFINER;the rollup catalogs without their grants +rollup rollup and refreshes it 15;16;17;18;19 2026-10-14 refresh_rollup returns one more than it folded;rollup_source_groups without SECURITY DEFINER;the rollup catalogs without their grants rollup rollup and that is not zero 15;16;17;18;19 never - -rollup rollup and the next refresh, with nothing changed since, recomputes nothing 15;16;17;18;19 2026-10-13 refresh_rollup not recounting a changed group that is still there;refresh_rollup refusing any change but appends (phase 1, main before this branch) +rollup rollup and the next refresh, with nothing changed since, recomputes nothing 15;16;17;18;19 2026-10-14 refresh_rollup not recounting a changed group that is still there;refresh_rollup refusing any change but appends (phase 1, main before this branch);refresh_rollup returns one more than it folded rollup rollup and the recent buckets equal a recompute, while the expired ones keep their history 15;16;17;18;19 2026-10-13 refresh_rollup bucketing with date_trunc, which follows the session's zone;refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup not dropping groups retention retired;row_tid mapping 290 rows a block instead of the AM's -rollup rollup and the refresh after the rewrite recomputes nothing 15;16;17;18;19 2026-10-13 refresh_rollup not forgetting a changed group that is gone;refresh_rollup refusing any change but appends (phase 1, main before this branch) +rollup rollup and the refresh after the rewrite recomputes nothing 15;16;17;18;19 2026-10-14 refresh_rollup not forgetting a changed group that is gone;refresh_rollup refusing any change but appends (phase 1, main before this branch);refresh_rollup returns one more than it folded rollup rollup and the refresh rewrote no partial outside the changed groups' time range 15;16;17;18;19 2026-10-13 refresh_rollup recomputing every bucket once any group changed rollup rollup and the refusal names rebase_rollup 15;16;17;18;19 2026-10-14 refresh_rollup without the source_oid check: a restored source under the recorded storage id is folded twice;the tree before #1377: pgcolumnar.rollup not dumped, no source_oid rollup rollup and the restored rollup equals a full recompute 15;16;17;18;19 2026-10-14 refresh_rollup without the source_oid check: a restored source under the recorded storage id is folded twice;the tree before #1377: pgcolumnar.rollup not dumped, no source_oid @@ -2952,12 +2952,12 @@ rollup rollup row_tid refuses a row number below 1 (22003), and row 1 is accepte rollup rollup the database holding a rollup dumps 15;16;17;18;19 never - rollup rollup the dump restores into another database 15;16;17;18;19 never - rollup rollup the dump restores into its own database after DROP EXTENSION, again under the recorded storage id 15;16;17;18;19 never - -rollup rollup the first refresh folds every row 15;16;17;18;19 2026-10-13 refresh_rollup folding only the first new group -rollup rollup the history rollup folded every row 15;16;17;18;19 2026-10-13 refresh_rollup folding only the first new group +rollup rollup the first refresh folds every row 15;16;17;18;19 2026-10-14 refresh_rollup folding only the first new group;refresh_rollup returns one more than it folded +rollup rollup the history rollup folded every row 15;16;17;18;19 2026-10-14 refresh_rollup folding only the first new group;refresh_rollup returns one more than it folded rollup rollup the history rollup holds old buckets 15;16;17;18;19 never - -rollup rollup the next refresh folds the rows committed during the last one, once 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup recording the folded groups from a fresh read after the fold;row_tid mapping 290 rows a block instead of the AM's +rollup rollup the next refresh folds the rows committed during the last one, once 15;16;17;18;19 2026-10-14 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup recording the folded groups from a fresh read after the fold;refresh_rollup returns one more than it folded;row_tid mapping 290 rows a block instead of the AM's rollup rollup the owner reads its source's groups through the definer helpers 15;16;17;18;19 never - -rollup rollup the paused refresh folded what was there before it began 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's +rollup rollup the paused refresh folded what was there before it began 15;16;17;18;19 2026-10-14 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup returns one more than it folded;row_tid mapping 290 rows a block instead of the AM's rollup rollup the refresh after a small append read the source at all 15;16;17;18;19 never - rollup rollup the restored rollup reads as a recompute of the restored source 15;16;17;18;19 never - rollup rollup the restored source holds the dumped rows under the storage id recorded for the old one 15;16;17;18;19 never - @@ -2965,10 +2965,10 @@ rollup rollup the rollup holds partials the delete below cannot reach 15;16;17;1 rollup rollup the rollup holds rows before the rewrite 15;16;17;18;19 never - rollup rollup the rollup table exists with the documented columns 15;16;17;18;19 never - rollup rollup the rollup to be dumped folds its source 15;16;17;18;19 never - -rollup rollup the second refresh folds new rows into the existing bucket 15;16;17;18;19 never - -rollup rollup the second refresh of the NULL-key rollup folded the append 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's +rollup rollup the second refresh folds new rows into the existing bucket 15;16;17;18;19 2026-10-14 refresh_rollup returns one more than it folded +rollup rollup the second refresh of the NULL-key rollup folded the append 15;16;17;18;19 2026-10-14 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup returns one more than it folded;row_tid mapping 290 rows a block instead of the AM's rollup rollup the zone rollups hold a row per key of the recompute 15;16;17;18;19 2026-10-13 refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's -rollup rollup two refreshes started together fold the new rows once between them 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup without the self-conflicting lock;row_tid mapping 290 rows a block instead of the AM's +rollup rollup two refreshes started together fold the new rows once between them 15;16;17;18;19 2026-10-14 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup returns one more than it folded;refresh_rollup without the self-conflicting lock;row_tid mapping 290 rows a block instead of the AM's serializable serializable a serializable read of a columnar table holds SIRead locks on columnar's row_group and delete_vector catalog 15;16;17;18;19 2026-10-15 PgColumnarCatalogSnapshot returns a SNAPSHOT_SELF copy, so a serializable read takes no SIRead lock on columnar's catalog serializable serializable at REPEATABLE READ both doctors' transactions commit and no one stays on call, in one row group and in two, as on heap 15;16;17;18;19 never - serializable serializable at REPEATABLE READ both ward transactions commit and two are on call, by a scan and by an index, as on heap 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 6ee71cac9..f83a54eed 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -2706,4 +2706,16 @@ suites_not_covered 224 # never 2527 + 2 = 2529 # dated 820 + 1 = 821 # 2529 + 821 == 3350 -checks_never_observed_red 2529 +# +# #1372 review (OffgridwithJD): 2529 -> 2528. "the second refresh folds new rows into the +# existing bucket" loses the premise prefix: refresh_rollup's return value is product +# output. refresh_rollup returning one more than it folded reddens it, dating it, and +# fifteen other rollup arms that count what a refresh folded, all already dated. +# Five-major runs: rollup 85/85 on 15-19, and 108/108 on PG18 after the rebase onto +# d5c0a5e, which brought in 23 checks (#1391's row_tid arm still runs last). +# +# ledger rows 3350 = 3350 +# never 2529 - 1 = 2528 +# dated 821 + 1 = 822 +# 2528 + 822 == 3350 +checks_never_observed_red 2528 From 19b880c2644fd63ca0623fca44f3ef7994aa816f Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 22:10:19 +0000 Subject: [PATCH 20/20] test: a savepoint rolled back, or a statement failed, after a flush (#1341) A row group flushed inside a savepoint is a catalog row written under the subtransaction, so the abort must hide it, and the savepoint's buffered rows and delete marks must be thrown away. No test rolled back a savepoint after a flush and then read the table, so item 40 of #1341 had no test. savepoint_flush and its pytest twin compare a columnar table with heap, with row groups of 1000 rows: a flushed INSERT rolled back to a savepoint, at READ COMMITTED and at REPEATABLE READ; the same unique keys inserted again after ROLLBACK TO; an UPDATE of every row rolled back to a savepoint; and an INSERT and an UPDATE that fail after flushing, alone and inside a savepoint. Four readers check each result, inside the transaction and after COMMIT: a plain sequential scan, the custom scan, the vectorized count and the primary key's index. Premises pin each reader's plan and show each write flushed. 28 checks, 28/28 in both twins on PG15-19. Four mutants of the abort path date 20 of them on PG17, and both twins fail first on the same check with the same values under each. Co-Authored-By: Claude Opus 5.5 Squashed for the rebase; it replaces: - test: savepoint_flush's bookkeeping restated on main 2d1a77f0, and the budget says the dropped mutants crash only on an assert build (#1341) - test: savepoint_flush's UPDATE premises measure the difference from BEGIN, so the twin's can fail (#1341) - test: savepoint_flush runs each shape on fresh tables, and both UPDATEs prove they flushed (#1341) - test: a savepoint rolled back, or a statement failed, after a flush (#1341) --- test/check_ledger.tsv | 30 +++ test/check_ledger_budget.txt | 25 ++- test/pytest/TESTS.md | 11 ++ test/pytest/expected_tests.txt | 3 +- test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_savepoint_flush.py | 277 ++++++++++++++++++++++++++++ test/run_all_versions.sh | 1 + test/savepoint_flush.sh | 246 ++++++++++++++++++++++++ 8 files changed, 592 insertions(+), 2 deletions(-) create mode 100644 test/pytest/test_savepoint_flush.py create mode 100755 test/savepoint_flush.sh diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index a543b0391..40f578002 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2969,6 +2969,36 @@ rollup rollup the second refresh folds new rows into the existing bucket 15;16;1 rollup rollup the second refresh of the NULL-key rollup folded the append 15;16;17;18;19 2026-10-14 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup returns one more than it folded;row_tid mapping 290 rows a block instead of the AM's rollup rollup the zone rollups hold a row per key of the recompute 15;16;17;18;19 2026-10-13 refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's rollup rollup two refreshes started together fold the new rows once between them 15;16;17;18;19 2026-10-14 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup returns one more than it folded;refresh_rollup without the self-conflicting lock;row_tid mapping 290 rows a block instead of the AM's +savepoint_flush savepoint_flush a savepoint's INSERT runs, and ROLLBACK TO it, on both tables 15;16;17;18;19 never - +savepoint_flush savepoint_flush after ROLLBACK TO, in the same transaction, every reader sees the rows as they were, as heap does 15;16;17;18;19 2026-10-15 every delete_vector read is under SnapshotAny, so marks an aborted savepoint flushed stay applied;the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList) +savepoint_flush savepoint_flush after ROLLBACK TO, in the same transaction, no reader sees the savepoint's rows, as heap does 15;16;17;18;19 2026-10-15 the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList) +savepoint_flush savepoint_flush after ROLLBACK TO, the same unique keys insert again in the same transaction, as on heap 15;16;17;18;19 2026-10-15 the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList) +savepoint_flush savepoint_flush an INSERT that fails after flushing is refused on both tables 15;16;17;18;19 never - +savepoint_flush savepoint_flush an UPDATE of every row in a savepoint runs, and ROLLBACK TO it, on both tables 15;16;17;18;19 never - +savepoint_flush savepoint_flush and after COMMIT, each key is there once, as heap does 15;16;17;18;19 2026-10-15 the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList) +savepoint_flush savepoint_flush and after COMMIT, no reader sees them, as heap does 15;16;17;18;19 2026-10-15 the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList) +savepoint_flush savepoint_flush and after COMMIT, only the rows written after the failed INSERT are there, as heap does 15;16;17;18;19 2026-10-15 the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList);the savepoint abort promotes its buffered rows and delete marks to the parent instead of discarding them +savepoint_flush savepoint_flush and after COMMIT, the rows are as they were plus the later UPDATE, as heap does 15;16;17;18;19 2026-10-15 every delete_vector read is under SnapshotAny, so marks an aborted savepoint flushed stay applied;the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList);the savepoint abort promotes its buffered rows and delete marks to the parent instead of discarding them +savepoint_flush savepoint_flush and after COMMIT, the rows are as they were, as heap does 15;16;17;18;19 2026-10-15 every delete_vector read is under SnapshotAny, so marks an aborted savepoint flushed stay applied;the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList) +savepoint_flush savepoint_flush and every reader sees each key once, with the second insert's values, as heap does 15;16;17;18;19 2026-10-15 the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList) +savepoint_flush savepoint_flush and in the same transaction every reader sees only the rows written after it, as heap does 15;16;17;18;19 2026-10-15 the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList);the savepoint abort promotes its buffered rows and delete marks to the parent instead of discarding them +savepoint_flush savepoint_flush and in the same transaction every reader sees the rows as they were, plus the later UPDATE, as heap does 15;16;17;18;19 2026-10-15 every delete_vector read is under SnapshotAny, so marks an aborted savepoint flushed stay applied;the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList);the savepoint abort promotes its buffered rows and delete marks to the parent instead of discarding them +savepoint_flush savepoint_flush and no reader sees any of its rows, as heap does 15;16;17;18;19 2026-10-15 the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList) +savepoint_flush savepoint_flush at REPEATABLE READ, a savepoint's INSERT runs, and ROLLBACK TO it, on both tables 15;16;17;18;19 never - +savepoint_flush savepoint_flush at REPEATABLE READ, after ROLLBACK TO, no reader sees the savepoint's rows, as heap does 15;16;17;18;19 2026-10-15 the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList) +savepoint_flush savepoint_flush at REPEATABLE READ, inside the savepoint, every reader saw the INSERT, as heap does 15;16;17;18;19 never - +savepoint_flush savepoint_flush in a savepoint, an INSERT that fails after flushing raises its error and nothing else, on both tables 15;16;17;18;19 never - +savepoint_flush savepoint_flush in a savepoint, an UPDATE of every row that fails at the last raises its error and nothing else, on both tables 15;16;17;18;19 2026-10-15 the row-group list is read under SnapshotAny, so groups an aborted savepoint flushed stay listed (PgColumnarReadRowGroupList) +savepoint_flush savepoint_flush inside the savepoint, every reader saw the INSERT, as heap does 15;16;17;18;19 never - +savepoint_flush savepoint_flush inside the savepoint, the UPDATE changed each row once, as heap does 15;16;17;18;19 never - +savepoint_flush savepoint_flush the failing INSERT's shape flushes row groups before its 3500th row 15;16;17;18;19 never - +savepoint_flush savepoint_flush the failing UPDATE flushed new versions and delete marks before it failed 15;16;17;18;19 never - +savepoint_flush savepoint_flush the idx reader reads the primary key's index 15;16;17;18;19 never - +savepoint_flush savepoint_flush the savepoint's INSERT flushed row groups before ROLLBACK TO 15;16;17;18;19 never - +savepoint_flush savepoint_flush the savepoint's UPDATE flushed new versions and delete marks before ROLLBACK TO 15;16;17;18;19 never - +savepoint_flush savepoint_flush the scan reader is the custom scan with no vectorized aggregate 15;16;17;18;19 never - +savepoint_flush savepoint_flush the seq reader is a plain sequential scan, not the custom scan 15;16;17;18;19 never - +savepoint_flush savepoint_flush the vec reader is a vectorized aggregate 15;16;17;18;19 never - serializable serializable a serializable read of a columnar table holds SIRead locks on columnar's row_group and delete_vector catalog 15;16;17;18;19 2026-10-15 PgColumnarCatalogSnapshot returns a SNAPSHOT_SELF copy, so a serializable read takes no SIRead lock on columnar's catalog serializable serializable at REPEATABLE READ both doctors' transactions commit and no one stays on call, in one row group and in two, as on heap 15;16;17;18;19 never - serializable serializable at REPEATABLE READ both ward transactions commit and two are on call, by a scan and by an index, as on heap 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index f83a54eed..60dbfb047 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -2718,4 +2718,27 @@ suites_not_covered 224 # never 2529 - 1 = 2528 # dated 821 + 1 = 822 # 2528 + 822 == 3350 -checks_never_observed_red 2528 +# +# #1341, a savepoint rolled back, or a statement failed, after a flush +# (jdatcmd): 2528 -> 2544. savepoint_flush enters with all 30 checks from +# five-major runs (30/30 on each of 15-19). Every shape starts from fresh tables, +# so each date below was observed with that shape alone (OffgridwithJD, review of +# #1430). 14 are dated on PG17 by three mutants: +# - the row-group list read under SnapshotAny: every arm that reads the table +# after an aborted savepoint, and the failing UPDATE's error arm, whose +# transaction then fails on the rows that came back; +# - every delete_vector read under SnapshotAny: the rolled-back UPDATE and the +# failing UPDATE, whose marks are now flushed before it fails; +# - the abort promoting its buffers to the parent instead of discarding them: +# the failing INSERT and failing UPDATE arms. +# The two "abort keeps its buffer" mutants of the first version are not used: on +# an assert-enabled build, after #1403, both trip a flushedUnder assertion at commit +# and crash the server, so their reds are the crash. On a build without asserts they +# run and redden 6 and 3 arms (OffgridwithJD, review of #1430). The 16 never are the premises, the +# "it ran" arms and the reads inside a savepoint. +# +# ledger rows 3350 + 30 = 3380 +# never 2528 + 16 = 2544 +# dated 822 + 14 = 836 +# 2544 + 836 == 3380 +checks_never_observed_red 2544 diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 52480759f..87009bb81 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -174,6 +174,7 @@ behaviour, the source of that number is named. - [126. test_objstore_crlf.py: a CR or LF in an object-store URL is refused before it is sent](#126-test_objstore_crlfpy-a-cr-or-lf-in-an-object-store-url-is-refused-before-it-is-sent) - [127. test_objstore_allowlist.py: the endpoint allow-list, empty by default](#127-test_objstore_allowlistpy-the-endpoint-allow-list-empty-by-default) - [128. test_objstore_userinfo.py: userinfo in a URL is refused on every entry point](#128-test_objstore_userinfopy-userinfo-in-a-url-is-refused-on-every-entry-point) +- [129. test_savepoint_flush.py: a savepoint rolled back after a flush](#129-test_savepoint_flushpy-a-savepoint-rolled-back-after-a-flush) ## 1. How to read a test in here @@ -7153,3 +7154,13 @@ The controls are what make the arms mean anything. With no endpoint configured e | `test_userinfo_in_a_url_is_refused_by_the_parse_guard` | read_parquet and export_parquet over http, export_arrow, and the s3 and gs bucket authority, read and write: 22023, naming userinfo | | `test_a_clean_url_reaches_the_refusal_it_should` | a clean s3 URL demands an endpoint (naming `AWS_ENDPOINT_URL`), gs demands a credential, a malformed URL and an `@` in a key are not userinfo, a clean non-allowed endpoint is still 42501, and the backend is alive | | `test_userinfo_in_a_configured_endpoint_is_refused` | both endpoint shapes, `u:p@` and `user@`, are 22023 naming userinfo and the endpoint; a clean endpoint gets past the parse | + +## 129. test_savepoint_flush.py: a savepoint rolled back after a flush + +The pytest twin of `test/savepoint_flush.sh` (#1341), every check by the shell's name. A row group flushed inside a savepoint is a catalog row written under the subtransaction, so the abort must hide it; rows still buffered, and buffered delete marks, must be thrown away. With row groups of 1000 rows, each shape runs on a columnar table and on heap, and heap is the answer: a flushed `INSERT` rolled back to a savepoint, at READ COMMITTED and at REPEATABLE READ; the same unique keys inserted again after `ROLLBACK TO`; an `UPDATE` of every row rolled back to a savepoint; and an `INSERT` and an `UPDATE` that fail after flushing, alone and inside a savepoint. Four readers compare each result, inside the transaction and after `COMMIT`: a plain sequential scan, the custom scan, the vectorized count and the primary key's index. Every shape starts from fresh tables, so one that goes wrong cannot fail the ones after it. Premises check that each reader takes its path and that each write really flushed: row groups for the INSERTs, and for both UPDATEs the inserts into `row_group` and `delete_vector` that `pg_stat_xact_all_tables` counts between `BEGIN` and `ROLLBACK TO`, two or more of each. The failing UPDATE reads its own table half-way through, through `sp_probe`, which flushes its delete marks before it fails. Each transaction runs one statement at a time and goes on after an error, as `psql` does with `ON_ERROR_STOP` off. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_a_savepoint_rolled_back_after_a_flush` | the reader paths, the flushes, and every shape answering as heap does inside the transaction and after it | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 786ac5aa1..59af69adc 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -721,4 +721,5 @@ guard_tests 438 # test_analyze_function.py, by collection over the cluster files on this tree. # cluster_tests 598 -> 599 for #1341's test_min_and_max_keep_each_columns_collation in # test_rollup.py, by collection over the cluster files on this tree. -cluster_tests 599 +# cluster_tests 599 -> 600 for #1341's test_savepoint_flush.py, one test, by collection +cluster_tests 600 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index e28cec9b5..ffa87bfdf 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -150,6 +150,7 @@ "rewrite_storage_oid", "rls_direct_storage", "rollup", + "savepoint_flush", "scan_decode_cost", "serializable", "settings_reference", diff --git a/test/pytest/test_savepoint_flush.py b/test/pytest/test_savepoint_flush.py new file mode 100644 index 000000000..5f3e2fcbf --- /dev/null +++ b/test/pytest/test_savepoint_flush.py @@ -0,0 +1,277 @@ +"""Rolling back to a savepoint, or failing a statement, after rows have already +been flushed into row groups (#1341). The pytest twin of `test/savepoint_flush.sh`, +every check by the shell's name. + +A flushed group is a catalog row written under the subtransaction, so the abort +must hide it; rows still buffered, and buffered delete marks, must be thrown away. + +Each shape runs on a columnar table and on heap, and heap is the answer. The +columnar table writes row groups of 1000 rows, so every write crosses several +flushes. Each transaction runs one statement at a time and goes on after an error, +as psql does with ON_ERROR_STOP off, and records labelled values (label=value). +The two tables are compared label by label, inside the transaction and after it. +Both tables run every transaction, so a columnar failure cannot leave heap +untouched and the comparison vacuous. Every shape starts from fresh tables, so a +shape that goes wrong cannot make the ones after it fail for its rows. +""" +import psycopg + +AMS = ("pgcolumnar", "heap") +SESSION = ["SET pgcolumnar.stripe_row_limit = 1000", "SET statement_timeout = '60s'"] + +# What every reader sees: a plain sequential scan through the table AM, the custom +# scan without vectorization, the vectorized count, and the primary key's index. +# Each is the settings that choose its path and its query. +READERS = { + "seq": (["SET pgcolumnar.enable_custom_scan = off"], + "SELECT 'seq=' || count(*) || '|' || coalesce(sum(id), 0) || '|' || coalesce(sum(v), 0) FROM %t"), + "scan": (["SET pgcolumnar.enable_vectorization = off"], + "SELECT 'scan=' || count(*) || '|' || coalesce(sum(id), 0) || '|' || coalesce(sum(v), 0) FROM %t"), + "vec": (["SET pgcolumnar.enable_vectorization = on"], + "SELECT 'vec=' || count(*) || '|' || coalesce(sum(id), 0) FROM %t"), + "idx": (["SET pgcolumnar.enable_custom_scan = off", "SET enable_seqscan = off", + "SET enable_bitmapscan = off"], + "SELECT 'idx=' || count(*) || '|' || coalesce(sum(v), 0) FROM %t WHERE id > 1000"), +} +RESET = ["RESET pgcolumnar.enable_custom_scan", "RESET pgcolumnar.enable_vectorization", + "RESET enable_seqscan", "RESET enable_bitmapscan"] +# This transaction's inserts into row_group and delete_vector. pg_stat_xact_all_tables +# still counts a subtransaction's inserts after it aborts, so it shows what a +# rolled-back write flushed. +# Read once after BEGIN ("b") and once after ROLLBACK TO ("a"); the premise is the +# difference, since this one connection's counters include every earlier shape. +def _catalog_ins(tag): + return [f"SELECT 'rg_{tag}=' || n_tup_ins FROM pg_stat_xact_all_tables " + "WHERE relid = 'pgcolumnar.row_group'::regclass", + f"SELECT 'dv_{tag}=' || n_tup_ins FROM pg_stat_xact_all_tables " + "WHERE relid = 'pgcolumnar.delete_vector'::regclass"] +# Reads the table at row 2500 -- a scan starting flushes the delete marks buffered +# so far -- and divides by zero at row 5000. +PROBE = ("CREATE FUNCTION sp_probe(t regclass, id int) RETURNS int LANGUAGE plpgsql AS $$ " + "BEGIN IF id = 2500 THEN EXECUTE format('SELECT count(*) FROM %s', t); END IF; " + "RETURN 1 / (5000 - id) * 0; END $$") +GROUPS = ("SELECT 'groups=' || count(*) FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('sp_pgcolumnar')") + + +def _reads(prefix=""): + out = [] + for name, (sets, query) in READERS.items(): + out += sets + [query.replace(f"'{name}=", f"'{prefix}{name}=")] + RESET + return out + + +READS = _reads() +READS_IN = _reads("in_") + + +def _xact(conn, statements): + """Run `statements` (with %t for the table) on both tables, one at a time. + + -> {am: (lines, errors)}: each row's first value, and each error's first line. + """ + out = {} + for am in AMS: + lines, errors = [], [] + for sql in SESSION + statements: + try: + with conn.cursor() as cur: + cur.execute(sql.replace("%t", f"sp_{am}")) + if cur.description: + lines += [str(r[0]) for r in cur.fetchall()] + except psycopg.Error as exc: + errors.append(str(exc).splitlines()[0]) + if conn.info.transaction_status != psycopg.pq.TransactionStatus.IDLE: + conn.execute("ROLLBACK") # a psql session would have ended here + out[am] = (lines, errors) + return out + + +def _pick(out, am, label): + return next((ln.split("=", 1)[1] for ln in out[am][0] if ln.startswith(f"{label}=")), "") + + +def _same(out, *labels): + diff = "" + for label in labels: + c, h = _pick(out, "pgcolumnar", label), _pick(out, "heap", label) + if not h: + diff += f" {label}: heap printed nothing ({out['heap']})" + elif c != h: + diff += f" {label}: columnar=[{c}] heap=[{h}]" + return diff or "same" + + +def _errors(out, msg=None): + bad = "" + want = 1 if msg else 0 + for am in AMS: + errs = out[am][1] + if len(errs) != want or (msg and msg not in errs[0]): + bad += f" {am}: {len(errs)} errors, first [{errs[0] if errs else ''}]" + return bad or "as expected" + + +def _grew(out): + g = [int(v.split("=", 1)[1]) for v in out["pgcolumnar"][0] if v.startswith("groups=")] + return "yes" if len(g) >= 2 and g[1] - g[0] >= 2 else f"no (groups read: {g})" + + +def _flushed(out): + """yes when both catalogs gained two rows or more between the b and a readings.""" + vals = {k: _pick(out, "pgcolumnar", k) for k in ("rg_b", "dv_b", "rg_a", "dv_a")} + if not all(v.isdigit() for v in vals.values()): + return f"no (readings {vals})" + rg, dv = int(vals["rg_a"]) - int(vals["rg_b"]), int(vals["dv_a"]) - int(vals["dv_b"]) + return "yes" if rg >= 2 and dv >= 2 else f"no (row_group inserts [{rg}], delete_vector inserts [{dv}])" + + +def _fresh(conn, n=2000): + """Both tables dropped and made again, holding rows 1..n.""" + with conn.cursor() as cur: + for sql in SESSION: + cur.execute(sql) + for am in AMS: + cur.execute(f"DROP TABLE IF EXISTS sp_{am}") + cur.execute(f"CREATE TABLE sp_{am} (id int PRIMARY KEY, v int) USING {am}") + cur.execute(f"INSERT INTO sp_{am} SELECT g, 0 FROM generate_series(1, {n}) g") + + +def _plan(conn, reader): + sets, query = READERS[reader] + with conn.cursor() as cur: + for sql in SESSION + sets: + cur.execute(sql) + cur.execute("EXPLAIN (COSTS OFF) " + query.replace("%t", "sp_pgcolumnar")) + plan = "\n".join(r[0] for r in cur.fetchall()) + for sql in RESET: + cur.execute(sql) + return plan + + +def _count(plan, *words): + return sum(1 for ln in plan.splitlines() if any(w in ln for w in words)) + + +def test_a_savepoint_rolled_back_after_a_flush(pgc_conn, expect): + pgc_conn.execute(PROBE) + _fresh(pgc_conn) + + # ---- the readers take the paths they are named for ---- + p = _plan(pgc_conn, "seq") + expect.text(f"{_count(p, 'Seq Scan')}|{_count(p, 'Custom Scan')}", "1|0", + "the seq reader is a plain sequential scan, not the custom scan") + p = _plan(pgc_conn, "scan") + expect.text(f"{_count(p, 'Custom Scan')}|{_count(p, 'Columnar Vectorized Aggregates')}", "1|0", + "the scan reader is the custom scan with no vectorized aggregate") + expect.text(str(_count(_plan(pgc_conn, "vec"), "Columnar Vectorized Aggregates")), "1", + "the vec reader is a vectorized aggregate") + p = _plan(pgc_conn, "idx") + expect.text(f"{_count(p, 'Index Scan', 'Index Only Scan')}|{_count(p, 'Seq Scan', 'Custom Scan')}", "1|0", + "the idx reader reads the primary key's index") + + every = ("seq", "scan", "vec", "idx") + + # ---- a flushed INSERT rolled back to a savepoint ---- + _fresh(pgc_conn) + out = _xact(pgc_conn, ["BEGIN", GROUPS, "SAVEPOINT s", + "INSERT INTO %t SELECT g, 1 FROM generate_series(10001, 15000) g", + GROUPS, "SELECT 'inside=' || count(*) FROM %t", + "ROLLBACK TO s"] + READS + ["COMMIT"]) + expect.text(_errors(out), "as expected", "a savepoint's INSERT runs, and ROLLBACK TO it, on both tables") + expect.text(_grew(out), "yes", "the savepoint's INSERT flushed row groups before ROLLBACK TO") + expect.text(_same(out, "inside"), "same", "inside the savepoint, every reader saw the INSERT, as heap does") + expect.text(_same(out, *every), "same", + "after ROLLBACK TO, in the same transaction, no reader sees the savepoint's rows, as heap does") + expect.text(_same(_xact(pgc_conn, READS), *every), "same", + "and after COMMIT, no reader sees them, as heap does") + + # ---- the same, under one snapshot ---- + # At REPEATABLE READ the reads after ROLLBACK TO use the snapshot the reads + # inside the savepoint used, so only the abort itself can hide the groups. + _fresh(pgc_conn) + out = _xact(pgc_conn, ["BEGIN ISOLATION LEVEL REPEATABLE READ", "SELECT 1 FROM %t LIMIT 1", + "SAVEPOINT s", + "INSERT INTO %t SELECT g, 1 FROM generate_series(10001, 15000) g"] + + READS_IN + ["ROLLBACK TO s"] + READS + ["COMMIT"]) + expect.text(_errors(out), "as expected", + "at REPEATABLE READ, a savepoint's INSERT runs, and ROLLBACK TO it, on both tables") + expect.text(_same(out, "in_seq", "in_scan", "in_vec", "in_idx"), "same", + "at REPEATABLE READ, inside the savepoint, every reader saw the INSERT, as heap does") + expect.text(_same(out, *every), "same", + "at REPEATABLE READ, after ROLLBACK TO, no reader sees the savepoint's rows, as heap does") + + # ---- the same unique keys again after ROLLBACK TO ---- + _fresh(pgc_conn) + out = _xact(pgc_conn, ["BEGIN", "SAVEPOINT s", + "INSERT INTO %t SELECT g, 2 FROM generate_series(10001, 15000) g", + "ROLLBACK TO s", + "INSERT INTO %t SELECT g, 3 FROM generate_series(10001, 15000) g"] + + READS + ["COMMIT"]) + expect.text(_errors(out), "as expected", + "after ROLLBACK TO, the same unique keys insert again in the same transaction, as on heap") + expect.text(_same(out, *every), "same", + "and every reader sees each key once, with the second insert's values, as heap does") + expect.text(_same(_xact(pgc_conn, READS), *every), "same", + "and after COMMIT, each key is there once, as heap does") + + # ---- a large UPDATE rolled back to a savepoint ---- + _fresh(pgc_conn, 5000) + out = _xact(pgc_conn, ["BEGIN"] + _catalog_ins("b") + ["SAVEPOINT s", "UPDATE %t SET v = v + 10", + "SELECT 'inside=' || count(*) || '|' || sum(v) FROM %t", + "ROLLBACK TO s"] + _catalog_ins("a") + READS + ["COMMIT"]) + expect.text(_errors(out), "as expected", + "an UPDATE of every row in a savepoint runs, and ROLLBACK TO it, on both tables") + expect.text(_flushed(out), "yes", + "the savepoint's UPDATE flushed new versions and delete marks before ROLLBACK TO") + expect.text(_same(out, "inside"), "same", + "inside the savepoint, the UPDATE changed each row once, as heap does") + expect.text(_same(out, *every), "same", + "after ROLLBACK TO, in the same transaction, every reader sees the rows as they were, as heap does") + expect.text(_same(_xact(pgc_conn, READS), *every), "same", + "and after COMMIT, the rows are as they were, as heap does") + + # ---- a statement that fails after it has flushed ---- + # Rows are generated in order, so the division by zero comes at the 3500th row, + # after three flushes. The same statement without the failing row shows the flushes. + fail = "INSERT INTO %t SELECT g, 1 / (g - 23500) FROM generate_series(20001, 24000) g" + _fresh(pgc_conn) + out = _xact(pgc_conn, ["BEGIN", GROUPS, + "INSERT INTO %t SELECT g, 1 / (g - 99999) FROM generate_series(20001, 24000) g", + GROUPS, "ROLLBACK"]) + expect.text(_grew(out), "yes", "the failing INSERT's shape flushes row groups before its 3500th row") + + _fresh(pgc_conn) + expect.text(_errors(_xact(pgc_conn, [fail]), "division by zero"), "as expected", + "an INSERT that fails after flushing is refused on both tables") + expect.text(_same(_xact(pgc_conn, READS), *every), "same", + "and no reader sees any of its rows, as heap does") + + _fresh(pgc_conn) + out = _xact(pgc_conn, ["BEGIN", "SAVEPOINT s", fail, "ROLLBACK TO s", + "INSERT INTO %t VALUES (30001, 5), (30002, 5)"] + READS + ["COMMIT"]) + expect.text(_errors(out, "division by zero"), "as expected", + "in a savepoint, an INSERT that fails after flushing raises its error and nothing else, on both tables") + expect.text(_same(out, *every), "same", + "and in the same transaction every reader sees only the rows written after it, as heap does") + expect.text(_same(_xact(pgc_conn, READS), *every), "same", + "and after COMMIT, only the rows written after the failed INSERT are there, as heap does") + + # An UPDATE of every row that fails at the last row it reaches (5000, the last + # in insert order). By then it has flushed new versions, and sp_probe's read at + # row 2500 flushed the delete marks of the rows before it. The premise reads both + # catalogs' inserts right after ROLLBACK TO, before anything else writes. + _fresh(pgc_conn, 5000) + out = _xact(pgc_conn, ["BEGIN"] + _catalog_ins("b") + ["SAVEPOINT s", "UPDATE %t SET v = v + sp_probe('%t', id)", + "ROLLBACK TO s"] + _catalog_ins("a") + ["UPDATE %t SET v = v + 100 WHERE id = 1"] + + READS + ["COMMIT"]) + expect.text(_errors(out, "division by zero"), "as expected", + "in a savepoint, an UPDATE of every row that fails at the last raises its error and nothing else, " + "on both tables") + expect.text(_flushed(out), "yes", + "the failing UPDATE flushed new versions and delete marks before it failed") + expect.text(_same(out, *every), "same", + "and in the same transaction every reader sees the rows as they were, plus the later UPDATE, " + "as heap does") + expect.text(_same(_xact(pgc_conn, READS), *every), "same", + "and after COMMIT, the rows are as they were plus the later UPDATE, as heap does") diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index bd7adbd97..f3961c4bf 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -296,6 +296,7 @@ SUITES=( rls_direct_storage rollup row_triggers + savepoint_flush scan_decode_cost scan_direction serializable diff --git a/test/savepoint_flush.sh b/test/savepoint_flush.sh new file mode 100755 index 000000000..a9504a4e0 --- /dev/null +++ b/test/savepoint_flush.sh @@ -0,0 +1,246 @@ +#!/usr/bin/env bash +# +# Rolling back to a savepoint, or failing a statement, after rows have already been +# flushed into row groups (#1341). A flushed group is a catalog row written under +# the subtransaction, so the abort must hide it; rows still buffered, and buffered +# delete marks, must be thrown away; and the group-list memo built while the +# savepoint was live must not keep serving the aborted groups. +# +# Each shape runs on a columnar table and on heap, and heap is the answer. The +# columnar table writes row groups of 1000 rows, so every write below crosses +# several flushes. Each transaction prints labelled values (label=value), and the +# two tables are compared label by label, inside the transaction and after it. +# +# EVERY SHAPE STARTS FROM FRESH TABLES. Otherwise a shape that goes wrong poisons +# the ones after it, and they fail for its rows rather than their own (OffgridwithJD, +# review of #1430). +# +# Usage: test/savepoint_flush.sh [PG_CONFIG] +# Written fresh for pgColumnar. + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +SET="SET pgcolumnar.stripe_row_limit = 1000; SET statement_timeout = '60s';" +AMS="pgcolumnar heap" +groups() { # groups -> how many row groups the columnar table has, as this session sees + echo "SELECT 'groups=' || count(*) FROM pgcolumnar.row_group + WHERE storage_id = pgcolumnar.get_storage_id('sp_pgcolumnar');" +} +# What every reader sees: a plain sequential scan through the table AM, the custom +# scan without vectorization, the vectorized count, and the primary key's index. +# Each is a pair of the settings that choose its path and its query. +R_SEQ=("SET pgcolumnar.enable_custom_scan = off;" + "SELECT 'seq=' || count(*) || '|' || coalesce(sum(id), 0) || '|' || coalesce(sum(v), 0) FROM %t;") +R_SCAN=("SET pgcolumnar.enable_vectorization = off;" + "SELECT 'scan=' || count(*) || '|' || coalesce(sum(id), 0) || '|' || coalesce(sum(v), 0) FROM %t;") +R_VEC=("SET pgcolumnar.enable_vectorization = on;" + "SELECT 'vec=' || count(*) || '|' || coalesce(sum(id), 0) FROM %t;") +R_IDX=("SET pgcolumnar.enable_custom_scan = off; SET enable_seqscan = off; SET enable_bitmapscan = off;" + "SELECT 'idx=' || count(*) || '|' || coalesce(sum(v), 0) FROM %t WHERE id > 1000;") +RESET="RESET pgcolumnar.enable_custom_scan; RESET pgcolumnar.enable_vectorization; + RESET enable_seqscan; RESET enable_bitmapscan;" +READS="" +for r in R_SEQ R_SCAN R_VEC R_IDX; do + declare -n rr="$r"; READS="$READS ${rr[0]} ${rr[1]} $RESET" +done +unset -n rr +READS_IN="${READS//\'seq=/\'in_seq=}"; READS_IN="${READS_IN//\'scan=/\'in_scan=}" +READS_IN="${READS_IN//\'vec=/\'in_vec=}"; READS_IN="${READS_IN//\'idx=/\'in_idx=}" +declare -A OUT +xact() { # xact SQL: run SQL on both tables; %t is the table, %r READS, %i READS_IN + # Both tables, always: stopping at a columnar error would leave heap unchanged + # too, and the "as heap does" arms after it would compare two untouched tables. + local am sql + for am in $AMS; do + sql="${1//%r/$READS}"; sql="${sql//%i/$READS_IN}" + OUT[$am]="$(psql_run_at "$SET ${sql//%t/sp_$am}" 2>&1)" + done +} +psql_run_at() { # psql_run_at SQL: a script, one statement at a time, each value a line + # Not -c: psql sends a -c string as one query, and an error in it skips the + # rest, so the ROLLBACK TO after a failing statement would never run. + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -v ON_ERROR_STOP=0 -qAt <<<"$1" +} +pick() { # pick AM LABEL -> the value the last xact printed under LABEL for AM + grep -m1 "^$2=" <<<"${OUT[$1]}" | cut -d= -f2- +} +same() { # same LABEL... -> "same" when both tables printed alike under every LABEL + local l c h diff="" + for l in "$@"; do + c="$(pick pgcolumnar "$l")"; h="$(pick heap "$l")" + [ -n "$h" ] || { diff="$diff $l: heap printed nothing (${OUT[heap]:0:200})"; continue; } + [ "$c" = "$h" ] || diff="$diff $l: columnar=[$c] heap=[$h]" + done + echo "${diff:-same}" +} +errors() { # errors [MSG] -> "as expected" when each table's last xact raised one + # error containing MSG (no MSG: none at all), else what each table raised + local am n e want=0 bad="" + [ -n "${1:-}" ] && want=1 + for am in $AMS; do + n="$(grep -c ERROR <<<"${OUT[$am]}")"; e="$(grep -m1 ERROR <<<"${OUT[$am]}")" + { [ "$n" = "$want" ] && { [ "$want" = 0 ] || [[ "$e" == *"$1"* ]]; }; } || + bad="$bad $am: $n errors, first [$e]" + done + echo "${bad:-as expected}" +} +grew() { # grew BEFORE AFTER -> yes when the columnar table gained two groups or more + [ -n "$1" ] && [ -n "$2" ] && [ "$(($2 - $1))" -ge 2 ] && echo yes || echo "no ($1 groups before, $2 after)" +} + +fresh() { # fresh [N]: both tables dropped and made again, holding rows 1..N (2000) + local am + for am in $AMS; do + psql_run "$SET DROP TABLE IF EXISTS sp_$am; CREATE TABLE sp_$am (id int PRIMARY KEY, v int) USING $am; + INSERT INTO sp_$am SELECT g, 0 FROM generate_series(1, ${1:-2000}) g;" + done +} +# catalog_ins TAG -> this transaction's inserts so far into row_group and delete_vector, +# as rg_TAG= and dv_TAG=. pg_stat_xact_all_tables still counts a subtransaction's +# inserts after it aborts, so it shows what a rolled-back write flushed. Read once +# after BEGIN ("b") and once after ROLLBACK TO ("a"); the premise is the difference, +# so the reading cannot include earlier work in the same session, as the twin's one +# connection does (OffgridwithJD, review of #1430). +catalog_ins() { + echo "SELECT 'rg_$1=' || n_tup_ins FROM pg_stat_xact_all_tables WHERE relid = 'pgcolumnar.row_group'::regclass; + SELECT 'dv_$1=' || n_tup_ins FROM pg_stat_xact_all_tables WHERE relid = 'pgcolumnar.delete_vector'::regclass;" +} +flushed() { # flushed -> yes when both catalogs gained two rows or more between b and a + local rg dv k + for k in rg_b dv_b rg_a dv_a; do + local "$k=$(grep -m1 "^$k=" <<<"${OUT[pgcolumnar]}" | cut -d= -f2)" + done + rg=$(( ${rg_a:-0} - ${rg_b:-0} )); dv=$(( ${dv_a:-0} - ${dv_b:-0} )) + [ -n "$rg_a" ] && [ -n "$rg_b" ] && [ "$rg" -ge 2 ] && [ "$dv" -ge 2 ] && echo yes || + echo "no (row_group inserts [$rg], delete_vector inserts [$dv])" +} +# sp_probe(table, id): reads the table when it reaches row 2500 -- a scan starting +# flushes the delete marks buffered so far -- and divides by zero at row 5000. +psql_run "CREATE FUNCTION sp_probe(t regclass, id int) RETURNS int LANGUAGE plpgsql AS \$\$ + BEGIN + IF id = 2500 THEN EXECUTE format('SELECT count(*) FROM %s', t); END IF; + RETURN 1 / (5000 - id) * 0; + END \$\$;" +fresh + +# ---- the readers take the paths they are named for -------------------------------- +plan() { # plan READER -> the columnar table's plan for that reader's query + declare -n rr="$1" + q "$SET ${rr[0]} EXPLAIN (COSTS OFF) ${rr[1]//%t/sp_pgcolumnar}" +} +check "the seq reader is a plain sequential scan, not the custom scan" \ + "$(plan R_SEQ | grep -c 'Seq Scan')|$(plan R_SEQ | grep -c 'Custom Scan')" "1|0" +check "the scan reader is the custom scan with no vectorized aggregate" \ + "$(plan R_SCAN | grep -c 'Custom Scan')|$(plan R_SCAN | grep -c 'Columnar Vectorized Aggregates')" "1|0" +check "the vec reader is a vectorized aggregate" \ + "$(plan R_VEC | grep -c 'Columnar Vectorized Aggregates')" "1" +check "the idx reader reads the primary key's index" \ + "$(plan R_IDX | grep -c 'Index Scan\|Index Only Scan')|$(plan R_IDX | grep -c 'Seq Scan\|Custom Scan')" "1|0" + +# ---- a flushed INSERT rolled back to a savepoint --------------------------------- +fresh +xact "BEGIN; $(groups) SAVEPOINT s; + INSERT INTO %t SELECT g, 1 FROM generate_series(10001, 15000) g; + $(groups) SELECT 'inside=' || count(*) FROM %t; + ROLLBACK TO s; %r COMMIT;" +check "a savepoint's INSERT runs, and ROLLBACK TO it, on both tables" "$(errors)" "as expected" +check "the savepoint's INSERT flushed row groups before ROLLBACK TO" \ + "$(grew "$(grep -m1 '^groups=' <<<"${OUT[pgcolumnar]}" | cut -d= -f2)" \ + "$(grep '^groups=' <<<"${OUT[pgcolumnar]}" | sed -n 2p | cut -d= -f2)")" "yes" +check "inside the savepoint, every reader saw the INSERT, as heap does" "$(same inside)" "same" +check "after ROLLBACK TO, in the same transaction, no reader sees the savepoint's rows, as heap does" \ + "$(same seq scan vec idx)" "same" +xact "%r" +check "and after COMMIT, no reader sees them, as heap does" "$(same seq scan vec idx)" "same" + +# ---- the same, under one snapshot ----------------------------------------------- +# At REPEATABLE READ the reads after ROLLBACK TO use the snapshot the reads inside +# the savepoint used, so only the abort itself can hide the savepoint's groups. +# (A scan held open across the abort, through a cursor, is native_fetch_group_memo's.) +fresh +xact "BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT 1 FROM %t LIMIT 1; SAVEPOINT s; + INSERT INTO %t SELECT g, 1 FROM generate_series(10001, 15000) g; + %i ROLLBACK TO s; %r COMMIT;" +check "at REPEATABLE READ, a savepoint's INSERT runs, and ROLLBACK TO it, on both tables" "$(errors)" "as expected" +check "at REPEATABLE READ, inside the savepoint, every reader saw the INSERT, as heap does" \ + "$(same in_seq in_scan in_vec in_idx)" "same" +check "at REPEATABLE READ, after ROLLBACK TO, no reader sees the savepoint's rows, as heap does" \ + "$(same seq scan vec idx)" "same" + +# ---- the same unique keys again after ROLLBACK TO -------------------------------- +fresh +xact "BEGIN; SAVEPOINT s; + INSERT INTO %t SELECT g, 2 FROM generate_series(10001, 15000) g; + ROLLBACK TO s; + INSERT INTO %t SELECT g, 3 FROM generate_series(10001, 15000) g; + %r COMMIT;" +check "after ROLLBACK TO, the same unique keys insert again in the same transaction, as on heap" "$(errors)" "as expected" +check "and every reader sees each key once, with the second insert's values, as heap does" \ + "$(same seq scan vec idx)" "same" +xact "%r" +check "and after COMMIT, each key is there once, as heap does" "$(same seq scan vec idx)" "same" + +# ---- a large UPDATE rolled back to a savepoint ----------------------------------- +fresh 5000 +xact "BEGIN; $(catalog_ins b) SAVEPOINT s; + UPDATE %t SET v = v + 10; + SELECT 'inside=' || count(*) || '|' || sum(v) FROM %t; + ROLLBACK TO s; $(catalog_ins a) %r COMMIT;" +check "an UPDATE of every row in a savepoint runs, and ROLLBACK TO it, on both tables" "$(errors)" "as expected" +check "the savepoint's UPDATE flushed new versions and delete marks before ROLLBACK TO" "$(flushed)" "yes" +check "inside the savepoint, the UPDATE changed each row once, as heap does" "$(same inside)" "same" +check "after ROLLBACK TO, in the same transaction, every reader sees the rows as they were, as heap does" \ + "$(same seq scan vec idx)" "same" +xact "%r" +check "and after COMMIT, the rows are as they were, as heap does" "$(same seq scan vec idx)" "same" + +# ---- a statement that fails after it has flushed --------------------------------- +# Rows are generated in order, so the division by zero comes at the 3500th row, +# after three flushes. The same statement without the failing row shows the flushes. +FAIL="INSERT INTO %t SELECT g, 1 / (g - 23500) FROM generate_series(20001, 24000) g;" +fresh +xact "BEGIN; $(groups) INSERT INTO %t SELECT g, 1 / (g - 99999) FROM generate_series(20001, 24000) g; + $(groups) ROLLBACK;" +check "the failing INSERT's shape flushes row groups before its 3500th row" \ + "$(grew "$(grep -m1 '^groups=' <<<"${OUT[pgcolumnar]}" | cut -d= -f2)" \ + "$(grep '^groups=' <<<"${OUT[pgcolumnar]}" | sed -n 2p | cut -d= -f2)")" "yes" + +fresh +xact "$FAIL" +check "an INSERT that fails after flushing is refused on both tables" \ + "$(errors 'division by zero')" "as expected" +xact "%r" +check "and no reader sees any of its rows, as heap does" "$(same seq scan vec idx)" "same" + +fresh +xact "BEGIN; SAVEPOINT s; $FAIL + ROLLBACK TO s; INSERT INTO %t VALUES (30001, 5), (30002, 5); + %r COMMIT;" +check "in a savepoint, an INSERT that fails after flushing raises its error and nothing else, on both tables" \ + "$(errors 'division by zero')" "as expected" +check "and in the same transaction every reader sees only the rows written after it, as heap does" \ + "$(same seq scan vec idx)" "same" +xact "%r" +check "and after COMMIT, only the rows written after the failed INSERT are there, as heap does" "$(same seq scan vec idx)" "same" + +# An UPDATE of every row that fails at the last row it reaches (5000, the last in +# insert order). By then it has flushed new versions, and sp_probe's read at row 2500 +# flushed the delete marks of the rows before it. The premise reads both catalogs' +# inserts right after ROLLBACK TO, before anything else in the transaction writes. +fresh 5000 +xact "BEGIN; $(catalog_ins b) SAVEPOINT s; + UPDATE %t SET v = v + sp_probe('%t', id); + ROLLBACK TO s; $(catalog_ins a) UPDATE %t SET v = v + 100 WHERE id = 1; + %r COMMIT;" +check "in a savepoint, an UPDATE of every row that fails at the last raises its error and nothing else, on both tables" \ + "$(errors 'division by zero')" "as expected" +check "the failing UPDATE flushed new versions and delete marks before it failed" "$(flushed)" "yes" +check "and in the same transaction every reader sees the rows as they were, plus the later UPDATE, as heap does" \ + "$(same seq scan vec idx)" "same" +xact "%r" +check "and after COMMIT, the rows are as they were plus the later UPDATE, as heap does" "$(same seq scan vec idx)" "same" + +pgc_summary