From 03c31deb3d58faaedb35e7babbe28fd8bd90ade2 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 23:16:41 +0000 Subject: [PATCH 1/3] test: native_gap gains its pytest twin, and its indexed lookup reads the index (#1341) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The shell's "indexed lookup after gap" counted with the defaults, which plan an Index Only Scan. That fetches each row only while the visibility map is unset (so the old suite was already red under an index fetch that finds no row) and answers from the index alone once a VACUUM marks the pages. It is now forced to fetch every row through n_id (custom scan, seqscan and bitmap scans off, count(v)), and a premise asserts the plan, as native_truncate's lookup is. A guard, not a fix. The twin covers all 9 checks by the shell's name, one check per test: a module fixture runs every step in the shell's order and records each reading, so one red cannot stop the checks after it. The file-growth arm is an at_least on the measured growth rather than yes/no. Four runs on PG18 (none, gap fill removed as before 6a165821, the test hook advancing nothing, index fetch finding no row) redden the same names in the shell and in the twin as shipped: 0, 6, 1 and 1. Ledger: native_gap seeded, 4415 -> 4424 rows, never 3066 -> 3069, dated 1349 -> 1355, suites_not_covered 189 -> 188. cluster_tests 754 -> 763, TESTS.md §177, and the stem leaves unported_suites.txt. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 9 ++ test/check_ledger_budget.txt | 17 +++- test/native_gap.sh | 10 ++- test/pytest/TESTS.md | 21 +++++ test/pytest/expected_tests.txt | 4 +- test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_native_gap.py | 135 ++++++++++++++++++++++++++++ test/pytest/unported_suites.txt | 1 - 8 files changed, 193 insertions(+), 5 deletions(-) create mode 100644 test/pytest/test_native_gap.py diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index fe55cb196..a5e51fee3 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2566,6 +2566,15 @@ native_fetch_projection native_fetch_projection premise: there is a needed-set m native_fetch_projection native_fetch_projection the all-covering projection reconstructs every row 15;16;17;18;19 never - native_fetch_projection native_fetch_projection the projection reads every row to start with 15;16;17;18;19 never - native_fetch_projection native_fetch_projection the wide fixture has 41 columns 15;16;17;18;19 never - +native_gap native_gap baseline parity 15;16;17;18;19 never - +native_gap native_gap baseline row count 15;16;17;18;19 never - +native_gap native_gap gap was physically materialized 15;16;17;18;19 2026-10-16 columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors;columnar_vacuum.c: debug_advance_reserved_offset advances nothing, so no gap is made +native_gap native_gap indexed lookup after gap 15;16;17;18;19 2026-10-16 columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors;columnar_tableam.c: an index fetch finds no row +native_gap native_gap parity after delete on gapped layout 15;16;17;18;19 2026-10-16 columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors +native_gap native_gap parity after gapped insert 15;16;17;18;19 2026-10-16 columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors +native_gap native_gap parity after post-gap insert 15;16;17;18;19 2026-10-16 columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors +native_gap native_gap premise: the indexed lookup reads n_id 15;16;17;18;19 never - +native_gap native_gap row count after gapped insert 15;16;17;18;19 2026-10-16 columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors native_groupagg native_groupagg a NULL key counts as a group in the bucket bound: four groups under a cap of 3 are declined and all answered 15;16;17;18;19 2026-10-16 the bucket bound left out the NULL group and the runtime cap was off, so the node took the query and answered it (mutation, #1339) native_groupagg native_groupagg a cached plan run under a lower cap stops at the runtime cap, with its hint 15;16;17;18;19 never - native_groupagg native_groupagg a date_bin key is bounded as a truncating key is 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 77c228e30..9df35a643 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 189 +suites_not_covered 188 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3560,4 +3560,17 @@ suites_not_covered 189 # never 3055 + 11 = 3066 # dated 1342 + 7 = 1349 # 3066 + 1349 == 4415 -checks_never_observed_red 3066 +# +# #1341, native_gap seeded, with its twin (OffgridwithJD): 3066 -> 3069, and +# suites_not_covered 189 -> 188. The suite had no rows. It enters with all 9 checks +# from five-major runs (9/9 on each of 15-19). 6 are dated by three mutations: the +# gap fill removed (6a165821's pre-fix write path, where the write lands short of +# its block and errors), the test hook advancing nothing, and an index fetch finding +# no row (the indexed lookup, now forced through n_id with a premise). Three arrive +# never: the two baseline checks and the index premise. +# +# ledger rows 4415 + 9 = 4424 +# never 3066 + 3 = 3069 +# dated 1349 + 6 = 1355 +# 3069 + 1355 == 4424 +checks_never_observed_red 3069 diff --git a/test/native_gap.sh b/test/native_gap.sh index 155a47e1b..9b9e81d78 100755 --- a/test/native_gap.sh +++ b/test/native_gap.sh @@ -56,8 +56,16 @@ psql_run "INSERT INTO h SELECT g, md5(g::text) FROM generate_series(6001, 7000) psql_run "INSERT INTO n SELECT g, md5(g::text) FROM generate_series(6001, 7000) g;" check "parity after post-gap insert" "$(hash_n)" "$(hash_h)" psql_run "CREATE INDEX n_id ON n (id);" +# Forced to fetch every row through n_id, and asserted (#1341). With the defaults +# this count plans an Index Only Scan (PG17 and PG18), which fetches each row only +# while the visibility map leaves its page unmarked: once a VACUUM marks them it +# answers from the index alone, whatever the fetch does. count(v), not count(*), +# with the custom scan, seqscan and bitmap scans off, so the plan is an Index Scan. +IDX_SETS="SET enable_seqscan = off; SET enable_bitmapscan = off; SET pgcolumnar.enable_custom_scan = off;" +check "premise: the indexed lookup reads n_id" \ + "$(pgc_uses_row_fetch "$IDX_SETS" "SELECT count(v) FROM n WHERE id BETWEEN 1 AND 7000")" yes check "indexed lookup after gap" \ - "$(q 'SELECT count(*) FROM n WHERE id BETWEEN 1 AND 7000;')" "7000" + "$(q "$IDX_SETS SELECT count(v) FROM n WHERE id BETWEEN 1 AND 7000;" | tail -1)" "7000" # a delete + reader still resolves correctly across the gapped layout. psql_run "DELETE FROM h WHERE id % 7 = 0;" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index cc0c55443..0500a1591 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -222,6 +222,7 @@ behaviour, the source of that number is named. - [174. test_sorted_projection.py: vacuum_sorted reorders storage and changes no answer](#174-test_sorted_projectionpy-vacuum_sorted-reorders-storage-and-changes-no-answer) - [175. test_native_ios.py: an index-only scan skips the fetch on all-visible row groups](#175-test_native_iospy-an-index-only-scan-skips-the-fetch-on-all-visible-row-groups) - [176. test_native_truncate.py: end-truncation shrinks the file and loses no row](#176-test_native_truncatepy-end-truncation-shrinks-the-file-and-loses-no-row) +- [177. test_native_gap.py: a write beyond EOF fills the gap and loses no row](#177-test_native_gappy-a-write-beyond-eof-fills-the-gap-and-loses-no-row) ## 1. How to read a test in here @@ -7836,3 +7837,23 @@ The pytest twin of `test/native_truncate.sh` (ported for #1341), every check by | `test_truncate_shrinks_the_file_and_keeps_every_row` | a truncate with nothing freed returns 0; after delete and compact it returns blocks, the file shrinks, the rows match the heap, and a second truncate returns 0 | | `test_the_table_stays_usable_and_the_guc_turns_it_off` | reinsert and an index lookup through the index after truncation; with the GUC off truncate returns 0 and the file keeps its size, with it on the file shrinks | | `test_truncate_refuses_a_transaction_block_and_reuse_stays_clean` | refused inside BEGIN with the file and rows unchanged; truncate, reinsert and compaction keep parity | + +## 177. test_native_gap.py: a write beyond EOF fills the gap and loses no row + +The pytest twin of `test/native_gap.sh` (ported for #1341), every check by the shell's name. The metapage highwater can sit ahead of the physical EOF, which is the state an aborted or crash-interrupted end-truncation leaves. A write whose target block lies beyond EOF must fill the gap with empty pages and succeed. A test-only hook, bound by the test as the shell binds it, pushes the highwater 50 pages past EOF. The next insert must keep every row against a heap mirror and grow the file by at least 40 pages. The table then stays usable: another insert, an index lookup forced through the index and asserted to read it, and a delete. + +### Every test + +Each check is its own test over a module fixture that runs every step in the shell's order and records each reading, so one red cannot stop the checks after it. + +| test | what it holds | +| --- | --- | +| `test_baseline_row_count` | 3000 rows before the gap | +| `test_baseline_parity` | and they match the heap | +| `test_row_count_after_gapped_insert` | 6000 rows after the insert across a 50-page gap | +| `test_parity_after_gapped_insert` | and they match the heap | +| `test_gap_was_physically_materialized` | the file grew by more than 40 pages | +| `test_parity_after_post_gap_insert` | a later insert keeps parity | +| `test_the_indexed_lookup_reads_n_id` | the forced lookup plans an Index Scan on `n_id` | +| `test_indexed_lookup_after_gap` | and finds all 7000 rows | +| `test_parity_after_delete_on_gapped_layout` | a delete keeps parity | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index e3dbd8992..92556a1b6 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -823,4 +823,6 @@ guard_tests 440 # one test, by collection over the cluster files on this tree. # cluster_tests 751 -> 754 for #1341's test_native_truncate.py, the twin of native_truncate.sh: # three tests, by collection over the cluster files on this tree. -cluster_tests 754 +# cluster_tests 754 -> 763 for #1341's test_native_gap.py, the twin of native_gap.sh: +# nine tests, one per check, by collection over the cluster files on this tree. +cluster_tests 763 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 9ac042055..1d512bb1f 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -135,6 +135,7 @@ "native_fetch_coalesce", "native_fetch_position", "native_fetch_projection", + "native_gap", "native_groupagg", "native_groupagg_batch", "native_index", diff --git a/test/pytest/test_native_gap.py b/test/pytest/test_native_gap.py new file mode 100644 index 000000000..ecc2eb237 --- /dev/null +++ b/test/pytest/test_native_gap.py @@ -0,0 +1,135 @@ +"""The gap-tolerant write path (Phase F). The pytest twin of `test/native_gap.sh`, every +check by the shell's name. + +The metapage highwater can sit ahead of the physical EOF: the state an aborted or +crash-interrupted end-truncation leaves. A write whose target block is beyond EOF fills +the gap with empty pages and succeeds. A test-only hook, bound here as the shell binds +it, pushes the highwater 50 pages past EOF; then the writes across the gap succeed, the +rows match a heap mirror, the file grows by the gap, and the table stays usable: more +inserts, an index read through, a delete. The fixture runs every step and records +each reading, and each check is its own test, so one red cannot hide the next. +""" +import psycopg +import pytest + +HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) " + "FROM (SELECT _row::text AS t FROM ({}) _row) _s") +IDX_SETS = ("SET enable_seqscan = off", "SET enable_bitmapscan = off", "SET pgcolumnar.enable_custom_scan = off") +LOOKUP = "SELECT count(v) FROM n WHERE id BETWEEN 1 AND 7000" + + +def _one(c, sql): + try: + return str(c.execute(sql).fetchone()[0]) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _size(c): + return int(c.execute("SELECT pg_relation_size('n')").fetchone()[0]) + + +def _hashes(c): + """-> (columnar, heap) set digests of the rows.""" + return _one(c, HASH.format("SELECT id, v FROM n")), _one(c, HASH.format("SELECT id, v FROM h")) + + +def _both(c, sql): + """Run the statement on the heap mirror, then on the columnar table; a write the + gap refuses is left for the checks to see, as psql_run leaves it.""" + c.execute(sql.format(t="h")) + try: + c.execute(sql.format(t="n")) + except psycopg.Error as exc: + print(f"-- columnar write failed: {exc.diag.message_primary}") + + +@pytest.fixture(scope="module") +def gap(pgc_cluster): + """Every step of the shell, in its order, with each reading recorded rather than + asserted, so each check is its own test and one red cannot stop the rest (#1517 + review): 3000 rows, the highwater pushed 50 pages past EOF, 3000 more across the + gap, 1000 more after it, an index read through, a delete.""" + db = "pgc_gap" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + # the internal test hook, deliberately not in the shipped catalog + c.execute("CREATE FUNCTION pgcolumnar.debug_advance_reserved_offset(regclass, int) " + "RETURNS void AS 'pgcolumnar', 'pgcolumnar_debug_advance_reserved_offset' LANGUAGE C") + c.execute("CREATE TABLE h (id int, v text)") + c.execute("CREATE TABLE n (id int, v text) USING pgcolumnar") + c.execute("SELECT pgcolumnar.set_options('n', stripe_row_limit => 1000, chunk_group_row_limit => 1000)") + r = {} + _both(c, "INSERT INTO {t} SELECT g, md5(g::text) FROM generate_series(1, 3000) g") + r["baseline count"], r["baseline"] = _one(c, "SELECT count(*) FROM n"), _hashes(c) + before = _size(c) + + # the highwater 50 pages past the physical EOF: the next write targets beyond it + c.execute("SELECT pgcolumnar.debug_advance_reserved_offset('n', 50)") + _both(c, "INSERT INTO {t} SELECT g, md5(g::text) FROM generate_series(3001, 6000) g") + r["gapped count"], r["gapped"] = _one(c, "SELECT count(*) FROM n"), _hashes(c) + r["growth"] = _size(c) - before + print(f"-- file size: before-gap={before} after-gap={before + r['growth']}") + + _both(c, "INSERT INTO {t} SELECT g, md5(g::text) FROM generate_series(6001, 7000) g") + r["post-gap"] = _hashes(c) + c.execute("CREATE INDEX n_id ON n (id)") + # forced to fetch every row through n_id: by default this count is an Index + # Only Scan, which stops fetching once a VACUUM marks the pages all-visible + for s in IDX_SETS: + c.execute(s) + try: + r["plan"] = "\n".join(row[0] for row in c.execute(f"EXPLAIN (COSTS OFF) {LOOKUP}")) + r["lookup"] = _one(c, LOOKUP) + finally: + for g in ("enable_seqscan", "enable_bitmapscan", "pgcolumnar.enable_custom_scan"): + c.execute(f"RESET {g}") + + _both(c, "DELETE FROM {t} WHERE id % 7 = 0") + r["deleted"] = _hashes(c) + yield r + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_baseline_row_count(gap, expect): + expect.text(gap["baseline count"], "3000", "baseline row count") + + +def test_baseline_parity(gap, expect): + expect.text(*gap["baseline"], "baseline parity") + + +def test_row_count_after_gapped_insert(gap, expect): + expect.text(gap["gapped count"], "6000", "row count after gapped insert") + + +def test_parity_after_gapped_insert(gap, expect): + expect.text(*gap["gapped"], "parity after gapped insert") + + +def test_gap_was_physically_materialized(gap, expect): + # the 50-page gap and the new rows, with slack: at least 40 pages of growth + expect.at_least(gap["growth"], 40 * 8192 + 1, "gap was physically materialized") + + +def test_parity_after_post_gap_insert(gap, expect): + expect.text(*gap["post-gap"], "parity after post-gap insert") + + +def test_the_indexed_lookup_reads_n_id(gap, expect): + expect.text("yes" if "Index Scan using n_id" in gap["plan"] else "no", "yes", + "premise: the indexed lookup reads n_id") + + +def test_indexed_lookup_after_gap(gap, expect): + expect.text(gap["lookup"], "7000", "indexed lookup after gap") + + +def test_parity_after_delete_on_gapped_layout(gap, expect): + expect.text(*gap["deleted"], "parity after delete on gapped layout") diff --git a/test/pytest/unported_suites.txt b/test/pytest/unported_suites.txt index ae0bd9e62..8f7c7cc42 100644 --- a/test/pytest/unported_suites.txt +++ b/test/pytest/unported_suites.txt @@ -88,7 +88,6 @@ native_fetch_interrupt native_fetch_sort_context native_fold_deferral native_fold_skipguard -native_gap native_groupagg_wide_cost native_late_materialization native_lazy_slot From 9bc605ce19a5c77c0f3206782ace0516b646b616 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 23:26:58 +0000 Subject: [PATCH 2/3] test: native_roundtrip gains its pytest twin, and asserts the read crosses its row groups (#1341) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The suite's header says the read crosses several row groups, and nothing checked it. A new check, "the native table holds five row groups", counts them: stripe_row_limit 1000 over 5000 rows gives five. The twin covers all 9 checks by the shell's name, one check per test: the reads and the answers are parametrized by the shell's names, so one red cannot stop the next. Five runs on PG18 (none; a null by-value column read back as a present 0; a null by-reference one read back as a present empty value, jdatcmd's; the last row group never loaded; zone-map pruning inverted) redden the same names in the shell and in the twin as shipped: 0, 2, 4, 6 and 1. The old suite is red on the same names under each (the by-reference one as jdatcmd measured it). Ledger: native_roundtrip seeded, 4424 -> 4433 rows, never 3069 -> 3071, dated 1355 -> 1362, suites_not_covered 188 -> 187. cluster_tests 763 -> 772, TESTS.md §178, and the stem leaves unported_suites.txt. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 9 +++ test/check_ledger_budget.txt | 18 +++++- test/native_roundtrip.sh | 4 ++ test/pytest/TESTS.md | 16 ++++++ test/pytest/expected_tests.txt | 4 +- test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_native_roundtrip.py | 83 ++++++++++++++++++++++++++++ test/pytest/unported_suites.txt | 1 - 8 files changed, 132 insertions(+), 4 deletions(-) create mode 100644 test/pytest/test_native_roundtrip.py diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index a5e51fee3..c63224142 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -3070,6 +3070,15 @@ native_repack native_repack the columnar table is unharmed: content hash 19 2026 native_repack native_repack the columnar table is unharmed: row count 19 2026-10-16 columnar_tableam.c: relation_copy_for_cluster returns without raising, so a rewrite copies nothing native_repack native_repack the columnar table is unharmed: storage options 19 never - native_repack native_repack without an identity index it is refused by PostgreSQL, not by us 19 never - +native_roundtrip native_roundtrip native aggregate (sum over a scanned column) 15;16;17;18;19 2026-10-17 columnar_reader.c: the native reader never loads the last row group +native_roundtrip native_roundtrip native count with filter 15;16;17;18;19 2026-10-17 columnar_reader.c: the native reader never loads the last row group +native_roundtrip native_roundtrip native filtered scan 15;16;17;18;19 2026-10-17 columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: zone-map pruning skips the groups that can match and keeps the ones that cannot +native_roundtrip native_roundtrip native null handling (f) 15;16;17;18;19 2026-10-17 columnar_reader.c: a null by-value column reads back as a present 0;columnar_reader.c: the native reader never loads the last row group +native_roundtrip native_roundtrip native null handling (v) 15;16;17;18;19 2026-10-17 columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: the native reader never loads the last row group +native_roundtrip native_roundtrip native projection (subset of columns) 15;16;17;18;19 2026-10-17 columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: the native reader never loads the last row group +native_roundtrip native_roundtrip native reads back all columns 15;16;17;18;19 2026-10-17 columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: a null by-value column reads back as a present 0;columnar_reader.c: the native reader never loads the last row group +native_roundtrip native_roundtrip native row count 15;16;17;18;19 never - +native_roundtrip native_roundtrip the native table holds five row groups 15;16;17;18;19 never - native_saop_pushdown native_saop_pushdown IN-list count matches the OR-literal equivalent 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) native_saop_pushdown native_saop_pushdown NOT IN builds no range (0 removed) and stays correct 15;16;17;18;19 never - native_saop_pushdown native_saop_pushdown NULL-element count is exact 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 9df35a643..098fd4dba 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 188 +suites_not_covered 187 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3573,4 +3573,18 @@ suites_not_covered 188 # never 3066 + 3 = 3069 # dated 1349 + 6 = 1355 # 3069 + 1355 == 4424 -checks_never_observed_red 3069 +# +# #1341, native_roundtrip seeded, with its twin (OffgridwithJD): 3069 -> 3071, and +# suites_not_covered 188 -> 187. The suite had no rows. It enters with all 9 checks +# from five-major runs (9/9 on each of 15-19), one of them new: the table holds five +# row groups, so the read crosses them. 7 are dated by four mutations in the native +# reader: a null by-value column read back as a present 0, a null by-reference one +# read back as a present empty value (jdatcmd's, #1518 review), the last row group +# never loaded, and zone-map pruning inverted. Two arrive never: the row count, +# which the metadata path answers, and the row-group count. +# +# ledger rows 4424 + 9 = 4433 +# never 3069 + 2 = 3071 +# dated 1355 + 7 = 1362 +# 3071 + 1362 == 4433 +checks_never_observed_red 3071 diff --git a/test/native_roundtrip.sh b/test/native_roundtrip.sh index 32bd146bb..05bd7f37a 100755 --- a/test/native_roundtrip.sh +++ b/test/native_roundtrip.sh @@ -29,6 +29,10 @@ psql_run "INSERT INTO h $GEN;" psql_run "INSERT INTO n $GEN;" check "native row count" "$(q 'SELECT count(*) FROM n;')" "5000" +# the header's claim, asserted (#1341): stripe_row_limit 1000 over 5000 rows, so the +# read crosses row groups rather than decoding one +check "the native table holds five row groups" \ + "$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('n');")" "5" check "native reads back all columns" \ "$(pgc_set_hash 'SELECT id, k, v, f, b FROM n')" \ "$(pgc_set_hash 'SELECT id, k, v, f, b FROM h')" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 0500a1591..5f1aabb8e 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -223,6 +223,7 @@ behaviour, the source of that number is named. - [175. test_native_ios.py: an index-only scan skips the fetch on all-visible row groups](#175-test_native_iospy-an-index-only-scan-skips-the-fetch-on-all-visible-row-groups) - [176. test_native_truncate.py: end-truncation shrinks the file and loses no row](#176-test_native_truncatepy-end-truncation-shrinks-the-file-and-loses-no-row) - [177. test_native_gap.py: a write beyond EOF fills the gap and loses no row](#177-test_native_gappy-a-write-beyond-eof-fills-the-gap-and-loses-no-row) +- [178. test_native_roundtrip.py: a native table reads back what the heap holds](#178-test_native_roundtrippy-a-native-table-reads-back-what-the-heap-holds) ## 1. How to read a test in here @@ -7857,3 +7858,18 @@ Each check is its own test over a module fixture that runs every step in the she | `test_the_indexed_lookup_reads_n_id` | the forced lookup plans an Index Scan on `n_id` | | `test_indexed_lookup_after_gap` | and finds all 7000 rows | | `test_parity_after_delete_on_gapped_layout` | a delete keeps parity | + +## 178. test_native_roundtrip.py: a native table reads back what the heap holds + +The pytest twin of `test/native_roundtrip.sh` (ported for #1341), every check by the shell's name. A native-format table with nulls in a text and a float column is read back through the native reader and compared with a heap mirror holding the same rows. Its 5000 rows are split into five row groups, and a check asserts the count. The row count is answered from metadata, so only the hashed reads see a lost row group. The comparisons cover every column, a filtered scan, a projection, the null counts, an aggregate and a filtered count. Rows are hashed order-independently, so the heap is the oracle. + +### Every test + +Each check is its own test, parametrized by the shell's name, so one red cannot stop the others. + +| test | what it holds | +| --- | --- | +| `test_native_row_count` | 5000 rows, as metadata counts them | +| `test_the_native_table_holds_five_row_groups` | five row groups, so the read crosses them | +| `test_a_read_matches_the_heap[...]` | every column, a filtered scan and a projection match the heap, one test per shell name | +| `test_a_null_count_or_aggregate_matches_the_heap[...]` | the null counts of the text and float columns, `sum(k)` and `count(*) WHERE b` match the heap, one test per shell name | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 92556a1b6..9a5485c17 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -825,4 +825,6 @@ guard_tests 440 # three tests, by collection over the cluster files on this tree. # cluster_tests 754 -> 763 for #1341's test_native_gap.py, the twin of native_gap.sh: # nine tests, one per check, by collection over the cluster files on this tree. -cluster_tests 763 +# cluster_tests 763 -> 772 for #1341's test_native_roundtrip.py, the twin of native_roundtrip.sh: +# nine tests, one per check, by collection over the cluster files on this tree. +cluster_tests 772 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 1d512bb1f..362b46cfc 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -156,6 +156,7 @@ "native_reclaim_frag", "native_recluster", "native_repack", + "native_roundtrip", "native_saop_pushdown", "native_truncate", "native_vacuum_race", diff --git a/test/pytest/test_native_roundtrip.py b/test/pytest/test_native_roundtrip.py new file mode 100644 index 000000000..6ebd44a34 --- /dev/null +++ b/test/pytest/test_native_roundtrip.py @@ -0,0 +1,83 @@ +"""The native round trip (Phase D3). The pytest twin of `test/native_roundtrip.sh`, every +check by the shell's name. + +A native-format table is read back through the native reader and must match a heap +mirror: scalars with nulls across five row groups, a filter, a projection, an +aggregate and a filtered count, compared order-independently so the heap is the +oracle. Each check is its own test. +""" +import psycopg +import pytest + +HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) " + "FROM (SELECT _row::text AS t FROM ({}) _row) _s") +GEN = """SELECT g, + (g * 100000)::bigint, + CASE WHEN g % 9 = 0 THEN NULL ELSE 'row-' || g END, + CASE WHEN g % 6 = 0 THEN NULL ELSE (g::float8 / 7) END, + (g % 2 = 0) + FROM generate_series(1, 5000) g""" + + +def _one(c, sql): + try: + return str(c.execute(sql).fetchone()[0]) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +@pytest.fixture(scope="module") +def rt(pgc_cluster): + db = "pgc_roundtrip" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + c.execute("CREATE TABLE h (id int, k bigint, v text, f float8, b bool)") + c.execute("CREATE TABLE n (id int, k bigint, v text, f float8, b bool) USING pgcolumnar") + # a small row-group limit, so the read crosses several row groups + c.execute("SELECT pgcolumnar.set_options('n', stripe_row_limit => 1000)") + c.execute(f"INSERT INTO h {GEN}") + c.execute(f"INSERT INTO n {GEN}") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def _pair(c, tmpl): + """-> (columnar, heap) answers of `tmpl` with {t} for the table.""" + return _one(c, tmpl.format(t="n")), _one(c, tmpl.format(t="h")) + + +# one check per test, by the shell's name: one red cannot stop the next (#1518 review) +READS = (("native reads back all columns", "SELECT id, k, v, f, b FROM {t}"), + ("native filtered scan", "SELECT id, v FROM {t} WHERE k BETWEEN 100000000 AND 200000000"), + ("native projection (subset of columns)", "SELECT v, b FROM {t}")) +ANSWERS = (("native null handling (v)", "SELECT count(*) FROM {t} WHERE v IS NULL"), + ("native null handling (f)", "SELECT count(*) FROM {t} WHERE f IS NULL"), + ("native aggregate (sum over a scanned column)", "SELECT sum(k) FROM {t}"), + ("native count with filter", "SELECT count(*) FROM {t} WHERE b")) + + +def test_native_row_count(rt, expect): + # answered from metadata, so a reader that drops a row group still counts 5000 + expect.text(_one(rt, "SELECT count(*) FROM n"), "5000", "native row count") + + +def test_the_native_table_holds_five_row_groups(rt, expect): + expect.text(_one(rt, "SELECT count(*) FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('n')"), "5", + "the native table holds five row groups") + + +@pytest.mark.parametrize("name,sql", READS, ids=[n for n, _ in READS]) +def test_a_read_matches_the_heap(rt, expect, name, sql): + expect.text(*_pair(rt, HASH.format(sql)), name) + + +@pytest.mark.parametrize("name,sql", ANSWERS, ids=[n for n, _ in ANSWERS]) +def test_a_null_count_or_aggregate_matches_the_heap(rt, expect, name, sql): + expect.text(*_pair(rt, sql), name) diff --git a/test/pytest/unported_suites.txt b/test/pytest/unported_suites.txt index 8f7c7cc42..97f8452b4 100644 --- a/test/pytest/unported_suites.txt +++ b/test/pytest/unported_suites.txt @@ -107,7 +107,6 @@ native_read_parquet native_reclaim_reconcile native_rewrite native_rewrite_conc -native_roundtrip native_sort_by native_toasted_write native_varlena_bound From e53ba5d62c91c57f80ca79cd2a7aa3dc7388802f Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 23:36:31 +0000 Subject: [PATCH 3/3] test: native_bloom gains its pytest twin (#1341) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The twin covers all 7 checks by the shell's name, one check per test, so one red cannot stop the next. Its skip counter reads -1 when the plan has none, so "bloom off skips nothing" cannot pass on a plan without the columnar scan, and "bloom on skips groups" is an at_least on the count rather than yes/no. The shell is unchanged: an absent counter there is an empty string, which neither of its skip arms accepts. Six runs on PG18 (none; the bloom consulted whatever the GUC says; the GUC inverted, jdatcmd's; never consulted; answering absent for every value; never written) redden the same names in the shell and in the twin as shipped: 0, 1, 2, 1, 1 and 2. Ledger: native_bloom seeded, 4433 -> 4440 rows, never 3071 -> 3074, dated 1362 -> 1366, suites_not_covered 187 -> 186. cluster_tests 772 -> 779, TESTS.md §179, and the stem leaves unported_suites.txt. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 7 ++ test/check_ledger_budget.txt | 17 ++++- test/pytest/TESTS.md | 19 ++++++ test/pytest/expected_tests.txt | 4 +- test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_native_bloom.py | 101 ++++++++++++++++++++++++++++ test/pytest/unported_suites.txt | 1 - 7 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 test/pytest/test_native_bloom.py diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index c63224142..843bafe81 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2367,6 +2367,13 @@ native_batch_fold_projection native_batch_fold_projection ref is a non-empty res native_batch_fold_projection native_batch_fold_projection the folded aggregate equals the non-fold aggregate (payload columns correct) 15;16;17;18;19 2026-10-16 columnar_vector.c: the batch fold's payload list omits the table's last column native_batch_fold_projection native_batch_fold_projection the non-fold aggregate equals the generator's own answer 15;16;17;18;19 never - native_batch_fold_projection native_batch_fold_projection the ungrouped batch fold actually ran (Batch Fold: yes) 15;16;17;18;19 2026-10-16 columnar_vector.c: the ungrouped aggregate never takes the batch fold (row path instead) +native_bloom native_bloom absent value parity 15;16;17;18;19 never - +native_bloom native_bloom bloom off skips nothing (min/max cannot) 15;16;17;18;19 2026-10-16 columnar_reader.c: the bloom is consulted whatever pgcolumnar.enable_bloom_filter says +native_bloom native_bloom bloom on skips groups 15;16;17;18;19 2026-10-16 columnar_reader.c: the reader never consults the bloom;columnar_write_state.c: the writer builds no bloom +native_bloom native_bloom bloom rows written 15;16;17;18;19 2026-10-16 columnar_write_state.c: the writer builds no bloom +native_bloom native_bloom equality probe parity 15;16;17;18;19 2026-10-16 columnar_reader.c: the bloom answers absent for every value, so a matching group is pruned +native_bloom native_bloom row count 15;16;17;18;19 never - +native_bloom native_bloom the plan under test is a columnar custom scan 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound a sequential scan of the same poisoned chunk is refused (XX001) 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound an index fetch of a chunk whose page_length is 2^32 too large is refused (XX001) 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound backend survived the sequential refusal 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 098fd4dba..fb3be53c2 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 187 +suites_not_covered 186 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3587,4 +3587,17 @@ suites_not_covered 187 # never 3069 + 2 = 3071 # dated 1355 + 7 = 1362 # 3071 + 1362 == 4433 -checks_never_observed_red 3071 +# +# #1341, native_bloom seeded, with its twin (OffgridwithJD): 3071 -> 3074, and +# suites_not_covered 187 -> 186. The suite had no rows. It enters with all 7 checks +# from five-major runs (7/7 on each of 15-19). 4 are dated by four mutations: the +# bloom consulted whatever the GUC says, never consulted, answering absent for every +# value (a false negative pruning the match), and never written. Three arrive never: +# the row count, the custom-scan premise, and the absent-value parity, whose value +# lies outside every group's range, so min/max prunes it before the bloom is asked. +# +# ledger rows 4433 + 7 = 4440 +# never 3071 + 3 = 3074 +# dated 1362 + 4 = 1366 +# 3074 + 1366 == 4440 +checks_never_observed_red 3074 diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 5f1aabb8e..f8f539f14 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -224,6 +224,7 @@ behaviour, the source of that number is named. - [176. test_native_truncate.py: end-truncation shrinks the file and loses no row](#176-test_native_truncatepy-end-truncation-shrinks-the-file-and-loses-no-row) - [177. test_native_gap.py: a write beyond EOF fills the gap and loses no row](#177-test_native_gappy-a-write-beyond-eof-fills-the-gap-and-loses-no-row) - [178. test_native_roundtrip.py: a native table reads back what the heap holds](#178-test_native_roundtrippy-a-native-table-reads-back-what-the-heap-holds) +- [179. test_native_bloom.py: a bloom filter prunes the equality probe min/max cannot](#179-test_native_bloompy-a-bloom-filter-prunes-the-equality-probe-minmax-cannot) ## 1. How to read a test in here @@ -7873,3 +7874,21 @@ Each check is its own test, parametrized by the shell's name, so one red cannot | `test_the_native_table_holds_five_row_groups` | five row groups, so the read crosses them | | `test_a_read_matches_the_heap[...]` | every column, a filtered scan and a projection match the heap, one test per shell name | | `test_a_null_count_or_aggregate_matches_the_heap[...]` | the null counts of the text and float columns, `sum(k)` and `count(*) WHERE b` match the heap, one test per shell name | + +## 179. test_native_bloom.py: a bloom filter prunes the equality probe min/max cannot + +The pytest twin of `test/native_bloom.sh` (ported for #1341), every check by the shell's name. A native table carries a bloom filter per column chunk over each hashable column. An equality probe on an unsorted column can then skip row groups that provably lack the value, even where the min/max zone map cannot. The fixture spreads its values so that every group's range spans the domain. Groups are therefore skipped only with `pgcolumnar.enable_bloom_filter` on, and the probe's answer matches a heap mirror. The skip counter reads -1 when the plan has none, so a plan without the columnar scan can't pass as "skipped nothing". + +### Every test + +Each check is its own test, so one red cannot stop the next. + +| test | what it holds | +| --- | --- | +| `test_row_count` | 20480 rows | +| `test_bloom_rows_written` | one filter per row group for the tag column | +| `test_the_plan_under_test_is_a_columnar_custom_scan` | the probe plans the columnar scan, the only source of the skip counter | +| `test_equality_probe_parity` | the probe for a present value matches the heap | +| `test_bloom_on_skips_groups` | the probe skips groups with the bloom on | +| `test_bloom_off_skips_nothing` | and none with it off | +| `test_absent_value_parity` | a value outside every group's range matches the heap | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 9a5485c17..2dd6aa942 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -827,4 +827,6 @@ guard_tests 440 # nine tests, one per check, by collection over the cluster files on this tree. # cluster_tests 763 -> 772 for #1341's test_native_roundtrip.py, the twin of native_roundtrip.sh: # nine tests, one per check, by collection over the cluster files on this tree. -cluster_tests 772 +# cluster_tests 772 -> 779 for #1341's test_native_bloom.py, the twin of native_bloom.sh: +# seven tests, one per check, by collection over the cluster files on this tree. +cluster_tests 779 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 362b46cfc..070838982 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -122,6 +122,7 @@ "native_agg", "native_backend_crash", "native_batch_fold_projection", + "native_bloom", "native_chunk_length_bound", "native_decode_gate_width", "native_decode_gating", diff --git a/test/pytest/test_native_bloom.py b/test/pytest/test_native_bloom.py new file mode 100644 index 000000000..57fdaf54f --- /dev/null +++ b/test/pytest/test_native_bloom.py @@ -0,0 +1,101 @@ +"""Native per-chunk bloom filters (Phase D5b). The pytest twin of `test/native_bloom.sh`, +every check by the shell's name. + +A native table carries a bloom filter per column chunk over hashable columns, so an +equality probe on an unsorted column skips row groups that provably lack the value, +where min/max cannot. The values are spread so every group's range spans the domain: +groups are skipped only with the bloom on, and the answer matches a heap mirror. +""" +import re + +import psycopg +import pytest + +HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) " + "FROM (SELECT _row::text AS t FROM ({}) _row) _s") +# a well-spread tag over a large domain: each 2048-row group's [min,max] spans almost +# all of it, so min/max cannot prune an in-range probe and the bloom can +GEN = "SELECT g, ((g * 104729) % 1000000) AS tag FROM generate_series(1, 20480) g" +REMOVED = re.compile(r"Columnar Chunk Groups Removed by Filter: (\d+)") + + +def _one(c, sql): + try: + return str(c.execute(sql).fetchone()[0]) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _skipped(c, sql, bloom): + """Groups the scan removed with the bloom `bloom`, or -1 when the plan reports no + counter: a plan without the columnar scan must not read as "removed nothing".""" + c.execute(f"SET pgcolumnar.enable_bloom_filter = {bloom}") + try: + plan = "\n".join(r[0] for r in c.execute( + f"EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) {sql}")) + finally: + c.execute("RESET pgcolumnar.enable_bloom_filter") + m = REMOVED.search(plan) + return int(m.group(1)) if m else -1 + + +@pytest.fixture(scope="module") +def bloom(pgc_cluster): + """-> (conn, probe): the two tables, and a tag that really occurs, from the heap.""" + db = "pgc_bloom" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + c.execute("CREATE TABLE h (id int, tag int)") + c.execute("CREATE TABLE n (id int, tag int) USING pgcolumnar") + c.execute("SELECT pgcolumnar.set_options('n', stripe_row_limit => 2048, chunk_group_row_limit => 1024)") + c.execute(f"INSERT INTO h {GEN}") + c.execute(f"INSERT INTO n {GEN}") + yield c, _one(c, "SELECT tag FROM h WHERE id = 7777") + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +# one check per test, so one red cannot stop the next (#1519 review) +def test_row_count(bloom, expect): + expect.text(_one(bloom[0], "SELECT count(*) FROM n"), "20480", "row count") + + +def test_bloom_rows_written(bloom, expect): + # the hashable tag column, one filter per row group + expect.text(_one(bloom[0], "SELECT count(*) FROM pgcolumnar.bloom WHERE storage_id = " + "pgcolumnar.get_storage_id('n') AND column_index = 1"), "10", "bloom rows written") + + +def test_the_plan_under_test_is_a_columnar_custom_scan(bloom, expect): + c, probe = bloom + plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) SELECT id FROM n WHERE tag = {probe}")) + expect.text("yes" if "Columnar Projected Columns" in plan else "no", "yes", + "the plan under test is a columnar custom scan") + + +def test_equality_probe_parity(bloom, expect): + c, probe = bloom + expect.text(_one(c, HASH.format(f"SELECT id, tag FROM n WHERE tag = {probe}")), + _one(c, HASH.format(f"SELECT id, tag FROM h WHERE tag = {probe}")), "equality probe parity") + + +def test_bloom_on_skips_groups(bloom, expect): + c, probe = bloom + expect.at_least(_skipped(c, f"SELECT id FROM n WHERE tag = {probe}", "on"), 1, "bloom on skips groups") + + +def test_bloom_off_skips_nothing(bloom, expect): + # the values are spread so every group's min/max spans the domain + c, probe = bloom + expect.num(_skipped(c, f"SELECT id FROM n WHERE tag = {probe}", "off"), 0, + "bloom off skips nothing (min/max cannot)") + + +def test_absent_value_parity(bloom, expect): + expect.text(_one(bloom[0], "SELECT count(*) FROM n WHERE tag = 999999999"), + _one(bloom[0], "SELECT count(*) FROM h WHERE tag = 999999999"), "absent value parity") diff --git a/test/pytest/unported_suites.txt b/test/pytest/unported_suites.txt index 97f8452b4..89960f625 100644 --- a/test/pytest/unported_suites.txt +++ b/test/pytest/unported_suites.txt @@ -75,7 +75,6 @@ logical_subscriber maintenance_due native_agg_addcolumn native_agg_deletes -native_bloom native_cancel native_cluster native_compact