diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index fe55cb196..843bafe81 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2367,6 +2367,13 @@ native_batch_fold_projection native_batch_fold_projection ref is a non-empty res native_batch_fold_projection native_batch_fold_projection the folded aggregate equals the non-fold aggregate (payload columns correct) 15;16;17;18;19 2026-10-16 columnar_vector.c: the batch fold's payload list omits the table's last column native_batch_fold_projection native_batch_fold_projection the non-fold aggregate equals the generator's own answer 15;16;17;18;19 never - native_batch_fold_projection native_batch_fold_projection the ungrouped batch fold actually ran (Batch Fold: yes) 15;16;17;18;19 2026-10-16 columnar_vector.c: the ungrouped aggregate never takes the batch fold (row path instead) +native_bloom native_bloom absent value parity 15;16;17;18;19 never - +native_bloom native_bloom bloom off skips nothing (min/max cannot) 15;16;17;18;19 2026-10-16 columnar_reader.c: the bloom is consulted whatever pgcolumnar.enable_bloom_filter says +native_bloom native_bloom bloom on skips groups 15;16;17;18;19 2026-10-16 columnar_reader.c: the reader never consults the bloom;columnar_write_state.c: the writer builds no bloom +native_bloom native_bloom bloom rows written 15;16;17;18;19 2026-10-16 columnar_write_state.c: the writer builds no bloom +native_bloom native_bloom equality probe parity 15;16;17;18;19 2026-10-16 columnar_reader.c: the bloom answers absent for every value, so a matching group is pruned +native_bloom native_bloom row count 15;16;17;18;19 never - +native_bloom native_bloom the plan under test is a columnar custom scan 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound a sequential scan of the same poisoned chunk is refused (XX001) 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound an index fetch of a chunk whose page_length is 2^32 too large is refused (XX001) 15;16;17;18;19 never - native_chunk_length_bound native_chunk_length_bound backend survived the sequential refusal 15;16;17;18;19 never - @@ -2566,6 +2573,15 @@ native_fetch_projection native_fetch_projection premise: there is a needed-set m native_fetch_projection native_fetch_projection the all-covering projection reconstructs every row 15;16;17;18;19 never - native_fetch_projection native_fetch_projection the projection reads every row to start with 15;16;17;18;19 never - native_fetch_projection native_fetch_projection the wide fixture has 41 columns 15;16;17;18;19 never - +native_gap native_gap baseline parity 15;16;17;18;19 never - +native_gap native_gap baseline row count 15;16;17;18;19 never - +native_gap native_gap gap was physically materialized 15;16;17;18;19 2026-10-16 columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors;columnar_vacuum.c: debug_advance_reserved_offset advances nothing, so no gap is made +native_gap native_gap indexed lookup after gap 15;16;17;18;19 2026-10-16 columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors;columnar_tableam.c: an index fetch finds no row +native_gap native_gap parity after delete on gapped layout 15;16;17;18;19 2026-10-16 columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors +native_gap native_gap parity after gapped insert 15;16;17;18;19 2026-10-16 columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors +native_gap native_gap parity after post-gap insert 15;16;17;18;19 2026-10-16 columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors +native_gap native_gap premise: the indexed lookup reads n_id 15;16;17;18;19 never - +native_gap native_gap row count after gapped insert 15;16;17;18;19 2026-10-16 columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors native_groupagg native_groupagg a NULL key counts as a group in the bucket bound: four groups under a cap of 3 are declined and all answered 15;16;17;18;19 2026-10-16 the bucket bound left out the NULL group and the runtime cap was off, so the node took the query and answered it (mutation, #1339) native_groupagg native_groupagg a cached plan run under a lower cap stops at the runtime cap, with its hint 15;16;17;18;19 never - native_groupagg native_groupagg a date_bin key is bounded as a truncating key is 15;16;17;18;19 never - @@ -3061,6 +3077,15 @@ native_repack native_repack the columnar table is unharmed: content hash 19 2026 native_repack native_repack the columnar table is unharmed: row count 19 2026-10-16 columnar_tableam.c: relation_copy_for_cluster returns without raising, so a rewrite copies nothing native_repack native_repack the columnar table is unharmed: storage options 19 never - native_repack native_repack without an identity index it is refused by PostgreSQL, not by us 19 never - +native_roundtrip native_roundtrip native aggregate (sum over a scanned column) 15;16;17;18;19 2026-10-17 columnar_reader.c: the native reader never loads the last row group +native_roundtrip native_roundtrip native count with filter 15;16;17;18;19 2026-10-17 columnar_reader.c: the native reader never loads the last row group +native_roundtrip native_roundtrip native filtered scan 15;16;17;18;19 2026-10-17 columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: zone-map pruning skips the groups that can match and keeps the ones that cannot +native_roundtrip native_roundtrip native null handling (f) 15;16;17;18;19 2026-10-17 columnar_reader.c: a null by-value column reads back as a present 0;columnar_reader.c: the native reader never loads the last row group +native_roundtrip native_roundtrip native null handling (v) 15;16;17;18;19 2026-10-17 columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: the native reader never loads the last row group +native_roundtrip native_roundtrip native projection (subset of columns) 15;16;17;18;19 2026-10-17 columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: the native reader never loads the last row group +native_roundtrip native_roundtrip native reads back all columns 15;16;17;18;19 2026-10-17 columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: a null by-value column reads back as a present 0;columnar_reader.c: the native reader never loads the last row group +native_roundtrip native_roundtrip native row count 15;16;17;18;19 never - +native_roundtrip native_roundtrip the native table holds five row groups 15;16;17;18;19 never - native_saop_pushdown native_saop_pushdown IN-list count matches the OR-literal equivalent 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) native_saop_pushdown native_saop_pushdown NOT IN builds no range (0 removed) and stays correct 15;16;17;18;19 never - native_saop_pushdown native_saop_pushdown NULL-element count is exact 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 77c228e30..fb3be53c2 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 189 +suites_not_covered 186 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3560,4 +3560,44 @@ suites_not_covered 189 # never 3055 + 11 = 3066 # dated 1342 + 7 = 1349 # 3066 + 1349 == 4415 -checks_never_observed_red 3066 +# +# #1341, native_gap seeded, with its twin (OffgridwithJD): 3066 -> 3069, and +# suites_not_covered 189 -> 188. The suite had no rows. It enters with all 9 checks +# from five-major runs (9/9 on each of 15-19). 6 are dated by three mutations: the +# gap fill removed (6a165821's pre-fix write path, where the write lands short of +# its block and errors), the test hook advancing nothing, and an index fetch finding +# no row (the indexed lookup, now forced through n_id with a premise). Three arrive +# never: the two baseline checks and the index premise. +# +# ledger rows 4415 + 9 = 4424 +# never 3066 + 3 = 3069 +# dated 1349 + 6 = 1355 +# 3069 + 1355 == 4424 +# +# #1341, native_roundtrip seeded, with its twin (OffgridwithJD): 3069 -> 3071, and +# suites_not_covered 188 -> 187. The suite had no rows. It enters with all 9 checks +# from five-major runs (9/9 on each of 15-19), one of them new: the table holds five +# row groups, so the read crosses them. 7 are dated by four mutations in the native +# reader: a null by-value column read back as a present 0, a null by-reference one +# read back as a present empty value (jdatcmd's, #1518 review), the last row group +# never loaded, and zone-map pruning inverted. Two arrive never: the row count, +# which the metadata path answers, and the row-group count. +# +# ledger rows 4424 + 9 = 4433 +# never 3069 + 2 = 3071 +# dated 1355 + 7 = 1362 +# 3071 + 1362 == 4433 +# +# #1341, native_bloom seeded, with its twin (OffgridwithJD): 3071 -> 3074, and +# suites_not_covered 187 -> 186. The suite had no rows. It enters with all 7 checks +# from five-major runs (7/7 on each of 15-19). 4 are dated by four mutations: the +# bloom consulted whatever the GUC says, never consulted, answering absent for every +# value (a false negative pruning the match), and never written. Three arrive never: +# the row count, the custom-scan premise, and the absent-value parity, whose value +# lies outside every group's range, so min/max prunes it before the bloom is asked. +# +# ledger rows 4433 + 7 = 4440 +# never 3071 + 3 = 3074 +# dated 1362 + 4 = 1366 +# 3074 + 1366 == 4440 +checks_never_observed_red 3074 diff --git a/test/native_gap.sh b/test/native_gap.sh index 155a47e1b..9b9e81d78 100755 --- a/test/native_gap.sh +++ b/test/native_gap.sh @@ -56,8 +56,16 @@ psql_run "INSERT INTO h SELECT g, md5(g::text) FROM generate_series(6001, 7000) psql_run "INSERT INTO n SELECT g, md5(g::text) FROM generate_series(6001, 7000) g;" check "parity after post-gap insert" "$(hash_n)" "$(hash_h)" psql_run "CREATE INDEX n_id ON n (id);" +# Forced to fetch every row through n_id, and asserted (#1341). With the defaults +# this count plans an Index Only Scan (PG17 and PG18), which fetches each row only +# while the visibility map leaves its page unmarked: once a VACUUM marks them it +# answers from the index alone, whatever the fetch does. count(v), not count(*), +# with the custom scan, seqscan and bitmap scans off, so the plan is an Index Scan. +IDX_SETS="SET enable_seqscan = off; SET enable_bitmapscan = off; SET pgcolumnar.enable_custom_scan = off;" +check "premise: the indexed lookup reads n_id" \ + "$(pgc_uses_row_fetch "$IDX_SETS" "SELECT count(v) FROM n WHERE id BETWEEN 1 AND 7000")" yes check "indexed lookup after gap" \ - "$(q 'SELECT count(*) FROM n WHERE id BETWEEN 1 AND 7000;')" "7000" + "$(q "$IDX_SETS SELECT count(v) FROM n WHERE id BETWEEN 1 AND 7000;" | tail -1)" "7000" # a delete + reader still resolves correctly across the gapped layout. psql_run "DELETE FROM h WHERE id % 7 = 0;" diff --git a/test/native_roundtrip.sh b/test/native_roundtrip.sh index 32bd146bb..05bd7f37a 100755 --- a/test/native_roundtrip.sh +++ b/test/native_roundtrip.sh @@ -29,6 +29,10 @@ psql_run "INSERT INTO h $GEN;" psql_run "INSERT INTO n $GEN;" check "native row count" "$(q 'SELECT count(*) FROM n;')" "5000" +# the header's claim, asserted (#1341): stripe_row_limit 1000 over 5000 rows, so the +# read crosses row groups rather than decoding one +check "the native table holds five row groups" \ + "$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('n');")" "5" check "native reads back all columns" \ "$(pgc_set_hash 'SELECT id, k, v, f, b FROM n')" \ "$(pgc_set_hash 'SELECT id, k, v, f, b FROM h')" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index cc0c55443..f8f539f14 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -222,6 +222,9 @@ behaviour, the source of that number is named. - [174. test_sorted_projection.py: vacuum_sorted reorders storage and changes no answer](#174-test_sorted_projectionpy-vacuum_sorted-reorders-storage-and-changes-no-answer) - [175. test_native_ios.py: an index-only scan skips the fetch on all-visible row groups](#175-test_native_iospy-an-index-only-scan-skips-the-fetch-on-all-visible-row-groups) - [176. test_native_truncate.py: end-truncation shrinks the file and loses no row](#176-test_native_truncatepy-end-truncation-shrinks-the-file-and-loses-no-row) +- [177. test_native_gap.py: a write beyond EOF fills the gap and loses no row](#177-test_native_gappy-a-write-beyond-eof-fills-the-gap-and-loses-no-row) +- [178. test_native_roundtrip.py: a native table reads back what the heap holds](#178-test_native_roundtrippy-a-native-table-reads-back-what-the-heap-holds) +- [179. test_native_bloom.py: a bloom filter prunes the equality probe min/max cannot](#179-test_native_bloompy-a-bloom-filter-prunes-the-equality-probe-minmax-cannot) ## 1. How to read a test in here @@ -7836,3 +7839,56 @@ The pytest twin of `test/native_truncate.sh` (ported for #1341), every check by | `test_truncate_shrinks_the_file_and_keeps_every_row` | a truncate with nothing freed returns 0; after delete and compact it returns blocks, the file shrinks, the rows match the heap, and a second truncate returns 0 | | `test_the_table_stays_usable_and_the_guc_turns_it_off` | reinsert and an index lookup through the index after truncation; with the GUC off truncate returns 0 and the file keeps its size, with it on the file shrinks | | `test_truncate_refuses_a_transaction_block_and_reuse_stays_clean` | refused inside BEGIN with the file and rows unchanged; truncate, reinsert and compaction keep parity | + +## 177. test_native_gap.py: a write beyond EOF fills the gap and loses no row + +The pytest twin of `test/native_gap.sh` (ported for #1341), every check by the shell's name. The metapage highwater can sit ahead of the physical EOF, which is the state an aborted or crash-interrupted end-truncation leaves. A write whose target block lies beyond EOF must fill the gap with empty pages and succeed. A test-only hook, bound by the test as the shell binds it, pushes the highwater 50 pages past EOF. The next insert must keep every row against a heap mirror and grow the file by at least 40 pages. The table then stays usable: another insert, an index lookup forced through the index and asserted to read it, and a delete. + +### Every test + +Each check is its own test over a module fixture that runs every step in the shell's order and records each reading, so one red cannot stop the checks after it. + +| test | what it holds | +| --- | --- | +| `test_baseline_row_count` | 3000 rows before the gap | +| `test_baseline_parity` | and they match the heap | +| `test_row_count_after_gapped_insert` | 6000 rows after the insert across a 50-page gap | +| `test_parity_after_gapped_insert` | and they match the heap | +| `test_gap_was_physically_materialized` | the file grew by more than 40 pages | +| `test_parity_after_post_gap_insert` | a later insert keeps parity | +| `test_the_indexed_lookup_reads_n_id` | the forced lookup plans an Index Scan on `n_id` | +| `test_indexed_lookup_after_gap` | and finds all 7000 rows | +| `test_parity_after_delete_on_gapped_layout` | a delete keeps parity | + +## 178. test_native_roundtrip.py: a native table reads back what the heap holds + +The pytest twin of `test/native_roundtrip.sh` (ported for #1341), every check by the shell's name. A native-format table with nulls in a text and a float column is read back through the native reader and compared with a heap mirror holding the same rows. Its 5000 rows are split into five row groups, and a check asserts the count. The row count is answered from metadata, so only the hashed reads see a lost row group. The comparisons cover every column, a filtered scan, a projection, the null counts, an aggregate and a filtered count. Rows are hashed order-independently, so the heap is the oracle. + +### Every test + +Each check is its own test, parametrized by the shell's name, so one red cannot stop the others. + +| test | what it holds | +| --- | --- | +| `test_native_row_count` | 5000 rows, as metadata counts them | +| `test_the_native_table_holds_five_row_groups` | five row groups, so the read crosses them | +| `test_a_read_matches_the_heap[...]` | every column, a filtered scan and a projection match the heap, one test per shell name | +| `test_a_null_count_or_aggregate_matches_the_heap[...]` | the null counts of the text and float columns, `sum(k)` and `count(*) WHERE b` match the heap, one test per shell name | + +## 179. test_native_bloom.py: a bloom filter prunes the equality probe min/max cannot + +The pytest twin of `test/native_bloom.sh` (ported for #1341), every check by the shell's name. A native table carries a bloom filter per column chunk over each hashable column. An equality probe on an unsorted column can then skip row groups that provably lack the value, even where the min/max zone map cannot. The fixture spreads its values so that every group's range spans the domain. Groups are therefore skipped only with `pgcolumnar.enable_bloom_filter` on, and the probe's answer matches a heap mirror. The skip counter reads -1 when the plan has none, so a plan without the columnar scan can't pass as "skipped nothing". + +### Every test + +Each check is its own test, so one red cannot stop the next. + +| test | what it holds | +| --- | --- | +| `test_row_count` | 20480 rows | +| `test_bloom_rows_written` | one filter per row group for the tag column | +| `test_the_plan_under_test_is_a_columnar_custom_scan` | the probe plans the columnar scan, the only source of the skip counter | +| `test_equality_probe_parity` | the probe for a present value matches the heap | +| `test_bloom_on_skips_groups` | the probe skips groups with the bloom on | +| `test_bloom_off_skips_nothing` | and none with it off | +| `test_absent_value_parity` | a value outside every group's range matches the heap | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index e3dbd8992..2dd6aa942 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -823,4 +823,10 @@ guard_tests 440 # one test, by collection over the cluster files on this tree. # cluster_tests 751 -> 754 for #1341's test_native_truncate.py, the twin of native_truncate.sh: # three tests, by collection over the cluster files on this tree. -cluster_tests 754 +# cluster_tests 754 -> 763 for #1341's test_native_gap.py, the twin of native_gap.sh: +# nine tests, one per check, by collection over the cluster files on this tree. +# cluster_tests 763 -> 772 for #1341's test_native_roundtrip.py, the twin of native_roundtrip.sh: +# nine tests, one per check, by collection over the cluster files on this tree. +# cluster_tests 772 -> 779 for #1341's test_native_bloom.py, the twin of native_bloom.sh: +# seven tests, one per check, by collection over the cluster files on this tree. +cluster_tests 779 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 9ac042055..070838982 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -122,6 +122,7 @@ "native_agg", "native_backend_crash", "native_batch_fold_projection", + "native_bloom", "native_chunk_length_bound", "native_decode_gate_width", "native_decode_gating", @@ -135,6 +136,7 @@ "native_fetch_coalesce", "native_fetch_position", "native_fetch_projection", + "native_gap", "native_groupagg", "native_groupagg_batch", "native_index", @@ -155,6 +157,7 @@ "native_reclaim_frag", "native_recluster", "native_repack", + "native_roundtrip", "native_saop_pushdown", "native_truncate", "native_vacuum_race", diff --git a/test/pytest/test_native_bloom.py b/test/pytest/test_native_bloom.py new file mode 100644 index 000000000..57fdaf54f --- /dev/null +++ b/test/pytest/test_native_bloom.py @@ -0,0 +1,101 @@ +"""Native per-chunk bloom filters (Phase D5b). The pytest twin of `test/native_bloom.sh`, +every check by the shell's name. + +A native table carries a bloom filter per column chunk over hashable columns, so an +equality probe on an unsorted column skips row groups that provably lack the value, +where min/max cannot. The values are spread so every group's range spans the domain: +groups are skipped only with the bloom on, and the answer matches a heap mirror. +""" +import re + +import psycopg +import pytest + +HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) " + "FROM (SELECT _row::text AS t FROM ({}) _row) _s") +# a well-spread tag over a large domain: each 2048-row group's [min,max] spans almost +# all of it, so min/max cannot prune an in-range probe and the bloom can +GEN = "SELECT g, ((g * 104729) % 1000000) AS tag FROM generate_series(1, 20480) g" +REMOVED = re.compile(r"Columnar Chunk Groups Removed by Filter: (\d+)") + + +def _one(c, sql): + try: + return str(c.execute(sql).fetchone()[0]) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _skipped(c, sql, bloom): + """Groups the scan removed with the bloom `bloom`, or -1 when the plan reports no + counter: a plan without the columnar scan must not read as "removed nothing".""" + c.execute(f"SET pgcolumnar.enable_bloom_filter = {bloom}") + try: + plan = "\n".join(r[0] for r in c.execute( + f"EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) {sql}")) + finally: + c.execute("RESET pgcolumnar.enable_bloom_filter") + m = REMOVED.search(plan) + return int(m.group(1)) if m else -1 + + +@pytest.fixture(scope="module") +def bloom(pgc_cluster): + """-> (conn, probe): the two tables, and a tag that really occurs, from the heap.""" + db = "pgc_bloom" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + c.execute("CREATE TABLE h (id int, tag int)") + c.execute("CREATE TABLE n (id int, tag int) USING pgcolumnar") + c.execute("SELECT pgcolumnar.set_options('n', stripe_row_limit => 2048, chunk_group_row_limit => 1024)") + c.execute(f"INSERT INTO h {GEN}") + c.execute(f"INSERT INTO n {GEN}") + yield c, _one(c, "SELECT tag FROM h WHERE id = 7777") + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +# one check per test, so one red cannot stop the next (#1519 review) +def test_row_count(bloom, expect): + expect.text(_one(bloom[0], "SELECT count(*) FROM n"), "20480", "row count") + + +def test_bloom_rows_written(bloom, expect): + # the hashable tag column, one filter per row group + expect.text(_one(bloom[0], "SELECT count(*) FROM pgcolumnar.bloom WHERE storage_id = " + "pgcolumnar.get_storage_id('n') AND column_index = 1"), "10", "bloom rows written") + + +def test_the_plan_under_test_is_a_columnar_custom_scan(bloom, expect): + c, probe = bloom + plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) SELECT id FROM n WHERE tag = {probe}")) + expect.text("yes" if "Columnar Projected Columns" in plan else "no", "yes", + "the plan under test is a columnar custom scan") + + +def test_equality_probe_parity(bloom, expect): + c, probe = bloom + expect.text(_one(c, HASH.format(f"SELECT id, tag FROM n WHERE tag = {probe}")), + _one(c, HASH.format(f"SELECT id, tag FROM h WHERE tag = {probe}")), "equality probe parity") + + +def test_bloom_on_skips_groups(bloom, expect): + c, probe = bloom + expect.at_least(_skipped(c, f"SELECT id FROM n WHERE tag = {probe}", "on"), 1, "bloom on skips groups") + + +def test_bloom_off_skips_nothing(bloom, expect): + # the values are spread so every group's min/max spans the domain + c, probe = bloom + expect.num(_skipped(c, f"SELECT id FROM n WHERE tag = {probe}", "off"), 0, + "bloom off skips nothing (min/max cannot)") + + +def test_absent_value_parity(bloom, expect): + expect.text(_one(bloom[0], "SELECT count(*) FROM n WHERE tag = 999999999"), + _one(bloom[0], "SELECT count(*) FROM h WHERE tag = 999999999"), "absent value parity") diff --git a/test/pytest/test_native_gap.py b/test/pytest/test_native_gap.py new file mode 100644 index 000000000..ecc2eb237 --- /dev/null +++ b/test/pytest/test_native_gap.py @@ -0,0 +1,135 @@ +"""The gap-tolerant write path (Phase F). The pytest twin of `test/native_gap.sh`, every +check by the shell's name. + +The metapage highwater can sit ahead of the physical EOF: the state an aborted or +crash-interrupted end-truncation leaves. A write whose target block is beyond EOF fills +the gap with empty pages and succeeds. A test-only hook, bound here as the shell binds +it, pushes the highwater 50 pages past EOF; then the writes across the gap succeed, the +rows match a heap mirror, the file grows by the gap, and the table stays usable: more +inserts, an index read through, a delete. The fixture runs every step and records +each reading, and each check is its own test, so one red cannot hide the next. +""" +import psycopg +import pytest + +HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) " + "FROM (SELECT _row::text AS t FROM ({}) _row) _s") +IDX_SETS = ("SET enable_seqscan = off", "SET enable_bitmapscan = off", "SET pgcolumnar.enable_custom_scan = off") +LOOKUP = "SELECT count(v) FROM n WHERE id BETWEEN 1 AND 7000" + + +def _one(c, sql): + try: + return str(c.execute(sql).fetchone()[0]) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _size(c): + return int(c.execute("SELECT pg_relation_size('n')").fetchone()[0]) + + +def _hashes(c): + """-> (columnar, heap) set digests of the rows.""" + return _one(c, HASH.format("SELECT id, v FROM n")), _one(c, HASH.format("SELECT id, v FROM h")) + + +def _both(c, sql): + """Run the statement on the heap mirror, then on the columnar table; a write the + gap refuses is left for the checks to see, as psql_run leaves it.""" + c.execute(sql.format(t="h")) + try: + c.execute(sql.format(t="n")) + except psycopg.Error as exc: + print(f"-- columnar write failed: {exc.diag.message_primary}") + + +@pytest.fixture(scope="module") +def gap(pgc_cluster): + """Every step of the shell, in its order, with each reading recorded rather than + asserted, so each check is its own test and one red cannot stop the rest (#1517 + review): 3000 rows, the highwater pushed 50 pages past EOF, 3000 more across the + gap, 1000 more after it, an index read through, a delete.""" + db = "pgc_gap" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + # the internal test hook, deliberately not in the shipped catalog + c.execute("CREATE FUNCTION pgcolumnar.debug_advance_reserved_offset(regclass, int) " + "RETURNS void AS 'pgcolumnar', 'pgcolumnar_debug_advance_reserved_offset' LANGUAGE C") + c.execute("CREATE TABLE h (id int, v text)") + c.execute("CREATE TABLE n (id int, v text) USING pgcolumnar") + c.execute("SELECT pgcolumnar.set_options('n', stripe_row_limit => 1000, chunk_group_row_limit => 1000)") + r = {} + _both(c, "INSERT INTO {t} SELECT g, md5(g::text) FROM generate_series(1, 3000) g") + r["baseline count"], r["baseline"] = _one(c, "SELECT count(*) FROM n"), _hashes(c) + before = _size(c) + + # the highwater 50 pages past the physical EOF: the next write targets beyond it + c.execute("SELECT pgcolumnar.debug_advance_reserved_offset('n', 50)") + _both(c, "INSERT INTO {t} SELECT g, md5(g::text) FROM generate_series(3001, 6000) g") + r["gapped count"], r["gapped"] = _one(c, "SELECT count(*) FROM n"), _hashes(c) + r["growth"] = _size(c) - before + print(f"-- file size: before-gap={before} after-gap={before + r['growth']}") + + _both(c, "INSERT INTO {t} SELECT g, md5(g::text) FROM generate_series(6001, 7000) g") + r["post-gap"] = _hashes(c) + c.execute("CREATE INDEX n_id ON n (id)") + # forced to fetch every row through n_id: by default this count is an Index + # Only Scan, which stops fetching once a VACUUM marks the pages all-visible + for s in IDX_SETS: + c.execute(s) + try: + r["plan"] = "\n".join(row[0] for row in c.execute(f"EXPLAIN (COSTS OFF) {LOOKUP}")) + r["lookup"] = _one(c, LOOKUP) + finally: + for g in ("enable_seqscan", "enable_bitmapscan", "pgcolumnar.enable_custom_scan"): + c.execute(f"RESET {g}") + + _both(c, "DELETE FROM {t} WHERE id % 7 = 0") + r["deleted"] = _hashes(c) + yield r + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_baseline_row_count(gap, expect): + expect.text(gap["baseline count"], "3000", "baseline row count") + + +def test_baseline_parity(gap, expect): + expect.text(*gap["baseline"], "baseline parity") + + +def test_row_count_after_gapped_insert(gap, expect): + expect.text(gap["gapped count"], "6000", "row count after gapped insert") + + +def test_parity_after_gapped_insert(gap, expect): + expect.text(*gap["gapped"], "parity after gapped insert") + + +def test_gap_was_physically_materialized(gap, expect): + # the 50-page gap and the new rows, with slack: at least 40 pages of growth + expect.at_least(gap["growth"], 40 * 8192 + 1, "gap was physically materialized") + + +def test_parity_after_post_gap_insert(gap, expect): + expect.text(*gap["post-gap"], "parity after post-gap insert") + + +def test_the_indexed_lookup_reads_n_id(gap, expect): + expect.text("yes" if "Index Scan using n_id" in gap["plan"] else "no", "yes", + "premise: the indexed lookup reads n_id") + + +def test_indexed_lookup_after_gap(gap, expect): + expect.text(gap["lookup"], "7000", "indexed lookup after gap") + + +def test_parity_after_delete_on_gapped_layout(gap, expect): + expect.text(*gap["deleted"], "parity after delete on gapped layout") diff --git a/test/pytest/test_native_roundtrip.py b/test/pytest/test_native_roundtrip.py new file mode 100644 index 000000000..6ebd44a34 --- /dev/null +++ b/test/pytest/test_native_roundtrip.py @@ -0,0 +1,83 @@ +"""The native round trip (Phase D3). The pytest twin of `test/native_roundtrip.sh`, every +check by the shell's name. + +A native-format table is read back through the native reader and must match a heap +mirror: scalars with nulls across five row groups, a filter, a projection, an +aggregate and a filtered count, compared order-independently so the heap is the +oracle. Each check is its own test. +""" +import psycopg +import pytest + +HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) " + "FROM (SELECT _row::text AS t FROM ({}) _row) _s") +GEN = """SELECT g, + (g * 100000)::bigint, + CASE WHEN g % 9 = 0 THEN NULL ELSE 'row-' || g END, + CASE WHEN g % 6 = 0 THEN NULL ELSE (g::float8 / 7) END, + (g % 2 = 0) + FROM generate_series(1, 5000) g""" + + +def _one(c, sql): + try: + return str(c.execute(sql).fetchone()[0]) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +@pytest.fixture(scope="module") +def rt(pgc_cluster): + db = "pgc_roundtrip" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + c.execute("CREATE TABLE h (id int, k bigint, v text, f float8, b bool)") + c.execute("CREATE TABLE n (id int, k bigint, v text, f float8, b bool) USING pgcolumnar") + # a small row-group limit, so the read crosses several row groups + c.execute("SELECT pgcolumnar.set_options('n', stripe_row_limit => 1000)") + c.execute(f"INSERT INTO h {GEN}") + c.execute(f"INSERT INTO n {GEN}") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def _pair(c, tmpl): + """-> (columnar, heap) answers of `tmpl` with {t} for the table.""" + return _one(c, tmpl.format(t="n")), _one(c, tmpl.format(t="h")) + + +# one check per test, by the shell's name: one red cannot stop the next (#1518 review) +READS = (("native reads back all columns", "SELECT id, k, v, f, b FROM {t}"), + ("native filtered scan", "SELECT id, v FROM {t} WHERE k BETWEEN 100000000 AND 200000000"), + ("native projection (subset of columns)", "SELECT v, b FROM {t}")) +ANSWERS = (("native null handling (v)", "SELECT count(*) FROM {t} WHERE v IS NULL"), + ("native null handling (f)", "SELECT count(*) FROM {t} WHERE f IS NULL"), + ("native aggregate (sum over a scanned column)", "SELECT sum(k) FROM {t}"), + ("native count with filter", "SELECT count(*) FROM {t} WHERE b")) + + +def test_native_row_count(rt, expect): + # answered from metadata, so a reader that drops a row group still counts 5000 + expect.text(_one(rt, "SELECT count(*) FROM n"), "5000", "native row count") + + +def test_the_native_table_holds_five_row_groups(rt, expect): + expect.text(_one(rt, "SELECT count(*) FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('n')"), "5", + "the native table holds five row groups") + + +@pytest.mark.parametrize("name,sql", READS, ids=[n for n, _ in READS]) +def test_a_read_matches_the_heap(rt, expect, name, sql): + expect.text(*_pair(rt, HASH.format(sql)), name) + + +@pytest.mark.parametrize("name,sql", ANSWERS, ids=[n for n, _ in ANSWERS]) +def test_a_null_count_or_aggregate_matches_the_heap(rt, expect, name, sql): + expect.text(*_pair(rt, sql), name) diff --git a/test/pytest/unported_suites.txt b/test/pytest/unported_suites.txt index ae0bd9e62..89960f625 100644 --- a/test/pytest/unported_suites.txt +++ b/test/pytest/unported_suites.txt @@ -75,7 +75,6 @@ logical_subscriber maintenance_due native_agg_addcolumn native_agg_deletes -native_bloom native_cancel native_cluster native_compact @@ -88,7 +87,6 @@ native_fetch_interrupt native_fetch_sort_context native_fold_deferral native_fold_skipguard -native_gap native_groupagg_wide_cost native_late_materialization native_lazy_slot @@ -108,7 +106,6 @@ native_read_parquet native_reclaim_reconcile native_rewrite native_rewrite_conc -native_roundtrip native_sort_by native_toasted_write native_varlena_bound