test: native_bloom gains its pytest twin (#1341)

test/check_ledger.tsv

@@ -2367,6 +2367,13 @@ native_batch_fold_projection	native_batch_fold_projection	ref is a non-empty res
 native_batch_fold_projection	native_batch_fold_projection	the folded aggregate equals the non-fold aggregate (payload columns correct)	15;16;17;18;19	2026-10-16	columnar_vector.c: the batch fold's payload list omits the table's last column
 native_batch_fold_projection	native_batch_fold_projection	the non-fold aggregate equals the generator's own answer	15;16;17;18;19	never	-
 native_batch_fold_projection	native_batch_fold_projection	the ungrouped batch fold actually ran (Batch Fold: yes)	15;16;17;18;19	2026-10-16	columnar_vector.c: the ungrouped aggregate never takes the batch fold (row path instead)
+native_bloom	native_bloom	absent value parity	15;16;17;18;19	never	-
+native_bloom	native_bloom	bloom off skips nothing (min/max cannot)	15;16;17;18;19	2026-10-16	columnar_reader.c: the bloom is consulted whatever pgcolumnar.enable_bloom_filter says
+native_bloom	native_bloom	bloom on skips groups	15;16;17;18;19	2026-10-16	columnar_reader.c: the reader never consults the bloom;columnar_write_state.c: the writer builds no bloom
+native_bloom	native_bloom	bloom rows written	15;16;17;18;19	2026-10-16	columnar_write_state.c: the writer builds no bloom
+native_bloom	native_bloom	equality probe parity	15;16;17;18;19	2026-10-16	columnar_reader.c: the bloom answers absent for every value, so a matching group is pruned
+native_bloom	native_bloom	row count	15;16;17;18;19	never	-
+native_bloom	native_bloom	the plan under test is a columnar custom scan	15;16;17;18;19	never	-
 native_chunk_length_bound	native_chunk_length_bound	a sequential scan of the same poisoned chunk is refused (XX001)	15;16;17;18;19	never	-
 native_chunk_length_bound	native_chunk_length_bound	an index fetch of a chunk whose page_length is 2^32 too large is refused (XX001)	15;16;17;18;19	never	-
 native_chunk_length_bound	native_chunk_length_bound	backend survived the sequential refusal	15;16;17;18;19	never	-
@@ -2566,6 +2573,15 @@ native_fetch_projection	native_fetch_projection	premise: there is a needed-set m
 native_fetch_projection	native_fetch_projection	the all-covering projection reconstructs every row	15;16;17;18;19	never	-
 native_fetch_projection	native_fetch_projection	the projection reads every row to start with	15;16;17;18;19	never	-
 native_fetch_projection	native_fetch_projection	the wide fixture has 41 columns	15;16;17;18;19	never	-
+native_gap	native_gap	baseline parity	15;16;17;18;19	never	-
+native_gap	native_gap	baseline row count	15;16;17;18;19	never	-
+native_gap	native_gap	gap was physically materialized	15;16;17;18;19	2026-10-16	columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors;columnar_vacuum.c: debug_advance_reserved_offset advances nothing, so no gap is made
+native_gap	native_gap	indexed lookup after gap	15;16;17;18;19	2026-10-16	columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors;columnar_tableam.c: an index fetch finds no row
+native_gap	native_gap	parity after delete on gapped layout	15;16;17;18;19	2026-10-16	columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors
+native_gap	native_gap	parity after gapped insert	15;16;17;18;19	2026-10-16	columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors
+native_gap	native_gap	parity after post-gap insert	15;16;17;18;19	2026-10-16	columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors
+native_gap	native_gap	premise: the indexed lookup reads n_id	15;16;17;18;19	never	-
+native_gap	native_gap	row count after gapped insert	15;16;17;18;19	2026-10-16	columnar_storage.c: PgColumnarWriteLogicalData does not fill a gap beyond EOF (6a165821 reverted), so the write lands short of its block and errors
 native_groupagg	native_groupagg	a NULL key counts as a group in the bucket bound: four groups under a cap of 3 are declined and all answered	15;16;17;18;19	2026-10-16	the bucket bound left out the NULL group and the runtime cap was off, so the node took the query and answered it (mutation, #1339)
 native_groupagg	native_groupagg	a cached plan run under a lower cap stops at the runtime cap, with its hint	15;16;17;18;19	never	-
 native_groupagg	native_groupagg	a date_bin key is bounded as a truncating key is	15;16;17;18;19	never	-
@@ -3061,6 +3077,15 @@ native_repack	native_repack	the columnar table is unharmed: content hash	19	2026
 native_repack	native_repack	the columnar table is unharmed: row count	19	2026-10-16	columnar_tableam.c: relation_copy_for_cluster returns without raising, so a rewrite copies nothing
 native_repack	native_repack	the columnar table is unharmed: storage options	19	never	-
 native_repack	native_repack	without an identity index it is refused by PostgreSQL, not by us	19	never	-
+native_roundtrip	native_roundtrip	native aggregate (sum over a scanned column)	15;16;17;18;19	2026-10-17	columnar_reader.c: the native reader never loads the last row group
+native_roundtrip	native_roundtrip	native count with filter	15;16;17;18;19	2026-10-17	columnar_reader.c: the native reader never loads the last row group
+native_roundtrip	native_roundtrip	native filtered scan	15;16;17;18;19	2026-10-17	columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: zone-map pruning skips the groups that can match and keeps the ones that cannot
+native_roundtrip	native_roundtrip	native null handling (f)	15;16;17;18;19	2026-10-17	columnar_reader.c: a null by-value column reads back as a present 0;columnar_reader.c: the native reader never loads the last row group
+native_roundtrip	native_roundtrip	native null handling (v)	15;16;17;18;19	2026-10-17	columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: the native reader never loads the last row group
+native_roundtrip	native_roundtrip	native projection (subset of columns)	15;16;17;18;19	2026-10-17	columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: the native reader never loads the last row group
+native_roundtrip	native_roundtrip	native reads back all columns	15;16;17;18;19	2026-10-17	columnar_reader.c: a null by-reference column reads back as a present empty value;columnar_reader.c: a null by-value column reads back as a present 0;columnar_reader.c: the native reader never loads the last row group
+native_roundtrip	native_roundtrip	native row count	15;16;17;18;19	never	-
+native_roundtrip	native_roundtrip	the native table holds five row groups	15;16;17;18;19	never	-
 native_saop_pushdown	native_saop_pushdown	IN-list count matches the OR-literal equivalent	15;16;17;18;19	2026-10-16	columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out)
 native_saop_pushdown	native_saop_pushdown	NOT IN builds no range (0 removed) and stays correct	15;16;17;18;19	never	-
 native_saop_pushdown	native_saop_pushdown	NULL-element count is exact	15;16;17;18;19	never	-

test/check_ledger_budget.txt

@@ -15,7 +15,7 @@
 #
 #   Adding a check to a suite that is already covered does not move it, which is
 #   what makes it safe to bound.
-suites_not_covered 189
+suites_not_covered 186
 #
 # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become
 #   one.
@@ -3560,4 +3560,44 @@ suites_not_covered 189
 #     never         3055 + 11 = 3066
 #     dated         1342 +  7 = 1349
 #     3066 + 1349 == 4415
-checks_never_observed_red 3066
+#
+# #1341, native_gap seeded, with its twin (OffgridwithJD): 3066 -> 3069, and
+# suites_not_covered 189 -> 188. The suite had no rows. It enters with all 9 checks
+# from five-major runs (9/9 on each of 15-19). 6 are dated by three mutations: the
+# gap fill removed (6a165821's pre-fix write path, where the write lands short of
+# its block and errors), the test hook advancing nothing, and an index fetch finding
+# no row (the indexed lookup, now forced through n_id with a premise). Three arrive
+# never: the two baseline checks and the index premise.
+#
+#     ledger rows   4415 +  9 = 4424
+#     never         3066 +  3 = 3069
+#     dated         1349 +  6 = 1355
+#     3069 + 1355 == 4424
+#
+# #1341, native_roundtrip seeded, with its twin (OffgridwithJD): 3069 -> 3071, and
+# suites_not_covered 188 -> 187. The suite had no rows. It enters with all 9 checks
+# from five-major runs (9/9 on each of 15-19), one of them new: the table holds five
+# row groups, so the read crosses them. 7 are dated by four mutations in the native
+# reader: a null by-value column read back as a present 0, a null by-reference one
+# read back as a present empty value (jdatcmd's, #1518 review), the last row group
+# never loaded, and zone-map pruning inverted. Two arrive never: the row count,
+# which the metadata path answers, and the row-group count.
+#
+#     ledger rows   4424 +  9 = 4433
+#     never         3069 +  2 = 3071
+#     dated         1355 +  7 = 1362
+#     3071 + 1362 == 4433
+#
+# #1341, native_bloom seeded, with its twin (OffgridwithJD): 3071 -> 3074, and
+# suites_not_covered 187 -> 186. The suite had no rows. It enters with all 7 checks
+# from five-major runs (7/7 on each of 15-19). 4 are dated by four mutations: the
+# bloom consulted whatever the GUC says, never consulted, answering absent for every
+# value (a false negative pruning the match), and never written. Three arrive never:
+# the row count, the custom-scan premise, and the absent-value parity, whose value
+# lies outside every group's range, so min/max prunes it before the bloom is asked.
+#
+#     ledger rows   4433 +  7 = 4440
+#     never         3071 +  3 = 3074
+#     dated         1362 +  4 = 1366
+#     3074 + 1366 == 4440
+checks_never_observed_red 3074

test/native_gap.sh

@@ -56,8 +56,16 @@ psql_run "INSERT INTO h SELECT g, md5(g::text) FROM generate_series(6001, 7000)
 psql_run "INSERT INTO n SELECT g, md5(g::text) FROM generate_series(6001, 7000) g;"
 check "parity after post-gap insert" "$(hash_n)" "$(hash_h)"
 psql_run "CREATE INDEX n_id ON n (id);"
+# Forced to fetch every row through n_id, and asserted (#1341). With the defaults
+# this count plans an Index Only Scan (PG17 and PG18), which fetches each row only
+# while the visibility map leaves its page unmarked: once a VACUUM marks them it
+# answers from the index alone, whatever the fetch does. count(v), not count(*),
+# with the custom scan, seqscan and bitmap scans off, so the plan is an Index Scan.
+IDX_SETS="SET enable_seqscan = off; SET enable_bitmapscan = off; SET pgcolumnar.enable_custom_scan = off;"
+check "premise: the indexed lookup reads n_id" \
+	"$(pgc_uses_row_fetch "$IDX_SETS" "SELECT count(v) FROM n WHERE id BETWEEN 1 AND 7000")" yes
 check "indexed lookup after gap" \
-	"$(q 'SELECT count(*) FROM n WHERE id BETWEEN 1 AND 7000;')" "7000"
+	"$(q "$IDX_SETS SELECT count(v) FROM n WHERE id BETWEEN 1 AND 7000;" | tail -1)" "7000"
 
 # a delete + reader still resolves correctly across the gapped layout.
 psql_run "DELETE FROM h WHERE id % 7 = 0;"

test/native_roundtrip.sh

@@ -29,6 +29,10 @@ psql_run "INSERT INTO h $GEN;"
 psql_run "INSERT INTO n $GEN;"
 
 check "native row count" "$(q 'SELECT count(*) FROM n;')" "5000"
+# the header's claim, asserted (#1341): stripe_row_limit 1000 over 5000 rows, so the
+# read crosses row groups rather than decoding one
+check "the native table holds five row groups" \
+	"$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('n');")" "5"
 check "native reads back all columns" \
 	"$(pgc_set_hash 'SELECT id, k, v, f, b FROM n')" \
 	"$(pgc_set_hash 'SELECT id, k, v, f, b FROM h')"

test/pytest/TESTS.md

@@ -222,6 +222,9 @@ behaviour, the source of that number is named.
 - [174. test_sorted_projection.py: vacuum_sorted reorders storage and changes no answer](#174-test_sorted_projectionpy-vacuum_sorted-reorders-storage-and-changes-no-answer)
 - [175. test_native_ios.py: an index-only scan skips the fetch on all-visible row groups](#175-test_native_iospy-an-index-only-scan-skips-the-fetch-on-all-visible-row-groups)
 - [176. test_native_truncate.py: end-truncation shrinks the file and loses no row](#176-test_native_truncatepy-end-truncation-shrinks-the-file-and-loses-no-row)
+- [177. test_native_gap.py: a write beyond EOF fills the gap and loses no row](#177-test_native_gappy-a-write-beyond-eof-fills-the-gap-and-loses-no-row)
+- [178. test_native_roundtrip.py: a native table reads back what the heap holds](#178-test_native_roundtrippy-a-native-table-reads-back-what-the-heap-holds)
+- [179. test_native_bloom.py: a bloom filter prunes the equality probe min/max cannot](#179-test_native_bloompy-a-bloom-filter-prunes-the-equality-probe-minmax-cannot)
 
 ## 1. How to read a test in here
 
@@ -7836,3 +7839,56 @@ The pytest twin of `test/native_truncate.sh` (ported for #1341), every check by
 | `test_truncate_shrinks_the_file_and_keeps_every_row` | a truncate with nothing freed returns 0; after delete and compact it returns blocks, the file shrinks, the rows match the heap, and a second truncate returns 0 |
 | `test_the_table_stays_usable_and_the_guc_turns_it_off` | reinsert and an index lookup through the index after truncation; with the GUC off truncate returns 0 and the file keeps its size, with it on the file shrinks |
 | `test_truncate_refuses_a_transaction_block_and_reuse_stays_clean` | refused inside BEGIN with the file and rows unchanged; truncate, reinsert and compaction keep parity |
+
+## 177. test_native_gap.py: a write beyond EOF fills the gap and loses no row
+
+The pytest twin of `test/native_gap.sh` (ported for #1341), every check by the shell's name. The metapage highwater can sit ahead of the physical EOF, which is the state an aborted or crash-interrupted end-truncation leaves. A write whose target block lies beyond EOF must fill the gap with empty pages and succeed. A test-only hook, bound by the test as the shell binds it, pushes the highwater 50 pages past EOF. The next insert must keep every row against a heap mirror and grow the file by at least 40 pages. The table then stays usable: another insert, an index lookup forced through the index and asserted to read it, and a delete.
+
+### Every test
+
+Each check is its own test over a module fixture that runs every step in the shell's order and records each reading, so one red cannot stop the checks after it.
+
+| test | what it holds |
+| --- | --- |
+| `test_baseline_row_count` | 3000 rows before the gap |
+| `test_baseline_parity` | and they match the heap |
+| `test_row_count_after_gapped_insert` | 6000 rows after the insert across a 50-page gap |
+| `test_parity_after_gapped_insert` | and they match the heap |
+| `test_gap_was_physically_materialized` | the file grew by more than 40 pages |
+| `test_parity_after_post_gap_insert` | a later insert keeps parity |
+| `test_the_indexed_lookup_reads_n_id` | the forced lookup plans an Index Scan on `n_id` |
+| `test_indexed_lookup_after_gap` | and finds all 7000 rows |
+| `test_parity_after_delete_on_gapped_layout` | a delete keeps parity |
+
+## 178. test_native_roundtrip.py: a native table reads back what the heap holds
+
+The pytest twin of `test/native_roundtrip.sh` (ported for #1341), every check by the shell's name. A native-format table with nulls in a text and a float column is read back through the native reader and compared with a heap mirror holding the same rows. Its 5000 rows are split into five row groups, and a check asserts the count. The row count is answered from metadata, so only the hashed reads see a lost row group. The comparisons cover every column, a filtered scan, a projection, the null counts, an aggregate and a filtered count. Rows are hashed order-independently, so the heap is the oracle.
+
+### Every test
+
+Each check is its own test, parametrized by the shell's name, so one red cannot stop the others.
+
+| test | what it holds |
+| --- | --- |
+| `test_native_row_count` | 5000 rows, as metadata counts them |
+| `test_the_native_table_holds_five_row_groups` | five row groups, so the read crosses them |
+| `test_a_read_matches_the_heap[...]` | every column, a filtered scan and a projection match the heap, one test per shell name |
+| `test_a_null_count_or_aggregate_matches_the_heap[...]` | the null counts of the text and float columns, `sum(k)` and `count(*) WHERE b` match the heap, one test per shell name |
+
+## 179. test_native_bloom.py: a bloom filter prunes the equality probe min/max cannot
+
+The pytest twin of `test/native_bloom.sh` (ported for #1341), every check by the shell's name. A native table carries a bloom filter per column chunk over each hashable column. An equality probe on an unsorted column can then skip row groups that provably lack the value, even where the min/max zone map cannot. The fixture spreads its values so that every group's range spans the domain. Groups are therefore skipped only with `pgcolumnar.enable_bloom_filter` on, and the probe's answer matches a heap mirror. The skip counter reads -1 when the plan has none, so a plan without the columnar scan can't pass as "skipped nothing".
+
+### Every test
+
+Each check is its own test, so one red cannot stop the next.
+
+| test | what it holds |
+| --- | --- |
+| `test_row_count` | 20480 rows |
+| `test_bloom_rows_written` | one filter per row group for the tag column |
+| `test_the_plan_under_test_is_a_columnar_custom_scan` | the probe plans the columnar scan, the only source of the skip counter |
+| `test_equality_probe_parity` | the probe for a present value matches the heap |
+| `test_bloom_on_skips_groups` | the probe skips groups with the bloom on |
+| `test_bloom_off_skips_nothing` | and none with it off |
+| `test_absent_value_parity` | a value outside every group's range matches the heap |

test/pytest/expected_tests.txt

@@ -823,4 +823,10 @@ guard_tests 440
 # one test, by collection over the cluster files on this tree.
 # cluster_tests 751 -> 754 for #1341's test_native_truncate.py, the twin of native_truncate.sh:
 # three tests, by collection over the cluster files on this tree.
-cluster_tests 754
+# cluster_tests 754 -> 763 for #1341's test_native_gap.py, the twin of native_gap.sh:
+# nine tests, one per check, by collection over the cluster files on this tree.
+# cluster_tests 763 -> 772 for #1341's test_native_roundtrip.py, the twin of native_roundtrip.sh:
+# nine tests, one per check, by collection over the cluster files on this tree.
+# cluster_tests 772 -> 779 for #1341's test_native_bloom.py, the twin of native_bloom.sh:
+# seven tests, one per check, by collection over the cluster files on this tree.
+cluster_tests 779

test/pytest/test_compare_to_bash.py

@@ -122,6 +122,7 @@
             "native_agg",
             "native_backend_crash",
             "native_batch_fold_projection",
+            "native_bloom",
             "native_chunk_length_bound",
             "native_decode_gate_width",
             "native_decode_gating",
@@ -135,6 +136,7 @@
             "native_fetch_coalesce",
             "native_fetch_position",
             "native_fetch_projection",
+            "native_gap",
             "native_groupagg",
             "native_groupagg_batch",
             "native_index",
@@ -155,6 +157,7 @@
             "native_reclaim_frag",
             "native_recluster",
             "native_repack",
+            "native_roundtrip",
             "native_saop_pushdown",
             "native_truncate",
             "native_vacuum_race",