diff --git a/src/columnar_vector.c b/src/columnar_vector.c index 87b795118..a2c6e725f 100644 --- a/src/columnar_vector.c +++ b/src/columnar_vector.c @@ -4196,9 +4196,11 @@ pgcolumnar_batch_gates_ok(const PgColumnarAggSpec *specs, int naggs, * PgColumnarQualsExactlyKeyed demands one EXACT key per clause, so those keys * never serve as the fold's WHERE even if pgcolumnar_clause_to_predicate were * later taught to accept a SAOP. The byval gather guard below is a second, - * independent reason LIKE and other varlena filters fall back. Both - * native_saop_pushdown.sh's vector-agg arms and ungrouped_vector_agg.sh's - * #715 arms go red if this regresses. + * independent reason LIKE and other varlena filters fall back. + * ungrouped_vector_agg.sh's #715 arms go red if this regresses. For a SAOP + * it is one of three gates: allConvertible above and the sk_flags test on + * the keys below refuse it too, so native_saop_pushdown.sh's vector-agg + * arms go red only with all three removed (measured, #1341). */ if (!PgColumnarQualsExactlyKeyed(quals, scanrelid, tupdesc)) return false; diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 10a8cd6b2..4cf738e9a 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2795,6 +2795,59 @@ native_repack native_repack the columnar table is unharmed: content hash 19 2026 native_repack native_repack the columnar table is unharmed: row count 19 2026-10-16 columnar_tableam.c: relation_copy_for_cluster returns without raising, so a rewrite copies nothing native_repack native_repack the columnar table is unharmed: storage options 19 never - native_repack native_repack without an identity index it is refused by PostgreSQL, not by us 19 never - +native_saop_pushdown native_saop_pushdown IN-list count matches the OR-literal equivalent 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) +native_saop_pushdown native_saop_pushdown NOT IN builds no range (0 removed) and stays correct 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown NULL-element count is exact 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown SK_SEARCHARRAY reaches the dedicated array predicate arm 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown a C-collated text IN-list prunes (19 removed) 15;16;17;18;19 2026-10-16 columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted);columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) +native_saop_pushdown native_saop_pushdown a NULL element is ignored for the range, not a bailout (19 removed) 15;16;17;18;19 2026-10-16 columnar_customscan.c: a NULL element makes pgcolumnar_saop_scankey build no key, not skip the element;columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted) +native_saop_pushdown native_saop_pushdown a by-reference text set strips NULL without crashing (17 removed) 15;16;17;18;19 2026-10-16 columnar_customscan.c: a NULL element makes pgcolumnar_saop_scankey build no key, not skip the element;columnar_customscan.c: every IN-list takes the two-key [min,max] hull, never the set key;columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted);columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out);columnar_reader.c: the zone map tests only a set key's first element +native_saop_pushdown native_saop_pushdown a contiguous IN-list agrees with its hull (19 removed) 15;16;17;18;19 2026-10-16 columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted);columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) +native_saop_pushdown native_saop_pushdown a correlated PARAM_EXEC array is not mis-pruned (results stay correct) 15;16;17;18;19 2026-10-16 columnar_customscan.c: the operand walker calls a correlated PARAM_EXEC stable, so it is frozen at Begin (the backend crashes) +native_saop_pushdown native_saop_pushdown a cross-type element array (int col, bigint[] list) prunes (19 removed) 15;16;17;18;19 2026-10-16 columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted);columnar_reader.c: a cross-type key is dropped by the reader +native_saop_pushdown native_saop_pushdown a domain-element array prunes (19 removed) 15;16;17;18;19 2026-10-16 columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted);columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) +native_saop_pushdown native_saop_pushdown a generic-plan parameterized array prunes (19 removed) 15;16;17;18;19 2026-10-16 columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted);columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) +native_saop_pushdown native_saop_pushdown a large IN-list falls back to two bounded hull filters 15;16;17;18;19 2026-10-16 columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted);columnar_customscan.c: the set-key bound one long, so 129 elements stay a set key +native_saop_pushdown native_saop_pushdown a mixed literal+param IN-list prunes on a generic plan (19 removed) 15;16;17;18;19 2026-10-16 columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted);columnar_customscan.c: pgcolumnar_stabilize_quals freezes only a bare Param, not an ArrayExpr holding one;columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) +native_saop_pushdown native_saop_pushdown a multi-value integer set strips NULL and still prunes 17 groups 15;16;17;18;19 2026-10-16 columnar_customscan.c: a NULL element makes pgcolumnar_saop_scankey build no key, not skip the element;columnar_customscan.c: every IN-list takes the two-key [min,max] hull, never the set key;columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted);columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out);columnar_reader.c: the zone map tests only a set key's first element +native_saop_pushdown native_saop_pushdown a multi-value set bloom-prunes all overlapping groups 15;16;17;18;19 2026-10-16 columnar_customscan.c: every IN-list takes the two-key [min,max] hull, never the set key;columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted) +native_saop_pushdown native_saop_pushdown a one-element IN-list (parser-collapsed to =) bloom-prunes (20 removed) 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown a range-spanning IN-list prunes between its elements (18 removed, exact count) 15;16;17;18;19 2026-10-16 columnar_customscan.c: every IN-list takes the two-key [min,max] hull, never the set key;columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted);columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out);columnar_reader.c: the zone map tests only a set key's first element +native_saop_pushdown native_saop_pushdown a scattered IN-list prunes by element, not only by its hull (17 removed) 15;16;17;18;19 2026-10-16 columnar_customscan.c: every IN-list takes the two-key [min,max] hull, never the set key;columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted);columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out);columnar_reader.c: the zone map tests only a set key's first element +native_saop_pushdown native_saop_pushdown a single-DISTINCT-value SAOP list ('{25,25}') bloom-prunes (20 removed) 15;16;17;18;19 2026-10-16 columnar_customscan.c: a single-distinct-value list emitted as the [v,v] range, not an equality key;columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted) +native_saop_pushdown native_saop_pushdown an IN-list confined to the last group prunes (19 of 20 removed) 15;16;17;18;19 2026-10-16 columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted);columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) +native_saop_pushdown native_saop_pushdown an IN-list pushes one set filter (intent) 15;16;17;18;19 2026-10-16 columnar_customscan.c: every IN-list takes the two-key [min,max] hull, never the set key;columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted) +native_saop_pushdown native_saop_pushdown an all-NULL array counts zero and does not crash 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown an empty array counts zero and does not crash 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown backend alive 15;16;17;18;19 2026-10-16 columnar_customscan.c: the operand walker calls a correlated PARAM_EXEC stable, so it is frozen at Begin (the backend crashes) +native_saop_pushdown native_saop_pushdown contiguous per-element pruning keeps the exact answer 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) +native_saop_pushdown native_saop_pushdown cross-type count is exact 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown domain-element count is exact 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) +native_saop_pushdown native_saop_pushdown exactly 128 elements still use one set filter 15;16;17;18;19 2026-10-16 columnar_customscan.c: every IN-list takes the two-key [min,max] hull, never the set key;columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted);columnar_customscan.c: the set-key bound one short, so 128 elements fall back to the hull +native_saop_pushdown native_saop_pushdown fixture premise: 25 is inside every group's range (range keys prune 0) 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown fixture premise: equality on the overlap column bloom-prunes all 20 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown parameterized array count matches the literal count 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) +native_saop_pushdown native_saop_pushdown premise: plain-key special flags have one reject expression 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown premise: the reject expression was extracted, not blank 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown scattered per-element pruning keeps the exact answer 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out);columnar_reader.c: the zone map tests only a set key's first element +native_saop_pushdown native_saop_pushdown single-value IN-list count is exact (0 rows) 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown text IN-list count is exact 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) +native_saop_pushdown native_saop_pushdown the IN-list set filter is a usable skip predicate 15;16;17;18;19 2026-10-16 columnar_customscan.c: every IN-list takes the two-key [min,max] hull, never the set key;columnar_customscan.c: pgcolumnar_saop_scankey builds no key for any ScalarArrayOpExpr (the #704 fix reverted) +native_saop_pushdown native_saop_pushdown the batch fold accepts the equivalent exact range quals (Batch Fold: yes) 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown the batch fold refuses the IN-list (Batch Fold: no) 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown the bloom-only multi-value set returns the exact empty answer 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown the by-reference text NULL set stays exact 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out);columnar_reader.c: the zone map tests only a set key's first element +native_saop_pushdown native_saop_pushdown the large-list fallback keeps the exact answer 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown the multi-value integer NULL set stays exact 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out);columnar_reader.c: the zone map tests only a set key's first element +native_saop_pushdown native_saop_pushdown the predicate builder's reject expression contains SK_ISNULL 15;16;17;18;19 never - +native_saop_pushdown native_saop_pushdown the predicate builder's reject expression contains SK_ORDER_BY 15;16;17;18;19 2026-10-16 columnar_reader.c: the reject expression keeps only SK_ISNULL +native_saop_pushdown native_saop_pushdown the predicate builder's reject expression contains SK_ROW_END 15;16;17;18;19 2026-10-16 columnar_reader.c: the reject expression keeps only SK_ISNULL +native_saop_pushdown native_saop_pushdown the predicate builder's reject expression contains SK_ROW_HEADER 15;16;17;18;19 2026-10-16 columnar_reader.c: the reject expression keeps only SK_ISNULL +native_saop_pushdown native_saop_pushdown the predicate builder's reject expression contains SK_ROW_MEMBER 15;16;17;18;19 2026-10-16 columnar_reader.c: the reject expression keeps only SK_ISNULL +native_saop_pushdown native_saop_pushdown the predicate builder's reject expression contains SK_SEARCHNOTNULL 15;16;17;18;19 2026-10-16 columnar_reader.c: the reject expression keeps only SK_ISNULL +native_saop_pushdown native_saop_pushdown the predicate builder's reject expression contains SK_SEARCHNULL 15;16;17;18;19 2026-10-16 columnar_reader.c: the reject expression keeps only SK_ISNULL +native_saop_pushdown native_saop_pushdown vector-agg count over an IN-list is exact (conservative keys not folded) 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) +native_saop_pushdown native_saop_pushdown vector-agg path engaged for the guard arm (plan shape) 15;16;17;18;19 never - native_vecskip native_vecskip and the aggregate skips the same seven vectors 15;16;17;18;19 2026-10-16 columnar_reader.c: no vector skip at all (zone-map rule off, refine_skipvec and qual_skipvec return at once) native_vecskip native_vecskip cross-vector range parity 15;16;17;18;19 never - native_vecskip native_vecskip empty range parity 15;16;17;18;19 never - @@ -3319,23 +3372,23 @@ replication replication with replay paused the standby does NOT see the new rows replication replication with the module back, the replayed table reads correctly 15;16;17;18;19 2026-10-16 columnar_storage.c: PgColumnarWriteLogicalData logs no data or gap page (if (false) for both RelationNeedsWAL) replication replication with the same content the primary reads 15;16;17;18;19 never - rewrite_storage_oid rewrite_storage_oid a matview created WITH DATA leaves its storage row pointing at itself 15;16;17;18;19 2026-10-12 the repair gate skips every relation, as if no inheritor were locked by the statement;the rewrite repair gate no longer covers CreateTableAsStmt, so a matview created WITH DATA keeps the transient's OID +rewrite_storage_oid rewrite_storage_oid a table that was not rewritten still points at itself 15;16;17;18;19 2026-10-16 the storage repoint after a rewrite updated the first storage row, not the rewritten table's (mutation, #1339) rewrite_storage_oid rewrite_storage_oid a type rewrite leaves the storage row pointing at the live table 15;16;17;18;19 2026-10-12 the repair gate skips every relation, as if no inheritor were locked by the statement;the rewrite hook does not retarget storage.relation_oid rewrite_storage_oid rewrite_storage_oid and REFRESH still leaves it pointing at itself 15;16;17;18;19 2026-10-12 the repair gate skips every relation, as if no inheritor were locked by the statement;the rewrite repair gate no longer covers RefreshMatViewStmt rewrite_storage_oid rewrite_storage_oid and a rewrite through the parent still repoints every partition's storage row 15;16;17;18;19 2026-10-12 the repair gate skips every relation, as if no inheritor were locked by the statement rewrite_storage_oid rewrite_storage_oid control: reading lp_1 shows up as blocks read from it 15;16;17;18;19 never - rewrite_storage_oid rewrite_storage_oid premise: CREATE TABLE ... AS leaves its storage row pointing at itself 15;16;17;18;19 never - -rewrite_storage_oid rewrite_storage_oid a table that was not rewritten still points at itself 15;16;17;18;19 2026-10-16 the storage repoint after a rewrite updated the first storage row, not the rewritten table's (mutation, #1339) rewrite_storage_oid rewrite_storage_oid premise: both columnar partitions hold their rows 15;16;17;18;19 never - +rewrite_storage_oid rewrite_storage_oid premise: the storage row points at the live table before the rewrite 15;16;17;18;19 never - +rewrite_storage_oid rewrite_storage_oid premise: the table holds every inserted row 15;16;17;18;19 never - rewrite_storage_oid rewrite_storage_oid the CREATE TABLE AS table holds them too 15;16;17;18;19 2026-10-16 a bulk load (CREATE TABLE AS, REFRESH, ALTER TABLE rewrite) threw its buffered rows away at finish_bulk_insert (mutation, #1339) rewrite_storage_oid rewrite_storage_oid the matview holds the rows it was created with 15;16;17;18;19 2026-10-16 a bulk load (CREATE TABLE AS, REFRESH, ALTER TABLE rewrite) threw its buffered rows away at finish_bulk_insert (mutation, #1339) rewrite_storage_oid rewrite_storage_oid the refreshed matview still holds every row 15;16;17;18;19 2026-10-16 a bulk load (CREATE TABLE AS, REFRESH, ALTER TABLE rewrite) threw its buffered rows away at finish_bulk_insert (mutation, #1339) -rewrite_storage_oid rewrite_storage_oid the rewritten table still holds every row 15;16;17;18;19 2026-10-16 a bulk load (CREATE TABLE AS, REFRESH, ALTER TABLE rewrite) threw its buffered rows away at finish_bulk_insert (mutation, #1339) -rewrite_storage_oid rewrite_storage_oid premise: the storage row points at the live table before the rewrite 15;16;17;18;19 never - -rewrite_storage_oid rewrite_storage_oid premise: the table holds every inserted row 15;16;17;18;19 never - rewrite_storage_oid rewrite_storage_oid the repair gate does not read a partition the ALTER did not touch: ONLY lp ALTER COLUMN v SET DEFAULT 0 15;16;17;18;19 2026-10-12 the repair gate opens every inheritor with NoLock again, so it reads the metapage of a partition the ALTER never locked (an Assert on assert builds) rewrite_storage_oid rewrite_storage_oid the repair gate does not read a partition the ALTER did not touch: lp ATTACH PARTITION lp_3 FOR VALUES FROM ('2026-03-01') TO ('2026-04-01') 15;16;17;18;19 2026-10-12 the repair gate opens every inheritor with NoLock again, so it reads the metapage of a partition the ALTER never locked (an Assert on assert builds) rewrite_storage_oid rewrite_storage_oid the repair gate does not read a partition the ALTER did not touch: lp DETACH PARTITION lp_2 15;16;17;18;19 2026-10-12 the repair gate opens every inheritor with NoLock again, so it reads the metapage of a partition the ALTER never locked (an Assert on assert builds) rewrite_storage_oid rewrite_storage_oid the repair gate does not read a partition the ALTER did not touch: lp DETACH PARTITION lp_2 CONCURRENTLY 15;16;17;18;19 2026-10-12 the repair gate opens every inheritor with NoLock again, so it reads the metapage of a partition the ALTER never locked (an Assert on assert builds) +rewrite_storage_oid rewrite_storage_oid the rewritten table still holds every row 15;16;17;18;19 2026-10-16 a bulk load (CREATE TABLE AS, REFRESH, ALTER TABLE rewrite) threw its buffered rows away at finish_bulk_insert (mutation, #1339) rls_direct_storage rls_direct_storage a table with NO policy is still readable by the same role 15;16;17;18;19 never - rls_direct_storage rls_direct_storage and no imported row landed in the restricted table 15;16;17;18;19 2026-10-14 PgColumnarRequireNoRowSecurity never refuses (the RLS guard deleted) rls_direct_storage rls_direct_storage and read_projection tells it the same thing, not that RLS exists 15;16;17;18;19 2026-10-14 read_projection checks RLS before the table ACL diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 37c628816..dca2488fb 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 202 +suites_not_covered 201 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3256,4 +3256,25 @@ suites_not_covered 202 # never 2889 + 34 = 2923 # dated 1118 + 7 = 1125 # 2923 + 1125 == 4048 -checks_never_observed_red 2923 +# +# #1341, native_saop_pushdown seeded, with its twin (OffgridwithJD): 2923 -> 2941, +# and suites_not_covered 202 -> 201. The suite had no rows. It enters with all 53 +# checks from five-major runs (53/53 on each of 15-19). 35 are dated by twelve +# mutations of the set key: the #704 fix reverted, the hull always taken, the +# 128-element bound one short and one long, a NULL element bailing out, the single +# distinct value emitted as a range, the reader's bloom and zone-map set loops, the +# freeze of a mixed list and of a correlated PARAM_EXEC (which crashes the backend), +# a cross-type key dropped, and the reader's reject expression cut to SK_ISNULL. +# Eighteen arrive never: four premises; the NOT IN arm and the three batch-fold +# arms, each behind more than one gate (useOr and the equality strategy; the fold's +# allConvertible, exactness and sk_flags gates), so no single-gate mutation reddens +# them; the parser-collapsed one-element list, which never reaches the set key; +# seven exact counts no set-key mutation moves; SK_ISNULL, which cannot leave the +# reject expression without reddening its premise; and the SK_SEARCHARRAY arm. The +# merge also re-sorted five rewrite_storage_oid rows #1475 renamed in place. +# +# ledger rows 4048 + 53 = 4101 +# never 2923 + 18 = 2941 +# dated 1125 + 35 = 1160 +# 2941 + 1160 == 4101 +checks_never_observed_red 2941 diff --git a/test/native_saop_pushdown.sh b/test/native_saop_pushdown.sh index 2a0c2f435..644b9ee5c 100755 --- a/test/native_saop_pushdown.sh +++ b/test/native_saop_pushdown.sh @@ -162,8 +162,10 @@ check "the bloom-only multi-value set returns the exact empty answer" \ # `IN (25)` parses to plain `= 25` (the parser collapses a one-element list), # so it exercises the OpExpr path; the `= ANY('{25,25}')` arm below is the -# SAOP shape and is the removal proof for the single-distinct-value -# equality-key refinement (as two range keys it removes 0). +# SAOP shape. As two range keys [25,25] it removes 0, so it reddens when the +# single-distinct-value case is emitted as a range. It does NOT prove the +# equality-key refinement is needed: without it the list takes the set path, +# whose bloom loop probes 25 too and removes all 20 (measured). check "a one-element IN-list (parser-collapsed to =) bloom-prunes (20 removed)" \ "$(groups_removed 'SELECT count(*) FROM t WHERE ov IN (25);')" "20" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index baa90b2bf..a0f21f926 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -209,6 +209,7 @@ behaviour, the source of that number is named. - [161. test_native_parquet_schema.py: parquet_schema round trip](#161-test_native_parquet_schemapy-parquet_schema-round-trip) - [162. test_native_vecskip.py: per-vector skipping by the zone maps](#162-test_native_vecskippy-per-vector-skipping-by-the-zone-maps) - [163. test_native_index_projection.py: CREATE INDEX projects only what it needs](#163-test_native_index_projectionpy-create-index-projects-only-what-it-needs) +- [164. test_native_saop_pushdown.py: IN-lists and = ANY(array) prune row groups](#164-test_native_saop_pushdownpy-in-lists-and--anyarray-prune-row-groups) ## 1. How to read a test in here @@ -7641,3 +7642,20 @@ The pytest twin of `test/native_index_projection.sh` (ported for #1341), every c | `test_each_build_projects_only_what_it_needs` | every serial and parallel build projects only its key, expression and predicate columns | | `test_the_parallel_built_index_agrees_and_holds_each_row_once` | the parallel-built index agrees with a sequential scan and holds each row once | | `test_amcheck_finds_every_index_sound` | `bt_index_check` finds all eleven indexes sound | + +## 164. test_native_saop_pushdown.py: IN-lists and = ANY(array) prune row groups + +The pytest twin of `test/native_saop_pushdown.sh` (ported for #1341), every check by the shell's name. The scan-key builder keeps an IN-list or `= ANY(array)` of up to 128 non-NULL elements as one set predicate (#704), and a group survives when any element can lie in its zone map and bloom filter. Larger sets keep the bounded [min,max] hull. The fixture is 40000 rows in 20 monotonic groups, plus an overlap column whose every group has the range [10,88] and only even values, so only the bloom filter can prune an odd probe. NOT IN builds no keys. An empty or all-NULL array neither crashes nor mis-counts. A generic plan's array parameter is frozen at Begin, and a correlated one is not. The batch fold never takes the conservative set keys as its exact filter. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_a_set_prunes_by_element_and_keeps_the_exact_answer` | an IN-list pushes one usable set filter that removes 19 groups, a scattered set prunes by element rather than by its hull, and both sides of the 128-element bound | +| `test_nulls_types_and_collations_prune_exactly` | NULL elements, a by-reference text set, a cross-type array, a C-collated list and a domain-element array each prune and count exactly | +| `test_only_the_bloom_filter_can_prune_the_overlap_column` | with 25 inside every group's range, the bloom filter alone removes all 20 groups for a set, a single value and a single distinct value | +| `test_the_conservative_arms_build_nothing_wrong` | a range-spanning list prunes between its elements, NOT IN builds no range, and empty and all-NULL arrays count zero | +| `test_a_parameter_array_is_frozen_at_begin` | a generic plan's array parameter and a mixed literal-and-parameter list prune | +| `test_the_batch_fold_never_takes_the_set_keys_as_exact` | the batch fold refuses the IN-list but accepts the equivalent range, and the vectorized count stays exact | +| `test_a_correlated_array_is_not_frozen_and_the_backend_lives` | a correlated LATERAL array is not frozen, and the session's backend is still alive; a crash is a named red | +| `test_the_reader_refuses_every_special_flag_but_the_array_one` | the reader's reject expression names every special scan-key flag, and SK_SEARCHARRAY reaches its own arm | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 59387c959..58ea62469 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -789,4 +789,6 @@ guard_tests 438 # one test, by collection over the cluster files on this tree. # cluster_tests 698 -> 702 for #1341's test_native_index_projection.py, the twin of native_index_projection.sh: # four tests, by collection over the cluster files on this tree. -cluster_tests 702 +# cluster_tests 702 -> 710 for #1341's test_native_saop_pushdown.py, the twin of native_saop_pushdown.sh: +# eight tests, by collection over the cluster files on this tree. +cluster_tests 710 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 78a88434e..a4276febb 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -143,6 +143,7 @@ "native_reclaim_frag", "native_recluster", "native_repack", + "native_saop_pushdown", "native_vacuum_race", "native_vecskip", "native_writer", @@ -411,7 +412,7 @@ def test_the_loop_reader_invents_nothing_in_this_corpus(expect): expect.text(", ".join(files), "build_refusal, differential, join_runtime_filter, " "native_index_projection, native_reclaim, native_repack, " - "native_vacuum_race, native_vecskip, sorted_pathkeys", + "native_saop_pushdown, native_vacuum_race, native_vecskip, sorted_pathkeys", "and it is these files, so another appearing is a diff a reviewer " "sees rather than a number that moved") diff --git a/test/pytest/test_native_saop_pushdown.py b/test/pytest/test_native_saop_pushdown.py new file mode 100644 index 000000000..894867158 --- /dev/null +++ b/test/pytest/test_native_saop_pushdown.py @@ -0,0 +1,258 @@ +"""Chunk-group skipping works for IN-list / = ANY(array) predicates (#704, #1341). The +pytest twin of `test/native_saop_pushdown.sh`, every check by the shell's name. + +The scan-key builder keeps up to 128 non-NULL elements as one set predicate: a group +survives when any element can lie in its zone map and bloom filter, and larger sets keep +the bounded [min,max] hull. 40000 rows in 20 monotonic groups, plus an overlap column +whose every group has the range [10,88] and only even values, so an odd probe inside it +can be pruned by the bloom filter alone. The hazards each have an arm: NOT IN builds no +keys, an empty or all-NULL array neither crashes nor mis-counts, a generic plan's array +parameter is frozen at Begin, a correlated PARAM_EXEC array is not, and the batch fold +never takes the conservative set keys as its exact filter. Last, the reader's flag guard: +every special scan-key flag but SK_SEARCHARRAY is refused, and that one has its own arm. +""" +import pathlib +import re + +import psycopg +import pytest + +SRC = pathlib.Path(__file__).resolve().parents[2] / "src" / "columnar_reader.c" +REMOVED = "Chunk Groups Removed by Filter" + + +def _one(c, sql): + return c.execute(sql).fetchone()[0] + + +def _plan(c, sql, sets=()): + for s in sets: + c.execute(s) + try: + return "\n".join(r[0] for r in c.execute(f"EXPLAIN (ANALYZE, TIMING off, SUMMARY off) {sql}")) + finally: + for s in sets: + c.execute(f"RESET {s.split()[1].split('=')[0]}") + + +def _counter(plan, label): + """-> the first number after `label: `, or -1 when the plan has no such line: a plan + that is not the columnar scan reads as its own red, not as 0 groups removed.""" + m = re.search(rf"{re.escape(label)}: (\d+)", plan) + return int(m.group(1)) if m else -1 + + +def _answer(c, sql): + """-> the first value as text, or the error: a crash is a named red, not a lost test.""" + try: + return str(_one(c, sql)) + except psycopg.Error as exc: + return f"error: {exc}" + + +def _removed(c, sql): + return _counter(_plan(c, sql), REMOVED) + + +def _generic(c, name, params, body, run): + """-> the rows of `run` on a forced generic plan of `body`, prepared as `name`.""" + c.execute("SET plan_cache_mode = force_generic_plan") + try: + c.execute(f"PREPARE {name}({params}) AS {body}") + try: + return [r[0] for r in c.execute(run)] + finally: + c.execute(f"DEALLOCATE {name}") + finally: + c.execute("RESET plan_cache_mode") + + +@pytest.fixture(scope="module") +def saop(pgc_cluster): + """-> a connection to a database holding the shell's t and thr.""" + db = "pgc_saop" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + # prepare_threshold=None: the arms PREPARE and DEALLOCATE by name themselves + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True, prepare_threshold=None) as c: + c.execute("CREATE EXTENSION pgcolumnar") + # 40000 rows / 2000 = 20 monotonic groups; values >= 39000 live in the last one. + # ov = (g%40)*2+10 gives every group the same even-valued range [10,88]. + c.execute("SET pgcolumnar.stripe_row_limit = 2000") + c.execute("CREATE DOMAIN intdom AS int") + c.execute('CREATE TABLE t (id int, ts int, txt text COLLATE "C", ov int) USING pgcolumnar') + c.execute("INSERT INTO t SELECT g, g, lpad(g::text, 8, '0'), (g % 40)*2 + 10 " + "FROM generate_series(1,40000) g") + c.execute("RESET pgcolumnar.stripe_row_limit") + c.execute("CREATE TABLE thr (lo int[])") + c.execute("INSERT INTO thr VALUES ('{39100}'),('{100,200}')") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_a_set_prunes_by_element_and_keeps_the_exact_answer(saop, expect): + c = saop + # intent, usability and work done are three counters, so intent cannot impersonate work + plan = _plan(c, "SELECT count(*) FROM t WHERE ts IN (39100, 39200)") + expect.num(_counter(plan, "Columnar Pushed-Down Filters"), 1, "an IN-list pushes one set filter (intent)") + expect.num(_counter(plan, "Columnar Usable Skip Predicates"), 1, + "the IN-list set filter is a usable skip predicate") + expect.num(_counter(plan, REMOVED), 19, "an IN-list confined to the last group prunes (19 of 20 removed)") + expect.num(_one(c, "SELECT count(*) FROM t WHERE ts IN (39100, 39200)"), + _one(c, "SELECT count(*) FROM t WHERE ts = 39100 OR ts = 39200"), + "IN-list count matches the OR-literal equivalent") + + # the scattered set's hull spans nearly the whole table, so only per-element pruning removes 17 + expect.num(_removed(c, "SELECT count(*) FROM t WHERE ts IN (100, 20100, 38100)"), 17, + "a scattered IN-list prunes by element, not only by its hull (17 removed)") + expect.num(_one(c, "SELECT count(*) FROM t WHERE ts IN (100, 20100, 38100)"), 3, + "scattered per-element pruning keeps the exact answer") + expect.num(_removed(c, "SELECT count(*) FROM t WHERE ts IN (100, 101, 102)"), 19, + "a contiguous IN-list agrees with its hull (19 removed)") + expect.num(_one(c, "SELECT count(*) FROM t WHERE ts IN (100, 101, 102)"), 3, + "contiguous per-element pruning keeps the exact answer") + + # both sides of the 128-element bound + limit = ",".join(str(i) for i in range(1, 129)) + expect.num(_counter(_plan(c, f"SELECT count(*) FROM t WHERE ts IN ({limit})"), "Columnar Pushed-Down Filters"), 1, + "exactly 128 elements still use one set filter") + large = ",".join(str(i) for i in range(1, 130)) + expect.num(_counter(_plan(c, f"SELECT count(*) FROM t WHERE ts IN ({large})"), "Columnar Pushed-Down Filters"), 2, + "a large IN-list falls back to two bounded hull filters") + expect.num(_one(c, f"SELECT count(*) FROM t WHERE ts IN ({large})"), 129, + "the large-list fallback keeps the exact answer") + + +def test_nulls_types_and_collations_prune_exactly(saop, expect): + c = saop + for name, sql, want in ( + ("a NULL element is ignored for the range, not a bailout (19 removed)", + "SELECT count(*) FROM t WHERE ts = ANY (ARRAY[39100, NULL]::int[])", 19), + ("a multi-value integer set strips NULL and still prunes 17 groups", + "SELECT count(*) FROM t WHERE ts = ANY (ARRAY[100,20100,38100,NULL]::int[])", 17), + ("a by-reference text set strips NULL without crashing (17 removed)", + "SELECT count(*) FROM t WHERE txt = ANY (ARRAY['00000100','00020100','00038100',NULL]::text[])", 17), + ("a cross-type element array (int col, bigint[] list) prunes (19 removed)", + "SELECT count(*) FROM t WHERE ts = ANY ('{39100,39200}'::bigint[])", 19), + ("a C-collated text IN-list prunes (19 removed)", + "SELECT count(*) FROM t WHERE txt IN ('00039100', '00039200')", 19), + # a domain-element cast arrives with the BASE type's OID in the array header + ("a domain-element array prunes (19 removed)", + "SELECT count(*) FROM t WHERE ts = ANY ('{39100,39200}'::intdom[])", 19), + ): + expect.num(_removed(c, sql), want, name) + for name, sql, want in ( + ("NULL-element count is exact", "SELECT count(*) FROM t WHERE ts = ANY (ARRAY[39100, NULL]::int[])", 1), + ("the multi-value integer NULL set stays exact", + "SELECT count(*) FROM t WHERE ts = ANY (ARRAY[100,20100,38100,NULL]::int[])", 3), + ("the by-reference text NULL set stays exact", + "SELECT count(*) FROM t WHERE txt = ANY (ARRAY['00000100','00020100','00038100',NULL]::text[])", 3), + ("cross-type count is exact", "SELECT count(*) FROM t WHERE ts = ANY ('{39100,39200}'::bigint[])", 2), + ("text IN-list count is exact", "SELECT count(*) FROM t WHERE txt IN ('00039100', '00039200')", 2), + ("domain-element count is exact", "SELECT count(*) FROM t WHERE ts = ANY ('{39100,39200}'::intdom[])", 2), + ): + expect.num(_one(c, sql), want, name) + + +def test_only_the_bloom_filter_can_prune_the_overlap_column(saop, expect): + c = saop + # premises first: 25 lies inside every group's range, so the 20-removed arms are the bloom probe + expect.num(_removed(c, "SELECT count(*) FROM t WHERE ov >= 25 AND ov <= 25"), 0, + "fixture premise: 25 is inside every group's range (range keys prune 0)") + expect.num(_removed(c, "SELECT count(*) FROM t WHERE ov = 25"), 20, + "fixture premise: equality on the overlap column bloom-prunes all 20") + expect.num(_removed(c, "SELECT count(*) FROM t WHERE ov = ANY ('{25,27,29}'::int[])"), 20, + "a multi-value set bloom-prunes all overlapping groups") + expect.num(_one(c, "SELECT count(*) FROM t WHERE ov = ANY ('{25,27,29}'::int[])"), 0, + "the bloom-only multi-value set returns the exact empty answer") + # IN (25) is parsed to `= 25`; '{25,25}' is the SAOP shape of the single-distinct-value refinement + expect.num(_removed(c, "SELECT count(*) FROM t WHERE ov IN (25)"), 20, + "a one-element IN-list (parser-collapsed to =) bloom-prunes (20 removed)") + expect.num(_removed(c, "SELECT count(*) FROM t WHERE ov = ANY ('{25,25}'::int[])"), 20, + "a single-DISTINCT-value SAOP list ('{25,25}') bloom-prunes (20 removed)") + expect.num(_one(c, "SELECT count(*) FROM t WHERE ov IN (25)"), 0, "single-value IN-list count is exact (0 rows)") + + +def test_the_conservative_arms_build_nothing_wrong(saop, expect): + c = saop + q = "SELECT count(*) FROM t WHERE ts IN (100, 39900)" + expect.text(f"{_removed(c, q)}/{_one(c, q)}", "18/2", + "a range-spanning IN-list prunes between its elements (18 removed, exact count)") + # confined to one group: a positive range wrongly derived from NOT IN would collapse the count + q = "SELECT count(*) FROM t WHERE ts NOT IN (19000, 19500)" + expect.text(f"{_removed(c, q)}/{_one(c, q)}", "0/39998", "NOT IN builds no range (0 removed) and stays correct") + expect.num(_one(c, "SELECT count(*) FROM t WHERE ts = ANY ('{}'::int[])"), 0, + "an empty array counts zero and does not crash") + expect.num(_one(c, "SELECT count(*) FROM t WHERE ts = ANY (ARRAY[NULL,NULL]::int[])"), 0, + "an all-NULL array counts zero and does not crash") + + +def test_a_parameter_array_is_frozen_at_begin(saop, expect): + c = saop + body = "SELECT count(*) FROM t WHERE ts = ANY ($1)" + ex = "EXPLAIN (ANALYZE, TIMING off, SUMMARY off) " + rows = _generic(c, "p", "int[]", body, ex + "EXECUTE p('{39100,39200}')") + expect.num(_counter("\n".join(rows), REMOVED), 19, "a generic-plan parameterized array prunes (19 removed)") + expect.num(_generic(c, "pc", "int[]", body, "EXECUTE pc('{39100,39200}')")[0], 2, + "parameterized array count matches the literal count") + # IN (literal, $1) stays an unfolded ArrayExpr on a generic plan: the freeze must evaluate it whole + rows = _generic(c, "m", "int", "SELECT count(*) FROM t WHERE ts IN (39100, $1)", ex + "EXECUTE m(39200)") + expect.num(_counter("\n".join(rows), REMOVED), 19, + "a mixed literal+param IN-list prunes on a generic plan (19 removed)") + + +def test_the_batch_fold_never_takes_the_set_keys_as_exact(saop, expect): + c = saop + vec = ("SET pgcolumnar.enable_ungrouped_vector_agg=on",) + plan = _plan(c, "SELECT count(*) FROM t WHERE ts IN (39100, 39150)", vec) + expect.num(plan.count("Columnar Vectorized Aggregates"), 1, + "vector-agg path engaged for the guard arm (plan shape)") + # the no/yes pair: the clause gate is what excludes the SAOP, not another eligibility rule + expect.num(plan.count("Columnar Batch Fold: no"), 1, "the batch fold refuses the IN-list (Batch Fold: no)") + expect.num(_plan(c, "SELECT count(*) FROM t WHERE ts >= 39100 AND ts <= 39150", vec) + .count("Columnar Batch Fold: yes"), 1, + "the batch fold accepts the equivalent exact range quals (Batch Fold: yes)") + c.execute(vec[0]) + try: + expect.num(_one(c, "SELECT count(*) FROM t WHERE ts IN (39100, 39150)"), 2, + "vector-agg count over an IN-list is exact (conservative keys not folded)") + finally: + c.execute("RESET pgcolumnar.enable_ungrouped_vector_agg") + + +def test_a_correlated_array_is_not_frozen_and_the_backend_lives(saop, expect): + c = saop + # the two thr rows land in different groups, so a frozen first array would lose rows; + # freezing the PARAM_EXEC at Begin crashes the backend instead, and that is a red here too + expect.text(_answer(c, "SELECT sum(c) FROM thr, LATERAL " + "(SELECT count(*) c FROM t WHERE t.ts = ANY (thr.lo)) s"), "3", + "a correlated PARAM_EXEC array is not mis-pruned (results stay correct)") + expect.text(_answer(c, "SELECT 1"), "1", "backend alive") + + +def test_the_reader_refuses_every_special_flag_but_the_array_one(expect): + src = SRC.read_text() + expect.num(sum("key->sk_flags & (SK_ISNULL" in l for l in src.splitlines()), 1, + "premise: plain-key special flags have one reject expression") + # from the `if (key->sk_flags & (` line through the first line holding `))`, as the shell's awk range + guard, inside = "", False + for line in src.splitlines(): + if not inside and re.search(r"if \(key->sk_flags & \(", line): + inside = True + if inside: + guard += re.sub(r"\s", "", line) + if "))" in line: + break + expect.text("yes" if guard and "sk_flags" in guard else f"extracted=<{guard or 'empty'}>", "yes", + "premise: the reject expression was extracted, not blank") + # membership, not adjacency: the flag list is a set + for flag in ("SK_ISNULL", "SK_ROW_HEADER", "SK_ROW_MEMBER", "SK_ROW_END", "SK_SEARCHNULL", + "SK_SEARCHNOTNULL", "SK_ORDER_BY"): + expect.text("yes" if flag in guard else f"absent from <{guard}>", "yes", + f"the predicate builder's reject expression contains {flag}") + expect.num(len(re.findall(r"out\[n\]\.searchArray = \(key->sk_flags & SK_SEARCHARRAY\)", src)), 1, + "SK_SEARCHARRAY reaches the dedicated array predicate arm")