diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 053c08f87..ce94bc74b 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -3766,35 +3766,35 @@ native_ctas native_ctas serial CTAS row count 15;16;17;18;19 2026-10-22 columnar native_ctas native_ctas serial CTAS still succeeds 15;16;17;18;19 2026-10-22 columnar_metadata.c: the storage-row write for a relation created in this transaction aborts the backend native_ctas native_ctas the parallel CTAS produced exactly one storage row 15;16;17;18;19 2026-10-22 columnar_metadata.c: the storage-row write for a relation created in this transaction aborts the backend native_ctas native_ctas the second writer started while the first held its transaction open 15;16;17;18;19 2026-10-22 columnar_metadata.c: the storage-row write under the first-writer lock aborts the backend +native_decode_gate_width native_decode_gate_width EXPLAIN reports Columnar Vector Decodes (the observable) 15;16;17;18;19 2026-10-23 columnar_customscan.c: the custom-scan path is never added (if (!pgcolumnar_enable_custom_scan) became if (true)) +native_decode_gate_width native_decode_gate_width back to the shipped default 15;16;17;18;19 2026-10-23 columnar_tableam.c: pgcolumnar.qual_skipvec_min_payload_cols boot value 20 changed to 21 native_decode_gate_width native_decode_gate_width control, gate forced on: a matches-everything narrow qual skips no vector (full decode) 15;16;17;18;19 never - native_decode_gate_width native_decode_gate_width control, gate forced on: a matches-everything wide qual skips no vector (full decode) 15;16;17;18;19 never - native_decode_gate_width native_decode_gate_width correctness: narrow needle matches the heap either way 15;16;17;18;19 never - native_decode_gate_width native_decode_gate_width correctness: the full scan matches the heap 15;16;17;18;19 never - native_decode_gate_width native_decode_gate_width correctness: wide needle, gate OFF, returns the identical rows (output-invariant) 15;16;17;18;19 never - native_decode_gate_width native_decode_gate_width correctness: wide needle, gate ON, matches the heap oracle 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width gate suppressed for the invariance arm 15;16;17;18;19 2026-10-23 columnar_tableam.c: pgcolumnar.qual_skipvec_min_payload_cols maximum 100000 changed to 23 native_decode_gate_width native_decode_gate_width narrow, default: a zero-match LIKE decodes EVERY projected column of every vector -> NOT gated 15;16;17;18;19 2026-10-16 columnar_reader.c: phase-2 gating runs whatever the projection's payload width native_decode_gate_width native_decode_gate_width narrow, gate off (GUC=0): the SAME narrow projection now gates -> qual column alone 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: EXPLAIN reports Columnar Vector Decodes (the observable) 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: back to the shipped default 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: gate suppressed for the invariance arm 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: no vector is ruled out by VALUE (that is 1b-ii's path, not phase 2) 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: no whole row group is pruned 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: one row group, so every count below is one group's 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: still at the default before arm 1 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: the LIKE is unprunable (no usable skip predicate) 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width no vector is ruled out by VALUE (that is 1b-ii's path, not phase 2) 15;16;17;18;19 2026-10-23 columnar_customscan.c: the custom-scan path is never added (if (!pgcolumnar_enable_custom_scan) became if (true)) +native_decode_gate_width native_decode_gate_width no whole row group is pruned 15;16;17;18;19 2026-10-23 columnar_customscan.c: the custom-scan path is never added (if (!pgcolumnar_enable_custom_scan) became if (true)) +native_decode_gate_width native_decode_gate_width one row group, so every count below is one group's 15;16;17;18;19 2026-10-23 columnar_write_state.c: a table's stripe_row_limit option applied divided by 100 native_decode_gate_width native_decode_gate_width premise: the control matches every row 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: the narrow plan is a columnar custom scan with projection 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: the narrow query takes the late-materialization path 15;16;17;18;19 never - native_decode_gate_width native_decode_gate_width premise: the needle matches exactly one vector's worth of rows 15;16;17;18;19 never - native_decode_gate_width native_decode_gate_width premise: the nomatch predicate matches no row 15;16;17;18;19 never - native_decode_gate_width native_decode_gate_width premise: the rejected probe GUC qual_skipvec_min_skip_pct was NOT shipped 15;16;17;18;19 never - native_decode_gate_width native_decode_gate_width premise: the rejected probe GUC qual_skipvec_probe_vecs was NOT shipped 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: the wide plan is a columnar custom scan with projection 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: the wide query takes the late-materialization path 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: the width gate is now disabled (0) 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: the width threshold is now above the projection width 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: the width threshold now equals the projection width 15;16;17;18;19 never - -native_decode_gate_width native_decode_gate_width premise: the width-gate GUC exists and defaults to the shipped 20 15;16;17;18;19 never - +native_decode_gate_width native_decode_gate_width still at the default before arm 1 15;16;17;18;19 2026-10-23 columnar_tableam.c: pgcolumnar.qual_skipvec_min_payload_cols boot value 20 changed to 21 +native_decode_gate_width native_decode_gate_width the LIKE is unprunable (no usable skip predicate) 15;16;17;18;19 2026-10-23 columnar_customscan.c: the custom-scan path is never added (if (!pgcolumnar_enable_custom_scan) became if (true)) +native_decode_gate_width native_decode_gate_width the narrow plan is a columnar custom scan with projection 15;16;17;18;19 2026-10-23 columnar_customscan.c: the custom-scan path is never added (if (!pgcolumnar_enable_custom_scan) became if (true)) +native_decode_gate_width native_decode_gate_width the narrow query takes the late-materialization path 15;16;17;18;19 2026-10-23 columnar_customscan.c: the custom-scan path is never added (if (!pgcolumnar_enable_custom_scan) became if (true)) +native_decode_gate_width native_decode_gate_width the wide plan is a columnar custom scan with projection 15;16;17;18;19 2026-10-23 columnar_customscan.c: the custom-scan path is never added (if (!pgcolumnar_enable_custom_scan) became if (true)) +native_decode_gate_width native_decode_gate_width the wide query takes the late-materialization path 15;16;17;18;19 2026-10-23 columnar_customscan.c: the custom-scan path is never added (if (!pgcolumnar_enable_custom_scan) became if (true)) +native_decode_gate_width native_decode_gate_width the width gate is now disabled (0) 15;16;17;18;19 2026-10-23 columnar_tableam.c: pgcolumnar.qual_skipvec_min_payload_cols minimum 0 changed to 1 +native_decode_gate_width native_decode_gate_width the width threshold is now above the projection width 15;16;17;18;19 2026-10-23 columnar_tableam.c: pgcolumnar.qual_skipvec_min_payload_cols maximum 100000 changed to 23 +native_decode_gate_width native_decode_gate_width the width threshold now equals the projection width 15;16;17;18;19 2026-10-23 columnar_tableam.c: pgcolumnar.qual_skipvec_min_payload_cols maximum 100000 changed to 23 +native_decode_gate_width native_decode_gate_width the width-gate GUC exists and defaults to the shipped 20 15;16;17;18;19 2026-10-23 columnar_tableam.c: pgcolumnar.qual_skipvec_min_payload_cols boot value 20 changed to 21 native_decode_gate_width native_decode_gate_width wide, default: a one-vector LIKE decodes the qual column plus payload for that vector only 15;16;17;18;19 never - native_decode_gate_width native_decode_gate_width wide, default: a zero-match LIKE decodes the qual column and NO payload -> gated 15;16;17;18;19 never - native_decode_gate_width native_decode_gate_width wide, default: and reports the payload vectors skipped 15;16;17;18;19 never - @@ -3808,27 +3808,27 @@ native_decode_gating native_decode_gating and reports them all filtered before m native_decode_gating native_decode_gating and skips every vector 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluation between the decode passes never runs native_decode_gating native_decode_gating and the full scan across every group's mask matches too 15;16;17;18;19 never - native_decode_gating native_decode_gating deletes + multiple groups + unprunable qual: columnar matches the heap 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped -native_decode_gating native_decode_gating premise: no vector is ruled out by VALUE either (that is 1b-ii's path) 15;16;17;18;19 never - -native_decode_gating native_decode_gating premise: no whole row group is pruned 15;16;17;18;19 never - -native_decode_gating native_decode_gating premise: one row group, so every count below is one group's 15;16;17;18;19 never - -native_decode_gating native_decode_gating premise: rows really were deleted (so the mask is non-NULL) 15;16;17;18;19 never - -native_decode_gating native_decode_gating premise: the columnar table spans more than one row group 15;16;17;18;19 never - +native_decode_gating native_decode_gating no vector is ruled out by VALUE either (that is 1b-ii's path) 15;16;17;18;19 2026-10-23 columnar_customscan.c: the custom-scan path is never added (if (!pgcolumnar_enable_custom_scan) became if (true)) +native_decode_gating native_decode_gating no whole row group is pruned 15;16;17;18;19 2026-10-23 columnar_customscan.c: the custom-scan path is never added (if (!pgcolumnar_enable_custom_scan) became if (true)) +native_decode_gating native_decode_gating one row group, so every count below is one group's 15;16;17;18;19 2026-10-23 columnar_write_state.c: a table's stripe_row_limit option applied divided by 100 native_decode_gating native_decode_gating premise: the control matches every row 15;16;17;18;19 never - native_decode_gating native_decode_gating premise: the half-needle survives in one vector, half of its rows 15;16;17;18;19 never - -native_decode_gating native_decode_gating premise: the needle plan is a columnar custom scan 15;16;17;18;19 never - -native_decode_gating native_decode_gating premise: the needle query takes the late-materialization path 15;16;17;18;19 never - native_decode_gating native_decode_gating premise: the needle really matches one vector's worth of rows 15;16;17;18;19 never - native_decode_gating native_decode_gating premise: the nomatch predicate matches no row 15;16;17;18;19 never - -native_decode_gating native_decode_gating premise: the reader has no usable skip predicate for this LIKE 15;16;17;18;19 never - -native_decode_gating native_decode_gating premise: the width gate is pinned off, so this narrow projection still gates 15;16;17;18;19 never - native_decode_gating native_decode_gating rejections inside a surviving vector are counted once 15;16;17;18;19 2026-10-16 columnar_reader.c: the producer counts a gated group's rejections a second time +native_decode_gating native_decode_gating rows really were deleted (so the mask is non-NULL) 15;16;17;18;19 2026-10-23 columnar_tableam.c: pgcolumnar_tuple_delete returns TM_Ok without PgColumnarMarkRowDeleted native_decode_gating native_decode_gating rows removed by filter across deleted rows count only live rows 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator counts a deleted row as filtered;columnar_reader.c: the producer counts a gated group's rejections a second time +native_decode_gating native_decode_gating the columnar table spans more than one row group 15;16;17;18;19 2026-10-23 columnar_write_state.c: a table's stripe_row_limit option applied multiplied by 100 native_decode_gating native_decode_gating the control decodes every column of every vector 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped native_decode_gating native_decode_gating the control returns every row, values intact 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating the needle plan is a columnar custom scan 15;16;17;18;19 2026-10-23 columnar_customscan.c: the custom-scan path is never added (if (!pgcolumnar_enable_custom_scan) became if (true)) native_decode_gating native_decode_gating the needle query returns exactly the matching rows 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluator marks every vector skipped +native_decode_gating native_decode_gating the needle query takes the late-materialization path 15;16;17;18;19 2026-10-23 columnar_customscan.c: the custom-scan path is never added (if (!pgcolumnar_enable_custom_scan) became if (true)) native_decode_gating native_decode_gating the needle removes exactly the non-matching rows, counted once 15;16;17;18;19 never - native_decode_gating native_decode_gating the needle skips exactly the vectors holding no match 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluation between the decode passes never runs;columnar_reader.c: the per-vector qual evaluator marks every vector skipped native_decode_gating native_decode_gating the nomatch query returns nothing 15;16;17;18;19 never - +native_decode_gating native_decode_gating the reader has no usable skip predicate for this LIKE 15;16;17;18;19 2026-10-23 columnar_customscan.c: the custom-scan path is never added (if (!pgcolumnar_enable_custom_scan) became if (true)) +native_decode_gating native_decode_gating the width gate is pinned off, so this narrow projection still gates 15;16;17;18;19 2026-10-23 columnar_tableam.c: pgcolumnar.qual_skipvec_min_payload_cols minimum 0 changed to 1 native_decode_gating native_decode_gating the zero-match query removes every row by filter, counted once 15;16;17;18;19 never - native_delete_visibility_paths native_delete_visibility_paths a DELETE whose qual deletes another row of the table succeeds and leaves heap's rows (#1394) 15;16;17;18;19 2026-10-16 PgColumnarUpsertDeleteVector without the command advance (main before #1394's fix);columnar_reader.c: the row-at-a-time scan reads a deleted row as live native_delete_visibility_paths native_delete_visibility_paths a deleted row is invisible via the index 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row ignores the flushed delete vector diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 2cea03ef9..5d2a3cb83 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -7021,4 +7021,22 @@ suites_not_covered 28 # never 4545 + 0 = 4545 # dated 3418 + 0 = 3418 # 4545 + 3418 == 7963 -checks_never_observed_red 4545 +# +# #1339, premises that a one-line pgcolumnar change reddens, in native_decode_gate_width +# and native_decode_gating (OffgridwithJD): 4545 -> 4520. Twenty-five checks read +# pgcolumnar's own output (its plan, its EXPLAIN counters, its GUC's default and range, +# its row-group catalog, its delete) under a premise: label, the shape jdatcmd flagged +# on #1691, #1698 and #1700. Each loses the prefix, keeps its row under the new name +# and is dated by the one-line mutant that reddens it in the shell: no custom scan (13), +# the width gate's default 21 (3), its minimum 1 (2), its maximum 23 (3), a table's +# stripe_row_limit /100 (2) and x100 (1), and a delete that marks nothing (1). Each +# twin is one test, so it names the first of these per mutant; both twins now leave a +# refused gate SET to the premise that reads it, as the shell does, rather than end the +# test unnamed. The fixture-count premises now read the heap mirror and stay premises, +# and the two rejected-GUC premises stay: no one-line change ships a GUC. +# +# ledger rows 7963 + 0 = 7963 +# never 4545 - 25 = 4520 +# dated 3418 + 25 = 3443 +# 4520 + 3443 == 7963 +checks_never_observed_red 4520 diff --git a/test/native_decode_gate_width.sh b/test/native_decode_gate_width.sh index 176830fd6..1202a4d99 100755 --- a/test/native_decode_gate_width.sh +++ b/test/native_decode_gate_width.sh @@ -100,7 +100,7 @@ NARROW_CONTROL="SELECT tag, p1, p2 FROM nw WHERE tag LIKE '%row%'" # SHOW erroring (GUC undefined) -> q is empty -> this fails: it is also the guard # that the extern + DefineCustomIntVariable actually shipped. -check "premise: the width-gate GUC exists and defaults to the shipped $DEFAULT_GUC" \ +check "the width-gate GUC exists and defaults to the shipped $DEFAULT_GUC" \ "$(gate_now)" "$DEFAULT_GUC" # The rejected selectivity-probe idea's two GUCs must NOT have shipped. SHOW of an # unknown parameter errors; q swallows the error and returns empty, so an empty @@ -117,7 +117,7 @@ check "premise: the rejected probe GUC qual_skipvec_probe_vecs was NOT shipped" "$(show_state pgcolumnar.qual_skipvec_probe_vecs)" "42704" check "premise: the rejected probe GUC qual_skipvec_min_skip_pct was NOT shipped" \ "$(show_state pgcolumnar.qual_skipvec_min_skip_pct)" "42704" -check "premise: one row group, so every count below is one group's" \ +check "one row group, so every count below is one group's" \ "$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('nw');")" \ "1" @@ -129,31 +129,33 @@ PLAN_WIDE_NEEDLE="$(explain_of "$WIDE_NEEDLE")" PLAN_NARROW_NEEDLE="$(explain_of "SELECT tag, p1, p2 FROM nw WHERE tag LIKE '%needle%'")" PLAN_NARROW_NOMATCH="$(explain_of "$NARROW_NOMATCH")" -check "premise: the wide plan is a columnar custom scan with projection" \ +check "the wide plan is a columnar custom scan with projection" \ "$(has_line "$PLAN_WIDE_NEEDLE" 'Columnar Projected Columns')" "yes" -check "premise: the narrow plan is a columnar custom scan with projection" \ +check "the narrow plan is a columnar custom scan with projection" \ "$(has_line "$PLAN_NARROW_NOMATCH" 'Columnar Projected Columns')" "yes" -check "premise: the LIKE is unprunable (no usable skip predicate)" \ +check "the LIKE is unprunable (no usable skip predicate)" \ "$(counter_in "$PLAN_WIDE_NEEDLE" 'Columnar Usable Skip Predicates')" "0" -check "premise: no whole row group is pruned" \ +check "no whole row group is pruned" \ "$(counter_in "$PLAN_WIDE_NEEDLE" 'Columnar Chunk Groups Removed by Filter')" "0" -check "premise: no vector is ruled out by VALUE (that is 1b-ii's path, not phase 2)" \ +check "no vector is ruled out by VALUE (that is 1b-ii's path, not phase 2)" \ "$(counter_in "$PLAN_WIDE_NEEDLE" 'Columnar Vectors Ruled Out by Value')" "0" # Phase 2 rides on 1a's callback: both projections must take the # late-materialization path or no decode count below measures phase 2. -check "premise: the wide query takes the late-materialization path" \ +check "the wide query takes the late-materialization path" \ "$(has_line "$PLAN_WIDE_NEEDLE" 'Columnar Rows Filtered Before Materialization')" "yes" -check "premise: the narrow query takes the late-materialization path" \ +check "the narrow query takes the late-materialization path" \ "$(has_line "$PLAN_NARROW_NEEDLE" 'Columnar Rows Filtered Before Materialization')" "yes" -check "premise: EXPLAIN reports Columnar Vector Decodes (the observable)" \ +check "EXPLAIN reports Columnar Vector Decodes (the observable)" \ "$(has_line "$PLAN_NARROW_NOMATCH" 'Columnar Vector Decodes')" "yes" +# The fixture's shape, read from the heap mirror the same generator filled (#1339): +# read from the columnar table, a one-line reader change reddened these premises. check "premise: the needle matches exactly one vector's worth of rows" \ - "$(q "SELECT count(*) FROM nw WHERE tag LIKE '%needle%';")" "1024" + "$(q "SELECT count(*) FROM hw WHERE tag LIKE '%needle%';")" "1024" check "premise: the nomatch predicate matches no row" \ - "$(q "SELECT count(*) FROM nw WHERE tag LIKE '%zzzzz%';")" "0" + "$(q "SELECT count(*) FROM hw WHERE tag LIKE '%zzzzz%';")" "0" check "premise: the control matches every row" \ - "$(q "SELECT count(*) FROM nw WHERE tag LIKE '%row%';")" "$ROWS" + "$(q "SELECT count(*) FROM hw WHERE tag LIKE '%row%';")" "$ROWS" # ---- ARM 1: at the shipped default, WIDE gates and NARROW does not ----------- # @@ -165,7 +167,7 @@ check "premise: the control matches every row" \ # VECTORS*NARROW_NCOLS is the whole signal, and it is exactly the payload the gate # chose to spare or not. -check "premise: still at the default before arm 1" "$(gate_now)" "$DEFAULT_GUC" +check "still at the default before arm 1" "$(gate_now)" "$DEFAULT_GUC" PLAN_WIDE_NOMATCH="$(explain_of "$WIDE_NOMATCH")" check "wide, default: a zero-match LIKE decodes the qual column and NO payload -> gated" \ @@ -193,7 +195,7 @@ check "narrow, default: a zero-match LIKE decodes EVERY projected column of ever # scan now decodes the qual column alone. set_gate 0 -check "premise: the width gate is now disabled (0)" "$(gate_now)" "0" +check "the width gate is now disabled (0)" "$(gate_now)" "0" check "narrow, gate off (GUC=0): the SAME narrow projection now gates -> qual column alone" \ "$(counter_in "$(explain_of "$NARROW_NOMATCH")" 'Columnar Vector Decodes')" "$VECTORS" @@ -215,13 +217,13 @@ check "control, gate forced on: a matches-everything narrow qual skips no vector # would fail the suppression check. set_gate "$NCOLS_WIDE" # 25 > 24 payload cols -> must NOT gate -check "premise: the width threshold is now above the projection width" "$(gate_now)" "$NCOLS_WIDE" +check "the width threshold is now above the projection width" "$(gate_now)" "$NCOLS_WIDE" check "wide, gate raised above width: the SAME wide SELECT * stops gating -> full decode" \ "$(counter_in "$(explain_of "$WIDE_NOMATCH")" 'Columnar Vector Decodes')" \ "$(( VECTORS * NCOLS_WIDE ))" set_gate "$PCOLS" # exactly 24 == 24 payload cols -> gates (>=) -check "premise: the width threshold now equals the projection width" "$(gate_now)" "$PCOLS" +check "the width threshold now equals the projection width" "$(gate_now)" "$PCOLS" check "wide, gate AT the width (>= boundary): gates again -> qual column alone" \ "$(counter_in "$(explain_of "$WIDE_NOMATCH")" 'Columnar Vector Decodes')" "$VECTORS" @@ -231,7 +233,7 @@ check "wide, gate AT the width (>= boundary): gates again -> qual column alone" # it against the heap oracle AND against itself with the gate on vs off. reset_gate -check "premise: back to the shipped default" "$(gate_now)" "$DEFAULT_GUC" +check "back to the shipped default" "$(gate_now)" "$DEFAULT_GUC" HASH_HEAP_NEEDLE="$(pgc_set_hash "SELECT * FROM hw WHERE tag LIKE '%needle%'")" HASH_COL_NEEDLE_GATED="$(pgc_set_hash "$WIDE_NEEDLE")" @@ -239,7 +241,7 @@ check "correctness: wide needle, gate ON, matches the heap oracle" \ "$HASH_COL_NEEDLE_GATED" "$HASH_HEAP_NEEDLE" set_gate "$NCOLS_WIDE" # gate OFF for the wide projection -check "premise: gate suppressed for the invariance arm" "$(gate_now)" "$NCOLS_WIDE" +check "gate suppressed for the invariance arm" "$(gate_now)" "$NCOLS_WIDE" check "correctness: wide needle, gate OFF, returns the identical rows (output-invariant)" \ "$(pgc_set_hash "$WIDE_NEEDLE")" "$HASH_COL_NEEDLE_GATED" diff --git a/test/native_decode_gating.sh b/test/native_decode_gating.sh index 2b8f9fc7b..399eacd13 100755 --- a/test/native_decode_gating.sh +++ b/test/native_decode_gating.sh @@ -84,38 +84,40 @@ PLAN_CONTROL="$(explain_of "$CONTROLQ")" # ---- premises -------------------------------------------------------------- -check "premise: the needle plan is a columnar custom scan" \ +check "the needle plan is a columnar custom scan" \ "$(has_line "$PLAN_NEEDLE" 'Columnar Projected Columns')" "yes" -check "premise: one row group, so every count below is one group's" \ +check "one row group, so every count below is one group's" \ "$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('n');")" \ "1" -check "premise: the width gate is pinned off, so this narrow projection still gates" \ +check "the width gate is pinned off, so this narrow projection still gates" \ "$(q "SHOW pgcolumnar.qual_skipvec_min_payload_cols;")" "0" # The whole point of the fixture is that this qual is NOT admitted to the reader. # If a future #426 taught the reader to prune an infix LIKE, numPredicates would # be non-zero, 1b-ii would do the ruling out, and this suite would pass without # phase 2 existing. So assert the unprunable shape rather than assume it. -check "premise: the reader has no usable skip predicate for this LIKE" \ +check "the reader has no usable skip predicate for this LIKE" \ "$(counter_in "$PLAN_NEEDLE" 'Columnar Usable Skip Predicates')" "0" -check "premise: no whole row group is pruned" \ +check "no whole row group is pruned" \ "$(counter_in "$PLAN_NEEDLE" 'Columnar Chunk Groups Removed by Filter')" "0" -check "premise: no vector is ruled out by VALUE either (that is 1b-ii's path)" \ +check "no vector is ruled out by VALUE either (that is 1b-ii's path)" \ "$(counter_in "$PLAN_NEEDLE" 'Columnar Vectors Ruled Out by Value')" "0" # Phase 2 rides on 1a's callback, so 1a's path must be the one in use. Its # counter is present only when late materialization is active, which is also what # puts the qual on the scan with tag as a qual column. If this line is absent the # qual is being applied somewhere else and no count below measures phase 2. -check "premise: the needle query takes the late-materialization path" \ +check "the needle query takes the late-materialization path" \ "$(has_line "$PLAN_NEEDLE" 'Columnar Rows Filtered Before Materialization')" "yes" +# The fixture's shape, read from the heap mirror the same generator filled (#1339): +# read from the columnar table, a one-line reader change reddened these premises. check "premise: the needle really matches one vector's worth of rows" \ - "$(q "SELECT count(*) FROM n WHERE tag LIKE '%needle%';")" "1024" + "$(q "SELECT count(*) FROM h WHERE tag LIKE '%needle%';")" "1024" check "premise: the nomatch predicate matches no row" \ - "$(q "SELECT count(*) FROM n WHERE tag LIKE '%zzzzz%';")" "0" + "$(q "SELECT count(*) FROM h WHERE tag LIKE '%zzzzz%';")" "0" check "premise: the control matches every row" \ - "$(q "SELECT count(*) FROM n WHERE tag LIKE '%row%';")" "$ROWS" + "$(q "SELECT count(*) FROM h WHERE tag LIKE '%row%';")" "$ROWS" # ---- the observable, and its unit ------------------------------------------ @@ -182,7 +184,7 @@ check "and skips every vector" \ HALFQ="SELECT * FROM n WHERE tag LIKE '%needle%' AND p1 % 2 = 0" PLAN_HALF="$(explain_of "$HALFQ")" check "premise: the half-needle survives in one vector, half of its rows" \ - "$(q "SELECT count(*) || '|' || count(DISTINCT (p1 - 1) / 1024) FROM n WHERE tag LIKE '%needle%' AND p1 % 2 = 0;")" "512|1" + "$(q "SELECT count(*) || '|' || count(DISTINCT (p1 - 1) / 1024) FROM h WHERE tag LIKE '%needle%' AND p1 % 2 = 0;")" "512|1" check "rejections inside a surviving vector are counted once" \ "$(counter_in "$PLAN_HALF" 'Rows Removed by Filter')" "$(( ROWS - 512 ))" @@ -223,13 +225,13 @@ psql_run "DELETE FROM hm WHERE p1 % 17 = 0;" # unprunable qual too, so it is run with the width gate out of reach (#1341). psql_run "SET pgcolumnar.qual_skipvec_min_payload_cols = 100000; DELETE FROM m WHERE p1 % 17 = 0;" -check "premise: the columnar table spans more than one row group" \ +check "the columnar table spans more than one row group" \ "$(q "SELECT count(*) > 1 FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('m');")" \ "t" # count(*) rather than a count WHERE p1 % 17 = 0: that WHERE is an unprunable qual # too, so it ran through the very gating under test, and a gating mutant broke this # premise rather than the arms below (#1341). -check "premise: rows really were deleted (so the mask is non-NULL)" \ +check "rows really were deleted (so the mask is non-NULL)" \ "$(q "SELECT count(*) FROM m;")" "$(( MROWS - MROWS / 17 ))" check "deletes + multiple groups + unprunable qual: columnar matches the heap" \ diff --git a/test/pytest/test_native_decode_gate_width.py b/test/pytest/test_native_decode_gate_width.py index 56e3620eb..28bf58e89 100644 --- a/test/pytest/test_native_decode_gate_width.py +++ b/test/pytest/test_native_decode_gate_width.py @@ -73,8 +73,14 @@ def _has(plan, label): def _set_gate(conn, v): + """A REFUSED SET IS LEFT FOR THE PREMISE THAT READS IT BACK (#1339). The shell's + ALTER DATABASE fails quietly and its premise reads what stuck; raising here ended + the test unnamed when the GUC's range lost the value (0, or the width).""" with conn.cursor() as cur: - cur.execute(f"SET pgcolumnar.qual_skipvec_min_payload_cols = {v}") + try: + cur.execute(f"SET pgcolumnar.qual_skipvec_min_payload_cols = {v}") + except psycopg.Error: + pass def _reset_gate(conn): @@ -98,43 +104,45 @@ def test_the_width_gate(pgc_conn, expect): cur.execute(f"INSERT INTO nw {gen}") expect.num(_gate_now(conn), DEFAULT_GUC, - f"premise: the width-gate GUC exists and defaults to the shipped {DEFAULT_GUC}") + f"the width-gate GUC exists and defaults to the shipped {DEFAULT_GUC}") expect.text(_show_state(conn, "pgcolumnar.qual_skipvec_probe_vecs"), "42704", "premise: the rejected probe GUC qual_skipvec_probe_vecs was NOT shipped") expect.text(_show_state(conn, "pgcolumnar.qual_skipvec_min_skip_pct"), "42704", "premise: the rejected probe GUC qual_skipvec_min_skip_pct was NOT shipped") expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.row_group " "WHERE storage_id = pgcolumnar.get_storage_id('nw')"), 1, - "premise: one row group, so every count below is one group's") + "one row group, so every count below is one group's") wide_needle = _plan(conn, WIDE_NEEDLE) narrow_needle = _plan(conn, NARROW_NEEDLE) narrow_nomatch = _plan(conn, NARROW_NOMATCH) expect.text(_has(wide_needle, "Columnar Projected Columns"), "yes", - "premise: the wide plan is a columnar custom scan with projection") + "the wide plan is a columnar custom scan with projection") expect.text(_has(narrow_nomatch, "Columnar Projected Columns"), "yes", - "premise: the narrow plan is a columnar custom scan with projection") + "the narrow plan is a columnar custom scan with projection") expect.num(_counter(wide_needle, "Columnar Usable Skip Predicates"), 0, - "premise: the LIKE is unprunable (no usable skip predicate)") + "the LIKE is unprunable (no usable skip predicate)") expect.num(_counter(wide_needle, "Columnar Chunk Groups Removed by Filter"), 0, - "premise: no whole row group is pruned") + "no whole row group is pruned") expect.num(_counter(wide_needle, "Columnar Vectors Ruled Out by Value"), 0, - "premise: no vector is ruled out by VALUE (that is 1b-ii's path, not phase 2)") + "no vector is ruled out by VALUE (that is 1b-ii's path, not phase 2)") expect.text(_has(wide_needle, "Columnar Rows Filtered Before Materialization"), "yes", - "premise: the wide query takes the late-materialization path") + "the wide query takes the late-materialization path") expect.text(_has(narrow_needle, "Columnar Rows Filtered Before Materialization"), "yes", - "premise: the narrow query takes the late-materialization path") + "the narrow query takes the late-materialization path") expect.text(_has(narrow_nomatch, "Columnar Vector Decodes"), "yes", - "premise: EXPLAIN reports Columnar Vector Decodes (the observable)") - expect.num(_one(conn, "SELECT count(*) FROM nw WHERE tag LIKE '%needle%'"), 1024, + "EXPLAIN reports Columnar Vector Decodes (the observable)") + # The fixture's shape, read from the heap mirror the same generator filled (#1339): + # read from the columnar table, a one-line reader change reddened these premises. + expect.num(_one(conn, "SELECT count(*) FROM hw WHERE tag LIKE '%needle%'"), 1024, "premise: the needle matches exactly one vector's worth of rows") - expect.num(_one(conn, "SELECT count(*) FROM nw WHERE tag LIKE '%zzzzz%'"), 0, + expect.num(_one(conn, "SELECT count(*) FROM hw WHERE tag LIKE '%zzzzz%'"), 0, "premise: the nomatch predicate matches no row") - expect.num(_one(conn, "SELECT count(*) FROM nw WHERE tag LIKE '%row%'"), ROWS, + expect.num(_one(conn, "SELECT count(*) FROM hw WHERE tag LIKE '%row%'"), ROWS, "premise: the control matches every row") # arm 1: at the shipped default, wide gates and narrow does not - expect.num(_gate_now(conn), DEFAULT_GUC, "premise: still at the default before arm 1") + expect.num(_gate_now(conn), DEFAULT_GUC, "still at the default before arm 1") wide_nomatch = _plan(conn, WIDE_NOMATCH) expect.num(_counter(wide_nomatch, "Columnar Vector Decodes"), VECTORS, "wide, default: a zero-match LIKE decodes the qual column and NO payload -> gated") @@ -150,7 +158,7 @@ def test_the_width_gate(pgc_conn, expect): # arm 2: the gate off, and the same narrow projection gates _set_gate(conn, 0) - expect.num(_gate_now(conn), 0, "premise: the width gate is now disabled (0)") + expect.num(_gate_now(conn), 0, "the width gate is now disabled (0)") expect.num(_counter(_plan(conn, NARROW_NOMATCH), "Columnar Vector Decodes"), VECTORS, "narrow, gate off (GUC=0): the SAME narrow projection now gates -> qual column alone") expect.num(_counter(_plan(conn, WIDE_CONTROL), "Columnar Vector Decodes"), VECTORS * NCOLS_WIDE, @@ -160,22 +168,22 @@ def test_the_width_gate(pgc_conn, expect): # arm 3: above the wide projection's width it stops; at exactly its width it gates _set_gate(conn, NCOLS_WIDE) - expect.num(_gate_now(conn), NCOLS_WIDE, "premise: the width threshold is now above the projection width") + expect.num(_gate_now(conn), NCOLS_WIDE, "the width threshold is now above the projection width") expect.num(_counter(_plan(conn, WIDE_NOMATCH), "Columnar Vector Decodes"), VECTORS * NCOLS_WIDE, "wide, gate raised above width: the SAME wide SELECT * stops gating -> full decode") _set_gate(conn, PCOLS) - expect.num(_gate_now(conn), PCOLS, "premise: the width threshold now equals the projection width") + expect.num(_gate_now(conn), PCOLS, "the width threshold now equals the projection width") expect.num(_counter(_plan(conn, WIDE_NOMATCH), "Columnar Vector Decodes"), VECTORS, "wide, gate AT the width (>= boundary): gates again -> qual column alone") # correctness: the gate never changes a result _reset_gate(conn) - expect.num(_gate_now(conn), DEFAULT_GUC, "premise: back to the shipped default") + expect.num(_gate_now(conn), DEFAULT_GUC, "back to the shipped default") gated = _rows(conn, WIDE_NEEDLE) expect.row_set(gated, _rows(conn, "SELECT * FROM hw WHERE tag LIKE '%needle%'"), "correctness: wide needle, gate ON, matches the heap oracle") _set_gate(conn, NCOLS_WIDE) - expect.num(_gate_now(conn), NCOLS_WIDE, "premise: gate suppressed for the invariance arm") + expect.num(_gate_now(conn), NCOLS_WIDE, "gate suppressed for the invariance arm") expect.row_set(_rows(conn, WIDE_NEEDLE), gated, "correctness: wide needle, gate OFF, returns the identical rows (output-invariant)") expect.row_set(_rows(conn, NARROW_NEEDLE), _rows(conn, "SELECT tag, p1, p2 FROM hw WHERE tag LIKE '%needle%'"), diff --git a/test/pytest/test_native_decode_gating.py b/test/pytest/test_native_decode_gating.py index e2a6e5148..6262e4747 100644 --- a/test/pytest/test_native_decode_gating.py +++ b/test/pytest/test_native_decode_gating.py @@ -12,6 +12,8 @@ """ import re +import psycopg + ROWS = 32768 VECTORS = 32 # ROWS / 1024 NCOLS = 6 # tag plus five payload columns @@ -32,6 +34,17 @@ def _rows(conn, sql): return cur.fetchall() +def _set_gate(conn, v): + """A REFUSED SET IS LEFT FOR THE PREMISE THAT READS IT BACK (#1339), as the shell's + PGC_EXTRA_CONF and psql -c leave it: raising here ended the test unnamed when the + GUC's range lost the value.""" + with conn.cursor() as cur: + try: + cur.execute(f"SET pgcolumnar.qual_skipvec_min_payload_cols = {v}") + except psycopg.Error: + pass + + def _run(conn, *sqls): with conn.cursor() as cur: for sql in sqls: @@ -65,31 +78,33 @@ def test_an_unprunable_qual_gates_payload_decode(pgc_conn, expect): conn = pgc_conn gen = (f"SELECT 'row' || CASE WHEN g <= 1024 THEN 'needle' ELSE 'miss' END, " f"g, g + 1, g + 2, g + 3, g + 4 FROM generate_series(1, {ROWS}) g") - _run(conn, "SET pgcolumnar.qual_skipvec_min_payload_cols = 0", - "CREATE TABLE h (tag text, p1 int, p2 int, p3 int, p4 int, p5 int)", + _set_gate(conn, 0) + _run(conn, "CREATE TABLE h (tag text, p1 int, p2 int, p3 int, p4 int, p5 int)", "CREATE TABLE n (tag text, p1 int, p2 int, p3 int, p4 int, p5 int) USING pgcolumnar", "SELECT pgcolumnar.set_options('n', stripe_row_limit => 65536, chunk_group_row_limit => 1024)", f"INSERT INTO h {gen}", f"INSERT INTO n {gen}") needle, nomatch, control = _plan(conn, NEEDLEQ), _plan(conn, NOMATCHQ), _plan(conn, CONTROLQ) - expect.text(_has(needle, "Columnar Projected Columns"), "yes", "premise: the needle plan is a columnar custom scan") + expect.text(_has(needle, "Columnar Projected Columns"), "yes", "the needle plan is a columnar custom scan") expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.row_group " "WHERE storage_id = pgcolumnar.get_storage_id('n')"), 1, - "premise: one row group, so every count below is one group's") + "one row group, so every count below is one group's") expect.text(_one(conn, "SHOW pgcolumnar.qual_skipvec_min_payload_cols"), "0", - "premise: the width gate is pinned off, so this narrow projection still gates") + "the width gate is pinned off, so this narrow projection still gates") expect.num(_counter(needle, "Columnar Usable Skip Predicates"), 0, - "premise: the reader has no usable skip predicate for this LIKE") - expect.num(_counter(needle, "Columnar Chunk Groups Removed by Filter"), 0, "premise: no whole row group is pruned") + "the reader has no usable skip predicate for this LIKE") + expect.num(_counter(needle, "Columnar Chunk Groups Removed by Filter"), 0, "no whole row group is pruned") expect.num(_counter(needle, "Columnar Vectors Ruled Out by Value"), 0, - "premise: no vector is ruled out by VALUE either (that is 1b-ii's path)") + "no vector is ruled out by VALUE either (that is 1b-ii's path)") expect.text(_has(needle, "Columnar Rows Filtered Before Materialization"), "yes", - "premise: the needle query takes the late-materialization path") - expect.num(_one(conn, "SELECT count(*) FROM n WHERE tag LIKE '%needle%'"), MATCHES, + "the needle query takes the late-materialization path") + # The fixture's shape, read from the heap mirror the same generator filled (#1339): + # read from the columnar table, a one-line reader change reddened these premises. + expect.num(_one(conn, "SELECT count(*) FROM h WHERE tag LIKE '%needle%'"), MATCHES, "premise: the needle really matches one vector's worth of rows") - expect.num(_one(conn, "SELECT count(*) FROM n WHERE tag LIKE '%zzzzz%'"), 0, + expect.num(_one(conn, "SELECT count(*) FROM h WHERE tag LIKE '%zzzzz%'"), 0, "premise: the nomatch predicate matches no row") - expect.num(_one(conn, "SELECT count(*) FROM n WHERE tag LIKE '%row%'"), ROWS, + expect.num(_one(conn, "SELECT count(*) FROM h WHERE tag LIKE '%row%'"), ROWS, "premise: the control matches every row") expect.text(_has(control, "Columnar Vector Decodes"), "yes", "EXPLAIN reports Columnar Vector Decodes") @@ -112,7 +127,7 @@ def test_an_unprunable_qual_gates_payload_decode(pgc_conn, expect): # Every row of the needle's surviving vector matches, so a producer that counted # its re-test's rejections twice reads right above; half of vector 0 passes here. - expect.text(_one(conn, "SELECT count(*) || '|' || count(DISTINCT (p1 - 1) / 1024) FROM n " + expect.text(_one(conn, "SELECT count(*) || '|' || count(DISTINCT (p1 - 1) / 1024) FROM h " "WHERE tag LIKE '%needle%' AND p1 % 2 = 0"), "512|1", "premise: the half-needle survives in one vector, half of its rows") expect.num(_counter(_plan(conn, HALFQ), "Rows Removed by Filter"), ROWS - 512, @@ -133,21 +148,28 @@ def test_deleted_rows_across_several_groups(pgc_conn, expect): conn = pgc_conn mgen = (f"SELECT 'x' || CASE WHEN g % 4096 = 7 THEN 'needle' ELSE 'miss' END || g::text, g, g * 2 " f"FROM generate_series(1, {MROWS}) g") - _run(conn, "SET pgcolumnar.qual_skipvec_min_payload_cols = 0", - "CREATE TABLE hm (tag text, p1 int, p2 int)", + _set_gate(conn, 0) + _run(conn, "CREATE TABLE hm (tag text, p1 int, p2 int)", "CREATE TABLE m (tag text, p1 int, p2 int) USING pgcolumnar", "SELECT pgcolumnar.set_options('m', stripe_row_limit => 8192, chunk_group_row_limit => 1024)", f"INSERT INTO hm {mgen}", f"INSERT INTO m {mgen}", - "DELETE FROM hm WHERE p1 % 17 = 0", - # the fixture's DELETE must not depend on the gating under test - "SET pgcolumnar.qual_skipvec_min_payload_cols = 100000", "DELETE FROM m WHERE p1 % 17 = 0", - "SET pgcolumnar.qual_skipvec_min_payload_cols = 0") + "DELETE FROM hm WHERE p1 % 17 = 0") + # The fixture's DELETE must not depend on the gating under test, so it runs with + # the gate out of reach. The SET and the DELETE go together, as the shell's one + # psql -c: a refused SET skips the DELETE there, and the premise below names it, + # rather than an unnamed psycopg error ending this test (#1756 review). + try: + _run(conn, "SET pgcolumnar.qual_skipvec_min_payload_cols = 100000", + "DELETE FROM m WHERE p1 % 17 = 0") + except psycopg.Error: + pass + _set_gate(conn, 0) expect.text(str(_one(conn, "SELECT count(*) > 1 FROM pgcolumnar.row_group " "WHERE storage_id = pgcolumnar.get_storage_id('m')")), "True", - "premise: the columnar table spans more than one row group") + "the columnar table spans more than one row group") # count(*), not a count WHERE p1 % 17 = 0: that WHERE would run through the gating expect.num(_one(conn, "SELECT count(*) FROM m"), MROWS - MROWS // 17, - "premise: rows really were deleted (so the mask is non-NULL)") + "rows really were deleted (so the mask is non-NULL)") expect.row_set(_rows(conn, "SELECT * FROM m WHERE tag LIKE '%needle%'"), _rows(conn, "SELECT * FROM hm WHERE tag LIKE '%needle%'"), "deletes + multiple groups + unprunable qual: columnar matches the heap")