diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 39328e2cc..f39ed7f04 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2477,6 +2477,24 @@ native_fetch_coalesce native_fetch_coalesce premise: fetching one projected colu native_fetch_coalesce native_fetch_coalesce premise: the coalescing helper holds both the bound and the validity copy 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce the validity copy is bounded by the chunk length before it runs 15;16;17;18;19 never - native_fetch_coalesce native_fetch_coalesce the wide fetch returns the projected values 15;16;17;18;19 2026-10-14 every column of a coalesced fetch is copied from the start of the span (off = 0) +native_fetch_projection native_fetch_projection a deleted row stops being read through the projection 15;16;17;18;19 2026-10-16 columnar_projection.c: read_projection ignores the base row's liveness +native_fetch_projection native_fetch_projection an uncovered column still reads correctly after a delete 15;16;17;18;19 2026-10-16 columnar_projection.c: reconstruct's uncovered set misses the table's last column +native_fetch_projection native_fetch_projection and does not decode the row to find out 15;16;17;18;19 2026-10-16 columnar_tableam.c: index deletion decodes each bottom-up row to decide +native_fetch_projection native_fetch_projection and its values are right with nothing decoded from the base 15;16;17;18;19 never - +native_fetch_projection native_fetch_projection and stops being reconstructed too 15;16;17;18;19 2026-10-16 columnar_projection.c: reconstruct ignores the projected fetch's liveness answer +native_fetch_projection native_fetch_projection every needed-set membership test consults that flag rather than a null set 15;16;17;18;19 2026-10-16 columnar_reader.c: one needed-set membership test no longer consults the all-columns flag +native_fetch_projection native_fetch_projection every reconstructed row matches the base table 15;16;17;18;19 2026-10-16 columnar_projection.c: reconstruct's uncovered set misses the table's last column +native_fetch_projection native_fetch_projection index deletion asks no row whether it is live: it vouches for nothing 15;16;17;18;19 2026-10-16 columnar_tableam.c: index deletion asks each bottom-up row whether it is live again +native_fetch_projection native_fetch_projection neither caller decodes every column: no full decode in the projection path 15;16;17;18;19 2026-10-16 columnar_projection.c: a full decode of the base row planted beside the liveness check (the #157 regression) +native_fetch_projection native_fetch_projection premise: the column-set caller is not counted as a full decode 15;16;17;18;19 never - +native_fetch_projection native_fetch_projection premise: the column-set entry point is called at all 15;16;17;18;19 never - +native_fetch_projection native_fetch_projection premise: the explicit all-columns flag is declared 15;16;17;18;19 never - +native_fetch_projection native_fetch_projection premise: the index deletion function was extracted, not blank 15;16;17;18;19 never - +native_fetch_projection native_fetch_projection premise: the visibility-only entry point is called at all 15;16;17;18;19 never - +native_fetch_projection native_fetch_projection premise: there is a needed-set membership test to guard 15;16;17;18;19 never - +native_fetch_projection native_fetch_projection the all-covering projection reconstructs every row 15;16;17;18;19 never - +native_fetch_projection native_fetch_projection the projection reads every row to start with 15;16;17;18;19 never - +native_fetch_projection native_fetch_projection the wide fixture has 41 columns 15;16;17;18;19 never - native_groupagg native_groupagg a NULL key counts as a group in the bucket bound: four groups under a cap of 3 are declined and all answered 15;16;17;18;19 2026-10-16 the bucket bound left out the NULL group and the runtime cap was off, so the node took the query and answered it (mutation, #1339) native_groupagg native_groupagg a cached plan run under a lower cap stops at the runtime cap, with its hint 15;16;17;18;19 never - native_groupagg native_groupagg a date_bin key is bounded as a truncating key is 15;16;17;18;19 never - @@ -2571,6 +2589,75 @@ native_groupagg native_groupagg toggle sum signed-zero (-0 preserved) [node fire native_groupagg native_groupagg while a matching-type predicate still gets one 15;16;17;18;19 never - native_groupagg native_groupagg with no time bound at all, a truncating key is bounded from the zone maps 15;16;17;18;19 never - native_groupagg native_groupagg with only a lower time bound, a truncating key is still bounded 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch <> filter: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the batch gates' exactness test dropped, so a <> filter's rows are folded +native_groupagg_batch native_groupagg_batch <> filter: the excluded rows really are excluded 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch <> filter: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: the batch gates' exactness test dropped, so a <> filter's rows are folded +native_groupagg_batch native_groupagg_batch add column: ANALYZE reports the row-path run 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped node's ANALYZE prints the predicted fold, not what ran (the #602 shape) +native_groupagg_batch native_groupagg_batch add column: plain EXPLAIN reports the prediction 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch add column: the answer is still right 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch count(*): answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch count(*): the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch count(*): the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch date key: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch date key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch date key: the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch deletes: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch deletes: the fold still ran 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch expression key: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: an expression key's -1 attidx is not refused by the grouped fold's eligibility +native_groupagg_batch native_groupagg_batch expression key: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: an expression key's -1 attidx is not refused by the grouped fold's eligibility +native_groupagg_batch native_groupagg_batch filtered: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row +native_groupagg_batch native_groupagg_batch filtered: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch filtered: the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch float8 key: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch float8 key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch float8 key: the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch float8 zero: -0.0 and 0.0 are one group, exactly as the heap says 15;16;17;18;19 2026-10-16 columnar_vector.c: float8 admitted as a simple key, so its group keys are hashed and compared by bits +native_groupagg_batch native_groupagg_batch float8 zero: and a positive one beside it 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch float8 zero: and the fold actually ran on it 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch float8 zero: the fixture really stores a NEGATIVE zero 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch float8 zero: the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch float8 zero: the grouped sums match the heap 15;16;17;18;19 2026-10-16 columnar_vector.c: float8 admitted as a simple key, so its group keys are hashed and compared by bits +native_groupagg_batch native_groupagg_batch min/max: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch min/max: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: min and max admitted as batch fold kinds +native_groupagg_batch native_groupagg_batch parallel: and they equal the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter;columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch parallel: our parallel-aware grouped node is under the Gather 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch parallel: premise: the value arm's own plan launches workers 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch parallel: the answers equal the serial fold, byte for byte 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter +native_groupagg_batch native_groupagg_batch parallel: the fold ran in the partial node (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch parallel: workers actually launched for that node 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch plain: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch plain: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch plain: the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch premise: fixture loaded every row (columnar) 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch premise: fixture loaded every row (heap) 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch premise: the <> filter excludes something at all 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch premise: the delete removed rows 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch premise: the fixture really has null keys 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch premise: the fixture spans more than one row group 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch skipped vectors: and their sums match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a skipped vector's present slots are not consumed, so later vectors misalign;columnar_vector.c: the grouped fold applies no scan key per row +native_groupagg_batch native_groupagg_batch skipped vectors: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a skipped vector's present slots are not consumed, so later vectors misalign;columnar_vector.c: the grouped fold applies no scan key per row +native_groupagg_batch native_groupagg_batch skipped vectors: premise: a row group was still read 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch skipped vectors: premise: the scan really skipped vectors 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch skipped vectors: premise: the wide arm is still the grouped node 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch skipped vectors: the fold ran 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch skipped vectors: the fold ran on the four-payload arm 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch text key: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the batch gates' by-value test dropped, so a text key reaches the gather +native_groupagg_batch native_groupagg_batch text key: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: the batch gates' by-value test dropped, so a text key reaches the gather;columnar_vector.c: the grouped path is not planned for a text group key +native_groupagg_batch native_groupagg_batch timestamp key: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch timestamp key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch timestamp key: the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch toggle-differential: filtered 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: filtered [premise: node fires with the path on] 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch toggle-differential: filtered [premise: the off arm is not the grouped node] 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch toggle-differential: plain 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: plain [premise: node fires with the path on] 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch toggle-differential: plain [premise: the off arm is not the grouped node] 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch toggle-differential: two keys 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: two keys [premise: node fires with the path on] 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch toggle-differential: two keys [premise: the off arm is not the grouped node] 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch two keys: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch two keys: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch two keys: the grouped node is planned 15;16;17;18;19 never - native_index_projection native_index_projection a parallel build projects one column of twenty, not all twenty 15;16;17;18;19 never - native_index_projection native_index_projection a parallel expression build projects its expression's columns 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index build's projection omits ii_Expressions' columns native_index_projection native_index_projection a parallel partial build projects the predicate's columns too 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index build's projection omits ii_Predicate's columns diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 83a2a80bb..4d97cc201 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 201 +suites_not_covered 199 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3285,4 +3285,39 @@ suites_not_covered 201 # never 2941 + 1 = 2942 # dated 1160 + 0 = 1160 # 2942 + 1160 == 4102 -checks_never_observed_red 2942 +# +# #1341, native_groupagg_batch seeded, with its twin (OffgridwithJD): 2942 -> 2980, +# and suites_not_covered 201 -> 200. The suite had no rows. It enters with all 69 +# checks from five-major runs (69/69 on each of 15-19). 31 are dated by twelve +# mutations of the grouped fold: the fold never called, float8 keys compared by +# bits, the delete mask ignored, a skipped vector's slots not consumed, no scan key +# applied, the by-value, exactness and expression-key gates dropped, min and max +# admitted, the partial node without its shared counter, ANALYZE printing the +# prediction, and the grouped path refusing a text key. Thirty-eight arrive never: +# sixteen premises and the two float8 fixture facts; the eight "grouped node is +# planned" arms and the two parallel plan arms, which no fold mutation reaches; +# seven heap-mirror oracles that none of these mutations moves; the <> count, read +# by an ungrouped count(*); and the add-column prediction and answer. +# +# ledger rows 4102 + 69 = 4171 +# never 2942 + 38 = 2980 +# dated 1160 + 31 = 1191 +# 2980 + 1191 == 4171 +# +# #1341, native_fetch_projection seeded, with its twin (OffgridwithJD): 2980 -> 2990, +# and suites_not_covered 200 -> 199. The suite had no rows. It enters with all 18 +# checks from five-major runs (18/18 on each of 15-19). 8 are dated by seven +# mutations: reconstruct's uncovered set missing the last column, read_projection +# and reconstruct ignoring liveness, a full decode planted beside the liveness +# check, index deletion asking each row whether it is live and decoding each row, +# and an unguarded needed-set test. Ten arrive never: seven premises; the fixture's +# column count and the opening row count; and the two all-covering-projection +# arms, which an empty set read as "every column" cannot redden, because decoding +# everything still answers right (measured: the old convention restored leaves +# both green). +# +# ledger rows 4171 + 18 = 4189 +# never 2980 + 10 = 2990 +# dated 1191 + 8 = 1199 +# 2990 + 1199 == 4189 +checks_never_observed_red 2990 diff --git a/test/native_fetch_projection.sh b/test/native_fetch_projection.sh index 28d1f85b1..75411153a 100755 --- a/test/native_fetch_projection.sh +++ b/test/native_fetch_projection.sh @@ -54,7 +54,9 @@ psql_run "DROP TABLE IF EXISTS fp_w; CREATE TABLE fp_w (id int$cols) USING pgcolumnar; INSERT INTO fp_w SELECT g$sel FROM generate_series(1, $ROWS) g;" >/dev/null -check "the wide fixture has $((NCOLS + 1)) columns" \ +# A literal name, so the ledger and the twin's grader can read it. The +# all-covering projection below names c1..c40, so NCOLS is 40 in practice. +check "the wide fixture has 41 columns" \ "$(q "SELECT count(*) FROM information_schema.columns WHERE table_name = 'fp_w';")" "$((NCOLS + 1))" @@ -139,6 +141,20 @@ check "and its values are right with nothing decoded from the base" \ # every check above while giving back what the change was for. SRC="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/src" +# Every source arm below reads CODE, not prose. The index-deletion arm used to +# pass on a comment: pgcolumnar_index_delete_tuples stopped asking any row +# whether it is live (it vouches for nothing, #1341), and the only +# PgColumnarRowIsLive left in it was the comment explaining why. String literals +# are kept, so a "/*" inside one is not read as a comment. +code_of() { # code_of FILE -> FILE with its C comments blanked, line count kept + python3 -c ' +import re, sys +s = open(sys.argv[1]).read() +pat = re.compile(r"\"(?:\\.|[^\"\\])*\"|\x27(?:\\.|[^\x27\\])*\x27|/\*.*?\*/|//[^\n]*", re.S) +sys.stdout.write(pat.sub(lambda m: m.group(0) if m.group(0)[0] in "\"\x27" else "\n" * m.group(0).count("\n"), s)) +' "$1" +} + # THE ARM NAMED "DECODES NOTHING" COULD NOT SEE A DECODE. Both of these counted a # CALL SITE, with its argument text, and pinned it at a literal 1 -- so they # asserted that a particular call is still written the way it was written, which is @@ -162,9 +178,10 @@ SRC="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/src" # # The two premises keep the zero from being vacuous: a file that called NOTHING # would also report zero full decodes. -_np_live="$(grep -c 'PgColumnarRowIsLive(' "$SRC/columnar_projection.c")" -_np_cols="$(grep -c 'PgColumnarReadRowByNumberCols(' "$SRC/columnar_projection.c")" -_np_full="$(grep -c 'PgColumnarReadRowByNumber(' "$SRC/columnar_projection.c")" +_np_code="$(code_of "$SRC/columnar_projection.c")" +_np_live="$(grep -c 'PgColumnarRowIsLive(' <<<"$_np_code")" +_np_cols="$(grep -c 'PgColumnarReadRowByNumberCols(' <<<"$_np_code")" +_np_full="$(grep -c 'PgColumnarReadRowByNumber(' <<<"$_np_code")" check "premise: the visibility-only entry point is called at all" \ "$([ "${_np_live:-0}" -ge 1 ] && echo yes || echo no)" "yes" @@ -198,10 +215,19 @@ check "neither caller decodes every column: no full decode in the projection pat # string appears legitimately elsewhere now that the index fetch also asks only # for liveness, and a whole-file count turned that correct second use into a # failure. -deltuples="$(awk '/^pgcolumnar_index_delete_tuples\(/,/^}/' "$SRC/columnar_tableam.c")" +# +# The first arm was "index deletion asks whether the row is live". It stopped +# being true when the function stopped asking (an answer from this backend's +# snapshot erased index entries of rows live to everyone else), and it stayed +# green on the comment that says so. The property now is the absence: a +# per-row liveness test here is the regression, so it is what reddens. +deltuples="$(code_of "$SRC/columnar_tableam.c" | awk '/^pgcolumnar_index_delete_tuples\(/,/^}/')" + +check "premise: the index deletion function was extracted, not blank" \ + "$(case "$deltuples" in *ndeltids*) echo yes ;; *) echo "extracted=<${deltuples:-empty}>" ;; esac)" "yes" -check "index deletion asks whether the row is live" \ - "$(case "$deltuples" in *PgColumnarRowIsLive*) echo yes ;; *) echo no ;; esac)" "yes" +check "index deletion asks no row whether it is live: it vouches for nothing" \ + "$(case "$deltuples" in *PgColumnarRowIsLive*) echo "no (asks per row)" ;; *) echo yes ;; esac)" "yes" check "and does not decode the row to find out" \ "$(case "$deltuples" in *PgColumnarReadRowByNumber*) echo "no (still decodes)" ;; @@ -222,9 +248,10 @@ check "and does not decode the row to find out" \ # first. Honest callers move both counts together; an unguarded test moves only # the total. Both numbers are printed in the arm so the comparison is the # reconciliation rather than a bare verdict. -_afc_tests="$(grep -c 'bms_is_member(c, needed)' "$SRC/columnar_reader.c")" -_afc_guarded="$(grep -c '!allColumns && !bms_is_member(c, needed)' "$SRC/columnar_reader.c")" -_afc_flags="$(grep -c 'bool allColumns' "$SRC/columnar_reader.c")" +_afc_code="$(code_of "$SRC/columnar_reader.c")" +_afc_tests="$(grep -c 'bms_is_member(c, needed)' <<<"$_afc_code")" +_afc_guarded="$(grep -c '!allColumns && !bms_is_member(c, needed)' <<<"$_afc_code")" +_afc_flags="$(grep -c 'bool allColumns' <<<"$_afc_code")" check "premise: there is a needed-set membership test to guard" \ "$([ "${_afc_tests:-0}" -ge 1 ] && echo yes || echo no)" "yes" diff --git a/test/native_groupagg_batch.sh b/test/native_groupagg_batch.sh index dfa2ebe91..5bde7ce08 100755 --- a/test/native_groupagg_batch.sh +++ b/test/native_groupagg_batch.sh @@ -170,13 +170,16 @@ for pair in "text key:$Q_TEXTKEY" "expression key:$Q_EXPRKEY" \ "min/max:$Q_MINMAX" "<> filter:$Q_NEQ"; do label="${pair%%:*}"; tmpl="${pair#*:}" qcol="${tmpl//%T/gbb}" + # The decline must be the FOLD's, read off the grouped node. This branch was + # a check_skip, and it never fired on 15-19: with the node gone the gate + # under test is unreachable, so a skip there was a silent pass. A shape the + # node stops taking is now a red under the arm's own name. if [ "$(is_groupvec "$qcol")" = yes ]; then check_text "$label: the fold declined" \ "$(fold_of "$ANALYZE_PFX" "$qcol")" no else - # The node itself declined the shape; there is no fold line to read and - # nothing for this suite to gate. Say so rather than assert a missing line. - check_skip "$label" "SKIP $label: the grouped node is not planned for this shape" "the grouped node is not planned for this shape" + check_text "$label: the fold declined" \ + "the grouped node is not planned" no fi agree "$label: answers match the heap mirror" "$tmpl" done @@ -315,7 +318,7 @@ agree_in gbs "skipped vectors: answers match the heap mirror" "$Q_SKIP" Q_SKIP4="SELECT k, count(*), sum(p1), sum(p2), sum(p3), sum(p4) FROM %T WHERE sel = 1 GROUP BY k" check_text "skipped vectors: premise: the wide arm is still the grouped node" \ "$(is_groupvec "${Q_SKIP4//%T/gbs}")" yes -check_text "skipped vectors: four payload columns all stay in step" \ +check_text "skipped vectors: the fold ran on the four-payload arm" \ "$(fold_of "$ANALYZE_PFX" "${Q_SKIP4//%T/gbs}")" yes agree_in gbs "skipped vectors: and their sums match the heap mirror" "$Q_SKIP4" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index a0f21f926..babd7df85 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -210,6 +210,8 @@ behaviour, the source of that number is named. - [162. test_native_vecskip.py: per-vector skipping by the zone maps](#162-test_native_vecskippy-per-vector-skipping-by-the-zone-maps) - [163. test_native_index_projection.py: CREATE INDEX projects only what it needs](#163-test_native_index_projectionpy-create-index-projects-only-what-it-needs) - [164. test_native_saop_pushdown.py: IN-lists and = ANY(array) prune row groups](#164-test_native_saop_pushdownpy-in-lists-and--anyarray-prune-row-groups) +- [165. test_native_groupagg_batch.py: the grouped aggregate folds column-at-a-time](#165-test_native_groupagg_batchpy-the-grouped-aggregate-folds-column-at-a-time) +- [166. test_native_fetch_projection.py: a fetch by row number decodes only what its caller needs](#166-test_native_fetch_projectionpy-a-fetch-by-row-number-decodes-only-what-its-caller-needs) ## 1. How to read a test in here @@ -7659,3 +7661,33 @@ The pytest twin of `test/native_saop_pushdown.sh` (ported for #1341), every chec | `test_the_batch_fold_never_takes_the_set_keys_as_exact` | the batch fold refuses the IN-list but accepts the equivalent range, and the vectorized count stays exact | | `test_a_correlated_array_is_not_frozen_and_the_backend_lives` | a correlated LATERAL array is not frozen, and the session's backend is still alive; a crash is a named red | | `test_the_reader_refuses_every_special_flag_but_the_array_one` | the reader's reject expression names every special scan-key flag, and SK_SEARCHARRAY reaches its own arm | + +## 165. test_native_groupagg_batch.py: the grouped aggregate folds column-at-a-time + +The pytest twin of `test/native_groupagg_batch.sh` (ported for #1341), every check by the shell's name. The grouped vectorized aggregate folds a whole row group at a time (#708), and two things are proved separately. The answers must be unchanged: each shape agrees with a heap mirror, and with the same rows when the path is off. The work must happen: "Columnar Batch Fold: yes" under EXPLAIN ANALYZE is the only check that goes red when the fold stops running. Each shape the fold must decline is read off the grouped node, so a shape the node stops taking is a red, not a skip. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_fold_takes_every_eligible_shape_and_answers_as_the_heap_does` | seven shapes (plain, filtered, two keys, float8, count(*), timestamp and date keys) plan the grouped node, fold, and match the heap | +| `test_the_fold_declines_every_ineligible_shape_and_still_answers` | a text key, an expression key, min/max and a `<>` filter make the fold decline on the grouped node, and each still matches the heap | +| `test_the_fold_honours_the_delete_mask` | after deletes the fold still runs and matches the heap | +| `test_the_fold_equals_the_row_path_over_the_same_rows` | three shapes answer the same with the path off and on, and the off arm really is not the grouped node | +| `test_float8_keys_group_by_the_types_equality_not_by_bits` | a stored -0.0 and 0.0 are one group on the fold, as on the heap | +| `test_the_fold_steps_over_skipped_vectors` | a scan that really skips vectors folds, and its one-column and four-column sums match the heap | +| `test_the_parallel_partial_fold_equals_the_serial_one` | our parallel-aware node launches workers and folds, and equals the serial fold and the heap | +| `test_a_column_added_after_some_row_groups_runs_the_row_path` | plain EXPLAIN predicts the fold, ANALYZE reports the row path that ran, and the answer is right | + +## 166. test_native_fetch_projection.py: a fetch by row number decodes only what its caller needs + +The pytest twin of `test/native_fetch_projection.sh` (ported for #1341), every check by the shell's name. A row fetched by number decoded every column whatever the caller wanted (#157). Two narrower entry points exist beside it: one decodes a given set of columns, and one answers visibility and decodes nothing. Neither has a SQL surface, so they are exercised through `pgcolumnar.read_projection` and `pgcolumnar.reconstruct_via_projection` on a 41-column table, against the base table as the oracle. The source arms read code with its comments blanked, as the shell's `code_of` does. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_projected_fetch_returns_the_base_tables_rows_and_sees_deletes` | every reconstructed row matches the base table, a deleted row stops being read or reconstructed, and an uncovered column still reads right | +| `test_a_projection_covering_every_column_decodes_nothing_and_is_right` | with the computed base-column set empty, every row is reconstructed with the right values | +| `test_no_caller_decodes_every_column_and_index_deletion_reads_no_row` | the projection path calls no full decode, and index deletion asks no row whether it is live | +| `test_every_needed_set_test_consults_the_all_columns_flag` | every needed-set membership test in the reader consults the explicit all-columns flag | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 58ea62469..1df468344 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -791,4 +791,8 @@ guard_tests 438 # four tests, by collection over the cluster files on this tree. # cluster_tests 702 -> 710 for #1341's test_native_saop_pushdown.py, the twin of native_saop_pushdown.sh: # eight tests, by collection over the cluster files on this tree. -cluster_tests 710 +# cluster_tests 710 -> 718 for #1341's test_native_groupagg_batch.py, the twin of native_groupagg_batch.sh: +# eight tests, by collection over the cluster files on this tree. +# cluster_tests 718 -> 722 for #1341's test_native_fetch_projection.py, the twin of native_fetch_projection.sh: +# four tests, by collection over the cluster files on this tree. +cluster_tests 722 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index a4276febb..4c323a6c0 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -130,7 +130,9 @@ "native_dml", "native_encdesc_golden", "native_fetch_coalesce", + "native_fetch_projection", "native_groupagg", + "native_groupagg_batch", "native_index_fetch_stripe_cost", "native_index_projection", "native_join_vector_agg", @@ -410,7 +412,7 @@ def test_the_loop_reader_invents_nothing_in_this_corpus(expect): "every name the loop reader returns appears verbatim in the file it " "came from") expect.text(", ".join(files), - "build_refusal, differential, join_runtime_filter, " + "build_refusal, differential, join_runtime_filter, native_groupagg_batch, " "native_index_projection, native_reclaim, native_repack, " "native_saop_pushdown, native_vacuum_race, native_vecskip, sorted_pathkeys", "and it is these files, so another appearing is a diff a reviewer " diff --git a/test/pytest/test_native_fetch_projection.py b/test/pytest/test_native_fetch_projection.py new file mode 100644 index 000000000..a2536e8a0 --- /dev/null +++ b/test/pytest/test_native_fetch_projection.py @@ -0,0 +1,141 @@ +"""Fetch-by-row-number with a column projection (#157, #1341). The pytest twin of +`test/native_fetch_projection.sh`, every check by the shell's name. + +A row fetched by number decoded every column whatever the caller wanted. Two narrower +entry points exist beside it: one decodes a given set of columns, one answers +visibility and decodes nothing. They have no SQL surface of their own, so they are +exercised through their callers, pgcolumnar.read_projection and +pgcolumnar.reconstruct_via_projection, on a 41-column table: the output is unchanged +against the base table, a deleted row stops being read, and a projection covering +every column reconstructs right with nothing decoded from the base. Then the source: +neither caller decodes every column, index deletion asks no row whether it is live, +and every needed-set membership test consults the explicit all-columns flag. The +source arms read code with its comments blanked, as the shell's code_of does. +""" +import pathlib +import re + +import psycopg +import pytest + +ROWS = 20000 +NCOLS = 40 +SRC = pathlib.Path(__file__).resolve().parents[2] / "src" +COLS = "".join(f", c{i} bigint" for i in range(1, NCOLS + 1)) +SEL = "".join(f", g * {i}" for i in range(1, NCOLS + 1)) +_COMMENT = re.compile(r"\"(?:\\.|[^\"\\])*\"|'(?:\\.|[^'\\])*'|/\*.*?\*/|//[^\n]*", re.S) + + +def _code(name): + """-> the source file with its C comments blanked (line count kept), literals kept.""" + s = (SRC / name).read_text() + return _COMMENT.sub(lambda m: m.group(0) if m.group(0)[0] in "\"'" else "\n" * m.group(0).count("\n"), s) + + +def _lines(code, pat): + """-> how many lines hold `pat`, as grep -c counts.""" + return sum(pat in line for line in code.splitlines()) + + +def _one(c, sql): + return c.execute(sql).fetchone()[0] + + +@pytest.fixture(scope="module") +def fp(pgc_cluster): + """-> a connection to a database holding fp_w and its projection fp_p.""" + db = "pgc_fetchproj" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + c.execute(f"CREATE TABLE fp_w (id int{COLS}) USING pgcolumnar") + c.execute(f"INSERT INTO fp_w SELECT g{SEL} FROM generate_series(1, {ROWS}) g") + c.execute("SELECT pgcolumnar.add_projection('fp_w', 'fp_p', ARRAY['id','c1'], ARRAY['id'])") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_the_projected_fetch_returns_the_base_tables_rows_and_sees_deletes(fp, expect): + c = fp + expect.num(_one(c, "SELECT count(*) FROM information_schema.columns WHERE table_name = 'fp_w'"), + NCOLS + 1, "the wide fixture has 41 columns") + # the oracle is the base table, which does not go through the projected fetch + last = NCOLS + 1 + expect.num(_one(c, f"""WITH viaproj AS ( + SELECT pgcolumnar.reconstruct_via_projection('fp_w','fp_p') AS r + ), direct AS ( + SELECT id::text || '|' || c1::text || '|' || c2::text || '|' || c{NCOLS}::text AS d FROM fp_w + ) + SELECT count(*) FROM viaproj + WHERE split_part(r, '|', 1) || '|' || split_part(r, '|', 2) || '|' || + split_part(r, '|', 3) || '|' || split_part(r, '|', {last}) + NOT IN (SELECT d FROM direct)"""), 0, "every reconstructed row matches the base table") + + expect.num(_one(c, "SELECT count(*) FROM pgcolumnar.read_projection('fp_w','fp_p')"), ROWS, + "the projection reads every row to start with") + c.execute("DELETE FROM fp_w WHERE id <= 100") + expect.num(_one(c, "SELECT count(*) FROM pgcolumnar.read_projection('fp_w','fp_p')"), ROWS - 100, + "a deleted row stops being read through the projection") + expect.num(_one(c, "SELECT count(*) FROM pgcolumnar.reconstruct_via_projection('fp_w','fp_p')"), ROWS - 100, + "and stops being reconstructed too") + # a value from an uncovered column, so the set was not narrowed by one column too many + expect.text(_one(c, f"SELECT split_part(r, '|', {last}) " + "FROM pgcolumnar.reconstruct_via_projection('fp_w','fp_p') AS r " + "WHERE split_part(r, '|', 1) = '101'"), + _one(c, f"SELECT c{NCOLS}::text FROM fp_w WHERE id = 101"), + "an uncovered column still reads correctly after a delete") + + +def test_a_projection_covering_every_column_decodes_nothing_and_is_right(fp, expect): + c = fp + # the computed set of base columns comes out EMPTY here, which once read as "every column" + c.execute(f"CREATE TABLE fp_all (id int{COLS}) USING pgcolumnar") + c.execute(f"INSERT INTO fp_all SELECT g{SEL} FROM generate_series(1, 2000) g") + every = ", ".join(["'id'"] + [f"'c{i}'" for i in range(1, NCOLS + 1)]) + c.execute(f"SELECT pgcolumnar.add_projection('fp_all', 'fp_ap', ARRAY[{every}], ARRAY['id'])") + expect.num(_one(c, "SELECT count(*) FROM pgcolumnar.reconstruct_via_projection('fp_all','fp_ap')"), 2000, + "the all-covering projection reconstructs every row") + expect.text(_one(c, f"SELECT split_part(r, '|', 2) || '/' || split_part(r, '|', {NCOLS + 1}) " + "FROM pgcolumnar.reconstruct_via_projection('fp_all','fp_ap') AS r " + "WHERE split_part(r, '|', 1) = '77'"), + _one(c, f"SELECT c1::text || '/' || c{NCOLS}::text FROM fp_all WHERE id = 77"), + "and its values are right with nothing decoded from the base") + + +def test_no_caller_decodes_every_column_and_index_deletion_reads_no_row(expect): + proj = _code("columnar_projection.c") + # a want-zero on the full decode; the premises keep a file that calls nothing from passing + expect.at_least(_lines(proj, "PgColumnarRowIsLive("), 1, "premise: the visibility-only entry point is called at all") + expect.at_least(_lines(proj, "PgColumnarReadRowByNumberCols("), 1, + "premise: the column-set entry point is called at all") + expect.num(_lines("PgColumnarReadRowByNumberCols(a, b)", "PgColumnarReadRowByNumber("), 0, + "premise: the column-set caller is not counted as a full decode") + expect.text(f"{_lines(proj, 'PgColumnarReadRowByNumber(')} full decode(s)", "0 full decode(s)", + "neither caller decodes every column: no full decode in the projection path") + + lines = _code("columnar_tableam.c").splitlines() + start = next((i for i, l in enumerate(lines) if l.startswith("pgcolumnar_index_delete_tuples(")), None) + end = next((i for i in range(start, len(lines)) if lines[i].startswith("}")), len(lines)) if start is not None else 0 + deltuples = "\n".join(lines[start:end + 1]) if start is not None else "" + expect.text("yes" if "ndeltids" in deltuples else f"extracted=<{deltuples or 'empty'}>", "yes", + "premise: the index deletion function was extracted, not blank") + expect.text("no (asks per row)" if "PgColumnarRowIsLive" in deltuples else "yes", "yes", + "index deletion asks no row whether it is live: it vouches for nothing") + expect.text("no (still decodes)" if "PgColumnarReadRowByNumber" in deltuples else "yes", "yes", + "and does not decode the row to find out") + + +def test_every_needed_set_test_consults_the_all_columns_flag(expect): + reader = _code("columnar_reader.c").splitlines() + tests = sum("bms_is_member(c, needed)" in l for l in reader) + guarded = sum("!allColumns && !bms_is_member(c, needed)" in l for l in reader) + expect.at_least(tests, 1, "premise: there is a needed-set membership test to guard") + expect.at_least(sum("bool allColumns" in l for l in reader), 1, "premise: the explicit all-columns flag is declared") + # honest callers move both counts together; an unguarded test moves only the total + expect.text(f"guarded {guarded} of {tests}", f"guarded {tests} of {tests}", + "every needed-set membership test consults that flag rather than a null set") diff --git a/test/pytest/test_native_groupagg_batch.py b/test/pytest/test_native_groupagg_batch.py new file mode 100644 index 000000000..182e7884c --- /dev/null +++ b/test/pytest/test_native_groupagg_batch.py @@ -0,0 +1,272 @@ +"""The grouped vectorized aggregate folds column-at-a-time (#708, #1341). The pytest twin +of `test/native_groupagg_batch.sh`, every check by the shell's name. + +Two things are proved separately. The ANSWERS: every shape agrees with a heap mirror, +and with the same rows on the path off. The WORK: "Columnar Batch Fold: yes" on the +grouped node under EXPLAIN ANALYZE, the only assertion that goes red when the fold +stops running. Eligibility both ways: a by-reference key, an expression key, min/max +and a `<>` filter each make the fold decline, and each still answers right. Then the +delete mask, -0.0 and 0.0 as one float8 group, a scan that really skips vectors, the +parallel partial fold, and a column added after some row groups (predicted yes, ran +no). +""" +import re + +import psycopg +import pytest + +ANALYZE = "EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF)" +PLAIN = "EXPLAIN (COSTS OFF)" +MARKER = "Columnar Vectorized Group Keys" +DDL = "k int, k8 bigint, f float8, t text, ts timestamp, d date, v int, w int" +GEN = """SELECT CASE WHEN g % 997 = 0 THEN NULL ELSE g % 8 END, + (g % 5)::bigint, + (g % 4)::float8, + 't' || (g % 6), + timestamp '2024-01-01 00:00:00' + ((g % 7) * interval '1 day'), + date '2024-03-01' + (g % 9), + g, + g % 100 + FROM generate_series(1,200000) g""" +SGEN = """SELECT g % 8, CASE WHEN (g / 1024) % 40 = 0 THEN 1 ELSE 0 END, + g, g % 3, g % 5, g % 7, g % 11 + FROM generate_series(1,200000) g""" +PAR = ("max_parallel_workers_per_gather=4", "parallel_setup_cost=0", "parallel_tuple_cost=0", + "min_parallel_table_scan_size=0", "min_parallel_index_scan_size=0", + "pgcolumnar.enable_parallel_vector_agg=on") +Q_PLAIN = "SELECT k, count(*), sum(v) FROM %T GROUP BY k" +Q_FILT = "SELECT k, count(*), sum(v) FROM %T WHERE v < 50000 GROUP BY k" +Q_TWOKEY = "SELECT k, k8, count(*), sum(w) FROM %T GROUP BY k, k8" +Q_SKIP = "SELECT k, count(*), sum(p1) FROM %T WHERE sel = 1 GROUP BY k" +Q_SKIP4 = "SELECT k, count(*), sum(p1), sum(p2), sum(p3), sum(p4) FROM %T WHERE sel = 1 GROUP BY k" +Q_PAR = "SELECT k, count(*), sum(v) FROM gbb GROUP BY k" + + +def _rows(c, sql): + """-> the rows as sorted text tuples, NULL as "\\0" so it never equals ''. A query that + errors returns its error as the one row, so the comparison reddens by name.""" + try: + return sorted(tuple("\0" if v is None else str(v) for v in r) for r in c.execute(sql)) + except psycopg.Error as exc: + return [(f"error: {exc.diag.message_primary}",)] + + +def _one(c, sql): + return c.execute(sql).fetchone()[0] + + +def _text(c, sql): + """-> the rows' first column joined, or the error: a plan that errors has no fold line.""" + try: + return "\n".join(str(r[0]) for r in c.execute(sql)) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _groupvec(c, sql): + return "yes" if MARKER in _text(c, f"{PLAIN} {sql}") else "no" + + +def _fold(c, pfx, sql): + """-> the grouped node's "Columnar Batch Fold" value, '' when the plan has none.""" + m = re.search(r"Columnar Batch Fold: (yes|no)", _text(c, f"{pfx} {sql}")) + return m.group(1) if m else "" + + +def _num(plan, label): + m = re.search(rf"{re.escape(label)}: (\d+)", plan) + return int(m.group(1)) if m else 0 + + +def _agree(c, tbl, tmpl): + """-> (columnar rows, heap rows). An empty columnar side against a full heap is a red, + and expect.rows refuses two empty sides (#418).""" + return _rows(c, tmpl.replace("%T", tbl)), _rows(c, tmpl.replace("%T", f"{tbl}_h")) + + +class _Gucs: + def __init__(self, c, settings): + self.c, self.settings = c, settings + + def __enter__(self): + for s in self.settings: + self.c.execute(f"SET {s}") + + def __exit__(self, *exc): + for s in self.settings: + self.c.execute(f"RESET {s.split('=')[0]}") + + +@pytest.fixture(scope="module") +def gbb(pgc_cluster): + """-> a connection to a database holding gbb and its heap mirror, the path on.""" + db = "pgc_gab" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + a.execute(f"ALTER DATABASE {db} SET pgcolumnar.enable_group_vectorization = on") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + c.execute(f"CREATE TABLE gbb({DDL}) USING pgcolumnar") + c.execute(f"INSERT INTO gbb {GEN}") + c.execute(f"CREATE TABLE gbb_h({DDL})") + c.execute(f"INSERT INTO gbb_h {GEN}") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_the_fold_takes_every_eligible_shape_and_answers_as_the_heap_does(gbb, expect): + c = gbb + expect.num(_one(c, "SELECT count(*) FROM gbb"), 200000, "premise: fixture loaded every row (columnar)") + expect.num(_one(c, "SELECT count(*) FROM gbb_h"), 200000, "premise: fixture loaded every row (heap)") + expect.num(_one(c, "SELECT count(*) FROM gbb WHERE k IS NULL"), + _one(c, "SELECT count(*) FROM gbb_h WHERE k IS NULL"), "premise: the fixture really has null keys") + expect.num(_one(c, "SELECT (count(*) > 1)::int FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('gbb'::regclass)"), 1, + "premise: the fixture spans more than one row group") + for label, tmpl in ( + ("plain", Q_PLAIN), + ("filtered", Q_FILT), + ("two keys", Q_TWOKEY), + ("float8 key", "SELECT f, count(*), sum(v) FROM %T GROUP BY f"), + ("count(*)", "SELECT k, count(*) FROM %T GROUP BY k"), + ("timestamp key", "SELECT ts, count(*), sum(v) FROM %T GROUP BY ts"), + ("date key", "SELECT d, count(*), sum(w) FROM %T GROUP BY d"), + ): + q = tmpl.replace("%T", "gbb") + expect.text(_groupvec(c, q), "yes", f"{label}: the grouped node is planned") + expect.text(_fold(c, ANALYZE, q), "yes", f"{label}: the fold ran (work done)") + col, heap = _agree(c, "gbb", tmpl) + expect.rows(col, heap, f"{label}: answers match the heap mirror") + + +def test_the_fold_declines_every_ineligible_shape_and_still_answers(gbb, expect): + c = gbb + for label, tmpl in ( + ("text key", "SELECT t, count(*), sum(v) FROM %T GROUP BY t"), + ("expression key", "SELECT k + 1, count(*) FROM %T GROUP BY k + 1"), + ("min/max", "SELECT k, min(v), max(v) FROM %T GROUP BY k"), + ("<> filter", "SELECT k, count(*), sum(v) FROM %T WHERE v <> 5 GROUP BY k"), + ): + q = tmpl.replace("%T", "gbb") + # the decline must be the fold's, not the node's absence + expect.text(_fold(c, ANALYZE, q) if _groupvec(c, q) == "yes" else "the grouped node is not planned", + "no", f"{label}: the fold declined") + col, heap = _agree(c, "gbb", tmpl) + expect.rows(col, heap, f"{label}: answers match the heap mirror") + # the #715 gate: `<>` builds no scan key, so a fold that took it would count excluded rows + expect.num(_one(c, "SELECT count(*) FROM gbb WHERE v <> 5"), _one(c, "SELECT count(*) FROM gbb_h WHERE v <> 5"), + "<> filter: the excluded rows really are excluded") + expect.num(_one(c, "SELECT (count(*) < 200000)::int FROM gbb_h WHERE v <> 5"), 1, + "premise: the <> filter excludes something at all") + + +def test_the_fold_honours_the_delete_mask(gbb, expect): + c = gbb + c.execute("DELETE FROM gbb WHERE v % 7 = 0") + c.execute("DELETE FROM gbb_h WHERE v % 7 = 0") + expect.num(_one(c, "SELECT (count(*) < 200000)::int FROM gbb"), 1, "premise: the delete removed rows") + expect.text(_fold(c, ANALYZE, Q_PLAIN.replace("%T", "gbb")), "yes", "deletes: the fold still ran") + col, heap = _agree(c, "gbb", Q_PLAIN) + expect.rows(col, heap, "deletes: answers match the heap mirror") + + +def test_the_fold_equals_the_row_path_over_the_same_rows(gbb, expect): + c = gbb + off = ("pgcolumnar.enable_group_vectorization=off",) + for label, tmpl in ( + ("toggle-differential: plain", Q_PLAIN), + ("toggle-differential: filtered", Q_FILT), + ("toggle-differential: two keys", Q_TWOKEY), + ): + q = tmpl.replace("%T", "gbb") + with _Gucs(c, off): + rows_off = _rows(c, q) + node_off = _groupvec(c, q) + expect.text(_groupvec(c, q), "yes", f"{label} [premise: node fires with the path on]") + rows_on = _rows(c, q) + expect.text(node_off, "no", f"{label} [premise: the off arm is not the grouped node]") + expect.rows(rows_on, rows_off, label) + + +def test_float8_keys_group_by_the_types_equality_not_by_bits(gbb, expect): + c = gbb + c.execute("CREATE TABLE gbz(f float8, v int) USING pgcolumnar") + c.execute("INSERT INTO gbz VALUES ('0'::float8, 1), ('-0'::float8, 2), ('0'::float8, 4), " + "('NaN'::float8, 8), ('NaN'::float8, 16)") + c.execute("CREATE TABLE gbz_h(f float8, v int)") + c.execute("INSERT INTO gbz_h SELECT * FROM gbz") + # '-0'::float8, not -0.0: an untyped -0.0 goes through numeric and arrives as +0.0 + expect.text(_one(c, "SELECT encode(float8send(f),'hex') FROM gbz WHERE v = 2"), "8000000000000000", + "float8 zero: the fixture really stores a NEGATIVE zero") + expect.text(_one(c, "SELECT encode(float8send(f),'hex') FROM gbz WHERE v = 1"), "0000000000000000", + "float8 zero: and a positive one beside it") + q = "SELECT f, sum(v) FROM gbz GROUP BY f" + expect.text(_groupvec(c, q), "yes", "float8 zero: the grouped node is planned") + expect.text(_fold(c, ANALYZE, q), "yes", "float8 zero: and the fold actually ran on it") + # the rows of the folding query itself: an outer count(*) would change the plan + expect.num(len(_rows(c, q)), len(_rows(c, "SELECT f, sum(v) FROM gbz_h GROUP BY f")), + "float8 zero: -0.0 and 0.0 are one group, exactly as the heap says") + expect.rows(_rows(c, q), _rows(c, "SELECT f, sum(v) FROM gbz_h GROUP BY f"), + "float8 zero: the grouped sums match the heap") + + +def test_the_fold_steps_over_skipped_vectors(gbb, expect): + c = gbb + c.execute("CREATE TABLE gbs(k int, sel int, p1 int, p2 int, p3 int, p4 int, p5 int) USING pgcolumnar") + c.execute("SELECT pgcolumnar.set_options('gbs', stripe_row_limit => 65536, chunk_group_row_limit => 1024)") + c.execute(f"INSERT INTO gbs {SGEN}") + c.execute("CREATE TABLE gbs_h(k int, sel int, p1 int, p2 int, p3 int, p4 int, p5 int)") + c.execute(f"INSERT INTO gbs_h {SGEN}") + plan = _text(c, f"{ANALYZE} {Q_SKIP.replace('%T', 'gbs')}") + m = re.search(r"Columnar Batch Fold: (yes|no)", plan) + expect.text(m.group(1) if m else "", "yes", "skipped vectors: the fold ran") + expect.at_least(_num(plan, "Columnar Vectors Skipped"), 1, "skipped vectors: premise: the scan really skipped vectors") + expect.at_least(_num(plan, "Columnar Chunk Groups Read"), 1, "skipped vectors: premise: a row group was still read") + col, heap = _agree(c, "gbs", Q_SKIP) + expect.rows(col, heap, "skipped vectors: answers match the heap mirror") + # four payload columns, so a cursor that misaligns on any one shows as a wrong sum + q4 = Q_SKIP4.replace("%T", "gbs") + expect.text(_groupvec(c, q4), "yes", "skipped vectors: premise: the wide arm is still the grouped node") + expect.text(_fold(c, ANALYZE, q4), "yes", "skipped vectors: the fold ran on the four-payload arm") + col, heap = _agree(c, "gbs", Q_SKIP4) + expect.rows(col, heap, "skipped vectors: and their sums match the heap mirror") + + +def test_the_parallel_partial_fold_equals_the_serial_one(gbb, expect): + c = gbb + with _Gucs(c, PAR): + pea = _text(c, f"{ANALYZE} {Q_PAR}") + rows_par = _rows(c, Q_PAR) + expect.text("yes" if re.search(r"Parallel Custom Scan \(PgColumnarScan\)", pea, re.I) and MARKER in pea else "no", + "yes", "parallel: our parallel-aware grouped node is under the Gather") + expect.text("yes" if re.search(r"Workers Launched: [1-9]", pea, re.I) and MARKER in pea else "no", + "yes", "parallel: workers actually launched for that node") + m = re.search(r"Columnar Batch Fold: (yes|no)", pea) + expect.text(m.group(1) if m else "", "yes", "parallel: the fold ran in the partial node (work done)") + with _Gucs(c, ("max_parallel_workers_per_gather=0",)): + rows_serial = _rows(c, Q_PAR) + expect.rows(rows_par, rows_serial, "parallel: the answers equal the serial fold, byte for byte") + expect.rows(rows_par, _rows(c, "SELECT k, count(*), sum(v) FROM gbb_h GROUP BY k"), + "parallel: and they equal the heap mirror") + # the value arm's own plan, read under the same settings: without workers it is serial against serial + with _Gucs(c, PAR): + own = _text(c, f"{ANALYZE} {Q_PAR}") + expect.text("yes" if re.search(r"Workers Launched: [1-9]", own, re.I) else "no", "yes", + "parallel: premise: the value arm's own plan launches workers") + + +def test_a_column_added_after_some_row_groups_runs_the_row_path(gbb, expect): + c = gbb + c.execute("CREATE TABLE gba(k int) USING pgcolumnar") + c.execute("INSERT INTO gba SELECT g % 8 FROM generate_series(1,50000) g") + c.execute("ALTER TABLE gba ADD COLUMN v int") + c.execute("INSERT INTO gba SELECT g % 8, g FROM generate_series(1,10000) g") + q = "SELECT k, count(*), sum(v) FROM gba GROUP BY k" + expect.text(_fold(c, PLAIN, q), "yes", "add column: plain EXPLAIN reports the prediction") + expect.text(_fold(c, ANALYZE, q), "no", "add column: ANALYZE reports the row-path run") + expect.num(_one(c, "SELECT sum(s) FROM (SELECT sum(v) AS s FROM gba GROUP BY k) x"), + _one(c, "SELECT sum(g) FROM generate_series(1,10000) g"), "add column: the answer is still right")