diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 39328e2cc..14178d65a 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2571,6 +2571,75 @@ native_groupagg native_groupagg toggle sum signed-zero (-0 preserved) [node fire native_groupagg native_groupagg while a matching-type predicate still gets one 15;16;17;18;19 never - native_groupagg native_groupagg with no time bound at all, a truncating key is bounded from the zone maps 15;16;17;18;19 never - native_groupagg native_groupagg with only a lower time bound, a truncating key is still bounded 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch <> filter: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the batch gates' exactness test dropped, so a <> filter's rows are folded +native_groupagg_batch native_groupagg_batch <> filter: the excluded rows really are excluded 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch <> filter: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: the batch gates' exactness test dropped, so a <> filter's rows are folded +native_groupagg_batch native_groupagg_batch add column: ANALYZE reports the row-path run 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped node's ANALYZE prints the predicted fold, not what ran (the #602 shape) +native_groupagg_batch native_groupagg_batch add column: plain EXPLAIN reports the prediction 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch add column: the answer is still right 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch count(*): answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch count(*): the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch count(*): the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch date key: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch date key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch date key: the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch deletes: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch deletes: the fold still ran 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch expression key: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: an expression key's -1 attidx is not refused by the grouped fold's eligibility +native_groupagg_batch native_groupagg_batch expression key: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: an expression key's -1 attidx is not refused by the grouped fold's eligibility +native_groupagg_batch native_groupagg_batch filtered: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row +native_groupagg_batch native_groupagg_batch filtered: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch filtered: the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch float8 key: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch float8 key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch float8 key: the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch float8 zero: -0.0 and 0.0 are one group, exactly as the heap says 15;16;17;18;19 2026-10-16 columnar_vector.c: float8 admitted as a simple key, so its group keys are hashed and compared by bits +native_groupagg_batch native_groupagg_batch float8 zero: and a positive one beside it 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch float8 zero: and the fold actually ran on it 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch float8 zero: the fixture really stores a NEGATIVE zero 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch float8 zero: the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch float8 zero: the grouped sums match the heap 15;16;17;18;19 2026-10-16 columnar_vector.c: float8 admitted as a simple key, so its group keys are hashed and compared by bits +native_groupagg_batch native_groupagg_batch min/max: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch min/max: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: min and max admitted as batch fold kinds +native_groupagg_batch native_groupagg_batch parallel: and they equal the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter;columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch parallel: our parallel-aware grouped node is under the Gather 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch parallel: premise: the value arm's own plan launches workers 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch parallel: the answers equal the serial fold, byte for byte 15;16;17;18;19 2026-10-16 columnar_vector.c: a partial grouped node folds without the shared row-group counter +native_groupagg_batch native_groupagg_batch parallel: the fold ran in the partial node (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch parallel: workers actually launched for that node 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch plain: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch plain: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch plain: the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch premise: fixture loaded every row (columnar) 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch premise: fixture loaded every row (heap) 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch premise: the <> filter excludes something at all 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch premise: the delete removed rows 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch premise: the fixture really has null keys 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch premise: the fixture spans more than one row group 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch skipped vectors: and their sums match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a skipped vector's present slots are not consumed, so later vectors misalign;columnar_vector.c: the grouped fold applies no scan key per row +native_groupagg_batch native_groupagg_batch skipped vectors: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: a skipped vector's present slots are not consumed, so later vectors misalign;columnar_vector.c: the grouped fold applies no scan key per row +native_groupagg_batch native_groupagg_batch skipped vectors: premise: a row group was still read 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch skipped vectors: premise: the scan really skipped vectors 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch skipped vectors: premise: the wide arm is still the grouped node 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch skipped vectors: the fold ran 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch skipped vectors: the fold ran on the four-payload arm 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch text key: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the batch gates' by-value test dropped, so a text key reaches the gather +native_groupagg_batch native_groupagg_batch text key: the fold declined 15;16;17;18;19 2026-10-16 columnar_vector.c: the batch gates' by-value test dropped, so a text key reaches the gather;columnar_vector.c: the grouped path is not planned for a text group key +native_groupagg_batch native_groupagg_batch timestamp key: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch timestamp key: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch timestamp key: the grouped node is planned 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch toggle-differential: filtered 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold applies no scan key per row;columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: filtered [premise: node fires with the path on] 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch toggle-differential: filtered [premise: the off arm is not the grouped node] 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch toggle-differential: plain 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: plain [premise: node fires with the path on] 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch toggle-differential: plain [premise: the off arm is not the grouped node] 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch toggle-differential: two keys 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold ignores the delete mask +native_groupagg_batch native_groupagg_batch toggle-differential: two keys [premise: node fires with the path on] 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch toggle-differential: two keys [premise: the off arm is not the grouped node] 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch two keys: answers match the heap mirror 15;16;17;18;19 never - +native_groupagg_batch native_groupagg_batch two keys: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) +native_groupagg_batch native_groupagg_batch two keys: the grouped node is planned 15;16;17;18;19 never - native_index_projection native_index_projection a parallel build projects one column of twenty, not all twenty 15;16;17;18;19 never - native_index_projection native_index_projection a parallel expression build projects its expression's columns 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index build's projection omits ii_Expressions' columns native_index_projection native_index_projection a parallel partial build projects the predicate's columns too 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index build's projection omits ii_Predicate's columns diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 83a2a80bb..3b955b160 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 201 +suites_not_covered 200 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3285,4 +3285,22 @@ suites_not_covered 201 # never 2941 + 1 = 2942 # dated 1160 + 0 = 1160 # 2942 + 1160 == 4102 -checks_never_observed_red 2942 +# +# #1341, native_groupagg_batch seeded, with its twin (OffgridwithJD): 2942 -> 2980, +# and suites_not_covered 201 -> 200. The suite had no rows. It enters with all 69 +# checks from five-major runs (69/69 on each of 15-19). 31 are dated by twelve +# mutations of the grouped fold: the fold never called, float8 keys compared by +# bits, the delete mask ignored, a skipped vector's slots not consumed, no scan key +# applied, the by-value, exactness and expression-key gates dropped, min and max +# admitted, the partial node without its shared counter, ANALYZE printing the +# prediction, and the grouped path refusing a text key. Thirty-eight arrive never: +# sixteen premises and the two float8 fixture facts; the eight "grouped node is +# planned" arms and the two parallel plan arms, which no fold mutation reaches; +# seven heap-mirror oracles that none of these mutations moves; the <> count, read +# by an ungrouped count(*); and the add-column prediction and answer. +# +# ledger rows 4102 + 69 = 4171 +# never 2942 + 38 = 2980 +# dated 1160 + 31 = 1191 +# 2980 + 1191 == 4171 +checks_never_observed_red 2980 diff --git a/test/native_groupagg_batch.sh b/test/native_groupagg_batch.sh index dfa2ebe91..5bde7ce08 100755 --- a/test/native_groupagg_batch.sh +++ b/test/native_groupagg_batch.sh @@ -170,13 +170,16 @@ for pair in "text key:$Q_TEXTKEY" "expression key:$Q_EXPRKEY" \ "min/max:$Q_MINMAX" "<> filter:$Q_NEQ"; do label="${pair%%:*}"; tmpl="${pair#*:}" qcol="${tmpl//%T/gbb}" + # The decline must be the FOLD's, read off the grouped node. This branch was + # a check_skip, and it never fired on 15-19: with the node gone the gate + # under test is unreachable, so a skip there was a silent pass. A shape the + # node stops taking is now a red under the arm's own name. if [ "$(is_groupvec "$qcol")" = yes ]; then check_text "$label: the fold declined" \ "$(fold_of "$ANALYZE_PFX" "$qcol")" no else - # The node itself declined the shape; there is no fold line to read and - # nothing for this suite to gate. Say so rather than assert a missing line. - check_skip "$label" "SKIP $label: the grouped node is not planned for this shape" "the grouped node is not planned for this shape" + check_text "$label: the fold declined" \ + "the grouped node is not planned" no fi agree "$label: answers match the heap mirror" "$tmpl" done @@ -315,7 +318,7 @@ agree_in gbs "skipped vectors: answers match the heap mirror" "$Q_SKIP" Q_SKIP4="SELECT k, count(*), sum(p1), sum(p2), sum(p3), sum(p4) FROM %T WHERE sel = 1 GROUP BY k" check_text "skipped vectors: premise: the wide arm is still the grouped node" \ "$(is_groupvec "${Q_SKIP4//%T/gbs}")" yes -check_text "skipped vectors: four payload columns all stay in step" \ +check_text "skipped vectors: the fold ran on the four-payload arm" \ "$(fold_of "$ANALYZE_PFX" "${Q_SKIP4//%T/gbs}")" yes agree_in gbs "skipped vectors: and their sums match the heap mirror" "$Q_SKIP4" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index a0f21f926..6583b1fbc 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -210,6 +210,7 @@ behaviour, the source of that number is named. - [162. test_native_vecskip.py: per-vector skipping by the zone maps](#162-test_native_vecskippy-per-vector-skipping-by-the-zone-maps) - [163. test_native_index_projection.py: CREATE INDEX projects only what it needs](#163-test_native_index_projectionpy-create-index-projects-only-what-it-needs) - [164. test_native_saop_pushdown.py: IN-lists and = ANY(array) prune row groups](#164-test_native_saop_pushdownpy-in-lists-and--anyarray-prune-row-groups) +- [165. test_native_groupagg_batch.py: the grouped aggregate folds column-at-a-time](#165-test_native_groupagg_batchpy-the-grouped-aggregate-folds-column-at-a-time) ## 1. How to read a test in here @@ -7659,3 +7660,20 @@ The pytest twin of `test/native_saop_pushdown.sh` (ported for #1341), every chec | `test_the_batch_fold_never_takes_the_set_keys_as_exact` | the batch fold refuses the IN-list but accepts the equivalent range, and the vectorized count stays exact | | `test_a_correlated_array_is_not_frozen_and_the_backend_lives` | a correlated LATERAL array is not frozen, and the session's backend is still alive; a crash is a named red | | `test_the_reader_refuses_every_special_flag_but_the_array_one` | the reader's reject expression names every special scan-key flag, and SK_SEARCHARRAY reaches its own arm | + +## 165. test_native_groupagg_batch.py: the grouped aggregate folds column-at-a-time + +The pytest twin of `test/native_groupagg_batch.sh` (ported for #1341), every check by the shell's name. The grouped vectorized aggregate folds a whole row group at a time (#708), and two things are proved separately. The answers must be unchanged: each shape agrees with a heap mirror, and with the same rows when the path is off. The work must happen: "Columnar Batch Fold: yes" under EXPLAIN ANALYZE is the only check that goes red when the fold stops running. Each shape the fold must decline is read off the grouped node, so a shape the node stops taking is a red, not a skip. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_fold_takes_every_eligible_shape_and_answers_as_the_heap_does` | seven shapes (plain, filtered, two keys, float8, count(*), timestamp and date keys) plan the grouped node, fold, and match the heap | +| `test_the_fold_declines_every_ineligible_shape_and_still_answers` | a text key, an expression key, min/max and a `<>` filter make the fold decline on the grouped node, and each still matches the heap | +| `test_the_fold_honours_the_delete_mask` | after deletes the fold still runs and matches the heap | +| `test_the_fold_equals_the_row_path_over_the_same_rows` | three shapes answer the same with the path off and on, and the off arm really is not the grouped node | +| `test_float8_keys_group_by_the_types_equality_not_by_bits` | a stored -0.0 and 0.0 are one group on the fold, as on the heap | +| `test_the_fold_steps_over_skipped_vectors` | a scan that really skips vectors folds, and its one-column and four-column sums match the heap | +| `test_the_parallel_partial_fold_equals_the_serial_one` | our parallel-aware node launches workers and folds, and equals the serial fold and the heap | +| `test_a_column_added_after_some_row_groups_runs_the_row_path` | plain EXPLAIN predicts the fold, ANALYZE reports the row path that ran, and the answer is right | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 58ea62469..c5406af1d 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -791,4 +791,6 @@ guard_tests 438 # four tests, by collection over the cluster files on this tree. # cluster_tests 702 -> 710 for #1341's test_native_saop_pushdown.py, the twin of native_saop_pushdown.sh: # eight tests, by collection over the cluster files on this tree. -cluster_tests 710 +# cluster_tests 710 -> 718 for #1341's test_native_groupagg_batch.py, the twin of native_groupagg_batch.sh: +# eight tests, by collection over the cluster files on this tree. +cluster_tests 718 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index a4276febb..74e1befa7 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -131,6 +131,7 @@ "native_encdesc_golden", "native_fetch_coalesce", "native_groupagg", + "native_groupagg_batch", "native_index_fetch_stripe_cost", "native_index_projection", "native_join_vector_agg", @@ -410,7 +411,7 @@ def test_the_loop_reader_invents_nothing_in_this_corpus(expect): "every name the loop reader returns appears verbatim in the file it " "came from") expect.text(", ".join(files), - "build_refusal, differential, join_runtime_filter, " + "build_refusal, differential, join_runtime_filter, native_groupagg_batch, " "native_index_projection, native_reclaim, native_repack, " "native_saop_pushdown, native_vacuum_race, native_vecskip, sorted_pathkeys", "and it is these files, so another appearing is a diff a reviewer " diff --git a/test/pytest/test_native_groupagg_batch.py b/test/pytest/test_native_groupagg_batch.py new file mode 100644 index 000000000..182e7884c --- /dev/null +++ b/test/pytest/test_native_groupagg_batch.py @@ -0,0 +1,272 @@ +"""The grouped vectorized aggregate folds column-at-a-time (#708, #1341). The pytest twin +of `test/native_groupagg_batch.sh`, every check by the shell's name. + +Two things are proved separately. The ANSWERS: every shape agrees with a heap mirror, +and with the same rows on the path off. The WORK: "Columnar Batch Fold: yes" on the +grouped node under EXPLAIN ANALYZE, the only assertion that goes red when the fold +stops running. Eligibility both ways: a by-reference key, an expression key, min/max +and a `<>` filter each make the fold decline, and each still answers right. Then the +delete mask, -0.0 and 0.0 as one float8 group, a scan that really skips vectors, the +parallel partial fold, and a column added after some row groups (predicted yes, ran +no). +""" +import re + +import psycopg +import pytest + +ANALYZE = "EXPLAIN (ANALYZE, TIMING OFF, COSTS OFF, SUMMARY OFF)" +PLAIN = "EXPLAIN (COSTS OFF)" +MARKER = "Columnar Vectorized Group Keys" +DDL = "k int, k8 bigint, f float8, t text, ts timestamp, d date, v int, w int" +GEN = """SELECT CASE WHEN g % 997 = 0 THEN NULL ELSE g % 8 END, + (g % 5)::bigint, + (g % 4)::float8, + 't' || (g % 6), + timestamp '2024-01-01 00:00:00' + ((g % 7) * interval '1 day'), + date '2024-03-01' + (g % 9), + g, + g % 100 + FROM generate_series(1,200000) g""" +SGEN = """SELECT g % 8, CASE WHEN (g / 1024) % 40 = 0 THEN 1 ELSE 0 END, + g, g % 3, g % 5, g % 7, g % 11 + FROM generate_series(1,200000) g""" +PAR = ("max_parallel_workers_per_gather=4", "parallel_setup_cost=0", "parallel_tuple_cost=0", + "min_parallel_table_scan_size=0", "min_parallel_index_scan_size=0", + "pgcolumnar.enable_parallel_vector_agg=on") +Q_PLAIN = "SELECT k, count(*), sum(v) FROM %T GROUP BY k" +Q_FILT = "SELECT k, count(*), sum(v) FROM %T WHERE v < 50000 GROUP BY k" +Q_TWOKEY = "SELECT k, k8, count(*), sum(w) FROM %T GROUP BY k, k8" +Q_SKIP = "SELECT k, count(*), sum(p1) FROM %T WHERE sel = 1 GROUP BY k" +Q_SKIP4 = "SELECT k, count(*), sum(p1), sum(p2), sum(p3), sum(p4) FROM %T WHERE sel = 1 GROUP BY k" +Q_PAR = "SELECT k, count(*), sum(v) FROM gbb GROUP BY k" + + +def _rows(c, sql): + """-> the rows as sorted text tuples, NULL as "\\0" so it never equals ''. A query that + errors returns its error as the one row, so the comparison reddens by name.""" + try: + return sorted(tuple("\0" if v is None else str(v) for v in r) for r in c.execute(sql)) + except psycopg.Error as exc: + return [(f"error: {exc.diag.message_primary}",)] + + +def _one(c, sql): + return c.execute(sql).fetchone()[0] + + +def _text(c, sql): + """-> the rows' first column joined, or the error: a plan that errors has no fold line.""" + try: + return "\n".join(str(r[0]) for r in c.execute(sql)) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _groupvec(c, sql): + return "yes" if MARKER in _text(c, f"{PLAIN} {sql}") else "no" + + +def _fold(c, pfx, sql): + """-> the grouped node's "Columnar Batch Fold" value, '' when the plan has none.""" + m = re.search(r"Columnar Batch Fold: (yes|no)", _text(c, f"{pfx} {sql}")) + return m.group(1) if m else "" + + +def _num(plan, label): + m = re.search(rf"{re.escape(label)}: (\d+)", plan) + return int(m.group(1)) if m else 0 + + +def _agree(c, tbl, tmpl): + """-> (columnar rows, heap rows). An empty columnar side against a full heap is a red, + and expect.rows refuses two empty sides (#418).""" + return _rows(c, tmpl.replace("%T", tbl)), _rows(c, tmpl.replace("%T", f"{tbl}_h")) + + +class _Gucs: + def __init__(self, c, settings): + self.c, self.settings = c, settings + + def __enter__(self): + for s in self.settings: + self.c.execute(f"SET {s}") + + def __exit__(self, *exc): + for s in self.settings: + self.c.execute(f"RESET {s.split('=')[0]}") + + +@pytest.fixture(scope="module") +def gbb(pgc_cluster): + """-> a connection to a database holding gbb and its heap mirror, the path on.""" + db = "pgc_gab" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + a.execute(f"ALTER DATABASE {db} SET pgcolumnar.enable_group_vectorization = on") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + c.execute(f"CREATE TABLE gbb({DDL}) USING pgcolumnar") + c.execute(f"INSERT INTO gbb {GEN}") + c.execute(f"CREATE TABLE gbb_h({DDL})") + c.execute(f"INSERT INTO gbb_h {GEN}") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_the_fold_takes_every_eligible_shape_and_answers_as_the_heap_does(gbb, expect): + c = gbb + expect.num(_one(c, "SELECT count(*) FROM gbb"), 200000, "premise: fixture loaded every row (columnar)") + expect.num(_one(c, "SELECT count(*) FROM gbb_h"), 200000, "premise: fixture loaded every row (heap)") + expect.num(_one(c, "SELECT count(*) FROM gbb WHERE k IS NULL"), + _one(c, "SELECT count(*) FROM gbb_h WHERE k IS NULL"), "premise: the fixture really has null keys") + expect.num(_one(c, "SELECT (count(*) > 1)::int FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('gbb'::regclass)"), 1, + "premise: the fixture spans more than one row group") + for label, tmpl in ( + ("plain", Q_PLAIN), + ("filtered", Q_FILT), + ("two keys", Q_TWOKEY), + ("float8 key", "SELECT f, count(*), sum(v) FROM %T GROUP BY f"), + ("count(*)", "SELECT k, count(*) FROM %T GROUP BY k"), + ("timestamp key", "SELECT ts, count(*), sum(v) FROM %T GROUP BY ts"), + ("date key", "SELECT d, count(*), sum(w) FROM %T GROUP BY d"), + ): + q = tmpl.replace("%T", "gbb") + expect.text(_groupvec(c, q), "yes", f"{label}: the grouped node is planned") + expect.text(_fold(c, ANALYZE, q), "yes", f"{label}: the fold ran (work done)") + col, heap = _agree(c, "gbb", tmpl) + expect.rows(col, heap, f"{label}: answers match the heap mirror") + + +def test_the_fold_declines_every_ineligible_shape_and_still_answers(gbb, expect): + c = gbb + for label, tmpl in ( + ("text key", "SELECT t, count(*), sum(v) FROM %T GROUP BY t"), + ("expression key", "SELECT k + 1, count(*) FROM %T GROUP BY k + 1"), + ("min/max", "SELECT k, min(v), max(v) FROM %T GROUP BY k"), + ("<> filter", "SELECT k, count(*), sum(v) FROM %T WHERE v <> 5 GROUP BY k"), + ): + q = tmpl.replace("%T", "gbb") + # the decline must be the fold's, not the node's absence + expect.text(_fold(c, ANALYZE, q) if _groupvec(c, q) == "yes" else "the grouped node is not planned", + "no", f"{label}: the fold declined") + col, heap = _agree(c, "gbb", tmpl) + expect.rows(col, heap, f"{label}: answers match the heap mirror") + # the #715 gate: `<>` builds no scan key, so a fold that took it would count excluded rows + expect.num(_one(c, "SELECT count(*) FROM gbb WHERE v <> 5"), _one(c, "SELECT count(*) FROM gbb_h WHERE v <> 5"), + "<> filter: the excluded rows really are excluded") + expect.num(_one(c, "SELECT (count(*) < 200000)::int FROM gbb_h WHERE v <> 5"), 1, + "premise: the <> filter excludes something at all") + + +def test_the_fold_honours_the_delete_mask(gbb, expect): + c = gbb + c.execute("DELETE FROM gbb WHERE v % 7 = 0") + c.execute("DELETE FROM gbb_h WHERE v % 7 = 0") + expect.num(_one(c, "SELECT (count(*) < 200000)::int FROM gbb"), 1, "premise: the delete removed rows") + expect.text(_fold(c, ANALYZE, Q_PLAIN.replace("%T", "gbb")), "yes", "deletes: the fold still ran") + col, heap = _agree(c, "gbb", Q_PLAIN) + expect.rows(col, heap, "deletes: answers match the heap mirror") + + +def test_the_fold_equals_the_row_path_over_the_same_rows(gbb, expect): + c = gbb + off = ("pgcolumnar.enable_group_vectorization=off",) + for label, tmpl in ( + ("toggle-differential: plain", Q_PLAIN), + ("toggle-differential: filtered", Q_FILT), + ("toggle-differential: two keys", Q_TWOKEY), + ): + q = tmpl.replace("%T", "gbb") + with _Gucs(c, off): + rows_off = _rows(c, q) + node_off = _groupvec(c, q) + expect.text(_groupvec(c, q), "yes", f"{label} [premise: node fires with the path on]") + rows_on = _rows(c, q) + expect.text(node_off, "no", f"{label} [premise: the off arm is not the grouped node]") + expect.rows(rows_on, rows_off, label) + + +def test_float8_keys_group_by_the_types_equality_not_by_bits(gbb, expect): + c = gbb + c.execute("CREATE TABLE gbz(f float8, v int) USING pgcolumnar") + c.execute("INSERT INTO gbz VALUES ('0'::float8, 1), ('-0'::float8, 2), ('0'::float8, 4), " + "('NaN'::float8, 8), ('NaN'::float8, 16)") + c.execute("CREATE TABLE gbz_h(f float8, v int)") + c.execute("INSERT INTO gbz_h SELECT * FROM gbz") + # '-0'::float8, not -0.0: an untyped -0.0 goes through numeric and arrives as +0.0 + expect.text(_one(c, "SELECT encode(float8send(f),'hex') FROM gbz WHERE v = 2"), "8000000000000000", + "float8 zero: the fixture really stores a NEGATIVE zero") + expect.text(_one(c, "SELECT encode(float8send(f),'hex') FROM gbz WHERE v = 1"), "0000000000000000", + "float8 zero: and a positive one beside it") + q = "SELECT f, sum(v) FROM gbz GROUP BY f" + expect.text(_groupvec(c, q), "yes", "float8 zero: the grouped node is planned") + expect.text(_fold(c, ANALYZE, q), "yes", "float8 zero: and the fold actually ran on it") + # the rows of the folding query itself: an outer count(*) would change the plan + expect.num(len(_rows(c, q)), len(_rows(c, "SELECT f, sum(v) FROM gbz_h GROUP BY f")), + "float8 zero: -0.0 and 0.0 are one group, exactly as the heap says") + expect.rows(_rows(c, q), _rows(c, "SELECT f, sum(v) FROM gbz_h GROUP BY f"), + "float8 zero: the grouped sums match the heap") + + +def test_the_fold_steps_over_skipped_vectors(gbb, expect): + c = gbb + c.execute("CREATE TABLE gbs(k int, sel int, p1 int, p2 int, p3 int, p4 int, p5 int) USING pgcolumnar") + c.execute("SELECT pgcolumnar.set_options('gbs', stripe_row_limit => 65536, chunk_group_row_limit => 1024)") + c.execute(f"INSERT INTO gbs {SGEN}") + c.execute("CREATE TABLE gbs_h(k int, sel int, p1 int, p2 int, p3 int, p4 int, p5 int)") + c.execute(f"INSERT INTO gbs_h {SGEN}") + plan = _text(c, f"{ANALYZE} {Q_SKIP.replace('%T', 'gbs')}") + m = re.search(r"Columnar Batch Fold: (yes|no)", plan) + expect.text(m.group(1) if m else "", "yes", "skipped vectors: the fold ran") + expect.at_least(_num(plan, "Columnar Vectors Skipped"), 1, "skipped vectors: premise: the scan really skipped vectors") + expect.at_least(_num(plan, "Columnar Chunk Groups Read"), 1, "skipped vectors: premise: a row group was still read") + col, heap = _agree(c, "gbs", Q_SKIP) + expect.rows(col, heap, "skipped vectors: answers match the heap mirror") + # four payload columns, so a cursor that misaligns on any one shows as a wrong sum + q4 = Q_SKIP4.replace("%T", "gbs") + expect.text(_groupvec(c, q4), "yes", "skipped vectors: premise: the wide arm is still the grouped node") + expect.text(_fold(c, ANALYZE, q4), "yes", "skipped vectors: the fold ran on the four-payload arm") + col, heap = _agree(c, "gbs", Q_SKIP4) + expect.rows(col, heap, "skipped vectors: and their sums match the heap mirror") + + +def test_the_parallel_partial_fold_equals_the_serial_one(gbb, expect): + c = gbb + with _Gucs(c, PAR): + pea = _text(c, f"{ANALYZE} {Q_PAR}") + rows_par = _rows(c, Q_PAR) + expect.text("yes" if re.search(r"Parallel Custom Scan \(PgColumnarScan\)", pea, re.I) and MARKER in pea else "no", + "yes", "parallel: our parallel-aware grouped node is under the Gather") + expect.text("yes" if re.search(r"Workers Launched: [1-9]", pea, re.I) and MARKER in pea else "no", + "yes", "parallel: workers actually launched for that node") + m = re.search(r"Columnar Batch Fold: (yes|no)", pea) + expect.text(m.group(1) if m else "", "yes", "parallel: the fold ran in the partial node (work done)") + with _Gucs(c, ("max_parallel_workers_per_gather=0",)): + rows_serial = _rows(c, Q_PAR) + expect.rows(rows_par, rows_serial, "parallel: the answers equal the serial fold, byte for byte") + expect.rows(rows_par, _rows(c, "SELECT k, count(*), sum(v) FROM gbb_h GROUP BY k"), + "parallel: and they equal the heap mirror") + # the value arm's own plan, read under the same settings: without workers it is serial against serial + with _Gucs(c, PAR): + own = _text(c, f"{ANALYZE} {Q_PAR}") + expect.text("yes" if re.search(r"Workers Launched: [1-9]", own, re.I) else "no", "yes", + "parallel: premise: the value arm's own plan launches workers") + + +def test_a_column_added_after_some_row_groups_runs_the_row_path(gbb, expect): + c = gbb + c.execute("CREATE TABLE gba(k int) USING pgcolumnar") + c.execute("INSERT INTO gba SELECT g % 8 FROM generate_series(1,50000) g") + c.execute("ALTER TABLE gba ADD COLUMN v int") + c.execute("INSERT INTO gba SELECT g % 8, g FROM generate_series(1,10000) g") + q = "SELECT k, count(*), sum(v) FROM gba GROUP BY k" + expect.text(_fold(c, PLAIN, q), "yes", "add column: plain EXPLAIN reports the prediction") + expect.text(_fold(c, ANALYZE, q), "no", "add column: ANALYZE reports the row-path run") + expect.num(_one(c, "SELECT sum(s) FROM (SELECT sum(v) AS s FROM gba GROUP BY k) x"), + _one(c, "SELECT sum(g) FROM generate_series(1,10000) g"), "add column: the answer is still right")