test: native_groupagg_wide_cost gains its pytest twin, and reaches the fallback it is named for (#1341)

test/check_ledger.tsv

@@ -2767,6 +2767,11 @@ native_groupagg_batch	native_groupagg_batch	toggle-differential: two keys [premi
 native_groupagg_batch	native_groupagg_batch	two keys: answers match the heap mirror	15;16;17;18;19	2026-10-16	columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review)
 native_groupagg_batch	native_groupagg_batch	two keys: the fold ran (work done)	15;16;17;18;19	2026-10-16	columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof)
 native_groupagg_batch	native_groupagg_batch	two keys: the grouped node is planned	15;16;17;18;19	never	-
+native_groupagg_wide_cost	native_groupagg_wide_cost	backend alive	15;16;17;18;19	never	-
+native_groupagg_wide_cost	native_groupagg_wide_cost	premise: with parallelism free, a Gather displaces the serial scan	15;16;17;18;19	never	-
+native_groupagg_wide_cost	native_groupagg_wide_cost	the fallback prices the fold as a surviving serial scan does	15;16;17;18;19	2026-10-17	columnar_vector.c: the grouped aggregate's no-serial-survivor fallback prices its input with the bare seqscan formula;columnar_vector.c: the grouped vector aggregate path is never planned
+native_groupagg_wide_cost	native_groupagg_wide_cost	the fold path is used on the wide grouped aggregate	15;16;17;18;19	2026-10-17	columnar_vector.c: the grouped vector aggregate path is never planned
+native_groupagg_wide_cost	native_groupagg_wide_cost	the fold result matches a plain aggregate on a wide table	15;16;17;18;19	2026-10-17	columnar_vector.c: the grouped batch fold loses the first row of each row group
 native_index	native_index	duplicate id rejected	15;16;17;18;19	2026-10-16	columnar_tableam.c: pgcolumnar_index_fetch_tuple fetches the neighbouring row number;columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row
 native_index	native_index	index IN lookup	15;16;17;18;19	2026-10-16	columnar_tableam.c: pgcolumnar_index_fetch_tuple fetches the neighbouring row number;columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row
 native_index	native_index	index lookup miss returns nothing	15;16;17;18;19	never	-

test/check_ledger_budget.txt

@@ -15,7 +15,7 @@
 #
 #   Adding a check to a suite that is already covered does not move it, which is
 #   what makes it safe to bound.
-suites_not_covered 183
+suites_not_covered 182
 #
 # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become
 #   one.
@@ -3725,4 +3725,25 @@ suites_not_covered 183
 #     never         3081 +  4 = 3085
 #     dated         1387 + 15 = 1402
 #     3085 + 1402 == 4487
-checks_never_observed_red 3085
+#
+# #1341, native_groupagg_wide_cost seeded, with its twin (OffgridwithJD): 3085 -> 3087,
+# and suites_not_covered 183 -> 182. The suite had no rows. It enters with 5 checks
+# from five-major runs (5/5 on each of 15-19), after two fixes and two new arms. The
+# fallback it is named for never ran: with 40,000 rows a serial scan survives
+# add_path, so the fold was priced from it (traced), and the old suite is 3/3 with
+# the fallback pricing by the bare seqscan formula. Free parallelism now lets a Gather
+# displace the serial scan, hash and sort off keep the fold node the plan, and its
+# cost must equal the cost a surviving serial scan gives. The readings kept the last
+# line of "SET ...; SELECT", so a failed SELECT read "SET" on both sides and agreed
+# (the old suite is 3/3 with the query naming a missing column); the settings now
+# ride in PGOPTIONS and check_text refuses an empty side. The plan check matched any
+# custom scan, which the reference plan's input is too; it now reads the fold's own
+# line. 3 are dated by three mutations: the fallback pricing by the bare formula, the
+# grouped path never planned, and the batch fold losing a row. Two arrive never: the
+# Gather premise and "backend alive".
+#
+#     ledger rows   4487 + 5 = 4492
+#     never         3085 + 2 = 3087
+#     dated         1402 + 3 = 1405
+#     3087 + 1405 == 4492
+checks_never_observed_red 3087

test/native_groupagg_wide_cost.sh

@@ -3,8 +3,9 @@
 # aggregate's no-serial-survivor fallback must price its input scan the same way
 # the columnar scan node does (projected width, per-column decode CPU, zone-map
 # survival), not with the bare seqscan formula that under-priced it on a wide
-# table. This asserts the fold path is chosen and, decisively, that it returns
-# the same aggregate a plain (non-fold) plan does on a WIDE table.
+# table. This asserts the fold path is chosen, that it returns the same aggregate
+# a plain (non-fold) plan does on a WIDE table, and that the fallback prices the
+# fold node exactly as a surviving serial scan does.
 set -uo pipefail
 . "$(dirname "${BASH_SOURCE[0]}")/lib.sh"
 pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
@@ -17,27 +18,60 @@ VALS="g%16$(for i in $(seq 0 24); do printf ', g*%d' $((i + 1)); done)"
 q "INSERT INTO wa SELECT $VALS FROM generate_series(1, 40000) g" >/dev/null
 q "ANALYZE wa" >/dev/null
 
+# Each reading's settings ride in PGOPTIONS rather than a SET, so a failed query
+# prints nothing and check_text refuses it. The earlier form kept the last line of
+# "SET ...; SELECT ...", which for a failed SELECT was the SET's own tag: both
+# sides read "SET" and agreed.
+qn() {	# SQL [PGOPTIONS]
+	env PATH="$PGC_BINDIR:$PATH" PGOPTIONS="${2:-}" psql -h 127.0.0.1 -p "$PGC_PORT" \
+		-U postgres -d "$PGC_DB" -At -c "$1" 2>/dev/null || true
+}
+# the fold node's total cost, or empty when the plan is not the fold node
+fold_cost() {	# PGOPTIONS
+	qn "EXPLAIN SELECT g, sum(c0) FROM wa GROUP BY g;" "$1" \
+		| awk 'NR == 1 && /^Custom Scan \(PgColumnarScan\)/ { f = 1
+			if (match($0, /cost=[0-9.]+\.\.[0-9.]+/)) {
+				c = substr($0, RSTART + 5, RLENGTH - 5); sub(/^[0-9.]+\.\./, "", c) } }
+			f && /Columnar Batch Fold: yes/ { print c; exit }'
+}
+
 # The whole grouped result reduced to one comparable string, computed in SQL so
 # there is no shell ordering or row-count artifact.
 Q="SELECT string_agg(g||':'||s0||':'||s24||':'||c, '|' ORDER BY g)
    FROM (SELECT g, sum(c0) s0, sum(c24) s24, count(*) c FROM wa GROUP BY g) r"
 EXPLAINQ="SELECT g, sum(c0) FROM wa GROUP BY g"
 
 # reference: no vectorized grouping
-ref="$(q "SET pgcolumnar.enable_group_vectorization=off; SET pgcolumnar.enable_ungrouped_vector_agg=off; $Q" | tail -1)"
+ref="$(qn "$Q" "-c pgcolumnar.enable_group_vectorization=off -c pgcolumnar.enable_ungrouped_vector_agg=off")"
 
-# fold path: parallel ungrouped vector agg on (the shape that drops the serial
-# scan path and takes the fallback this fix corrects)
-FOLD="SET pgcolumnar.enable_ungrouped_vector_agg=on; SET pgcolumnar.enable_parallel_vector_agg=on; SET max_parallel_workers_per_gather=4;"
-got="$(q "$FOLD $Q" | tail -1)"
+FOLD="-c pgcolumnar.enable_ungrouped_vector_agg=on -c pgcolumnar.enable_parallel_vector_agg=on -c max_parallel_workers_per_gather=4"
+got="$(qn "$Q" "$FOLD")"
 
-plan="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -qtA \
-	-c "$FOLD" -c "EXPLAIN (COSTS OFF, VERBOSE) $EXPLAINQ" 2>/dev/null)"
-foldcnt="$(printf '%s' "$plan" | grep -ciE 'Batch Fold|Custom Scan')"
+# the fold node by its own line, not any custom scan: the reference plan's input
+# is a Custom Scan too
+plan="$(qn "EXPLAIN (COSTS OFF, VERBOSE) $EXPLAINQ" "$FOLD")"
+foldcnt="$(printf '%s' "$plan" | grep -c 'Columnar Batch Fold: yes')"
 echo "-- fold path markers: $foldcnt"
 
 check "the fold path is used on the wide grouped aggregate" \
 	"$([ "$foldcnt" -gt 0 ] && echo yes)" "yes"
-check "the fold result matches a plain aggregate on a wide table" "$got" "$ref"
+check_text "the fold result matches a plain aggregate on a wide table" "$got" "$ref"
+
+# --- the fallback this suite is named for ---------------------------------------
+#
+# With 40,000 rows a serial scan survives add_path, so the readings above price the
+# fold from it and the fallback never runs (traced: a bare-seqscan fallback leaves
+# every reading above unchanged). Free parallelism lets a Gather displace the
+# serial scan, which is when the fallback prices the scan itself; hash and sort off
+# keep the parallel finalize from winning, so the fold node is the plan and its
+# cost can be read. Measured on PG17: 800.19 both ways, and 503.01 through the
+# fallback with the bare seqscan formula.
+FREE="-c parallel_setup_cost=0 -c parallel_tuple_cost=0 -c min_parallel_table_scan_size=0 -c max_parallel_workers_per_gather=4"
+FREEPLAN="$(qn "EXPLAIN (COSTS OFF) $EXPLAINQ" "$FREE")"
+case "$FREEPLAN" in *Gather*) FREEGATHER=yes ;; *) FREEGATHER="no Gather in: $FREEPLAN" ;; esac
+check "premise: with parallelism free, a Gather displaces the serial scan" "$FREEGATHER" "yes"
+check_text "the fallback prices the fold as a surviving serial scan does" \
+	"$(fold_cost "$FREE -c enable_hashagg=off -c enable_sort=off")" "$(fold_cost "")"
+
 check "backend alive" "$(q 'SELECT 1')" "1"
 pgc_summary

test/pytest/TESTS.md

@@ -228,6 +228,7 @@ behaviour, the source of that number is named.
 - [180. test_native_toasted_write.py: a toasted value is written once and read back whole](#180-test_native_toasted_writepy-a-toasted-value-is-written-once-and-read-back-whole)
 - [181. test_native_varlena_bound.py: a corrupt varlena length is refused, not read past its buffer](#181-test_native_varlena_boundpy-a-corrupt-varlena-length-is-refused-not-read-past-its-buffer)
 - [182. test_native_agg_addcolumn.py: a metadata aggregate over an added column reads what the metadata cannot hold](#182-test_native_agg_addcolumnpy-a-metadata-aggregate-over-an-added-column-reads-what-the-metadata-cannot-hold)
+- [183. test_native_groupagg_wide_cost.py: the grouped fold prices its own input scan as the scan node does](#183-test_native_groupagg_wide_costpy-the-grouped-fold-prices-its-own-input-scan-as-the-scan-node-does)
 
 ## 1. How to read a test in here
 
@@ -8047,3 +8048,19 @@ Each check is its own test over a module fixture that runs every step in the she
 | `test_the_oracle_reads_the_rows` | premise: the oracle's connection plans `Custom Scan (PgColumnarScan)`, so the two sides take different paths; an empty or other plan cannot pass |
 | `test_the_scan_reads_an_added_columns_default_for_the_rows_written_before_it` | the oracle's own `count(d)` and `sum(d)` are 3000 and 21000: both paths read old groups through the reader's missing value, so agreement alone cannot see it lost |
 | `test_the_metadata_path_agrees_with_the_scan` | parametrized by the shell's 16 names: count, sum, min, max and avg over the added columns, count(*), and the same across both sides of the ALTER and after a delete |
+
+## 183. test_native_groupagg_wide_cost.py: the grouped fold prices its own input scan as the scan node does
+
+The pytest twin of `test/native_groupagg_wide_cost.sh` (ported for #1341), every check by the shell's name. The grouped vector aggregate prices the scan it performs from the cheapest serial scan path that survived add_path. When none survived, because a Gather over the partial path dominated it, it prices that scan itself, and must do so as the columnar scan node does, not with the bare seqscan formula (#12). On the suite's 40,000-row wide table a serial scan survives, so the fold and result readings never reach that fallback. Free parallelism (`parallel_setup_cost`, `parallel_tuple_cost` and `min_parallel_table_scan_size` at 0) lets a Gather displace it, and with `enable_hashagg` and `enable_sort` off the parallel finalize cannot win, so the fold node is the plan and its cost can be read. Each reading's settings ride in the connection's options, as the shell's PGOPTIONS do.
+
+### Every test
+
+Each check is its own test over a module fixture that takes every reading in the shell's order, each on a connection of its own. A failed query is a `query_error()`, refused on either side by name.
+
+| test | what it holds |
+| --- | --- |
+| `test_the_fold_path_is_used_on_the_wide_grouped_aggregate` | the plan carries `Columnar Batch Fold: yes`, not merely a custom scan, which the reference plan's input is too |
+| `test_the_fold_result_matches_a_plain_aggregate_on_a_wide_table` | the folded per-group sums and counts equal the plan without vectorized grouping |
+| `test_with_parallelism_free_a_gather_displaces_the_serial_scan` | premise: with parallelism free the plan is a Gather, the state in which the fallback prices the scan |
+| `test_the_fallback_prices_the_fold_as_a_surviving_serial_scan_does` | the fold node's cost through the fallback equals its cost when a serial scan survives (800.19 on PG17; 503.01 with the bare seqscan formula) |
+| `test_backend_alive` | the backend still answers |

test/pytest/expected_tests.txt

@@ -852,4 +852,6 @@ guard_tests 442
 # test (1 -> 25 tests), by collection over the cluster files.
 # cluster_tests 990 -> 1006 for #1341, test_iceberg_data_files.py ported by name, one check
 # per test (5 -> 21 tests), by collection over the cluster files.
-cluster_tests 1006
+# cluster_tests 1006 -> 1011 for #1341's test_native_groupagg_wide_cost.py, the twin of
+# native_groupagg_wide_cost.sh: five tests, one per check, by collection over the cluster files.
+cluster_tests 1011

test/pytest/test_compare_to_bash.py

@@ -143,6 +143,7 @@
             "native_gap",
             "native_groupagg",
             "native_groupagg_batch",
+            "native_groupagg_wide_cost",
             "native_index",
             "native_index_fetch_stripe_cost",
             "native_index_projection",

test/pytest/test_native_groupagg_wide_cost.py

@@ -0,0 +1,124 @@
+"""The grouped vector aggregate's input scan, priced on a wide table. The pytest twin
+of `test/native_groupagg_wide_cost.sh`, every check by the shell's name (#1341).
+
+The grouped fold node prices the scan it performs from the cheapest serial scan path
+that survived add_path. When none did -- a Gather over the partial path dominated it
+-- it prices the scan itself, and must do so the way the columnar scan node does
+(projected-width I/O, per-column decode CPU, zone-map survival), not with the bare
+seqscan formula that under-priced it on a wide table (#12).
+
+With 40,000 rows a serial scan survives, so the fold and result readings never reach
+that fallback. Free parallelism lets a Gather displace the serial scan; with hash and
+sort off the parallel finalize cannot win, so the fold node is the plan and its cost
+can be read. It must equal the cost a surviving serial scan gives.
+
+Each reading's settings ride in the connection's options, as the shell's PGOPTIONS
+do, and each reading has a connection of its own, as the shell's psql calls do. ON A
+PRIVATE DATABASE, built once by a module fixture that takes every reading in the
+shell's order; each test asserts one, so one red cannot stop the next (#1339). A
+failed query is a `query_error()`, refused on either side by name.
+"""
+import re
+
+import psycopg
+import pytest
+
+from pgc_vacuity import query_error
+
+DB = "pgc_groupagg_wide_cost"
+COLS = 25
+Q = ("SELECT string_agg(g||':'||s0||':'||s24||':'||c, '|' ORDER BY g) "
+     "FROM (SELECT g, sum(c0) s0, sum(c24) s24, count(*) c FROM wa GROUP BY g) r")
+EXPLAINQ = "SELECT g, sum(c0) FROM wa GROUP BY g"
+REF = "-c pgcolumnar.enable_group_vectorization=off -c pgcolumnar.enable_ungrouped_vector_agg=off"
+FOLD = ("-c pgcolumnar.enable_ungrouped_vector_agg=on -c pgcolumnar.enable_parallel_vector_agg=on "
+        "-c max_parallel_workers_per_gather=4")
+FREE = ("-c parallel_setup_cost=0 -c parallel_tuple_cost=0 -c min_parallel_table_scan_size=0 "
+        "-c max_parallel_workers_per_gather=4")
+FOLD_LINE = "Columnar Batch Fold: yes"
+COST = re.compile(r"^Custom Scan \(PgColumnarScan\)\s+\(cost=[0-9.]+\.\.([0-9.]+) ")
+
+
+def _failed(exc):
+    return query_error(str(exc).splitlines()[0] if str(exc) else type(exc).__name__)
+
+
+def _rows(dsn, sql, options=""):
+    """-> every row's first column as text, on a connection of its own, or the
+    failure as a query_error()."""
+    try:
+        with psycopg.connect(dsn, autocommit=True, options=options) as conn:
+            return ["" if r[0] is None else str(r[0]) for r in conn.execute(sql)]
+    except psycopg.Error as exc:
+        return _failed(exc)
+
+
+def _value(dsn, sql, options=""):
+    rows = _rows(dsn, sql, options)
+    return rows if isinstance(rows, str) else (rows[0] if rows else "")
+
+
+def _fold_cost(dsn, options):
+    """-> the fold node's total cost, "" when the plan is not the fold node, or the
+    failure as a query_error()."""
+    plan = _rows(dsn, f"EXPLAIN {EXPLAINQ}", options)
+    if isinstance(plan, str):
+        return plan
+    m = COST.match(plan[0]) if plan else None
+    return m.group(1) if m and any(FOLD_LINE in line for line in plan) else ""
+
+
+@pytest.fixture(scope="module")
+def wide(pgc_cluster):
+    with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a:
+        a.execute(f"DROP DATABASE IF EXISTS {DB} WITH (FORCE)")
+        a.execute(f"CREATE DATABASE {DB}")
+    try:
+        dsn = pgc_cluster.dsn(dbname=DB)
+        cols = ", ".join(f"c{i} bigint" for i in range(COLS))
+        vals = ", ".join(f"g*{i + 1}" for i in range(COLS))
+        with psycopg.connect(dsn, autocommit=True) as c:
+            for sql in ("CREATE EXTENSION pgcolumnar",
+                        f"CREATE TABLE wa (g int, {cols}) USING pgcolumnar",
+                        f"INSERT INTO wa SELECT g%16, {vals} FROM generate_series(1, 40000) g",
+                        "ANALYZE wa"):
+                try:
+                    c.execute(sql)
+                except psycopg.Error as exc:
+                    print(f"-- step failed: {str(exc).splitlines()[0] if str(exc) else exc!r}")
+        r = {"ref": _value(dsn, Q, REF), "got": _value(dsn, Q, FOLD)}
+        plan = _rows(dsn, f"EXPLAIN (COSTS OFF, VERBOSE) {EXPLAINQ}", FOLD)
+        r["fold markers"] = plan if isinstance(plan, str) else sum(FOLD_LINE in line for line in plan)
+        gather = _rows(dsn, f"EXPLAIN (COSTS OFF) {EXPLAINQ}", FREE)
+        r["gather"] = gather if isinstance(gather, str) else (
+            "yes" if any("Gather" in line for line in gather) else "")
+        r["fallback cost"] = _fold_cost(dsn, f"{FREE} -c enable_hashagg=off -c enable_sort=off")
+        r["serial cost"] = _fold_cost(dsn, "")
+        r["alive"] = _value(dsn, "SELECT 1")
+        yield r
+    finally:
+        with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a:
+            a.execute(f"DROP DATABASE IF EXISTS {DB} WITH (FORCE)")
+
+
+def test_the_fold_path_is_used_on_the_wide_grouped_aggregate(wide, expect):
+    m = wide["fold markers"]
+    expect.text(m if isinstance(m, str) else ("yes" if m > 0 else ""), "yes",
+                "the fold path is used on the wide grouped aggregate")
+
+
+def test_the_fold_result_matches_a_plain_aggregate_on_a_wide_table(wide, expect):
+    expect.text(wide["got"], wide["ref"], "the fold result matches a plain aggregate on a wide table")
+
+
+def test_with_parallelism_free_a_gather_displaces_the_serial_scan(wide, expect):
+    expect.text(wide["gather"], "yes", "premise: with parallelism free, a Gather displaces the serial scan")
+
+
+def test_the_fallback_prices_the_fold_as_a_surviving_serial_scan_does(wide, expect):
+    expect.text(wide["fallback cost"], wide["serial cost"],
+                "the fallback prices the fold as a surviving serial scan does")
+
+
+def test_backend_alive(wide, expect):
+    expect.text(wide["alive"], "1", "backend alive")

test/pytest/unported_suites.txt

@@ -86,7 +86,6 @@ native_fetch_interrupt
 native_fetch_sort_context
 native_fold_deferral
 native_fold_skipguard
-native_groupagg_wide_cost
 native_late_materialization
 native_lazy_slot
 native_metadata_flush