From cf98e1b67ba17a0fca018bfb72976da614c7edc6 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 20:49:36 +0000 Subject: [PATCH 1/6] test: the group estimate must bound a time bucket from the zone maps (#1292), red Four arms in both twins, written before the estimator changes. Each asserts the estimate is at least the true group count (under-estimating is the harmful direction) and at most four times it: no time bound at all, date_trunc estimate 300000 for 11 groups only a lower time bound, date_trunc estimate 299970 for 11 groups a date_bin key estimate 299940 for 11 groups date_bin with no time bound estimate 300000 for 3 groups They replace 'a truncating key with no range bound gets no bound from us': the owner chose on #1292 to gate the grouped path on this estimate, and a gate cannot use one 60x to 3600x high. That arm pinned the old design rather than a defect, and it is replaced, not weakened. Co-Authored-By: Claude Opus 5.5 --- test/native_groupagg.sh | 33 +++++++++++++++++++++++++---- test/pytest/test_native_groupagg.py | 29 +++++++++++++++++++------ 2 files changed, 52 insertions(+), 10 deletions(-) diff --git a/test/native_groupagg.sh b/test/native_groupagg.sh index 6a90f3999..2bb210ce4 100755 --- a/test/native_groupagg.sh +++ b/test/native_groupagg.sh @@ -450,10 +450,35 @@ plain_before=$(gbest "SELECT host, avg(v) FROM gb GROUP BY host") check "an informed estimate (plain column) is not replaced" \ "$([ "$plain_before" -ge 3900 ] && [ "$plain_before" -le 4100 ] && echo yes || echo no)" "yes" -# No Const bound on the key means no range, so no bound. The estimate stays whatever core -# produced; asserting it is LARGE is the point, since a bound would have shrunk it. -check "a truncating key with no range bound gets no bound from us" \ - "$([ "$(gbest "SELECT date_trunc('minute', ts) b, avg(v) FROM gb GROUP BY b")" -gt 1000 ] && echo yes || echo no)" "yes" +# A MISSING END OF THE RANGE COMES FROM THE ZONE MAPS (#1292). This arm used to assert +# the opposite -- "a truncating key with no range bound gets no bound from us" -- and that +# was the design until the owner chose, on #1292, to gate the grouped path on this +# estimate. A gate cannot use a number 60x to 3600x too high (measured on #1292), and a +# dashboard query bounds only one side ("the last day"). The zone maps hold every row +# group's minimum and maximum of the key's column, which is the range the scan reads. +# +# Each arm states BOTH directions: at least the true group count, because an +# under-estimate is the direction that under-sizes the hash table and would let the gate +# pass a query the runtime cap then refuses; and at most 4x, which is the accuracy a gate +# needs. Each reports the estimate and the truth when it fails. +gbwithin() { # gbwithin QUERY -> "ok" when actual <= estimate <= 4*actual + local est act + est="$(gbest "$1")" + act="$(q "SELECT count(*) FROM ($1) z;")" + if [ -n "$est" ] && [ -n "$act" ] && [ "$act" -ge 1 ] && [ "$est" -ge "$act" ] && [ "$est" -le $((act * 4)) ]; then + echo ok + else + echo "estimate ${est:-none} for ${act:-none} groups" + fi +} +check "with no time bound at all, a truncating key is bounded from the zone maps" \ + "$(gbwithin "SELECT date_trunc('minute', ts) b, avg(v) FROM gb GROUP BY b")" "ok" +check "with only a lower time bound, a truncating key is still bounded" \ + "$(gbwithin "SELECT date_trunc('minute', ts) b, avg(v) FROM gb WHERE ts >= '2026-01-01' GROUP BY b")" "ok" +check "a date_bin key is bounded as a truncating key is" \ + "$(gbwithin "SELECT date_bin('1 minute', ts, timestamptz '2026-01-01') b, avg(v) FROM gb $W GROUP BY b")" "ok" +check "and a date_bin key with no time bound is bounded from the zone maps" \ + "$(gbwithin "SELECT date_bin('5 minutes', ts, timestamptz '2026-01-01') b, avg(v) FROM gb GROUP BY b")" "ok" # A user function with date_trunc's SHAPE must not get date_trunc's BOUND. PL/pgSQL # because an SQL function would be inlined and lose the shape. Matching on shape alone # estimated this at 722 against 499,999 actual, a 692x UNDER-estimate, which is the diff --git a/test/pytest/test_native_groupagg.py b/test/pytest/test_native_groupagg.py index 2bd1b3e54..9d089f2fb 100644 --- a/test/pytest/test_native_groupagg.py +++ b/test/pytest/test_native_groupagg.py @@ -617,12 +617,29 @@ def test_the_group_estimate_bound_is_accurate_and_narrow(pgc_conn, expect): expect.text("yes" if 3900 <= plain <= 4100 else f"no [{plain}]", "yes", "an informed estimate (plain column) is not replaced") - # No Const bound on the key means no range, so no bound. Asserting the estimate - # is LARGE is the point: a bound would have shrunk it. - unbounded = _estimate(pgc_conn, - "SELECT date_trunc('minute', ts) b, avg(v) FROM gb GROUP BY b") - expect.text("yes" if unbounded > 1000 else f"no [{unbounded}]", "yes", - "a truncating key with no range bound gets no bound from us") + # A MISSING END OF THE RANGE COMES FROM THE ZONE MAPS (#1292). This used to assert + # that a key with no Const bound gets no bound at all. The owner chose on #1292 to + # gate the grouped path on this estimate, and a gate cannot use one that is 60x to + # 3600x high (measured there), while a dashboard query bounds only one side. Each + # case asserts both directions: at least the true count, because under-estimating + # is the harmful direction, and at most four times it. + def within(sql): + est = _estimate(pgc_conn, sql) + act = int(_rows(pgc_conn, f"SELECT count(*) FROM ({sql}) z")[0][0]) + ok = act >= 1 and act <= est <= 4 * act + return "ok" if ok else f"estimate {est} for {act} groups" + + expect.text(within("SELECT date_trunc('minute', ts) b, avg(v) FROM gb GROUP BY b"), + "ok", "with no time bound at all, a truncating key is bounded from the zone maps") + expect.text(within("SELECT date_trunc('minute', ts) b, avg(v) FROM gb " + "WHERE ts >= '2026-01-01' GROUP BY b"), + "ok", "with only a lower time bound, a truncating key is still bounded") + expect.text(within("SELECT date_bin('1 minute', ts, timestamptz '2026-01-01') b, avg(v) " + f"FROM gb {GB_WINDOW} GROUP BY b"), + "ok", "a date_bin key is bounded as a truncating key is") + expect.text(within("SELECT date_bin('5 minutes', ts, timestamptz '2026-01-01') b, avg(v) " + "FROM gb GROUP BY b"), + "ok", "and a date_bin key with no time bound is bounded from the zone maps") # A user function with date_trunc's SHAPE must not get date_trunc's BOUND. # PL/pgSQL because an SQL function would be inlined and lose the shape. Matching From df6ec1029edf616946fa784815af367df2b102fd Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 20:58:55 +0000 Subject: [PATCH 2/6] opt: estimate time-bucket groups from the zone maps, and recognise date_bin (#1292) GROUP BY date_trunc(unit, ts) was estimated near the row count unless the query bounded ts on both sides, and date_bin was not recognised at all: 300,000 groups for 11 on native_groupagg's fixture, 60x to 3,600x on #1292's measurements. The owner chose to gate the vectorized grouping path on this estimate, and a gate cannot use that number. - A missing end of the range comes from the zone maps: the smallest and largest value of the key's column over EVERY row group (not a sample, which could miss the maximum and under-estimate), clipped to any bound the query gives. A group with values but no usable zone map for the column means no bound. - date_bin(stride, ts, origin) is bounded as date_trunc is, the width from the stride; a stride with months is refused, as date_bin does. - A timestamp Const against a timestamptz column still never sets the range. Test first (d88f084, red): four arms in both twins assert each estimate is at least the true group count and at most four times it. They replace 'a truncating key with no range bound gets no bound from us', which pinned the old design. 'a mixed timestamp/timestamptz predicate gets NO bound' is replaced by the property it protected: such a predicate never narrows the estimate below the true count, on a window where misreading wall clock as UTC gives 50 for 150. Mutations, both twins, PG17: zone-map fill disabled the 3 zone-map arms red date_bin not recognised the 2 date_bin arms red mixed Const accepted 'estimate 50 for 150 groups', red (my first date_bin mutation was 'false && A || B', which && binds as '(false && A) || B' and never applied; redone with parentheses) native_groupagg green on PG15 to 19 (PG18 runs the ICU arm too); harness_selftest 1213/1213 after removing the replaced arm's row from lossy_arms.tsv; parity 40 literal + 1 template, 0 missing; guard half 403 + test_build_refusal (red on unmodified main here too). Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 12 +++ src/columnar_vector.c | 127 +++++++++++++++++++++++++++- test/lossy_arms.tsv | 1 - test/native_groupagg.sh | 18 +++- test/pytest/TESTS.md | 4 +- test/pytest/test_native_groupagg.py | 28 +++--- 6 files changed, 173 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index eee5185c8..1c522a1a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,18 @@ true until the next version shipped. ### Changed +- The planner estimates the number of groups for a time-bucket key from the + table's own data (#1292). `GROUP BY date_trunc(unit, ts)` and, newly, + `GROUP BY date_bin(stride, ts, origin)` were estimated near the row count + unless the query bounded `ts` on both sides: 300,000 groups for 11 on the test + fixture, and 60x to 3,600x too high on #1292's measurements. A missing end of + the range now comes from the zone maps, the smallest and largest value of `ts` + in every row group, so a query bounded on one side only, or not at all, is + estimated within a small factor of the truth and never below it. This is + the first step towards turning the vectorized grouping path on by default. + A `timestamp` constant compared against a `timestamptz` column still does not + set the range. + - Five selftest parts and `rewrite_storage_oid` no longer hold a check that has never been red (#1236). Nine checks now carry a mutation that reddens them, each measured on all five majors: 570's fold arm, two in 290, one in 560, three diff --git a/src/columnar_vector.c b/src/columnar_vector.c index 4baeb346e..6d3d6f3b9 100644 --- a/src/columnar_vector.c +++ b/src/columnar_vector.c @@ -2041,6 +2041,79 @@ pgcolumnar_groupagg_outmap(List *exprs, List *groupKeys, Index scanrelid, * value is used to make a plan cheaper, so an under-estimate would * under-price the arm and under-size the Finalize's hash table. */ +/* + * pgcolumnar_zonemap_time_range + * The smallest and largest value of a timestamp column over every row group, + * from the zone maps (#1292). Returns false when any group that holds values + * has no usable zone map for the column -- a column added after the group was + * written, say -- because a range that leaves out one group is not a bound. + */ +static bool +pgcolumnar_zonemap_time_range(PlannerInfo *root, RelOptInfo *input_rel, Var *tsvar, + double *lo, double *hi) +{ + RangeTblEntry *rte = planner_rt_fetch(input_rel->relid, root); + Relation r; + TupleDesc td; + Form_pg_attribute att; + uint64 storageId; + Snapshot snap; + List *groups; + ListCell *lc; + PgColumnarZoneMapSession sess; + bool any = false; + bool ok = true; + int col = tsvar->varattno - 1; + + if (rte == NULL || !OidIsValid(rte->relid) || tsvar->varattno <= 0) + return false; + r = table_open(rte->relid, AccessShareLock); + td = RelationGetDescr(r); + if (col >= td->natts) + { + table_close(r, AccessShareLock); + return false; + } + att = TupleDescAttr(td, col); + storageId = PgColumnarStorageId(r); + snap = PgColumnarCatalogSnapshot(GetActiveSnapshot()); + groups = PgColumnarReadRowGroupList(storageId, snap); + memset(&sess, 0, sizeof(sess)); + sess.what = "group estimate"; + + foreach(lc, groups) + { + NativeRowGroupMetadata *rg = (NativeRowGroupMetadata *) lfirst(lc); + NativeZoneMapMetadata *z = PgColumnarReadZoneMapForColumn(storageId, + rg->groupNumber, + col, snap, &sess); + char *cur; + double vmin, + vmax; + + if (z == NULL || !z->hasMinMax) + { + if (z != NULL && z->valueCount == 0) + continue; /* all null: contributes no bucket */ + ok = false; + break; + } + cur = (char *) z->minimum; + vmin = (double) DatumGetTimestamp(PgColumnarDecodeValue(att, &cur, + z->minimum + z->minimumLen, CurrentMemoryContext)); + cur = (char *) z->maximum; + vmax = (double) DatumGetTimestamp(PgColumnarDecodeValue(att, &cur, + z->maximum + z->maximumLen, CurrentMemoryContext)); + *lo = any ? Min(*lo, vmin) : vmin; + *hi = any ? Max(*hi, vmax) : vmax; + any = true; + } + PgColumnarCloseZoneMapSession(&sess); + list_free_deep(groups); + table_close(r, AccessShareLock); + return ok && any; +} + static double pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel, Node *expr) @@ -2079,6 +2152,36 @@ pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel, * deliberately: its argument is a duration rather than a point in time, so a * range on it does not describe a scanned window. */ + if (f->funcid == F_DATE_BIN_INTERVAL_TIMESTAMP_TIMESTAMP || + f->funcid == F_DATE_BIN_INTERVAL_TIMESTAMPTZ_TIMESTAMPTZ) + { + /* + * date_bin(stride, source, origin) (#1292): the same bucketing with the + * width given as an interval. A stride with months is not a fixed width, + * and date_bin itself refuses one, so it gets no bound here either. + */ + Const *stride; + Interval *iv; + + if (list_length(f->args) != 3 || !IsA(linitial(f->args), Const) || + !IsA(lsecond(f->args), Var)) + return 0.0; + stride = (Const *) linitial(f->args); + if (stride->constisnull || stride->consttype != INTERVALOID) + return 0.0; + iv = DatumGetIntervalP(stride->constvalue); + if (iv->month != 0) + return 0.0; + width = (double) iv->time + (double) iv->day * (double) USECS_PER_DAY; + if (width <= 0) + return 0.0; + tsvar = (Var *) lsecond(f->args); + if (tsvar->varno != input_rel->relid || tsvar->varlevelsup != 0) + return 0.0; + if (tsvar->vartype != TIMESTAMPOID && tsvar->vartype != TIMESTAMPTZOID) + return 0.0; + goto have_width; + } if (f->funcid != F_DATE_TRUNC_TEXT_TIMESTAMP && f->funcid != F_DATE_TRUNC_TEXT_TIMESTAMPTZ && f->funcid != F_DATE_TRUNC_TEXT_TIMESTAMPTZ_TEXT) @@ -2115,6 +2218,8 @@ pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel, else if (pg_strcasecmp(unitstr, "century") == 0) width = 36500.0 * USECS_PER_DAY; else if (pg_strcasecmp(unitstr, "millennium") == 0) width = 365000.0 * USECS_PER_DAY; else return 0.0; + +have_width: /* * month, quarter and year use their SHORTEST possible length above, so the * bucket count is over-estimated rather than under-estimated. This must stay @@ -2180,7 +2285,27 @@ pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel, } } - if (!havelo || !havehi || hi <= lo) + /* + * A MISSING END COMES FROM THE ZONE MAPS (#1292), and both ends are clipped to + * them when they are known. Every row group's whole-chunk zone map holds the + * minimum and maximum of the key's column, so together they bound every value + * the scan can read: deletes only shrink that, never widen it. All groups are + * read, not a sample -- a sampled maximum can miss the true one, which would + * UNDER-estimate, the direction a gate on this number must never take. + */ + { + double zlo, + zhi; + + if (pgcolumnar_zonemap_time_range(root, input_rel, tsvar, &zlo, &zhi)) + { + lo = havelo ? Max(lo, zlo) : zlo; + hi = havehi ? Min(hi, zhi) : zhi; + havelo = havehi = true; + } + } + + if (!havelo || !havehi || hi < lo) return 0.0; return floor((hi - lo) / width) + 2.0; } diff --git a/test/lossy_arms.tsv b/test/lossy_arms.tsv index 92053f53b..2f99561de 100644 --- a/test/lossy_arms.tsv +++ b/test/lossy_arms.tsv @@ -136,7 +136,6 @@ native_gap gap was physically materialized native_groupagg a lookalike function with the same signature gets NO bound native_groupagg and it is an UPPER bound: the true group count is at or below it native_groupagg a non-time expression key gets no bound either -native_groupagg a truncating key with no range bound gets no bound from us native_reclaim_frag coalesce merges adjacent frees (fewer free_space rows) native_reclaim_frag coalesce never larger than whole-range reuse native_reclaim repeated recluster reuses space (file plateaus) diff --git a/test/native_groupagg.sh b/test/native_groupagg.sh index 2bb210ce4..61ec04600 100755 --- a/test/native_groupagg.sh +++ b/test/native_groupagg.sh @@ -500,9 +500,21 @@ psql_run "CREATE TABLE gbm (ts timestamptz, v float8) USING pgcolumnar; gbmest() { env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ -d "$PGC_DB" -At -q -c "SET TimeZone='Asia/Kolkata';" -c "$GB" \ -c "EXPLAIN (COSTS ON) $1" 2>&1 | grep -oE "rows=[0-9]+" | head -1 | cut -d= -f2; } -check "a mixed timestamp/timestamptz predicate gets NO bound" \ - "$([ "$(gbmest "SELECT date_trunc('minute',ts) b, avg(v) FROM gbm - WHERE ts >= timestamp '2026-01-01 00:00' AND ts < timestamp '2026-01-01 12:00' GROUP BY b")" -gt 50000 ] && echo yes || echo no)" "yes" +# WHAT THE MIXED-TYPE RULE PROTECTS, stated directly (#1292). This asserted "gets NO +# bound", which was the observable sign that the mixed predicate was not used. With the +# zone maps supplying a range, there is a bound either way, so the arm now asserts the +# property itself: a wall-clock window read as UTC must never narrow the estimate below +# the true group count. Under Asia/Kolkata, 04:00-08:00 wall clock is 22:30-02:30 UTC, +# 150 minutes of data; read as UTC it would clip the data to 04:00-04:48, about 50. +gbmcount() { env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -At -q -c "SET TimeZone='Asia/Kolkata';" -c "SELECT count(*) FROM ($1) z;" 2>&1; } +GBM_MIXED="SELECT date_trunc('minute',ts) b, avg(v) FROM gbm + WHERE ts >= timestamp '2026-01-01 04:00' AND ts < timestamp '2026-01-01 08:00' GROUP BY b" +gbm_est="$(gbmest "$GBM_MIXED")"; gbm_act="$(gbmcount "$GBM_MIXED")" +check "premise: the mixed-type window holds data" \ + "$([ "${gbm_act:-0}" -ge 100 ] && echo yes || echo "no (${gbm_act:-none} groups)")" "yes" +check "a mixed timestamp/timestamptz predicate never narrows the estimate below the true count" \ + "$([ "${gbm_est:-0}" -ge "${gbm_act:-1}" ] && echo yes || echo "no (estimate ${gbm_est:-none} for ${gbm_act:-none} groups)")" "yes" check "while a matching-type predicate still gets one" \ "$([ "$(gbmest "SELECT date_trunc('minute',ts) b, avg(v) FROM gbm WHERE ts >= timestamptz '2026-01-01 00:00+00' AND ts < timestamptz '2026-01-01 12:00+00' GROUP BY b")" -lt 5000 ] && echo yes || echo no)" "yes" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index a4437f9c5..17df4b827 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -5646,8 +5646,8 @@ every major for this pair, which is the designed third state and not a failure. | `test_avg_float8_overflows_the_way_core_overflows` | the node fires and raises out of range, which a row comparison cannot express | | `test_degenerate_inputs_produce_no_groups` | a predicate matching nothing, and an empty table | | `test_the_path_pays_for_the_folding_it_does` | the isolated #349 arm and its two node premises | -| `test_the_group_estimate_bound_is_accurate_and_narrow` | the bound is 12 and is an upper bound; an informed estimate is untouched; no range, a lookalike function and a non-time key each get no bound | -| `test_a_mixed_timestamp_predicate_gets_no_bound` | a cross-type predicate under a non-UTC TimeZone gets none, while the matching-type one still gets a bound | +| `test_the_group_estimate_bound_is_accurate_and_narrow` | the bound is 12 and is an upper bound; an informed estimate is untouched; with no time bound, with only a lower one, and for `date_bin`, the estimate is between the true group count and four times it, from the zone maps (#1292); a lookalike function and a non-time key get no bound | +| `test_a_mixed_timestamp_predicate_never_narrows_the_bound` | a cross-type predicate under a non-UTC TimeZone never narrows the estimate below the true group count (#1292); the matching-type one still gets a tight bound | ## 72. test_analyze_function.py: statistics collected by reading, not by sampling diff --git a/test/pytest/test_native_groupagg.py b/test/pytest/test_native_groupagg.py index 9d089f2fb..5a6526e9a 100644 --- a/test/pytest/test_native_groupagg.py +++ b/test/pytest/test_native_groupagg.py @@ -658,7 +658,7 @@ def within(sql): "a non-time expression key gets no bound either") -def test_a_mixed_timestamp_predicate_gets_no_bound(pgc_conn, expect): +def test_a_mixed_timestamp_predicate_never_narrows_the_bound(pgc_conn, expect): """PostgreSQL has cross-type operators, so a `timestamp` Const compared against a `timestamptz` column stays bare and reaches the extraction instead of being wrapped in a cast. Both are int64 microseconds on different scales -- wall clock @@ -674,18 +674,26 @@ def test_a_mixed_timestamp_predicate_gets_no_bound(pgc_conn, expect): _exec(pgc_conn, "SET TimeZone='Asia/Kolkata'") _gb_session(pgc_conn) - mixed = _estimate(pgc_conn, - "SELECT date_trunc('minute',ts) b, avg(v) FROM gbm " - "WHERE ts >= timestamp '2026-01-01 00:00' " - "AND ts < timestamp '2026-01-01 12:00' GROUP BY b") + # THE PROPERTY THE MIXED-TYPE RULE PROTECTS, stated directly (#1292). This used to + # assert "gets NO bound", the observable sign that the mixed predicate was unused. + # With the zone maps supplying a range there is a bound either way, so the arm now + # asserts that a wall-clock window read as UTC never narrows the estimate below the + # truth. 04:00-08:00 Kolkata is 22:30-02:30 UTC, 150 minutes of this data; read as + # UTC it would clip the data to 04:00-04:48, about 50. + mixed_sql = ("SELECT date_trunc('minute',ts) b, avg(v) FROM gbm " + "WHERE ts >= timestamp '2026-01-01 04:00' " + "AND ts < timestamp '2026-01-01 08:00' GROUP BY b") + mixed = _estimate(pgc_conn, mixed_sql) + mixed_actual = int(_rows(pgc_conn, f"SELECT count(*) FROM ({mixed_sql}) z")[0][0]) matching = _estimate(pgc_conn, "SELECT date_trunc('minute',ts) b, avg(v) FROM gbm " "WHERE ts >= timestamptz '2026-01-01 00:00+00' " "AND ts < timestamptz '2026-01-01 12:00+00' GROUP BY b") - print(f"-- mixed-type predicate {mixed}, matching-type predicate {matching}") - # The two arms are a pair: "no bound" and "a bound" have to be distinguishable - # on this fixture, and core's own estimate here is six figures, so they are. - expect.text("yes" if mixed > 50000 else f"no [{mixed}]", "yes", - "a mixed timestamp/timestamptz predicate gets NO bound") + print(f"-- mixed-type predicate {mixed} for {mixed_actual} groups, " + f"matching-type predicate {matching}") + expect.at_least(mixed_actual, 100, "premise: the mixed-type window holds data") + expect.text("yes" if mixed >= mixed_actual + else f"no (estimate {mixed} for {mixed_actual} groups)", "yes", + "a mixed timestamp/timestamptz predicate never narrows the estimate below the true count") expect.text("yes" if matching < 5000 else f"no [{matching}]", "yes", "while a matching-type predicate still gets one") From 4f8b231e385eabf51402958b3865707e891a99cc Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 21:09:33 +0000 Subject: [PATCH 3/6] opt: read a column's zone maps in one catalog scan for the bucket bound (#1304) Review finding, my own: the bound probed zone_map_pkey once per row group on every plan. Planning buffers at 67 groups went 109 -> 313. PgColumnarReadWholeChunkZoneMapsForColumn reads every group's whole-chunk row for one column in one scan keyed on storage_id; the helper compares its count with the group list, so a group with no row for the column still means no bound. Test first: 'planning a bucketed GROUP BY scans zone_map once, not once per row group', counted as the planning transaction's own scans on zone_map and zone_map_pkey (a catalog index scan counts on the INDEX: the table's counter alone read 0 against 50 real probes). Red at got [50] want [1]; green after. Planning buffers, PG17 non-assert, second plan in one session: 1M rows, 7 groups base 34 per-group 58 one scan 50 10M rows, 67 groups base 109 per-group 313 one scan 246 Still about 2 buffers per group: a scan keyed on storage_id walks every zone-map row of the table (every column and vector). Raised on #1304. Co-Authored-By: Claude Opus 5.5 --- src/columnar.h | 2 ++ src/columnar_metadata.c | 45 +++++++++++++++++++++++++++++++++++++++++ src/columnar_vector.c | 29 ++++++++++++++++---------- test/native_groupagg.sh | 23 +++++++++++++++++++++ 4 files changed, 88 insertions(+), 11 deletions(-) diff --git a/src/columnar.h b/src/columnar.h index 71021f787..ed42a7430 100644 --- a/src/columnar.h +++ b/src/columnar.h @@ -606,6 +606,8 @@ extern NativeZoneMapMetadata *PgColumnarReadZoneMapForColumn(uint64 storageId, Snapshot snapshot, PgColumnarZoneMapSession *sess); extern void PgColumnarCloseZoneMapSession(PgColumnarZoneMapSession *sess); +extern List *PgColumnarReadWholeChunkZoneMapsForColumn(uint64 storageId, int columnIndex, + Snapshot snapshot); extern void PgColumnarDeleteMetadata(uint64 storageId); /* declared retention, read by pgcolumnar.expire (#403 item 5a) */ diff --git a/src/columnar_metadata.c b/src/columnar_metadata.c index fa2684f8d..c137d2861 100644 --- a/src/columnar_metadata.c +++ b/src/columnar_metadata.c @@ -3506,6 +3506,51 @@ PgColumnarReadZoneMapList(uint64 storageId, uint64 groupNumber, Snapshot snapsho * is (storage_id, group_number, column_index, vector_index), so this probes * the exact column. Returns NULL when the column has no whole-chunk zone map. */ +/* + * PgColumnarReadWholeChunkZoneMapsForColumn + * Every row group's whole-chunk zone map for one column, in ONE catalog scan + * keyed on the storage id (#1304 review). The planner's time-bucket bound needs + * all of them, and one probe per group cost about three buffers per group on + * every plan. Groups come back in zone_map_pkey order; a group with no row for + * the column is simply absent, so a caller needing all groups compares counts. + */ +List * +PgColumnarReadWholeChunkZoneMapsForColumn(uint64 storageId, int columnIndex, + Snapshot snapshot) +{ + Relation rel = open_columnar_table("zone_map", AccessShareLock); + TupleDesc tupdesc = RelationGetDescr(rel); + ScanKeyData key[1]; + SysScanDesc scan; + Oid idxOid; + HeapTuple tuple; + List *result = NIL; + + ScanKeyInit(&key[0], Anum_zone_map_storage_id, BTEqualStrategyNumber, + F_INT8EQ, Int64GetDatum((int64) storageId)); + idxOid = pgcolumnar_scan_index_oid(rel, "zone_map_pkey"); + scan = systable_beginscan(rel, idxOid, OidIsValid(idxOid), snapshot, 1, key); + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + bool isnull; + int16 col = DatumGetInt16( + heap_getattr(tuple, Anum_zone_map_column_index, tupdesc, &isnull)); + int32 vecIndex = DatumGetInt32( + heap_getattr(tuple, Anum_zone_map_vector_index, tupdesc, &isnull)); + uint64 groupNumber; + + if (col != columnIndex || vecIndex != -1) + continue; /* this column's whole-chunk rows only */ + groupNumber = (uint64) DatumGetInt64( + heap_getattr(tuple, Anum_zone_map_group_number, tupdesc, &isnull)); + result = lappend(result, + zonemap_from_tuple(tuple, tupdesc, storageId, groupNumber, -1)); + } + systable_endscan(scan); + table_close(rel, AccessShareLock); + return result; +} + NativeZoneMapMetadata * PgColumnarReadZoneMapForColumn(uint64 storageId, uint64 groupNumber, int columnIndex, Snapshot snapshot, diff --git a/src/columnar_vector.c b/src/columnar_vector.c index 6d3d6f3b9..8363dc338 100644 --- a/src/columnar_vector.c +++ b/src/columnar_vector.c @@ -2059,8 +2059,8 @@ pgcolumnar_zonemap_time_range(PlannerInfo *root, RelOptInfo *input_rel, Var *tsv uint64 storageId; Snapshot snap; List *groups; + List *zones; ListCell *lc; - PgColumnarZoneMapSession sess; bool any = false; bool ok = true; int col = tsvar->varattno - 1; @@ -2077,23 +2077,30 @@ pgcolumnar_zonemap_time_range(PlannerInfo *root, RelOptInfo *input_rel, Var *tsv att = TupleDescAttr(td, col); storageId = PgColumnarStorageId(r); snap = PgColumnarCatalogSnapshot(GetActiveSnapshot()); + + /* + * TWO CATALOG SCANS, NOT ONE PER GROUP (#1304 review): the group list, for the + * count, and every group's zone map for this column. A group with no zone-map + * row for the column is missing from the second list, and a range that leaves + * out a group is not a bound, so the counts must agree. + */ groups = PgColumnarReadRowGroupList(storageId, snap); - memset(&sess, 0, sizeof(sess)); - sess.what = "group estimate"; + zones = PgColumnarReadWholeChunkZoneMapsForColumn(storageId, col, snap); + if (list_length(zones) != list_length(groups)) + ok = false; - foreach(lc, groups) + foreach(lc, zones) { - NativeRowGroupMetadata *rg = (NativeRowGroupMetadata *) lfirst(lc); - NativeZoneMapMetadata *z = PgColumnarReadZoneMapForColumn(storageId, - rg->groupNumber, - col, snap, &sess); + NativeZoneMapMetadata *z = (NativeZoneMapMetadata *) lfirst(lc); char *cur; double vmin, vmax; - if (z == NULL || !z->hasMinMax) + if (!ok) + break; + if (!z->hasMinMax) { - if (z != NULL && z->valueCount == 0) + if (z->valueCount == 0) continue; /* all null: contributes no bucket */ ok = false; break; @@ -2108,8 +2115,8 @@ pgcolumnar_zonemap_time_range(PlannerInfo *root, RelOptInfo *input_rel, Var *tsv *hi = any ? Max(*hi, vmax) : vmax; any = true; } - PgColumnarCloseZoneMapSession(&sess); list_free_deep(groups); + list_free_deep(zones); table_close(r, AccessShareLock); return ok && any; } diff --git a/test/native_groupagg.sh b/test/native_groupagg.sh index 61ec04600..8bbe41cf2 100755 --- a/test/native_groupagg.sh +++ b/test/native_groupagg.sh @@ -479,6 +479,29 @@ check "a date_bin key is bounded as a truncating key is" \ "$(gbwithin "SELECT date_bin('1 minute', ts, timestamptz '2026-01-01') b, avg(v) FROM gb $W GROUP BY b")" "ok" check "and a date_bin key with no time bound is bounded from the zone maps" \ "$(gbwithin "SELECT date_bin('5 minutes', ts, timestamptz '2026-01-01') b, avg(v) FROM gb GROUP BY b")" "ok" + +# THE RANGE COSTS ONE CATALOG SCAN, NOT ONE PER ROW GROUP (#1304 review). The first +# version probed zone_map_pkey once per group, on every plan: 109 -> 313 planning +# buffers at 67 groups, about 3 per group, so ~20,000 per plan at a billion rows. Counted +# as the planning transaction's own zone_map scans, which no other session can move (the +# #1287 instrument), on the table AND its index: a catalog index scan counts on the +# INDEX, and reading the table's counter alone gave 0 against 50 real probes. No WHERE, +# so the filter's survival estimate -- which samples zone +# maps on purpose -- is not in the count; 50 groups, so one scan and one per group are +# far apart. +psql_run "CREATE TABLE gbz (ts timestamptz, v float8) USING pgcolumnar; + SELECT pgcolumnar.set_options('gbz', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO gbz SELECT '2026-01-01'::timestamptz + g * interval '1 second', g FROM generate_series(1,50000) g; + ANALYZE gbz;" >/dev/null 2>&1 +gbz_groups="$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('gbz');")" +gbz_scans="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -At -q \ + -c "$GB" -c "BEGIN" -c "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM gbz GROUP BY b" \ + -c "SELECT pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass) + + pg_stat_get_xact_numscans('pgcolumnar.zone_map_pkey'::regclass)" -c "ROLLBACK" 2>&1 | grep -E '^[0-9]+$' | tail -1)" +check "premise: the fixture has many row groups, so one scan and one per group differ" \ + "$([ "${gbz_groups:-0}" -ge 40 ] && echo yes || echo "no (${gbz_groups:-none} groups)")" "yes" +check_num "planning a bucketed GROUP BY scans zone_map once, not once per row group" \ + "${gbz_scans:-}" "1" # A user function with date_trunc's SHAPE must not get date_trunc's BOUND. PL/pgSQL # because an SQL function would be inlined and lose the shape. Matching on shape alone # estimated this at 722 against 499,999 actual, a 692x UNDER-estimate, which is the From 1d77885972935bbc0f872475bef45ac41df41d78 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 21:51:51 +0000 Subject: [PATCH 4/6] test: the pytest twin counts the bound's zone_map scans, as the .sh twin does (#1304) compare_to_bash graded the native_groupagg pair incomplete on CI: the .sh suite's one-scan arm and its premise had no pytest twin. This adds them, with the same fixture and the same instrument. Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_native_groupagg.py | 33 +++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/pytest/test_native_groupagg.py b/test/pytest/test_native_groupagg.py index 5a6526e9a..560f1df49 100644 --- a/test/pytest/test_native_groupagg.py +++ b/test/pytest/test_native_groupagg.py @@ -658,6 +658,39 @@ def within(sql): "a non-time expression key gets no bound either") +def test_the_bound_reads_the_zone_maps_in_one_catalog_scan(pgc_conn, expect): + """The range costs one catalog scan, not one per row group (#1304 review). + + The first version probed zone_map_pkey once per group, on every plan: 109 -> 313 + planning buffers at 67 groups, about 3 per group. Counted as the planning + transaction's own scans, which no other session can move, on the table AND its + index, because a catalog index scan counts on the index. No WHERE, so the + filter's survival estimate, which samples zone maps on purpose, is not counted. + """ + _exec(pgc_conn, "CREATE TABLE gbz (ts timestamptz, v float8) USING pgcolumnar") + _exec(pgc_conn, "SELECT pgcolumnar.set_options('gbz', stripe_row_limit => 1000, " + "chunk_group_row_limit => 1000)") + _exec(pgc_conn, "INSERT INTO gbz SELECT '2026-01-01'::timestamptz + g * interval '1 second', g " + "FROM generate_series(1,50000) g") + _exec(pgc_conn, "ANALYZE gbz") + _gb_session(pgc_conn) + + groups = int(_one(pgc_conn, "SELECT count(*) FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('gbz')")) + _exec(pgc_conn, "BEGIN") + try: + _exec(pgc_conn, "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM gbz GROUP BY b") + scans = int(_one(pgc_conn, + "SELECT pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass) " + "+ pg_stat_get_xact_numscans('pgcolumnar.zone_map_pkey'::regclass)")) + finally: + _exec(pgc_conn, "ROLLBACK") + print(f"-- {groups} row groups, {scans} zone_map scans while planning") + expect.at_least(groups, 40, + "premise: the fixture has many row groups, so one scan and one per group differ") + expect.num(scans, 1, "planning a bucketed GROUP BY scans zone_map once, not once per row group") + + def test_a_mixed_timestamp_predicate_never_narrows_the_bound(pgc_conn, expect): """PostgreSQL has cross-type operators, so a `timestamp` Const compared against a `timestamptz` column stays bare and reaches the extraction instead of being From 9449132b245265dc4d602e256e3b8fb6cf843d18 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 21:53:21 +0000 Subject: [PATCH 5/6] test: the sizing suite's wrong estimate comes from arithmetic, not date_trunc (#1304) #1304 bounds a truncating time key from the zone maps, so date_trunc('day', ts) now estimates 7 groups for 6 real ones. The premise that the estimate is wrong went red on CI, and the jump-bound arm, which also used date_trunc, would have passed without testing anything. v / 100000 and v / 60 keep the wrong estimate, since neither PostgreSQL nor pgColumnar sees through integer division. Measured on PG16: estimate 500000 for 6 real groups, and 8334 groups allocate 65536 entries. With the jump bound removed (COLUMNAR_GROUPAGG_JUMP set to 1 << 20), the jump-bound arm fails: 1048576 entries for a need of 16384. Co-Authored-By: Claude Opus 5.5 --- test/groupagg_table_sizing.sh | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/test/groupagg_table_sizing.sh b/test/groupagg_table_sizing.sh index e4b9f32e0..708a2cb84 100755 --- a/test/groupagg_table_sizing.sh +++ b/test/groupagg_table_sizing.sh @@ -31,9 +31,10 @@ # # THE OVER-ESTIMATE ARM came out of review and is why the sizing happens on the # first grow rather than at Begin. estimate_num_groups cannot see through a -# function, so `GROUP BY date_trunc('day', ts)` over a high-cardinality timestamp -# reaches this node with an estimate of every row against a handful of real -# groups. Sizing at Begin allocated 131,072 entries for 47 groups: 128x the +# expression, so `GROUP BY v / 100000` over a high-cardinality column reaches +# this node with an estimate of every row against a handful of real groups. +# (The arm used date_trunc('day', ts) until #1304 bounded truncating time keys +# from the zone maps, which made that estimate right and this arm vacuous.) Sizing at Begin allocated 131,072 entries for 47 groups: 128x the # memory of growing from nothing, to save zero rehashing, on the most ordinary # grouping in analytics. A table that never outgrows its first 1024 entries must # never be charged for a wrong estimate. @@ -137,10 +138,10 @@ check "the table is never larger than the real group count needs (#810 review)" || echo "OVERSIZED ($CAP entries for $NGROUPS groups, which need $NEEDED)")" "bounded" # ---- the over-estimate arm (review of #810) ------------------------------- -EXPR_Q="SELECT date_trunc('day', ts) d, count(*) FROM gts_expr GROUP BY 1" +EXPR_Q="SELECT v / 100000 d, count(*) FROM gts_expr GROUP BY 1" EXPR_PLAN="$(explain_with "$BIG" "$EXPR_Q")" EXPR_CAP="$(field "$EXPR_PLAN" 'Columnar Group Table Capacity')" -EXPR_REAL="$(q1 "SELECT count(*) FROM (SELECT date_trunc('day', ts) FROM gts_expr GROUP BY 1) s")" +EXPR_REAL="$(q1 "SELECT count(*) FROM (SELECT v / 100000 FROM gts_expr GROUP BY 1) s")" EXPR_EST="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -Atq \ -c "$ON $BIG EXPLAIN (COSTS ON) $EXPR_Q" 2>&1 | sed -n 's/.*rows=\([0-9]*\).*width.*/\1/p' | head -1)" echo "-- expression key: planner estimate $EXPR_EST, real groups $EXPR_REAL, capacity ${EXPR_CAP:-0}" @@ -157,17 +158,17 @@ check "a wrong estimate cannot inflate a table that never needed to grow (#810 r || echo "INFLATED ($EXPR_CAP entries for $EXPR_REAL real groups)")" "not inflated" check "and the expression key still answers correctly" "$EXPR_REAL" \ - "$(q1 "SELECT count(DISTINCT date_trunc('day', ts)) FROM gts_expr")" + "$(q1 "SELECT count(DISTINCT v / 100000) FROM gts_expr")" # ---- the jump bound ------------------------------------------------------- # The arm above cannot see the jump bound: 6 groups fit in the starting 1024, so # the table never grows and no jump is taken. This one has enough real groups to # force a grow while keeping the estimate wildly wrong, which is the only shape # where the bound is observable. -MIN_Q="SELECT date_trunc('minute', ts) d, count(*) FROM gts_expr GROUP BY 1" +MIN_Q="SELECT v / 60 d, count(*) FROM gts_expr GROUP BY 1" MIN_PLAN="$(explain_with "$BIG" "$MIN_Q")" MIN_CAP="$(field "$MIN_PLAN" 'Columnar Group Table Capacity')" -MIN_REAL="$(q1 "SELECT count(*) FROM (SELECT date_trunc('minute', ts) FROM gts_expr GROUP BY 1) s")" +MIN_REAL="$(q1 "SELECT count(*) FROM (SELECT v / 60 FROM gts_expr GROUP BY 1) s")" MIN_NEED="$(awk -v g="${MIN_REAL:-1}" 'BEGIN { c = 1024; while (c * 0.7 < g) c *= 2; printf "%d", c }')" echo "-- minute key: real groups $MIN_REAL need $MIN_NEED entries; allocated ${MIN_CAP:-0}" @@ -181,7 +182,7 @@ check "a grow cannot jump to a wrong estimate, only toward it (#810 review)" \ 'BEGIN { print (c > 0 && c <= 8 * n) ? "bounded" : "UNBOUNDED (" c " for a need of " n ")" }')" "bounded" check "and the minute key answers correctly too" "$MIN_REAL" \ - "$(q1 "SELECT count(DISTINCT date_trunc('minute', ts)) FROM gts_expr")" + "$(q1 "SELECT count(DISTINCT v / 60) FROM gts_expr")" check "and the answer is the same under a small work_mem" \ "$(q1 "SET work_mem='1MB'; SELECT count(*) FROM (SELECT k, sum(v) FROM gts GROUP BY k) s")" "$NGROUPS" From 4aeccd0c72cc5139ad314a050c0e425277747ac0 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 21:54:27 +0000 Subject: [PATCH 6/6] test: register the one-scan pytest arm in TESTS.md and the cluster count (#1304) cluster_tests 494 -> 495, collected over the cluster files on this tree. Co-Authored-By: Claude Opus 5.5 --- test/pytest/TESTS.md | 1 + test/pytest/expected_tests.txt | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 17df4b827..e2433eb6d 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -5647,6 +5647,7 @@ every major for this pair, which is the designed third state and not a failure. | `test_degenerate_inputs_produce_no_groups` | a predicate matching nothing, and an empty table | | `test_the_path_pays_for_the_folding_it_does` | the isolated #349 arm and its two node premises | | `test_the_group_estimate_bound_is_accurate_and_narrow` | the bound is 12 and is an upper bound; an informed estimate is untouched; with no time bound, with only a lower one, and for `date_bin`, the estimate is between the true group count and four times it, from the zone maps (#1292); a lookalike function and a non-time key get no bound | +| `test_the_bound_reads_the_zone_maps_in_one_catalog_scan` | planning a bucketed GROUP BY over 50 row groups scans zone_map and its index once in all, not once per group (#1304) | | `test_a_mixed_timestamp_predicate_never_narrows_the_bound` | a cross-type predicate under a non-UTC TimeZone never narrows the estimate below the true group count (#1292); the matching-type one still gets a tight bound | ## 72. test_analyze_function.py: statistics collected by reading, not by sampling diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 019b63b3d..df9bd7c6e 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -551,4 +551,7 @@ guard_tests 413 # rebased tree, not added up. # 495 -> 496 for #1288: test_storage_row_repair.py. Re-derived by collection on # 038600e (after #1303): `496 tests collected`. -cluster_tests 497 +# +# cluster_tests 497 -> 498 for #1304's test_the_bound_reads_the_zone_maps_in_one_catalog_scan, +# collected over the cluster files on the tree rebased onto 503f080 (#1286). +cluster_tests 498