diff --git a/CHANGELOG.md b/CHANGELOG.md index eee5185c..1c522a1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,18 @@ true until the next version shipped. ### Changed +- The planner estimates the number of groups for a time-bucket key from the + table's own data (#1292). `GROUP BY date_trunc(unit, ts)` and, newly, + `GROUP BY date_bin(stride, ts, origin)` were estimated near the row count + unless the query bounded `ts` on both sides: 300,000 groups for 11 on the test + fixture, and 60x to 3,600x too high on #1292's measurements. A missing end of + the range now comes from the zone maps, the smallest and largest value of `ts` + in every row group, so a query bounded on one side only, or not at all, is + estimated within a small factor of the truth and never below it. This is + the first step towards turning the vectorized grouping path on by default. + A `timestamp` constant compared against a `timestamptz` column still does not + set the range. + - Five selftest parts and `rewrite_storage_oid` no longer hold a check that has never been red (#1236). Nine checks now carry a mutation that reddens them, each measured on all five majors: 570's fold arm, two in 290, one in 560, three diff --git a/src/columnar.h b/src/columnar.h index 71021f78..ed42a743 100644 --- a/src/columnar.h +++ b/src/columnar.h @@ -606,6 +606,8 @@ extern NativeZoneMapMetadata *PgColumnarReadZoneMapForColumn(uint64 storageId, Snapshot snapshot, PgColumnarZoneMapSession *sess); extern void PgColumnarCloseZoneMapSession(PgColumnarZoneMapSession *sess); +extern List *PgColumnarReadWholeChunkZoneMapsForColumn(uint64 storageId, int columnIndex, + Snapshot snapshot); extern void PgColumnarDeleteMetadata(uint64 storageId); /* declared retention, read by pgcolumnar.expire (#403 item 5a) */ diff --git a/src/columnar_metadata.c b/src/columnar_metadata.c index fa2684f8..c137d286 100644 --- a/src/columnar_metadata.c +++ b/src/columnar_metadata.c @@ -3506,6 +3506,51 @@ PgColumnarReadZoneMapList(uint64 storageId, uint64 groupNumber, Snapshot snapsho * is (storage_id, group_number, column_index, vector_index), so this probes * the exact column. Returns NULL when the column has no whole-chunk zone map. */ +/* + * PgColumnarReadWholeChunkZoneMapsForColumn + * Every row group's whole-chunk zone map for one column, in ONE catalog scan + * keyed on the storage id (#1304 review). The planner's time-bucket bound needs + * all of them, and one probe per group cost about three buffers per group on + * every plan. Groups come back in zone_map_pkey order; a group with no row for + * the column is simply absent, so a caller needing all groups compares counts. + */ +List * +PgColumnarReadWholeChunkZoneMapsForColumn(uint64 storageId, int columnIndex, + Snapshot snapshot) +{ + Relation rel = open_columnar_table("zone_map", AccessShareLock); + TupleDesc tupdesc = RelationGetDescr(rel); + ScanKeyData key[1]; + SysScanDesc scan; + Oid idxOid; + HeapTuple tuple; + List *result = NIL; + + ScanKeyInit(&key[0], Anum_zone_map_storage_id, BTEqualStrategyNumber, + F_INT8EQ, Int64GetDatum((int64) storageId)); + idxOid = pgcolumnar_scan_index_oid(rel, "zone_map_pkey"); + scan = systable_beginscan(rel, idxOid, OidIsValid(idxOid), snapshot, 1, key); + while (HeapTupleIsValid(tuple = systable_getnext(scan))) + { + bool isnull; + int16 col = DatumGetInt16( + heap_getattr(tuple, Anum_zone_map_column_index, tupdesc, &isnull)); + int32 vecIndex = DatumGetInt32( + heap_getattr(tuple, Anum_zone_map_vector_index, tupdesc, &isnull)); + uint64 groupNumber; + + if (col != columnIndex || vecIndex != -1) + continue; /* this column's whole-chunk rows only */ + groupNumber = (uint64) DatumGetInt64( + heap_getattr(tuple, Anum_zone_map_group_number, tupdesc, &isnull)); + result = lappend(result, + zonemap_from_tuple(tuple, tupdesc, storageId, groupNumber, -1)); + } + systable_endscan(scan); + table_close(rel, AccessShareLock); + return result; +} + NativeZoneMapMetadata * PgColumnarReadZoneMapForColumn(uint64 storageId, uint64 groupNumber, int columnIndex, Snapshot snapshot, diff --git a/src/columnar_vector.c b/src/columnar_vector.c index 4baeb346..8363dc33 100644 --- a/src/columnar_vector.c +++ b/src/columnar_vector.c @@ -2041,6 +2041,86 @@ pgcolumnar_groupagg_outmap(List *exprs, List *groupKeys, Index scanrelid, * value is used to make a plan cheaper, so an under-estimate would * under-price the arm and under-size the Finalize's hash table. */ +/* + * pgcolumnar_zonemap_time_range + * The smallest and largest value of a timestamp column over every row group, + * from the zone maps (#1292). Returns false when any group that holds values + * has no usable zone map for the column -- a column added after the group was + * written, say -- because a range that leaves out one group is not a bound. + */ +static bool +pgcolumnar_zonemap_time_range(PlannerInfo *root, RelOptInfo *input_rel, Var *tsvar, + double *lo, double *hi) +{ + RangeTblEntry *rte = planner_rt_fetch(input_rel->relid, root); + Relation r; + TupleDesc td; + Form_pg_attribute att; + uint64 storageId; + Snapshot snap; + List *groups; + List *zones; + ListCell *lc; + bool any = false; + bool ok = true; + int col = tsvar->varattno - 1; + + if (rte == NULL || !OidIsValid(rte->relid) || tsvar->varattno <= 0) + return false; + r = table_open(rte->relid, AccessShareLock); + td = RelationGetDescr(r); + if (col >= td->natts) + { + table_close(r, AccessShareLock); + return false; + } + att = TupleDescAttr(td, col); + storageId = PgColumnarStorageId(r); + snap = PgColumnarCatalogSnapshot(GetActiveSnapshot()); + + /* + * TWO CATALOG SCANS, NOT ONE PER GROUP (#1304 review): the group list, for the + * count, and every group's zone map for this column. A group with no zone-map + * row for the column is missing from the second list, and a range that leaves + * out a group is not a bound, so the counts must agree. + */ + groups = PgColumnarReadRowGroupList(storageId, snap); + zones = PgColumnarReadWholeChunkZoneMapsForColumn(storageId, col, snap); + if (list_length(zones) != list_length(groups)) + ok = false; + + foreach(lc, zones) + { + NativeZoneMapMetadata *z = (NativeZoneMapMetadata *) lfirst(lc); + char *cur; + double vmin, + vmax; + + if (!ok) + break; + if (!z->hasMinMax) + { + if (z->valueCount == 0) + continue; /* all null: contributes no bucket */ + ok = false; + break; + } + cur = (char *) z->minimum; + vmin = (double) DatumGetTimestamp(PgColumnarDecodeValue(att, &cur, + z->minimum + z->minimumLen, CurrentMemoryContext)); + cur = (char *) z->maximum; + vmax = (double) DatumGetTimestamp(PgColumnarDecodeValue(att, &cur, + z->maximum + z->maximumLen, CurrentMemoryContext)); + *lo = any ? Min(*lo, vmin) : vmin; + *hi = any ? Max(*hi, vmax) : vmax; + any = true; + } + list_free_deep(groups); + list_free_deep(zones); + table_close(r, AccessShareLock); + return ok && any; +} + static double pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel, Node *expr) @@ -2079,6 +2159,36 @@ pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel, * deliberately: its argument is a duration rather than a point in time, so a * range on it does not describe a scanned window. */ + if (f->funcid == F_DATE_BIN_INTERVAL_TIMESTAMP_TIMESTAMP || + f->funcid == F_DATE_BIN_INTERVAL_TIMESTAMPTZ_TIMESTAMPTZ) + { + /* + * date_bin(stride, source, origin) (#1292): the same bucketing with the + * width given as an interval. A stride with months is not a fixed width, + * and date_bin itself refuses one, so it gets no bound here either. + */ + Const *stride; + Interval *iv; + + if (list_length(f->args) != 3 || !IsA(linitial(f->args), Const) || + !IsA(lsecond(f->args), Var)) + return 0.0; + stride = (Const *) linitial(f->args); + if (stride->constisnull || stride->consttype != INTERVALOID) + return 0.0; + iv = DatumGetIntervalP(stride->constvalue); + if (iv->month != 0) + return 0.0; + width = (double) iv->time + (double) iv->day * (double) USECS_PER_DAY; + if (width <= 0) + return 0.0; + tsvar = (Var *) lsecond(f->args); + if (tsvar->varno != input_rel->relid || tsvar->varlevelsup != 0) + return 0.0; + if (tsvar->vartype != TIMESTAMPOID && tsvar->vartype != TIMESTAMPTZOID) + return 0.0; + goto have_width; + } if (f->funcid != F_DATE_TRUNC_TEXT_TIMESTAMP && f->funcid != F_DATE_TRUNC_TEXT_TIMESTAMPTZ && f->funcid != F_DATE_TRUNC_TEXT_TIMESTAMPTZ_TEXT) @@ -2115,6 +2225,8 @@ pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel, else if (pg_strcasecmp(unitstr, "century") == 0) width = 36500.0 * USECS_PER_DAY; else if (pg_strcasecmp(unitstr, "millennium") == 0) width = 365000.0 * USECS_PER_DAY; else return 0.0; + +have_width: /* * month, quarter and year use their SHORTEST possible length above, so the * bucket count is over-estimated rather than under-estimated. This must stay @@ -2180,7 +2292,27 @@ pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel, } } - if (!havelo || !havehi || hi <= lo) + /* + * A MISSING END COMES FROM THE ZONE MAPS (#1292), and both ends are clipped to + * them when they are known. Every row group's whole-chunk zone map holds the + * minimum and maximum of the key's column, so together they bound every value + * the scan can read: deletes only shrink that, never widen it. All groups are + * read, not a sample -- a sampled maximum can miss the true one, which would + * UNDER-estimate, the direction a gate on this number must never take. + */ + { + double zlo, + zhi; + + if (pgcolumnar_zonemap_time_range(root, input_rel, tsvar, &zlo, &zhi)) + { + lo = havelo ? Max(lo, zlo) : zlo; + hi = havehi ? Min(hi, zhi) : zhi; + havelo = havehi = true; + } + } + + if (!havelo || !havehi || hi < lo) return 0.0; return floor((hi - lo) / width) + 2.0; } diff --git a/test/groupagg_table_sizing.sh b/test/groupagg_table_sizing.sh index e4b9f32e..708a2cb8 100755 --- a/test/groupagg_table_sizing.sh +++ b/test/groupagg_table_sizing.sh @@ -31,9 +31,10 @@ # # THE OVER-ESTIMATE ARM came out of review and is why the sizing happens on the # first grow rather than at Begin. estimate_num_groups cannot see through a -# function, so `GROUP BY date_trunc('day', ts)` over a high-cardinality timestamp -# reaches this node with an estimate of every row against a handful of real -# groups. Sizing at Begin allocated 131,072 entries for 47 groups: 128x the +# expression, so `GROUP BY v / 100000` over a high-cardinality column reaches +# this node with an estimate of every row against a handful of real groups. +# (The arm used date_trunc('day', ts) until #1304 bounded truncating time keys +# from the zone maps, which made that estimate right and this arm vacuous.) Sizing at Begin allocated 131,072 entries for 47 groups: 128x the # memory of growing from nothing, to save zero rehashing, on the most ordinary # grouping in analytics. A table that never outgrows its first 1024 entries must # never be charged for a wrong estimate. @@ -137,10 +138,10 @@ check "the table is never larger than the real group count needs (#810 review)" || echo "OVERSIZED ($CAP entries for $NGROUPS groups, which need $NEEDED)")" "bounded" # ---- the over-estimate arm (review of #810) ------------------------------- -EXPR_Q="SELECT date_trunc('day', ts) d, count(*) FROM gts_expr GROUP BY 1" +EXPR_Q="SELECT v / 100000 d, count(*) FROM gts_expr GROUP BY 1" EXPR_PLAN="$(explain_with "$BIG" "$EXPR_Q")" EXPR_CAP="$(field "$EXPR_PLAN" 'Columnar Group Table Capacity')" -EXPR_REAL="$(q1 "SELECT count(*) FROM (SELECT date_trunc('day', ts) FROM gts_expr GROUP BY 1) s")" +EXPR_REAL="$(q1 "SELECT count(*) FROM (SELECT v / 100000 FROM gts_expr GROUP BY 1) s")" EXPR_EST="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -Atq \ -c "$ON $BIG EXPLAIN (COSTS ON) $EXPR_Q" 2>&1 | sed -n 's/.*rows=\([0-9]*\).*width.*/\1/p' | head -1)" echo "-- expression key: planner estimate $EXPR_EST, real groups $EXPR_REAL, capacity ${EXPR_CAP:-0}" @@ -157,17 +158,17 @@ check "a wrong estimate cannot inflate a table that never needed to grow (#810 r || echo "INFLATED ($EXPR_CAP entries for $EXPR_REAL real groups)")" "not inflated" check "and the expression key still answers correctly" "$EXPR_REAL" \ - "$(q1 "SELECT count(DISTINCT date_trunc('day', ts)) FROM gts_expr")" + "$(q1 "SELECT count(DISTINCT v / 100000) FROM gts_expr")" # ---- the jump bound ------------------------------------------------------- # The arm above cannot see the jump bound: 6 groups fit in the starting 1024, so # the table never grows and no jump is taken. This one has enough real groups to # force a grow while keeping the estimate wildly wrong, which is the only shape # where the bound is observable. -MIN_Q="SELECT date_trunc('minute', ts) d, count(*) FROM gts_expr GROUP BY 1" +MIN_Q="SELECT v / 60 d, count(*) FROM gts_expr GROUP BY 1" MIN_PLAN="$(explain_with "$BIG" "$MIN_Q")" MIN_CAP="$(field "$MIN_PLAN" 'Columnar Group Table Capacity')" -MIN_REAL="$(q1 "SELECT count(*) FROM (SELECT date_trunc('minute', ts) FROM gts_expr GROUP BY 1) s")" +MIN_REAL="$(q1 "SELECT count(*) FROM (SELECT v / 60 FROM gts_expr GROUP BY 1) s")" MIN_NEED="$(awk -v g="${MIN_REAL:-1}" 'BEGIN { c = 1024; while (c * 0.7 < g) c *= 2; printf "%d", c }')" echo "-- minute key: real groups $MIN_REAL need $MIN_NEED entries; allocated ${MIN_CAP:-0}" @@ -181,7 +182,7 @@ check "a grow cannot jump to a wrong estimate, only toward it (#810 review)" \ 'BEGIN { print (c > 0 && c <= 8 * n) ? "bounded" : "UNBOUNDED (" c " for a need of " n ")" }')" "bounded" check "and the minute key answers correctly too" "$MIN_REAL" \ - "$(q1 "SELECT count(DISTINCT date_trunc('minute', ts)) FROM gts_expr")" + "$(q1 "SELECT count(DISTINCT v / 60) FROM gts_expr")" check "and the answer is the same under a small work_mem" \ "$(q1 "SET work_mem='1MB'; SELECT count(*) FROM (SELECT k, sum(v) FROM gts GROUP BY k) s")" "$NGROUPS" diff --git a/test/lossy_arms.tsv b/test/lossy_arms.tsv index 92053f53..2f99561d 100644 --- a/test/lossy_arms.tsv +++ b/test/lossy_arms.tsv @@ -136,7 +136,6 @@ native_gap gap was physically materialized native_groupagg a lookalike function with the same signature gets NO bound native_groupagg and it is an UPPER bound: the true group count is at or below it native_groupagg a non-time expression key gets no bound either -native_groupagg a truncating key with no range bound gets no bound from us native_reclaim_frag coalesce merges adjacent frees (fewer free_space rows) native_reclaim_frag coalesce never larger than whole-range reuse native_reclaim repeated recluster reuses space (file plateaus) diff --git a/test/native_groupagg.sh b/test/native_groupagg.sh index 6a90f399..8bbe41cf 100755 --- a/test/native_groupagg.sh +++ b/test/native_groupagg.sh @@ -450,10 +450,58 @@ plain_before=$(gbest "SELECT host, avg(v) FROM gb GROUP BY host") check "an informed estimate (plain column) is not replaced" \ "$([ "$plain_before" -ge 3900 ] && [ "$plain_before" -le 4100 ] && echo yes || echo no)" "yes" -# No Const bound on the key means no range, so no bound. The estimate stays whatever core -# produced; asserting it is LARGE is the point, since a bound would have shrunk it. -check "a truncating key with no range bound gets no bound from us" \ - "$([ "$(gbest "SELECT date_trunc('minute', ts) b, avg(v) FROM gb GROUP BY b")" -gt 1000 ] && echo yes || echo no)" "yes" +# A MISSING END OF THE RANGE COMES FROM THE ZONE MAPS (#1292). This arm used to assert +# the opposite -- "a truncating key with no range bound gets no bound from us" -- and that +# was the design until the owner chose, on #1292, to gate the grouped path on this +# estimate. A gate cannot use a number 60x to 3600x too high (measured on #1292), and a +# dashboard query bounds only one side ("the last day"). The zone maps hold every row +# group's minimum and maximum of the key's column, which is the range the scan reads. +# +# Each arm states BOTH directions: at least the true group count, because an +# under-estimate is the direction that under-sizes the hash table and would let the gate +# pass a query the runtime cap then refuses; and at most 4x, which is the accuracy a gate +# needs. Each reports the estimate and the truth when it fails. +gbwithin() { # gbwithin QUERY -> "ok" when actual <= estimate <= 4*actual + local est act + est="$(gbest "$1")" + act="$(q "SELECT count(*) FROM ($1) z;")" + if [ -n "$est" ] && [ -n "$act" ] && [ "$act" -ge 1 ] && [ "$est" -ge "$act" ] && [ "$est" -le $((act * 4)) ]; then + echo ok + else + echo "estimate ${est:-none} for ${act:-none} groups" + fi +} +check "with no time bound at all, a truncating key is bounded from the zone maps" \ + "$(gbwithin "SELECT date_trunc('minute', ts) b, avg(v) FROM gb GROUP BY b")" "ok" +check "with only a lower time bound, a truncating key is still bounded" \ + "$(gbwithin "SELECT date_trunc('minute', ts) b, avg(v) FROM gb WHERE ts >= '2026-01-01' GROUP BY b")" "ok" +check "a date_bin key is bounded as a truncating key is" \ + "$(gbwithin "SELECT date_bin('1 minute', ts, timestamptz '2026-01-01') b, avg(v) FROM gb $W GROUP BY b")" "ok" +check "and a date_bin key with no time bound is bounded from the zone maps" \ + "$(gbwithin "SELECT date_bin('5 minutes', ts, timestamptz '2026-01-01') b, avg(v) FROM gb GROUP BY b")" "ok" + +# THE RANGE COSTS ONE CATALOG SCAN, NOT ONE PER ROW GROUP (#1304 review). The first +# version probed zone_map_pkey once per group, on every plan: 109 -> 313 planning +# buffers at 67 groups, about 3 per group, so ~20,000 per plan at a billion rows. Counted +# as the planning transaction's own zone_map scans, which no other session can move (the +# #1287 instrument), on the table AND its index: a catalog index scan counts on the +# INDEX, and reading the table's counter alone gave 0 against 50 real probes. No WHERE, +# so the filter's survival estimate -- which samples zone +# maps on purpose -- is not in the count; 50 groups, so one scan and one per group are +# far apart. +psql_run "CREATE TABLE gbz (ts timestamptz, v float8) USING pgcolumnar; + SELECT pgcolumnar.set_options('gbz', stripe_row_limit => 1000, chunk_group_row_limit => 1000); + INSERT INTO gbz SELECT '2026-01-01'::timestamptz + g * interval '1 second', g FROM generate_series(1,50000) g; + ANALYZE gbz;" >/dev/null 2>&1 +gbz_groups="$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('gbz');")" +gbz_scans="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -At -q \ + -c "$GB" -c "BEGIN" -c "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM gbz GROUP BY b" \ + -c "SELECT pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass) + + pg_stat_get_xact_numscans('pgcolumnar.zone_map_pkey'::regclass)" -c "ROLLBACK" 2>&1 | grep -E '^[0-9]+$' | tail -1)" +check "premise: the fixture has many row groups, so one scan and one per group differ" \ + "$([ "${gbz_groups:-0}" -ge 40 ] && echo yes || echo "no (${gbz_groups:-none} groups)")" "yes" +check_num "planning a bucketed GROUP BY scans zone_map once, not once per row group" \ + "${gbz_scans:-}" "1" # A user function with date_trunc's SHAPE must not get date_trunc's BOUND. PL/pgSQL # because an SQL function would be inlined and lose the shape. Matching on shape alone # estimated this at 722 against 499,999 actual, a 692x UNDER-estimate, which is the @@ -475,9 +523,21 @@ psql_run "CREATE TABLE gbm (ts timestamptz, v float8) USING pgcolumnar; gbmest() { env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ -d "$PGC_DB" -At -q -c "SET TimeZone='Asia/Kolkata';" -c "$GB" \ -c "EXPLAIN (COSTS ON) $1" 2>&1 | grep -oE "rows=[0-9]+" | head -1 | cut -d= -f2; } -check "a mixed timestamp/timestamptz predicate gets NO bound" \ - "$([ "$(gbmest "SELECT date_trunc('minute',ts) b, avg(v) FROM gbm - WHERE ts >= timestamp '2026-01-01 00:00' AND ts < timestamp '2026-01-01 12:00' GROUP BY b")" -gt 50000 ] && echo yes || echo no)" "yes" +# WHAT THE MIXED-TYPE RULE PROTECTS, stated directly (#1292). This asserted "gets NO +# bound", which was the observable sign that the mixed predicate was not used. With the +# zone maps supplying a range, there is a bound either way, so the arm now asserts the +# property itself: a wall-clock window read as UTC must never narrow the estimate below +# the true group count. Under Asia/Kolkata, 04:00-08:00 wall clock is 22:30-02:30 UTC, +# 150 minutes of data; read as UTC it would clip the data to 04:00-04:48, about 50. +gbmcount() { env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -At -q -c "SET TimeZone='Asia/Kolkata';" -c "SELECT count(*) FROM ($1) z;" 2>&1; } +GBM_MIXED="SELECT date_trunc('minute',ts) b, avg(v) FROM gbm + WHERE ts >= timestamp '2026-01-01 04:00' AND ts < timestamp '2026-01-01 08:00' GROUP BY b" +gbm_est="$(gbmest "$GBM_MIXED")"; gbm_act="$(gbmcount "$GBM_MIXED")" +check "premise: the mixed-type window holds data" \ + "$([ "${gbm_act:-0}" -ge 100 ] && echo yes || echo "no (${gbm_act:-none} groups)")" "yes" +check "a mixed timestamp/timestamptz predicate never narrows the estimate below the true count" \ + "$([ "${gbm_est:-0}" -ge "${gbm_act:-1}" ] && echo yes || echo "no (estimate ${gbm_est:-none} for ${gbm_act:-none} groups)")" "yes" check "while a matching-type predicate still gets one" \ "$([ "$(gbmest "SELECT date_trunc('minute',ts) b, avg(v) FROM gbm WHERE ts >= timestamptz '2026-01-01 00:00+00' AND ts < timestamptz '2026-01-01 12:00+00' GROUP BY b")" -lt 5000 ] && echo yes || echo no)" "yes" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index a4437f9c..e2433eb6 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -5646,8 +5646,9 @@ every major for this pair, which is the designed third state and not a failure. | `test_avg_float8_overflows_the_way_core_overflows` | the node fires and raises out of range, which a row comparison cannot express | | `test_degenerate_inputs_produce_no_groups` | a predicate matching nothing, and an empty table | | `test_the_path_pays_for_the_folding_it_does` | the isolated #349 arm and its two node premises | -| `test_the_group_estimate_bound_is_accurate_and_narrow` | the bound is 12 and is an upper bound; an informed estimate is untouched; no range, a lookalike function and a non-time key each get no bound | -| `test_a_mixed_timestamp_predicate_gets_no_bound` | a cross-type predicate under a non-UTC TimeZone gets none, while the matching-type one still gets a bound | +| `test_the_group_estimate_bound_is_accurate_and_narrow` | the bound is 12 and is an upper bound; an informed estimate is untouched; with no time bound, with only a lower one, and for `date_bin`, the estimate is between the true group count and four times it, from the zone maps (#1292); a lookalike function and a non-time key get no bound | +| `test_the_bound_reads_the_zone_maps_in_one_catalog_scan` | planning a bucketed GROUP BY over 50 row groups scans zone_map and its index once in all, not once per group (#1304) | +| `test_a_mixed_timestamp_predicate_never_narrows_the_bound` | a cross-type predicate under a non-UTC TimeZone never narrows the estimate below the true group count (#1292); the matching-type one still gets a tight bound | ## 72. test_analyze_function.py: statistics collected by reading, not by sampling diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 019b63b3..df9bd7c6 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -551,4 +551,7 @@ guard_tests 413 # rebased tree, not added up. # 495 -> 496 for #1288: test_storage_row_repair.py. Re-derived by collection on # 038600e (after #1303): `496 tests collected`. -cluster_tests 497 +# +# cluster_tests 497 -> 498 for #1304's test_the_bound_reads_the_zone_maps_in_one_catalog_scan, +# collected over the cluster files on the tree rebased onto 503f080 (#1286). +cluster_tests 498 diff --git a/test/pytest/test_native_groupagg.py b/test/pytest/test_native_groupagg.py index 2bd1b3e5..560f1df4 100644 --- a/test/pytest/test_native_groupagg.py +++ b/test/pytest/test_native_groupagg.py @@ -617,12 +617,29 @@ def test_the_group_estimate_bound_is_accurate_and_narrow(pgc_conn, expect): expect.text("yes" if 3900 <= plain <= 4100 else f"no [{plain}]", "yes", "an informed estimate (plain column) is not replaced") - # No Const bound on the key means no range, so no bound. Asserting the estimate - # is LARGE is the point: a bound would have shrunk it. - unbounded = _estimate(pgc_conn, - "SELECT date_trunc('minute', ts) b, avg(v) FROM gb GROUP BY b") - expect.text("yes" if unbounded > 1000 else f"no [{unbounded}]", "yes", - "a truncating key with no range bound gets no bound from us") + # A MISSING END OF THE RANGE COMES FROM THE ZONE MAPS (#1292). This used to assert + # that a key with no Const bound gets no bound at all. The owner chose on #1292 to + # gate the grouped path on this estimate, and a gate cannot use one that is 60x to + # 3600x high (measured there), while a dashboard query bounds only one side. Each + # case asserts both directions: at least the true count, because under-estimating + # is the harmful direction, and at most four times it. + def within(sql): + est = _estimate(pgc_conn, sql) + act = int(_rows(pgc_conn, f"SELECT count(*) FROM ({sql}) z")[0][0]) + ok = act >= 1 and act <= est <= 4 * act + return "ok" if ok else f"estimate {est} for {act} groups" + + expect.text(within("SELECT date_trunc('minute', ts) b, avg(v) FROM gb GROUP BY b"), + "ok", "with no time bound at all, a truncating key is bounded from the zone maps") + expect.text(within("SELECT date_trunc('minute', ts) b, avg(v) FROM gb " + "WHERE ts >= '2026-01-01' GROUP BY b"), + "ok", "with only a lower time bound, a truncating key is still bounded") + expect.text(within("SELECT date_bin('1 minute', ts, timestamptz '2026-01-01') b, avg(v) " + f"FROM gb {GB_WINDOW} GROUP BY b"), + "ok", "a date_bin key is bounded as a truncating key is") + expect.text(within("SELECT date_bin('5 minutes', ts, timestamptz '2026-01-01') b, avg(v) " + "FROM gb GROUP BY b"), + "ok", "and a date_bin key with no time bound is bounded from the zone maps") # A user function with date_trunc's SHAPE must not get date_trunc's BOUND. # PL/pgSQL because an SQL function would be inlined and lose the shape. Matching @@ -641,7 +658,40 @@ def test_the_group_estimate_bound_is_accurate_and_narrow(pgc_conn, expect): "a non-time expression key gets no bound either") -def test_a_mixed_timestamp_predicate_gets_no_bound(pgc_conn, expect): +def test_the_bound_reads_the_zone_maps_in_one_catalog_scan(pgc_conn, expect): + """The range costs one catalog scan, not one per row group (#1304 review). + + The first version probed zone_map_pkey once per group, on every plan: 109 -> 313 + planning buffers at 67 groups, about 3 per group. Counted as the planning + transaction's own scans, which no other session can move, on the table AND its + index, because a catalog index scan counts on the index. No WHERE, so the + filter's survival estimate, which samples zone maps on purpose, is not counted. + """ + _exec(pgc_conn, "CREATE TABLE gbz (ts timestamptz, v float8) USING pgcolumnar") + _exec(pgc_conn, "SELECT pgcolumnar.set_options('gbz', stripe_row_limit => 1000, " + "chunk_group_row_limit => 1000)") + _exec(pgc_conn, "INSERT INTO gbz SELECT '2026-01-01'::timestamptz + g * interval '1 second', g " + "FROM generate_series(1,50000) g") + _exec(pgc_conn, "ANALYZE gbz") + _gb_session(pgc_conn) + + groups = int(_one(pgc_conn, "SELECT count(*) FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('gbz')")) + _exec(pgc_conn, "BEGIN") + try: + _exec(pgc_conn, "EXPLAIN SELECT date_trunc('minute', ts) b, avg(v) FROM gbz GROUP BY b") + scans = int(_one(pgc_conn, + "SELECT pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass) " + "+ pg_stat_get_xact_numscans('pgcolumnar.zone_map_pkey'::regclass)")) + finally: + _exec(pgc_conn, "ROLLBACK") + print(f"-- {groups} row groups, {scans} zone_map scans while planning") + expect.at_least(groups, 40, + "premise: the fixture has many row groups, so one scan and one per group differ") + expect.num(scans, 1, "planning a bucketed GROUP BY scans zone_map once, not once per row group") + + +def test_a_mixed_timestamp_predicate_never_narrows_the_bound(pgc_conn, expect): """PostgreSQL has cross-type operators, so a `timestamp` Const compared against a `timestamptz` column stays bare and reaches the extraction instead of being wrapped in a cast. Both are int64 microseconds on different scales -- wall clock @@ -657,18 +707,26 @@ def test_a_mixed_timestamp_predicate_gets_no_bound(pgc_conn, expect): _exec(pgc_conn, "SET TimeZone='Asia/Kolkata'") _gb_session(pgc_conn) - mixed = _estimate(pgc_conn, - "SELECT date_trunc('minute',ts) b, avg(v) FROM gbm " - "WHERE ts >= timestamp '2026-01-01 00:00' " - "AND ts < timestamp '2026-01-01 12:00' GROUP BY b") + # THE PROPERTY THE MIXED-TYPE RULE PROTECTS, stated directly (#1292). This used to + # assert "gets NO bound", the observable sign that the mixed predicate was unused. + # With the zone maps supplying a range there is a bound either way, so the arm now + # asserts that a wall-clock window read as UTC never narrows the estimate below the + # truth. 04:00-08:00 Kolkata is 22:30-02:30 UTC, 150 minutes of this data; read as + # UTC it would clip the data to 04:00-04:48, about 50. + mixed_sql = ("SELECT date_trunc('minute',ts) b, avg(v) FROM gbm " + "WHERE ts >= timestamp '2026-01-01 04:00' " + "AND ts < timestamp '2026-01-01 08:00' GROUP BY b") + mixed = _estimate(pgc_conn, mixed_sql) + mixed_actual = int(_rows(pgc_conn, f"SELECT count(*) FROM ({mixed_sql}) z")[0][0]) matching = _estimate(pgc_conn, "SELECT date_trunc('minute',ts) b, avg(v) FROM gbm " "WHERE ts >= timestamptz '2026-01-01 00:00+00' " "AND ts < timestamptz '2026-01-01 12:00+00' GROUP BY b") - print(f"-- mixed-type predicate {mixed}, matching-type predicate {matching}") - # The two arms are a pair: "no bound" and "a bound" have to be distinguishable - # on this fixture, and core's own estimate here is six figures, so they are. - expect.text("yes" if mixed > 50000 else f"no [{mixed}]", "yes", - "a mixed timestamp/timestamptz predicate gets NO bound") + print(f"-- mixed-type predicate {mixed} for {mixed_actual} groups, " + f"matching-type predicate {matching}") + expect.at_least(mixed_actual, 100, "premise: the mixed-type window holds data") + expect.text("yes" if mixed >= mixed_actual + else f"no (estimate {mixed} for {mixed_actual} groups)", "yes", + "a mixed timestamp/timestamptz predicate never narrows the estimate below the true count") expect.text("yes" if matching < 5000 else f"no [{matching}]", "yes", "while a matching-type predicate still gets one")