opt: estimate time-bucket groups from the zone maps, and recognise date_bin (#1292)

CHANGELOG.md

@@ -74,6 +74,18 @@ true until the next version shipped.
 
 ### Changed
 
+- The planner estimates the number of groups for a time-bucket key from the
+  table's own data (#1292). `GROUP BY date_trunc(unit, ts)` and, newly,
+  `GROUP BY date_bin(stride, ts, origin)` were estimated near the row count
+  unless the query bounded `ts` on both sides: 300,000 groups for 11 on the test
+  fixture, and 60x to 3,600x too high on #1292's measurements. A missing end of
+  the range now comes from the zone maps, the smallest and largest value of `ts`
+  in every row group, so a query bounded on one side only, or not at all, is
+  estimated within a small factor of the truth and never below it. This is
+  the first step towards turning the vectorized grouping path on by default.
+  A `timestamp` constant compared against a `timestamptz` column still does not
+  set the range.
+
 - Five selftest parts and `rewrite_storage_oid` no longer hold a check that has
   never been red (#1236). Nine checks now carry a mutation that reddens them,
   each measured on all five majors: 570's fold arm, two in 290, one in 560, three

src/columnar.h

@@ -606,6 +606,8 @@ extern NativeZoneMapMetadata *PgColumnarReadZoneMapForColumn(uint64 storageId,
 									 Snapshot snapshot,
 									 PgColumnarZoneMapSession *sess);
 extern void PgColumnarCloseZoneMapSession(PgColumnarZoneMapSession *sess);
+extern List *PgColumnarReadWholeChunkZoneMapsForColumn(uint64 storageId, int columnIndex,
+													   Snapshot snapshot);
 extern void PgColumnarDeleteMetadata(uint64 storageId);
 
 /* declared retention, read by pgcolumnar.expire (#403 item 5a) */

src/columnar_metadata.c

@@ -3506,6 +3506,51 @@ PgColumnarReadZoneMapList(uint64 storageId, uint64 groupNumber, Snapshot snapsho
  *		is (storage_id, group_number, column_index, vector_index), so this probes
  *		the exact column. Returns NULL when the column has no whole-chunk zone map.
  */
+/*
+ * PgColumnarReadWholeChunkZoneMapsForColumn
+ *		Every row group's whole-chunk zone map for one column, in ONE catalog scan
+ *		keyed on the storage id (#1304 review). The planner's time-bucket bound needs
+ *		all of them, and one probe per group cost about three buffers per group on
+ *		every plan. Groups come back in zone_map_pkey order; a group with no row for
+ *		the column is simply absent, so a caller needing all groups compares counts.
+ */
+List *
+PgColumnarReadWholeChunkZoneMapsForColumn(uint64 storageId, int columnIndex,
+										  Snapshot snapshot)
+{
+	Relation	rel = open_columnar_table("zone_map", AccessShareLock);
+	TupleDesc	tupdesc = RelationGetDescr(rel);
+	ScanKeyData key[1];
+	SysScanDesc scan;
+	Oid			idxOid;
+	HeapTuple	tuple;
+	List	   *result = NIL;
+
+	ScanKeyInit(&key[0], Anum_zone_map_storage_id, BTEqualStrategyNumber,
+				F_INT8EQ, Int64GetDatum((int64) storageId));
+	idxOid = pgcolumnar_scan_index_oid(rel, "zone_map_pkey");
+	scan = systable_beginscan(rel, idxOid, OidIsValid(idxOid), snapshot, 1, key);
+	while (HeapTupleIsValid(tuple = systable_getnext(scan)))
+	{
+		bool		isnull;
+		int16		col = DatumGetInt16(
+			heap_getattr(tuple, Anum_zone_map_column_index, tupdesc, &isnull));
+		int32		vecIndex = DatumGetInt32(
+			heap_getattr(tuple, Anum_zone_map_vector_index, tupdesc, &isnull));
+		uint64		groupNumber;
+
+		if (col != columnIndex || vecIndex != -1)
+			continue;			/* this column's whole-chunk rows only */
+		groupNumber = (uint64) DatumGetInt64(
+			heap_getattr(tuple, Anum_zone_map_group_number, tupdesc, &isnull));
+		result = lappend(result,
+						 zonemap_from_tuple(tuple, tupdesc, storageId, groupNumber, -1));
+	}
+	systable_endscan(scan);
+	table_close(rel, AccessShareLock);
+	return result;
+}
+
 NativeZoneMapMetadata *
 PgColumnarReadZoneMapForColumn(uint64 storageId, uint64 groupNumber,
 							   int columnIndex, Snapshot snapshot,

src/columnar_vector.c

@@ -2041,6 +2041,86 @@ pgcolumnar_groupagg_outmap(List *exprs, List *groupKeys, Index scanrelid,
  *		value is used to make a plan cheaper, so an under-estimate would
  *		under-price the arm and under-size the Finalize's hash table.
  */
+/*
+ * pgcolumnar_zonemap_time_range
+ *		The smallest and largest value of a timestamp column over every row group,
+ *		from the zone maps (#1292). Returns false when any group that holds values
+ *		has no usable zone map for the column -- a column added after the group was
+ *		written, say -- because a range that leaves out one group is not a bound.
+ */
+static bool
+pgcolumnar_zonemap_time_range(PlannerInfo *root, RelOptInfo *input_rel, Var *tsvar,
+							  double *lo, double *hi)
+{
+	RangeTblEntry *rte = planner_rt_fetch(input_rel->relid, root);
+	Relation	r;
+	TupleDesc	td;
+	Form_pg_attribute att;
+	uint64		storageId;
+	Snapshot	snap;
+	List	   *groups;
+	List	   *zones;
+	ListCell   *lc;
+	bool		any = false;
+	bool		ok = true;
+	int			col = tsvar->varattno - 1;
+
+	if (rte == NULL || !OidIsValid(rte->relid) || tsvar->varattno <= 0)
+		return false;
+	r = table_open(rte->relid, AccessShareLock);
+	td = RelationGetDescr(r);
+	if (col >= td->natts)
+	{
+		table_close(r, AccessShareLock);
+		return false;
+	}
+	att = TupleDescAttr(td, col);
+	storageId = PgColumnarStorageId(r);
+	snap = PgColumnarCatalogSnapshot(GetActiveSnapshot());
+
+	/*
+	 * TWO CATALOG SCANS, NOT ONE PER GROUP (#1304 review): the group list, for the
+	 * count, and every group's zone map for this column. A group with no zone-map
+	 * row for the column is missing from the second list, and a range that leaves
+	 * out a group is not a bound, so the counts must agree.
+	 */
+	groups = PgColumnarReadRowGroupList(storageId, snap);
+	zones = PgColumnarReadWholeChunkZoneMapsForColumn(storageId, col, snap);
+	if (list_length(zones) != list_length(groups))
+		ok = false;
+
+	foreach(lc, zones)
+	{
+		NativeZoneMapMetadata *z = (NativeZoneMapMetadata *) lfirst(lc);
+		char	   *cur;
+		double		vmin,
+					vmax;
+
+		if (!ok)
+			break;
+		if (!z->hasMinMax)
+		{
+			if (z->valueCount == 0)
+				continue;		/* all null: contributes no bucket */
+			ok = false;
+			break;
+		}
+		cur = (char *) z->minimum;
+		vmin = (double) DatumGetTimestamp(PgColumnarDecodeValue(att, &cur,
+										z->minimum + z->minimumLen, CurrentMemoryContext));
+		cur = (char *) z->maximum;
+		vmax = (double) DatumGetTimestamp(PgColumnarDecodeValue(att, &cur,
+										z->maximum + z->maximumLen, CurrentMemoryContext));
+		*lo = any ? Min(*lo, vmin) : vmin;
+		*hi = any ? Max(*hi, vmax) : vmax;
+		any = true;
+	}
+	list_free_deep(groups);
+	list_free_deep(zones);
+	table_close(r, AccessShareLock);
+	return ok && any;
+}
+
 static double
 pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel,
 							   Node *expr)
@@ -2079,6 +2159,36 @@ pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel,
 	 * deliberately: its argument is a duration rather than a point in time, so a
 	 * range on it does not describe a scanned window.
 	 */
+	if (f->funcid == F_DATE_BIN_INTERVAL_TIMESTAMP_TIMESTAMP ||
+		f->funcid == F_DATE_BIN_INTERVAL_TIMESTAMPTZ_TIMESTAMPTZ)
+	{
+		/*
+		 * date_bin(stride, source, origin) (#1292): the same bucketing with the
+		 * width given as an interval. A stride with months is not a fixed width,
+		 * and date_bin itself refuses one, so it gets no bound here either.
+		 */
+		Const	   *stride;
+		Interval   *iv;
+
+		if (list_length(f->args) != 3 || !IsA(linitial(f->args), Const) ||
+			!IsA(lsecond(f->args), Var))
+			return 0.0;
+		stride = (Const *) linitial(f->args);
+		if (stride->constisnull || stride->consttype != INTERVALOID)
+			return 0.0;
+		iv = DatumGetIntervalP(stride->constvalue);
+		if (iv->month != 0)
+			return 0.0;
+		width = (double) iv->time + (double) iv->day * (double) USECS_PER_DAY;
+		if (width <= 0)
+			return 0.0;
+		tsvar = (Var *) lsecond(f->args);
+		if (tsvar->varno != input_rel->relid || tsvar->varlevelsup != 0)
+			return 0.0;
+		if (tsvar->vartype != TIMESTAMPOID && tsvar->vartype != TIMESTAMPTZOID)
+			return 0.0;
+		goto have_width;
+	}
 	if (f->funcid != F_DATE_TRUNC_TEXT_TIMESTAMP &&
 		f->funcid != F_DATE_TRUNC_TEXT_TIMESTAMPTZ &&
 		f->funcid != F_DATE_TRUNC_TEXT_TIMESTAMPTZ_TEXT)
@@ -2115,6 +2225,8 @@ pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel,
 	else if (pg_strcasecmp(unitstr, "century") == 0)		width = 36500.0 * USECS_PER_DAY;
 	else if (pg_strcasecmp(unitstr, "millennium") == 0)		width = 365000.0 * USECS_PER_DAY;
 	else return 0.0;
+
+have_width:
 	/*
 	 * month, quarter and year use their SHORTEST possible length above, so the
 	 * bucket count is over-estimated rather than under-estimated. This must stay
@@ -2180,7 +2292,27 @@ pgcolumnar_truncating_time_bound(PlannerInfo *root, RelOptInfo *input_rel,
 		}
 	}
 
-	if (!havelo || !havehi || hi <= lo)
+	/*
+	 * A MISSING END COMES FROM THE ZONE MAPS (#1292), and both ends are clipped to
+	 * them when they are known. Every row group's whole-chunk zone map holds the
+	 * minimum and maximum of the key's column, so together they bound every value
+	 * the scan can read: deletes only shrink that, never widen it. All groups are
+	 * read, not a sample -- a sampled maximum can miss the true one, which would
+	 * UNDER-estimate, the direction a gate on this number must never take.
+	 */
+	{
+		double		zlo,
+					zhi;
+
+		if (pgcolumnar_zonemap_time_range(root, input_rel, tsvar, &zlo, &zhi))
+		{
+			lo = havelo ? Max(lo, zlo) : zlo;
+			hi = havehi ? Min(hi, zhi) : zhi;
+			havelo = havehi = true;
+		}
+	}
+
+	if (!havelo || !havehi || hi < lo)
 		return 0.0;
 	return floor((hi - lo) / width) + 2.0;
 }

test/groupagg_table_sizing.sh

@@ -31,9 +31,10 @@
 #
 # THE OVER-ESTIMATE ARM came out of review and is why the sizing happens on the
 # first grow rather than at Begin. estimate_num_groups cannot see through a
-# function, so `GROUP BY date_trunc('day', ts)` over a high-cardinality timestamp
-# reaches this node with an estimate of every row against a handful of real
-# groups. Sizing at Begin allocated 131,072 entries for 47 groups: 128x the
+# expression, so `GROUP BY v / 100000` over a high-cardinality column reaches
+# this node with an estimate of every row against a handful of real groups.
+# (The arm used date_trunc('day', ts) until #1304 bounded truncating time keys
+# from the zone maps, which made that estimate right and this arm vacuous.) Sizing at Begin allocated 131,072 entries for 47 groups: 128x the
 # memory of growing from nothing, to save zero rehashing, on the most ordinary
 # grouping in analytics. A table that never outgrows its first 1024 entries must
 # never be charged for a wrong estimate.
@@ -137,10 +138,10 @@ check "the table is never larger than the real group count needs (#810 review)"
 	   || echo "OVERSIZED ($CAP entries for $NGROUPS groups, which need $NEEDED)")" "bounded"
 
 # ---- the over-estimate arm (review of #810) -------------------------------
-EXPR_Q="SELECT date_trunc('day', ts) d, count(*) FROM gts_expr GROUP BY 1"
+EXPR_Q="SELECT v / 100000 d, count(*) FROM gts_expr GROUP BY 1"
 EXPR_PLAN="$(explain_with "$BIG" "$EXPR_Q")"
 EXPR_CAP="$(field "$EXPR_PLAN" 'Columnar Group Table Capacity')"
-EXPR_REAL="$(q1 "SELECT count(*) FROM (SELECT date_trunc('day', ts) FROM gts_expr GROUP BY 1) s")"
+EXPR_REAL="$(q1 "SELECT count(*) FROM (SELECT v / 100000 FROM gts_expr GROUP BY 1) s")"
 EXPR_EST="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -Atq \
 	-c "$ON $BIG EXPLAIN (COSTS ON) $EXPR_Q" 2>&1 | sed -n 's/.*rows=\([0-9]*\).*width.*/\1/p' | head -1)"
 echo "-- expression key: planner estimate $EXPR_EST, real groups $EXPR_REAL, capacity ${EXPR_CAP:-0}"
@@ -157,17 +158,17 @@ check "a wrong estimate cannot inflate a table that never needed to grow (#810 r
 	   || echo "INFLATED ($EXPR_CAP entries for $EXPR_REAL real groups)")" "not inflated"
 
 check "and the expression key still answers correctly" "$EXPR_REAL" \
-	"$(q1 "SELECT count(DISTINCT date_trunc('day', ts)) FROM gts_expr")"
+	"$(q1 "SELECT count(DISTINCT v / 100000) FROM gts_expr")"
 
 # ---- the jump bound -------------------------------------------------------
 # The arm above cannot see the jump bound: 6 groups fit in the starting 1024, so
 # the table never grows and no jump is taken. This one has enough real groups to
 # force a grow while keeping the estimate wildly wrong, which is the only shape
 # where the bound is observable.
-MIN_Q="SELECT date_trunc('minute', ts) d, count(*) FROM gts_expr GROUP BY 1"
+MIN_Q="SELECT v / 60 d, count(*) FROM gts_expr GROUP BY 1"
 MIN_PLAN="$(explain_with "$BIG" "$MIN_Q")"
 MIN_CAP="$(field "$MIN_PLAN" 'Columnar Group Table Capacity')"
-MIN_REAL="$(q1 "SELECT count(*) FROM (SELECT date_trunc('minute', ts) FROM gts_expr GROUP BY 1) s")"
+MIN_REAL="$(q1 "SELECT count(*) FROM (SELECT v / 60 FROM gts_expr GROUP BY 1) s")"
 MIN_NEED="$(awk -v g="${MIN_REAL:-1}" 'BEGIN { c = 1024; while (c * 0.7 < g) c *= 2; printf "%d", c }')"
 echo "-- minute key: real groups $MIN_REAL need $MIN_NEED entries; allocated ${MIN_CAP:-0}"
 
@@ -181,7 +182,7 @@ check "a grow cannot jump to a wrong estimate, only toward it (#810 review)" \
 	   'BEGIN { print (c > 0 && c <= 8 * n) ? "bounded" : "UNBOUNDED (" c " for a need of " n ")" }')" "bounded"
 
 check "and the minute key answers correctly too" "$MIN_REAL" \
-	"$(q1 "SELECT count(DISTINCT date_trunc('minute', ts)) FROM gts_expr")"
+	"$(q1 "SELECT count(DISTINCT v / 60) FROM gts_expr")"
 
 check "and the answer is the same under a small work_mem" \
 	"$(q1 "SET work_mem='1MB'; SELECT count(*) FROM (SELECT k, sum(v) FROM gts GROUP BY k) s")" "$NGROUPS"

test/lossy_arms.tsv

@@ -136,7 +136,6 @@ native_gap	gap was physically materialized
 native_groupagg	a lookalike function with the same signature gets NO bound
 native_groupagg	and it is an UPPER bound: the true group count is at or below it
 native_groupagg	a non-time expression key gets no bound either
-native_groupagg	a truncating key with no range bound gets no bound from us
 native_reclaim_frag	coalesce merges adjacent frees (fewer free_space rows)
 native_reclaim_frag	coalesce never larger than whole-range reuse
 native_reclaim	repeated recluster reuses space (file plateaus)