From b9af0bf6f80d6b098921571229a5e37c5c80c362 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 12:55:40 +0000 Subject: [PATCH 01/11] test: rollups phase 2, dirty buckets, red first (#1294) A delete, an update or a rewrite in a rollup's source must make the buckets of the changed row groups dirty, and a refresh must recompute them from the source and leave every other partial alone. Retained history is never recomputed: a refresh that would recompute a bucket holding a retired group's rows is refused with 0A000, and the rollup is unchanged. Arms, in both twins: delete, an update that moves rows across buckets, a delete plus an append straddling its bucket, compact_rewrite, a deleted NULL-ts row, the two history cases (refused, and folding next to history), history retired by the same refresh, a DST day, and the record kept tidy (recount, forget, retired groups dropped). On main (PG17) the shell twin is 7 red, each 0A000 or a stale rollup. Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_rollup.py | 141 +++++++++++++++++++++++++++++++++++-- test/rollup.sh | 110 +++++++++++++++++++++++++++-- 2 files changed, 240 insertions(+), 11 deletions(-) diff --git a/test/pytest/test_rollup.py b/test/pytest/test_rollup.py index 84143bbfb..8ce3e9235 100644 --- a/test/pytest/test_rollup.py +++ b/test/pytest/test_rollup.py @@ -78,6 +78,15 @@ def _refresh(conn, rollup="gauge_15m"): return _state(conn, f"SELECT pgcolumnar.refresh_rollup('{rollup}')") +def _folds(conn, rollup="gauge_15m"): + # "folded", or the SQLSTATE: a count of five digits would read as one. + try: + _one(conn, f"SELECT pgcolumnar.refresh_rollup('{rollup}')") + return "folded" + except psycopg.Error as e: + return e.sqlstate or "?" + + def test_rollup_phase_one(pgc_own_db, pgc_cluster, expect): conn = pgc_own_db dsn = pgc_cluster.dsn(dbname=_one(conn, "SELECT current_database()")) @@ -201,13 +210,53 @@ def identity(): expect.text(_refresh(conn), "600", "after a rebase the next refresh folds exactly the appended rows") expect.text(_matches(conn), _oracle(conn), "and after the rebase the rollup equals a full recompute") - # Phase 1 refuses anything but appends. - before = _matches(conn) - expect.at_least(_one(conn, "SELECT count(*) FROM gauge_15m"), 1, "the rollup holds rows before the delete") + # Phase 2: a delete, update or rewrite makes the buckets of the changed groups + # dirty; the refresh recomputes them and leaves every other partial alone (a + # recomputed partial is a new row, so a new ctid). The groups were last + # recorded by rebase_rollup, so this also needs rebase to record time ranges. + far = "bucket >= '2026-02-04 00:00:00+00'" # past every group the delete reaches + at_far = ("SELECT count(*) || '|' || md5(string_agg(ctid::text, ',' ORDER BY ctid))" + f" FROM gauge_15m__partials WHERE {far}") + far_before = _one(conn, at_far) + expect.at_least(_one(conn, f"SELECT count(*) FROM gauge_15m__partials WHERE {far}"), 1, + "premise: the rollup holds partials the delete below cannot reach") with conn.cursor() as cur: cur.execute("DELETE FROM feed WHERE sensor = 4 AND at < '2026-02-01 01:00:00+00'") - expect.text(_refresh(conn), "0A000", "a refresh after a delete is refused until phase 2") - expect.text(_matches(conn), before, "and the refused refresh changed nothing") + expect.text(_folds(conn), "folded", + "a refresh after a delete is not refused") + expect.text(_matches(conn), _oracle(conn), "and after a delete the rollup equals a full recompute") + expect.text(_one(conn, at_far), far_before, + "and the refresh rewrote no partial outside the changed groups' time range") + expect.text(_refresh(conn), "0", "and the next refresh, with nothing changed since, recomputes nothing") + + with conn.cursor() as cur: + cur.execute("UPDATE feed SET at = at + interval '3 days 7 hours'" + " WHERE at >= '2026-02-02 08:00:00+00' AND at < '2026-02-02 08:10:00+00'") + _refresh(conn) + expect.text(_matches(conn), _oracle(conn), + "after an update that moves rows to another bucket the rollup equals a full recompute") + + # A dirty bucket and a new group in one refresh: the recompute already counts + # the new rows in the dirty bucket. 150 rows an hour apart outrun any group. + with conn.cursor() as cur: + cur.execute("DELETE FROM feed WHERE sensor = 2 AND at >= '2026-02-02 20:00:00+00'" + " AND at < '2026-02-02 20:15:00+00'") + cur.execute("INSERT INTO feed SELECT '2026-02-02 20:05:00+00'::timestamptz + g * interval '1 hour'," + " 2, g * 3, g / 7.0 FROM generate_series(0, 149) g") + _refresh(conn) + expect.text(_matches(conn), _oracle(conn), + "after a delete and an append that straddles its bucket, a refresh counts the new rows once") + + with conn.cursor() as cur: + cur.execute("DELETE FROM feed WHERE at >= '2026-02-05 00:00:00+00' AND at < '2026-02-05 09:00:00+00'") + try: + rewrote = _one(conn, "SELECT pgcolumnar.compact_rewrite('feed', 0.02)") + except psycopg.Error: + rewrote = -1 + expect.at_least(rewrote, 1, "premise: compact_rewrite rewrote a row group") + _refresh(conn) + expect.text(_matches(conn), _oracle(conn), "after compact_rewrite the rollup equals a full recompute") + expect.text(_refresh(conn), "0", "and the refresh after the rewrite recomputes nothing") # The TID range of each row group comes from the access method's own mapping # (row_tid), so it holds exactly the group's rows at any block size. @@ -243,6 +292,11 @@ def identity(): "a rollup with NULL keys and NULL timestamps equals a full recompute") expect.at_least(_one(conn, f"SELECT count(*) FROM ({holes_recompute}) z WHERE sensor IS NULL OR bucket IS NULL"), 2, "premise: and the fixture has NULL-key and NULL-bucket groups") + with conn.cursor() as cur: + cur.execute("DELETE FROM holes WHERE at IS NULL AND sensor = 1") + _refresh(conn, "holes_15m") + expect.text(_digest(conn, "SELECT * FROM holes_15m"), _digest(conn, holes_recompute), + "after a delete of rows whose ts is NULL the NULL-bucket rollup equals a full recompute") # Retention keeps history: groups expire retires (pgcolumnar.retired_group, # #1314) leave the snapshot, and their buckets keep what they folded. @@ -265,6 +319,10 @@ def identity(): expect.at_least(_one(conn, "SELECT pgcolumnar.expire('aged')"), 1, "premise: expire retired the old row groups") expect.text(f"{_refresh(conn, 'aged_hourly')}|{_one(conn, hist)}", f"0|{before_hist}", "a refresh after retention folds nothing, refuses nothing, and keeps every bucket") + expect.num(_one(conn, "SELECT count(*) FROM pgcolumnar.rollup_group rg JOIN pgcolumnar.retired_group t" + " ON t.group_number = rg.group_number AND t.storage_id = pgcolumnar.get_storage_id('aged')" + " WHERE rg.rollup = 'aged_hourly'::regclass"), 0, + "and the rollup no longer records the groups retention retired") with conn.cursor() as cur: cur.execute("INSERT INTO aged SELECT now() - interval '5 hours' + g * interval '3 s', g % 83" " FROM generate_series(1, 700) g") @@ -278,6 +336,79 @@ def identity(): expect.text(f"{recent}|{still_old}", f"{recomputed}|{old_buckets}", "and the recent buckets equal a recompute, while the expired ones keep their history") + # History and dirty buckets: a bucket that also holds a retired group's rows + # cannot be recomputed, so that refresh is refused and changes nothing; a + # delete whose buckets hold no history folds. One group per INSERT: OLD + # retires, KEPT shares OLD's hour H and reaches past the cutoff, LATE is + # days later. + h = f"date_bin('1 hour', now() - interval '20 days', '{ORIGIN}')" + + def layer(table): + with conn.cursor() as cur: + cur.execute(f"CREATE TABLE {table} (at timestamptz, reading int) USING pgcolumnar") + cur.execute(f"SELECT pgcolumnar.set_options('{table}', stripe_row_limit => 5000)") + cur.execute(f"INSERT INTO {table} SELECT {h} + g * interval '7 s', g FROM generate_series(1, 300) g") + cur.execute(f"INSERT INTO {table} SELECT {h} + interval '40 min' + g * interval '10 min', g" + " FROM generate_series(1, 1000) g") + cur.execute(f"INSERT INTO {table} SELECT {h} + interval '12 days' + g * interval '1 min', g" + " FROM generate_series(1, 900) g") + cur.execute(f"SELECT pgcolumnar.create_rollup('{table}_hourly', '{table}', 'at', '1 hour', '{ORIGIN}'," + " '{}', '{count,max}', '{*,reading}')") + cur.execute(f"SELECT pgcolumnar.refresh_rollup('{table}_hourly')") + cur.execute(f"SELECT pgcolumnar.set_options('{table}', ttl_column => 'at', ttl_interval => '14 days')") + cur.execute(f"SELECT pgcolumnar.expire('{table}')") + layer("layered") + expect.text(_one(conn, f"SELECT count(*) FILTER (WHERE at < {h} + interval '40 min') || '|'" + f" || count(*) FILTER (WHERE at >= {h} + interval '40 min' AND at < {h} + interval '1 hour')" + " FROM layered"), "0|1", + "premise: expire retired the group in bucket D alone and kept the one that shares it") + lq = "SELECT md5(string_agg(bucket || ':' || count || ':' || max_reading, ',' ORDER BY bucket)) FROM layered_hourly" + lr = (f"SELECT md5(string_agg(b || ':' || n || ':' || m, ',' ORDER BY b)) FROM (SELECT date_bin('1 hour', at," + f" '{ORIGIN}') b, count(*) n, max(reading) m FROM layered GROUP BY 1) x") + with conn.cursor() as cur: + cur.execute(f"DELETE FROM layered WHERE at >= {h} + interval '12 days 5 hours' AND reading % 3 = 0") + expect.text(f"{_folds(conn, 'layered_hourly')}|{_one(conn, f'{lq} WHERE bucket > {h}')}" + f"|{_one(conn, f'SELECT count || chr(58) || max_reading FROM layered_hourly WHERE bucket = {h}')}", + f"folded|{_one(conn, f'{lr} WHERE b > {h}')}|301:300", + "a delete in buckets with no history, next to retired ones, folds: " + "those buckets equal a recompute and D keeps its history") + before_layered = _one(conn, lq) + with conn.cursor() as cur: + cur.execute(f"DELETE FROM layered WHERE at >= {h} + interval '2 hours' AND at < {h} + interval '3 hours'") + expect.text(f"{_refresh(conn, 'layered_hourly')}|{_one(conn, lq)}", f"0A000|{before_layered}", + "a delete in a bucket that holds retired history is refused, and the rollup is unchanged") + + # No refresh between expire and the delete: the same refresh retires the + # history that blocks it. + layer("relayered") + rq = lq.replace("layered_hourly", "relayered_hourly") + before_re = _one(conn, rq) + with conn.cursor() as cur: + cur.execute(f"DELETE FROM relayered WHERE at >= {h} + interval '2 hours' AND at < {h} + interval '3 hours'") + expect.text(f"{_refresh(conn, 'relayered_hourly')}|{_one(conn, rq)}", f"0A000|{before_re}", + "a delete in a bucket whose history the same refresh retires is refused, and the rollup is unchanged") + + # A day is 24 hours to date_bin in every zone. In Europe/Berlin 2026-03-29 is + # 23 hours long; the UTC day bucket must be recomputed to its end (rows run to + # 23:40 UTC). + with conn.cursor() as cur: + cur.execute("CREATE TABLE springs (at timestamptz, reading int) USING pgcolumnar") + cur.execute("SELECT pgcolumnar.set_options('springs', stripe_row_limit => 4000)") + cur.execute("INSERT INTO springs SELECT '2026-03-28 00:00:00+00'::timestamptz + g * interval '4 min', g" + " FROM generate_series(1, 715) g") + cur.execute(f"SELECT pgcolumnar.create_rollup('springs_daily', 'springs', 'at', '1 day', '{ORIGIN}'," + " '{}', '{count,max}', '{*,reading}')") + cur.execute("SELECT pgcolumnar.refresh_rollup('springs_daily')") + cur.execute("DELETE FROM springs WHERE reading = 2") + cur.execute("SET TimeZone = 'Europe/Berlin'") + cur.execute("SELECT pgcolumnar.refresh_rollup('springs_daily')") + cur.execute("SET TimeZone = 'UTC'") + expect.text(_one(conn, "SELECT string_agg(bucket || ':' || count || ':' || max_reading, ',' ORDER BY bucket)" + " FROM springs_daily"), + _one(conn, f"SELECT string_agg(b || ':' || n || ':' || m, ',' ORDER BY b) FROM (SELECT" + f" date_bin('1 day', at, '{ORIGIN}') b, count(*) n, max(reading) m FROM springs GROUP BY 1) x"), + "a refresh in a zone with a DST change recomputes a whole day bucket") + # Ordinary roles: row-level security on the rollup catalogs is what is tested. with conn.cursor() as cur: for r in ("rp_owner", "rp_other"): diff --git a/test/rollup.sh b/test/rollup.sh index 5d8cb08a6..250ecac58 100755 --- a/test/rollup.sh +++ b/test/rollup.sh @@ -181,13 +181,49 @@ check_num "after a rebase the next refresh folds exactly the appended rows" \ "$(q "SELECT pgcolumnar.refresh_rollup('r_hourly');")" "800" check "and after the rebase the rollup equals a full recompute" "$(matches)" "$(oracle)" -# Phase 1 folds appends only; anything else is refused, and the rollup is untouched. -before="$(exact "SELECT * FROM r_hourly")" -check_num "the rollup holds rows before the delete" "$(at_least "$(q "SELECT count(*) FROM r_hourly;")" 1)" "1" +# Phase 2 (#1294): a delete, an update or a rewrite in the source makes the +# buckets of the row groups it changed DIRTY. A refresh recomputes those from the +# source, folds new groups into the rest, and leaves every other partial alone. +# The groups were last recorded by rebase_rollup above, so these arms also need +# rebase to have recorded each group's time range. +# where TEXT -> the partials matching it, by physical position: a bucket that is +# recomputed or updated gets a new row, so a new ctid. +partials_at() { q "SELECT count(*) || '|' || md5(string_agg(ctid::text, ',' ORDER BY ctid)) FROM r_hourly__partials WHERE $1;"; } +FAR="bucket >= '2026-03-05 00:00:00+00'" # beyond the time range of every group the delete below reaches +far_before="$(partials_at "$FAR")" +check_num "premise: the rollup holds partials the delete below cannot reach" \ + "$(at_least "$(q "SELECT count(*) FROM r_hourly__partials WHERE $FAR;")" 1)" "1" q "DELETE FROM src WHERE dev = 3 AND ts < '2026-03-01 06:00:00+00';" >/dev/null -check "a refresh after a delete is refused until phase 2" \ - "$(sqlstate "SELECT pgcolumnar.refresh_rollup('r_hourly');")" "0A000" -check "and the refused refresh changed nothing" "$(exact "SELECT * FROM r_hourly")" "$before" +check "a refresh after a delete is not refused" \ + "$(sqlstate_or "SELECT pgcolumnar.refresh_rollup('r_hourly');" | sed 's/^[0-9]*$/folded/')" "folded" +check "and after a delete the rollup equals a full recompute" "$(matches)" "$(oracle)" +check "and the refresh rewrote no partial outside the changed groups' time range" "$(partials_at "$FAR")" "$far_before" +check_num "and the next refresh, with nothing changed since, recomputes nothing" \ + "$(q "SELECT pgcolumnar.refresh_rollup('r_hourly');")" "0" + +# An update moves rows to a bucket five days on: the old bucket loses them (its +# group is dirty), the new one gains them (an appended group). +q "UPDATE src SET ts = ts + interval '5 days' WHERE ts >= '2026-03-03 10:00:00+00' AND ts < '2026-03-03 10:20:00+00';" >/dev/null +q "SELECT pgcolumnar.refresh_rollup('r_hourly');" >/dev/null +check "after an update that moves rows to another bucket the rollup equals a full recompute" "$(matches)" "$(oracle)" + +# A dirty bucket and a new group in the same refresh: the new group's rows in the +# dirty bucket are counted by the recompute, so the fold must skip them. +q "DELETE FROM src WHERE dev = 5 AND ts >= '2026-03-04 02:00:00+00' AND ts < '2026-03-04 03:00:00+00';" >/dev/null +# 200 rows every 30 minutes from 02:30: 100 hours, past any 4000-row group's 41. +q "INSERT INTO src SELECT '2026-03-04 02:30:00+00'::timestamptz + g * interval '30 min', 5, g, g / 3.0 + FROM generate_series(0, 199) g;" >/dev/null +q "SELECT pgcolumnar.refresh_rollup('r_hourly');" >/dev/null +check "after a delete and an append that straddles its bucket, a refresh counts the new rows once" "$(matches)" "$(oracle)" + +# compact_rewrite drops a group's dead rows by rewriting the group. +q "DELETE FROM src WHERE ts >= '2026-03-06 00:00:00+00' AND ts < '2026-03-06 12:00:00+00';" >/dev/null +check_num "premise: compact_rewrite rewrote a row group" \ + "$(at_least "$(q "SELECT pgcolumnar.compact_rewrite('src', 0.01);")" 1)" "1" +q "SELECT pgcolumnar.refresh_rollup('r_hourly');" >/dev/null +check "after compact_rewrite the rollup equals a full recompute" "$(matches)" "$(oracle)" +check_num "and the refresh after the rewrite recomputes nothing" \ + "$(q "SELECT pgcolumnar.refresh_rollup('r_hourly');")" "0" # Refused at creation. # Each new group is read through a TID range built from its row numbers. The @@ -221,6 +257,11 @@ check "a rollup with NULL keys and NULL timestamps equals a full recompute" \ "$(exact "$NKR")|$(q "SELECT count(*) FROM ($NKR) z WHERE dev IS NULL;") $(q "SELECT count(*) FROM ($NKR) z WHERE bucket IS NULL;")" check_num "premise: and the fixture has NULL-key and NULL-bucket groups" \ "$(at_least "$(q "SELECT count(*) FROM ($NKR) z WHERE dev IS NULL OR bucket IS NULL;")" 2)" "2" +# A deleted row whose ts is NULL dirties the NULL bucket, which has no time range. +q "DELETE FROM nk WHERE ts IS NULL AND dev = 2;" >/dev/null +q "SELECT pgcolumnar.refresh_rollup('r_nk');" >/dev/null +check "after a delete of rows whose ts is NULL the NULL-bucket rollup equals a full recompute" \ + "$(exact "SELECT * FROM r_nk")" "$(exact "$NKR")" # Retention keeps history (owner's decision on #1294 point 2): groups that expire # retires are recorded in pgcolumnar.retired_group (#1314), and a refresh drops @@ -241,12 +282,69 @@ check_num "premise: expire retired the old row groups" "$(at_least "${retired:-0 check "a refresh after retention folds nothing, refuses nothing, and keeps every bucket" \ "$(sqlstate_or "SELECT pgcolumnar.refresh_rollup('r_hist');")|$(q "SELECT md5(string_agg(bucket || ':' || count || ':' || sum_v, ',' ORDER BY bucket)) FROM r_hist;")" \ "0|$hb" +check_num "and the rollup no longer records the groups retention retired" \ + "$(q "SELECT count(*) FROM pgcolumnar.rollup_group rg JOIN pgcolumnar.retired_group t + ON t.group_number = rg.group_number AND t.storage_id = pgcolumnar.get_storage_id('hist') + WHERE rg.rollup = 'r_hist'::regclass;")" "0" q "INSERT INTO hist SELECT now() - interval '2 hours' + g * interval '1 s', g FROM generate_series(1, 500) g;" >/dev/null check_num "and after retention an append still folds" "$(q "SELECT pgcolumnar.refresh_rollup('r_hist');")" "500" check "and the recent buckets equal a recompute, while the expired ones keep their history" \ "$(q "SELECT md5(string_agg(bucket || ':' || count || ':' || sum_v, ',' ORDER BY bucket)) FROM r_hist WHERE bucket >= now() - interval '30 days';")|$(q "SELECT count(*) FROM r_hist WHERE bucket < now() - interval '100 days';")" \ "$(q "SELECT md5(string_agg(b || ':' || n || ':' || s, ',' ORDER BY b)) FROM (SELECT date_bin('1 day', ts, '$ORIGIN') b, count(*) n, sum(v)::numeric s FROM hist WHERE ts >= now() - interval '30 days' GROUP BY 1) x WHERE b >= now() - interval '30 days';")|$ob" +# History and dirty buckets. A bucket that also holds a retired group's rows +# cannot be recomputed: the source no longer has them. Such a refresh is refused +# and changes nothing; a delete whose buckets hold no history folds. Three groups, +# one per INSERT: OLD retires; KEPT shares OLD's bucket D and reaches past the +# cutoff, so expire keeps it; LATE starts days after D. +D="date_bin('1 day', now() - interval '40 days', '$ORIGIN')" +layered() { # layered TABLE: the three groups, a daily rollup r_TABLE refreshed once, then expire + q "CREATE TABLE $1 (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('$1', stripe_row_limit => 2000); + INSERT INTO $1 SELECT $D + interval '2 hours' + g * interval '1 min', g FROM generate_series(1, 500) g; + INSERT INTO $1 SELECT $D + interval '12 hours' + g * interval '15 min', g FROM generate_series(1, 1200) g; + INSERT INTO $1 SELECT $D + interval '20 days' + g * interval '5 min', g FROM generate_series(1, 600) g;" >/dev/null + q "SELECT pgcolumnar.create_rollup('r_$1', '$1', 'ts', '1 day', '$ORIGIN', '{}', '{count,sum}', '{*,v}'); + SELECT pgcolumnar.refresh_rollup('r_$1'); + SELECT pgcolumnar.set_options('$1', ttl_column => 'ts', ttl_interval => '30 days');" >/dev/null + q "SELECT pgcolumnar.expire('$1');" >/dev/null +} +layered hd +check "premise: expire retired the group in bucket D alone and kept the one that shares it" \ + "$(q "SELECT count(*) FILTER (WHERE ts < $D + interval '12 hours') || '|' || count(*) FILTER (WHERE ts >= $D + interval '12 hours' AND ts < $D + interval '1 day') FROM hd;")" "0|47" +HDQ="SELECT md5(string_agg(bucket || ':' || count || ':' || sum_v, ',' ORDER BY bucket)) FROM r_hd" +HDR="SELECT md5(string_agg(b || ':' || n || ':' || s, ',' ORDER BY b)) FROM (SELECT date_bin('1 day', ts, '$ORIGIN') b, count(*) n, sum(v)::numeric s FROM hd GROUP BY 1) x" +q "DELETE FROM hd WHERE ts >= $D + interval '20 days 12 hours' AND v % 2 = 0;" >/dev/null +check "a delete in buckets with no history, next to retired ones, folds: those buckets equal a recompute and D keeps its history" \ + "$(sqlstate_or "SELECT pgcolumnar.refresh_rollup('r_hd');" | sed 's/^[0-9]*$/folded/')|$(q "$HDQ WHERE bucket > $D;")|$(q "SELECT count || ':' || sum_v FROM r_hd WHERE bucket = $D;")" \ + "folded|$(q "$HDR WHERE b > $D;")|547:$((500 * 501 / 2 + 47 * 48 / 2))" +hd_before="$(q "$HDQ;")" +q "DELETE FROM hd WHERE ts >= $D + interval '12 hours' AND ts < $D + interval '13 hours';" >/dev/null +check "a delete in a bucket that holds retired history is refused, and the rollup is unchanged" \ + "$(sqlstate "SELECT pgcolumnar.refresh_rollup('r_hd');")|$(q "$HDQ;")" "0A000|$hd_before" +# The same, with no refresh between expire and the delete: the history that +# blocks the recompute is retired by the very refresh that would recompute it. +layered hd2 +hd2_before="$(q "SELECT md5(string_agg(bucket || ':' || count || ':' || sum_v, ',' ORDER BY bucket)) FROM r_hd2;")" +q "DELETE FROM hd2 WHERE ts >= $D + interval '12 hours' AND ts < $D + interval '13 hours';" >/dev/null +check "a delete in a bucket whose history the same refresh retires is refused, and the rollup is unchanged" \ + "$(sqlstate "SELECT pgcolumnar.refresh_rollup('r_hd2');")|$(q "SELECT md5(string_agg(bucket || ':' || count || ':' || sum_v, ',' ORDER BY bucket)) FROM r_hd2;")" \ + "0A000|$hd2_before" + +# A day bucket is 24 hours (date_bin's day), whatever the session's zone. In +# America/New_York, 2026-03-08 is 23 hours long, and the dirty bucket of that UTC +# day must still be recomputed up to its end: rows run to 23:30 UTC. +q "CREATE TABLE dst (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('dst', stripe_row_limit => 5000); + INSERT INTO dst SELECT '2026-03-07 00:00:00+00'::timestamptz + g * interval '5 min', g FROM generate_series(1, 570) g; + SELECT pgcolumnar.create_rollup('r_dst', 'dst', 'ts', '1 day', '$ORIGIN', '{}', '{count,sum}', '{*,v}'); + SELECT pgcolumnar.refresh_rollup('r_dst'); + DELETE FROM dst WHERE v = 1;" >/dev/null +q "SET TimeZone = 'America/New_York'; SELECT pgcolumnar.refresh_rollup('r_dst');" >/dev/null +check "a refresh in a zone with a DST change recomputes a whole day bucket" \ + "$(q "SELECT string_agg(bucket || ':' || count || ':' || sum_v, ',' ORDER BY bucket) FROM r_dst;" | tail -1)" \ + "$(q "SELECT string_agg(b || ':' || n || ':' || s, ',' ORDER BY b) FROM (SELECT date_bin('1 day', ts, '$ORIGIN') b, count(*) n, sum(v)::numeric s FROM dst GROUP BY 1) x;")" + # Not only for superusers: a superuser bypasses row-level security, so every arm # above ran past the policies on pgcolumnar.rollup and rollup_group. A table's # owner can roll it up; another role sees nothing of it and cannot refresh it. From 4525e3490ba6a22a5bd9af64fff6a62ccab5749e Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 12:55:40 +0000 Subject: [PATCH 02/11] feat: rollups phase 2, dirty buckets (#1294) refresh_rollup no longer refuses deletes and rewrites in the source. In one statement, so one snapshot, it: - treats a recorded group that is gone, or whose row or deleted count moved, as changed, unless expire retired it; - recomputes from the source every bucket a changed group's recorded time range covers, plus the NULL bucket when the group had rows without a ts; - folds the new groups' rows into the clean buckets only, because the recompute already counts their rows in the dirty ones; - refuses with 0A000, writing nothing, when a dirty bucket meets the history of groups expire retired. rollup_group records each group's ts range at fold time, since a group that is gone has no zone maps left. The retired groups' ranges collect in the rollup's new history column (tstzmultirange), not in rollup_group: after a rewrite, the group numbers that rebase records can repeat a retired group's number. rebase_rollup now reads each group's range from its rows. The width is added as plain seconds, because a timestamptz plus '1 day' in the session's zone is 23 or 25 hours across a DST change. Co-Authored-By: Claude Opus 5.5 --- pgcolumnar--1.0-alpha4--1.0-alpha5.sql | 177 +++++++++++++++++++------ pgcolumnar--1.0-alpha5.sql | 177 +++++++++++++++++++------ 2 files changed, 276 insertions(+), 78 deletions(-) diff --git a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql index 54fc7a518..17bfd2c1e 100644 --- a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql @@ -1015,14 +1015,21 @@ COMMENT ON FUNCTION pgcolumnar.column_extent(regclass, name) /* --------------------------------------------------------------------------- - * Incremental rollups, phase 1: append-only (#1294). + * Incremental rollups (#1294). * * A rollup is a VIEW over a partials table. The partials table holds, per * (bucket, group_by...) key, the parts each aggregate combines from: count, sum, * min, max, and for avg a sum and a count. The view turns those into the * aggregates. refresh_rollup folds the rows of row groups that are new since the - * last refresh, in ONE statement, so the group list it reads, the rows it folds - * and the groups it records all come from one snapshot (#1294 point 1). + * last refresh, and recomputes the buckets of groups that were deleted from or + * rewritten since (phase 2, dirty buckets), in ONE statement: the group list it + * reads, the rows it folds and the groups it records all come from one snapshot + * (#1294 point 1). + * + * Each recorded group carries the time range of the rows folded from it, + * because a group that is gone has no zone maps left to ask. Retention keeps + * history: the ranges of the groups expire retired collect in the rollup's + * history, and a refresh that would recompute a bucket in it is refused. * * pgcolumnar.rollup and pgcolumnar.rollup_group are readable and writable by * anyone, row-level security limiting each role to the rollups it owns: the @@ -1038,7 +1045,11 @@ CREATE TABLE pgcolumnar.rollup ( origin timestamptz NOT NULL, group_by name[] NOT NULL, agg_functions text[] NOT NULL, - agg_columns name[] NOT NULL + agg_columns name[] NOT NULL, + -- the time ranges of the groups retention retired: their buckets keep rows + -- the source no longer has, so they are never recomputed + history tstzmultirange NOT NULL DEFAULT '{}', + history_has_null boolean NOT NULL DEFAULT false ); CREATE TABLE pgcolumnar.rollup_group ( @@ -1046,6 +1057,10 @@ CREATE TABLE pgcolumnar.rollup_group ( group_number bigint NOT NULL, row_count bigint NOT NULL, deleted_count bigint NOT NULL, + -- the range of ts_column over the rows folded; NULL when none had a ts + ts_min timestamptz, + ts_max timestamptz, + ts_has_null boolean NOT NULL, PRIMARY KEY (rollup, group_number) ); @@ -1262,6 +1277,8 @@ DECLARE c text; g name; folded bigint; + refused boolean; + wabs interval; BEGIN -- A REPEATABLE READ caller's snapshot predates the lock below, so a refresh -- that waited for another would fold the same groups again. @@ -1282,24 +1299,10 @@ BEGIN USING ERRCODE = 'object_not_in_prerequisite_state', HINT = 'If the rollup is current, pgcolumnar.rebase_rollup records the source''s new row groups.'; END IF; - -- Retention keeps history (the owner's decision on #1294): a recorded group - -- that expire retired leaves the snapshot, and its buckets keep the rows they - -- folded. Retirement is permanent, so this need not share the fold's snapshot. - DELETE FROM pgcolumnar.rollup_group rg - WHERE rg.rollup = r.rollup - AND rg.group_number IN (SELECT t.group_number FROM pgcolumnar.rollup_source_retired(r.source) t); - -- Phase 1 folds appends only: any other recorded group that is gone, or has - -- new deletions, needs phase 2's dirty buckets. - SELECT string_agg(rg.group_number::text, ', ' ORDER BY rg.group_number) INTO changed - FROM pgcolumnar.rollup_group rg - LEFT JOIN pgcolumnar.rollup_source_groups(r.source) s USING (group_number) - WHERE rg.rollup = r.rollup - AND (s.group_number IS NULL OR s.deleted_count <> rg.deleted_count); - IF changed IS NOT NULL THEN - RAISE EXCEPTION 'rollup %: row groups % of its source were deleted from or rewritten', rollup, changed - USING ERRCODE = 'feature_not_supported', - DETAIL = 'This version folds appended row groups only; changed ones need dirty buckets (phase 2).'; - END IF; + -- The width as plain seconds: a timestamptz plus '1 day' is a calendar day + -- in the session's zone, 23 or 25 hours across a DST change, while date_bin + -- counts a day as 24 hours. + wabs := make_interval(secs => extract(epoch FROM r.width)::float8); FOREACH g IN ARRAY r.group_by LOOP keys := keys || format(', %I', g); @@ -1357,23 +1360,61 @@ BEGIN SELECT string_agg(format('%I', x), ', ' ORDER BY o) INTO rowcols FROM unnest(rcols) WITH ORDINALITY u(x, o); - -- ONE statement: the group list, the rows of the new groups (one TID range - -- per group, so only their row groups are read), the fold into the partials - -- and the record of the groups folded all see the same snapshot. A group - -- that commits while this runs is in none of them, and the next refresh - -- takes it. pgcolumnar.rollup_pause holds the statement open for that arm. + -- ONE statement, so one snapshot, for all of it: + -- * the groups. A recorded group that expire retired (keep history, the + -- owner's decision on #1294) leaves the record and its range joins the + -- rollup's history. Any other recorded group that is gone, or whose row + -- or deleted count moved, is CHANGED. + -- * the dirty buckets: every bucket a changed group's recorded range covers, + -- and the NULL bucket when it had rows without a ts. Their partials are + -- deleted and recomputed from the source's live rows. + -- * the rows of the new groups (one TID range per group, so only their row + -- groups are read), folded into the clean buckets only: the recompute + -- already counts their rows in the dirty ones. + -- * the record: changed groups recounted or forgotten, new ones added with + -- the range of their rows. + -- A group that commits while this runs is in none of it, and the next + -- refresh takes it. Reading retirement here, not in an earlier statement, + -- keeps a group expire retires meanwhile from looking deleted. + -- pgcolumnar.rollup_pause holds the statement open for that arm. EXECUTE format($f$ WITH cur AS ( SELECT * FROM pgcolumnar.rollup_source_groups(%1$L::regclass) + ), ret AS ( + SELECT * FROM pgcolumnar.rollup_source_retired(%1$L::regclass) ), held AS ( SELECT pg_sleep(CASE WHEN current_setting('pgcolumnar.rollup_pause', true) = 'on' THEN 2 ELSE 0 END) AS slept + ), rec AS ( + SELECT rg.* FROM pgcolumnar.rollup_group rg WHERE rg.rollup = %2$L::regclass + ), retiring AS ( + SELECT rg.* FROM rec rg WHERE rg.group_number IN (SELECT t.group_number FROM ret t) + ), changed AS ( + SELECT rg.group_number, rg.ts_min, rg.ts_max, rg.ts_has_null, + c.row_count AS now_rows, c.deleted_count AS now_deleted + FROM rec rg LEFT JOIN cur c USING (group_number) + WHERE rg.group_number NOT IN (SELECT t.group_number FROM ret t) + AND (c.group_number IS NULL OR c.row_count <> rg.row_count + OR c.deleted_count <> rg.deleted_count) + ), dirty AS ( + SELECT coalesce(range_agg(tstzrange(date_bin(%4$L::interval, ch.ts_min, %6$L::timestamptz), + date_bin(%4$L::interval, ch.ts_max, %6$L::timestamptz) + + %17$L::interval)) + FILTER (WHERE ch.ts_min IS NOT NULL), '{}') AS b, + coalesce(bool_or(ch.ts_has_null), false) AS n + FROM changed ch + ), kept AS ( + SELECT x.history + coalesce((SELECT range_agg(tstzrange(t.ts_min, t.ts_max, '[]')) FROM retiring t + WHERE t.ts_min IS NOT NULL), '{}') AS m, + x.history_has_null OR coalesce((SELECT bool_or(t.ts_has_null) FROM retiring t), false) AS n + FROM pgcolumnar.rollup x WHERE x.rollup = %2$L::regclass + ), blocked AS ( + SELECT 1 FROM dirty d, kept k WHERE d.b && k.m OR (d.n AND k.n) ), new_groups AS ( SELECT c.* FROM cur c - WHERE NOT EXISTS (SELECT 1 FROM pgcolumnar.rollup_group rg - WHERE rg.rollup = %2$L::regclass AND rg.group_number = c.group_number) + WHERE NOT EXISTS (SELECT 1 FROM rec rg WHERE rg.group_number = c.group_number) ), rows AS MATERIALIZED ( - SELECT s.* FROM new_groups ng, held, + SELECT ng.group_number AS pgcolumnar__group, s.* FROM new_groups ng, held, LATERAL (SELECT %3$s FROM %1$s WHERE ctid >= pgcolumnar.row_tid(ng.first_row_number) AND ctid <= pgcolumnar.row_tid(ng.first_row_number + ng.row_count - 1) @@ -1381,9 +1422,28 @@ BEGIN -- PostgreSQL builds a TID range scan only from a relation's own -- restrictions, and as a join clause the range read every group. OFFSET 0) s + ), fold AS ( + SELECT w.* FROM rows w, dirty d + WHERE NOT CASE WHEN w.%5$I IS NULL THEN d.n + ELSE d.b @> date_bin(%4$L::interval, w.%5$I, %6$L::timestamptz) END + ), recomputed AS MATERIALIZED ( + -- one range predicate per span of dirty buckets, so zone maps prune + SELECT s.* FROM dirty d, held, unnest(d.b) u(span), + LATERAL (SELECT %3$s FROM %1$s + WHERE %5$I >= lower(u.span) AND %5$I < upper(u.span) OFFSET 0) s + UNION ALL + SELECT s.* FROM dirty d, held, + LATERAL (SELECT %3$s FROM %1$s WHERE %5$I IS NULL OFFSET 0) s + WHERE d.n ), delta AS ( SELECT date_bin(%4$L::interval, %5$I, %6$L::timestamptz) AS bucket%7$s%8$s - FROM rows GROUP BY 1%9$s + FROM fold GROUP BY 1%9$s + ), redo AS ( + SELECT date_bin(%4$L::interval, %5$I, %6$L::timestamptz) AS bucket%7$s%8$s + FROM recomputed GROUP BY 1%9$s + ), cleared AS ( + DELETE FROM %10$s p USING dirty d + WHERE CASE WHEN p.bucket IS NULL THEN d.n ELSE d.b @> p.bucket END ), upd AS ( UPDATE %10$s p SET %11$s FROM delta d WHERE p.bucket IS NOT DISTINCT FROM d.bucket%12$s @@ -1392,11 +1452,34 @@ BEGIN INSERT INTO %10$s (bucket%7$s%14$s) SELECT d.bucket%15$s%16$s FROM delta d WHERE NOT EXISTS (SELECT 1 FROM upd p WHERE p.bucket IS NOT DISTINCT FROM d.bucket%12$s) + ), reins AS ( + INSERT INTO %10$s (bucket%7$s%14$s) + SELECT d.bucket%15$s%16$s FROM redo d + ), forgotten AS ( + DELETE FROM pgcolumnar.rollup_group rg + WHERE rg.rollup = %2$L::regclass + AND (rg.group_number IN (SELECT t.group_number FROM retiring t) + OR rg.group_number IN (SELECT ch.group_number FROM changed ch WHERE ch.now_rows IS NULL)) + ), recounted AS ( + UPDATE pgcolumnar.rollup_group rg + SET row_count = ch.now_rows, deleted_count = ch.now_deleted + FROM changed ch + WHERE rg.rollup = %2$L::regclass AND rg.group_number = ch.group_number + AND ch.now_rows IS NOT NULL ), recorded AS ( INSERT INTO pgcolumnar.rollup_group - SELECT %2$L::regclass, ng.group_number, ng.row_count, ng.deleted_count FROM new_groups ng + SELECT %2$L::regclass, ng.group_number, ng.row_count, ng.deleted_count, + x.lo, x.hi, coalesce(x.has_null, false) + FROM new_groups ng + LEFT JOIN (SELECT w.pgcolumnar__group, min(w.%5$I) AS lo, max(w.%5$I) AS hi, + bool_or(w.%5$I IS NULL) AS has_null + FROM rows w GROUP BY 1) x ON x.pgcolumnar__group = ng.group_number + ), historied AS ( + UPDATE pgcolumnar.rollup x SET history = k.m, history_has_null = k.n + FROM kept k WHERE x.rollup = %2$L::regclass AND EXISTS (SELECT 1 FROM retiring) ) - SELECT count(*) FROM rows + SELECT (SELECT count(*) FROM fold) + (SELECT count(*) FROM recomputed), + EXISTS (SELECT 1 FROM blocked) $f$, r.source, r.rollup, rowcols, r.width, r.ts_column, r.origin, keys, dsel, @@ -1405,14 +1488,21 @@ BEGIN (SELECT coalesce(string_agg(format(', p.%I', x), ''), '') FROM unnest(r.group_by) x), icols, (SELECT coalesce(string_agg(format(', d.%I', x), ''), '') FROM unnest(r.group_by) x), - isel) - INTO folded; + isel, wabs) + INTO folded, refused; + -- Raised after the statement, so everything it wrote rolls back with it. + IF refused THEN + RAISE EXCEPTION 'rollup %: a changed row group of its source reaches buckets whose raw rows retention removed', rollup + USING ERRCODE = 'feature_not_supported', + DETAIL = 'Recomputing those buckets from the source would drop the history they keep.', + HINT = 'If the rollup is right as it stands, pgcolumnar.rebase_rollup records the source''s current row groups.'; + END IF; RETURN folded; END $rr$; COMMENT ON FUNCTION pgcolumnar.refresh_rollup(regclass) - IS 'fold the rows of the row groups appended to a rollup''s source since its last refresh, in one statement; returns the rows folded (#1294, phase 1: appends only)'; + IS 'bring a rollup up to date in one statement: fold the rows of row groups appended to its source since the last refresh, and recompute the buckets of groups deleted from or rewritten since, except buckets that keep retained history; returns the rows folded and recomputed (#1294)'; CREATE FUNCTION pgcolumnar.rebase_rollup(rollup regclass) RETURNS bigint @@ -1430,10 +1520,19 @@ BEGIN EXECUTE format('LOCK TABLE %s IN SHARE UPDATE EXCLUSIVE MODE', r.partials); -- The caller asserts the rollup is current: the source's groups are recorded -- as folded, and the rollup's rows are not touched (#1294 point 9). + -- Each group's time range is read from its rows, one TID range per group, + -- so a rebase reads the whole source. The history stays as it was. DELETE FROM pgcolumnar.rollup_group rg WHERE rg.rollup = r.rollup; - INSERT INTO pgcolumnar.rollup_group - SELECT r.rollup, s.group_number, s.row_count, s.deleted_count - FROM pgcolumnar.rollup_source_groups(r.source) s; + EXECUTE format($f$ + INSERT INTO pgcolumnar.rollup_group + SELECT %2$L::regclass, s.group_number, s.row_count, s.deleted_count, + x.lo, x.hi, coalesce(x.has_null, false) + FROM pgcolumnar.rollup_source_groups(%1$L::regclass) s, + LATERAL (SELECT min(%3$I) AS lo, max(%3$I) AS hi, bool_or(%3$I IS NULL) AS has_null + FROM %1$s + WHERE ctid >= pgcolumnar.row_tid(s.first_row_number) + AND ctid <= pgcolumnar.row_tid(s.first_row_number + s.row_count - 1)) x + $f$, r.source, r.rollup, r.ts_column); GET DIAGNOSTICS n = ROW_COUNT; UPDATE pgcolumnar.rollup x SET source_storage = pgcolumnar.get_storage_id(r.source) WHERE x.rollup = r.rollup; diff --git a/pgcolumnar--1.0-alpha5.sql b/pgcolumnar--1.0-alpha5.sql index d742a4cdb..73363576b 100644 --- a/pgcolumnar--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha5.sql @@ -2205,14 +2205,21 @@ END; $create_ahead$; /* --------------------------------------------------------------------------- - * Incremental rollups, phase 1: append-only (#1294). + * Incremental rollups (#1294). * * A rollup is a VIEW over a partials table. The partials table holds, per * (bucket, group_by...) key, the parts each aggregate combines from: count, sum, * min, max, and for avg a sum and a count. The view turns those into the * aggregates. refresh_rollup folds the rows of row groups that are new since the - * last refresh, in ONE statement, so the group list it reads, the rows it folds - * and the groups it records all come from one snapshot (#1294 point 1). + * last refresh, and recomputes the buckets of groups that were deleted from or + * rewritten since (phase 2, dirty buckets), in ONE statement: the group list it + * reads, the rows it folds and the groups it records all come from one snapshot + * (#1294 point 1). + * + * Each recorded group carries the time range of the rows folded from it, + * because a group that is gone has no zone maps left to ask. Retention keeps + * history: the ranges of the groups expire retired collect in the rollup's + * history, and a refresh that would recompute a bucket in it is refused. * * pgcolumnar.rollup and pgcolumnar.rollup_group are readable and writable by * anyone, row-level security limiting each role to the rollups it owns: the @@ -2228,7 +2235,11 @@ CREATE TABLE pgcolumnar.rollup ( origin timestamptz NOT NULL, group_by name[] NOT NULL, agg_functions text[] NOT NULL, - agg_columns name[] NOT NULL + agg_columns name[] NOT NULL, + -- the time ranges of the groups retention retired: their buckets keep rows + -- the source no longer has, so they are never recomputed + history tstzmultirange NOT NULL DEFAULT '{}', + history_has_null boolean NOT NULL DEFAULT false ); CREATE TABLE pgcolumnar.rollup_group ( @@ -2236,6 +2247,10 @@ CREATE TABLE pgcolumnar.rollup_group ( group_number bigint NOT NULL, row_count bigint NOT NULL, deleted_count bigint NOT NULL, + -- the range of ts_column over the rows folded; NULL when none had a ts + ts_min timestamptz, + ts_max timestamptz, + ts_has_null boolean NOT NULL, PRIMARY KEY (rollup, group_number) ); @@ -2452,6 +2467,8 @@ DECLARE c text; g name; folded bigint; + refused boolean; + wabs interval; BEGIN -- A REPEATABLE READ caller's snapshot predates the lock below, so a refresh -- that waited for another would fold the same groups again. @@ -2472,24 +2489,10 @@ BEGIN USING ERRCODE = 'object_not_in_prerequisite_state', HINT = 'If the rollup is current, pgcolumnar.rebase_rollup records the source''s new row groups.'; END IF; - -- Retention keeps history (the owner's decision on #1294): a recorded group - -- that expire retired leaves the snapshot, and its buckets keep the rows they - -- folded. Retirement is permanent, so this need not share the fold's snapshot. - DELETE FROM pgcolumnar.rollup_group rg - WHERE rg.rollup = r.rollup - AND rg.group_number IN (SELECT t.group_number FROM pgcolumnar.rollup_source_retired(r.source) t); - -- Phase 1 folds appends only: any other recorded group that is gone, or has - -- new deletions, needs phase 2's dirty buckets. - SELECT string_agg(rg.group_number::text, ', ' ORDER BY rg.group_number) INTO changed - FROM pgcolumnar.rollup_group rg - LEFT JOIN pgcolumnar.rollup_source_groups(r.source) s USING (group_number) - WHERE rg.rollup = r.rollup - AND (s.group_number IS NULL OR s.deleted_count <> rg.deleted_count); - IF changed IS NOT NULL THEN - RAISE EXCEPTION 'rollup %: row groups % of its source were deleted from or rewritten', rollup, changed - USING ERRCODE = 'feature_not_supported', - DETAIL = 'This version folds appended row groups only; changed ones need dirty buckets (phase 2).'; - END IF; + -- The width as plain seconds: a timestamptz plus '1 day' is a calendar day + -- in the session's zone, 23 or 25 hours across a DST change, while date_bin + -- counts a day as 24 hours. + wabs := make_interval(secs => extract(epoch FROM r.width)::float8); FOREACH g IN ARRAY r.group_by LOOP keys := keys || format(', %I', g); @@ -2547,23 +2550,61 @@ BEGIN SELECT string_agg(format('%I', x), ', ' ORDER BY o) INTO rowcols FROM unnest(rcols) WITH ORDINALITY u(x, o); - -- ONE statement: the group list, the rows of the new groups (one TID range - -- per group, so only their row groups are read), the fold into the partials - -- and the record of the groups folded all see the same snapshot. A group - -- that commits while this runs is in none of them, and the next refresh - -- takes it. pgcolumnar.rollup_pause holds the statement open for that arm. + -- ONE statement, so one snapshot, for all of it: + -- * the groups. A recorded group that expire retired (keep history, the + -- owner's decision on #1294) leaves the record and its range joins the + -- rollup's history. Any other recorded group that is gone, or whose row + -- or deleted count moved, is CHANGED. + -- * the dirty buckets: every bucket a changed group's recorded range covers, + -- and the NULL bucket when it had rows without a ts. Their partials are + -- deleted and recomputed from the source's live rows. + -- * the rows of the new groups (one TID range per group, so only their row + -- groups are read), folded into the clean buckets only: the recompute + -- already counts their rows in the dirty ones. + -- * the record: changed groups recounted or forgotten, new ones added with + -- the range of their rows. + -- A group that commits while this runs is in none of it, and the next + -- refresh takes it. Reading retirement here, not in an earlier statement, + -- keeps a group expire retires meanwhile from looking deleted. + -- pgcolumnar.rollup_pause holds the statement open for that arm. EXECUTE format($f$ WITH cur AS ( SELECT * FROM pgcolumnar.rollup_source_groups(%1$L::regclass) + ), ret AS ( + SELECT * FROM pgcolumnar.rollup_source_retired(%1$L::regclass) ), held AS ( SELECT pg_sleep(CASE WHEN current_setting('pgcolumnar.rollup_pause', true) = 'on' THEN 2 ELSE 0 END) AS slept + ), rec AS ( + SELECT rg.* FROM pgcolumnar.rollup_group rg WHERE rg.rollup = %2$L::regclass + ), retiring AS ( + SELECT rg.* FROM rec rg WHERE rg.group_number IN (SELECT t.group_number FROM ret t) + ), changed AS ( + SELECT rg.group_number, rg.ts_min, rg.ts_max, rg.ts_has_null, + c.row_count AS now_rows, c.deleted_count AS now_deleted + FROM rec rg LEFT JOIN cur c USING (group_number) + WHERE rg.group_number NOT IN (SELECT t.group_number FROM ret t) + AND (c.group_number IS NULL OR c.row_count <> rg.row_count + OR c.deleted_count <> rg.deleted_count) + ), dirty AS ( + SELECT coalesce(range_agg(tstzrange(date_bin(%4$L::interval, ch.ts_min, %6$L::timestamptz), + date_bin(%4$L::interval, ch.ts_max, %6$L::timestamptz) + + %17$L::interval)) + FILTER (WHERE ch.ts_min IS NOT NULL), '{}') AS b, + coalesce(bool_or(ch.ts_has_null), false) AS n + FROM changed ch + ), kept AS ( + SELECT x.history + coalesce((SELECT range_agg(tstzrange(t.ts_min, t.ts_max, '[]')) FROM retiring t + WHERE t.ts_min IS NOT NULL), '{}') AS m, + x.history_has_null OR coalesce((SELECT bool_or(t.ts_has_null) FROM retiring t), false) AS n + FROM pgcolumnar.rollup x WHERE x.rollup = %2$L::regclass + ), blocked AS ( + SELECT 1 FROM dirty d, kept k WHERE d.b && k.m OR (d.n AND k.n) ), new_groups AS ( SELECT c.* FROM cur c - WHERE NOT EXISTS (SELECT 1 FROM pgcolumnar.rollup_group rg - WHERE rg.rollup = %2$L::regclass AND rg.group_number = c.group_number) + WHERE NOT EXISTS (SELECT 1 FROM rec rg WHERE rg.group_number = c.group_number) ), rows AS MATERIALIZED ( - SELECT s.* FROM new_groups ng, held, + SELECT ng.group_number AS pgcolumnar__group, s.* FROM new_groups ng, held, LATERAL (SELECT %3$s FROM %1$s WHERE ctid >= pgcolumnar.row_tid(ng.first_row_number) AND ctid <= pgcolumnar.row_tid(ng.first_row_number + ng.row_count - 1) @@ -2571,9 +2612,28 @@ BEGIN -- PostgreSQL builds a TID range scan only from a relation's own -- restrictions, and as a join clause the range read every group. OFFSET 0) s + ), fold AS ( + SELECT w.* FROM rows w, dirty d + WHERE NOT CASE WHEN w.%5$I IS NULL THEN d.n + ELSE d.b @> date_bin(%4$L::interval, w.%5$I, %6$L::timestamptz) END + ), recomputed AS MATERIALIZED ( + -- one range predicate per span of dirty buckets, so zone maps prune + SELECT s.* FROM dirty d, held, unnest(d.b) u(span), + LATERAL (SELECT %3$s FROM %1$s + WHERE %5$I >= lower(u.span) AND %5$I < upper(u.span) OFFSET 0) s + UNION ALL + SELECT s.* FROM dirty d, held, + LATERAL (SELECT %3$s FROM %1$s WHERE %5$I IS NULL OFFSET 0) s + WHERE d.n ), delta AS ( SELECT date_bin(%4$L::interval, %5$I, %6$L::timestamptz) AS bucket%7$s%8$s - FROM rows GROUP BY 1%9$s + FROM fold GROUP BY 1%9$s + ), redo AS ( + SELECT date_bin(%4$L::interval, %5$I, %6$L::timestamptz) AS bucket%7$s%8$s + FROM recomputed GROUP BY 1%9$s + ), cleared AS ( + DELETE FROM %10$s p USING dirty d + WHERE CASE WHEN p.bucket IS NULL THEN d.n ELSE d.b @> p.bucket END ), upd AS ( UPDATE %10$s p SET %11$s FROM delta d WHERE p.bucket IS NOT DISTINCT FROM d.bucket%12$s @@ -2582,11 +2642,34 @@ BEGIN INSERT INTO %10$s (bucket%7$s%14$s) SELECT d.bucket%15$s%16$s FROM delta d WHERE NOT EXISTS (SELECT 1 FROM upd p WHERE p.bucket IS NOT DISTINCT FROM d.bucket%12$s) + ), reins AS ( + INSERT INTO %10$s (bucket%7$s%14$s) + SELECT d.bucket%15$s%16$s FROM redo d + ), forgotten AS ( + DELETE FROM pgcolumnar.rollup_group rg + WHERE rg.rollup = %2$L::regclass + AND (rg.group_number IN (SELECT t.group_number FROM retiring t) + OR rg.group_number IN (SELECT ch.group_number FROM changed ch WHERE ch.now_rows IS NULL)) + ), recounted AS ( + UPDATE pgcolumnar.rollup_group rg + SET row_count = ch.now_rows, deleted_count = ch.now_deleted + FROM changed ch + WHERE rg.rollup = %2$L::regclass AND rg.group_number = ch.group_number + AND ch.now_rows IS NOT NULL ), recorded AS ( INSERT INTO pgcolumnar.rollup_group - SELECT %2$L::regclass, ng.group_number, ng.row_count, ng.deleted_count FROM new_groups ng + SELECT %2$L::regclass, ng.group_number, ng.row_count, ng.deleted_count, + x.lo, x.hi, coalesce(x.has_null, false) + FROM new_groups ng + LEFT JOIN (SELECT w.pgcolumnar__group, min(w.%5$I) AS lo, max(w.%5$I) AS hi, + bool_or(w.%5$I IS NULL) AS has_null + FROM rows w GROUP BY 1) x ON x.pgcolumnar__group = ng.group_number + ), historied AS ( + UPDATE pgcolumnar.rollup x SET history = k.m, history_has_null = k.n + FROM kept k WHERE x.rollup = %2$L::regclass AND EXISTS (SELECT 1 FROM retiring) ) - SELECT count(*) FROM rows + SELECT (SELECT count(*) FROM fold) + (SELECT count(*) FROM recomputed), + EXISTS (SELECT 1 FROM blocked) $f$, r.source, r.rollup, rowcols, r.width, r.ts_column, r.origin, keys, dsel, @@ -2595,14 +2678,21 @@ BEGIN (SELECT coalesce(string_agg(format(', p.%I', x), ''), '') FROM unnest(r.group_by) x), icols, (SELECT coalesce(string_agg(format(', d.%I', x), ''), '') FROM unnest(r.group_by) x), - isel) - INTO folded; + isel, wabs) + INTO folded, refused; + -- Raised after the statement, so everything it wrote rolls back with it. + IF refused THEN + RAISE EXCEPTION 'rollup %: a changed row group of its source reaches buckets whose raw rows retention removed', rollup + USING ERRCODE = 'feature_not_supported', + DETAIL = 'Recomputing those buckets from the source would drop the history they keep.', + HINT = 'If the rollup is right as it stands, pgcolumnar.rebase_rollup records the source''s current row groups.'; + END IF; RETURN folded; END $rr$; COMMENT ON FUNCTION pgcolumnar.refresh_rollup(regclass) - IS 'fold the rows of the row groups appended to a rollup''s source since its last refresh, in one statement; returns the rows folded (#1294, phase 1: appends only)'; + IS 'bring a rollup up to date in one statement: fold the rows of row groups appended to its source since the last refresh, and recompute the buckets of groups deleted from or rewritten since, except buckets that keep retained history; returns the rows folded and recomputed (#1294)'; CREATE FUNCTION pgcolumnar.rebase_rollup(rollup regclass) RETURNS bigint @@ -2620,10 +2710,19 @@ BEGIN EXECUTE format('LOCK TABLE %s IN SHARE UPDATE EXCLUSIVE MODE', r.partials); -- The caller asserts the rollup is current: the source's groups are recorded -- as folded, and the rollup's rows are not touched (#1294 point 9). + -- Each group's time range is read from its rows, one TID range per group, + -- so a rebase reads the whole source. The history stays as it was. DELETE FROM pgcolumnar.rollup_group rg WHERE rg.rollup = r.rollup; - INSERT INTO pgcolumnar.rollup_group - SELECT r.rollup, s.group_number, s.row_count, s.deleted_count - FROM pgcolumnar.rollup_source_groups(r.source) s; + EXECUTE format($f$ + INSERT INTO pgcolumnar.rollup_group + SELECT %2$L::regclass, s.group_number, s.row_count, s.deleted_count, + x.lo, x.hi, coalesce(x.has_null, false) + FROM pgcolumnar.rollup_source_groups(%1$L::regclass) s, + LATERAL (SELECT min(%3$I) AS lo, max(%3$I) AS hi, bool_or(%3$I IS NULL) AS has_null + FROM %1$s + WHERE ctid >= pgcolumnar.row_tid(s.first_row_number) + AND ctid <= pgcolumnar.row_tid(s.first_row_number + s.row_count - 1)) x + $f$, r.source, r.rollup, r.ts_column); GET DIAGNOSTICS n = ROW_COUNT; UPDATE pgcolumnar.rollup x SET source_storage = pgcolumnar.get_storage_id(r.source) WHERE x.rollup = r.rollup; From 0232aafbff92c38d04c67141f479d180b7d67d5c Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 12:57:20 +0000 Subject: [PATCH 03/11] docs: rollups recompute deletes and keep history (#1294) The how-to, the SQL reference and the CHANGELOG said a refresh refuses any delete. Phase 2 recomputes the changed buckets instead, and refuses only when a bucket to recompute holds retained history. refresh_rollup now returns the rows it read, folded and recomputed. docs_style 58/58. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 19 ++++++++++++++++--- docs/how-to.md | 28 +++++++++++++++++++--------- docs/sql-reference.md | 19 +++++++++++++------ 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4416dbdf..83634090b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -184,9 +184,8 @@ true until the next version shipped. concurrent refreshes fold each row once. The rollup equals a full recompute: exactly for counts, integer sums and averages, min and max, and within 1e-12 for `float8` averages. Rows that `expire` retires keep their buckets in the - rollup, so the rollup's history outlives the raw data. This phase refuses any - other delete or update in the source (`0A000`) and a rewritten source - (`55000`, until `pgcolumnar.rebase_rollup`). Each group's TID range comes from + rollup, so the rollup's history outlives the raw data. A rewritten source is + refused (`55000`, until `pgcolumnar.rebase_rollup`). Each group's TID range comes from the access method's own mapping (`pgcolumnar.row_tid`), not a block size restated in SQL. The rollup catalogs carry row-level security, so each role sees only the @@ -194,6 +193,20 @@ true until the next version shipped. pulled up into a join read every row group (114 blocks of a 103-block table for one new group), because PostgreSQL builds a TID range scan only from a relation's own restrictions; `OFFSET 0` keeps it a scan (15 blocks). +- Incremental rollups, phase 2: dirty buckets (#1294). `refresh_rollup` no longer + refuses deletes, updates and `compact_rewrite` in the source. In the same one + statement, it recomputes from the source every bucket a changed row group's + rows fall in, and folds new groups into the other buckets only, so a row is + never counted twice. Each recorded group now carries the time range of its + rows, since a group that is gone has no zone maps left. A recompute never + touches history: when a bucket to recompute holds rows retention dropped, the + refresh is refused (`0A000`) and writes nothing. The retired groups' ranges + are kept as a `tstzmultirange` on `pgcolumnar.rollup`. `rebase_rollup` now reads + each group's range from its rows. Found while it was written: retirement read + in a statement before the fold would let an `expire` that commits in between + look like a delete and lose history, so it is read in the fold's own snapshot; + and a bucket's end is its start plus the width in plain seconds, because a + `timestamptz` plus `'1 day'` is 23 or 25 hours across a DST change. - `pgcolumnar.tier_partition(partition, server, path, lock_wait)` moves a cold columnar partition to a Parquet file and attaches the file in its place as a foreign partition with the same name, bounds, grants and comment (#1295). It diff --git a/docs/how-to.md b/docs/how-to.md index c266458e5..bae9a8bab 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -568,9 +568,10 @@ When a partition is dropped by retention, its pgcolumnar catalog rows go with it ## Keep a rollup up to date -A rollup keeps an aggregate per time bucket, such as hourly averages, and folds -in only the rows added since the last refresh. Queries read the rollup instead -of the raw table. +A rollup keeps an aggregate per time bucket, such as hourly averages. A refresh +folds in the rows added since the last refresh, and recomputes only the buckets +whose rows were deleted or updated. Queries read the rollup instead of the raw +table. ```sql SELECT pgcolumnar.create_rollup( @@ -579,7 +580,7 @@ SELECT pgcolumnar.create_rollup( agg_functions => '{count,sum,min,max,avg}', agg_columns => '{*,value,value,value,value}'); -SELECT pgcolumnar.refresh_rollup('metrics_hourly'); -- returns rows folded +SELECT pgcolumnar.refresh_rollup('metrics_hourly'); -- returns rows read SELECT * FROM metrics_hourly WHERE bucket >= now() - interval '1 day'; ``` @@ -603,11 +604,20 @@ average of a `float8` column can differ from a full recompute in the last digits maintenance daemon, drops old rows of the source, the rollup keeps its rows for those buckets. So, for example, hourly averages can outlive the raw data. -**This version folds appended rows only.** If rows of the source are deleted or -updated in any other way, `refresh_rollup` refuses (`0A000`) and changes -nothing. If the source -is rewritten, for example by `pgcolumnar.vacuum`, `refresh_rollup` refuses with -`55000`. If the rollup was current before the rewrite, run +**Deletes and updates are recomputed.** Rows can be deleted or updated, or +`pgcolumnar.compact_rewrite` can rewrite a row group. The next refresh then +recomputes, from the source, each bucket that the changed row groups covered. +It leaves the other buckets as they are. + +**A refresh never recomputes history.** A bucket can hold rows that retention +dropped. If such a bucket needs a recompute, `refresh_rollup` refuses (`0A000`) +and changes nothing, because a recompute would lose those rows. A delete that reaches a row group +spanning such a bucket is refused too, even when the deleted rows are recent. +If the rollup is right as it stands, run `pgcolumnar.rebase_rollup` to accept +the source as it is now. + +If the source is rewritten, for example by `pgcolumnar.vacuum`, `refresh_rollup` +refuses with `55000`. If the rollup was current before the rewrite, run `pgcolumnar.rebase_rollup` to record the rewritten table and continue. Run `refresh_rollup` at READ COMMITTED. A role that is not a superuser needs diff --git a/docs/sql-reference.md b/docs/sql-reference.md index e8e86221c..0c64f8908 100644 --- a/docs/sql-reference.md +++ b/docs/sql-reference.md @@ -636,11 +636,16 @@ See [Keep a rollup up to date](how-to.md#keep-a-rollup-up-to-date). ### pgcolumnar.refresh_rollup(rollup regclass) returns bigint -Folds the rows of the row groups added to the source since the last refresh, -and returns the number of rows folded. Runs at READ COMMITTED only (`25000`). +Brings the rollup up to date in one statement, and returns the number of +source rows it read. It folds the rows of the row groups added to the source +since the last refresh. For each row group deleted from or rewritten since, it +recomputes from the source the buckets that the group's rows fall in, and +nothing else. Runs at READ COMMITTED only (`25000`). + Row groups that `pgcolumnar.expire` retired are dropped from the refresh's -record, and the rollup keeps their buckets. Refuses after any other delete or -update in the source (`0A000`), and after a rewrite of the source (`55000`). +record, and the rollup keeps their buckets as history. Refuses, changing +nothing, when a bucket to recompute holds history (`0A000`), and after a rewrite +of the source (`55000`). ### pgcolumnar.row_tid(row_number bigint) returns tid @@ -650,8 +655,10 @@ number. Row numbers start at 1 (`22003` otherwise). ### pgcolumnar.rebase_rollup(rollup regclass) returns bigint Records the source's current row groups as folded, without changing the rollup, -and returns the number of row groups recorded. Use it after a rewrite of the -source, when the rollup was current before the rewrite. +and returns the number of row groups recorded. It reads every row group to +record the time range of its rows, and keeps the rollup's history. Use it after +a rewrite of the source, or after a refusal to recompute history, when the +rollup is right as it stands. ### pgcolumnar.tier_partition(partition regclass, server name, path text, lock_wait interval DEFAULT '10 seconds') returns bigint From 6e83a07e98a9128ee0ae3e617a59b7195891bfb0 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 13:06:48 +0000 Subject: [PATCH 04/11] refactor: a changed group is one gone or with new deletions (#1294) The row count was compared too, as a guard. Mutated away, it left all 70 checks green on 15-19, compact_rewrite arm included. No arm needs it, so it is dropped rather than kept untested. Co-Authored-By: Claude Opus 5.5 --- pgcolumnar--1.0-alpha4--1.0-alpha5.sql | 7 +++---- pgcolumnar--1.0-alpha5.sql | 7 +++---- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql index 17bfd2c1e..28bec6628 100644 --- a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql @@ -1363,8 +1363,8 @@ BEGIN -- ONE statement, so one snapshot, for all of it: -- * the groups. A recorded group that expire retired (keep history, the -- owner's decision on #1294) leaves the record and its range joins the - -- rollup's history. Any other recorded group that is gone, or whose row - -- or deleted count moved, is CHANGED. + -- rollup's history. Any other recorded group that is gone, or whose + -- deleted count moved, is CHANGED. -- * the dirty buckets: every bucket a changed group's recorded range covers, -- and the NULL bucket when it had rows without a ts. Their partials are -- deleted and recomputed from the source's live rows. @@ -1394,8 +1394,7 @@ BEGIN c.row_count AS now_rows, c.deleted_count AS now_deleted FROM rec rg LEFT JOIN cur c USING (group_number) WHERE rg.group_number NOT IN (SELECT t.group_number FROM ret t) - AND (c.group_number IS NULL OR c.row_count <> rg.row_count - OR c.deleted_count <> rg.deleted_count) + AND (c.group_number IS NULL OR c.deleted_count <> rg.deleted_count) ), dirty AS ( SELECT coalesce(range_agg(tstzrange(date_bin(%4$L::interval, ch.ts_min, %6$L::timestamptz), date_bin(%4$L::interval, ch.ts_max, %6$L::timestamptz) diff --git a/pgcolumnar--1.0-alpha5.sql b/pgcolumnar--1.0-alpha5.sql index 73363576b..ca7114249 100644 --- a/pgcolumnar--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha5.sql @@ -2553,8 +2553,8 @@ BEGIN -- ONE statement, so one snapshot, for all of it: -- * the groups. A recorded group that expire retired (keep history, the -- owner's decision on #1294) leaves the record and its range joins the - -- rollup's history. Any other recorded group that is gone, or whose row - -- or deleted count moved, is CHANGED. + -- rollup's history. Any other recorded group that is gone, or whose + -- deleted count moved, is CHANGED. -- * the dirty buckets: every bucket a changed group's recorded range covers, -- and the NULL bucket when it had rows without a ts. Their partials are -- deleted and recomputed from the source's live rows. @@ -2584,8 +2584,7 @@ BEGIN c.row_count AS now_rows, c.deleted_count AS now_deleted FROM rec rg LEFT JOIN cur c USING (group_number) WHERE rg.group_number NOT IN (SELECT t.group_number FROM ret t) - AND (c.group_number IS NULL OR c.row_count <> rg.row_count - OR c.deleted_count <> rg.deleted_count) + AND (c.group_number IS NULL OR c.deleted_count <> rg.deleted_count) ), dirty AS ( SELECT coalesce(range_agg(tstzrange(date_bin(%4$L::interval, ch.ts_min, %6$L::timestamptz), date_bin(%4$L::interval, ch.ts_max, %6$L::timestamptz) From 4d84c122b498159ba23690fd69a8631cf4a3e77c Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 13:16:06 +0000 Subject: [PATCH 05/11] test: ledger rows for rollups phase 2 (#1294) Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 1fb66591b..bb7232c1f 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -1993,23 +1993,32 @@ rewrite_storage_oid rewrite_storage_oid the repair gate does not read a partitio rewrite_storage_oid rewrite_storage_oid the repair gate does not read a partition the ALTER did not touch: lp DETACH PARTITION lp_2 15;16;17;18;19 2026-10-12 the repair gate opens every inheritor with NoLock again, so it reads the metapage of a partition the ALTER never locked (an Assert on assert builds) rewrite_storage_oid rewrite_storage_oid the repair gate does not read a partition the ALTER did not touch: lp DETACH PARTITION lp_2 CONCURRENTLY 15;16;17;18;19 2026-10-12 the repair gate opens every inheritor with NoLock again, so it reads the metapage of a partition the ALTER never locked (an Assert on assert builds) rollup rollup * with anything but count is refused 15;16;17;18;19 2026-10-13 create_rollup allowing * with any function -rollup rollup a refresh after a delete is refused until phase 2 15;16;17;18;19 2026-10-13 refresh_rollup without the changed-group refusal +rollup rollup a delete in a bucket that holds retired history is refused, and the rollup is unchanged 15;16;17;18;19 2026-10-13 refresh_rollup not storing the retired groups' ranges as history;refresh_rollup recording new groups without their time range;refresh_rollup without the history refusal +rollup rollup a delete in a bucket whose history the same refresh retires is refused, and the rollup is unchanged 15;16;17;18;19 2026-10-13 refresh_rollup blocking on stored history only, not on groups the same refresh retires;refresh_rollup recording new groups without their time range;refresh_rollup without the history refusal +rollup rollup a delete in buckets with no history, next to retired ones, folds: those buckets equal a recompute and D keeps its history 15;16;17;18;19 2026-10-13 refresh_rollup recomputing every bucket once any group changed;refresh_rollup recording new groups without their time range;refresh_rollup refusing any change but appends (phase 1, main before this branch) +rollup rollup a refresh after a delete is not refused 15;16;17;18;19 2026-10-13 refresh_rollup refusing any change but appends (phase 1, main before this branch) rollup rollup a refresh after a small append reads under a fifth of the source 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup's per-group subquery without OFFSET 0, so it becomes a join and reads every group rollup rollup a refresh after appends folds exactly the appended rows, late ones included 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's rollup rollup a refresh after retention folds nothing, refuses nothing, and keeps every bucket 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup not dropping groups retention retired rollup rollup a refresh after the source was rewritten is refused, naming rebase_rollup 15;16;17;18;19 2026-10-13 refresh_rollup without the source identity check rollup rollup a refresh at REPEATABLE READ is refused 15;16;17;18;19 2026-10-13 refresh_rollup at any isolation level rollup rollup a refresh folds a bulk append 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's +rollup rollup a refresh in a zone with a DST change recomputes a whole day bucket 15;16;17;18;19 2026-10-13 refresh_rollup adding the width in the session's zone;refresh_rollup recomputing every bucket once any group changed;refresh_rollup recording new groups without their time range;refresh_rollup refusing any change but appends (phase 1, main before this branch) rollup rollup a refresh with nothing new folds nothing 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group rollup rollup a role that cannot read the source cannot create a rollup of it 15;16;17;18;19 2026-10-13 get_storage_id without its SELECT check, so create_rollup no longer refuses a caller that cannot read the source rollup rollup a rollup kept in a heap table refreshes and equals a full recompute 15;16;17;18;19 2026-10-13 create_rollup ignoring access_method;refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's rollup rollup a rollup with NULL keys and NULL timestamps equals a full recompute 15;16;17;18;19 2026-10-13 refresh_rollup combining min with greatest;refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup matching keys with = instead of IS NOT DISTINCT FROM;row_tid mapping 290 rows a block instead of the AM's rollup rollup a table's owner, not a superuser, creates a rollup of it 15;16;17;18;19 2026-10-13 the rollup catalogs without their grants rollup rollup a width with months is refused 15;16;17;18;19 2026-10-13 create_rollup without the months refusal +rollup rollup after a delete and an append that straddles its bucket, a refresh counts the new rows once 15;16;17;18;19 2026-10-13 rebase_rollup recording no time ranges;refresh_rollup folding new groups' rows into dirty buckets too;refresh_rollup recomputing every bucket once any group changed;refresh_rollup refusing any change but appends (phase 1, main before this branch) +rollup rollup after a delete of rows whose ts is NULL the NULL-bucket rollup equals a full recompute 15;16;17;18;19 2026-10-13 refresh_rollup never dirtying the NULL bucket;refresh_rollup recomputing every bucket once any group changed;refresh_rollup refusing any change but appends (phase 1, main before this branch) rollup rollup after a rebase the next refresh folds exactly the appended rows 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;row_tid mapping 290 rows a block instead of the AM's +rollup rollup after an update that moves rows to another bucket the rollup equals a full recompute 15;16;17;18;19 2026-10-13 rebase_rollup recording no time ranges;refresh_rollup recomputing every bucket once any group changed;refresh_rollup refusing any change but appends (phase 1, main before this branch) +rollup rollup after compact_rewrite the rollup equals a full recompute 15;16;17;18;19 2026-10-13 rebase_rollup recording no time ranges;refresh_rollup folding new groups' rows into dirty buckets too;refresh_rollup recomputing every bucket once any group changed;refresh_rollup recording new groups without their time range;refresh_rollup refusing any change but appends (phase 1, main before this branch) rollup rollup aggregate functions and columns of different lengths are refused 15;16;17;18;19 2026-10-13 create_rollup without the equal-length check rollup rollup an aggregate column the source does not have is refused 15;16;17;18;19 2026-10-13 create_rollup without the column-exists check rollup rollup an aggregate that does not combine is refused 15;16;17;18;19 2026-10-13 create_rollup without the five-function allow-list +rollup rollup and after a delete the rollup equals a full recompute 15;16;17;18;19 2026-10-13 rebase_rollup recording no time ranges;refresh_rollup recomputing every bucket once any group changed;refresh_rollup refusing any change but appends (phase 1, main before this branch) rollup rollup and after retention an append still folds 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup not dropping groups retention retired;row_tid mapping 290 rows a block instead of the AM's rollup rollup and after the bulk append the rollup equals a full recompute 15;16;17;18;19 2026-10-13 refresh_rollup combining min with greatest;refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup recording the folded groups from a fresh read after the fold;row_tid mapping 290 rows a block instead of the AM's rollup rollup and after the first refresh the rollup equals a full recompute 15;16;17;18;19 2026-10-13 refresh_rollup folding only the first new group @@ -2024,15 +2033,21 @@ rollup rollup and no refused rollup left a table behind 15;16;17;18;19 2026-10-1 rollup rollup and reads what a recompute gives 15;16;17;18;19 2026-10-13 refresh_rollup bucketing with date_trunc, which follows the session's zone;rollup_source_groups without SECURITY DEFINER;the rollup catalogs without their grants rollup rollup and refreshes it 15;16;17;18;19 2026-10-13 rollup_source_groups without SECURITY DEFINER;the rollup catalogs without their grants rollup rollup and that is not zero 15;16;17;18;19 never - +rollup rollup and the next refresh, with nothing changed since, recomputes nothing 15;16;17;18;19 2026-10-13 refresh_rollup not recounting a changed group that is still there;refresh_rollup refusing any change but appends (phase 1, main before this branch) rollup rollup and the recent buckets equal a recompute, while the expired ones keep their history 15;16;17;18;19 2026-10-13 refresh_rollup bucketing with date_trunc, which follows the session's zone;refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup not dropping groups retention retired;row_tid mapping 290 rows a block instead of the AM's -rollup rollup and the refused refresh changed nothing 15;16;17;18;19 never - +rollup rollup and the refresh after the rewrite recomputes nothing 15;16;17;18;19 2026-10-13 refresh_rollup not forgetting a changed group that is gone;refresh_rollup refusing any change but appends (phase 1, main before this branch) +rollup rollup and the refresh rewrote no partial outside the changed groups' time range 15;16;17;18;19 2026-10-13 refresh_rollup recomputing every bucket once any group changed +rollup rollup and the rollup no longer records the groups retention retired 15;16;17;18;19 2026-10-13 refresh_rollup keeping retired groups in its record rollup rollup and the rollup still equals a full recompute 15;16;17;18;19 2026-10-13 refresh_rollup combining min with greatest;refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's rollup rollup another role sees none of its rollup catalog rows 15;16;17;18;19 2026-10-13 the rollup catalogs without row-level security;the rollup catalogs without their grants rollup rollup each row group's TID range holds exactly that group's rows 15;16;17;18;19 2026-10-13 row_tid mapping 290 rows a block instead of the AM's rollup rollup premise: and the fixture has NULL-key and NULL-bucket groups 15;16;17;18;19 never - +rollup rollup premise: compact_rewrite rewrote a row group 15;16;17;18;19 never - +rollup rollup premise: expire retired the group in bucket D alone and kept the one that shares it 15;16;17;18;19 never - rollup rollup premise: expire retired the old row groups 15;16;17;18;19 never - rollup rollup premise: pgcolumnar.vacuum gave the source's row groups new identities 15;16;17;18;19 never - rollup rollup premise: the rollup functions exist 15;16;17;18;19 never - +rollup rollup premise: the rollup holds partials the delete below cannot reach 15;16;17;18;19 never - rollup rollup premise: the source holds every row 15;16;17;18;19 never - rollup rollup rebase_rollup records the source's current row groups 15;16;17;18;19 2026-10-13 rebase_rollup returning 0 rollup rollup refreshes from sessions in different time zones build the same rollup 15;16;17;18;19 2026-10-13 refresh_rollup bucketing with date_trunc, which follows the session's zone @@ -2042,7 +2057,6 @@ rollup rollup the history rollup holds old buckets 15;16;17;18;19 never - rollup rollup the next refresh folds the rows committed during the last one, once 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;refresh_rollup recording the folded groups from a fresh read after the fold;row_tid mapping 290 rows a block instead of the AM's rollup rollup the paused refresh folded what was there before it began 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's rollup rollup the refresh after a small append read the source at all 15;16;17;18;19 never - -rollup rollup the rollup holds rows before the delete 15;16;17;18;19 never - rollup rollup the rollup holds rows before the rewrite 15;16;17;18;19 never - rollup rollup the rollup table exists with the documented columns 15;16;17;18;19 never - rollup rollup the second refresh of the NULL-key rollup folded the append 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup folding only the first new group;row_tid mapping 290 rows a block instead of the AM's From 0b71e985a29a6232e448cc81317ce1ecfdb61904 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 13:16:55 +0000 Subject: [PATCH 06/11] test: restate the ledger budget for rollups phase 2, 1799 -> 1800 (#1294) Co-Authored-By: Claude Opus 5.5 --- test/check_ledger_budget.txt | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index df207daad..26c9c5897 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1576,4 +1576,23 @@ suites_not_covered 246 # never 1796 + 3 = 1799 # dated 463 + 8 = 471 # 1799 + 471 == 2270 -checks_never_observed_red 1799 +# +# #1294 phase 2, dirty buckets (jdatcmd): 1799 -> 1800. rollup gains 17 checks +# from a five-major run (70/70 on each of 15-19) and loses three. Fourteen of +# the new are dated by twelve mutations of the refresh and of rebase_rollup, and +# by the phase-1 code itself (these tests on main's refresh, 10 red on each +# major); three arrive `never`, all premises (compact_rewrite rewrote a group, +# expire kept the straddling group, partials beyond the delete's reach). One +# more mutation, ignoring a changed row count, reddened nothing, and the +# condition was removed from the code instead of kept untested. The three +# removed rows are phase 1's refusal of a delete and its two companions: phase +# 2 replaced that behaviour, so "a refresh after a delete is refused until +# phase 2" goes with its history (dated, refresh_rollup without the +# changed-group refusal), and "the refused refresh changed nothing" and "the +# rollup holds rows before the delete" go as never. +# +# ledger rows 2270 - 3 + 17 = 2284 +# never 1799 - 2 + 3 = 1800 +# dated 471 - 1 + 14 = 484 +# 1800 + 484 == 2284 +checks_never_observed_red 1800 From d6fdf62cdd0c2b10bebbceda7c8e7c9acffde68a Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 13:22:43 +0000 Subject: [PATCH 07/11] test: the pytest DST arm's bucket must contain the switch (#1294) ORIGIN in the pytest twin is 06:00 UTC, so the day bucket the first fixture aimed at began after Berlin's 01:00 UTC switch, and a session-zone '1 day' was 24 hours there: the arm stayed green under the mutation that adds the width in the session's zone. The rows now end at 05:40 UTC in the bucket [03-28 06:00, 03-29 06:00), which the switch is in. Under that mutation the arm is red on PG17; under each of the other eleven, the twin is red at the arm the shell twin names. Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_rollup.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/pytest/test_rollup.py b/test/pytest/test_rollup.py index 8ce3e9235..33d886e99 100644 --- a/test/pytest/test_rollup.py +++ b/test/pytest/test_rollup.py @@ -388,13 +388,15 @@ def layer(table): expect.text(f"{_refresh(conn, 'relayered_hourly')}|{_one(conn, rq)}", f"0A000|{before_re}", "a delete in a bucket whose history the same refresh retires is refused, and the rollup is unchanged") - # A day is 24 hours to date_bin in every zone. In Europe/Berlin 2026-03-29 is - # 23 hours long; the UTC day bucket must be recomputed to its end (rows run to - # 23:40 UTC). + # A day is 24 hours to date_bin in every zone. Europe/Berlin moves its clocks + # at 01:00 UTC on 2026-03-29, inside the bucket [03-28 06:00, 03-29 06:00) UTC + # (ORIGIN is at 06:00), so that bucket is 23 hours long in Berlin. It must be + # recomputed to its end: the rows run to 05:40 UTC. (Rows ending at 23:40 UTC + # first, a bucket the switch is not in, left this arm green under the mutation.) with conn.cursor() as cur: cur.execute("CREATE TABLE springs (at timestamptz, reading int) USING pgcolumnar") cur.execute("SELECT pgcolumnar.set_options('springs', stripe_row_limit => 4000)") - cur.execute("INSERT INTO springs SELECT '2026-03-28 00:00:00+00'::timestamptz + g * interval '4 min', g" + cur.execute("INSERT INTO springs SELECT '2026-03-27 06:00:00+00'::timestamptz + g * interval '4 min', g" " FROM generate_series(1, 715) g") cur.execute(f"SELECT pgcolumnar.create_rollup('springs_daily', 'springs', 'at', '1 day', '{ORIGIN}'," " '{}', '{count,max}', '{*,reading}')") From 7046662596473645f4b702ddbbded3c5c2160157 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 13:25:47 +0000 Subject: [PATCH 08/11] docs: TESTS.md section 97 covers rollups phase 2 (#1294) Co-Authored-By: Claude Opus 5.5 --- test/pytest/TESTS.md | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index fb85826b7..265d8cb1c 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -142,7 +142,7 @@ behaviour, the source of that number is named. - [94. test_approx_aggregates.py: hll and tdigest over columnar tables, the compatibility statement](#94-test_approx_aggregatespy-hll-and-tdigest-over-columnar-tables-the-compatibility-statement) - [95. test_docs_how_to_sections.py: the pg_partman how-to keeps its own heading](#95-test_docs_how_to_sectionspy-the-pg_partman-how-to-keeps-its-own-heading) - [96. test_autovacuum_expire.py: the daemon expires retention tables only when told to](#96-test_autovacuum_expirepy-the-daemon-expires-retention-tables-only-when-told-to) -- [97. test_rollup.py: incremental rollups fold appended row groups, and equal a full recompute](#97-test_rolluppy-incremental-rollups-fold-appended-row-groups-and-equal-a-full-recompute) +- [97. test_rollup.py: incremental rollups fold appended row groups, recompute changed ones, and equal a full recompute](#97-test_rolluppy-incremental-rollups-fold-appended-row-groups-recompute-changed-ones-and-equal-a-full-recompute) - [98. test_timeseries_recipes.py: the gap-fill, LOCF and interpolation recipes answer what they claim](#98-test_timeseries_recipespy-the-gap-fill-locf-and-interpolation-recipes-answer-what-they-claim) - [99. test_suite_exit_trap.py: a suite's own EXIT trap must still stop its cluster](#99-test_suite_exit_trappy-a-suites-own-exit-trap-must-still-stop-its-cluster) - [100. test_tier_partition.py: tier_partition moves a cold partition to Parquet, exactly once](#100-test_tier_partitionpy-tier_partition-moves-a-cold-partition-to-parquet-exactly-once) @@ -6603,9 +6603,9 @@ Written red first: on main the fixture raises `unrecognized configuration parame | `test_the_daemon_yields_the_lock_it_holds_inside_expire` | the worker holds ShareUpdateExclusiveLock inside `expire`; a session's ACCESS EXCLUSIVE is granted within 8 s of a 30 s hold, and the log shows the cancel | | `test_a_table_dropped_mid_pass_is_skipped_and_the_pass_goes_on` | while the worker holds `ex_hold`, `ex_gone` is dropped; after the cancel the same pass logs `ex_gone` as skipped and still expires `ex_after` | -## 97. test_rollup.py: incremental rollups fold appended row groups, and equal a full recompute +## 97. test_rollup.py: incremental rollups fold appended row groups, recompute changed ones, and equal a full recompute -Port of `rollup.sh` (#1294, phase 1: append-only). `create_rollup` makes a view over a partials table; `refresh_rollup` folds the rows of the row groups appended since the last refresh, in ONE statement, so the group list, the folded rows and the recorded groups share a snapshot. The design, its twelve points and the owner's decisions are on #1294. +Port of `rollup.sh` (#1294, phases 1 and 2). `create_rollup` makes a view over a partials table; `refresh_rollup` folds the rows of the row groups appended since the last refresh, in ONE statement, so the group list, the folded rows and the recorded groups share a snapshot. The design, its twelve points and the owner's decisions are on #1294. THE ORACLE is a full recompute of the same GROUP BY, every time. Exactness was fixed by the design: count, integer sum, min, max and the avg of an integer column exact; the avg of `float8` within a relative 1e-12. This file uses an integer avg; the shell twin a float one. @@ -6619,13 +6619,17 @@ From @jdatcmd's review of #1319: the refresh restated 291 rows a block in SQL, w A role that cannot SELECT the source cannot create a rollup of it (42501). `create_rollup` has no check of its own for this: it reads the source's storage id through `get_storage_id`, which refuses such a caller, and the arm was red only when that check was removed. `entry_point_privilege.sh` buckets `create_rollup` as relation-taking by its second argument and `row_tid` as taking no relation. -Removal proofs, each red on its arm: no `OFFSET 0` (the cost arm), no isolation check, no lock (two refreshes double-fold), no identity check, no changed-group check, the folded groups recorded from a fresh read (rows committed mid-refresh lost), min combined as max, no row-level security, a rebase that clears the rollup, `=` in place of `IS NOT DISTINCT FROM` (only the NULL arm), retired groups kept in the snapshot, and `row_tid` with 290 rows a block. +Phase 2, dirty buckets: a delete, an update or a `compact_rewrite` in the source makes the buckets of the changed row groups dirty, and the refresh recomputes them from the source in the same statement, folding new groups into the clean buckets only. A bucket that holds retained history is never recomputed: such a refresh is refused (`0A000`) and writes nothing, including when the history is retired by that same refresh. Arms: a delete (and no partial outside the changed groups' range rewritten), an update moving rows to another bucket, a delete plus an append straddling its bucket, `compact_rewrite`, a deleted row with a NULL ts, the two history cases, a day bucket refreshed in Europe/Berlin across its DST switch, and the record kept tidy (a recounted group, a forgotten one, retired groups dropped). Writing the DST arm showed that the first fixture's bucket began after the switch, because this file's ORIGIN is 06:00 UTC, so the arm was green under the mutation it was for; the rows were moved into the bucket that contains the switch. + +Phase 2 mutations, each red on PG17 at the arm the shell twin names: new rows folded into dirty buckets too, every bucket recomputed, no history refusal, blocking on stored history only, a rebase without ranges, the NULL bucket never dirty, the width added in the session's zone, new groups recorded without ranges, no recount, no forget, retired groups kept in the record, and the retired ranges never stored. + +Phase 1 removal proofs, each red on its arm: no `OFFSET 0` (the cost arm), no isolation check, no lock (two refreshes double-fold), no identity check, no changed-group check, the folded groups recorded from a fresh read (rows committed mid-refresh lost), min combined as max, no row-level security, a rebase that clears the rollup, `=` in place of `IS NOT DISTINCT FROM` (only the NULL arm), retired groups kept in the snapshot, and `row_tid` with 290 rows a block. ### Every test | test | what it holds | | --- | --- | -| `test_rollup_phase_one` | the rollup equals a full recompute after the first refresh, appends with late rows, a bulk append, and a rebase; a refresh with nothing new folds nothing; rows committed during a refresh are folded by the next, once; two concurrent refreshes fold each row once; zones agree; a refresh after a small append reads under a fifth of the source; a rewritten source is refused until `rebase_rollup`; a delete is refused (phase 1); a heap rollup behaves the same; REPEATABLE READ is refused; an owner who is not a superuser creates and refreshes a rollup, and another role sees none of it; bad definitions are refused with `22023` | +| `test_rollup_phase_one` | the rollup equals a full recompute after the first refresh, appends with late rows, a bulk append, and a rebase; a refresh with nothing new folds nothing; rows committed during a refresh are folded by the next, once; two concurrent refreshes fold each row once; zones agree; a refresh after a small append reads under a fifth of the source; a rewritten source is refused until `rebase_rollup`; after a delete, an update, a straddling append, `compact_rewrite` or a deleted NULL-ts row the rollup equals a full recompute, rewriting no partial outside the changed groups' range; a recompute that would reach retained history is refused with `0A000` and changes nothing; a day bucket is recomputed whole across a DST switch; a heap rollup behaves the same; REPEATABLE READ is refused; an owner who is not a superuser creates and refreshes a rollup, and another role sees none of it; bad definitions are refused with `22023` | ## 98. test_timeseries_recipes.py: the gap-fill, LOCF and interpolation recipes answer what they claim From 1548430a5d11b620f012fcf8534e9750a1ea2f6d Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 13:54:53 +0000 Subject: [PATCH 09/11] test: a delete in a group expire then retires is refused, red first (#1328 review) Found by @OffgridwithJD: rows deleted from a group that expire retires before the next refresh are folded into history, and their buckets keep counting them (1000 where the source had 600), with no error. The arm, in both twins, requires the refresh to refuse with 0A000 and leave the rollup unchanged. On PG17 before the fix the shell twin read ok and 1000. Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_rollup.py | 21 +++++++++++++++++++++ test/rollup.sh | 18 ++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/test/pytest/test_rollup.py b/test/pytest/test_rollup.py index 33d886e99..11e672f01 100644 --- a/test/pytest/test_rollup.py +++ b/test/pytest/test_rollup.py @@ -388,6 +388,27 @@ def layer(table): expect.text(f"{_refresh(conn, 'relayered_hourly')}|{_one(conn, rq)}", f"0A000|{before_re}", "a delete in a bucket whose history the same refresh retires is refused, and the rollup is unchanged") + # Rows deleted from a group expire then retires (found on #1328): they cannot + # be recomputed, and history would keep counting them, so the refresh refuses. + with conn.cursor() as cur: + cur.execute("CREATE TABLE thinned (at timestamptz, reading int) USING pgcolumnar") + cur.execute("SELECT pgcolumnar.set_options('thinned', stripe_row_limit => 2000)") + cur.execute("INSERT INTO thinned SELECT now() - interval '45 days' + g * interval '2 s', g" + " FROM generate_series(1, 1500) g") + cur.execute("INSERT INTO thinned SELECT now() - interval '2 days' + g * interval '2 s', g" + " FROM generate_series(1, 1500) g") + cur.execute(f"SELECT pgcolumnar.create_rollup('thinned_hourly', 'thinned', 'at', '1 hour', '{ORIGIN}'," + " '{}', '{count,max}', '{*,reading}')") + cur.execute("SELECT pgcolumnar.refresh_rollup('thinned_hourly')") + cur.execute("DELETE FROM thinned WHERE at < now() - interval '14 days' AND reading % 5 = 0") + cur.execute("SELECT pgcolumnar.set_options('thinned', ttl_column => 'at', ttl_interval => '14 days')") + expect.num(_one(conn, "SELECT pgcolumnar.expire('thinned')"), 1, + "premise: expire retired the group the delete thinned") + tq = "SELECT string_agg(bucket || ':' || count || ':' || max_reading, ',' ORDER BY bucket) FROM thinned_hourly" + before_thin = _one(conn, tq) + expect.text(f"{_refresh(conn, 'thinned_hourly')}|{_one(conn, tq)}", f"0A000|{before_thin}", + "a delete in a group that expire then retires is refused, and the rollup is unchanged") + # A day is 24 hours to date_bin in every zone. Europe/Berlin moves its clocks # at 01:00 UTC on 2026-03-29, inside the bucket [03-28 06:00, 03-29 06:00) UTC # (ORIGIN is at 06:00), so that bucket is 23 hours long in Berlin. It must be diff --git a/test/rollup.sh b/test/rollup.sh index 250ecac58..8662648ac 100755 --- a/test/rollup.sh +++ b/test/rollup.sh @@ -331,6 +331,24 @@ check "a delete in a bucket whose history the same refresh retires is refused, a "$(sqlstate "SELECT pgcolumnar.refresh_rollup('r_hd2');")|$(q "SELECT md5(string_agg(bucket || ':' || count || ':' || sum_v, ',' ORDER BY bucket)) FROM r_hd2;")" \ "0A000|$hd2_before" +# Rows deleted from a group that expire then retires, before the next refresh: the +# rows are gone, so their buckets cannot be recomputed, and folding the group's +# range into history would keep counting them. Found by @OffgridwithJD on #1328 +# (1000 counted where the source had 600). The refresh is refused instead. +q "CREATE TABLE dx (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('dx', stripe_row_limit => 1000); + INSERT INTO dx SELECT now() - interval '60 days' + g * interval '1 s', g FROM generate_series(1, 1000) g; + INSERT INTO dx SELECT now() - interval '1 day' + g * interval '1 s', g FROM generate_series(1, 1000) g; + SELECT pgcolumnar.create_rollup('r_dx', 'dx', 'ts', '1 day', '$ORIGIN', '{}', '{count}', '{*}'); + SELECT pgcolumnar.refresh_rollup('r_dx'); + DELETE FROM dx WHERE ts < now() - interval '30 days' AND v <= 400; + SELECT pgcolumnar.set_options('dx', ttl_column => 'ts', ttl_interval => '30 days');" >/dev/null +check_num "premise: expire retired the group the delete thinned" "$(q "SELECT pgcolumnar.expire('dx');")" "1" +dx_before="$(q "SELECT string_agg(bucket || ':' || count, ',' ORDER BY bucket) FROM r_dx;")" +check "a delete in a group that expire then retires is refused, and the rollup is unchanged" \ + "$(sqlstate "SELECT pgcolumnar.refresh_rollup('r_dx');")|$(q "SELECT string_agg(bucket || ':' || count, ',' ORDER BY bucket) FROM r_dx;")" \ + "0A000|$dx_before" + # A day bucket is 24 hours (date_bin's day), whatever the session's zone. In # America/New_York, 2026-03-08 is 23 hours long, and the dirty bucket of that UTC # day must still be recomputed up to its end: rows run to 23:30 UTC. From f6b2bf5811fca22c7658b8890f0fba7ebbc78d78 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 13:54:53 +0000 Subject: [PATCH 10/11] fix: refuse a refresh when rows were deleted from a group expire then retired (#1328 review) expire now records each retired group's deleted count in retired_group, read before the group's delete vector goes with it. A refresh that retires a recorded group whose count moved since the rollup last folded it refuses with 0A000: the rows are gone, so the buckets cannot be recomputed, and keeping the group's range as history would keep counting them. Co-Authored-By: Claude Opus 5.5 --- pgcolumnar--1.0-alpha4--1.0-alpha5.sql | 17 +++++++++++++---- pgcolumnar--1.0-alpha5.sql | 17 +++++++++++++---- src/columnar.h | 3 ++- src/columnar_metadata.c | 10 +++++++--- src/columnar_vacuum.c | 13 ++++++++++++- 5 files changed, 47 insertions(+), 13 deletions(-) diff --git a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql index 28bec6628..58c7f0dae 100644 --- a/pgcolumnar--1.0-alpha4--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha4--1.0-alpha5.sql @@ -972,13 +972,17 @@ $create_ahead$; * refresh must tell a group retired by retention from one deleted or * rewritten; this is where it looks. Nothing but expire writes here: a * compaction's retired groups are a change to the data, not retention. + * deleted_count is the group's deleted count when expire retired it: rows + * deleted after a rollup's last refresh cannot be recomputed once the group is + * gone, and this is how the refresh sees that (found on #1328). * Deleted with the storage's other metadata. * ------------------------------------------------------------------------- */ CREATE TABLE pgcolumnar.retired_group ( storage_id bigint NOT NULL, group_number bigint NOT NULL, - retired_at timestamptz NOT NULL + retired_at timestamptz NOT NULL, + deleted_count bigint NOT NULL ); CREATE UNIQUE INDEX retired_group_pkey ON pgcolumnar.retired_group USING btree (storage_id, group_number); @@ -1110,14 +1114,14 @@ $rsg$; * no GRANT; the caller must be able to SELECT the source. */ CREATE FUNCTION pgcolumnar.rollup_source_retired(source regclass) - RETURNS TABLE (group_number bigint) + RETURNS TABLE (group_number bigint, deleted_count bigint) LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $rsr$ BEGIN PERFORM pgcolumnar.require_caller_select(source); RETURN QUERY - SELECT r.group_number FROM pgcolumnar.retired_group r + SELECT r.group_number, r.deleted_count FROM pgcolumnar.retired_group r WHERE r.storage_id = pgcolumnar.get_storage_id(source); END $rsr$; @@ -1409,6 +1413,11 @@ BEGIN FROM pgcolumnar.rollup x WHERE x.rollup = %2$L::regclass ), blocked AS ( SELECT 1 FROM dirty d, kept k WHERE d.b && k.m OR (d.n AND k.n) + UNION ALL + -- rows deleted since the last refresh from a group expire then + -- retired: they are gone, so their buckets cannot be recomputed + SELECT 1 FROM retiring t JOIN ret x USING (group_number) + WHERE x.deleted_count <> t.deleted_count ), new_groups AS ( SELECT c.* FROM cur c WHERE NOT EXISTS (SELECT 1 FROM rec rg WHERE rg.group_number = c.group_number) @@ -1493,7 +1502,7 @@ BEGIN IF refused THEN RAISE EXCEPTION 'rollup %: a changed row group of its source reaches buckets whose raw rows retention removed', rollup USING ERRCODE = 'feature_not_supported', - DETAIL = 'Recomputing those buckets from the source would drop the history they keep.', + DETAIL = 'Recomputing those buckets from the source would drop the history they keep, or the rows to recompute them from are gone.', HINT = 'If the rollup is right as it stands, pgcolumnar.rebase_rollup records the source''s current row groups.'; END IF; RETURN folded; diff --git a/pgcolumnar--1.0-alpha5.sql b/pgcolumnar--1.0-alpha5.sql index ca7114249..23fdd601f 100644 --- a/pgcolumnar--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha5.sql @@ -330,13 +330,17 @@ CREATE INDEX free_space_fit * refresh must tell a group retired by retention from one deleted or * rewritten; this is where it looks. Nothing but expire writes here: a * compaction's retired groups are a change to the data, not retention. + * deleted_count is the group's deleted count when expire retired it: rows + * deleted after a rollup's last refresh cannot be recomputed once the group is + * gone, and this is how the refresh sees that (found on #1328). * Deleted with the storage's other metadata. * ------------------------------------------------------------------------- */ CREATE TABLE pgcolumnar.retired_group ( storage_id bigint NOT NULL, group_number bigint NOT NULL, - retired_at timestamptz NOT NULL + retired_at timestamptz NOT NULL, + deleted_count bigint NOT NULL ); CREATE UNIQUE INDEX retired_group_pkey ON pgcolumnar.retired_group USING btree (storage_id, group_number); @@ -2300,14 +2304,14 @@ $rsg$; * no GRANT; the caller must be able to SELECT the source. */ CREATE FUNCTION pgcolumnar.rollup_source_retired(source regclass) - RETURNS TABLE (group_number bigint) + RETURNS TABLE (group_number bigint, deleted_count bigint) LANGUAGE plpgsql STABLE SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $rsr$ BEGIN PERFORM pgcolumnar.require_caller_select(source); RETURN QUERY - SELECT r.group_number FROM pgcolumnar.retired_group r + SELECT r.group_number, r.deleted_count FROM pgcolumnar.retired_group r WHERE r.storage_id = pgcolumnar.get_storage_id(source); END $rsr$; @@ -2599,6 +2603,11 @@ BEGIN FROM pgcolumnar.rollup x WHERE x.rollup = %2$L::regclass ), blocked AS ( SELECT 1 FROM dirty d, kept k WHERE d.b && k.m OR (d.n AND k.n) + UNION ALL + -- rows deleted since the last refresh from a group expire then + -- retired: they are gone, so their buckets cannot be recomputed + SELECT 1 FROM retiring t JOIN ret x USING (group_number) + WHERE x.deleted_count <> t.deleted_count ), new_groups AS ( SELECT c.* FROM cur c WHERE NOT EXISTS (SELECT 1 FROM rec rg WHERE rg.group_number = c.group_number) @@ -2683,7 +2692,7 @@ BEGIN IF refused THEN RAISE EXCEPTION 'rollup %: a changed row group of its source reaches buckets whose raw rows retention removed', rollup USING ERRCODE = 'feature_not_supported', - DETAIL = 'Recomputing those buckets from the source would drop the history they keep.', + DETAIL = 'Recomputing those buckets from the source would drop the history they keep, or the rows to recompute them from are gone.', HINT = 'If the rollup is right as it stands, pgcolumnar.rebase_rollup records the source''s current row groups.'; END IF; RETURN folded; diff --git a/src/columnar.h b/src/columnar.h index df0fca080..cb0f2c359 100644 --- a/src/columnar.h +++ b/src/columnar.h @@ -610,7 +610,8 @@ extern void PgColumnarCloseZoneMapSession(PgColumnarZoneMapSession *sess); extern List *PgColumnarReadWholeChunkZoneMapsForColumn(uint64 storageId, int columnIndex, Snapshot snapshot); extern void PgColumnarDeleteMetadata(uint64 storageId); -extern void PgColumnarRecordRetention(uint64 storageId, uint64 groupNumber); +extern void PgColumnarRecordRetention(uint64 storageId, uint64 groupNumber, + uint64 deletedCount); /* pgcolumnar.column_extent (#1304 option 5): columnar_metadata.c */ typedef struct NativeExtentRow diff --git a/src/columnar_metadata.c b/src/columnar_metadata.c index 37228e17a..21efd3add 100644 --- a/src/columnar_metadata.c +++ b/src/columnar_metadata.c @@ -152,7 +152,8 @@ #define Anum_retired_group_storage_id 1 #define Anum_retired_group_group_number 2 #define Anum_retired_group_retired_at 3 -#define Natts_retired_group 3 +#define Anum_retired_group_deleted_count 4 +#define Natts_retired_group 4 /* attribute numbers for columnar.delete_vector (spec 7.5) */ #define Anum_delete_vector_storage_id 1 @@ -985,11 +986,13 @@ record_free_space(uint64 storageId, uint64 fileOffset, uint64 byteLength) * Record that pgcolumnar.expire retired a row group for retention (#1294). * An incremental rollup that keeps history must not recompute a bucket whose * raw rows retention dropped, and this row is how its refresh tells such a - * group from one a delete or a rewrite removed. Called only by expire, in its + * group from one a delete or a rewrite removed. deletedCount is the group's + * deleted count at retirement, so a refresh can tell that rows were deleted + * since it last folded the group (#1328). Called only by expire, in its * transaction, so a rolled-back expire leaves no record. */ void -PgColumnarRecordRetention(uint64 storageId, uint64 groupNumber) +PgColumnarRecordRetention(uint64 storageId, uint64 groupNumber, uint64 deletedCount) { Relation rel = open_columnar_table("retired_group", RowExclusiveLock); Datum values[Natts_retired_group]; @@ -1000,6 +1003,7 @@ PgColumnarRecordRetention(uint64 storageId, uint64 groupNumber) values[Anum_retired_group_storage_id - 1] = Int64GetDatum((int64) storageId); values[Anum_retired_group_group_number - 1] = Int64GetDatum((int64) groupNumber); values[Anum_retired_group_retired_at - 1] = TimestampTzGetDatum(GetCurrentTransactionStartTimestamp()); + values[Anum_retired_group_deleted_count - 1] = Int64GetDatum((int64) deletedCount); tuple = heap_form_tuple(RelationGetDescr(rel), values, nulls); CatalogTupleInsert(rel, tuple); diff --git a/src/columnar_vacuum.c b/src/columnar_vacuum.c index 0507e9d0d..3e38a0019 100644 --- a/src/columnar_vacuum.c +++ b/src/columnar_vacuum.c @@ -2626,9 +2626,20 @@ pgcolumnar_expire(PG_FUNCTION_ARGS) att->attcollation, maxv, cutoff)); if (c < 0) { + /* + * The deleted count is read before the group's delete vector goes + * with it: the same stored count a rollup's refresh compares (#1328). + */ + uint64 deleted = 0; + ListCell *dc; + + foreach(dc, PgColumnarReadDeleteVectorList(storageId, rg->groupNumber, + GetActiveSnapshot())) + deleted += (uint64) ((DeleteVectorMetadata *) lfirst(dc))->deletedCount; + PgColumnarVMClearForRowRange(rel, rg->firstRowNumber, rg->rowCount); PgColumnarRetireGroup(storageId, rg->groupNumber); - PgColumnarRecordRetention(storageId, rg->groupNumber); + PgColumnarRecordRetention(storageId, rg->groupNumber, deleted); retired++; } } From 9683118bb133cdc513699f3ed807c715075c487f Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 14:05:50 +0000 Subject: [PATCH 11/11] test: ledger rows for the delete-then-expire refusal, 1800 -> 1801 (#1328 review) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++++ test/check_ledger.tsv | 2 ++ test/check_ledger_budget.txt | 13 ++++++++++++- 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83634090b..afc861fe2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -207,6 +207,10 @@ true until the next version shipped. look like a delete and lose history, so it is read in the fold's own snapshot; and a bucket's end is its start plus the width in plain seconds, because a `timestamptz` plus `'1 day'` is 23 or 25 hours across a DST change. + From @OffgridwithJD's review: rows deleted from a group that `expire` then + retires, before the next refresh, cannot be recomputed and would be kept as + history, so such a refresh is refused (`0A000`) too; `expire` now records each + retired group's deleted count in `pgcolumnar.retired_group` for this. - `pgcolumnar.tier_partition(partition, server, path, lock_wait)` moves a cold columnar partition to a Parquet file and attaches the file in its place as a foreign partition with the same name, bounds, grants and comment (#1295). It diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index bb7232c1f..b19f8618f 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -1995,6 +1995,7 @@ rewrite_storage_oid rewrite_storage_oid the repair gate does not read a partitio rollup rollup * with anything but count is refused 15;16;17;18;19 2026-10-13 create_rollup allowing * with any function rollup rollup a delete in a bucket that holds retired history is refused, and the rollup is unchanged 15;16;17;18;19 2026-10-13 refresh_rollup not storing the retired groups' ranges as history;refresh_rollup recording new groups without their time range;refresh_rollup without the history refusal rollup rollup a delete in a bucket whose history the same refresh retires is refused, and the rollup is unchanged 15;16;17;18;19 2026-10-13 refresh_rollup blocking on stored history only, not on groups the same refresh retires;refresh_rollup recording new groups without their time range;refresh_rollup without the history refusal +rollup rollup a delete in a group that expire then retires is refused, and the rollup is unchanged 15;16;17;18;19 2026-10-13 expire recording a retired group's deleted count as 0;refresh_rollup not refusing a retiring group whose deleted count moved rollup rollup a delete in buckets with no history, next to retired ones, folds: those buckets equal a recompute and D keeps its history 15;16;17;18;19 2026-10-13 refresh_rollup recomputing every bucket once any group changed;refresh_rollup recording new groups without their time range;refresh_rollup refusing any change but appends (phase 1, main before this branch) rollup rollup a refresh after a delete is not refused 15;16;17;18;19 2026-10-13 refresh_rollup refusing any change but appends (phase 1, main before this branch) rollup rollup a refresh after a small append reads under a fifth of the source 15;16;17;18;19 2026-10-13 refresh_rollup folding every group, ignoring the recorded snapshot;refresh_rollup's per-group subquery without OFFSET 0, so it becomes a join and reads every group @@ -2044,6 +2045,7 @@ rollup rollup each row group's TID range holds exactly that group's rows 15;16;1 rollup rollup premise: and the fixture has NULL-key and NULL-bucket groups 15;16;17;18;19 never - rollup rollup premise: compact_rewrite rewrote a row group 15;16;17;18;19 never - rollup rollup premise: expire retired the group in bucket D alone and kept the one that shares it 15;16;17;18;19 never - +rollup rollup premise: expire retired the group the delete thinned 15;16;17;18;19 never - rollup rollup premise: expire retired the old row groups 15;16;17;18;19 never - rollup rollup premise: pgcolumnar.vacuum gave the source's row groups new identities 15;16;17;18;19 never - rollup rollup premise: the rollup functions exist 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 26c9c5897..e474aa664 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1595,4 +1595,15 @@ suites_not_covered 246 # never 1799 - 2 + 3 = 1800 # dated 471 - 1 + 14 = 484 # 1800 + 484 == 2284 -checks_never_observed_red 1800 +# +# #1328 review (OffgridwithJD found rows deleted from a group that expire then +# retires were kept in history): 1800 -> 1801. rollup gains two checks from a +# five-major run (72/72 on each of 15-19): the refusal, dated by two mutations +# (the refusal removed, and expire recording the deleted count as 0), each red +# at it alone on every major; and its premise, that expire retired the group. +# +# ledger rows 2284 + 2 = 2286 +# never 1800 + 1 = 1801 +# dated 484 + 1 = 485 +# 1801 + 485 == 2286 +checks_never_observed_red 1801