From 280849e6a7f9d8d0ed38f0e4e2d0127a0b5f54fc Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 22:41:07 +0000 Subject: [PATCH 1/7] test: the daemon expires TTL tables only when autovacuum_expire is on (#1290), red Sixteen checks over one daemon. Red on main at PG17, for the intended reasons: the switch does not exist; the daemon never retires the aged-out groups and logs nothing; it never holds a lock inside expire, so the yield arm has no window. The controls are green on main: with the daemon running passes (it compacts a deleted-heavy table in the window) an aged table keeps its rows; a table with no retention keeps every row; and an expire that fails with 42703 does not undo a compaction due on the same table. Co-Authored-By: Claude Opus 5.5 --- test/autovacuum_expire.sh | 154 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100755 test/autovacuum_expire.sh diff --git a/test/autovacuum_expire.sh b/test/autovacuum_expire.sh new file mode 100755 index 000000000..1877a611d --- /dev/null +++ b/test/autovacuum_expire.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash +# +# pgColumnar #1290: the maintenance daemon runs pgcolumnar.expire for every +# table that declares a retention, when -- and only when -- both +# pgcolumnar.autovacuum and pgcolumnar.autovacuum_expire are on. The second +# switch is off by default (the owner's decision on #1290): expire deletes +# rows, so turning the daemon on must not start deleting them by itself. +# +# The arms that matter: +# off daemon on, expire off: an aged-out table keeps its rows, while a +# deleted-heavy table IS compacted in the same window, so the daemon +# was demonstrably running passes and chose not to expire. +# on turning expire on retires the aged-out rows within a few naptimes; +# no row inside the retention is lost; a table with no retention, +# holding the same old timestamps, keeps every row and is never +# named in a "skipped" line. +# isolate an expire that errors (its TTL column was dropped) does not roll +# back a compact_rewrite due on the same table in the same pass. +# yield the daemon holds ShareUpdateExclusiveLock inside expire (the only +# verb due on that table) and gives it up to a session's +# AccessExclusive request well inside the hold. +# +# Timing: naptime is 2s, so the polls below cover several passes. +# +# Usage: test/autovacuum_expire.sh [PG_CONFIG] +# Written fresh for pgColumnar. + +set -uo pipefail +export PGC_EXTRA_CONF="pgcolumnar.autovacuum=on +pgcolumnar.autovacuum_naptime=2 +pgcolumnar.autovacuum_compact_threshold=0.1 +deadlock_timeout=1s" +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +CG=1000 +OLD="now() - interval '30 days'" +aged() { q "SELECT count(*) FROM $1 WHERE ts < now() - interval '1 day';"; } +fresh() { q "SELECT count(*) FROM $1 WHERE ts >= now() - interval '1 day';"; } +deleted_rows() { q "SELECT COALESCE(sum(deletedrows),0) FROM pgcolumnar.stats('$1');"; } + +# mk_ttl : two row groups 30 days old and one fresh, retention 1 day. +mk_ttl() { + psql_run "CREATE TABLE $1 (ts timestamptz, v int) USING pgcolumnar;" + psql_run "SELECT pgcolumnar.set_options('$1', chunk_group_row_limit => $CG, stripe_row_limit => $CG, + ttl_column => 'ts', ttl_interval => '1 day');" + psql_run "INSERT INTO $1 SELECT $OLD + g * interval '1 second', g FROM generate_series(1, 2*$CG) g;" + psql_run "INSERT INTO $1 SELECT now() + g * interval '1 second', g FROM generate_series(1, $CG) g;" +} + +# poll : re-run the command until it prints want. +poll() { + local i got + for i in $(seq 1 "$1"); do + got="$(eval "$2")" + [ "$got" = "$3" ] && break + sleep 1 + done + printf '%s' "$got" +} + +check "premise: the maintenance launcher is running" \ + "$(q "SELECT count(*) FROM pg_stat_activity WHERE backend_type = 'pgcolumnar autovacuum launcher';")" "1" +check "the daemon's expire switch exists and is OFF by default" \ + "$(q "SELECT current_setting('pgcolumnar.autovacuum') || '/' || current_setting('pgcolumnar.autovacuum_expire');")" "on/off" + +# ---- the fixture is one an expire would change ------------------------------ +mk_ttl ex_copy +check_num "premise: a manual expire on the same fixture retires its two aged groups" \ + "$(q "SELECT pgcolumnar.expire('ex_copy');")" "2" + +# ---- OFF: the daemon runs, compacts, and does not expire -------------------- +mk_ttl ex_off +psql_run "CREATE TABLE ex_busy (id int, v int) USING pgcolumnar;" +psql_run "SELECT pgcolumnar.set_options('ex_busy', chunk_group_row_limit => $CG);" +psql_run "INSERT INTO ex_busy SELECT g, g FROM generate_series(1, 20000) g;" +psql_run "DELETE FROM ex_busy WHERE (id * 2654435761)::bigint % 100 < 40;" +check "premise: the daemon compacted a deleted-heavy table, so it was running passes" \ + "$(poll 40 'deleted_rows ex_busy' 0)" "0" +sleep 4 # two more passes over ex_off +check_num "OFF: with autovacuum_expire off the daemon retired nothing" "$(aged ex_off)" "$((2*CG))" + +# ---- ON: aged rows go, fresh rows and the no-retention table stay ----------- +psql_run "CREATE TABLE ex_none (ts timestamptz, v int) USING pgcolumnar;" +psql_run "SELECT pgcolumnar.set_options('ex_none', chunk_group_row_limit => $CG, stripe_row_limit => $CG);" +psql_run "INSERT INTO ex_none SELECT $OLD + g * interval '1 second', g FROM generate_series(1, 2*$CG) g;" +psql_run "ALTER SYSTEM SET pgcolumnar.autovacuum_expire = on;" +psql_run "SELECT pg_reload_conf();" +check_num "ON: the daemon expired the aged-out rows without being asked" \ + "$(poll 40 'aged ex_off' 0)" "0" +check_num "ON: no row inside the retention was lost" "$(fresh ex_off)" "$CG" +check_num "ON: a table with no retention keeps every row" "$(aged ex_none)" "$((2*CG))" +check "ON: the daemon logged what it retired" \ + "$(grep -c 'pgcolumnar autovacuum: expire .*ex_off retired 2 row groups' "$PGC_LOGFILE" 2>/dev/null | awk '{print ($1 >= 1) ? "yes" : "no"}')" "yes" +check_num "ON: a table with no retention is never named in a skipped line" \ + "$(grep -c 'pgcolumnar autovacuum: skipped .*ex_none' "$PGC_LOGFILE" 2>/dev/null)" "0" + +# ---- isolate: a failing expire does not undo the compaction ----------------- +# Built with expire off, so neither verb runs until both conditions are set. +psql_run "ALTER SYSTEM SET pgcolumnar.autovacuum_expire = off;" +psql_run "SELECT pg_reload_conf();" +sleep 3 +psql_run "CREATE TABLE ex_bad (id int, ts timestamptz, v int) USING pgcolumnar;" +psql_run "SELECT pgcolumnar.set_options('ex_bad', chunk_group_row_limit => $CG, ttl_column => 'ts', ttl_interval => '1 day');" +psql_run "INSERT INTO ex_bad SELECT g, now(), g FROM generate_series(1, 20000) g;" +psql_run "DELETE FROM ex_bad WHERE (id * 2654435761)::bigint % 100 < 40;" +psql_run "ALTER TABLE ex_bad DROP COLUMN ts;" +# q() discards stderr, so the error is read as its SQLSTATE from psql directly. +sqlstate_of() { + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -qtA -v VERBOSITY=sqlstate -c "$1" 2>&1 | sed -n 's/^ERROR: \([0-9A-Z]\{5\}\).*/\1/p' | head -1 +} +check "premise: expire on a table whose TTL column is gone fails with undefined_column" \ + "$(sqlstate_of "SELECT pgcolumnar.expire('ex_bad');")" "42703" +psql_run "ALTER SYSTEM SET pgcolumnar.autovacuum_expire = on;" +psql_run "SELECT pg_reload_conf();" +check "isolate: a failing expire does not roll back the compaction due on the same table" \ + "$(poll 40 'deleted_rows ex_bad' 0)" "0" + +# ---- yield: the daemon gives up expire's lock to an AccessExclusive request -- +# ex_hold has a retention and nothing else due -- no deletes, no sort key -- so +# a ShareUpdateExclusiveLock the daemon worker holds on it is expire's. The +# other retention tables go first: with the hold on, the worker spends 30s in +# expire on EACH of them, and reached ex_hold only after the poll gave up. +psql_run "DROP TABLE ex_copy, ex_off, ex_bad;" +psql_run "ALTER SYSTEM SET pgcolumnar.maintenance_hold_ms = 30000;" +psql_run "SELECT pg_reload_conf();" +mk_ttl ex_hold +holds() { + q "SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid + WHERE l.relation = 'ex_hold'::regclass AND l.mode = 'ShareUpdateExclusiveLock' + AND l.granted AND a.backend_type = 'pgcolumnar autovacuum worker';" +} +check "premise: the daemon holds ShareUpdateExclusiveLock inside expire (the hold window is open)" \ + "$(poll 40 holds 1)" "1" +t0=$(date +%s) +STATE="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ + -d "$PGC_DB" -qtA 2>&1 </dev/null && echo fast || echo "slow(${elapsed}s)")" "fast" +sleep 1 +check "yield: the daemon logged cancelling expire" \ + "$([ "$(grep -c 'pgcolumnar autovacuum: skipped .*ex_hold.* canceling statement' "$PGC_LOGFILE" 2>/dev/null)" -ge 1 ] && echo yes || echo no)" "yes" +psql_run "ALTER SYSTEM RESET pgcolumnar.maintenance_hold_ms;" +psql_run "SELECT pg_reload_conf();" + +pgc_summary From 4f5a90ce268d93276ee16fedb95209fcba217840 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 22:41:07 +0000 Subject: [PATCH 2/7] feat: the maintenance daemon runs expire when autovacuum_expire is on (#1290) pgcolumnar.autovacuum_expire, off by default (the owner's decision on #1290): expire deletes rows, so turning the daemon on does not by itself apply a retention. When both switches are on, each pass runs pgcolumnar.expire for every table whose options declare both ttl_column and ttl_interval, before compact_rewrite and recluster, in its own transaction, so an expire that fails is logged and skipped without rolling back a compaction. expire now calls the maintenance hold after taking its lock, as compact_rewrite does, so the daemon's yield is observable there too. autovacuum_expire: 16/16 on PG17. Unchanged and green on PG17: autovacuum 11/11, autovacuum_yield 6/6, ttl_expire 51/51, maintenance_due 27/27. Co-Authored-By: Claude Opus 5.5 --- src/columnar.h | 1 + src/columnar_autovacuum.c | 90 ++++++++++++++++++++++++++++++++++++--- src/columnar_tableam.c | 13 +++++- src/columnar_vacuum.c | 3 ++ 4 files changed, 101 insertions(+), 6 deletions(-) diff --git a/src/columnar.h b/src/columnar.h index 075316577..562d4e1df 100644 --- a/src/columnar.h +++ b/src/columnar.h @@ -262,6 +262,7 @@ extern bool pgcolumnar_autovacuum; extern int pgcolumnar_autovacuum_naptime; extern double pgcolumnar_autovacuum_compact_threshold; extern double pgcolumnar_autovacuum_recluster_threshold; +extern bool pgcolumnar_autovacuum_expire; extern void PgColumnarAutovacuumRegister(void); extern int pgcolumnar_maintenance_hold_ms; /* dev/test: hold SUEL this long in a maintenance verb */ diff --git a/src/columnar_autovacuum.c b/src/columnar_autovacuum.c index 20827767d..335f3fac6 100644 --- a/src/columnar_autovacuum.c +++ b/src/columnar_autovacuum.c @@ -12,8 +12,10 @@ * * TWO INVARIANTS make this safe to run unattended: * - * 1. Only the ShareUpdateExclusiveLock verbs -- pgcolumnar.compact_rewrite and - * pgcolumnar.recluster (now self-gating, #415 part A) -- are ever called. + * 1. Only the ShareUpdateExclusiveLock verbs -- pgcolumnar.compact_rewrite, + * pgcolumnar.recluster (now self-gating, #415 part A) and, when + * pgcolumnar.autovacuum_expire is on, pgcolumnar.expire (#1290) -- are ever + * called. * Never vacuum()/vacuum_sorted()/cluster(), which take AccessExclusiveLock. * SUEL does not block readers or ordinary writers, so the daemon cannot block * production by construction. @@ -218,6 +220,68 @@ av_mark_as_autovacuum(void) #endif } +/* + * Run pgcolumnar.expire on one table that declares a retention (#1290), in its + * own transaction inside PG_TRY. + * + * ITS OWN TRANSACTION, NOT av_maintain_one's: expire deletes rows and can fail + * on its own account -- the declared retention column was dropped, or its type + * changed -- and a failure here must not roll back a compact_rewrite that was + * due on the same table. It runs FIRST, so a compaction does not rewrite row + * groups that are about to be retired. + * + * Only reached when pgcolumnar.autovacuum_expire is on. That switch is off by + * default (the owner's decision on #1290): turning the daemon on must not + * start deleting rows by itself. + */ +static void +av_expire_one(const char *qualname) +{ + StartTransactionCommand(); + PushActiveSnapshot(GetTransactionSnapshot()); + SetCurrentStatementStartTimestamp(); + + PG_TRY(); + { + char q[512]; + int64 retired = 0; + + if (SPI_connect() != SPI_OK_CONNECT) + elog(ERROR, "pgcolumnar autovacuum: SPI_connect failed"); + + snprintf(q, sizeof(q), "SELECT pgcolumnar.expire(%s)", + quote_literal_cstr(qualname)); + if (SPI_execute(q, false, 1) == SPI_OK_SELECT && SPI_processed == 1) + { + bool isnull; + Datum d = SPI_getbinval(SPI_tuptable->vals[0], + SPI_tuptable->tupdesc, 1, &isnull); + + if (!isnull) + retired = DatumGetInt64(d); + } + elog(LOG, "pgcolumnar autovacuum: expire %s retired " INT64_FORMAT " row groups", + qualname, retired); + + SPI_finish(); + PopActiveSnapshot(); + CommitTransactionCommand(); + } + PG_CATCH(); + { + MemoryContext ecxt = MemoryContextSwitchTo(TopMemoryContext); + ErrorData *ed = CopyErrorData(); + + elog(LOG, "pgcolumnar autovacuum: skipped expire on %s: %s", qualname, ed->message); + FreeErrorData(ed); + MemoryContextSwitchTo(ecxt); + + FlushErrorState(); + AbortCurrentTransaction(); + } + PG_END_TRY(); +} + /* * Run one maintenance verb on one table, each in its own transaction inside * PG_TRY: a query-cancel (the lock yield) or any error aborts just this op and @@ -358,7 +422,9 @@ pgcolumnar_av_worker_main(Datum arg) Oid dbid; uint32 conn_flags = 0; List *tables = NIL; + List *hasTtl = NIL; ListCell *lc; + ListCell *lt; memcpy(&dbid, MyBgworkerEntry->bgw_extra, sizeof(Oid)); @@ -373,16 +439,23 @@ pgcolumnar_av_worker_main(Datum arg) av_mark_as_autovacuum(); - /* Snapshot the columnar-table list once, in its own transaction. */ + /* + * Snapshot the columnar-table list once, in its own transaction, with + * whether each declares a retention. Both halves must be set, as + * PgColumnarReadTtl reads it: a half declaration is no retention, and + * calling expire on it would only log an error every pass. + */ StartTransactionCommand(); PushActiveSnapshot(GetTransactionSnapshot()); if (SPI_connect() == SPI_OK_CONNECT) { int ret = SPI_execute( - "SELECT quote_ident(n.nspname) || '.' || quote_ident(c.relname) " + "SELECT quote_ident(n.nspname) || '.' || quote_ident(c.relname), " + " COALESCE(o.ttl_column IS NOT NULL AND o.ttl_interval IS NOT NULL, false) " "FROM pg_class c " "JOIN pg_am a ON a.oid = c.relam " "JOIN pg_namespace n ON n.oid = c.relnamespace " + "LEFT JOIN pgcolumnar.options o ON o.regclass = c.oid " "WHERE a.amname = 'pgcolumnar' AND c.relkind = 'r'", true, 0); if (ret == SPI_OK_SELECT) @@ -397,9 +470,13 @@ pgcolumnar_av_worker_main(Datum arg) if (!isnull) { + bool ttlnull; + Datum t = SPI_getbinval(SPI_tuptable->vals[i], + SPI_tuptable->tupdesc, 2, &ttlnull); MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); tables = lappend(tables, pstrdup(TextDatumGetCString(d))); + hasTtl = lappend_int(hasTtl, !ttlnull && DatumGetBool(t)); MemoryContextSwitchTo(old); } } @@ -409,9 +486,12 @@ pgcolumnar_av_worker_main(Datum arg) PopActiveSnapshot(); CommitTransactionCommand(); - foreach(lc, tables) + forboth(lc, tables, lt, hasTtl) { CHECK_FOR_INTERRUPTS(); + if (pgcolumnar_autovacuum_expire && lfirst_int(lt)) + av_expire_one((const char *) lfirst(lc)); + CHECK_FOR_INTERRUPTS(); av_maintain_one((const char *) lfirst(lc)); } /* worker exits; the launcher starts a fresh one next naptime */ diff --git a/src/columnar_tableam.c b/src/columnar_tableam.c index 493450acc..c1e340065 100644 --- a/src/columnar_tableam.c +++ b/src/columnar_tableam.c @@ -94,6 +94,7 @@ bool pgcolumnar_autovacuum = false; /* #415 daemon master switch */ int pgcolumnar_autovacuum_naptime = 60; double pgcolumnar_autovacuum_compact_threshold = 0.2; double pgcolumnar_autovacuum_recluster_threshold = 0.05; +bool pgcolumnar_autovacuum_expire = false; int pgcolumnar_maintenance_hold_ms = 0; /* dev/test fault injection */ /* value set for columnar.compression (spec 5, 8.3) */ @@ -4201,7 +4202,7 @@ _PG_init(void) NULL, NULL, NULL); DefineCustomBoolVariable("pgcolumnar.autovacuum", - "Run the pgColumnar maintenance daemon (compact_rewrite / recluster).", + "Run the pgColumnar maintenance daemon (compact_rewrite / recluster, and expire when pgcolumnar.autovacuum_expire is on).", "Off by default. When on it runs ONLY the online ShareUpdateExclusiveLock " "verbs, and yields (like autovacuum) to any statement needing a stronger lock.", &pgcolumnar_autovacuum, @@ -4237,6 +4238,16 @@ _PG_init(void) 0, NULL, NULL, NULL); + DefineCustomBoolVariable("pgcolumnar.autovacuum_expire", + "Let the maintenance daemon run pgcolumnar.expire on tables that declare a retention.", + "Off by default, because expire deletes rows: turning the daemon on does not " + "by itself apply any retention. Has no effect while pgcolumnar.autovacuum is off.", + &pgcolumnar_autovacuum_expire, + false, + PGC_SIGHUP, + 0, + NULL, NULL, NULL); + DefineCustomIntVariable("pgcolumnar.maintenance_hold_ms", "Dev/test: hold ShareUpdateExclusiveLock this many ms inside a " "maintenance verb, interruptibly.", diff --git a/src/columnar_vacuum.c b/src/columnar_vacuum.c index 9984f2051..c0cac30f0 100644 --- a/src/columnar_vacuum.c +++ b/src/columnar_vacuum.c @@ -2376,6 +2376,9 @@ pgcolumnar_expire(PG_FUNCTION_ARGS) /* the lazy lock, as compact takes: readers and writers continue */ rel = table_open(relid, ShareUpdateExclusiveLock); + /* dev/test: hold SUEL so the daemon's yield is observable in expire too (#1290) */ + pgcolumnar_maintenance_hold(); + if (!PgColumnarIsColumnarRelation(relid)) { table_close(rel, ShareUpdateExclusiveLock); From 0b72518b6489b8c645c9c5938086e620bbb59510 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 22:48:29 +0000 Subject: [PATCH 3/7] test: the pytest twin of autovacuum_expire, and the suite registered (#1290) Independent of the shell suite: its own schema, connection, SQL and polling. compare_to_bash grades the pair 16 of 16. Red on main (PG16): the fixture raises 'unrecognized configuration parameter pgcolumnar.autovacuum_expire' and all five tests error on it. Green on this branch (PG17): 16 pass. Teardown resets every daemon setting the module changed, each on its own, because the cluster is shared and the daemon visits every database. Co-Authored-By: Claude Opus 5.5 --- test/pytest/TESTS.md | 23 +++ test/pytest/test_autovacuum_expire.py | 264 ++++++++++++++++++++++++++ test/pytest/test_compare_to_bash.py | 1 + test/run_all_versions.sh | 1 + 4 files changed, 289 insertions(+) create mode 100644 test/pytest/test_autovacuum_expire.py diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index ddf145768..c9166c104 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -141,6 +141,7 @@ behaviour, the source of that number is named. - [93. test_partition_lifecycle.py: partition_create_ahead keeps a time-partitioned parent ahead of now](#93-test_partition_lifecyclepy-partition_create_ahead-keeps-a-time-partitioned-parent-ahead-of-now) - [94. test_approx_aggregates.py: hll and tdigest over columnar tables, the compatibility statement](#94-test_approx_aggregatespy-hll-and-tdigest-over-columnar-tables-the-compatibility-statement) - [95. test_docs_how_to_sections.py: the pg_partman how-to keeps its own heading](#95-test_docs_how_to_sectionspy-the-pg_partman-how-to-keeps-its-own-heading) +- [96. test_autovacuum_expire.py: the daemon expires retention tables only when told to](#96-test_autovacuum_expirepy-the-daemon-expires-retention-tables-only-when-told-to) ## 1. How to read a test in here @@ -6570,3 +6571,25 @@ Red on main at d208bce in both twins (got 0, want 1); green with the heading res | test | what it holds | | --- | --- | | `test_the_how_to_keeps_its_pg_partman_section_under_its_own_heading` | the how-to has a "Manage time partitions with pg_partman" section, and it opens with the pg_partman paragraph its steps belong to | + +## 96. test_autovacuum_expire.py: the daemon expires retention tables only when told to + +Port of `autovacuum_expire.sh` (#1290). `pgcolumnar.autovacuum_expire` is off by default, the owner's decision on #1290: `expire` deletes rows, so turning the daemon on must not start deleting them by itself. With both switches on, each pass runs `expire` on every table that declares a retention, in its own transaction, before compaction. + +Independent of the shell suite: its own schema, connection, SQL and polling, and its fresh-row count differs (500 rows against the suite's 1000). The check names match, so `compare_to_bash` grades the pair one-for-one (16 of 16). + +WHERE THE PORT DIFFERS. The shell suite starts its server with the daemon on and a 2 s naptime in `postgresql.conf`. This corpus shares one cluster per worker, so the settings arrive by `ALTER SYSTEM` and a reload, and the switches are read back from a fresh session. Because the cluster is shared and the daemon visits every database, teardown resets every setting the module changed, each reset on its own, so a build without the switch cannot leave the others in force. + +THE YIELD ARM NEEDS A TABLE WHERE `expire` IS THE ONLY VERB DUE. `ex_hold` has a retention and no deletes or sort key, so a ShareUpdateExclusiveLock the daemon worker holds on it is `expire`'s. The other retention tables are dropped first: with the hold on, the worker spends 30 s inside `expire` on each of them, and the first version's poll gave up before the worker reached `ex_hold`. + +Written red first: on main the fixture raises `unrecognized configuration parameter "pgcolumnar.autovacuum_expire"`, and all five tests error on it. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_switch_exists_and_is_off_by_default` | the launcher runs; the switch reads `off` while the daemon is `on`; a manual `expire` on the same fixture retires its two aged groups | +| `test_with_the_switch_off_the_daemon_expires_nothing` | the daemon compacts a deleted-heavy table in the window, so it is running passes, and the aged table keeps all 2000 aged rows | +| `test_with_the_switch_on_the_daemon_expires_only_what_aged` | the aged rows go; the 500 fresh rows stay; a table with no retention keeps every row and is never named in a skipped line; the log names what was retired | +| `test_a_failing_expire_does_not_undo_the_compaction` | `expire` on a table whose retention column was dropped fails with `42703`, and the compaction due on that table still lands | +| `test_the_daemon_yields_the_lock_it_holds_inside_expire` | the worker holds ShareUpdateExclusiveLock inside `expire`; a session's ACCESS EXCLUSIVE is granted within 8 s of a 30 s hold, and the log shows the cancel | diff --git a/test/pytest/test_autovacuum_expire.py b/test/pytest/test_autovacuum_expire.py new file mode 100644 index 000000000..e7baf1508 --- /dev/null +++ b/test/pytest/test_autovacuum_expire.py @@ -0,0 +1,264 @@ +"""The maintenance daemon runs `pgcolumnar.expire` only when told to (#1290). + +`pgcolumnar.autovacuum_expire` is off by default -- the owner's decision on #1290 -- +because expire deletes rows, and turning the daemon on must not start deleting them +by itself. With both switches on, each pass expires every table that declares a +retention, in its own transaction, before compaction. + +Independent of `test/autovacuum_expire.sh`: its own schema, its own connection, its +own SQL and its own polling. Neither file reads the other. Check names match so the +pair can be graded by name. + +WHERE THIS PORT DIFFERS, AND WHY IT IS NOT COSMETIC. The shell suite starts its +server with the daemon already on and a 2s naptime in postgresql.conf. This corpus +shares one cluster per worker, so the settings arrive by ALTER SYSTEM and a reload, +and are read back from a fresh session before anything depends on them. And because +the cluster is shared, every setting this module changes is reset at teardown: +the daemon visits every database, so leaving `autovacuum_expire` on would expire +other modules' retention tables. +""" + +import re +import time + +import pytest + +SCHEMA = "pgc_avexp" +GROUP = 1000 +NAP = 2 +POLL_S = 40 +# Every daemon setting this module touches, so teardown can put each one back. +TOUCHED = ("pgcolumnar.autovacuum", "pgcolumnar.autovacuum_expire", + "pgcolumnar.autovacuum_naptime", "pgcolumnar.autovacuum_compact_threshold", + "pgcolumnar.maintenance_hold_ms", "deadlock_timeout") + + +def _one(conn, sql): + row = conn.execute(sql).fetchone() + return None if row is None else row[0] + + +def _fresh(cluster, sql): + """One value from a NEW session: a reload is absorbed at session start, so the + module's long-lived connection can report the value from before it.""" + import psycopg + + c = psycopg.connect(cluster.dsn(), autocommit=True) + try: + return _one(c, sql) + finally: + c.close() + + +def _sqlstate(conn, sql): + """-> the SQLSTATE `sql` raises, or 'noerror'.""" + import psycopg + + try: + conn.execute(sql) + except psycopg.Error as e: + return e.sqlstate or "unreadable" + return "noerror" + + +def _log_count(cluster, pattern): + path = cluster.datadir / "server.log" + if not path.exists(): + return -1 + return sum(1 for line in path.read_text(errors="replace").splitlines() + if re.search(pattern, line)) + + +def _until(fn, want, seconds=POLL_S): + """Re-read `fn()` once a second until it returns `want`; -> the last reading.""" + got = fn() + deadline = time.monotonic() + seconds + while got != want and time.monotonic() < deadline: + time.sleep(1) + got = fn() + return got + + +def _reload(conn, **gucs): + for k, v in gucs.items(): + conn.execute(f"ALTER SYSTEM SET {k.replace('__', '.')} = {v}") + conn.execute("SELECT pg_reload_conf()") + + +@pytest.fixture(scope="module") +def ax(pgc_cluster): + import psycopg + + conn = psycopg.connect(pgc_cluster.dsn(), autocommit=True) + conn.execute(f'DROP SCHEMA IF EXISTS "{SCHEMA}" CASCADE') + conn.execute(f'CREATE SCHEMA "{SCHEMA}"') + conn.execute(f'SET search_path TO "{SCHEMA}", public') + yield conn + # The hold first, so the daemon is not sitting in a 30s expire when the schema + # is dropped; then every other setting, so no later module meets this daemon. + # Each reset on its own: on a build without the switch, its RESET raises, and + # one raise must not leave the settings after it in force for later modules. + for g in TOUCHED: + try: + conn.execute(f"ALTER SYSTEM RESET {g}") + except psycopg.Error: + pass + conn.execute("SELECT pg_reload_conf()") + conn.execute(f'DROP SCHEMA IF EXISTS "{SCHEMA}" CASCADE') + conn.close() + + +def _aged(conn, t): + return _one(conn, f"SELECT count(*) FROM {t} WHERE ts < now() - interval '1 day'") + + +def _fresh_rows(conn, t): + return _one(conn, f"SELECT count(*) FROM {t} WHERE ts >= now() - interval '1 day'") + + +def _dead(conn, t): + return _one(conn, f"SELECT COALESCE(sum(deletedrows), 0) FROM pgcolumnar.stats('{t}')") + + +def _ttl_table(conn, t, fresh=GROUP // 2): + """Two whole row groups dated 40 days back, then `fresh` rows dated now.""" + conn.execute(f"CREATE TABLE {t} (ts timestamptz, v int) USING pgcolumnar") + conn.execute(f"SELECT pgcolumnar.set_options('{t}', chunk_group_row_limit => {GROUP}," + f" stripe_row_limit => {GROUP}, ttl_column => 'ts', ttl_interval => '1 day')") + conn.execute(f"INSERT INTO {t} SELECT now() - interval '40 days' + g * interval '1 minute', g" + f" FROM generate_series(1, {2 * GROUP}) g") + conn.execute(f"INSERT INTO {t} SELECT now() + g * interval '1 minute', g" + f" FROM generate_series(1, {fresh}) g") + + +def _deleted_heavy(conn, t, extra_cols=""): + conn.execute(f"CREATE TABLE {t} (id int{extra_cols}, v int) USING pgcolumnar") + conn.execute(f"SELECT pgcolumnar.set_options('{t}', chunk_group_row_limit => {GROUP})") + + +@pytest.fixture(scope="module") +def run(ax, pgc_cluster): + """The whole sequence, in the order the shell suite runs it, recorded once.""" + st = {} + _reload(ax, pgcolumnar__autovacuum="on", pgcolumnar__autovacuum_naptime=NAP, + pgcolumnar__autovacuum_compact_threshold=0.1, deadlock_timeout="'1s'") + st["launcher"] = _one(ax, "SELECT count(*) FROM pg_stat_activity" + " WHERE backend_type = 'pgcolumnar autovacuum launcher'") + st["switches"] = _fresh(pgc_cluster, "SELECT current_setting('pgcolumnar.autovacuum')" + " || '/' || current_setting('pgcolumnar.autovacuum_expire', true)") + + _ttl_table(ax, "ex_copy") + st["manual"] = _one(ax, "SELECT pgcolumnar.expire('ex_copy')") + + # OFF: the daemon is running passes -- it compacts this -- and expires nothing. + _ttl_table(ax, "ex_off") + _deleted_heavy(ax, "ex_busy") + ax.execute("INSERT INTO ex_busy SELECT g, g FROM generate_series(1, 20000) g") + ax.execute("DELETE FROM ex_busy WHERE (id * 2654435761)::bigint % 100 < 40") + st["busy_dead"] = _until(lambda: _dead(ax, "ex_busy"), 0) + time.sleep(2 * NAP) + st["off_aged"] = _aged(ax, "ex_off") + + # ON + ax.execute("CREATE TABLE ex_none (ts timestamptz, v int) USING pgcolumnar") + ax.execute(f"SELECT pgcolumnar.set_options('ex_none', chunk_group_row_limit => {GROUP}," + f" stripe_row_limit => {GROUP})") + ax.execute(f"INSERT INTO ex_none SELECT now() - interval '40 days' + g * interval '1 minute', g" + f" FROM generate_series(1, {2 * GROUP}) g") + _reload(ax, pgcolumnar__autovacuum_expire="on") + st["on_aged"] = _until(lambda: _aged(ax, "ex_off"), 0) + st["on_fresh"] = _fresh_rows(ax, "ex_off") + st["none_aged"] = _aged(ax, "ex_none") + st["logged_retired"] = _log_count(pgc_cluster, + rf"pgcolumnar autovacuum: expire .*\b{SCHEMA}\.ex_off retired 2 row groups") + st["none_skipped"] = _log_count(pgc_cluster, rf"pgcolumnar autovacuum: skipped .*\bex_none\b") + + # isolate: built with expire off, so neither verb runs until both are set. + _reload(ax, pgcolumnar__autovacuum_expire="off") + time.sleep(NAP + 1) + _deleted_heavy(ax, "ex_bad", extra_cols=", ts timestamptz") + ax.execute("SELECT pgcolumnar.set_options('ex_bad', ttl_column => 'ts', ttl_interval => '1 day')") + ax.execute("INSERT INTO ex_bad SELECT g, now(), g FROM generate_series(1, 20000) g") + ax.execute("DELETE FROM ex_bad WHERE (id * 2654435761)::bigint % 100 < 40") + ax.execute("ALTER TABLE ex_bad DROP COLUMN ts") + st["bad_state"] = _sqlstate(ax, "SELECT pgcolumnar.expire('ex_bad')") + _reload(ax, pgcolumnar__autovacuum_expire="on") + st["bad_dead"] = _until(lambda: _dead(ax, "ex_bad"), 0) + + # yield: ex_hold has a retention and nothing else due, so a lock the daemon + # worker holds on it is expire's. The other retention tables go first, because + # with the hold on the worker spends 30s inside expire on each of them. + ax.execute("DROP TABLE ex_copy, ex_off, ex_bad") + _reload(ax, pgcolumnar__maintenance_hold_ms=30000) + _ttl_table(ax, "ex_hold") + st["held"] = _until(lambda: _one( + ax, "SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid" + " WHERE l.relation = 'ex_hold'::regclass AND l.mode = 'ShareUpdateExclusiveLock'" + " AND l.granted AND a.backend_type = 'pgcolumnar autovacuum worker'"), 1) + + import psycopg + + t0 = time.monotonic() + lk = psycopg.connect(pgc_cluster.dsn(), autocommit=True) + try: + lk.execute(f'SET search_path TO "{SCHEMA}", public') + lk.execute("SET statement_timeout = '15s'") + try: + with lk.transaction(): + lk.execute("LOCK TABLE ex_hold IN ACCESS EXCLUSIVE MODE") + st["lock_state"] = "granted" + except psycopg.Error as e: + st["lock_state"] = e.sqlstate or "unreadable" + finally: + lk.close() + st["lock_s"] = time.monotonic() - t0 + time.sleep(1) + st["yield_logged"] = _log_count(pgc_cluster, + r"pgcolumnar autovacuum: skipped .*\bex_hold\b.*canceling statement") + _reload(ax, pgcolumnar__maintenance_hold_ms=0) + print(f"-- yield: lock {st['lock_state']} after {st['lock_s']:.1f}s;" + f" busy_dead={st['busy_dead']} on_aged={st['on_aged']} bad_dead={st['bad_dead']}") + return st + + +def test_the_switch_exists_and_is_off_by_default(run, expect): + expect.num(run["launcher"], 1, "premise: the maintenance launcher is running") + expect.text(run["switches"], "on/off", + "the daemon's expire switch exists and is OFF by default") + expect.num(run["manual"], 2, + "premise: a manual expire on the same fixture retires its two aged groups") + + +def test_with_the_switch_off_the_daemon_expires_nothing(run, expect): + expect.num(run["busy_dead"], 0, + "premise: the daemon compacted a deleted-heavy table, so it was running passes") + expect.num(run["off_aged"], 2 * GROUP, + "OFF: with autovacuum_expire off the daemon retired nothing") + + +def test_with_the_switch_on_the_daemon_expires_only_what_aged(run, expect): + expect.num(run["on_aged"], 0, "ON: the daemon expired the aged-out rows without being asked") + expect.num(run["on_fresh"], GROUP // 2, "ON: no row inside the retention was lost") + expect.num(run["none_aged"], 2 * GROUP, "ON: a table with no retention keeps every row") + expect.text("yes" if run["logged_retired"] >= 1 else "no", "yes", + "ON: the daemon logged what it retired") + expect.num(run["none_skipped"], 0, + "ON: a table with no retention is never named in a skipped line") + + +def test_a_failing_expire_does_not_undo_the_compaction(run, expect): + expect.text(run["bad_state"], "42703", + "premise: expire on a table whose TTL column is gone fails with undefined_column") + expect.num(run["bad_dead"], 0, + "isolate: a failing expire does not roll back the compaction due on the same table") + + +def test_the_daemon_yields_the_lock_it_holds_inside_expire(run, expect): + expect.num(run["held"], 1, + "premise: the daemon holds ShareUpdateExclusiveLock inside expire (the hold window is open)") + expect.text(run["lock_state"], "granted", + "yield: the AccessExclusive lock on a table in expire was granted") + expect.text("fast" if run["lock_s"] <= 8 else f"slow({run['lock_s']:.0f}s)", "fast", + "yield: and it was granted fast (<= 8s, far under the 30s hold)") + expect.text("yes" if run["yield_logged"] >= 1 else "no", "yes", + "yield: the daemon logged cancelling expire") diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index a401cc3b5..fe21b028b 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -90,6 +90,7 @@ "analyze_function", "analyze_reltuples", "approx_aggregates", + "autovacuum_expire", "base_scan_io", "catalog_delete_index", "catalog_plan_index", diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index 13c9014b2..b81030a1e 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -53,6 +53,7 @@ SUITES=( arrow_nested_import audit autovacuum + autovacuum_expire autovacuum_yield avro_manifest base_scan_io From ae5683839552f090956904e699fe33a02bdeafe2 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 22:49:35 +0000 Subject: [PATCH 4/7] test: cluster_tests 498 -> 503 for test_autovacuum_expire.py (#1290) Collected over the cluster files on the tree rebased onto dd08f4e. Co-Authored-By: Claude Opus 5.5 --- test/pytest/expected_tests.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 5e1be1046..0a925b1c2 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -565,4 +565,7 @@ guard_tests 415 # # cluster_tests 503 -> 504 for #1294's test_expire_records_what_it_retires_and_nothing_else, # collected over the cluster files on this branch rebased onto d208bce. -cluster_tests 504 +# +# cluster_tests 504 -> 509 for #1290's test_autovacuum_expire.py, by collection +# on this branch rebased onto c0ee868. +cluster_tests 509 From 628f0beb9d9ce10a92a33eff1cbd9b205d3ed60c Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 22:51:28 +0000 Subject: [PATCH 5/7] docs: expire runs by name or from the daemon with autovacuum_expire on (#1290) Every page that said expire never runs on its own, or that the daemon runs only compact_rewrite and recluster, now names the second switch and that it is off by default: features, how-to (with the two ALTER SYSTEM lines and the TimeZone caveat for date and timestamp columns), sql-reference, configuration (a row for the new setting), administration and ARCHITECTURE. Also expire's own doc comment and the options catalog comment, and the how-to prerequisite that still said timestamp or timestamptz only. docs_style green on PG17; ttl_expire 51/51 and autovacuum_expire 16/16 re-run after the comment change moved line numbers in columnar_vacuum.c. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 ++++++++++++++++ docs/ARCHITECTURE.md | 6 ++++-- docs/administration.md | 6 ++++++ docs/configuration.md | 5 +++-- docs/features.md | 12 +++++++----- docs/how-to.md | 27 ++++++++++++++++++++++----- docs/sql-reference.md | 16 ++++++++++------ pgcolumnar--1.0-alpha5.sql | 3 ++- src/columnar_vacuum.c | 7 ++++--- 9 files changed, 74 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index baeff910c..aa5d0accf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,22 @@ true until the next version shipped. and the alpha4 to alpha5 upgrade; `native_upgrade_converge` 21/21. ttl_expire and its pytest twin hold it: exactly the retired groups are recorded, a rolled-back expire and a `compact_rewrite` record nothing, and a drop clears the rows. +- The maintenance daemon can apply retention (#1290). With + `pgcolumnar.autovacuum_expire` on, as well as `pgcolumnar.autovacuum`, each pass + runs `pgcolumnar.expire` on every table whose options declare both + `ttl_column` and `ttl_interval`. The new setting is off by default, by the + owner's decision on #1290: `expire` deletes rows, so turning the daemon on does + not by itself apply a retention. `expire` runs first, in its own transaction, + so one that fails (its retention column was dropped) is logged and skipped + without undoing the compaction that follows. `expire` now calls the maintenance + hold after taking its lock, as `compact_rewrite` does, so the daemon's yield is + testable there too. A no-op `expire` costs one zone-map scan per row group: + 0.05 ms at 10 groups, 2.7 ms at 1000, measured on PG17. `test/autovacuum_expire.sh` + and an independent `test_autovacuum_expire.py` each hold 16 checks: the switch + is off by default; with it off, the daemon compacts but retires nothing; with it + on, the aged rows go, the fresh rows stay, and a table with no retention is never + touched or named; a failing `expire` does not undo a compaction; and the daemon + yields `expire`'s lock to an `ACCESS EXCLUSIVE` request within 8 s of a 30 s hold. - `test/format_downgrade.sh` and `test_format_downgrade.py` test #1137's promise with both real libraries. They build `v1.0-alpha4`, write with it, upgrade to the diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c6aab3d68..cb9bbf454 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -254,8 +254,10 @@ The maintenance daemon (`pgcolumnar.autovacuum`). A background-worker launcher starts one short-lived worker per database on a schedule. Each worker consults `pgcolumnar.maintenance_due` for the tables that have crossed the compact-rewrite or recluster thresholds, then runs `pgcolumnar.compact_rewrite` and -`pgcolumnar.recluster` on them through SPI, each in its own transaction. It calls -only those two `ShareUpdateExclusiveLock` verbs, and it registers as an +`pgcolumnar.recluster` on them through SPI, each in its own transaction. With +`pgcolumnar.autovacuum_expire` on, it first runs `pgcolumnar.expire` on each table +that declares a retention, in a transaction of its own. It calls only these +`ShareUpdateExclusiveLock` verbs, and it registers as an autovacuum-kind process so it yields to a stronger lock a session requests. ### columnar_unique.c diff --git a/docs/administration.md b/docs/administration.md index 138f20658..532ad09f1 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -214,6 +214,12 @@ It is off by default. When it is on, a launcher wakes every database. Each worker asks `pgcolumnar.maintenance_due()` which columnar tables have crossed a threshold, then runs the verb it recommends. +With `pgcolumnar.autovacuum_expire` also on (default off), each worker first +runs `pgcolumnar.expire` on every table that declares a retention. Each `expire` +runs in its own transaction. One that fails is logged and skipped, and the +compaction that follows still runs. The switch is separate because `expire` deletes rows: +turning the daemon on does not by itself apply a retention. + The daemon calls only the online `ShareUpdateExclusiveLock` verbs. It never calls `vacuum`, `vacuum_sorted`, or `cluster`. So it does not block readers or writers. It also yields the way autovacuum does: it cancels its own maintenance diff --git a/docs/configuration.md b/docs/configuration.md index 278542818..2b12a5f78 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -86,7 +86,8 @@ disk. It never changes the values that a table returns. | `pgcolumnar.index_min_blocks` | integer | `3` | Heap pages at or above which a keyed read of a `pgcolumnar` metadata catalog probes that catalog's index instead of reading it sequentially. These catalogs are shared by every columnar table in the database, so the same catalog is one page in a small database and hundreds in a large one, and below a few pages a sequential read is cheaper than a probe. The default is measured rather than chosen, and it is a default rather than a truth: the best value differs per catalog, because the hot catalogs are read more often per retired row group. Change it only to undo a regression you have measured. A very large value makes every converted lookup read its catalog sequentially, as 1.0-alpha4 and earlier did for most of them; `0` always probes. Range 0 to INT_MAX. | | `pgcolumnar.reclaim_coalesce` | boolean | `on` | During online compaction, split an oversized freed range on reuse and coalesce adjacent freed ranges, so space is reclaimed under fragmentation. Off reverts to whole-range reuse. | | `pgcolumnar.enable_end_truncation` | boolean | `off` | Allow `pgcolumnar.truncate()` to return trailing reclaimed blocks to the operating system. Off makes `pgcolumnar.truncate()` a no-op. Requires superuser to set. | -| `pgcolumnar.autovacuum` | boolean | `off` | Run the maintenance daemon. When on, it runs `compact_rewrite` and `recluster` on columnar tables that cross a threshold. It uses only `ShareUpdateExclusiveLock` and yields to any stronger lock. It never blocks a reader or a writer. See the [administration guide](administration.md#the-maintenance-daemon-pgcolumnarautovacuum). Reloadable, not a per-session setting. | +| `pgcolumnar.autovacuum` | boolean | `off` | Run the maintenance daemon. When on, it runs `compact_rewrite` and `recluster` on columnar tables that cross a threshold, and `expire` when `pgcolumnar.autovacuum_expire` is also on. It uses only `ShareUpdateExclusiveLock` and yields to any stronger lock. It never blocks a reader or a writer. See the [administration guide](administration.md#the-maintenance-daemon-pgcolumnarautovacuum). Reloadable, not a per-session setting. | +| `pgcolumnar.autovacuum_expire` | boolean | `off` | Let the daemon run `pgcolumnar.expire` on every table that declares a retention, once per sweep, before compaction. Off by default because `expire` deletes rows: turning the daemon on does not by itself apply a retention. No effect while `pgcolumnar.autovacuum` is off. Reloadable. | | `pgcolumnar.autovacuum_naptime` | integer | `60` | Seconds between daemon sweeps. Each sweep starts one worker per database. Range 1 to 86400. Reloadable. | | `pgcolumnar.autovacuum_compact_threshold` | float | `0.2` | Deleted fraction at which the daemon rewrites a table with `compact_rewrite`. Range 0.0 to 1.0. Reloadable. | | `pgcolumnar.autovacuum_recluster_threshold` | float | `0.05` | Appended fraction at which the daemon reclusters a table that has a recorded clustering key. Range 0.0 to 1.0. Reloadable. | @@ -164,7 +165,7 @@ SELECT pgcolumnar.set_options( | `compression_level` | integer | Level for the `zstd` codec, 1 to 22. | | `encode_effort` | name | `full` (default) or `fast`. How much work the writer spends choosing an encoding. See below. | | `sort_by` | name[] | Declared physical sort key (#288), applied by `pgcolumnar.vacuum_sorted(t)` with no columns. Column names, so it survives `pg_dump`/restore. Not auto-maintained; re-run after inserts. Cannot name a virtual generated column. Clear with `reset_options(t, sort_by => true)`. | -| `ttl_column` | name | The `date`, `timestamp` or `timestamptz` column a retention is measured on. On a `date` column the cutoff is truncated to the day, so a row dated on the cutoff is kept. Set it with `ttl_interval`; neither works alone. Nothing is deleted until you call `pgcolumnar.expire(t)` by name. | +| `ttl_column` | name | The `date`, `timestamp` or `timestamptz` column a retention is measured on. On a `date` column the cutoff is truncated to the day, so a row dated on the cutoff is kept. Set it with `ttl_interval`; neither works alone. Nothing is deleted until `pgcolumnar.expire(t)` runs: by name, or from the daemon with `pgcolumnar.autovacuum_expire` on. | | `ttl_interval` | interval | How long a row is kept, measured from `ttl_column`. `pgcolumnar.expire(t)` then drops row groups whose rows are all older than this. A group with one live row is kept whole. | **A `stripe_row_limit` below 1024 disables FSST on text columns.** A vector is a diff --git a/docs/features.md b/docs/features.md index 7b04ba842..e7f7c7b34 100644 --- a/docs/features.md +++ b/docs/features.md @@ -231,8 +231,9 @@ and writes continue. curve. - `pgcolumnar.expire(table)` drops row groups whose rows have all passed the retention declared with `set_options(ttl_column, ttl_interval)`. A group with - one live row is kept whole, so nothing inside the retention is dropped. It - never runs on its own; you call it by name. + one live row is kept whole, so nothing inside the retention is dropped. You + call it by name, or the maintenance daemon calls it when + `pgcolumnar.autovacuum_expire` is on. - `pgcolumnar.truncate(table)` returns the reclaimed end blocks to the operating system, taking a brief `AccessExclusiveLock` only when it is free. @@ -244,9 +245,10 @@ and writes continue. whether a table has crossed the compact and recluster thresholds. - `pgcolumnar.autovacuum` is a background maintenance daemon. When on, a per-database worker runs `compact_rewrite` and `recluster` on a schedule for tables that - `maintenance_due` reports as due. It runs only those two online operations, so it - never takes an exclusive lock, and it yields to a stronger lock a session - requests. + `maintenance_due` reports as due. With `pgcolumnar.autovacuum_expire` also on, + it runs `expire` on every table that declares a retention. It runs only these + online operations, so it never takes an exclusive lock, and it yields to a + stronger lock a session requests. ## Parallel bulk ingest diff --git a/docs/how-to.md b/docs/how-to.md index d8803ed65..3b6e2a3f1 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -292,8 +292,8 @@ to bound each call and keep it incremental. Declare how long rows are kept, then drop what has passed it. -Before you start: the column must be `timestamp` or `timestamptz`, and you must -own the table. +Before you start: the column must be `date`, `timestamp` or `timestamptz`, and +you must own the table. 1. Declare the retention. Both halves are required. @@ -316,9 +316,26 @@ has passed the retention, so a group holding one live row is kept whole. Some rows older than the interval therefore survive until the rest of their group ages out. Nothing inside the retention is ever dropped. -It takes `ShareUpdateExclusiveLock`, so reads and writes continue. It never runs -on its own: call it by name, or schedule it. Calling it on a table with no -declared retention raises an error rather than doing nothing. +It takes `ShareUpdateExclusiveLock`, so reads and writes continue. Calling it on +a table with no declared retention raises an error rather than doing nothing. + +3. To expire on a schedule instead, let the maintenance daemon run `expire`. + Both settings are needed; the second is off by default, so turning on the + daemon alone never deletes rows. + +```sql +ALTER SYSTEM SET pgcolumnar.autovacuum = on; +ALTER SYSTEM SET pgcolumnar.autovacuum_expire = on; +SELECT pg_reload_conf(); +``` + +The daemon then runs `expire` on every table that declares a retention, once +per `pgcolumnar.autovacuum_naptime` (default 60 seconds), and logs +`pgcolumnar autovacuum: expire
retired row groups`. + +For a `date` or `timestamp` retention column, the cutoff depends on the time +zone. The daemon uses the server's `TimeZone` setting, which may differ from +the time zone your sessions use. `pgcolumnar.reset_options` has no argument for the retention. To stop expiring, set `ttl_interval` to a span no row will reach. diff --git a/docs/sql-reference.md b/docs/sql-reference.md index 5d7aa05a2..76546e30c 100644 --- a/docs/sql-reference.md +++ b/docs/sql-reference.md @@ -34,9 +34,11 @@ PostgreSQL `CLUSTER`. Column names must exist and cannot be virtual generated columns. `ttl_column name` and `ttl_interval interval` declare a retention. They are read -only by [`pgcolumnar.expire`](#pgcolumnarexpiretablename-regclass-returns-bigint), -which you run yourself. Declaring a retention does not delete anything on its -own. Both are needed: either one alone means no retention. +only by [`pgcolumnar.expire`](#pgcolumnarexpiretablename-regclass-returns-bigint). +Declaring a retention does not delete anything on its own. `expire` runs when +you call it, or when the maintenance daemon calls it with +`pgcolumnar.autovacuum_expire` on. Both are needed: either one alone means no +retention. `ttl_interval` must be a positive interval. Zero and negative intervals raise `22023`. A negative interval would put the cutoff in the future, so `expire` @@ -46,7 +48,7 @@ would drop rows that are still inside their retention. SELECT pgcolumnar.set_options('events', sort_by => ARRAY['customer_id','ts']); SELECT pgcolumnar.reset_options('events', sort_by => true); -- clear it --- declare a retention; nothing is deleted until you call expire +-- declare a retention; nothing is deleted until expire runs SELECT pgcolumnar.set_options('events', ttl_column => 'ts', ttl_interval => '90 days'); ``` @@ -208,8 +210,10 @@ Drops row groups whose rows are all older than the retention declared by `set_options`. Returns the number of groups dropped. Holds only `ShareUpdateExclusiveLock`, so it runs against a live table. -**This deletes rows.** It runs only when you call it. No other operation applies -a retention, and `VACUUM` never does. +**This deletes rows.** It runs when you call it, or when the maintenance daemon +calls it with both `pgcolumnar.autovacuum` and `pgcolumnar.autovacuum_expire` +on. The second is off by default. No other operation applies a retention, and +`VACUUM` never does. Each group it drops is recorded in `pgcolumnar.retired_group`, in the same transaction. An incremental rollup reads that record to keep the summary of rows diff --git a/pgcolumnar--1.0-alpha5.sql b/pgcolumnar--1.0-alpha5.sql index b73059ab5..893cd1ae8 100644 --- a/pgcolumnar--1.0-alpha5.sql +++ b/pgcolumnar--1.0-alpha5.sql @@ -57,7 +57,8 @@ CREATE TABLE pgcolumnar.options ( encode_effort name, sort_by name[], -- declared physical sort key (#288) -- Declared retention (#403 item 5a), read only by pgcolumnar.expire. - -- Nothing drops rows on its own: expire is called by name. + -- Nothing drops rows on its own: expire is called by name, or by the + -- daemon when pgcolumnar.autovacuum_expire is on (off by default, #1290). ttl_column name, ttl_interval interval ); diff --git a/src/columnar_vacuum.c b/src/columnar_vacuum.c index c0cac30f0..5daeb0094 100644 --- a/src/columnar_vacuum.c +++ b/src/columnar_vacuum.c @@ -2341,9 +2341,10 @@ pgcolumnar_cluster_hilbert(PG_FUNCTION_ARGS) * without fetching. A fetch would correctly fail once the catalog rows * are gone; skipping the fetch is what made the ghosts visible. * - * This is called by name and never runs on its own. It deletes rows, and an - * operation a user runs for maintenance must not do that silently, so it is - * not wired into vacuum, compact or autovacuum. + * It deletes rows, so nothing calls it implicitly: not vacuum, not compact. + * The maintenance daemon calls it only when pgcolumnar.autovacuum_expire is + * on, a switch that is off by default (#1290), so turning the daemon on + * does not by itself apply a retention. */ Datum pgcolumnar_expire(PG_FUNCTION_ARGS) From a399511b07470f775f42156520c2e8af5c5b6ad2 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 23:11:03 +0000 Subject: [PATCH 6/7] fix: the daemon logs expire at LOG only when it retired something (#1290) From OffgridwithJD's early notes on #1307. A line per retention table per pass, forever, is noise; a pass that retires nothing now logs at DEBUG1, as av_maintain_one logs only a verb that ran. Red first in both twins: against 8d68155 the new arm read 2 'retired 0' lines at LOG (shell PG17, pytest PG16). Two arms answer the question asked there, whether a table dropped mid-pass is skipped: while the worker holds ex_hold, ex_gone is dropped, and after the cancel the same pass logs ex_gone as skipped and still expires ex_after. Both were green on 8d68155, so they hold existing behaviour; the mutation runs date them. The worker's table list is now ORDER BY c.oid, so 'the next table' is defined. how-to: a pass that retires nothing logs at DEBUG1; for a session-zone cutoff, run expire by hand from that session (also from those notes). autovacuum_expire 19/19 on PG17; the pytest twin 19 pass, 6 tests; compare_to_bash 19 of 19; cluster_tests 503 -> 504 by collection. Co-Authored-By: Claude Opus 5.5 --- docs/how-to.md | 6 ++++-- src/columnar_autovacuum.c | 11 +++++++++-- test/autovacuum_expire.sh | 26 +++++++++++++++++++++++++- test/pytest/TESTS.md | 5 +++-- test/pytest/expected_tests.txt | 5 ++++- test/pytest/test_autovacuum_expire.py | 20 +++++++++++++++++++- 6 files changed, 64 insertions(+), 9 deletions(-) diff --git a/docs/how-to.md b/docs/how-to.md index 3b6e2a3f1..e7d32c0ce 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -331,11 +331,13 @@ SELECT pg_reload_conf(); The daemon then runs `expire` on every table that declares a retention, once per `pgcolumnar.autovacuum_naptime` (default 60 seconds), and logs -`pgcolumnar autovacuum: expire
retired row groups`. +`pgcolumnar autovacuum: expire
retired row groups` when it retires +any. A pass that retires nothing logs at `DEBUG1` only. For a `date` or `timestamp` retention column, the cutoff depends on the time zone. The daemon uses the server's `TimeZone` setting, which may differ from -the time zone your sessions use. +the time zone your sessions use. For a cutoff in your session's time zone, run +`pgcolumnar.expire` by hand from that session. `pgcolumnar.reset_options` has no argument for the retention. To stop expiring, set `ttl_interval` to a span no row will reach. diff --git a/src/columnar_autovacuum.c b/src/columnar_autovacuum.c index 335f3fac6..f62845e27 100644 --- a/src/columnar_autovacuum.c +++ b/src/columnar_autovacuum.c @@ -260,7 +260,13 @@ av_expire_one(const char *qualname) if (!isnull) retired = DatumGetInt64(d); } - elog(LOG, "pgcolumnar autovacuum: expire %s retired " INT64_FORMAT " row groups", + /* + * LOG only when something was retired: a line per retention table per + * pass, forever, is noise an administrator learns to ignore + * (OffgridwithJD on #1307). av_maintain_one logs only a verb that ran. + */ + elog(retired > 0 ? LOG : DEBUG1, + "pgcolumnar autovacuum: expire %s retired " INT64_FORMAT " row groups", qualname, retired); SPI_finish(); @@ -456,7 +462,8 @@ pgcolumnar_av_worker_main(Datum arg) "JOIN pg_am a ON a.oid = c.relam " "JOIN pg_namespace n ON n.oid = c.relnamespace " "LEFT JOIN pgcolumnar.options o ON o.regclass = c.oid " - "WHERE a.amname = 'pgcolumnar' AND c.relkind = 'r'", + "WHERE a.amname = 'pgcolumnar' AND c.relkind = 'r' " + "ORDER BY c.oid", true, 0); if (ret == SPI_OK_SELECT) { diff --git a/test/autovacuum_expire.sh b/test/autovacuum_expire.sh index 1877a611d..7dfeb7aac 100755 --- a/test/autovacuum_expire.sh +++ b/test/autovacuum_expire.sh @@ -94,6 +94,12 @@ check "ON: the daemon logged what it retired" \ "$(grep -c 'pgcolumnar autovacuum: expire .*ex_off retired 2 row groups' "$PGC_LOGFILE" 2>/dev/null | awk '{print ($1 >= 1) ? "yes" : "no"}')" "yes" check_num "ON: a table with no retention is never named in a skipped line" \ "$(grep -c 'pgcolumnar autovacuum: skipped .*ex_none' "$PGC_LOGFILE" 2>/dev/null)" "0" +# Two more passes find nothing left to retire in ex_off. A line per table per +# pass, forever, is noise an administrator learns to ignore (OffgridwithJD on +# #1307), so a pass that retires nothing logs below LOG. +sleep 5 +check_num "a pass that retires nothing logs nothing at LOG for that table" \ + "$(grep -c 'pgcolumnar autovacuum: expire .*ex_off retired 0 row groups' "$PGC_LOGFILE" 2>/dev/null)" "0" # ---- isolate: a failing expire does not undo the compaction ----------------- # Built with expire off, so neither verb runs until both conditions are set. @@ -125,7 +131,19 @@ check "isolate: a failing expire does not roll back the compaction due on the sa psql_run "DROP TABLE ex_copy, ex_off, ex_bad;" psql_run "ALTER SYSTEM SET pgcolumnar.maintenance_hold_ms = 30000;" psql_run "SELECT pg_reload_conf();" -mk_ttl ex_hold +# One transaction, so the worker that lists them sees all three, in creation +# order: ex_hold, then ex_gone, which is dropped while the worker holds +# ex_hold, then ex_after, which the same pass must still reach. +psql_run "BEGIN; + CREATE TABLE ex_hold (ts timestamptz, v int) USING pgcolumnar; + CREATE TABLE ex_gone (ts timestamptz, v int) USING pgcolumnar; + CREATE TABLE ex_after (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options(t, chunk_group_row_limit => $CG, stripe_row_limit => $CG, + ttl_column => 'ts', ttl_interval => '1 day') FROM unnest(ARRAY['ex_hold','ex_gone','ex_after']::regclass[]) t; + INSERT INTO ex_hold SELECT $OLD + g * interval '1 second', g FROM generate_series(1, 2*$CG) g; + INSERT INTO ex_gone SELECT $OLD + g * interval '1 second', g FROM generate_series(1, 2*$CG) g; + INSERT INTO ex_after SELECT $OLD + g * interval '1 second', g FROM generate_series(1, 2*$CG) g; + COMMIT;" holds() { q "SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid WHERE l.relation = 'ex_hold'::regclass AND l.mode = 'ShareUpdateExclusiveLock' @@ -133,6 +151,7 @@ holds() { } check "premise: the daemon holds ShareUpdateExclusiveLock inside expire (the hold window is open)" \ "$(poll 40 holds 1)" "1" +psql_run "DROP TABLE ex_gone;" t0=$(date +%s) STATE="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ -d "$PGC_DB" -qtA 2>&1 </dev/null)" -ge 1 ] && echo yes || echo no)" "yes" +# After the cancel the same worker goes on down its list: ex_gone no longer +# exists, and ex_after is expired (after its own 30s hold). +check_num "dropped: and the pass goes on to expire the next table" "$(poll 45 'aged ex_after' 0)" "0" +check "dropped: a table dropped mid-pass is logged as skipped" \ + "$([ "$(grep -c 'pgcolumnar autovacuum: skipped expire on .*ex_gone' "$PGC_LOGFILE" 2>/dev/null)" -ge 1 ] && echo yes || echo no)" "yes" psql_run "ALTER SYSTEM RESET pgcolumnar.maintenance_hold_ms;" psql_run "SELECT pg_reload_conf();" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index c9166c104..1149c5377 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -6576,7 +6576,7 @@ Red on main at d208bce in both twins (got 0, want 1); green with the heading res Port of `autovacuum_expire.sh` (#1290). `pgcolumnar.autovacuum_expire` is off by default, the owner's decision on #1290: `expire` deletes rows, so turning the daemon on must not start deleting them by itself. With both switches on, each pass runs `expire` on every table that declares a retention, in its own transaction, before compaction. -Independent of the shell suite: its own schema, connection, SQL and polling, and its fresh-row count differs (500 rows against the suite's 1000). The check names match, so `compare_to_bash` grades the pair one-for-one (16 of 16). +Independent of the shell suite: its own schema, connection, SQL and polling, and its fresh-row count differs (500 rows against the suite's 1000). The check names match, so `compare_to_bash` grades the pair one-for-one (19 of 19). WHERE THE PORT DIFFERS. The shell suite starts its server with the daemon on and a 2 s naptime in `postgresql.conf`. This corpus shares one cluster per worker, so the settings arrive by `ALTER SYSTEM` and a reload, and the switches are read back from a fresh session. Because the cluster is shared and the daemon visits every database, teardown resets every setting the module changed, each reset on its own, so a build without the switch cannot leave the others in force. @@ -6590,6 +6590,7 @@ Written red first: on main the fixture raises `unrecognized configuration parame | --- | --- | | `test_the_switch_exists_and_is_off_by_default` | the launcher runs; the switch reads `off` while the daemon is `on`; a manual `expire` on the same fixture retires its two aged groups | | `test_with_the_switch_off_the_daemon_expires_nothing` | the daemon compacts a deleted-heavy table in the window, so it is running passes, and the aged table keeps all 2000 aged rows | -| `test_with_the_switch_on_the_daemon_expires_only_what_aged` | the aged rows go; the 500 fresh rows stay; a table with no retention keeps every row and is never named in a skipped line; the log names what was retired | +| `test_with_the_switch_on_the_daemon_expires_only_what_aged` | the aged rows go; the 500 fresh rows stay; a table with no retention keeps every row and is never named in a skipped line; the log names what was retired; later passes that retire nothing write no LOG line for the table | | `test_a_failing_expire_does_not_undo_the_compaction` | `expire` on a table whose retention column was dropped fails with `42703`, and the compaction due on that table still lands | | `test_the_daemon_yields_the_lock_it_holds_inside_expire` | the worker holds ShareUpdateExclusiveLock inside `expire`; a session's ACCESS EXCLUSIVE is granted within 8 s of a 30 s hold, and the log shows the cancel | +| `test_a_table_dropped_mid_pass_is_skipped_and_the_pass_goes_on` | while the worker holds `ex_hold`, `ex_gone` is dropped; after the cancel the same pass logs `ex_gone` as skipped and still expires `ex_after` | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 0a925b1c2..ff723a7aa 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -568,4 +568,7 @@ guard_tests 415 # # cluster_tests 504 -> 509 for #1290's test_autovacuum_expire.py, by collection # on this branch rebased onto c0ee868. -cluster_tests 509 +# +# cluster_tests 509 -> 510 for #1290's test_autovacuum_expire.py, by collection +# on this branch rebased onto c0ee868. +cluster_tests 510 diff --git a/test/pytest/test_autovacuum_expire.py b/test/pytest/test_autovacuum_expire.py index e7baf1508..3b10d2b41 100644 --- a/test/pytest/test_autovacuum_expire.py +++ b/test/pytest/test_autovacuum_expire.py @@ -172,6 +172,10 @@ def run(ax, pgc_cluster): st["logged_retired"] = _log_count(pgc_cluster, rf"pgcolumnar autovacuum: expire .*\b{SCHEMA}\.ex_off retired 2 row groups") st["none_skipped"] = _log_count(pgc_cluster, rf"pgcolumnar autovacuum: skipped .*\bex_none\b") + # More passes find nothing left in ex_off; those must not write at LOG. + time.sleep(2 * NAP + 1) + st["zero_logged"] = _log_count(pgc_cluster, + rf"pgcolumnar autovacuum: expire .*\b{SCHEMA}\.ex_off retired 0 row groups") # isolate: built with expire off, so neither verb runs until both are set. _reload(ax, pgcolumnar__autovacuum_expire="off") @@ -190,7 +194,11 @@ def run(ax, pgc_cluster): # with the hold on the worker spends 30s inside expire on each of them. ax.execute("DROP TABLE ex_copy, ex_off, ex_bad") _reload(ax, pgcolumnar__maintenance_hold_ms=30000) - _ttl_table(ax, "ex_hold") + # One transaction, so the listing worker sees all three in creation order: + # ex_hold (held), ex_gone (dropped during the hold), ex_after (still reached). + with ax.transaction(): + for t in ("ex_hold", "ex_gone", "ex_after"): + _ttl_table(ax, t) st["held"] = _until(lambda: _one( ax, "SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a ON a.pid = l.pid" " WHERE l.relation = 'ex_hold'::regclass AND l.mode = 'ShareUpdateExclusiveLock'" @@ -198,6 +206,7 @@ def run(ax, pgc_cluster): import psycopg + ax.execute("DROP TABLE ex_gone") t0 = time.monotonic() lk = psycopg.connect(pgc_cluster.dsn(), autocommit=True) try: @@ -215,6 +224,8 @@ def run(ax, pgc_cluster): time.sleep(1) st["yield_logged"] = _log_count(pgc_cluster, r"pgcolumnar autovacuum: skipped .*\bex_hold\b.*canceling statement") + st["after_aged"] = _until(lambda: _aged(ax, "ex_after"), 0, seconds=45) + st["gone_skipped"] = _log_count(pgc_cluster, r"pgcolumnar autovacuum: skipped expire on .*\bex_gone\b") _reload(ax, pgcolumnar__maintenance_hold_ms=0) print(f"-- yield: lock {st['lock_state']} after {st['lock_s']:.1f}s;" f" busy_dead={st['busy_dead']} on_aged={st['on_aged']} bad_dead={st['bad_dead']}") @@ -244,6 +255,7 @@ def test_with_the_switch_on_the_daemon_expires_only_what_aged(run, expect): "ON: the daemon logged what it retired") expect.num(run["none_skipped"], 0, "ON: a table with no retention is never named in a skipped line") + expect.num(run["zero_logged"], 0, "a pass that retires nothing logs nothing at LOG for that table") def test_a_failing_expire_does_not_undo_the_compaction(run, expect): @@ -262,3 +274,9 @@ def test_the_daemon_yields_the_lock_it_holds_inside_expire(run, expect): "yield: and it was granted fast (<= 8s, far under the 30s hold)") expect.text("yes" if run["yield_logged"] >= 1 else "no", "yes", "yield: the daemon logged cancelling expire") + + +def test_a_table_dropped_mid_pass_is_skipped_and_the_pass_goes_on(run, expect): + expect.num(run["after_aged"], 0, "dropped: and the pass goes on to expire the next table") + expect.text("yes" if run["gone_skipped"] >= 1 else "no", "yes", + "dropped: a table dropped mid-pass is logged as skipped") From f76849d1782edbf42b15cc89bff0c2b858fe8456 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Mon, 12 Oct 2026 23:22:28 +0000 Subject: [PATCH 7/7] test: ledger rows for autovacuum_expire, from five majors and thirteen mutations (#1290) 19 rows from clean runs on 15-19 at this tree's code (19/19 on each), then thirteen mutations, each run on all five majors and merged one log at a time with --mutation and --target. Sixteen rows are dated; the three still never are premises. The suite is now covered, so suites_not_covered stays 247, which is what failed CI on 8d68155. Census 1693 -> 1696, rows 1943 -> 1962. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 19 +++++++++++++++++++ test/check_ledger_budget.txt | 22 +++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index aada27184..7e8ab9ba3 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -51,6 +51,25 @@ approx_aggregates approx_aggregates tdigest over a columnar table gives exactly approx_aggregates approx_aggregates the columnar table returns the 200,000 rows written, over 50,000 devices 15;16;17;18;19 2026-10-13 the custom scan off by default, and the access-method scan skips every row whose row number ends in 999 approx_aggregates approx_aggregates the hll aggregate reads through the columnar scan 15;16;17;18;19 2026-10-13 pgcolumnar.enable_custom_scan defaults to off, in both its C initializer and its GUC boot value;the custom scan off by default, and the access-method scan skips every row whose row number ends in 999 approx_aggregates approx_aggregates the tdigest aggregate reads through the columnar scan 15;16;17;18;19 2026-10-13 pgcolumnar.enable_custom_scan defaults to off, in both its C initializer and its GUC boot value;the custom scan off by default, and the access-method scan skips every row whose row number ends in 999 +autovacuum_expire autovacuum_expire OFF: with autovacuum_expire off the daemon retired nothing 15;16;17;18;19 2026-10-12 pgcolumnar.autovacuum_expire defaults to on, in both its C initializer and its GUC boot value;the daemon's loop runs expire for a retention table whether or not pgcolumnar.autovacuum_expire is on +autovacuum_expire autovacuum_expire ON: a table with no retention is never named in a skipped line 15;16;17;18;19 2026-10-12 the daemon's loop runs expire on every columnar table when autovacuum_expire is on, the has_ttl filter removed +autovacuum_expire autovacuum_expire ON: a table with no retention keeps every row 15;16;17;18;19 2026-10-12 the daemon runs expire on every table and PgColumnarReadTtl invents a 1 day retention on ts where none is declared +autovacuum_expire autovacuum_expire ON: no row inside the retention was lost 15;16;17;18;19 2026-10-12 expire retires every row group, the cutoff comparison made always true +autovacuum_expire autovacuum_expire ON: the daemon expired the aged-out rows without being asked 15;16;17;18;19 2026-10-12 the daemon never runs expire, its call guarded by false +autovacuum_expire autovacuum_expire ON: the daemon logged what it retired 15;16;17;18;19 2026-10-12 expire retires every row group, the cutoff comparison made always true;the daemon never runs expire, its call guarded by false;the daemon's 'expire retired row groups' LOG line removed +autovacuum_expire autovacuum_expire a pass that retires nothing logs nothing at LOG for that table 15;16;17;18;19 2026-10-12 the daemon logs 'retired 0 row groups' at LOG on every pass +autovacuum_expire autovacuum_expire dropped: a table dropped mid-pass is logged as skipped 15;16;17;18;19 2026-10-12 the daemon worker exits after the first expire that fails, abandoning the rest of its pass;the daemon's 'skipped expire on ' LOG line removed +autovacuum_expire autovacuum_expire dropped: and the pass goes on to expire the next table 15;16;17;18;19 2026-10-12 the daemon worker exits after the first expire that fails, abandoning the rest of its pass +autovacuum_expire autovacuum_expire isolate: a failing expire does not roll back the compaction due on the same table 15;16;17;18;19 2026-10-12 an expire that fails makes the daemon skip that table's compaction, as one shared transaction would;the daemon worker exits after the first expire that fails, abandoning the rest of its pass +autovacuum_expire autovacuum_expire premise: a manual expire on the same fixture retires its two aged groups 15;16;17;18;19 2026-10-12 expire retires every row group, the cutoff comparison made always true +autovacuum_expire autovacuum_expire premise: expire on a table whose TTL column is gone fails with undefined_column 15;16;17;18;19 never - +autovacuum_expire autovacuum_expire premise: the daemon compacted a deleted-heavy table, so it was running passes 15;16;17;18;19 never - +autovacuum_expire autovacuum_expire premise: the daemon holds ShareUpdateExclusiveLock inside expire (the hold window is open) 15;16;17;18;19 2026-10-12 pgcolumnar_maintenance_hold removed from expire, so the daemon holds no lock inside expire long enough to yield;the daemon never runs expire, its call guarded by false;the daemon runs expire on every table and PgColumnarReadTtl invents a 1 day retention on ts where none is declared +autovacuum_expire autovacuum_expire premise: the maintenance launcher is running 15;16;17;18;19 never - +autovacuum_expire autovacuum_expire the daemon's expire switch exists and is OFF by default 15;16;17;18;19 2026-10-12 pgcolumnar.autovacuum_expire defaults to on, in both its C initializer and its GUC boot value +autovacuum_expire autovacuum_expire yield: and it was granted fast (<= 8s, far under the 30s hold) 15;16;17;18;19 2026-10-12 the daemon worker no longer sets PROC_IS_AUTOVACUUM, so core never cancels it for a stronger lock +autovacuum_expire autovacuum_expire yield: the AccessExclusive lock on a table in expire was granted 15;16;17;18;19 2026-10-12 the daemon worker no longer sets PROC_IS_AUTOVACUUM, so core never cancels it for a stronger lock +autovacuum_expire autovacuum_expire yield: the daemon logged cancelling expire 15;16;17;18;19 2026-10-12 pgcolumnar_maintenance_hold removed from expire, so the daemon holds no lock inside expire long enough to yield;the daemon never runs expire, its call guarded by false;the daemon runs expire on every table and PgColumnarReadTtl invents a 1 day retention on ts where none is declared;the daemon worker no longer sets PROC_IS_AUTOVACUUM, so core never cancels it for a stronger lock;the daemon's 'skipped expire on ' LOG line removed base_scan_io base_scan_io a base scan is not priced from sibling projection pages 15;16;17;18;19 2026-09-21 projPages = 0 (keep whole-file pages) base_scan_io base_scan_io premise: a covering projection exists 15;16;17;18;19 never - base_scan_io base_scan_io premise: adding the projection enlarged the relation file 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 280ab7deb..2c77ef3e7 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1390,4 +1390,24 @@ suites_not_covered 247 # never 1684 - 1 = 1683 # dated 300 + 1 = 301 # 1683 + 301 == 1984 -checks_never_observed_red 1683 +# +# #1290 (PR #1307): 1683 -> 1686. autovacuum_expire enters the ledger with 19 +# checks. The mutation runs were of this PR's code on its first base +# (503f080); the rebases onto #1304, #1306, #1312, #1315, #1316, #1318, #1313, #1314 and #1320 since then touched +# columnar_tableam.c, columnar_metadata.c, columnar_vector.c and columnar.h, +# none in the daemon's or expire's code, and the clean suite re-run on the +# rebased tree is 19/19 on each of 15-19. The suite is now covered, so +# suites_not_covered stays 247. Sixteen are dated by thirteen +# mutations, each merged one log at a time with --mutation and --target: +# the expire switch bypassed, forced off, defaulted on; the has_ttl filter +# removed, and with it a retention invented where none is declared; a failed +# expire skipping the compaction; the hold removed from expire; the worker no +# longer PROC_IS_AUTOVACUUM; the retired LOG line removed; 'retired 0' logged +# at LOG; the skipped LOG line removed; the worker exiting on the first failed +# expire; and expire retiring every group. The three still never are premises. +# +# ledger rows 1984 + 19 = 2003 +# never 1683 + 3 = 1686 +# dated 301 + 16 = 317 +# 1686 + 317 == 2003 +checks_never_observed_red 1686