From 539e6d62946c1a3c36d03f7b4b8aa8900df8943e Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 16:27:31 +0000 Subject: [PATCH 1/4] test: the daemon cuts a date retention at its database's TimeZone, in both twins, and how-to.md says so (#1341) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VjTxfQJj5pnii9wzmPP9Ho --- docs/how-to.md | 9 ++-- test/autovacuum_expire.sh | 67 +++++++++++++++++++++++ test/pytest/TESTS.md | 1 + test/pytest/expected_tests.txt | 4 +- test/pytest/test_autovacuum_expire.py | 76 +++++++++++++++++++++++++++ 5 files changed, 153 insertions(+), 4 deletions(-) diff --git a/docs/how-to.md b/docs/how-to.md index dc8104e4e..23d223a97 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -335,9 +335,12 @@ per `pgcolumnar.autovacuum_naptime` (default 60 seconds), and logs any. A pass that retires nothing logs at `DEBUG1` only. For a `date` or `timestamp` retention column, the cutoff depends on the time -zone. The daemon uses the server's `TimeZone` setting, which may differ from -the time zone your sessions use. For a cutoff in your session's time zone, run -`pgcolumnar.expire` by hand from that session. +zone. The daemon uses the `TimeZone` in effect for the database it connects to: +a database-level setting (`ALTER DATABASE ... SET TimeZone`) when there is one, +otherwise the server's `TimeZone`, including a change made by reloading the +configuration. Either may differ from the time zone your sessions use. For a +cutoff in your session's time zone, run `pgcolumnar.expire` by hand from that +session. `pgcolumnar.reset_options` has no argument for the retention. To stop expiring, set `ttl_interval` to a span no row will reach. diff --git a/test/autovacuum_expire.sh b/test/autovacuum_expire.sh index 01766135e..9a3fcf2fb 100755 --- a/test/autovacuum_expire.sh +++ b/test/autovacuum_expire.sh @@ -175,4 +175,71 @@ check "dropped: a table dropped mid-pass is logged as skipped" \ psql_run "ALTER SYSTEM RESET pgcolumnar.maintenance_hold_ms;" psql_run "SELECT pg_reload_conf();" +# ---- which TimeZone the daemon's expire cuts a date retention at (#1341) ------ +# +# For a date or timestamp retention column the cutoff is a calendar date, so it +# depends on the zone. how-to.md says the daemon uses the server's TimeZone. Measured +# (PG17), it does, and it follows a change once reloaded, but a database-level +# ALTER DATABASE ... SET TimeZone overrides it, as it does for any session connecting +# to that database. Two zones 25 hours apart, so they are never on the same date. +# +# One row group per day, six days either side of today (UTC); the expire retires the +# groups wholly before the cutoff, so the first surviving day IS the cutoff the +# daemon computed: today in its zone, less the one-day retention. +tz_mk() { # tz_mk TABLE: built with the daemon's expire off, so no pass sees half of it + psql_run "CREATE TABLE $1 (d date, v int) USING pgcolumnar;" + psql_run "SELECT pgcolumnar.set_options('$1', chunk_group_row_limit => 1000, stripe_row_limit => 1000, + ttl_column => 'd', ttl_interval => '1 day');" + for k in $(seq -6 6); do + psql_run "INSERT INTO $1 SELECT (now() AT TIME ZONE 'UTC')::date + $k, g FROM generate_series(1, 100) g;" + done +} +tz_cut() { q "SELECT ((now() AT TIME ZONE '$1')::date - 1)::text;"; } +tz_expire() { # tz_expire on|off + psql_run "ALTER SYSTEM SET pgcolumnar.autovacuum_expire = $1;" + psql_run "SELECT pg_reload_conf();" + [ "$1" = off ] && sleep 5 +} +# tz_first TABLE ZONE -> "match" if the first surviving day is the zone's cutoff, read +# just before and just after the daemon's passes (a zone's midnight may fall +# between); otherwise the day it holds and the two cutoffs it was compared with. +tz_first() { + local before after got last="" same=0 i + before="$(tz_cut "$2")" + tz_expire on + for i in $(seq 1 60); do + got="$(q "SELECT min(d)::text FROM $1;")" + if [ "$got" = "$last" ] && [ "$got" != "$tz_start" ]; then + same=$((same + 1)); [ "$same" -ge 8 ] && break + else + same=0 + fi + last="$got"; sleep 1 + done + after="$(tz_cut "$2")" + tz_expire off + { [ "$got" = "$before" ] || [ "$got" = "$after" ]; } && echo match || echo "$got (want $before or $after)" +} +psql_run "ALTER SYSTEM RESET pgcolumnar.maintenance_hold_ms;" +tz_expire off +psql_run "ALTER SYSTEM SET timezone = 'Pacific/Kiritimati';" +psql_run "SELECT pg_reload_conf();" +tz_mk tz_k +tz_start="$(q "SELECT min(d)::text FROM tz_k;")" +check "premise: the two zones are on different dates, so every arm below can tell them apart" \ + "$([ "$(tz_cut Pacific/Kiritimati)" != "$(tz_cut Pacific/Pago_Pago)" ] && echo apart || echo same)|$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('tz_k');")" \ + "apart|13" +check "the daemon cuts a date retention at the server's TimeZone" "$(tz_first tz_k Pacific/Kiritimati)" "match" +psql_run "ALTER SYSTEM SET timezone = 'Pacific/Pago_Pago';" +psql_run "SELECT pg_reload_conf();" +sleep 3 +tz_mk tz_p +check "and follows a change to it once the configuration is reloaded" "$(tz_first tz_p Pacific/Pago_Pago)" "match" +psql_run "ALTER DATABASE \"$PGC_DB\" SET timezone = 'Pacific/Kiritimati';" +tz_mk tz_d +check "a database-level TimeZone overrides the server's for the daemon too" "$(tz_first tz_d Pacific/Kiritimati)" "match" +psql_run "ALTER DATABASE \"$PGC_DB\" RESET timezone;" +psql_run "ALTER SYSTEM RESET timezone;" +psql_run "SELECT pg_reload_conf();" + pgc_summary diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 7c7ae453a..4425ba7e6 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -6677,6 +6677,7 @@ Written red first: on main the fixture raises `unrecognized configuration parame | `test_a_failing_expire_does_not_undo_the_compaction` | `expire` on a table whose retention column was dropped fails with `42703`, and the compaction due on that table still lands | | `test_the_daemon_yields_the_lock_it_holds_inside_expire` | the worker holds ShareUpdateExclusiveLock inside `expire`; a session's ACCESS EXCLUSIVE is granted within 8 s of a 30 s hold, and the log shows the cancel | | `test_a_table_dropped_mid_pass_is_skipped_and_the_pass_goes_on` | while the worker holds `ex_hold`, `ex_gone` is dropped; after the cancel the same pass logs `ex_gone` as skipped and still expires `ex_after` | +| `test_the_daemon_cuts_a_date_retention_at_its_databases_time_zone` | one row group per day: the first surviving day is the daemon's cutoff in the server's TimeZone, then in a changed one after a reload, and in a database-level TimeZone, which overrides the server's; two zones 25 hours apart, read before and after the passes so a midnight between cannot flake it | ## 97. test_rollup.py: incremental rollups fold appended row groups, recompute changed ones, and equal a full recompute diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index eb87b1ffe..783fc01c6 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -700,4 +700,6 @@ guard_tests 435 # collection over the cluster files on this tree (stacked on #1422). # cluster_tests 585 -> 586 for #1341's test_maintenance_crash.py, one test, by collection # cluster_tests 586 -> 587 for #1341's test_serializable.py, one test, by collection -cluster_tests 587 +# cluster_tests 587 -> 588 for #1341's daemon TimeZone test in test_autovacuum_expire.py, +# by collection over the cluster files on this tree. +cluster_tests 588 diff --git a/test/pytest/test_autovacuum_expire.py b/test/pytest/test_autovacuum_expire.py index 19e235ed1..37ea36548 100644 --- a/test/pytest/test_autovacuum_expire.py +++ b/test/pytest/test_autovacuum_expire.py @@ -280,3 +280,79 @@ def test_a_table_dropped_mid_pass_is_skipped_and_the_pass_goes_on(run, expect): expect.num(run["after_aged"], 0, "dropped: and the pass goes on to expire the next table") expect.text("yes" if run["gone_skipped"] >= 1 else "no", "yes", "dropped: a table dropped mid-pass is logged as skipped") + + +def test_the_daemon_cuts_a_date_retention_at_its_databases_time_zone(pgc_own_db, pgc_cluster, expect): + """Which TimeZone the daemon's expire uses for a date retention (#1341). how-to.md + said the server's; measured, it follows the server's, including a reloaded change, + and a database-level ALTER DATABASE ... SET TimeZone overrides it, as for any + session connecting to that database. Two zones 25 hours apart, never on the same + date. One row group per day, so the first surviving day IS the daemon's cutoff. + + A private database, so the database-level setting reaches no other module, and every + setting this changes is reset before it returns: the daemon visits every database.""" + conn = pgc_own_db + db = _one(conn, "SELECT current_database()") + touched = ("timezone", "pgcolumnar.autovacuum", "pgcolumnar.autovacuum_expire", "pgcolumnar.autovacuum_naptime") + + def cut(zone): + return _one(conn, f"SELECT ((now() AT TIME ZONE '{zone}')::date - 1)::text") + + def expire(on): + _reload(conn, pgcolumnar__autovacuum_expire="on" if on else "off") + if not on: + time.sleep(5) + + def mk(t): + conn.execute(f"CREATE TABLE {t} (d date, v int) USING pgcolumnar") + conn.execute(f"SELECT pgcolumnar.set_options('{t}', chunk_group_row_limit => 1000," + " stripe_row_limit => 1000, ttl_column => 'd', ttl_interval => '1 day')") + for k in range(-6, 7): + conn.execute(f"INSERT INTO {t} SELECT (now() AT TIME ZONE 'UTC')::date + {k}, g" + " FROM generate_series(1, 100) g") + + def first(t, zone, start): + """'match' if the first surviving day is the zone's cutoff, read just before and + just after the daemon's passes (a zone's midnight may fall between).""" + before = cut(zone) + expire(True) + got, last, same = None, None, 0 + for _ in range(60): + got = _one(conn, f"SELECT min(d)::text FROM {t}") + if got == last and got != start: + same += 1 + if same >= 8: + break + else: + same = 0 + last = got + time.sleep(1) + after = cut(zone) + expire(False) + return "match" if got in (before, after) else f"{got} (want {before} or {after})" + + try: + _reload(conn, pgcolumnar__autovacuum="on", pgcolumnar__autovacuum_naptime=NAP, + pgcolumnar__autovacuum_expire="off", timezone="'Pacific/Kiritimati'") + time.sleep(3) + mk("tz_k") + start = _one(conn, "SELECT min(d)::text FROM tz_k") + groups = _one(conn, "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('tz_k')") + expect.text(f"{'apart' if cut('Pacific/Kiritimati') != cut('Pacific/Pago_Pago') else 'same'}|{groups}", + "apart|13", "premise: the two zones are on different dates, so every arm below can tell them apart") + expect.text(first("tz_k", "Pacific/Kiritimati", start), "match", + "the daemon cuts a date retention at the server's TimeZone") + _reload(conn, timezone="'Pacific/Pago_Pago'") + time.sleep(3) + mk("tz_p") + expect.text(first("tz_p", "Pacific/Pago_Pago", start), "match", + "and follows a change to it once the configuration is reloaded") + conn.execute(f'ALTER DATABASE "{db}" SET timezone = \'Pacific/Kiritimati\'') + mk("tz_d") + expect.text(first("tz_d", "Pacific/Kiritimati", start), "match", + "a database-level TimeZone overrides the server's for the daemon too") + finally: + conn.execute(f'ALTER DATABASE "{db}" RESET timezone') + for g in touched: + conn.execute(f"ALTER SYSTEM RESET {g}") + conn.execute("SELECT pg_reload_conf()") From b044310ae1dc443e20a8b784fa196ec163e7579c Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 16:38:35 +0000 Subject: [PATCH 2/4] test: the daemon TimeZone helper keeps psql's echo out of its answer (#1341) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VjTxfQJj5pnii9wzmPP9Ho --- test/autovacuum_expire.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/autovacuum_expire.sh b/test/autovacuum_expire.sh index 9a3fcf2fb..d9fdfff44 100755 --- a/test/autovacuum_expire.sh +++ b/test/autovacuum_expire.sh @@ -195,9 +195,9 @@ tz_mk() { # tz_mk TABLE: built with the daemon's expire off, so no pass sees hal done } tz_cut() { q "SELECT ((now() AT TIME ZONE '$1')::date - 1)::text;"; } -tz_expire() { # tz_expire on|off - psql_run "ALTER SYSTEM SET pgcolumnar.autovacuum_expire = $1;" - psql_run "SELECT pg_reload_conf();" +tz_expire() { # tz_expire on|off (silent: tz_first's output is its answer) + psql_run "ALTER SYSTEM SET pgcolumnar.autovacuum_expire = $1;" >/dev/null + psql_run "SELECT pg_reload_conf();" >/dev/null [ "$1" = off ] && sleep 5 } # tz_first TABLE ZONE -> "match" if the first surviving day is the zone's cutoff, read From ac04189913772628685bc9cf14a8ecd85d22d10f Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 17:16:33 +0000 Subject: [PATCH 3/4] test: the daemon TimeZone arms enter the ledger, three dated by a pinned zone (#1341) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VjTxfQJj5pnii9wzmPP9Ho --- test/check_ledger.tsv | 4 ++++ test/check_ledger_budget.txt | 13 ++++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 0b0c702ee..734b4a672 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -208,14 +208,18 @@ autovacuum_expire autovacuum_expire ON: a table with no retention keeps every ro autovacuum_expire autovacuum_expire ON: no row inside the retention was lost 15;16;17;18;19 2026-10-12 expire retires every row group, the cutoff comparison made always true autovacuum_expire autovacuum_expire ON: the daemon expired the aged-out rows without being asked 15;16;17;18;19 2026-10-12 the daemon never runs expire, its call guarded by false autovacuum_expire autovacuum_expire ON: the daemon logged what it retired 15;16;17;18;19 2026-10-12 expire retires every row group, the cutoff comparison made always true;the daemon never runs expire, its call guarded by false;the daemon's 'expire retired row groups' LOG line removed +autovacuum_expire autovacuum_expire a database-level TimeZone overrides the server's for the daemon too 15;16;17;18;19 2026-10-14 the daemon pins its expire to UTC (run at 17:12 UTC, when Kiritimati is a day ahead) autovacuum_expire autovacuum_expire a manual expire on the same fixture retires its two aged groups 15;16;17;18;19 2026-10-12 expire retires every row group, the cutoff comparison made always true autovacuum_expire autovacuum_expire a pass that retires nothing logs nothing at LOG for that table 15;16;17;18;19 2026-10-12 the daemon logs 'retired 0 row groups' at LOG on every pass +autovacuum_expire autovacuum_expire and follows a change to it once the configuration is reloaded 15;16;17;18;19 2026-10-14 the daemon pins its expire to Pacific/Kiritimati autovacuum_expire autovacuum_expire dropped: a table dropped mid-pass is logged as skipped 15;16;17;18;19 2026-10-12 the daemon worker exits after the first expire that fails, abandoning the rest of its pass;the daemon's 'skipped expire on ' LOG line removed autovacuum_expire autovacuum_expire dropped: and the pass goes on to expire the next table 15;16;17;18;19 2026-10-12 the daemon worker exits after the first expire that fails, abandoning the rest of its pass autovacuum_expire autovacuum_expire isolate: a failing expire does not roll back the compaction due on the same table 15;16;17;18;19 2026-10-12 an expire that fails makes the daemon skip that table's compaction, as one shared transaction would;the daemon worker exits after the first expire that fails, abandoning the rest of its pass autovacuum_expire autovacuum_expire premise: expire on a table whose TTL column is gone fails with undefined_column 15;16;17;18;19 never - autovacuum_expire autovacuum_expire premise: the daemon compacted a deleted-heavy table, so it was running passes 15;16;17;18;19 never - autovacuum_expire autovacuum_expire premise: the maintenance launcher is running 15;16;17;18;19 never - +autovacuum_expire autovacuum_expire premise: the two zones are on different dates, so every arm below can tell them apart 15;16;17;18;19 never - +autovacuum_expire autovacuum_expire the daemon cuts a date retention at the server's TimeZone 15;16;17;18;19 2026-10-14 the daemon pins its expire to UTC (run at 17:12 UTC, when Kiritimati is a day ahead) autovacuum_expire autovacuum_expire the daemon holds ShareUpdateExclusiveLock inside expire (the hold window is open) 15;16;17;18;19 2026-10-12 pgcolumnar_maintenance_hold removed from expire, so the daemon holds no lock inside expire long enough to yield;the daemon never runs expire, its call guarded by false;the daemon runs expire on every table and PgColumnarReadTtl invents a 1 day retention on ts where none is declared autovacuum_expire autovacuum_expire the daemon's expire switch exists and is OFF by default 15;16;17;18;19 2026-10-12 pgcolumnar.autovacuum_expire defaults to on, in both its C initializer and its GUC boot value autovacuum_expire autovacuum_expire yield: and it was granted fast (<= 8s, far under the 30s hold) 15;16;17;18;19 2026-10-12 the daemon worker no longer sets PROC_IS_AUTOVACUUM, so core never cancels it for a stronger lock diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 95f2bdc18..cea363447 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -2522,4 +2522,15 @@ suites_not_covered 230 # never 2338 + 6 = 2344 # dated 764 + 3 = 767 # 2344 + 767 == 3111 -checks_never_observed_red 2344 +# +# #1341, the daemon's TimeZone for a date retention (OffgridwithJD): 2344 -> 2345. Four +# new autovacuum_expire checks. Three arms are dated by the daemon pinning its expire +# to one zone: UTC (at 17:12 UTC, a day behind Kiritimati) reddens the server-zone and +# database-zone arms; Kiritimati reddens the reload arm. The premise that the two +# zones are on different dates arrives never. Five-major runs: 23/23 on 15-19. +# +# ledger rows 3111 + 4 = 3115 +# never 2344 + 1 = 2345 +# dated 767 + 3 = 770 +# 2345 + 770 == 3115 +checks_never_observed_red 2345 From 2a5203ab69c10aa894b896893e7225f3055bd2d0 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Wed, 14 Oct 2026 19:48:02 +0000 Subject: [PATCH 4/4] docs: the daemon TimeZone sentence is split so docs_style passes (#1393 review) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01VjTxfQJj5pnii9wzmPP9Ho --- docs/how-to.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/how-to.md b/docs/how-to.md index 23d223a97..45a2c809f 100644 --- a/docs/how-to.md +++ b/docs/how-to.md @@ -335,10 +335,11 @@ per `pgcolumnar.autovacuum_naptime` (default 60 seconds), and logs any. A pass that retires nothing logs at `DEBUG1` only. For a `date` or `timestamp` retention column, the cutoff depends on the time -zone. The daemon uses the `TimeZone` in effect for the database it connects to: -a database-level setting (`ALTER DATABASE ... SET TimeZone`) when there is one, -otherwise the server's `TimeZone`, including a change made by reloading the -configuration. Either may differ from the time zone your sessions use. For a +zone. The daemon uses the `TimeZone` in effect for the database it connects to. +That is the database's own setting (`ALTER DATABASE ... SET TimeZone`) when it +has one. Otherwise it is the server's `TimeZone`, including a change made by +reloading the configuration. Either may differ from the time zone your sessions +use. For a cutoff in your session's time zone, run `pgcolumnar.expire` by hand from that session.