From 0e32c6c466561072d73e45afb7fccd7e3f29d06c Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 12:10:23 +0000 Subject: [PATCH 1/6] test: a no-op expire must read the extent, not every zone map, red (#1310) Seen red on PG17: fifty groups, fifty zone_map scans. The guards beside it are green on this tree: without the extent it still probes every group, with it the same groups retire, and aged rows written in expire's own transaction are retired. Co-Authored-By: Claude Opus 5.5 --- test/ttl_expire.sh | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/test/ttl_expire.sh b/test/ttl_expire.sh index 62de58a35..0d2d9fca8 100755 --- a/test/ttl_expire.sh +++ b/test/ttl_expire.sh @@ -525,4 +525,49 @@ psql_run "DROP TABLE ttl_rec;" check_num "dropping the table drops its retention record" \ "$(q "SELECT count(*) FROM pgcolumnar.retired_group WHERE storage_id = $_sid;")" "0" +# ---- a no-op expire reads the extent, not every zone map (#1310) ------------- +# Most of the daemon's passes retire nothing. When the retention column's extent +# (#1304 option 5) puts its minimum past the cutoff, no group can qualify, and +# expire answers 0 from one index lookup instead of one zone-map probe per group. +# Fifty groups, so a per-group read shows as fifty scans. The instrument is the +# one native_groupagg uses: the transaction's own scans of zone_map and its index, +# in a fresh connection, because the counters carry a backend's earlier reads. +psql_run "CREATE TABLE ttl_x (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('ttl_x', ttl_column => 'ts', ttl_interval => '3650 days'); + INSERT INTO ttl_x SELECT now() - interval '1 day' + g * interval '1 second', g FROM generate_series(1, 50000) g;" +_xsid="$(q "SELECT pgcolumnar.get_storage_id('ttl_x');")" +check_num "premise: the no-op fixture has fifty row groups" \ + "$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = $_xsid;")" "50" +_expire_scans() { # _expire_scans TABLE [SQL run first in the transaction] -> "n= scans=" + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At -q \ + -c "BEGIN" ${2:+-c "$2"} \ + -c "SELECT 'n=' || pgcolumnar.expire('$1') || ' scans=' || + (pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass) + + pg_stat_get_xact_numscans('pgcolumnar.zone_map_pkey'::regclass))" \ + -c "ROLLBACK" 2>&1 +} +check "a no-op expire reads no zone map when the extent's minimum is past the cutoff" \ + "$(_expire_scans ttl_x)" "n=0 scans=0" +check "without the extent the same expire probes every group's zone map" \ + "$(_expire_scans ttl_x "DELETE FROM pgcolumnar.column_extent WHERE storage_id = $_xsid")" "n=0 scans=50" + +# The extent may say a group can qualify, and then expire decides group by group as +# before: the same groups with or without it. +psql_run "CREATE TABLE ttl_y (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('ttl_y', ttl_column => 'ts', ttl_interval => '2 days'); + INSERT INTO ttl_y SELECT now() - interval '9 days' + g * interval '1 second', g FROM generate_series(1, 3000) g; + INSERT INTO ttl_y SELECT now() + g * interval '1 second', g FROM generate_series(1, 2000) g;" +_ysid="$(q "SELECT pgcolumnar.get_storage_id('ttl_y');")" +_y_without="$(_expire_scans ttl_y "DELETE FROM pgcolumnar.column_extent WHERE storage_id = $_ysid" | sed 's/ scans=.*//')" +check "premise: without the extent, expire retires the three aged groups" "$_y_without" "n=3" +check "with the extent, expire retires the same groups" "$(_expire_scans ttl_y | sed 's/ scans=.*//')" "$_y_without" + +# Old rows written earlier in expire's own transaction: their groups are visible to +# it before its extent row is written, so the early exit must count them. +psql_run "CREATE TABLE ttl_z (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('ttl_z', ttl_column => 'ts', ttl_interval => '2 days'); + INSERT INTO ttl_z SELECT now() + g * interval '1 second', g FROM generate_series(1, 2000) g;" +check "expire in the transaction that wrote aged rows retires their groups" \ + "$(_expire_scans ttl_z "INSERT INTO ttl_z SELECT now() - interval '9 days' + g * interval '1 second', g FROM generate_series(1, 3000) g" | sed 's/ scans=.*//')" "n=3" + pgc_summary From ab276aa386524a89d34eeeb281d643fc1fed79ba Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 12:10:58 +0000 Subject: [PATCH 2/6] feat: a no-op expire answers from the column's extent, without a zone-map probe per group (#1310) When the retention column's extent puts its minimum at or past the cutoff, no group's maximum is below it, so nothing can qualify. Built on #1304 option 5, whose bound includes the transaction's own flushed groups. ttl_expire.sh 65/65 on PG17. Co-Authored-By: Claude Opus 5.5 --- src/columnar_vacuum.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/columnar_vacuum.c b/src/columnar_vacuum.c index 5daeb0094..b4bd94e8a 100644 --- a/src/columnar_vacuum.c +++ b/src/columnar_vacuum.c @@ -2491,6 +2491,35 @@ pgcolumnar_expire(PG_FUNCTION_ARGS) } storageId = PgColumnarStorageId(rel); + + /* + * THE NO-OP PASS ENDS HERE (#1310). Most of the daemon's passes retire + * nothing, and the loop below proves that with one zone-map probe per group. + * The column's extent (#1304 option 5) bounds every value a group visible to + * this snapshot can hold, this transaction's own flushed groups included, so + * when its minimum is not below the cutoff no group's maximum is either, and + * nothing can qualify. A column with no value at all has nothing to retire + * either: its groups have no minimum and maximum, which the loop keeps. + * + * An extent can be wider than the data, never narrower, and a wider one only + * means no early exit. With no complete extent the loop decides as before. + */ + { + Datum emin, + emax; + bool hasValues; + + if (PgColumnarExtentBound(rel, attno - 1, GetActiveSnapshot(), + &emin, &emax, &hasValues) && + (!hasValues || + DatumGetInt32(FunctionCall2Coll(&tce->cmp_proc_finfo, att->attcollation, + emin, cutoff)) >= 0)) + { + table_close(rel, NoLock); + PG_RETURN_INT64(0); + } + } + groups = PgColumnarReadRowGroupList(storageId, GetActiveSnapshot()); /* From b2be9a1b1e148d1471dfc17a2ef54e9ace066b66 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 12:12:08 +0000 Subject: [PATCH 3/6] test: the pytest twin of #1310's arms, and a column left with only NULLs The twin was seen red on PG17 against the tree before the early exit: forty groups, forty zone_map scans. The NULL arm was written after the code and is dated by a mutation of the !hasValues clause, not by the red-first run. Check names carry no counts, since the twins' fixtures differ. Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_ttl_expire.py | 68 ++++++++++++++++++++++++++++++++++ test/ttl_expire.sh | 15 +++++++- 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/test/pytest/test_ttl_expire.py b/test/pytest/test_ttl_expire.py index 939c2dd54..3cf5258e3 100644 --- a/test/pytest/test_ttl_expire.py +++ b/test/pytest/test_ttl_expire.py @@ -663,3 +663,71 @@ def rec(sid): pgc_conn.execute("DROP TABLE ttl_rr") expect.num(_one(pgc_conn, "SELECT count(*) FROM pgcolumnar.retired_group WHERE storage_id = %s", (sid,)), 0, "dropping the table drops its retention record") + + +def test_a_no_op_expire_reads_the_extent_not_every_zone_map(pgc_conn, pgc_cluster, expect): + """Most of the daemon's passes retire nothing (#1310). When the retention + column's extent (#1304 option 5) puts its minimum past the cutoff, no group can + qualify, and expire answers 0 without a zone-map probe per group. + + Counted as native_groupagg counts: the transaction's own scans of zone_map and + its index, in a fresh connection, because the counters carry a backend's earlier + reads for about a second. + """ + dsn = pgc_cluster.dsn(dbname=_one(pgc_conn, "SELECT current_database()")) + path = _one(pgc_conn, "SHOW search_path") + + def run(table, first=None): + """'n= scans=', in one transaction rolled back.""" + with psycopg.connect(dsn) as c: + with c.cursor() as cur: + cur.execute(f"SET search_path TO {path}") + if first: + cur.execute(first) + cur.execute(f"SELECT pgcolumnar.expire('{table}')," + " pg_stat_get_xact_numscans('pgcolumnar.zone_map'::regclass)" + " + pg_stat_get_xact_numscans('pgcolumnar.zone_map_pkey'::regclass)") + n, scans = cur.fetchone() + c.rollback() + return f"n={n} scans={scans}" + + def table(name, interval, *inserts): + with pgc_conn.cursor() as cur: + cur.execute(f"CREATE TABLE {name} (ts timestamptz, v int) USING pgcolumnar") + cur.execute(f"SELECT pgcolumnar.set_options('{name}', stripe_row_limit => {GROUP}," + f" ttl_column => 'ts', ttl_interval => '{interval}')") + for sql in inserts: + cur.execute(sql) + return _one(pgc_conn, f"SELECT pgcolumnar.get_storage_id('{name}')") + + sid = table("ttl_nx", "4000 days", + f"INSERT INTO ttl_nx SELECT now() - interval '20 hours' + g * interval '1 s', g" + f" FROM generate_series(1, {40 * GROUP}) g") + expect.num(_one(pgc_conn, "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = %s", (sid,)), 40, + "premise: the no-op fixture has many row groups") + expect.text(run("ttl_nx"), "n=0 scans=0", + "a no-op expire reads no zone map when the extent's minimum is past the cutoff") + expect.text(run("ttl_nx", f"DELETE FROM pgcolumnar.column_extent WHERE storage_id = {sid}"), "n=0 scans=40", + "without the extent the same expire probes every group's zone map") + + sid = table("ttl_ny", "3 days", + f"INSERT INTO ttl_ny SELECT now() - interval '12 days' + g * interval '1 s', g" + f" FROM generate_series(1, {4 * GROUP}) g", + f"INSERT INTO ttl_ny SELECT now() + g * interval '1 s', g FROM generate_series(1, {GROUP}) g") + without = run("ttl_ny", f"DELETE FROM pgcolumnar.column_extent WHERE storage_id = {sid}").split(" ")[0] + expect.text(without, "n=4", "premise: without the extent, expire retires the aged groups") + expect.text(run("ttl_ny").split(" ")[0], without, "with the extent, expire retires the same groups") + + table("ttl_nz", "3 days", + f"INSERT INTO ttl_nz SELECT now() + g * interval '1 s', g FROM generate_series(1, {GROUP}) g") + expect.text(run("ttl_nz", f"INSERT INTO ttl_nz SELECT now() - interval '12 days' + g * interval '1 s', g" + f" FROM generate_series(1, {4 * GROUP}) g").split(" ")[0], "n=4", + "expire in the transaction that wrote aged rows retires their groups") + + table("ttl_nn", "3 days", + f"INSERT INTO ttl_nn SELECT now() - interval '12 days' + g * interval '1 s', g" + f" FROM generate_series(1, {3 * GROUP}) g", + f"INSERT INTO ttl_nn SELECT NULL, g FROM generate_series(1, {2 * GROUP}) g") + expect.num(_one(pgc_conn, "SELECT pgcolumnar.expire('ttl_nn')"), 3, + "premise: expire retires the aged groups and keeps those of NULLs") + expect.text(run("ttl_nn"), "n=0 scans=0", "a later expire on a column of only NULLs reads no zone map") diff --git a/test/ttl_expire.sh b/test/ttl_expire.sh index 0d2d9fca8..ed150d1ac 100755 --- a/test/ttl_expire.sh +++ b/test/ttl_expire.sh @@ -536,7 +536,7 @@ psql_run "CREATE TABLE ttl_x (ts timestamptz, v int) USING pgcolumnar; SELECT pgcolumnar.set_options('ttl_x', ttl_column => 'ts', ttl_interval => '3650 days'); INSERT INTO ttl_x SELECT now() - interval '1 day' + g * interval '1 second', g FROM generate_series(1, 50000) g;" _xsid="$(q "SELECT pgcolumnar.get_storage_id('ttl_x');")" -check_num "premise: the no-op fixture has fifty row groups" \ +check_num "premise: the no-op fixture has many row groups" \ "$(q "SELECT count(*) FROM pgcolumnar.row_group WHERE storage_id = $_xsid;")" "50" _expire_scans() { # _expire_scans TABLE [SQL run first in the transaction] -> "n= scans=" env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -X -At -q \ @@ -559,7 +559,7 @@ psql_run "CREATE TABLE ttl_y (ts timestamptz, v int) USING pgcolumnar; INSERT INTO ttl_y SELECT now() + g * interval '1 second', g FROM generate_series(1, 2000) g;" _ysid="$(q "SELECT pgcolumnar.get_storage_id('ttl_y');")" _y_without="$(_expire_scans ttl_y "DELETE FROM pgcolumnar.column_extent WHERE storage_id = $_ysid" | sed 's/ scans=.*//')" -check "premise: without the extent, expire retires the three aged groups" "$_y_without" "n=3" +check "premise: without the extent, expire retires the aged groups" "$_y_without" "n=3" check "with the extent, expire retires the same groups" "$(_expire_scans ttl_y | sed 's/ scans=.*//')" "$_y_without" # Old rows written earlier in expire's own transaction: their groups are visible to @@ -570,4 +570,15 @@ psql_run "CREATE TABLE ttl_z (ts timestamptz, v int) USING pgcolumnar; check "expire in the transaction that wrote aged rows retires their groups" \ "$(_expire_scans ttl_z "INSERT INTO ttl_z SELECT now() - interval '9 days' + g * interval '1 second', g FROM generate_series(1, 3000) g" | sed 's/ scans=.*//')" "n=3" +# A column left with no value at all: expire retires every group that held one, +# and the recompute it commits leaves a complete extent with no minimum. The NULL +# rows' groups are kept, and the next pass has nothing to read. +psql_run "CREATE TABLE ttl_n (ts timestamptz, v int) USING pgcolumnar; + SELECT pgcolumnar.set_options('ttl_n', ttl_column => 'ts', ttl_interval => '2 days'); + INSERT INTO ttl_n SELECT now() - interval '9 days' + g * interval '1 second', g FROM generate_series(1, 2000) g; + INSERT INTO ttl_n SELECT NULL, g FROM generate_series(1, 4000) g;" +check_num "premise: expire retires the aged groups and keeps those of NULLs" \ + "$(q "SELECT pgcolumnar.expire('ttl_n');")" "2" +check "a later expire on a column of only NULLs reads no zone map" "$(_expire_scans ttl_n)" "n=0 scans=0" + pgc_summary From a4229744b052dbcd428558215952757db85820e7 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 12:26:49 +0000 Subject: [PATCH 4/6] test: 'expire retired the three aged groups' is an arm, not a premise (#1236) A mutation of expire's early exit (the extent's maximum compared instead of its minimum) reddens it, so it measures the code under test. Renamed in both twins, at jdatcmd's request on #1304. Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_ttl_expire.py | 2 +- test/ttl_expire.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pytest/test_ttl_expire.py b/test/pytest/test_ttl_expire.py index 3cf5258e3..f2bd4c6ee 100644 --- a/test/pytest/test_ttl_expire.py +++ b/test/pytest/test_ttl_expire.py @@ -649,7 +649,7 @@ def rec(sid): expect.text(f"{rec(sid)}|{rgs(sid)}", "|1,2,3,4,5", "a rolled-back expire records nothing") n = _one(pgc_conn, "SELECT pgcolumnar.expire('ttl_rr')") - expect.num(n, 3, "premise: expire retired the three aged groups") + expect.num(n, 3, "expire retired the three aged groups") expect.text(rec(sid), "1,2,3", "expire records exactly the groups it retired") expect.text(rgs(sid), "4,5", "and they are the groups gone from the row-group catalog") diff --git a/test/ttl_expire.sh b/test/ttl_expire.sh index ed150d1ac..b5dac4f7a 100755 --- a/test/ttl_expire.sh +++ b/test/ttl_expire.sh @@ -508,7 +508,7 @@ psql_run "BEGIN; SELECT pgcolumnar.expire('ttl_rec'); ROLLBACK;" check "a rolled-back expire records nothing" "$(_rec)|$(_rgs)" "|1,2,3,4" _n="$(q "SELECT pgcolumnar.expire('ttl_rec');")" -check_num "premise: expire retired the three aged groups" "${_n:-0}" "3" +check_num "expire retired the three aged groups" "${_n:-0}" "3" check "expire records exactly the groups it retired" "$(_rec)" "1,2,3" check "and they are the groups gone from the row-group catalog" "$(_rgs)" "4" From 4e723691477df491efc9ea31f1ff7c4d178cb180 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 12:30:13 +0000 Subject: [PATCH 5/6] test: ttl_expire enters the ledger, 67 checks, 16 dated by four mutations (#1310) ttl_expire.sh 67/67 and test_ttl_expire.py 9/9 on each of PG15-19. Seeded at jdatcmd's request on #1304; suites_not_covered 247 -> 246. Ledger 2162 / 1756. cluster_tests 518 -> 519 by collection. TESTS.md and CHANGELOG. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++++ test/check_ledger.tsv | 67 ++++++++++++++++++++++++++++++++++ test/check_ledger_budget.txt | 23 +++++++++++- test/pytest/TESTS.md | 3 +- test/pytest/expected_tests.txt | 5 ++- 5 files changed, 102 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 657a731d1..d72be83ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,14 @@ true until the next version shipped. ### Added +- A `pgcolumnar.expire` that has nothing to retire now answers from the + retention column's extent in one index lookup, instead of one zone-map probe + per row group (#1310). Most passes of the maintenance daemon are such passes. When + the extent's minimum is not below the cutoff, no group can qualify; otherwise + `expire` decides group by group as before. Measured as zone-map scans for a + no-op `expire` over 50 groups: 50 before, 0 after. `test/ttl_expire.sh` and + `test_ttl_expire.py`; `ttl_expire` now enters the mutation ledger. + - The planner's estimate of how many time buckets a `GROUP BY date_trunc(...)` or `date_bin(...)` produces now reads a per-column extent (`pgcolumnar.column_extent`) in one index lookup, instead of every row group's diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index e5f4e4dca..2b17c703c 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2165,6 +2165,73 @@ timeseries_recipes timeseries_recipes the how-to gives the LOCF recipe 15;16;17; timeseries_recipes timeseries_recipes the how-to gives the gap-fill recipe 15;16;17;18;19 2026-10-13 the how-to's 'Fill empty buckets' heading renamed, so the recipe is no longer found timeseries_recipes timeseries_recipes the how-to gives the interpolation recipe 15;16;17;18;19 2026-10-13 the how-to's 'Interpolate between known values' heading renamed, so the recipe is no longer found timeseries_recipes timeseries_recipes the interpolation recipe fills a gap linearly, and leaves both ends NULL 15;16;17;18;19 2026-10-13 the how-to's 'Interpolate between known values' heading renamed, so the recipe is no longer found;the how-to's interpolation recipe uses prev_v in place of next_v, so every gap is filled flat;the how-to's recipes join the series to the aggregate with JOIN instead of LEFT JOIN +ttl_expire ttl_expire NO row still inside the retention was dropped (#403 item 5a) 15;16;17;18;19 never - +ttl_expire ttl_expire NO row still inside the retention was dropped, on a date column 15;16;17;18;19 never - +ttl_expire ttl_expire a compaction's retired groups are not recorded as retention 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire a date retention column is accepted, not refused (#1135) 15;16;17;18;19 never - +ttl_expire ttl_expire a group whose only NULLs have been deleted is retired 15;16;17;18;19 never - +ttl_expire ttl_expire a later expire on a column of only NULLs reads no zone map 15;16;17;18;19 2026-10-13 expire's early exit removed (the tree before #1310);the early exit not taken for a column with no value +ttl_expire ttl_expire a negative ttl_interval is refused with 22023 15;16;17;18;19 never - +ttl_expire ttl_expire a no-op expire reads no zone map when the extent's minimum is past the cutoff 15;16;17;18;19 2026-10-13 expire's early exit removed (the tree before #1310) +ttl_expire ttl_expire a rolled-back expire records nothing 15;16;17;18;19 never - +ttl_expire ttl_expire a row dated exactly on the cutoff is kept, so truncation errs toward keeping 15;16;17;18;19 never - +ttl_expire ttl_expire a table with no declared retention is an error, not a silent success 15;16;17;18;19 never - +ttl_expire ttl_expire and a zero ttl_interval is refused too 15;16;17;18;19 never - +ttl_expire ttl_expire and its rows are gone 15;16;17;18;19 never - +ttl_expire ttl_expire and no row older than Pacific/Midway's own cutoff survives 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire and no row older than UTC's own cutoff survives 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire and the date table really is smaller than it was 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire and the expired timestamps sharing the group were kept with them 15;16;17;18;19 never - +ttl_expire ttl_expire and the table lost exactly the groups it retired 15;16;17;18;19 never - +ttl_expire ttl_expire and the table really is smaller than it was 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire and they are the groups gone from the row-group catalog 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire and those NULL rows are still there 15;16;17;18;19 never - +ttl_expire ttl_expire control: a group whose NULLs are still live is NOT retired 15;16;17;18;19 never - +ttl_expire ttl_expire control: and a positive one is still accepted 15;16;17;18;19 never - +ttl_expire ttl_expire control: and the rows are all still there 15;16;17;18;19 never - +ttl_expire ttl_expire control: and those NULL rows survive 15;16;17;18;19 never - +ttl_expire ttl_expire dropping the table drops its retention record 15;16;17;18;19 never - +ttl_expire ttl_expire every remaining date row reads back its own value 15;16;17;18;19 never - +ttl_expire ttl_expire every remaining row reads back its own value 15;16;17;18;19 never - +ttl_expire ttl_expire expire does not retire a group that still holds NULL retention rows 15;16;17;18;19 never - +ttl_expire ttl_expire expire in Pacific/Midway keeps exactly the rows at or after that session's cutoff 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire expire in UTC keeps exactly the rows at or after that session's cutoff 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire expire in the transaction that wrote aged rows retires their groups 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum;the extent bound leaves out the transaction's own pending fold (#1304 option 5's fix) +ttl_expire ttl_expire expire on a date column retires at least one group 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire expire records exactly the groups it retired 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire expire retired the three aged groups 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire expire retires at least one group (#403 item 5a) 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire expire retires the all-visible expired group 15;16;17;18;19 never - +ttl_expire ttl_expire index-only scan does not return rows expire already retired 15;16;17;18;19 never - +ttl_expire ttl_expire premise: an index-only scan is chosen on the all-visible table 15;16;17;18;19 never - +ttl_expire ttl_expire premise: and VACUUM really did set visibility-map bits to clear 15;16;17;18;19 never - +ttl_expire ttl_expire premise: and every live row is past the retention 15;16;17;18;19 never - +ttl_expire ttl_expire premise: and rows dated exactly ON the cutoff, which decide the rounding 15;16;17;18;19 never - +ttl_expire ttl_expire premise: and the SAME row group holds expired rows and NULL ones 15;16;17;18;19 never - +ttl_expire ttl_expire premise: but the zone map still records the deleted NULLs 15;16;17;18;19 never - +ttl_expire ttl_expire premise: compact_rewrite retired a group 15;16;17;18;19 never - +ttl_expire ttl_expire premise: expire retires the aged groups and keeps those of NULLs 15;16;17;18;19 never - +ttl_expire ttl_expire premise: in Pacific/Midway the fixture straddles that session's cutoff 15;16;17;18;19 never - +ttl_expire ttl_expire premise: in UTC the fixture straddles that session's cutoff 15;16;17;18;19 never - +ttl_expire ttl_expire premise: no LIVE row holds a NULL retention value any more 15;16;17;18;19 never - +ttl_expire ttl_expire premise: one date per row group in Pacific/Midway, so no group straddles the cutoff 15;16;17;18;19 never - +ttl_expire ttl_expire premise: one date per row group in UTC, so no group straddles the cutoff 15;16;17;18;19 never - +ttl_expire ttl_expire premise: rows exist on both sides of the cutoff within one day of it 15;16;17;18;19 never - +ttl_expire ttl_expire premise: some rows are past the retention and some are not 15;16;17;18;19 never - +ttl_expire ttl_expire premise: the date fixture has rows on both sides of the cutoff 15;16;17;18;19 never - +ttl_expire ttl_expire premise: the date fixture is laid out in several row groups 15;16;17;18;19 never - +ttl_expire ttl_expire premise: the fixture is ONE row group, so the NULLs share it 15;16;17;18;19 never - +ttl_expire ttl_expire premise: the fixture is laid out in several row groups 15;16;17;18;19 never - +ttl_expire ttl_expire premise: the no-op fixture has many row groups 15;16;17;18;19 never - +ttl_expire ttl_expire premise: the record fixture has three aged-out row groups ahead of the current ones 15;16;17;18;19 never - +ttl_expire ttl_expire premise: the table holds NULL retention rows at all 15;16;17;18;19 never - +ttl_expire ttl_expire premise: without the extent, expire retires the aged groups 15;16;17;18;19 never - +ttl_expire ttl_expire running it again retires nothing, because nothing new expired 15;16;17;18;19 never - +ttl_expire ttl_expire seqscan agrees the expired rows are gone 15;16;17;18;19 never - +ttl_expire ttl_expire the index-only scan is still the plan after expire 15;16;17;18;19 never - +ttl_expire ttl_expire the retention is recorded where the other options live 15;16;17;18;19 never - +ttl_expire ttl_expire with the extent, expire retires the same groups 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire without the extent the same expire probes every group's zone map 15;16;17;18;19 never - validity_elision validity_elision a chunk claiming no bitmap while it holds fewer values than rows is refused 15;16;17;18;19 never - validity_elision validity_elision a column with no nulls stores no validity bitmap 15;16;17;18;19 2026-09-18 writer: flush_one_column's presentCount == rowCount forced false, so the bitmap is written even when the chunk holds no null validity_elision validity_elision a null-free column elides its bitmap beside a null-bearing one in the same row group 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 93b191383..3d6f745e9 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 247 +suites_not_covered 246 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -1540,4 +1540,23 @@ suites_not_covered 247 # never 1746 - 1 = 1745 # dated 446 + 1 = 447 # 1745 + 447 == 2192 -checks_never_observed_red 1745 +# +# #1310, a no-op expire answers from the extent (OffgridwithJD): 1745 -> 1796, +# and suites_not_covered 247 -> 246. ttl_expire enters the ledger with all 67 +# checks from a five-major run (ttl_expire.sh 67/67 on each of 15-19), at +# jdatcmd's request on #1304: it is the suite for the one function that deletes +# rows. Sixteen are dated by four mutations, each merged one log at a time with +# --mutation and --target: the early exit removed (the tree before #1310), the +# extent's maximum compared instead of its minimum (fourteen checks, among them +# twelve of the suite's existing arms), the exit not taken for a column with no +# value, and the extent bound without option 5's own-transaction fold. No +# premise is among them: 'expire retired the three aged groups' was a premise +# that the second mutation reddened, and was renamed to an arm first (#1236). +# The other fifty-one arrive never -- the suite's older arms, which no mutation +# of this change reaches, and its premises. +# +# ledger rows 2192 + 67 = 2259 +# never 1745 + 51 = 1796 +# dated 447 + 16 = 463 +# 1796 + 463 == 2259 +checks_never_observed_red 1796 diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 8a9eb7113..c6b5b6698 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -6195,8 +6195,9 @@ restored byte-identical. | `test_a_date_retention_column_truncates_toward_keeping` | the rounding, which only this arm notices | | `test_the_cutoff_is_the_sessions_own_date` | the cutoff is the session's, asserted as an identity rather than as a difference | | `test_expire_records_what_it_retires_and_nothing_else` | `expire` writes exactly the groups it retires to `pgcolumnar.retired_group` (#1294); a rolled-back `expire` and a `compact_rewrite` record nothing; dropping the table drops its rows | +| `test_a_no_op_expire_reads_the_extent_not_every_zone_map` | #1310: with the retention column's extent (#1304 option 5) past the cutoff, a no-op `expire` makes 0 zone-map scans where it made one per group; the same groups retire with or without the extent; aged rows written in `expire`'s own transaction retire; a column left with only NULLs needs no reads. Counted in a fresh connection, with the test schema's `search_path` carried over. Seen red first: 40 scans | -The shell twin is `test/ttl_expire.sh`, 51 checks. The two halves share no code: +The shell twin is `test/ttl_expire.sh`, 67 checks since #1310, all in the mutation ledger. The two halves share no code: different tables, different row counts, different retention windows, and the refusals asserted by SQLSTATE here (`55000` for no declared retention, `22023` for a non-positive interval) where the shell greps its message. diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 168a8877e..ff49a70c2 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -591,4 +591,7 @@ guard_tests 419 # # cluster_tests 523 -> 524 for option 5's test_a_transaction_reads_its_own_flushed_groups, # by collection on the same tree. -cluster_tests 524 +# +# cluster_tests 524 -> 525 for #1310's test_a_no_op_expire_reads_the_extent_not_every_zone_map, +# by collection on this branch (stacked on #1304 option 5). +cluster_tests 525 From 4fd0480dd0bc509268a148deccfaf7bb5a131c94 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Tue, 13 Oct 2026 13:07:45 +0000 Subject: [PATCH 6/6] fix: initialise expire's extent bounds, as #1325 does the planner's The same out-parameter shape gcc 13/14 with LTO reported as maybe-uninitialized on #1325; emin and emax are read only when hasValues is true. Co-Authored-By: Claude Opus 5.5 --- src/columnar_vacuum.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/columnar_vacuum.c b/src/columnar_vacuum.c index b4bd94e8a..0507e9d0d 100644 --- a/src/columnar_vacuum.c +++ b/src/columnar_vacuum.c @@ -2505,9 +2505,10 @@ pgcolumnar_expire(PG_FUNCTION_ARGS) * means no early exit. With no complete extent the loop decides as before. */ { - Datum emin, - emax; - bool hasValues; + /* read only when the column has a value; initialised for gcc's LTO pass */ + Datum emin = (Datum) 0, + emax = (Datum) 0; + bool hasValues = false; if (PgColumnarExtentBound(rel, attno - 1, GetActiveSnapshot(), &emin, &emax, &hasValues) &&