diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index cb4c291d7..4ac845e2d 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2250,23 +2250,25 @@ ttl_expire ttl_expire NO row still inside the retention was dropped (#403 item 5 ttl_expire ttl_expire NO row still inside the retention was dropped, on a date column 15;16;17;18;19 never - ttl_expire ttl_expire a compaction's retired groups are not recorded as retention 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum ttl_expire ttl_expire a date retention column is accepted, not refused (#1135) 15;16;17;18;19 never - -ttl_expire ttl_expire a group whose only NULLs have been deleted is retired 15;16;17;18;19 never - +ttl_expire ttl_expire a group whose only NULLs have been deleted is retired 15;16;17;18;19 2026-10-14 expire returning 0 whatever it retired ttl_expire ttl_expire a later expire on a column of only NULLs reads no zone map 15;16;17;18;19 2026-10-13 expire's early exit removed (the tree before #1310);the early exit not taken for a column with no value ttl_expire ttl_expire a negative ttl_interval is refused with 22023 15;16;17;18;19 never - ttl_expire ttl_expire a no-op expire reads no zone map when the extent's minimum is past the cutoff 15;16;17;18;19 2026-10-13 expire's early exit removed (the tree before #1310) ttl_expire ttl_expire a rolled-back expire records nothing 15;16;17;18;19 never - ttl_expire ttl_expire a row dated exactly on the cutoff is kept, so truncation errs toward keeping 15;16;17;18;19 never - ttl_expire ttl_expire a table with no declared retention is an error, not a silent success 15;16;17;18;19 never - +ttl_expire ttl_expire and VACUUM really did set visibility-map bits to clear 15;16;17;18;19 never - ttl_expire ttl_expire and a zero ttl_interval is refused too 15;16;17;18;19 never - ttl_expire ttl_expire and its rows are gone 15;16;17;18;19 never - -ttl_expire ttl_expire and no row older than Pacific/Midway's own cutoff survives 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum ttl_expire ttl_expire and no row older than UTC's own cutoff survives 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire and no row older than a zone on another date's own cutoff survives 15;16;17;18;19 2026-10-14 expire's date cutoff taken from the UTC wall clock, not the session's date;the early exit compares the extent's maximum to the cutoff instead of its minimum ttl_expire ttl_expire and the date table really is smaller than it was 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum ttl_expire ttl_expire and the expired timestamps sharing the group were kept with them 15;16;17;18;19 never - -ttl_expire ttl_expire and the table lost exactly the groups it retired 15;16;17;18;19 never - +ttl_expire ttl_expire and the table lost exactly the groups it retired 15;16;17;18;19 2026-10-14 expire returning 0 whatever it retired ttl_expire ttl_expire and the table really is smaller than it was 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum ttl_expire ttl_expire and they are the groups gone from the row-group catalog 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum ttl_expire ttl_expire and those NULL rows are still there 15;16;17;18;19 never - +ttl_expire ttl_expire but the zone map still records the deleted NULLs 15;16;17;18;19 never - ttl_expire ttl_expire control: a group whose NULLs are still live is NOT retired 15;16;17;18;19 never - ttl_expire ttl_expire control: and a positive one is still accepted 15;16;17;18;19 never - ttl_expire ttl_expire control: and the rows are all still there 15;16;17;18;19 never - @@ -2275,28 +2277,27 @@ ttl_expire ttl_expire dropping the table drops its retention record 15;16;17;18; ttl_expire ttl_expire every remaining date row reads back its own value 15;16;17;18;19 never - ttl_expire ttl_expire every remaining row reads back its own value 15;16;17;18;19 never - ttl_expire ttl_expire expire does not retire a group that still holds NULL retention rows 15;16;17;18;19 never - -ttl_expire ttl_expire expire in Pacific/Midway keeps exactly the rows at or after that session's cutoff 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum ttl_expire ttl_expire expire in UTC keeps exactly the rows at or after that session's cutoff 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum -ttl_expire ttl_expire expire in the transaction that wrote aged rows retires their groups 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum;the extent bound leaves out the transaction's own pending fold (#1304 option 5's fix) -ttl_expire ttl_expire expire on a date column retires at least one group 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire expire in a zone on another date keeps exactly the rows at or after that session's cutoff 15;16;17;18;19 2026-10-14 expire's date cutoff taken from the UTC wall clock, not the session's date;the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire expire in the transaction that wrote aged rows retires their groups 15;16;17;18;19 2026-10-14 expire returning 0 whatever it retired;the early exit compares the extent's maximum to the cutoff instead of its minimum;the extent bound leaves out the transaction's own pending fold (#1304 option 5's fix) +ttl_expire ttl_expire expire on a date column retires at least one group 15;16;17;18;19 2026-10-14 expire returning 0 whatever it retired;the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire expire on a timestamp column retires the groups wholly before the session's local cutoff 15;16;17;18;19 2026-10-14 expire's timestamp cutoff taken from the UTC wall clock, not the session's local time ttl_expire ttl_expire expire records exactly the groups it retired 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum -ttl_expire ttl_expire expire retired the three aged groups 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum -ttl_expire ttl_expire expire retires at least one group (#403 item 5a) 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum -ttl_expire ttl_expire expire retires the all-visible expired group 15;16;17;18;19 never - +ttl_expire ttl_expire expire retired the three aged groups 15;16;17;18;19 2026-10-14 expire returning 0 whatever it retired;the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire expire retires at least one group (#403 item 5a) 15;16;17;18;19 2026-10-14 expire returning 0 whatever it retired;the early exit compares the extent's maximum to the cutoff instead of its minimum +ttl_expire ttl_expire expire retires the aged groups and keeps those of NULLs 15;16;17;18;19 2026-10-14 expire returning 0 whatever it retired +ttl_expire ttl_expire expire retires the all-visible expired group 15;16;17;18;19 2026-10-14 expire returning 0 whatever it retired ttl_expire ttl_expire index-only scan does not return rows expire already retired 15;16;17;18;19 never - ttl_expire ttl_expire premise: an index-only scan is chosen on the all-visible table 15;16;17;18;19 never - -ttl_expire ttl_expire premise: and VACUUM really did set visibility-map bits to clear 15;16;17;18;19 never - ttl_expire ttl_expire premise: and every live row is past the retention 15;16;17;18;19 never - ttl_expire ttl_expire premise: and rows dated exactly ON the cutoff, which decide the rounding 15;16;17;18;19 never - ttl_expire ttl_expire premise: and the SAME row group holds expired rows and NULL ones 15;16;17;18;19 never - -ttl_expire ttl_expire premise: but the zone map still records the deleted NULLs 15;16;17;18;19 never - ttl_expire ttl_expire premise: compact_rewrite retired a group 15;16;17;18;19 never - -ttl_expire ttl_expire premise: expire retires the aged groups and keeps those of NULLs 15;16;17;18;19 never - -ttl_expire ttl_expire premise: in Pacific/Midway the fixture straddles that session's cutoff 15;16;17;18;19 never - ttl_expire ttl_expire premise: in UTC the fixture straddles that session's cutoff 15;16;17;18;19 never - +ttl_expire ttl_expire premise: in a zone on another date the fixture straddles that session's cutoff 15;16;17;18;19 never - ttl_expire ttl_expire premise: no LIVE row holds a NULL retention value any more 15;16;17;18;19 never - -ttl_expire ttl_expire premise: one date per row group in Pacific/Midway, so no group straddles the cutoff 15;16;17;18;19 never - ttl_expire ttl_expire premise: one date per row group in UTC, so no group straddles the cutoff 15;16;17;18;19 never - +ttl_expire ttl_expire premise: one date per row group in a zone on another date, so no group straddles the cutoff 15;16;17;18;19 never - ttl_expire ttl_expire premise: rows exist on both sides of the cutoff within one day of it 15;16;17;18;19 never - ttl_expire ttl_expire premise: some rows are past the retention and some are not 15;16;17;18;19 never - ttl_expire ttl_expire premise: the date fixture has rows on both sides of the cutoff 15;16;17;18;19 never - @@ -2305,14 +2306,16 @@ ttl_expire ttl_expire premise: the fixture is ONE row group, so the NULLs share ttl_expire ttl_expire premise: the fixture is laid out in several row groups 15;16;17;18;19 never - ttl_expire ttl_expire premise: the no-op fixture has many row groups 15;16;17;18;19 never - ttl_expire ttl_expire premise: the record fixture has three aged-out row groups ahead of the current ones 15;16;17;18;19 never - +ttl_expire ttl_expire premise: the second zone's date is not UTC's right now 15;16;17;18;19 never - ttl_expire ttl_expire premise: the table holds NULL retention rows at all 15;16;17;18;19 never - -ttl_expire ttl_expire premise: without the extent, expire retires the aged groups 15;16;17;18;19 never - +ttl_expire ttl_expire premise: the timestamp fixture is six groups of 1,000 rows 15;16;17;18;19 never - ttl_expire ttl_expire running it again retires nothing, because nothing new expired 15;16;17;18;19 never - ttl_expire ttl_expire seqscan agrees the expired rows are gone 15;16;17;18;19 never - ttl_expire ttl_expire the index-only scan is still the plan after expire 15;16;17;18;19 never - ttl_expire ttl_expire the retention is recorded where the other options live 15;16;17;18;19 never - ttl_expire ttl_expire with the extent, expire retires the same groups 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum ttl_expire ttl_expire without the extent the same expire probes every group's zone map 15;16;17;18;19 never - +ttl_expire ttl_expire without the extent, expire retires the aged groups 15;16;17;18;19 2026-10-14 expire returning 0 whatever it retired validity_elision validity_elision a chunk claiming no bitmap while it holds fewer values than rows is refused 15;16;17;18;19 never - validity_elision validity_elision a column with no nulls stores no validity bitmap 15;16;17;18;19 2026-09-18 writer: flush_one_column's presentCount == rowCount forced false, so the bitmap is written even when the chunk holds no null validity_elision validity_elision a null-free column elides its bitmap beside a null-bearing one in the same row group 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 01290e6c6..589b6be08 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -1803,4 +1803,19 @@ suites_not_covered 246 # never 1803 - 2 = 1801 # dated 533 + 4 + 2 = 539 # 1801 + 539 == 2340 -checks_never_observed_red 1801 +# +# #1339 audit, ttl_expire (OffgridwithJD): 1801 -> 1798. Three new rows: the +# timestamp retention arm, dated by expire's timestamp cutoff taken from the UTC +# wall clock; two premises (the second zone's date is not UTC's, the timestamp +# fixture's six groups), never. Eight rows renamed with their history: four checks +# named premise: that test expire, VACUUM or the zone map lose the prefix (#1236), +# and the four Pacific/Midway checks are named "a zone on another date", because the +# zone is now chosen at run time. Dated here: the two renamed expire arms and three +# existing arms first seen red (expire returning 0); the zone arms again by expire's +# date cutoff taken from the UTC wall clock. Five-major runs: 70/70 on each of 15-19. +# +# ledger rows 2340 + 3 = 2343 +# never 1801 + 2 - 2 - 3 = 1798 +# dated 539 + 1 + 2 + 3 = 545 +# 1798 + 545 == 2343 +checks_never_observed_red 1798 diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 757e9b758..d8b9ce5cf 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -6233,7 +6233,8 @@ restored byte-identical. | `test_a_non_positive_retention_is_refused` | a cutoff in the future would retire live groups: `22023`, with a positive control | | `test_a_deleted_null_stops_pinning_its_group` | `null_count` is a write-time record, not a live count | | `test_a_date_retention_column_truncates_toward_keeping` | the rounding, which only this arm notices | -| `test_the_cutoff_is_the_sessions_own_date` | the cutoff is the session's, asserted as an identity rather than as a difference | +| `test_the_cutoff_is_the_sessions_own_date` | the cutoff is the session's, asserted as an identity rather than as a difference, in UTC and in a zone chosen at run time to be on another date (#1339) | +| `test_a_timestamp_retention_column_uses_the_sessions_local_time` | a `timestamp` column's cutoff is the session's local now: in UTC+14, exactly the groups wholly before it are retired (#1339) | | `test_expire_records_what_it_retires_and_nothing_else` | `expire` writes exactly the groups it retires to `pgcolumnar.retired_group` (#1294); a rolled-back `expire` and a `compact_rewrite` record nothing; dropping the table drops its rows | | `test_a_no_op_expire_reads_the_extent_not_every_zone_map` | #1310: with the retention column's extent (#1304 option 5) past the cutoff, a no-op `expire` makes 0 zone-map scans where it made one per group; the same groups retire with or without the extent; aged rows written in `expire`'s own transaction retire; a column left with only NULLs needs no reads. Counted in a fresh connection, with the test schema's `search_path` carried over. Seen red first: 40 scans | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 6af188170..0d3048d8d 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -627,4 +627,6 @@ guard_tests 429 # # cluster_tests 530 -> 532 for #1339's two test_harness_deps.py arms (the needs-the-driver # kind and the --partition mode), by collection over the cluster files on this branch. -cluster_tests 532 +# cluster_tests 532 -> 533 for #1339's test_a_timestamp_retention_column_uses_the_sessions_local_time +# in test_ttl_expire.py, by collection over the cluster files on this branch. +cluster_tests 533 diff --git a/test/pytest/test_ttl_expire.py b/test/pytest/test_ttl_expire.py index f2bd4c6ee..dc23fcd7a 100644 --- a/test/pytest/test_ttl_expire.py +++ b/test/pytest/test_ttl_expire.py @@ -118,13 +118,14 @@ def test_expire_retires_expired_groups_and_keeps_the_straddling_one(pgc_conn, ex # below able to fail. Counted within a day either side, because a day is # wider than the ~2.1 days a 1,000-row group covers here only in the sense # that matters: rows on both sides of the cutoff close to it. - straddle = _one( - pgc_conn, - f"SELECT count(*) FROM ttl_ts WHERE ts >= {cutoff} - interval '1 day' " - f"AND ts < {cutoff} + interval '1 day'") - expect.at_least( - straddle, 1, - "premise: rows exist on both sides of the cutoff within one day of it") + # Each side counted on its own (#1339): one count over the two-day window was + # met by rows on one side alone. + below, above = (_one(pgc_conn, f"SELECT count(*) FROM ttl_ts WHERE ts >= {cutoff} - interval '1 day'" + f" AND ts < {cutoff}"), + _one(pgc_conn, f"SELECT count(*) FROM ttl_ts WHERE ts >= {cutoff}" + f" AND ts < {cutoff} + interval '1 day'")) + expect.text("straddled" if below > 0 and above > 0 else f"NO STRADDLE ({below}|{above})", "straddled", + "premise: rows exist on both sides of the cutoff within one day of it") with pgc_conn.cursor() as cur: cur.execute( @@ -206,13 +207,20 @@ def test_a_null_retention_value_keeps_its_group(pgc_conn, expect): # A TABLE-LEVEL COUNT CANNOT SEE A ROW GROUP, and passes just as readily on a # fixture where the NULLs sit in a group of their own -- which is the # arrangement this premise exists to exclude. Count the groups holding both. - expect.num( + # And THAT group holds an expired row (#1339): counting groups with NULLs alone + # was met by a group of nothing but NULLs. Rows reach their group through + # row_tid() over the group's row numbers. + expect.text( _one(pgc_conn, - "SELECT count(*) FROM (SELECT z.group_number FROM pgcolumnar.zone_map z " - "JOIN pgcolumnar.storage s ON s.storage_id = z.storage_id " - "WHERE s.relation_oid = 'ttl_nul'::regclass AND z.null_count > 0 " - "GROUP BY z.group_number) g"), - 1, + "WITH ng AS (SELECT DISTINCT z.storage_id, z.group_number FROM pgcolumnar.zone_map z" + " JOIN pgcolumnar.storage s ON s.storage_id = z.storage_id" + " WHERE s.relation_oid = 'ttl_nul'::regclass AND z.null_count > 0)" + " SELECT (SELECT count(*) FROM ng) || '|' ||" + " (SELECT count(*) > 0 FROM ttl_nul t WHERE t.ts < now() - interval '4 days'" + " AND t.ctid IN (SELECT pgcolumnar.row_tid(n) FROM pgcolumnar.row_group rg" + " JOIN ng USING (storage_id, group_number)," + " generate_series(rg.first_row_number, rg.first_row_number + rg.row_count - 1) n))"), + "1|true", "premise: and the SAME row group holds expired rows and NULL ones") expect.num( @@ -306,7 +314,7 @@ def test_an_index_only_scan_does_not_return_retired_rows(pgc_conn, expect): "SELECT CASE WHEN relallvisible > 0 THEN 'set' ELSE 'none' END " "FROM pg_class WHERE oid = 'ttl_ios'::regclass"), "set", - "premise: and VACUUM really did set visibility-map bits to clear") + "and VACUUM really did set visibility-map bits to clear") expect.at_least( _one(pgc_conn, "SELECT pgcolumnar.expire('ttl_ios')"), 1, @@ -418,7 +426,7 @@ def test_a_deleted_null_stops_pinning_its_group(pgc_conn, expect): "FROM pgcolumnar.zone_map z JOIN pgcolumnar.storage s " "ON s.storage_id = z.storage_id WHERE s.relation_oid = 'ttl_dn'::regclass"), "still recorded", - "premise: but the zone map still records the deleted NULLs") + "but the zone map still records the deleted NULLs") expect.num( _one(pgc_conn, "SELECT count(*) FROM ttl_dn WHERE ts >= now() - interval '120 days'"), @@ -553,8 +561,16 @@ def test_the_cutoff_is_the_sessions_own_date(pgc_conn, expect): check name, so a port over different zones asserts every property correctly and still cannot be graded against the suite it mirrors. """ - for tz in ("UTC", "Pacific/Midway"): - table = "ttl_tz_" + "".join(c if c.isalnum() else "_" for c in tz) + # The second zone is chosen so its date is not UTC's (#1339): Midway alone had + # UTC's date from 11:00 UTC on, and an expire that ignored the session zone + # passed there for half of every day. Arms name the zone by what it is, so their + # ledger keys do not move with the clock. + other = _one(pgc_conn, "SELECT CASE WHEN extract(hour FROM now() AT TIME ZONE 'UTC') < 11" + " THEN 'Pacific/Midway' ELSE 'Pacific/Kiritimati' END") + expect.text(str(_one(pgc_conn, "SELECT (now() AT TIME ZONE %s)::date <> (now() AT TIME ZONE 'UTC')::date", + (other,))), "True", "premise: the second zone's date is not UTC's right now") + for tz, lbl in (("UTC", "UTC"), (other, "a zone on another date")): + table = "ttl_tz_" + "".join(c if c.isalnum() else "_" for c in lbl) with pgc_conn.cursor() as cur: cur.execute(f"DROP TABLE IF EXISTS {table}") cur.execute(f"CREATE TABLE {table} (id int, d date, v text) USING pgcolumnar") @@ -591,7 +607,7 @@ def test_the_cutoff_is_the_sessions_own_date(pgc_conn, expect): expect.text( "straddles" if 0 < want < before else f"want {want} of {before}", "straddles", - f"premise: in {tz} the fixture straddles that session's cutoff") + f"premise: in {lbl} the fixture straddles that session's cutoff") # AND NO GROUP STRADDLES IT, which is what makes the identity below legal. # `expire` keeps a straddling group WHOLE, so "survivors == rows at or @@ -604,13 +620,37 @@ def test_the_cutoff_is_the_sessions_own_date(pgc_conn, expect): "one per group" if groups == dates and per_date == [GROUP] else f"{groups} groups, {dates} dates, rows per date {per_date}", "one per group", - f"premise: one date per row group in {tz}, so no group straddles the cutoff") + f"premise: one date per row group in {lbl}, so no group straddles the cutoff") expect.num( after, want, - f"expire in {tz} keeps exactly the rows at or after that session's cutoff") + f"expire in {lbl} keeps exactly the rows at or after that session's cutoff") expect.num( below, 0, - f"and no row older than {tz}'s own cutoff survives") + f"and no row older than {lbl}'s own cutoff survives") + + +def test_a_timestamp_retention_column_uses_the_sessions_local_time(pgc_conn, expect): + """A `timestamp` column has no zone, so expire's cutoff is the session's LOCAL now + minus the interval (#1339: no test had a timestamp retention column at all). + + In Kiritimati, UTC+14, the local cutoff and a UTC one are fourteen hours apart. + 6,000 rows 36 s apart are six groups of ten hours, starting 40 hours before the + local cutoff: four groups lie wholly before it and go, the fifth holds it and + stays. A cutoff taken in UTC would sit 26 hours in and retire two. + """ + with pgc_conn.cursor() as cur: + cur.execute("CREATE TABLE ttl_lt (id int, ts timestamp, v text) USING pgcolumnar") + cur.execute(f"SELECT pgcolumnar.set_options('ttl_lt', stripe_row_limit => {GROUP})") + cur.execute("SET TimeZone = 'Pacific/Kiritimati'") + cur.execute("INSERT INTO ttl_lt SELECT g, localtimestamp - interval '2 days' - interval '40 hours'" + " + (g - 1) * interval '36 s', 'v' || g FROM generate_series(1, 6000) g") + cur.execute("SELECT pgcolumnar.set_options('ttl_lt', ttl_column => 'ts', ttl_interval => '2 days')") + expect.num(_groups(pgc_conn, "ttl_lt"), 6, "premise: the timestamp fixture is six groups of 1,000 rows") + with pgc_conn.cursor() as cur: + cur.execute("SELECT pgcolumnar.expire('ttl_lt')") + cur.execute("RESET TimeZone") + expect.text(_one(pgc_conn, "SELECT count(*) || '|' || min(id) FROM ttl_lt"), "2000|4001", + "expire on a timestamp column retires the groups wholly before the session's local cutoff") def test_expire_records_what_it_retires_and_nothing_else(pgc_conn, expect): @@ -715,7 +755,7 @@ def table(name, interval, *inserts): f" FROM generate_series(1, {4 * GROUP}) g", f"INSERT INTO ttl_ny SELECT now() + g * interval '1 s', g FROM generate_series(1, {GROUP}) g") without = run("ttl_ny", f"DELETE FROM pgcolumnar.column_extent WHERE storage_id = {sid}").split(" ")[0] - expect.text(without, "n=4", "premise: without the extent, expire retires the aged groups") + expect.text(without, "n=4", "without the extent, expire retires the aged groups") expect.text(run("ttl_ny").split(" ")[0], without, "with the extent, expire retires the same groups") table("ttl_nz", "3 days", @@ -729,5 +769,5 @@ def table(name, interval, *inserts): f" FROM generate_series(1, {3 * GROUP}) g", f"INSERT INTO ttl_nn SELECT NULL, g FROM generate_series(1, {2 * GROUP}) g") expect.num(_one(pgc_conn, "SELECT pgcolumnar.expire('ttl_nn')"), 3, - "premise: expire retires the aged groups and keeps those of NULLs") + "expire retires the aged groups and keeps those of NULLs") expect.text(run("ttl_nn"), "n=0 scans=0", "a later expire on a column of only NULLs reads no zone map") diff --git a/test/ttl_expire.sh b/test/ttl_expire.sh index b5dac4f7a..3f16081e7 100755 --- a/test/ttl_expire.sh +++ b/test/ttl_expire.sh @@ -83,11 +83,13 @@ check "premise: some rows are past the retention and some are not" \ # The whole safety argument rests on this: a group that spans the cutoff exists, # so "drop every group with an expired row in it" and "drop every group whose # rows are all expired" give different answers on this fixture. -STRADDLE_ROWS="$(q "SELECT count(*) FROM ttl_t - WHERE ts >= ($CUTOFF) - make_interval(days => 1) - AND ts < ($CUTOFF) + make_interval(days => 1)")" +# Each side counted on its own (#1339): one count over the two-day window was met by +# rows on one side alone. +STRADDLE_ROWS="$(q "SELECT count(*) FILTER (WHERE ts >= ($CUTOFF) - make_interval(days => 1) AND ts < ($CUTOFF)) + || '|' || count(*) FILTER (WHERE ts >= ($CUTOFF) AND ts < ($CUTOFF) + make_interval(days => 1)) + FROM ttl_t")" check "premise: rows exist on both sides of the cutoff within one day of it" \ - "$([ "${STRADDLE_ROWS:-0}" -gt 0 ] && echo "straddled" || echo "NO STRADDLE")" "straddled" + "$(case "$STRADDLE_ROWS" in 0\|*|*\|0|'') echo "NO STRADDLE ($STRADDLE_ROWS)" ;; *) echo straddled ;; esac)" "straddled" # ---- declaring the retention ---------------------------------------------- psql_run "SELECT pgcolumnar.set_options('ttl_t', ttl_column => 'ts', @@ -152,14 +154,17 @@ check "premise: the table holds NULL retention rows at all" "$NULL_BEFORE" "10" # table-level count cannot see a row group, and passes just as readily on a # fixture where the NULL rows sit in a group of their own -- which is the # arrangement the premise exists to exclude. Count the groups that hold both. +# And THAT group holds an expired row (#1339): counting groups with NULLs alone was +# met by a group of nothing but NULLs. Rows are mapped to their group through +# row_tid() over the group's row numbers. check "premise: and the SAME row group holds expired rows and NULL ones" \ - "$(q "SELECT count(*) FROM ( - SELECT z.group_number - FROM pgcolumnar.zone_map z - JOIN pgcolumnar.storage s ON s.storage_id = z.storage_id - WHERE s.relation_oid = 'ttl_null'::regclass - AND z.null_count > 0 - GROUP BY z.group_number) g")" "1" + "$(q "WITH ng AS (SELECT DISTINCT z.storage_id, z.group_number + FROM pgcolumnar.zone_map z JOIN pgcolumnar.storage s ON s.storage_id = z.storage_id + WHERE s.relation_oid = 'ttl_null'::regclass AND z.null_count > 0) + SELECT (SELECT count(*) FROM ng) || '|' || + (SELECT count(*) > 0 FROM ttl_null t WHERE t.ts < now() - interval '3 days' + AND t.ctid IN (SELECT pgcolumnar.row_tid(n) FROM pgcolumnar.row_group rg JOIN ng USING (storage_id, group_number), + generate_series(rg.first_row_number, rg.first_row_number + rg.row_count - 1) n))")" "1|true" NULL_RETIRED="$(q "SELECT pgcolumnar.expire('ttl_null')")" NULL_AFTER="$(q "SELECT count(*) FROM ttl_null WHERE ts IS NULL")" NULL_ROWS="$(q "SELECT count(*) FROM ttl_null")" @@ -191,7 +196,7 @@ check "premise: an index-only scan is chosen on the all-visible table" \ # enable_bitmapscan being off -- NOT by the visibility-map bit the fix clears. # Stop PgColumnarVMSetVisibleForRelation writing bits and the plan is unchanged, # so the arm above cannot fail for the thing under test. Assert the bit itself. -check "premise: and VACUUM really did set visibility-map bits to clear" \ +check "and VACUUM really did set visibility-map bits to clear" \ "$(q "SELECT CASE WHEN relallvisible > 0 THEN 'set' ELSE 'none' END FROM pg_class WHERE oid = 'ttl_ios'::regclass")" "set" IOS_VM_BEFORE="$(q "SELECT relallvisible FROM pg_class WHERE oid = 'ttl_ios'::regclass")" @@ -295,7 +300,7 @@ psql_run "DELETE FROM ttl_dn WHERE ts IS NULL;" check "premise: no LIVE row holds a NULL retention value any more" \ "$(q 'SELECT count(*) FROM ttl_dn WHERE ts IS NULL')" "0" -check "premise: but the zone map still records the deleted NULLs" \ +check "but the zone map still records the deleted NULLs" \ "$(q "SELECT CASE WHEN sum(z.null_count) > 0 THEN 'still recorded' ELSE 'gone' END FROM pgcolumnar.zone_map z JOIN pgcolumnar.storage s ON s.storage_id = z.storage_id @@ -432,8 +437,18 @@ qz() { # qz ZONE SQL -c "SET TimeZone = '$1'; $2" 2>&1 | tail -1 } -for _tz in UTC Pacific/Midway; do - _t="ttl_tz_$(printf '%s' "$_tz" | tr -c 'A-Za-z0-9' '_')" +# THE SECOND ZONE IS CHOSEN SO ITS DATE IS NOT UTC'S (#1339). Midway alone had +# UTC's date from 11:00 UTC on, and an expire that ignored the session zone passed +# there for half of every day. Midway is a day behind before 11:00 UTC, Kiritimati +# a day ahead from 10:00 UTC, so one of them always differs. The arms name the zone +# by what it is, not which it is, so their ledger keys do not move with the clock. +_other="$(q "SELECT CASE WHEN extract(hour FROM now() AT TIME ZONE 'UTC') < 11 + THEN 'Pacific/Midway' ELSE 'Pacific/Kiritimati' END;")" +check "premise: the second zone's date is not UTC's right now" \ + "$(q "SELECT ((now() AT TIME ZONE '$_other')::date <> (now() AT TIME ZONE 'UTC')::date)::text;")" "true" +for _tz in UTC "$_other"; do + _lbl="UTC"; [ "$_tz" = UTC ] || _lbl="a zone on another date" + _t="ttl_tz_$(printf '%s' "$_lbl" | tr -c 'A-Za-z0-9' '_')" psql_run "DROP TABLE IF EXISTS $_t; CREATE TABLE $_t (id int, d date, v text) USING pgcolumnar;" psql_run "INSERT INTO $_t @@ -451,7 +466,7 @@ for _tz in UTC Pacific/Midway; do _below="$(qz "$_tz" "SELECT count(*) FROM $_t WHERE d < DATE '$_cut'")" echo "-- tz $_tz: cutoff $_cut, $_before rows -> $_after, retired $_ret group(s)" - check "premise: in $_tz the fixture straddles that session's cutoff" \ + check "premise: in $_lbl the fixture straddles that session's cutoff" \ "$([ "${_want:-0}" -gt 0 ] && [ "${_want:-0}" -lt "${_before:-0}" ] && echo straddles \ || echo "want $_want of $_before")" \ "straddles" @@ -478,15 +493,33 @@ for _tz in UTC Pacific/Midway; do # gate refuses that for exactly the reason the arm exists: a red in one zone # would be recorded against the other. The gate caught it on the first push, # on both majors, while all 256 suites reported PASS. - check "premise: one date per row group in $_tz, so no group straddles the cutoff" \ + check "premise: one date per row group in $_lbl, so no group straddles the cutoff" \ "$([ "${_gr:-0}" = "${_days:-x}" ] && [ "${_perday:-0}" = "1000" ] && echo "one per group" \ || echo "$_gr groups, $_days dates, rows per date [$_perday]")" \ "one per group" - check "expire in $_tz keeps exactly the rows at or after that session's cutoff" \ + check "expire in $_lbl keeps exactly the rows at or after that session's cutoff" \ "$_after" "$_want" - check "and no row older than $_tz's own cutoff survives" "$_below" "0" + check "and no row older than $_lbl's own cutoff survives" "$_below" "0" done +# ---- a timestamp retention column: the cutoff is the session's LOCAL time (#1339) -- +# A `timestamp` column has no zone, so expire computes its cutoff from the session's +# local now. Nothing tested that type at all. In Kiritimati, UTC+14, the local cutoff +# and a UTC one are fourteen hours apart. 6,000 rows 36 s apart are six groups of ten +# hours, starting 40 hours before the local cutoff: four groups lie wholly before it +# and go, the fifth holds it and stays. A cutoff taken in UTC would sit 26 hours in +# and retire two. +psql_run "CREATE TABLE ttl_lt (id int, ts timestamp, v text) USING pgcolumnar;" +qz Pacific/Kiritimati "INSERT INTO ttl_lt SELECT g, localtimestamp - interval '2 days' - interval '40 hours' + + (g - 1) * interval '36 s', 'v' || g FROM generate_series(1, 6000) g;" >/dev/null +psql_run "SELECT pgcolumnar.set_options('ttl_lt', ttl_column => 'ts', ttl_interval => '2 days');" +check "premise: the timestamp fixture is six groups of 1,000 rows" \ + "$(q "SELECT count(*) FROM pgcolumnar.row_group rg JOIN pgcolumnar.storage s USING (storage_id) + WHERE s.relation_oid = 'ttl_lt'::regclass;")" "6" +qz Pacific/Kiritimati "SELECT pgcolumnar.expire('ttl_lt')" >/dev/null +check "expire on a timestamp column retires the groups wholly before the session's local cutoff" \ + "$(q "SELECT count(*) || '|' || min(id) FROM ttl_lt;")" "2000|4001" + # ---- expire records what it retires, for rollups that keep history (#1294) ---- # An incremental rollup keeps its buckets when retention drops their raw rows (the # owner's decision on #1294), so its refresh must tell a group RETIRED BY RETENTION @@ -559,7 +592,7 @@ psql_run "CREATE TABLE ttl_y (ts timestamptz, v int) USING pgcolumnar; INSERT INTO ttl_y SELECT now() + g * interval '1 second', g FROM generate_series(1, 2000) g;" _ysid="$(q "SELECT pgcolumnar.get_storage_id('ttl_y');")" _y_without="$(_expire_scans ttl_y "DELETE FROM pgcolumnar.column_extent WHERE storage_id = $_ysid" | sed 's/ scans=.*//')" -check "premise: without the extent, expire retires the aged groups" "$_y_without" "n=3" +check "without the extent, expire retires the aged groups" "$_y_without" "n=3" check "with the extent, expire retires the same groups" "$(_expire_scans ttl_y | sed 's/ scans=.*//')" "$_y_without" # Old rows written earlier in expire's own transaction: their groups are visible to @@ -577,7 +610,7 @@ psql_run "CREATE TABLE ttl_n (ts timestamptz, v int) USING pgcolumnar; SELECT pgcolumnar.set_options('ttl_n', ttl_column => 'ts', ttl_interval => '2 days'); INSERT INTO ttl_n SELECT now() - interval '9 days' + g * interval '1 second', g FROM generate_series(1, 2000) g; INSERT INTO ttl_n SELECT NULL, g FROM generate_series(1, 4000) g;" -check_num "premise: expire retires the aged groups and keeps those of NULLs" \ +check_num "expire retires the aged groups and keeps those of NULLs" \ "$(q "SELECT pgcolumnar.expire('ttl_n');")" "2" check "a later expire on a column of only NULLs reads no zone map" "$(_expire_scans ttl_n)" "n=0 scans=0"