diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 3fb2ec680..fe55cb196 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -3114,6 +3114,24 @@ native_saop_pushdown native_saop_pushdown the predicate builder's reject express native_saop_pushdown native_saop_pushdown the predicate builder's reject expression contains SK_SEARCHNULL 15;16;17;18;19 2026-10-16 columnar_reader.c: the reject expression keeps only SK_ISNULL native_saop_pushdown native_saop_pushdown vector-agg count over an IN-list is exact (conservative keys not folded) 15;16;17;18;19 2026-10-16 columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out) native_saop_pushdown native_saop_pushdown vector-agg path engaged for the guard arm (plan shape) 15;16;17;18;19 never - +native_truncate native_truncate GUC off: truncate does nothing 15;16;17;18;19 2026-10-16 pgcolumnar.truncate() ignores pgcolumnar.enable_end_truncation (columnar_vacuum.c) +native_truncate native_truncate GUC on: file shrank 15;16;17;18;19 2026-10-16 pgcolumnar.truncate() returns 0 without truncating (columnar_vacuum.c) +native_truncate native_truncate GUC on: truncate reclaims 15;16;17;18;19 2026-10-16 pgcolumnar.truncate() ignores pgcolumnar.enable_end_truncation (columnar_vacuum.c);pgcolumnar.truncate() returns 0 without truncating (columnar_vacuum.c) +native_truncate native_truncate file shrank 15;16;17;18;19 2026-10-16 pgcolumnar.truncate() returns 0 without truncating (columnar_vacuum.c) +native_truncate native_truncate file unchanged after refused in-txn truncate 15;16;17;18;19 never - +native_truncate native_truncate indexed lookup after truncate 15;16;17;18;19 2026-10-16 pgcolumnar_index_fetch_tuple finds no row (columnar_tableam.c) +native_truncate native_truncate load parity 15;16;17;18;19 never - +native_truncate native_truncate parity after guard cycle 15;16;17;18;19 never - +native_truncate native_truncate parity after post-truncate insert 15;16;17;18;19 never - +native_truncate native_truncate parity after refused in-txn truncate 15;16;17;18;19 never - +native_truncate native_truncate parity after truncate 15;16;17;18;19 never - +native_truncate native_truncate parity after truncate + reinsert + compaction 15;16;17;18;19 never - +native_truncate native_truncate premise: the indexed lookup reads n_id 15;16;17;18;19 never - +native_truncate native_truncate row count after truncate 15;16;17;18;19 never - +native_truncate native_truncate second truncate is a no-op 15;16;17;18;19 never - +native_truncate native_truncate truncate before any delete is a no-op 15;16;17;18;19 never - +native_truncate native_truncate truncate refused inside a transaction block 15;16;17;18;19 2026-10-16 pgcolumnar.truncate() runs inside a transaction block, its IsTransactionBlock refusal removed (columnar_vacuum.c) +native_truncate native_truncate truncate returned blocks 15;16;17;18;19 2026-10-16 pgcolumnar.truncate() returns 0 without truncating (columnar_vacuum.c) native_vecskip native_vecskip and the aggregate skips the same seven vectors 15;16;17;18;19 2026-10-16 columnar_reader.c: no vector skip at all (zone-map rule off, refine_skipvec and qual_skipvec return at once) native_vecskip native_vecskip cross-vector range parity 15;16;17;18;19 never - native_vecskip native_vecskip empty range parity 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 9c79c6274..77c228e30 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 190 +suites_not_covered 189 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3545,4 +3545,19 @@ suites_not_covered 190 # never 3055 + 0 = 3055 # dated 1341 + 1 = 1342 # 3055 + 1342 == 4397 -checks_never_observed_red 3055 +# +# #1341, native_truncate seeded, with its twin (OffgridwithJD): 3055 -> 3066, and +# suites_not_covered 190 -> 189. The suite had no rows. It enters with all 18 checks +# from five-major runs (18/18 on each of 15-19), restated over #1516. 7 are +# dated by four mutations: the transaction-block refusal removed, the GUC not +# consulted (the GUC-off arm, which had sent SET and the call as one implicit +# transaction block and passed on the refusal, and the GUC-on reclaim after it), +# truncate returning 0 without truncating, and an index fetch finding no row. +# Eleven arrive never: the six heap-parity comparisons, the row count, the two +# no-op truncates, the unchanged file after the refused call, and the index premise. +# +# ledger rows 4397 + 18 = 4415 +# never 3055 + 11 = 3066 +# dated 1342 + 7 = 1349 +# 3066 + 1349 == 4415 +checks_never_observed_red 3066 diff --git a/test/native_truncate.sh b/test/native_truncate.sh index 088947d88..02c19ae14 100755 --- a/test/native_truncate.sh +++ b/test/native_truncate.sh @@ -60,16 +60,29 @@ psql_run "INSERT INTO h SELECT g, md5(g::text) FROM generate_series(30001, 40000 psql_run "INSERT INTO n SELECT g, md5(g::text) FROM generate_series(30001, 40000) g;" check "parity after post-truncate insert" "$(hash_n)" "$(hash_h)" psql_run "CREATE INDEX n_id ON n (id);" +# Forced to fetch every row through n_id, and asserted. With the defaults this +# count plans an Index Only Scan (PG17 and PG18), which fetches each row only while +# the visibility map leaves its page unmarked: once a VACUUM marks them it answers +# from the index alone, whatever the fetch does. count(v), not count(*), with the +# custom scan, seqscan and bitmap scans off, so the plan is an Index Scan. +IDX_SETS="SET enable_seqscan = off; SET enable_bitmapscan = off; SET pgcolumnar.enable_custom_scan = off;" +check "premise: the indexed lookup reads n_id" \ + "$(pgc_uses_row_fetch "$IDX_SETS" "SELECT count(v) FROM n WHERE id BETWEEN 1 AND 40000")" yes check "indexed lookup after truncate" \ - "$(q 'SELECT count(*) FROM n WHERE id BETWEEN 1 AND 40000;')" "25000" + "$(q "$IDX_SETS SELECT count(v) FROM n WHERE id BETWEEN 1 AND 40000;" | tail -1)" "25000" # GUC off: truncate is a no-op even with reclaimable trailing space. psql_run "DELETE FROM h WHERE id > 35000;" psql_run "DELETE FROM n WHERE id > 35000;" psql_run "SELECT pgcolumnar.compact('n');" size_guard="$(size)" -psql_run "SET pgcolumnar.enable_end_truncation = off; SELECT pgcolumnar.truncate('n');" >/dev/null -check "GUC off: truncate does nothing" "$(size)" "$size_guard" +# Two -c statements in one session, and the call's own answer checked. As one string, +# "SET ...; SELECT pgcolumnar.truncate('n')" is an implicit transaction block, which +# truncate refuses: the arm passed on an unchanged file whatever the GUC did, and +# stayed green with the GUC not consulted at all. +guc_off="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -Atq \ + -c "SET pgcolumnar.enable_end_truncation = off" -c "SELECT pgcolumnar.truncate('n')" 2>&1)" +check "GUC off: truncate does nothing" "$(size)|$guc_off" "$size_guard|0" check "GUC on: truncate reclaims" \ "$([ "$(q "SELECT pgcolumnar.truncate('n');")" -gt 0 ] && echo yes || echo no)" "yes" check "GUC on: file shrank" "$([ "$(size)" -lt "$size_guard" ] && echo yes || echo no)" "yes" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 507e1a853..cc0c55443 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -221,6 +221,7 @@ behaviour, the source of that number is named. - [173. test_native_fetch_bigcap.py: fetching from a row group above the cache cap](#173-test_native_fetch_bigcappy-fetching-from-a-row-group-above-the-cache-cap) - [174. test_sorted_projection.py: vacuum_sorted reorders storage and changes no answer](#174-test_sorted_projectionpy-vacuum_sorted-reorders-storage-and-changes-no-answer) - [175. test_native_ios.py: an index-only scan skips the fetch on all-visible row groups](#175-test_native_iospy-an-index-only-scan-skips-the-fetch-on-all-visible-row-groups) +- [176. test_native_truncate.py: end-truncation shrinks the file and loses no row](#176-test_native_truncatepy-end-truncation-shrinks-the-file-and-loses-no-row) ## 1. How to read a test in here @@ -7823,3 +7824,15 @@ The pytest twin of `test/native_ios.sh` (ported for #1341), every check by the s | --- | --- | | `test_an_all_visible_range_needs_no_fetch_and_vacuum_records_it` | the index-only scan is chosen, fetches nothing, matches the heap, and VACUUM records a page count no larger than the relation and most of it, as the heap's is | | `test_a_delete_clears_the_bit_and_is_never_returned` | after a delete the scan matches the heap and fetches where the bit was cleared | + +## 176. test_native_truncate.py: end-truncation shrinks the file and loses no row + +The pytest twin of `test/native_truncate.sh` (ported for #1341), every check by the shell's name. `pgcolumnar.truncate()` returns trailing reclaimed blocks to the OS. After the trailing half is deleted and compacted, the file shrinks and the rows still match a heap mirror. A second truncate has nothing to do, and the table stays usable: reinsert, index, and an indexed lookup forced through the index and asserted to read it. With `pgcolumnar.enable_end_truncation` off, truncate must return 0 and leave the file alone. The call is a statement of its own on both sides, because sent with the `SET` as one string it ran in an implicit transaction block and was refused, which passed whatever the GUC did. Inside a transaction block truncate is refused and changes nothing, because its shrink is not transactional, and a truncate followed by reinsert and compaction keeps parity. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_truncate_shrinks_the_file_and_keeps_every_row` | a truncate with nothing freed returns 0; after delete and compact it returns blocks, the file shrinks, the rows match the heap, and a second truncate returns 0 | +| `test_the_table_stays_usable_and_the_guc_turns_it_off` | reinsert and an index lookup through the index after truncation; with the GUC off truncate returns 0 and the file keeps its size, with it on the file shrinks | +| `test_truncate_refuses_a_transaction_block_and_reuse_stays_clean` | refused inside BEGIN with the file and rows unchanged; truncate, reinsert and compaction keep parity | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index bd1d7ae80..e3dbd8992 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -821,4 +821,6 @@ guard_tests 440 # when test_join_runtime_filter.py became test_native_join_runtime_filter.py: one test, by collection. # cluster_tests 750 -> 751 for #1341's completion of test_native_agg.py, the twin of native_agg.sh: # one test, by collection over the cluster files on this tree. -cluster_tests 751 +# cluster_tests 751 -> 754 for #1341's test_native_truncate.py, the twin of native_truncate.sh: +# three tests, by collection over the cluster files on this tree. +cluster_tests 754 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 58d1a46c3..9ac042055 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -156,6 +156,7 @@ "native_recluster", "native_repack", "native_saop_pushdown", + "native_truncate", "native_vacuum_race", "native_vecskip", "native_writer", diff --git a/test/pytest/test_native_truncate.py b/test/pytest/test_native_truncate.py new file mode 100644 index 000000000..9d53dc9e4 --- /dev/null +++ b/test/pytest/test_native_truncate.py @@ -0,0 +1,148 @@ +"""Physical end-truncation (Phase F). The pytest twin of `test/native_truncate.sh`, +every check by the shell's name. + +pgcolumnar.truncate() returns trailing reclaimed blocks to the OS. After a delete and a +compaction the file shrinks, the rows still match a heap mirror, a second truncate has +nothing to do, and the table stays usable: reinsert, index, indexed lookup. The GUC +turns it off. It refuses to run inside a transaction block, since its shrink is not +transactional, and a truncate followed by reinsert and compaction stays parity-clean +(the assert build's no-overlap validator runs inside compact). Concurrency is the +isolation specs' (specs/truncate_vs_*.spec). +""" +import psycopg +import pytest + +HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) " + "FROM (SELECT _row::text AS t FROM ({}) _row) _s") +IDX_SETS = ("SET enable_seqscan = off", "SET enable_bitmapscan = off", "SET pgcolumnar.enable_custom_scan = off") +LOOKUP = "SELECT count(v) FROM n WHERE id BETWEEN 1 AND 40000" + + +def _one(c, sql): + try: + return str(c.execute(sql).fetchone()[0]) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _size(c): + return int(c.execute("SELECT pg_relation_size('n')").fetchone()[0]) + + +def _hashes(c): + """-> (columnar, heap) set digests of the rows.""" + return _one(c, HASH.format("SELECT id, v FROM n")), _one(c, HASH.format("SELECT id, v FROM h")) + + +def _both(c, *sqls): + """Run each statement on the heap mirror and on the columnar table.""" + for sql in sqls: + c.execute(sql.format(t="h")) + c.execute(sql.format(t="n")) + + +def _truncated(c): + return int(c.execute("SELECT pgcolumnar.truncate('n')").fetchone()[0]) + + +@pytest.fixture(scope="module") +def nt(pgc_cluster): + db = "pgc_ntrunc" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + # end-truncation is opt-in (GUC default off); on for the whole database + a.execute(f"ALTER DATABASE {db} SET pgcolumnar.enable_end_truncation = on") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + c.execute("CREATE TABLE h (id int, v text)") + c.execute("CREATE TABLE n (id int, v text) USING pgcolumnar") + c.execute("SELECT pgcolumnar.set_options('n', stripe_row_limit => 1000, chunk_group_row_limit => 1000)") + _both(c, "INSERT INTO {t} SELECT g, md5(g::text) FROM generate_series(1, 30000) g") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_truncate_shrinks_the_file_and_keeps_every_row(nt, expect): + c = nt + expect.text(*_hashes(c), "load parity") + # nothing has been freed yet + expect.text(_one(c, "SELECT pgcolumnar.truncate('n')"), "0", "truncate before any delete is a no-op") + + # the trailing half deleted whole (groups 16..30), and compacted to free it + _both(c, "DELETE FROM {t} WHERE id > 15000") + c.execute("SELECT pgcolumnar.compact('n')") + before = _size(c) + trunc = _truncated(c) + print(f"-- truncate returned {trunc} blocks; before-trunc={before} after={_size(c)}") + expect.at_least(trunc, 1, "truncate returned blocks") + expect.at_least(before - _size(c), 1, "file shrank") + expect.text(*_hashes(c), "parity after truncate") + expect.text(_one(c, "SELECT count(*) FROM n"), "15000", "row count after truncate") + expect.text(_one(c, "SELECT pgcolumnar.truncate('n')"), "0", "second truncate is a no-op") + + +def test_the_table_stays_usable_and_the_guc_turns_it_off(nt, expect): + c = nt + _both(c, "INSERT INTO {t} SELECT g, md5(g::text) FROM generate_series(30001, 40000) g") + expect.text(*_hashes(c), "parity after post-truncate insert") + c.execute("CREATE INDEX n_id ON n (id)") + # forced to fetch every row through n_id, and asserted: by default this count is an + # Index Only Scan, which stops fetching once a VACUUM marks the pages all-visible + for s in IDX_SETS: + c.execute(s) + try: + plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {LOOKUP}")) + expect.text("yes" if "Index Scan using n_id" in plan else "no", "yes", "premise: the indexed lookup reads n_id") + expect.text(_one(c, LOOKUP), "25000", "indexed lookup after truncate") + finally: + for g in ("enable_seqscan", "enable_bitmapscan", "pgcolumnar.enable_custom_scan"): + c.execute(f"RESET {g}") + + _both(c, "DELETE FROM {t} WHERE id > 35000") + c.execute("SELECT pgcolumnar.compact('n')") + guard = _size(c) + # its own statement, and its answer checked: the shell once sent SET and the call + # as one string, an implicit transaction block that truncate refuses + c.execute("SET pgcolumnar.enable_end_truncation = off") + try: + off = _one(c, "SELECT pgcolumnar.truncate('n')") + finally: + c.execute("RESET pgcolumnar.enable_end_truncation") + expect.text(f"{_size(c)}|{off}", f"{guard}|0", "GUC off: truncate does nothing") + expect.at_least(_truncated(c), 1, "GUC on: truncate reclaims") + expect.at_least(guard - _size(c), 1, "GUC on: file shrank") + expect.text(*_hashes(c), "parity after guard cycle") + + +def test_truncate_refuses_a_transaction_block_and_reuse_stays_clean(nt, expect): + c = nt + # a ROLLBACK must never leave the metapage highwater lowered while the + # transactional free_space deletes roll back + _both(c, "DELETE FROM {t} WHERE id BETWEEN 30001 AND 33000") + c.execute("SELECT pgcolumnar.compact('n')") + before = _size(c) + refused = "no" + c.execute("BEGIN") + try: + c.execute("SELECT pgcolumnar.truncate('n')") + except psycopg.Error as exc: + if "cannot run inside a transaction block" in (exc.diag.message_primary or ""): + refused = "yes" + finally: + c.execute("ROLLBACK") + expect.text(refused, "yes", "truncate refused inside a transaction block") + expect.num(_size(c), before, "file unchanged after refused in-txn truncate") + expect.text(*_hashes(c), "parity after refused in-txn truncate") + + # an autocommit truncate, reinsert into the reclaimed space, then compaction: no + # double allocation over reused blocks + c.execute("SELECT pgcolumnar.truncate('n')") + _both(c, "INSERT INTO {t} SELECT g, md5(g::text) FROM generate_series(41001, 43000) g", + "DELETE FROM {t} WHERE id > 42000") + c.execute("SELECT pgcolumnar.compact_rewrite('n', 0.0)") + c.execute("SELECT pgcolumnar.compact('n')") + expect.text(*_hashes(c), "parity after truncate + reinsert + compaction") diff --git a/test/pytest/unported_suites.txt b/test/pytest/unported_suites.txt index 74c4dbd4f..ae0bd9e62 100644 --- a/test/pytest/unported_suites.txt +++ b/test/pytest/unported_suites.txt @@ -111,7 +111,6 @@ native_rewrite_conc native_roundtrip native_sort_by native_toasted_write -native_truncate native_varlena_bound native_vecdecode native_zonemap