test: native_truncate gains its pytest twin, and its GUC-off arm stops passing on a refusal (#1341)

test/check_ledger.tsv

@@ -3114,6 +3114,24 @@ native_saop_pushdown	native_saop_pushdown	the predicate builder's reject express
 native_saop_pushdown	native_saop_pushdown	the predicate builder's reject expression contains SK_SEARCHNULL	15;16;17;18;19	2026-10-16	columnar_reader.c: the reject expression keeps only SK_ISNULL
 native_saop_pushdown	native_saop_pushdown	vector-agg count over an IN-list is exact (conservative keys not folded)	15;16;17;18;19	2026-10-16	columnar_reader.c: a set key never probes the bloom filter (it probes zero values and rules the group out)
 native_saop_pushdown	native_saop_pushdown	vector-agg path engaged for the guard arm (plan shape)	15;16;17;18;19	never	-
+native_truncate	native_truncate	GUC off: truncate does nothing	15;16;17;18;19	2026-10-16	pgcolumnar.truncate() ignores pgcolumnar.enable_end_truncation (columnar_vacuum.c)
+native_truncate	native_truncate	GUC on: file shrank	15;16;17;18;19	2026-10-16	pgcolumnar.truncate() returns 0 without truncating (columnar_vacuum.c)
+native_truncate	native_truncate	GUC on: truncate reclaims	15;16;17;18;19	2026-10-16	pgcolumnar.truncate() ignores pgcolumnar.enable_end_truncation (columnar_vacuum.c);pgcolumnar.truncate() returns 0 without truncating (columnar_vacuum.c)
+native_truncate	native_truncate	file shrank	15;16;17;18;19	2026-10-16	pgcolumnar.truncate() returns 0 without truncating (columnar_vacuum.c)
+native_truncate	native_truncate	file unchanged after refused in-txn truncate	15;16;17;18;19	never	-
+native_truncate	native_truncate	indexed lookup after truncate	15;16;17;18;19	2026-10-16	pgcolumnar_index_fetch_tuple finds no row (columnar_tableam.c)
+native_truncate	native_truncate	load parity	15;16;17;18;19	never	-
+native_truncate	native_truncate	parity after guard cycle	15;16;17;18;19	never	-
+native_truncate	native_truncate	parity after post-truncate insert	15;16;17;18;19	never	-
+native_truncate	native_truncate	parity after refused in-txn truncate	15;16;17;18;19	never	-
+native_truncate	native_truncate	parity after truncate	15;16;17;18;19	never	-
+native_truncate	native_truncate	parity after truncate + reinsert + compaction	15;16;17;18;19	never	-
+native_truncate	native_truncate	premise: the indexed lookup reads n_id	15;16;17;18;19	never	-
+native_truncate	native_truncate	row count after truncate	15;16;17;18;19	never	-
+native_truncate	native_truncate	second truncate is a no-op	15;16;17;18;19	never	-
+native_truncate	native_truncate	truncate before any delete is a no-op	15;16;17;18;19	never	-
+native_truncate	native_truncate	truncate refused inside a transaction block	15;16;17;18;19	2026-10-16	pgcolumnar.truncate() runs inside a transaction block, its IsTransactionBlock refusal removed (columnar_vacuum.c)
+native_truncate	native_truncate	truncate returned blocks	15;16;17;18;19	2026-10-16	pgcolumnar.truncate() returns 0 without truncating (columnar_vacuum.c)
 native_vecskip	native_vecskip	and the aggregate skips the same seven vectors	15;16;17;18;19	2026-10-16	columnar_reader.c: no vector skip at all (zone-map rule off, refine_skipvec and qual_skipvec return at once)
 native_vecskip	native_vecskip	cross-vector range parity	15;16;17;18;19	never	-
 native_vecskip	native_vecskip	empty range parity	15;16;17;18;19	never	-

test/check_ledger_budget.txt

@@ -15,7 +15,7 @@
 #
 #   Adding a check to a suite that is already covered does not move it, which is
 #   what makes it safe to bound.
-suites_not_covered 190
+suites_not_covered 189
 #
 # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become
 #   one.
@@ -3545,4 +3545,19 @@ suites_not_covered 190
 #     never         3055 +  0 = 3055
 #     dated         1341 +  1 = 1342
 #     3055 + 1342 == 4397
-checks_never_observed_red 3055
+#
+# #1341, native_truncate seeded, with its twin (OffgridwithJD): 3055 -> 3066, and
+# suites_not_covered 190 -> 189. The suite had no rows. It enters with all 18 checks
+# from five-major runs (18/18 on each of 15-19), restated over #1516. 7 are
+# dated by four mutations: the transaction-block refusal removed, the GUC not
+# consulted (the GUC-off arm, which had sent SET and the call as one implicit
+# transaction block and passed on the refusal, and the GUC-on reclaim after it),
+# truncate returning 0 without truncating, and an index fetch finding no row.
+# Eleven arrive never: the six heap-parity comparisons, the row count, the two
+# no-op truncates, the unchanged file after the refused call, and the index premise.
+#
+#     ledger rows   4397 + 18 = 4415
+#     never         3055 + 11 = 3066
+#     dated         1342 +  7 = 1349
+#     3066 + 1349 == 4415
+checks_never_observed_red 3066

test/native_truncate.sh

@@ -60,16 +60,29 @@ psql_run "INSERT INTO h SELECT g, md5(g::text) FROM generate_series(30001, 40000
 psql_run "INSERT INTO n SELECT g, md5(g::text) FROM generate_series(30001, 40000) g;"
 check "parity after post-truncate insert" "$(hash_n)" "$(hash_h)"
 psql_run "CREATE INDEX n_id ON n (id);"
+# Forced to fetch every row through n_id, and asserted. With the defaults this
+# count plans an Index Only Scan (PG17 and PG18), which fetches each row only while
+# the visibility map leaves its page unmarked: once a VACUUM marks them it answers
+# from the index alone, whatever the fetch does. count(v), not count(*), with the
+# custom scan, seqscan and bitmap scans off, so the plan is an Index Scan.
+IDX_SETS="SET enable_seqscan = off; SET enable_bitmapscan = off; SET pgcolumnar.enable_custom_scan = off;"
+check "premise: the indexed lookup reads n_id" \
+	"$(pgc_uses_row_fetch "$IDX_SETS" "SELECT count(v) FROM n WHERE id BETWEEN 1 AND 40000")" yes
 check "indexed lookup after truncate" \
-	"$(q 'SELECT count(*) FROM n WHERE id BETWEEN 1 AND 40000;')" "25000"
+	"$(q "$IDX_SETS SELECT count(v) FROM n WHERE id BETWEEN 1 AND 40000;" | tail -1)" "25000"
 
 # GUC off: truncate is a no-op even with reclaimable trailing space.
 psql_run "DELETE FROM h WHERE id > 35000;"
 psql_run "DELETE FROM n WHERE id > 35000;"
 psql_run "SELECT pgcolumnar.compact('n');"
 size_guard="$(size)"
-psql_run "SET pgcolumnar.enable_end_truncation = off; SELECT pgcolumnar.truncate('n');" >/dev/null
-check "GUC off: truncate does nothing" "$(size)" "$size_guard"
+# Two -c statements in one session, and the call's own answer checked. As one string,
+# "SET ...; SELECT pgcolumnar.truncate('n')" is an implicit transaction block, which
+# truncate refuses: the arm passed on an unchanged file whatever the GUC did, and
+# stayed green with the GUC not consulted at all.
+guc_off="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -Atq \
+	-c "SET pgcolumnar.enable_end_truncation = off" -c "SELECT pgcolumnar.truncate('n')" 2>&1)"
+check "GUC off: truncate does nothing" "$(size)|$guc_off" "$size_guard|0"
 check "GUC on: truncate reclaims" \
 	"$([ "$(q "SELECT pgcolumnar.truncate('n');")" -gt 0 ] && echo yes || echo no)" "yes"
 check "GUC on: file shrank" "$([ "$(size)" -lt "$size_guard" ] && echo yes || echo no)" "yes"

test/pytest/TESTS.md

@@ -221,6 +221,7 @@ behaviour, the source of that number is named.
 - [173. test_native_fetch_bigcap.py: fetching from a row group above the cache cap](#173-test_native_fetch_bigcappy-fetching-from-a-row-group-above-the-cache-cap)
 - [174. test_sorted_projection.py: vacuum_sorted reorders storage and changes no answer](#174-test_sorted_projectionpy-vacuum_sorted-reorders-storage-and-changes-no-answer)
 - [175. test_native_ios.py: an index-only scan skips the fetch on all-visible row groups](#175-test_native_iospy-an-index-only-scan-skips-the-fetch-on-all-visible-row-groups)
+- [176. test_native_truncate.py: end-truncation shrinks the file and loses no row](#176-test_native_truncatepy-end-truncation-shrinks-the-file-and-loses-no-row)
 
 ## 1. How to read a test in here
 
@@ -7823,3 +7824,15 @@ The pytest twin of `test/native_ios.sh` (ported for #1341), every check by the s
 | --- | --- |
 | `test_an_all_visible_range_needs_no_fetch_and_vacuum_records_it` | the index-only scan is chosen, fetches nothing, matches the heap, and VACUUM records a page count no larger than the relation and most of it, as the heap's is |
 | `test_a_delete_clears_the_bit_and_is_never_returned` | after a delete the scan matches the heap and fetches where the bit was cleared |
+
+## 176. test_native_truncate.py: end-truncation shrinks the file and loses no row
+
+The pytest twin of `test/native_truncate.sh` (ported for #1341), every check by the shell's name. `pgcolumnar.truncate()` returns trailing reclaimed blocks to the OS. After the trailing half is deleted and compacted, the file shrinks and the rows still match a heap mirror. A second truncate has nothing to do, and the table stays usable: reinsert, index, and an indexed lookup forced through the index and asserted to read it. With `pgcolumnar.enable_end_truncation` off, truncate must return 0 and leave the file alone. The call is a statement of its own on both sides, because sent with the `SET` as one string it ran in an implicit transaction block and was refused, which passed whatever the GUC did. Inside a transaction block truncate is refused and changes nothing, because its shrink is not transactional, and a truncate followed by reinsert and compaction keeps parity.
+
+### Every test
+
+| test | what it holds |
+| --- | --- |
+| `test_truncate_shrinks_the_file_and_keeps_every_row` | a truncate with nothing freed returns 0; after delete and compact it returns blocks, the file shrinks, the rows match the heap, and a second truncate returns 0 |
+| `test_the_table_stays_usable_and_the_guc_turns_it_off` | reinsert and an index lookup through the index after truncation; with the GUC off truncate returns 0 and the file keeps its size, with it on the file shrinks |
+| `test_truncate_refuses_a_transaction_block_and_reuse_stays_clean` | refused inside BEGIN with the file and rows unchanged; truncate, reinsert and compaction keep parity |

test/pytest/expected_tests.txt

@@ -821,4 +821,6 @@ guard_tests 440
 # when test_join_runtime_filter.py became test_native_join_runtime_filter.py: one test, by collection.
 # cluster_tests 750 -> 751 for #1341's completion of test_native_agg.py, the twin of native_agg.sh:
 # one test, by collection over the cluster files on this tree.
-cluster_tests 751
+# cluster_tests 751 -> 754 for #1341's test_native_truncate.py, the twin of native_truncate.sh:
+# three tests, by collection over the cluster files on this tree.
+cluster_tests 754

test/pytest/test_compare_to_bash.py

@@ -156,6 +156,7 @@
             "native_recluster",
             "native_repack",
             "native_saop_pushdown",
+            "native_truncate",
             "native_vacuum_race",
             "native_vecskip",
             "native_writer",

test/pytest/test_native_truncate.py

@@ -0,0 +1,148 @@
+"""Physical end-truncation (Phase F). The pytest twin of `test/native_truncate.sh`,
+every check by the shell's name.
+
+pgcolumnar.truncate() returns trailing reclaimed blocks to the OS. After a delete and a
+compaction the file shrinks, the rows still match a heap mirror, a second truncate has
+nothing to do, and the table stays usable: reinsert, index, indexed lookup. The GUC
+turns it off. It refuses to run inside a transaction block, since its shrink is not
+transactional, and a truncate followed by reinsert and compaction stays parity-clean
+(the assert build's no-overlap validator runs inside compact). Concurrency is the
+isolation specs' (specs/truncate_vs_*.spec).
+"""
+import psycopg
+import pytest
+
+HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) "
+        "FROM (SELECT _row::text AS t FROM ({}) _row) _s")
+IDX_SETS = ("SET enable_seqscan = off", "SET enable_bitmapscan = off", "SET pgcolumnar.enable_custom_scan = off")
+LOOKUP = "SELECT count(v) FROM n WHERE id BETWEEN 1 AND 40000"
+
+
+def _one(c, sql):
+    try:
+        return str(c.execute(sql).fetchone()[0])
+    except psycopg.Error as exc:
+        return f"error: {exc.diag.message_primary}"
+
+
+def _size(c):
+    return int(c.execute("SELECT pg_relation_size('n')").fetchone()[0])
+
+
+def _hashes(c):
+    """-> (columnar, heap) set digests of the rows."""
+    return _one(c, HASH.format("SELECT id, v FROM n")), _one(c, HASH.format("SELECT id, v FROM h"))
+
+
+def _both(c, *sqls):
+    """Run each statement on the heap mirror and on the columnar table."""
+    for sql in sqls:
+        c.execute(sql.format(t="h"))
+        c.execute(sql.format(t="n"))
+
+
+def _truncated(c):
+    return int(c.execute("SELECT pgcolumnar.truncate('n')").fetchone()[0])
+
+
+@pytest.fixture(scope="module")
+def nt(pgc_cluster):
+    db = "pgc_ntrunc"
+    with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a:
+        a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)")
+        a.execute(f"CREATE DATABASE {db}")
+        # end-truncation is opt-in (GUC default off); on for the whole database
+        a.execute(f"ALTER DATABASE {db} SET pgcolumnar.enable_end_truncation = on")
+    try:
+        with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c:
+            c.execute("CREATE EXTENSION pgcolumnar")
+            c.execute("CREATE TABLE h (id int, v text)")
+            c.execute("CREATE TABLE n (id int, v text) USING pgcolumnar")
+            c.execute("SELECT pgcolumnar.set_options('n', stripe_row_limit => 1000, chunk_group_row_limit => 1000)")
+            _both(c, "INSERT INTO {t} SELECT g, md5(g::text) FROM generate_series(1, 30000) g")
+            yield c
+    finally:
+        with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a:
+            a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)")
+
+
+def test_truncate_shrinks_the_file_and_keeps_every_row(nt, expect):
+    c = nt
+    expect.text(*_hashes(c), "load parity")
+    # nothing has been freed yet
+    expect.text(_one(c, "SELECT pgcolumnar.truncate('n')"), "0", "truncate before any delete is a no-op")
+
+    # the trailing half deleted whole (groups 16..30), and compacted to free it
+    _both(c, "DELETE FROM {t} WHERE id > 15000")
+    c.execute("SELECT pgcolumnar.compact('n')")
+    before = _size(c)
+    trunc = _truncated(c)
+    print(f"-- truncate returned {trunc} blocks; before-trunc={before} after={_size(c)}")
+    expect.at_least(trunc, 1, "truncate returned blocks")
+    expect.at_least(before - _size(c), 1, "file shrank")
+    expect.text(*_hashes(c), "parity after truncate")
+    expect.text(_one(c, "SELECT count(*) FROM n"), "15000", "row count after truncate")
+    expect.text(_one(c, "SELECT pgcolumnar.truncate('n')"), "0", "second truncate is a no-op")
+
+
+def test_the_table_stays_usable_and_the_guc_turns_it_off(nt, expect):
+    c = nt
+    _both(c, "INSERT INTO {t} SELECT g, md5(g::text) FROM generate_series(30001, 40000) g")
+    expect.text(*_hashes(c), "parity after post-truncate insert")
+    c.execute("CREATE INDEX n_id ON n (id)")
+    # forced to fetch every row through n_id, and asserted: by default this count is an
+    # Index Only Scan, which stops fetching once a VACUUM marks the pages all-visible
+    for s in IDX_SETS:
+        c.execute(s)
+    try:
+        plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {LOOKUP}"))
+        expect.text("yes" if "Index Scan using n_id" in plan else "no", "yes", "premise: the indexed lookup reads n_id")
+        expect.text(_one(c, LOOKUP), "25000", "indexed lookup after truncate")
+    finally:
+        for g in ("enable_seqscan", "enable_bitmapscan", "pgcolumnar.enable_custom_scan"):
+            c.execute(f"RESET {g}")
+
+    _both(c, "DELETE FROM {t} WHERE id > 35000")
+    c.execute("SELECT pgcolumnar.compact('n')")
+    guard = _size(c)
+    # its own statement, and its answer checked: the shell once sent SET and the call
+    # as one string, an implicit transaction block that truncate refuses
+    c.execute("SET pgcolumnar.enable_end_truncation = off")
+    try:
+        off = _one(c, "SELECT pgcolumnar.truncate('n')")
+    finally:
+        c.execute("RESET pgcolumnar.enable_end_truncation")
+    expect.text(f"{_size(c)}|{off}", f"{guard}|0", "GUC off: truncate does nothing")
+    expect.at_least(_truncated(c), 1, "GUC on: truncate reclaims")
+    expect.at_least(guard - _size(c), 1, "GUC on: file shrank")
+    expect.text(*_hashes(c), "parity after guard cycle")
+
+
+def test_truncate_refuses_a_transaction_block_and_reuse_stays_clean(nt, expect):
+    c = nt
+    # a ROLLBACK must never leave the metapage highwater lowered while the
+    # transactional free_space deletes roll back
+    _both(c, "DELETE FROM {t} WHERE id BETWEEN 30001 AND 33000")
+    c.execute("SELECT pgcolumnar.compact('n')")
+    before = _size(c)
+    refused = "no"
+    c.execute("BEGIN")
+    try:
+        c.execute("SELECT pgcolumnar.truncate('n')")
+    except psycopg.Error as exc:
+        if "cannot run inside a transaction block" in (exc.diag.message_primary or ""):
+            refused = "yes"
+    finally:
+        c.execute("ROLLBACK")
+    expect.text(refused, "yes", "truncate refused inside a transaction block")
+    expect.num(_size(c), before, "file unchanged after refused in-txn truncate")
+    expect.text(*_hashes(c), "parity after refused in-txn truncate")
+
+    # an autocommit truncate, reinsert into the reclaimed space, then compaction: no
+    # double allocation over reused blocks
+    c.execute("SELECT pgcolumnar.truncate('n')")
+    _both(c, "INSERT INTO {t} SELECT g, md5(g::text) FROM generate_series(41001, 43000) g",
+          "DELETE FROM {t} WHERE id > 42000")
+    c.execute("SELECT pgcolumnar.compact_rewrite('n', 0.0)")
+    c.execute("SELECT pgcolumnar.compact('n')")
+    expect.text(*_hashes(c), "parity after truncate + reinsert + compaction")

test/pytest/unported_suites.txt

@@ -111,7 +111,6 @@ native_rewrite_conc
 native_roundtrip
 native_sort_by
 native_toasted_write
-native_truncate
 native_varlena_bound
 native_vecdecode
 native_zonemap