diff --git a/test/audit.sh b/test/audit.sh index 613e21c61..a75405a26 100755 --- a/test/audit.sh +++ b/test/audit.sh @@ -213,9 +213,26 @@ check "add-column aggregate" \ "$(q "SELECT sum(c) || '/' || count(d) || '/' || count(b) FROM ac;")" "218/6/1" # index scan over an added column +# +# FORCED THROUGH THE INDEX, AND ASSERTED (#1339). enable_seqscan alone leaves the +# columnar custom scan on, so this and "create index rows indexed once" below could +# be answered without reading the index they name. Both lookups now turn the +# custom scan and bitmap scans off (a bitmap scan would also collapse a duplicate +# index entry), and assert an Index Scan or Index Only Scan through their index. +IDXONLY="SET enable_seqscan=off; SET enable_bitmapscan=off; SET pgcolumnar.enable_custom_scan=off;" +reads_index() { # reads_index INDEX QUERY -> yes, or the plan's scan line + local plan + plan="$(q "$IDXONLY EXPLAIN (COSTS OFF) $2")" + case "$plan" in + *"Index Scan using $1 "*|*"Index Only Scan using $1 "*) echo yes ;; + *) echo "no: $(grep -m1 'Scan' <<<"$plan" | sed 's/^ *//')" ;; + esac +} q "CREATE INDEX ac_c_idx ON ac(c);" >/dev/null +check "premise: the add-column lookup reads ac_c_idx" \ + "$(reads_index ac_c_idx "SELECT count(*) FROM ac WHERE c=42")" "yes" check "add-column index scan" \ - "$(q "SET enable_seqscan=off; SELECT count(*) FROM ac WHERE c=42;")" "5" + "$(q "$IDXONLY SELECT count(*) FROM ac WHERE c=42;" | tail -1)" "5" q "DROP TABLE ac;" >/dev/null # --------------------------------------------------------------------------- @@ -369,8 +386,10 @@ LEAKS="$(run_pg "grep -c 'resource was not closed' '$LOGFILE'" || true)" check "create index no relation leak" "$LEAKS" "0" # and every row is indexed exactly once (no duplicate TIDs from a per-worker rescan) +check "premise: the indexed-once count reads idxleak_a_idx" \ + "$(reads_index idxleak_a_idx "SELECT count(*) FROM idxleak WHERE a>0")" "yes" check "create index rows indexed once" \ - "$(q "SET enable_seqscan=off; SELECT count(*) FROM idxleak WHERE a>0;")" "50000" + "$(q "$IDXONLY SELECT count(*) FROM idxleak WHERE a>0;" | tail -1)" "50000" q "DROP TABLE idxleak;" >/dev/null # --------------------------------------------------------------------------- diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 2a9ea5c9b..de7c53af7 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2479,6 +2479,21 @@ native_encdesc_golden native_encdesc_golden int column a rawLen parses as 1200 ( native_encdesc_golden native_encdesc_golden the table reads back all rows 15;16;17;18;19 never - native_encdesc_golden native_encdesc_golden trailing sharedTableLen is 0 (u32 at offset 6+13=19) 15;16;17;18;19 never - native_encdesc_golden native_encdesc_golden vectorCount parses as 1 (at offset 2) 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap a narrow projection over a wide group is cached too (#433) 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap columns that fit are cached even when their group does not (#433) 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap every payload survived the round trip at full length 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap premise: fetching one row touched a measurable number of buffers 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap premise: fetching ten rows touched a measurable number of buffers 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap premise: one row is fetched through an index scan 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap premise: ten rows are fetched through an index scan 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap premise: the fixture really has a row group above the cap 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap premise: the medium projection is fetched through an index scan 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap premise: the medium projection touched a measurable number of buffers 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap premise: the narrow projection is fetched through an index scan 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap premise: the narrow projection touched a measurable number of buffers 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap premise: the table loaded every row 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap premise: the wide column is the one over the cap, so this arm is #359's 15;16;17;18;19 never - +native_fetch_bigcap native_fetch_bigcap the fetched rows are the right rows 15;16;17;18;19 2026-10-16 columnar_tableam.c: the deferred slot decode reads tag as NULL on every 7th row number native_fetch_cache native_fetch_cache a hit re-checks the group's fileOffset against the row group 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks fileOffset native_fetch_cache native_fetch_cache a hit re-checks the group's firstRowNumber against the row group 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks firstRowNumber native_fetch_cache native_fetch_cache a hit re-checks the group's natts against the scan's column count 15;16;17;18;19 2026-10-16 columnar_reader.c: a fetch-cache hit no longer re-checks natts @@ -3243,21 +3258,22 @@ partition_lifecycle partition_lifecycle running the plan leaves only the partiti partition_lifecycle partition_lifecycle the parent now has the partitions it reported 15;16;17;18;19 2026-10-14 partition_create_ahead creating its last partition without attaching it partition_lifecycle partition_lifecycle the plan retires the partitions wholly before the cutoff, in range order 15;16;17;18;19 2026-10-13 a plain DROP, not refusing while the partition is still attached;the cutoff filter removed: every partition planned;the lower bound read where the upper is meant;the plan detaches but drops nothing;the plan detaches without CONCURRENTLY;the plan in catalog order, not range order partition_lifecycle partition_lifecycle they are contiguous, each one width wide 15;16;17;18;19 2026-10-12 partition_create_ahead leaving a minute between partitions -partition_rows partition_rows a duplicate key inserted through the parent is refused (23505), as on heap 15;16;17;18;19 never - -partition_rows partition_rows a duplicate of a moved row's new key is refused in its new partition (23505), as on heap 15;16;17;18;19 never - +partition_rows partition_rows a duplicate key inserted through the parent is refused (23505), as on heap 15;16;17;18;19 2026-10-16 columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row +partition_rows partition_rows a duplicate of a moved row's new key is refused in its new partition (23505), as on heap 15;16;17;18;19 2026-10-16 columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row partition_rows partition_rows a key a moved row gave up can be used again, as on heap 15;16;17;18;19 2026-10-15 a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete) -partition_rows partition_rows a second insert of the same new key through the parent waits, then is refused (23505), and one row is left, as on heap 15;16;17;18;19 never - +partition_rows partition_rows a second insert of the same new key through the parent waits, then is refused (23505), and one row is left, as on heap 15;16;17;18;19 2026-10-16 columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row partition_rows partition_rows a second session moving a row the first is moving waits, then fails with 40001, and the row exists once, as on heap 15;16;17;18;19 2026-10-15 a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete);the tree before #1421: a row moved between partitions is serialized only at flush, after its new version's unique check partition_rows partition_rows an UPDATE in place of a row another session is moving waits, then fails with 40001, as on heap 15;16;17;18;19 2026-10-15 a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete) partition_rows partition_rows an UPDATE moving 25,000 keyed rows to the other partition keeps under 32 MB likewise 15;16;17;18;19 2026-10-15 PgColumnarSerializeRowNow reads the delete vector into the caller's context, keeping a bitmap copy per row (the tree before the #1422 review fix) partition_rows partition_rows an UPDATE moving 49 rows to the other partition succeeds, as on heap 15;16;17;18;19 never - partition_rows partition_rows an UPDATE of 25,000 keyed rows keeps under 32 MB of memory between its first and last row 15;16;17;18;19 2026-10-15 PgColumnarSerializeRowNow reads the delete vector into the caller's context, keeping a bitmap copy per row (the tree before the #1422 review fix) -partition_rows partition_rows and after the races the parents hold the same rows 15;16;17;18;19 2026-10-15 a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete) +partition_rows partition_rows and after the races the parents hold the same rows 15;16;17;18;19 2026-10-16 a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete);columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row partition_rows partition_rows and each UPDATE reached every row once 15;16;17;18;19 never - partition_rows partition_rows and each partition holds what heap's does 15;16;17;18;19 2026-10-15 a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete) partition_rows partition_rows and the parent holds the same rows as heap's 15;16;17;18;19 2026-10-15 a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete) partition_rows partition_rows each keyed table's group carries delete marks before its UPDATE 15;16;17;18;19 never - -partition_rows partition_rows the primary key finds a moved row at its new key, and not at its old one, as on heap 15;16;17;18;19 2026-10-15 a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete) +partition_rows partition_rows premise: the columnar key lookup reads the primary key 15;16;17;18;19 never - +partition_rows partition_rows the primary key finds a moved row at its new key, and not at its old one, as on heap 15;16;17;18;19 2026-10-16 a row moving to another partition is not marked deleted in the partition it leaves (src/columnar_tableam.c tuple_delete);columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row partman_columnar partman_columnar and no row-group row behind 15;16;17;18;19 2026-10-12 the drop hook stops deleting row_group rows, so they outlive their deleted storage row partman_columnar partman_columnar and read back as heap does 15;16;17;18;19 2026-10-12 a sequential scan drops the last row group from its list, so every one-group partition reads empty partman_columnar partman_columnar and the rows land in columnar partitions 15;16;17;18;19 2026-10-12 a sequential scan drops the last row group from its list, so every one-group partition reads empty @@ -3883,6 +3899,35 @@ settings_reference settings_reference each documented default is the server's 15 settings_reference settings_reference every documented setting exists on the server 15;16;17;18;19 2026-10-15 the compression_level row renamed compression_levl in docs/configuration.md settings_reference settings_reference every setting the server defines is documented, the developer hooks aside 15;16;17;18;19 2026-10-15 the backticks around every setting name removed from docs/configuration.md;the compression_level row deleted from docs/configuration.md;the compression_level row renamed compression_levl in docs/configuration.md settings_reference settings_reference the reference documents at least thirty settings, so the comparisons below read something 15;16;17;18;19 2026-10-15 the backticks around every setting name removed from docs/configuration.md +sorted_projection sorted_projection control: the set oracle is order-blind by design 15;16;17;18;19 never - +sorted_projection sorted_projection count matches heap 15;16;17;18;19 never - +sorted_projection sorted_projection idempotent sorted rewrite 15;16;17;18;19 never - +sorted_projection sorted_projection index point lookup after sort 15;16;17;18;19 2026-10-16 columnar_vacuum.c: pgcolumnar_compact_relation no longer reindexes after its rewrite +sorted_projection sorted_projection index sees deletion after sort 15;16;17;18;19 2026-10-16 columnar_vacuum.c: pgcolumnar_compact_relation no longer reindexes after its rewrite +sorted_projection sorted_projection live count after delete+sort 15;16;17;18;19 never - +sorted_projection sorted_projection multi-key range 15;16;17;18;19 never - +sorted_projection sorted_projection post aggregate 15;16;17;18;19 never - +sorted_projection sorted_projection post equality 15;16;17;18;19 never - +sorted_projection sorted_projection post order-by 15;16;17;18;19 never - +sorted_projection sorted_projection post range 15;16;17;18;19 never - +sorted_projection sorted_projection post star 15;16;17;18;19 never - +sorted_projection sorted_projection post-sort prunes more chunk groups 15;16;17;18;19 2026-10-16 columnar_vacuum.c: pgcolumnar_compact_relation never sorts the rows it rewrites +sorted_projection sorted_projection pre aggregate 15;16;17;18;19 never - +sorted_projection sorted_projection pre equality 15;16;17;18;19 never - +sorted_projection sorted_projection pre order-by 15;16;17;18;19 never - +sorted_projection sorted_projection pre range 15;16;17;18;19 never - +sorted_projection sorted_projection pre star 15;16;17;18;19 never - +sorted_projection sorted_projection pre-sort prunes little 15;16;17;18;19 never - +sorted_projection sorted_projection premise: the deletion lookup reads t_col_id_idx 15;16;17;18;19 never - +sorted_projection sorted_projection premise: the ordered oracle agrees with itself 15;16;17;18;19 never - +sorted_projection sorted_projection premise: the ordered oracle is order-sensitive 15;16;17;18;19 never - +sorted_projection sorted_projection premise: the point lookup reads t_col_id_idx 15;16;17;18;19 never - +sorted_projection sorted_projection reject no sort columns 15;16;17;18;19 never - +sorted_projection sorted_projection reject non-columnar table 15;16;17;18;19 2026-10-16 columnar_vacuum.c: pgcolumnar_vacuum_sorted no longer checks for a columnar relation +sorted_projection sorted_projection reject non-sortable type 15;16;17;18;19 2026-10-16 columnar_vacuum.c: pgcolumnar_compact_relation never sorts the rows it rewrites +sorted_projection sorted_projection reject unknown column 15;16;17;18;19 never - +sorted_projection sorted_projection sum matches heap 15;16;17;18;19 never - +sorted_projection sorted_projection the plan under test is a columnar custom scan 15;16;17;18;19 never - statement_self_visibility statement_self_visibility INSERT INTO t SELECT FROM t runs to completion 15;16;17;18;19 2026-10-15 a serial scan re-reads its row-group list when it reaches the end, so it picks up groups its own statement flushed (src/columnar_reader.c pgcolumnar_next_group_index) statement_self_visibility statement_self_visibility UPDATE of every row runs to completion 15;16;17;18;19 2026-10-15 a serial scan re-reads its row-group list when it reaches the end, so it picks up groups its own statement flushed (src/columnar_reader.c pgcolumnar_next_group_index) statement_self_visibility statement_self_visibility an INSERT in a WITH clause reading its own table returns the rows it inserted, as heap does 15;16;17;18;19 2026-10-15 a serial scan re-reads its row-group list when it reaches the end, so it picks up groups its own statement flushed (src/columnar_reader.c pgcolumnar_next_group_index) diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index affa347e8..3a65a959b 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 193 +suites_not_covered 191 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3467,4 +3467,42 @@ suites_not_covered 193 # never 3019 + 2 - 4 = 3017 # dated 1311 + 4 = 1315 # 3017 + 1315 == 4332 -checks_never_observed_red 3017 +# +# #1341, native_fetch_bigcap seeded, with its twin (OffgridwithJD): 3017 -> 3031, +# and suites_not_covered 193 -> 192. The suite had no rows. It enters with all 15 +# checks from five-major runs (15/15 on each of 15-19). One is dated: the right-rows +# arm, by the deferred decode reading tag as NULL on every 7th row number. Fourteen +# arrive never: eleven premises; the payload length; and the two #433 ratios, which +# the cache missing every lookup and the cap dropping every column cannot move, +# because each re-reads only the projected columns. +# +# ledger rows 4332 + 15 = 4347 +# never 3017 + 14 = 3031 +# dated 1315 + 1 = 1316 +# 3031 + 1316 == 4347 +# +# #1339, partition_rows' primary-key arm reads the key it names (OffgridwithJD): +# 3031 -> 3029. One premise is new, never. With pgcolumnar_index_fetch_tuple +# finding no row, five arms are red: the three duplicate-key refusals entered never +# and are now dated, and the primary-key and races arms gain it in their history. +# +# ledger rows 4347 + 1 = 4348 +# never 3031 + 1 - 3 = 3029 +# dated 1316 + 3 = 1319 +# 3029 + 1319 == 4348 +# +# #1341, sorted_projection seeded, with its twin (OffgridwithJD): 3029 -> 3053, and +# suites_not_covered 192 -> 191. The suite had no rows. It enters with all 29 checks +# from five-major runs (29/29 on each of 15-19). 5 are dated by three mutations: +# the post-sort reindex removed (both index lookups), the compaction not sorting +# (the pruning arm and the json refusal, which the sort raises), and vacuum_sorted's +# columnar check removed. Twenty-four arrive never: the ten heap-oracle comparisons +# and the multi-key and post-delete counts, which no rewrite mutation here moves; +# the oracle and index premises; the pre-sort control; idempotence; and the +# unknown-column and no-sort-key refusals. +# +# ledger rows 4348 + 29 = 4377 +# never 3029 + 24 = 3053 +# dated 1319 + 5 = 1324 +# 3053 + 1324 == 4377 +checks_never_observed_red 3053 diff --git a/test/native_fetch_bigcap.sh b/test/native_fetch_bigcap.sh index d91c52091..9e26b7228 100755 --- a/test/native_fetch_bigcap.sh +++ b/test/native_fetch_bigcap.sh @@ -33,8 +33,13 @@ CAP=$((32 * 1024 * 1024)) # COLUMNAR_FETCH_CACHE_MAX_BYTES, src/columnar.h # by its own model, and then nothing measures the fetch path at all. So the index # path is forced, and every measured query asserts it got one. Whether the # planner should choose it is #434, and that is a separate suite. +# +# pgcolumnar.enable_custom_scan=off as well: enable_seqscan alone does not turn +# the columnar custom scan off (#1339). Every measured query asserts its plan, so +# a planner change cannot quietly take the arms off the fetch path. FORCE_INDEX="SET max_parallel_workers_per_gather=0; SET enable_seqscan=off; - SET enable_bitmapscan=off; SET pgcolumnar.enable_index_fetch_penalty=off;" + SET enable_bitmapscan=off; SET pgcolumnar.enable_index_fetch_penalty=off; + SET pgcolumnar.enable_custom_scan=off;" # --------------------------------------------------------------------------- # The fixture, whose premise is its ON-DISK SIZE. @@ -144,6 +149,11 @@ nbufs() { # $1 = number of rows, projecting one narrow column } } END { print t + 0 }' } +# The narrow and medium projections had no plan premise of their own; only the +# payload query's plan was asserted (#1339). They plan as Index Scans today, and +# these keep a planner change from turning their ratios into scan-against-scan. +check_num "premise: the narrow projection is fetched through an index scan" \ + "$(pgc_uses_row_fetch "$FORCE_INDEX" "SELECT sum(tag) FROM bigcap WHERE id BETWEEN 1 AND 10" | grep -c yes)" "1" N1=$(nbufs 1) N10=$(nbufs 10) echo "-- narrow projection buffers: 1 row = ${N1}, 10 rows = ${N10}" @@ -172,6 +182,8 @@ mbufs() { # $1 = number of rows, projecting three medium columns } } END { print t + 0 }' } +check_num "premise: the medium projection is fetched through an index scan" \ + "$(pgc_uses_row_fetch "$FORCE_INDEX" "SELECT count(m1 || m2 || m3) FROM bigcap WHERE id BETWEEN 1 AND 10" | grep -c yes)" "1" M1=$(mbufs 1) M10=$(mbufs 10) echo "-- three medium columns from an oversized group: 1 row = ${M1}, 10 rows = ${M10}" diff --git a/test/partition_rows.sh b/test/partition_rows.sh index 9e6be2d15..b282ee991 100755 --- a/test/partition_rows.sh +++ b/test/partition_rows.sh @@ -52,9 +52,19 @@ check "an UPDATE moving 49 rows to the other partition succeeds, as on heap" \ "$(both "UPDATE %p SET id = id + 1000 WHERE id < 50")" "ok|ok" check "and each partition holds what heap's does" "$(answers "$SPLIT")" "950/49|950/49" check "and the parent holds the same rows as heap's" "$(same "$CONTENT")" "same" -IDX="SET enable_seqscan = off; SET enable_bitmapscan = off;" +# The custom scan off too, and a premise that the columnar lookup reads the key: +# enable_seqscan alone leaves the columnar custom scan on. Measured (#1339): with +# the index fetch finding no row, this arm stayed green, because the columnar +# parent was answered by its partitions' custom scans, not by the primary key. +IDX="SET enable_seqscan = off; SET enable_bitmapscan = off; SET pgcolumnar.enable_custom_scan = off;" +# It plans as an Index Only Scan (it reads only id), which on a columnar table still +# asks each row whether it is live; the premise accepts either index node. +KEY_SQL="SELECT count(*) FILTER (WHERE id = 1001) || '/' || count(*) FILTER (WHERE id = 1) FROM %p WHERE id IN (1, 1001)" +check "premise: the columnar key lookup reads the primary key" \ + "$(case "$(q "$IDX EXPLAIN (COSTS OFF) ${KEY_SQL//%p/pr_pgcolumnar}")" in + *"Index Scan using"*|*"Index Only Scan using"*) echo yes ;; *) echo no ;; esac)" yes check "the primary key finds a moved row at its new key, and not at its old one, as on heap" \ - "$(answers "$IDX SELECT count(*) FILTER (WHERE id = 1001) || '/' || count(*) FILTER (WHERE id = 1) FROM %p WHERE id IN (1, 1001)")" \ + "$(answers "$IDX $KEY_SQL")" \ "1/0|1/0" # ---- the key on the parent ---------------------------------------------------------- diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index a82be591b..6d3efc7fd 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -218,6 +218,8 @@ behaviour, the source of that number is named. - [170. test_native_parquet_flba.py: uuid and DECIMAL read from fixed-length byte arrays](#170-test_native_parquet_flbapy-uuid-and-decimal-read-from-fixed-length-byte-arrays) - [171. test_native_parquet_multifile.py: directories and globs read as one relation](#171-test_native_parquet_multifilepy-directories-and-globs-read-as-one-relation) - [172. test_native_index.py: index lookups and unique enforcement fetch by row number](#172-test_native_indexpy-index-lookups-and-unique-enforcement-fetch-by-row-number) +- [173. test_native_fetch_bigcap.py: fetching from a row group above the cache cap](#173-test_native_fetch_bigcappy-fetching-from-a-row-group-above-the-cache-cap) +- [174. test_sorted_projection.py: vacuum_sorted reorders storage and changes no answer](#174-test_sorted_projectionpy-vacuum_sorted-reorders-storage-and-changes-no-answer) ## 1. How to read a test in here @@ -7773,3 +7775,26 @@ The pytest twin of `test/native_index.sh` (ported for #1341), every check by the | --- | --- | | `test_index_lookups_match_the_heap` | four lookups plan as Index Scans and match the heap, and a duplicate primary key is refused with the row count unchanged | | `test_a_standalone_unique_index_enforces` | a standalone unique index refuses a duplicate and admits a new key | + +## 173. test_native_fetch_bigcap.py: fetching from a row group above the cache cap + +The pytest twin of `test/native_fetch_bigcap.sh` (ported for #1341), every check by the shell's name. Above the 32 MB fetch-cache cap, the entry used to be dropped whole after every fetch, so each fetched row re-read the whole group (#433). The measure is buffers, a count rather than a time. The fixture's premise is its on-disk size, read back from `pgcolumnar.row_group`. Every measured query is forced onto the index path with the custom scan off, and asserts its plan. A payload wider than the cap re-decodes per fetch by design (#359). A narrow projection, and columns that fit, must still be cached. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_a_group_above_the_cap_is_read_once_for_what_fits` | the fixture has a group above the cap, every measured query is an index fetch, and ten rows of a narrow or a fitting projection cost under three times one row | +| `test_caching_changes_no_row` | the first ten rows equal the same rows read in key order, and every payload is full length | + +## 174. test_sorted_projection.py: vacuum_sorted reorders storage and changes no answer + +The pytest twin of `test/sorted_projection.sh` (ported for #1341), every check by the shell's name. `pgcolumnar.vacuum_sorted()` rewrites a columnar table physically sorted on a key. It must never change an answer: every query shape agrees with a heap oracle before and after, order-independently, and in order where the query orders. It must also reorder storage for real, so a narrow range on the sort key skips vectors it could not skip before. The rest covers a multi-column key, the reclaim of deleted rows with the index rebuilt, idempotence, and argument validation. Each index lookup is forced with the custom scan off and asserts that it reads the index. Each refusal must be the function's own, identified by its SQLSTATE. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_the_sorted_rewrite_changes_no_answer_and_really_reorders` | five query shapes match the heap before and after the sort, the ordered oracle is order-sensitive, and the sort makes the range skip vectors | +| `test_a_multi_key_sort_reclaims_and_rebuilds_the_index` | a two-column sort matches, deletes are reclaimed, the rebuilt index finds and hides the right rows, and a second sort changes nothing | +| `test_vacuum_sorted_refuses_what_it_cannot_sort` | a heap table, an unknown column, no sort key and a json key are each refused with their own SQLSTATE | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index fdbaa4406..294633d71 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -807,4 +807,8 @@ guard_tests 438 # three tests, by collection over the cluster files on this tree. # cluster_tests 740 -> 742 for #1341's test_native_index.py, the twin of native_index.sh: # two tests, by collection over the cluster files on this tree. -cluster_tests 742 +# cluster_tests 742 -> 744 for #1341's test_native_fetch_bigcap.py, the twin of native_fetch_bigcap.sh: +# two tests, by collection over the cluster files on this tree. +# cluster_tests 744 -> 747 for #1341's test_sorted_projection.py, the twin of sorted_projection.sh: +# three tests, by collection over the cluster files on this tree. +cluster_tests 747 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 6708c3f05..cd5850245 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -129,6 +129,7 @@ "native_dict_underfill", "native_dml", "native_encdesc_golden", + "native_fetch_bigcap", "native_fetch_cache", "native_fetch_coalesce", "native_fetch_position", @@ -187,6 +188,7 @@ "serializable", "settings_reference", "sorted_pathkeys", + "sorted_projection", "statement_self_visibility", "stats_privilege", "storage_row_repair", @@ -420,7 +422,8 @@ def test_the_loop_reader_invents_nothing_in_this_corpus(expect): expect.text(", ".join(files), "build_refusal, differential, join_runtime_filter, native_groupagg_batch, " "native_index_projection, native_parquet_pushdown, native_reclaim, native_repack, " - "native_saop_pushdown, native_vacuum_race, native_vecskip, sorted_pathkeys", + "native_saop_pushdown, native_vacuum_race, native_vecskip, sorted_pathkeys, " + "sorted_projection", "and it is these files, so another appearing is a diff a reviewer " "sees rather than a number that moved") diff --git a/test/pytest/test_native_fetch_bigcap.py b/test/pytest/test_native_fetch_bigcap.py new file mode 100644 index 000000000..1ee132908 --- /dev/null +++ b/test/pytest/test_native_fetch_bigcap.py @@ -0,0 +1,138 @@ +"""Fetching from a row group larger than the fetch-cache cap (#433, #1341). The pytest +twin of `test/native_fetch_bigcap.sh`, every check by the shell's name. + +Above the 32 MB cap the hit rate used to be zero by construction: the entry was +populated, used once and dropped whole, so every fetched row re-read the whole group. +The measure is BUFFERS, a count rather than a time. The fixture's premise is its +on-disk size, read back from pgcolumnar.row_group. Every measured query is forced +onto the index path, with the custom scan off, and asserts its plan. A payload wider +than the cap re-decodes per fetch by design (#359); a narrow projection, and columns +that fit, must be cached anyway. Correctness last. +""" +import psycopg +import pytest + +CAP = 32 * 1024 * 1024 +FORCE_INDEX = ("max_parallel_workers_per_gather=0", "enable_seqscan=off", "enable_bitmapscan=off", + "pgcolumnar.enable_index_fetch_penalty=off", "pgcolumnar.enable_custom_scan=off") + + +class _Sets: + def __init__(self, c, sets): + self.c, self.sets = c, sets + + def __enter__(self): + for s in self.sets: + self.c.execute(f"SET {s}") + + def __exit__(self, *exc): + for s in self.sets: + self.c.execute(f"RESET {s.split('=')[0]}") + + +def _one(c, sql): + try: + return c.execute(sql).fetchone()[0] + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _index_scan(c, sql): + """-> 1 when `sql` plans an Index Scan under FORCE_INDEX, as the shell counts it.""" + with _Sets(c, FORCE_INDEX): + return int(any("Index Scan" in r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {sql}"))) + + +def _bufs(c, sql): + """-> the buffers EXPLAIN (ANALYZE, BUFFERS) reports for `sql`, after one warming run, + summed as the shell's awk sums them: every shared/read/hit token's digits.""" + total = 0 + with _Sets(c, FORCE_INDEX): + c.execute(f"EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING OFF, FORMAT TEXT) {sql}").fetchall() + for (line,) in c.execute(f"EXPLAIN (ANALYZE, BUFFERS, COSTS OFF, TIMING OFF, FORMAT TEXT) {sql}"): + if "Buffers:" not in line: + continue + for tok in line.split(): + if tok.startswith(("shared", "read", "hit")): + digits = "".join(ch for ch in tok if ch.isdigit()) + if digits: + total += int(digits) + return total + + +def _ratio(a, b, bound): + """-> 'within' when a/b, rounded to two places, is at most `bound`, as check_ratio.""" + if a == 0 or b == 0: + return f"a side of the ratio is zero, so nothing was measured: a=[{a}] b=[{b}]" + r = round(a / b, 2) + return "within" if r <= bound else f"{r:.2f}x exceeds the {bound}x bound (a={a} b={b})" + + +@pytest.fixture(scope="module") +def bigcap(pgc_cluster): + db = "pgc_bigcap" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + c.execute("CREATE TABLE bigcap (id int, tag int, m1 text, m2 text, m3 text, payload text) USING pgcolumnar") + # per-row distinct hex from a subquery correlated on g, the codec off: this + # fixture compressed 17x and 25x in earlier attempts, so its size is asserted + c.execute("SELECT pgcolumnar.set_options('bigcap', stripe_row_limit => 10000, compression => 'none')") + c.execute("""INSERT INTO bigcap + SELECT g, g % 97, md5((g * 7)::text), md5((g * 11)::text), md5((g * 13)::text), + (SELECT string_agg(md5((g * 300 + s)::text), '') FROM generate_series(1, 256) s) + FROM generate_series(1, 20000) g""") + c.execute("CREATE INDEX bigcap_id ON bigcap (id)") + c.execute("ANALYZE bigcap") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_a_group_above_the_cap_is_read_once_for_what_fits(bigcap, expect): + c = bigcap + maxg = _one(c, "SELECT max(byte_length) FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('bigcap')") + print(f"-- largest row group: {maxg} bytes, cap {CAP}") + expect.at_least(maxg if isinstance(maxg, int) else 0, CAP + 1, + "premise: the fixture really has a row group above the cap") + expect.num(_one(c, "SELECT count(*) FROM bigcap"), 20000, "premise: the table loaded every row") + + payload = "SELECT sum(length(payload)) FROM bigcap WHERE id BETWEEN 1 AND {}" + expect.num(_index_scan(c, payload.format(1)), 1, "premise: one row is fetched through an index scan") + expect.num(_index_scan(c, payload.format(10)), 1, "premise: ten rows are fetched through an index scan") + b1, b10 = _bufs(c, payload.format(1)), _bufs(c, payload.format(10)) + print(f"-- buffers: 1 row = {b1}, 10 rows = {b10}") + # both sides a measurement before any comparison (#418): a failed EXPLAIN reads as 0 + expect.at_least(b1, 101, "premise: fetching one row touched a measurable number of buffers") + expect.at_least(b10, 101, "premise: fetching ten rows touched a measurable number of buffers") + # the payload alone exceeds the cap, so it re-decodes per fetch: #359's design, not #433 + expect.at_least(b10, b1 + 1, "premise: the wide column is the one over the cap, so this arm is #359's") + + narrow = "SELECT sum(tag) FROM bigcap WHERE id BETWEEN 1 AND {}" + expect.num(_index_scan(c, narrow.format(10)), 1, "premise: the narrow projection is fetched through an index scan") + n1, n10 = _bufs(c, narrow.format(1)), _bufs(c, narrow.format(10)) + print(f"-- narrow projection buffers: 1 row = {n1}, 10 rows = {n10}") + expect.at_least(n1, 1, "premise: the narrow projection touched a measurable number of buffers") + expect.text(_ratio(n10, n1, 3), "within", "a narrow projection over a wide group is cached too (#433)") + + medium = "SELECT count(m1 || m2 || m3) FROM bigcap WHERE id BETWEEN 1 AND {}" + expect.num(_index_scan(c, medium.format(10)), 1, "premise: the medium projection is fetched through an index scan") + m1, m10 = _bufs(c, medium.format(1)), _bufs(c, medium.format(10)) + print(f"-- three medium columns from an oversized group: 1 row = {m1}, 10 rows = {m10}") + expect.at_least(m1, 1, "premise: the medium projection touched a measurable number of buffers") + expect.text(_ratio(m10, m1, 3), "within", "columns that fit are cached even when their group does not (#433)") + + +def test_caching_changes_no_row(bigcap, expect): + c = bigcap + row = "md5(string_agg(id || ':' || tag || ':' || m1 || m2 || m3 || ':' || md5(payload), ',' ORDER BY id))" + expect.text(str(_one(c, f"SELECT {row} FROM bigcap WHERE id BETWEEN 1 AND 10")), + str(_one(c, f"SELECT {row} FROM (SELECT * FROM bigcap ORDER BY id LIMIT 10) s")), + "the fetched rows are the right rows") + expect.num(_one(c, "SELECT count(*) FROM bigcap WHERE length(payload) <> 8192"), 0, + "every payload survived the round trip at full length") diff --git a/test/pytest/test_partition_rows.py b/test/pytest/test_partition_rows.py index 533d95257..d5e321ce8 100644 --- a/test/pytest/test_partition_rows.py +++ b/test/pytest/test_partition_rows.py @@ -21,7 +21,9 @@ CONTENT = ("SELECT count(*) || '/' || md5(string_agg(id || ':' || k || ':' || v, ',' ORDER BY id)) " "FROM %p") SPLIT = "SELECT (SELECT count(*) FROM %p_1) || '/' || (SELECT count(*) FROM %p_2)" -IDX = ("SET enable_seqscan = off", "SET enable_bitmapscan = off") +# the custom scan off too: enable_seqscan alone leaves it on, and the columnar parent +# was answered by its partitions' custom scans rather than the primary key (#1339) +IDX = ("SET enable_seqscan = off", "SET enable_bitmapscan = off", "SET pgcolumnar.enable_custom_scan = off") def _connect(cluster, schema, name): @@ -118,6 +120,17 @@ def test_rows_cross_columnar_partitions_and_keys_hold_through_the_parent(pgc_clu expect.text(_answers(pgc_conn, SPLIT), "950/49|950/49", "and each partition holds what heap's does") c, h = _answers(pgc_conn, CONTENT).split("|") expect.text("same" if c == h else f"{c}|{h}", "same", "and the parent holds the same rows as heap's") + with pgc_conn.cursor() as cur: + for s in IDX: + cur.execute(s) + cur.execute("EXPLAIN (COSTS OFF) SELECT count(*) FILTER (WHERE id = 1001) || '/' || " + "count(*) FILTER (WHERE id = 1) FROM pr_pgcolumnar WHERE id IN (1, 1001)") + # an Index Only Scan here (only id is read), which still asks each row whether it is live + lookup = "yes" if any("Index Scan using" in r[0] or "Index Only Scan using" in r[0] + for r in cur.fetchall()) else "no" + for s in IDX: + cur.execute(s.replace("SET ", "RESET ").split(" =")[0]) + expect.text(lookup, "yes", "premise: the columnar key lookup reads the primary key") expect.text(_answers(pgc_conn, "SELECT count(*) FILTER (WHERE id = 1001) || '/' || " "count(*) FILTER (WHERE id = 1) FROM %p WHERE id IN (1, 1001)", IDX), "1/0|1/0", "the primary key finds a moved row at its new key, and not at its old one, as on heap") diff --git a/test/pytest/test_sorted_projection.py b/test/pytest/test_sorted_projection.py new file mode 100644 index 000000000..41ddc9c40 --- /dev/null +++ b/test/pytest/test_sorted_projection.py @@ -0,0 +1,172 @@ +"""vacuum_sorted rewrites a columnar table physically sorted on a key (gap 26, #1341). The +pytest twin of `test/sorted_projection.sh`, every check by the shell's name. + +The rewrite must never change an answer: every query shape agrees with a heap oracle +before and after it, order-independently, and in order where the query orders. It +must really reorder storage: a narrow range on the sort key skips vectors it could not +skip before. Then a multi-column key, deleted rows reclaimed with the index rebuilt +(each index lookup forced with the custom scan off, and asserting it reads the index), +idempotence, and argument validation. +""" +import psycopg +import pytest + +SET_HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) " + "FROM (SELECT _row::text AS t FROM ({}) _row) _s") +SEQ_HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY n)), $e$EMPTY$e$) " + "FROM (SELECT row_number() OVER () AS n, _row::text AS t FROM ({}) _row) _s") +IDXON = ("SET enable_seqscan=off", "SET enable_bitmapscan=off", "SET pgcolumnar.enable_custom_scan=off") + + +def _hash(c, tmpl, sql): + try: + return c.execute(tmpl.format(sql)).fetchone()[0] + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _one(c, sql): + try: + return str(c.execute(sql).fetchone()[0]) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _pair(c, tmpl, hashing=SET_HASH): + """-> (columnar, heap) digests of `tmpl` with %T for the table, as diff_query.""" + return _hash(c, hashing, tmpl.replace("%T", "t_col")), _hash(c, hashing, tmpl.replace("%T", "t_heap")) + + +def _vectors_skipped(c, sql): + for (line,) in c.execute(f"EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) {sql}"): + if "Vectors Skipped" in line: + return int(line.rsplit(" ", 1)[-1]) + return 0 + + +def _forced(c, sql, explain=False): + for s in IDXON: + c.execute(s) + try: + if explain: + plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {sql}")) + return "yes" if ("Index Scan using t_col_id_idx" in plan or "Index Only Scan using t_col_id_idx" in plan) \ + else f"no: {plan.splitlines()[0] if plan else ''}" + return _one(c, sql) + finally: + for g in ("enable_seqscan", "enable_bitmapscan", "pgcolumnar.enable_custom_scan"): + c.execute(f"RESET {g}") + + +def _refused(c, sql): + """-> the SQLSTATE `sql` is refused with, or 'succeeded': the refusal must be the + function's own, since with its columnar check gone a heap table still raised XX001.""" + try: + c.execute(sql) + return "succeeded" + except psycopg.Error as exc: + return exc.sqlstate or "?" + + +@pytest.fixture(scope="module") +def sp(pgc_cluster): + db = "pgc_sortproj" + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + a.execute(f"CREATE DATABASE {db}") + try: + with psycopg.connect(pgc_cluster.dsn(dbname=db), autocommit=True) as c: + c.execute("CREATE EXTENSION pgcolumnar") + # k scrambled against insert order, so before sorting every group spans the domain + c.execute("CREATE TABLE t_heap (id int, k int, v bigint, t text) USING heap") + c.execute("CREATE TABLE t_col (id int, k int, v bigint, t text) USING pgcolumnar") + c.execute("SELECT pgcolumnar.set_options('t_col', chunk_group_row_limit => 1000)") + c.execute("INSERT INTO t_heap SELECT g, ((g*7919)%1000), g::bigint*2, 'r'||g FROM generate_series(1,50000) g") + c.execute("INSERT INTO t_col SELECT * FROM t_heap") + yield c + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {db} WITH (FORCE)") + + +def test_the_sorted_rewrite_changes_no_answer_and_really_reorders(sp, expect): + c = sp + for label, tmpl in (("pre range", "SELECT id, v, t FROM %T WHERE k BETWEEN 100 AND 120"), + ("pre equality", "SELECT id FROM %T WHERE k = 500")): + col, heap = _pair(c, tmpl) + expect.text(col, heap, label) + # the premise behind the ordered comparisons, as lib.sh's pgc_check_ordered_oracle + fwd = _hash(c, SEQ_HASH, "SELECT g FROM generate_series(1,5) g ORDER BY g") + rev = _hash(c, SEQ_HASH, "SELECT g FROM generate_series(1,5) g ORDER BY g DESC") + again = _hash(c, SEQ_HASH, "SELECT g FROM generate_series(1,5) g ORDER BY g") + sfwd = _hash(c, SET_HASH, "SELECT g FROM generate_series(1,5) g ORDER BY g") + srev = _hash(c, SET_HASH, "SELECT g FROM generate_series(1,5) g ORDER BY g DESC") + expect.text("yes" if fwd != rev else "no", "yes", "premise: the ordered oracle is order-sensitive") + expect.text("yes" if fwd == again else "no", "yes", "premise: the ordered oracle agrees with itself") + expect.text("yes" if sfwd == srev else "no", "yes", "control: the set oracle is order-blind by design") + col, heap = _pair(c, "SELECT k, v FROM %T ORDER BY k, v", SEQ_HASH) + expect.text(col, heap, "pre order-by") + for label, tmpl in (("pre aggregate", "SELECT k, count(*), sum(v) FROM %T GROUP BY k"), + ("pre star", "SELECT * FROM %T WHERE id % 997 = 0")): + col, heap = _pair(c, tmpl) + expect.text(col, heap, label) + + narrow = "SELECT sum(v) FROM t_col WHERE k BETWEEN 100 AND 120" + plan = "\n".join(r[0] for r in c.execute(f"EXPLAIN (COSTS OFF) {narrow}")) + expect.text("yes" if "Columnar Projected Columns" in plan else "no", "yes", + "the plan under test is a columnar custom scan") + before = _vectors_skipped(c, narrow) + + c.execute("SELECT pgcolumnar.vacuum_sorted('t_col', 'k')") + for label, tmpl in (("post range", "SELECT id, v, t FROM %T WHERE k BETWEEN 100 AND 120"), + ("post equality", "SELECT id FROM %T WHERE k = 500")): + col, heap = _pair(c, tmpl) + expect.text(col, heap, label) + col, heap = _pair(c, "SELECT k, v FROM %T ORDER BY k, v", SEQ_HASH) + expect.text(col, heap, "post order-by") + for label, tmpl in (("post aggregate", "SELECT k, count(*), sum(v) FROM %T GROUP BY k"), + ("post star", "SELECT * FROM %T WHERE id % 997 = 0")): + col, heap = _pair(c, tmpl) + expect.text(col, heap, label) + + after = _vectors_skipped(c, narrow) + expect.num(before, 0, "pre-sort prunes little") + expect.text("more" if after > before else f"removed_before={before} removed_after={after}", "more", + "post-sort prunes more chunk groups") + + +def test_a_multi_key_sort_reclaims_and_rebuilds_the_index(sp, expect): + c = sp + c.execute("SELECT pgcolumnar.vacuum_sorted('t_col', 'k', 'v')") + col, heap = _pair(c, "SELECT id, v FROM %T WHERE k BETWEEN 100 AND 120") + expect.text(col, heap, "multi-key range") + + c.execute("CREATE INDEX t_col_id_idx ON t_col (id)") + c.execute("DELETE FROM t_heap WHERE id % 2 = 0") + c.execute("DELETE FROM t_col WHERE id % 2 = 0") + c.execute("SELECT pgcolumnar.vacuum_sorted('t_col', 'k')") + expect.text(_one(c, "SELECT count(*) FROM t_col"), "25000", "live count after delete+sort") + for label, tmpl in (("count matches heap", "SELECT count(*) FROM %T"), ("sum matches heap", "SELECT sum(v) FROM %T")): + col, heap = _pair(c, tmpl) + expect.text(col, heap, label) + # forced and asserted: with seqscan alone off the point lookup never read the index + expect.text(_forced(c, "SELECT t FROM t_col WHERE id = 4999", explain=True), "yes", + "premise: the point lookup reads t_col_id_idx") + expect.text(_forced(c, "SELECT t FROM t_col WHERE id = 4999"), "r4999", "index point lookup after sort") + expect.text(_forced(c, "SELECT count(*) FROM t_col WHERE id = 5000", explain=True), "yes", + "premise: the deletion lookup reads t_col_id_idx") + expect.text(_forced(c, "SELECT count(*) FROM t_col WHERE id = 5000"), "0", "index sees deletion after sort") + + h1 = _hash(c, SET_HASH, "SELECT * FROM t_col") + c.execute("SELECT pgcolumnar.vacuum_sorted('t_col', 'k')") + expect.text(_hash(c, SET_HASH, "SELECT * FROM t_col"), h1, "idempotent sorted rewrite") + + +def test_vacuum_sorted_refuses_what_it_cannot_sort(sp, expect): + c = sp + expect.text(_refused(c, "SELECT pgcolumnar.vacuum_sorted('t_heap', 'k')"), "42809", "reject non-columnar table") + expect.text(_refused(c, "SELECT pgcolumnar.vacuum_sorted('t_col', 'nope')"), "42703", "reject unknown column") + expect.text(_refused(c, "SELECT pgcolumnar.vacuum_sorted('t_col')"), "22023", "reject no sort columns") + c.execute("CREATE TABLE t_js (a int, j json) USING pgcolumnar") + c.execute("INSERT INTO t_js SELECT g, '{}'::json FROM generate_series(1,10) g") + expect.text(_refused(c, "SELECT pgcolumnar.vacuum_sorted('t_js', 'j')"), "42883", "reject non-sortable type") diff --git a/test/sorted_projection.sh b/test/sorted_projection.sh index f2d8367a0..3209611bd 100755 --- a/test/sorted_projection.sh +++ b/test/sorted_projection.sh @@ -29,13 +29,20 @@ groups_removed() { | grep -F "Vectors Skipped" | grep -oE '[0-9]+$' | head -1 } +# expect_error LABEL SQL SQLSTATE: the refusal must be THIS one, by its SQLSTATE. +# +# Any error used to count. Measured (#1341): with vacuum_sorted's columnar check +# removed, the heap table still raised -- XX001, reading the heap's file as +# columnar storage -- and "reject non-columnar table" stayed green on a refusal +# that was no longer there. And a red was recorded as "$label (expected error)", +# a name the passing run never records, so the ledger's row could not be dated; +# it is recorded under the label now. expect_error() { - local label="$1" sql="$2" - if psql_run "$sql" >/dev/null 2>&1; then - check "$label (expected error)" "succeeded" "error" - else - check "$label" "error" "error" - fi + local label="$1" sql="$2" want="$3" out state + out="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -X -At -v ON_ERROR_STOP=1 -v VERBOSITY=sqlstate -c "$sql" 2>&1 >/dev/null)" + state="$(sed -n 's/^ERROR: *\([0-9A-Z]\{5\}\).*/\1/p' <<<"$out" | head -1)" + check "$label" "${state:-succeeded}" "$want" } # Many small chunk groups; the sort key k is scrambled relative to insert order @@ -105,10 +112,28 @@ check "live count after delete+sort" "$(q "SELECT count(*) FROM t_col;")" "25000 diff_query "count matches heap" "SELECT count(*) FROM %T" diff_query "sum matches heap" "SELECT sum(v) FROM %T" # SET prints its own command tag; the scalar value is the last line. +# +# The custom scan off too, and a premise per arm that it reads t_col_id_idx: +# enable_seqscan alone leaves the columnar custom scan on. Measured (#1339): with +# vacuum_sorted's reindex removed, so the index points at the old row numbers, +# the point lookup stayed green, because it never read the index. The deletion +# lookup plans as an Index Only Scan (it reads only id), which still asks each +# row whether it is live, so the premise accepts either index node. +IDXON="SET enable_seqscan=off; SET enable_bitmapscan=off; SET pgcolumnar.enable_custom_scan=off;" +reads_index() { # reads_index QUERY -> yes, or the plan's scan line + local plan + plan="$(q "$IDXON EXPLAIN (COSTS OFF) $1")" + case "$plan" in + *"Index Scan using t_col_id_idx"*|*"Index Only Scan using t_col_id_idx"*) echo yes ;; + *) echo "no: $(grep -m1 'Scan' <<<"$plan" | sed 's/^ *//')" ;; + esac +} +check "premise: the point lookup reads t_col_id_idx" "$(reads_index "SELECT t FROM t_col WHERE id = 4999")" "yes" check "index point lookup after sort" \ - "$(q "SET enable_seqscan=off; SELECT t FROM t_col WHERE id = 4999;" | tail -1)" "r4999" + "$(q "$IDXON SELECT t FROM t_col WHERE id = 4999;" | tail -1)" "r4999" +check "premise: the deletion lookup reads t_col_id_idx" "$(reads_index "SELECT count(*) FROM t_col WHERE id = 5000")" "yes" check "index sees deletion after sort" \ - "$(q "SET enable_seqscan=off; SELECT count(*) FROM t_col WHERE id = 5000;" | tail -1)" "0" + "$(q "$IDXON SELECT count(*) FROM t_col WHERE id = 5000;" | tail -1)" "0" # --------------------------------------------------------------------------- # Idempotence: sorting an already-sorted table changes nothing observable. @@ -123,12 +148,12 @@ check "idempotent sorted rewrite" "$h2" "$h1" # Argument validation. # --------------------------------------------------------------------------- echo "-- argument validation" -expect_error "reject non-columnar table" "SELECT pgcolumnar.vacuum_sorted('t_heap', 'k');" -expect_error "reject unknown column" "SELECT pgcolumnar.vacuum_sorted('t_col', 'nope');" -expect_error "reject no sort columns" "SELECT pgcolumnar.vacuum_sorted('t_col');" +expect_error "reject non-columnar table" "SELECT pgcolumnar.vacuum_sorted('t_heap', 'k');" 42809 +expect_error "reject unknown column" "SELECT pgcolumnar.vacuum_sorted('t_col', 'nope');" 42703 +expect_error "reject no sort columns" "SELECT pgcolumnar.vacuum_sorted('t_col');" 22023 q "DROP TABLE IF EXISTS t_js;" >/dev/null q "CREATE TABLE t_js (a int, j json) USING pgcolumnar;" >/dev/null q "INSERT INTO t_js SELECT g, '{}'::json FROM generate_series(1,10) g;" >/dev/null -expect_error "reject non-sortable type" "SELECT pgcolumnar.vacuum_sorted('t_js', 'j');" +expect_error "reject non-sortable type" "SELECT pgcolumnar.vacuum_sorted('t_js', 'j');" 42883 pgc_summary