diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index eabc7e55f..6ffcdc5a5 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2426,14 +2426,21 @@ native_decode_gating native_decode_gating the needle removes exactly the non-mat native_decode_gating native_decode_gating the needle skips exactly the vectors holding no match 15;16;17;18;19 2026-10-16 columnar_reader.c: the per-vector qual evaluation between the decode passes never runs;columnar_reader.c: the per-vector qual evaluator marks every vector skipped native_decode_gating native_decode_gating the nomatch query returns nothing 15;16;17;18;19 never - native_decode_gating native_decode_gating the zero-match query removes every row by filter, counted once 15;16;17;18;19 never - -native_delete_visibility_paths native_delete_visibility_paths a DELETE whose qual deletes another row of the table succeeds and leaves heap's rows (#1394) 15;16;17;18;19 2026-10-14 PgColumnarUpsertDeleteVector without the command advance (main before #1394's fix) -native_delete_visibility_paths native_delete_visibility_paths a deleted row is invisible via the index 15;16;17;18;19 never - +native_delete_visibility_paths native_delete_visibility_paths a DELETE whose qual deletes another row of the table succeeds and leaves heap's rows (#1394) 15;16;17;18;19 2026-10-16 PgColumnarUpsertDeleteVector without the command advance (main before #1394's fix);columnar_reader.c: the row-at-a-time scan reads a deleted row as live +native_delete_visibility_paths native_delete_visibility_paths a deleted row is invisible via the index 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row ignores the flushed delete vector native_delete_visibility_paths native_delete_visibility_paths a live row is visible via the index 15;16;17;18;19 never - -native_delete_visibility_paths native_delete_visibility_paths a row deleted in-transaction is invisible in the same transaction 15;16;17;18;19 never - +native_delete_visibility_paths native_delete_visibility_paths a row deleted in-transaction is invisible in the same transaction 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row ignores the flushed delete vector native_delete_visibility_paths native_delete_visibility_paths aggregate over the whole table sees the live set 15;16;17;18;19 never - -native_delete_visibility_paths native_delete_visibility_paths index / bitmap scan sees the live set 15;16;17;18;19 never - -native_delete_visibility_paths native_delete_visibility_paths index-only scan sees the live set 15;16;17;18;19 never - -native_delete_visibility_paths native_delete_visibility_paths sequential scan sees the live set (row-emitting path) 15;16;17;18;19 never - +native_delete_visibility_paths native_delete_visibility_paths index / bitmap scan sees the live set 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row ignores the flushed delete vector +native_delete_visibility_paths native_delete_visibility_paths index-only scan sees the live set 15;16;17;18;19 2026-10-16 columnar_reader.c: pgcolumnar_fetch_row ignores the flushed delete vector +native_delete_visibility_paths native_delete_visibility_paths premise: the deletes span more than one row group 15;16;17;18;19 never - +native_delete_visibility_paths native_delete_visibility_paths premise: the index arm really plans an Index Scan 15;16;17;18;19 never - +native_delete_visibility_paths native_delete_visibility_paths premise: the index-only arm really plans an Index Only Scan 15;16;17;18;19 never - +native_delete_visibility_paths native_delete_visibility_paths premise: the sequential arm really plans a Seq Scan 15;16;17;18;19 never - +native_delete_visibility_paths native_delete_visibility_paths premise: the table holds all 5000 rows 15;16;17;18;19 never - +native_delete_visibility_paths native_delete_visibility_paths premise: the whole-table aggregate stays on the columnar path 15;16;17;18;19 never - +native_delete_visibility_paths native_delete_visibility_paths sequential scan sees the live set (row-emitting path) 15;16;17;18;19 2026-10-16 columnar_reader.c: the row-at-a-time scan reads a deleted row as live +native_delete_visibility_paths native_delete_visibility_paths the rolled-back delete left the row in place 15;16;17;18;19 never - native_dict_underfill native_dict_underfill backend survives the under-filled DICT chunk 15;16;17;18;19 2026-10-16 columnar_encoding.c: the DICT fill refusal replaced by a backend crash native_dict_underfill native_dict_underfill premise: the inflation rewrote the DICT chunk's descriptor 15;16;17;18;19 never - native_dict_underfill native_dict_underfill the text column is DICT-encoded (precondition) 15;16;17;18;19 never - @@ -2696,6 +2703,19 @@ native_groupagg_batch native_groupagg_batch toggle-differential: two keys [premi native_groupagg_batch native_groupagg_batch two keys: answers match the heap mirror 15;16;17;18;19 2026-10-16 columnar_vector.c: the grouped fold's gather drops the NULL-key flag, so the NULL group folds into k = 0 (jdatcmd's #1485 review) native_groupagg_batch native_groupagg_batch two keys: the fold ran (work done) 15;16;17;18;19 2026-10-16 columnar_vector.c: pgcolumnar_groupagg_build never calls the grouped batch fold (the removal proof) native_groupagg_batch native_groupagg_batch two keys: the grouped node is planned 15;16;17;18;19 never - +native_index native_index duplicate id rejected 15;16;17;18;19 2026-10-16 columnar_tableam.c: pgcolumnar_index_fetch_tuple fetches the neighbouring row number;columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row +native_index native_index index IN lookup 15;16;17;18;19 2026-10-16 columnar_tableam.c: pgcolumnar_index_fetch_tuple fetches the neighbouring row number;columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row +native_index native_index index lookup miss returns nothing 15;16;17;18;19 never - +native_index native_index index point lookup 15;16;17;18;19 2026-10-16 columnar_tableam.c: pgcolumnar_index_fetch_tuple fetches the neighbouring row number;columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row +native_index native_index index range aggregate 15;16;17;18;19 2026-10-16 columnar_tableam.c: pgcolumnar_index_fetch_tuple fetches the neighbouring row number;columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row +native_index native_index premise: the index IN lookup fetches through n_pkey 15;16;17;18;19 never - +native_index native_index premise: the index lookup miss fetches through n_pkey 15;16;17;18;19 never - +native_index native_index premise: the index point lookup fetches through n_pkey 15;16;17;18;19 never - +native_index native_index premise: the index range aggregate fetches through n_pkey 15;16;17;18;19 never - +native_index native_index row count 15;16;17;18;19 never - +native_index native_index row count unchanged after rejected dup 15;16;17;18;19 2026-10-16 columnar_tableam.c: pgcolumnar_index_fetch_tuple fetches the neighbouring row number;columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row +native_index native_index unique index admits a new key 15;16;17;18;19 never - +native_index native_index unique index rejects duplicate 15;16;17;18;19 2026-10-16 columnar_tableam.c: pgcolumnar_index_fetch_tuple fetches the neighbouring row number;columnar_tableam.c: pgcolumnar_index_fetch_tuple finds no row native_index_projection native_index_projection a parallel build projects one column of twenty, not all twenty 15;16;17;18;19 never - native_index_projection native_index_projection a parallel expression build projects its expression's columns 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index build's projection omits ii_Expressions' columns native_index_projection native_index_projection a parallel partial build projects the predicate's columns too 15;16;17;18;19 2026-10-16 columnar_tableam.c: the index build's projection omits ii_Predicate's columns @@ -2828,6 +2848,38 @@ native_parquet_flba native_parquet_flba pyarrow decimal128 values are exact 15;1 native_parquet_flba native_parquet_flba pyarrow uuid reads as uuid 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read native_parquet_flba native_parquet_flba uuid round-trips through export/read 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read native_parquet_flba native_parquet_flba whole-row round-trip == source 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a FIXED_LEN_BYTE_ARRAY uuid's bytes are reversed on read +native_parquet_multifile native_parquet_multifile EXPLAIN reports 3 files 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile EXPLAIN sums row groups across files 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile FDW(directory) == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile FIFO named *.parquet is skipped, does not block 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension;columnar_parquet_reader.c: any non-directory entry is a candidate, a FIFO included +native_parquet_multifile native_parquet_multifile _temporary and dot directories are not walked 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension;columnar_parquet_reader.c: the walk descends into names beginning with _ or . +native_parquet_multifile native_parquet_multifile a file whose name starts with _ is skipped too 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension;columnar_parquet_reader.c: the walk descends into names beginning with _ or . +native_parquet_multifile native_parquet_multifile a symlink loop terminates and does not multiply rows 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension;columnar_parquet_reader.c: the walk descends a symlinked directory +native_parquet_multifile native_parquet_multifile a symlinked directory is not descended 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension;columnar_parquet_reader.c: the walk descends a symlinked directory +native_parquet_multifile native_parquet_multifile a symlinked file is still read 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile a tree deeper than the bound is rejected, not truncated 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: the walk has no depth bound +native_parquet_multifile native_parquet_multifile count(*) over directory is correct 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile count(*) over directory reads 0 columns 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile cross-file predicate result == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile dangling symlink in a glob errors, not skipped 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a path that cannot be stat'ed is dropped instead of handed to the reader +native_parquet_multifile native_parquet_multifile dangling symlink in directory errors, not skipped 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a path that cannot be stat'ed is dropped instead of handed to the reader +native_parquet_multifile native_parquet_multifile directory ignores a subdir named *.parquet 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile directory of only *.parquet subdirs errors 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a directory holding no .parquet files resolves to no files instead of an error +native_parquet_multifile native_parquet_multifile directory read ignores non-parquet files 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile empty directory errors 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a directory holding no .parquet files resolves to no files instead of an error +native_parquet_multifile native_parquet_multifile glob matching only directories errors 15;16;17;18;19 never - +native_parquet_multifile native_parquet_multifile glob part-[01] == first two files 15;16;17;18;19 never - +native_parquet_multifile native_parquet_multifile glob skips a directory match, reads the files 15;16;17;18;19 never - +native_parquet_multifile native_parquet_multifile import_parquet(directory) == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile import_parquet(directory) row count 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile nested tree reads every level as one relation 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile nested tree row count 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile non-matching glob errors 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a glob matching nothing resolves to no files instead of an error +native_parquet_multifile native_parquet_multifile one column over directory reads 1 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile predicate skips the two non-matching files' groups 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile read_parquet(directory) == oracle 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked entry is read whatever its extension +native_parquet_multifile native_parquet_multifile schema-mismatched file in directory errors 15;16;17;18;19 never - +native_parquet_multifile native_parquet_multifile tree read order follows the sorted full path 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: a walked directory's files are not sorted;columnar_parquet_reader.c: a walked entry is read whatever its extension native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: predicate skips 3 of 4 groups 15;16;17;18;19 2026-10-16 columnar_parquet_reader.c: INT64 statistics are not decoded, so an INT64-backed DECIMAL never skips;columnar_parquet_reader.c: pqfdw_compute_skip never marks a row group skippable native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: skipping did not drop rows 15;16;17;18;19 never - native_parquet_pushdown native_parquet_pushdown INT64-backed DECIMAL: unfiltered scan skips nothing 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index cda61de08..154cd50aa 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 195 +suites_not_covered 193 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -3416,4 +3416,44 @@ suites_not_covered 195 # never 3002 + 4 = 3006 # dated 1261 + 11 = 1272 # 3006 + 1272 == 4278 -checks_never_observed_red 3006 +# +# #1341, native_parquet_multifile seeded, with its twin (OffgridwithJD): 3006 -> 3010, +# and suites_not_covered 195 -> 194. The suite had no rows. It enters with all 32 +# checks from five-major runs (32/32 on each of 15-19). 28 are dated by nine +# mutations of the directory walk and glob expansion: an empty directory and an +# unmatched glob reading as nothing, an unstat-able path dropped, a FIFO opened, +# _ and . names walked, a symlinked directory descended, the files unsorted, no +# depth bound, and every extension read. Four arrive never: the three glob arms +# (subset, directory match skipped, only directories) and the schema mismatch. +# +# ledger rows 4278 + 32 = 4310 +# never 3006 + 4 = 3010 +# dated 1272 + 28 = 1300 +# 3010 + 1300 == 4310 +# +# #1341, native_index seeded, with its twin (OffgridwithJD): 3010 -> 3017, and +# suites_not_covered 194 -> 193. The suite had no rows. It enters with all 13 +# checks from five-major runs (13/13 on each of 15-19). 6 are dated by two +# mutations of pgcolumnar_index_fetch_tuple: the neighbouring row fetched, and no +# row found. Seven arrive never: the four index-path premises, the row count, the +# lookup miss (which no fetch mutation turns into a hit), and the unique index +# admitting a new key. +# +# ledger rows 4310 + 13 = 4323 +# never 3010 + 7 = 3017 +# dated 1300 + 6 = 1306 +# 3017 + 1306 == 4323 +# +# #1339, native_delete_visibility_paths' shell arms take the paths they name +# (OffgridwithJD): 3017 -> 3019. Seven checks are new, all never: the six path and +# fixture premises the twin already had, and the rolled-back row. Five arms that +# entered never are now dated: the index / bitmap, index-only, deleted-row-via-index +# and in-transaction arms by pgcolumnar_fetch_row ignoring the delete vector, and +# the sequential arm by the row scan reading a deleted row as live. The #1394 arm +# gains the second in its history. +# +# ledger rows 4323 + 7 = 4330 +# never 3017 + 7 - 5 = 3019 +# dated 1306 + 5 = 1311 +# 3019 + 1311 == 4330 +checks_never_observed_red 3019 diff --git a/test/native_delete_visibility_paths.sh b/test/native_delete_visibility_paths.sh index 6cddc1c45..38776eb6d 100755 --- a/test/native_delete_visibility_paths.sh +++ b/test/native_delete_visibility_paths.sh @@ -26,8 +26,14 @@ q "CREATE EXTENSION IF NOT EXISTS pgcolumnar;" >/dev/null q "SET pgcolumnar.stripe_row_limit=1000; CREATE TABLE t (id int, v int) USING pgcolumnar; INSERT INTO t SELECT g, g FROM generate_series(1,5000) g; - CREATE UNIQUE INDEX t_id ON t (id); - DELETE FROM t WHERE id % 7 = 0;" >/dev/null + CREATE UNIQUE INDEX t_id ON t (id);" >/dev/null +check "premise: the table holds all 5000 rows" "$(q 'SELECT count(*) FROM t;')" "5000" +q "DELETE FROM t WHERE id % 7 = 0;" >/dev/null +# Each group builds its own mask, so one group would test the fold of one bitmap. +check "premise: the deletes span more than one row group" \ + "$(q "SELECT (count(*) >= 2)::int FROM pgcolumnar.storage s + JOIN pgcolumnar.row_group rg USING (storage_id) + WHERE s.relation_oid = 't'::regclass;")" "1" # Live count = 5000 - floor(5000/7) = 5000 - 714 = 4286. Five row groups. LIVE=4286 @@ -36,27 +42,58 @@ LIVE=4286 # multi-statement send. last() { q "$1" | tail -1; } -# A qual on the non-indexed column forces a real row-emitting sequential scan -# (which applies the delete mask per row), not the unqualified count(*) metadata -# fast-path (which subtracts delete counts and never reaches the bit test). +# EACH ARM FORCES THE PATH IT IS NAMED FOR, AND ASSERTS THE NODE FIRST (#1339). +# The arms set enable_seqscan/enable_indexonlyscan and trusted the names, but none +# of the enable_* scan GUCs governs Custom Scan (PgColumnarScan): measured, the +# index / bitmap and index-only arms planned as the columnar custom scan, the same +# path as the sequential arm, so three of four named paths were one path counted +# three times. pgcolumnar.enable_custom_scan=off is what moves the plan; the +# pytest twin has forced each path this way since it was written. +NO_COLUMNAR="SET pgcolumnar.enable_custom_scan = off;" +path_plan() { # path_plan SETS QUERY NODE -> yes, or the plan's node lines + local plan + plan="$(q "$1 EXPLAIN (COSTS OFF) $2")" + case "$plan" in + *"$3"*) echo yes ;; + *) echo "no: $(grep -E 'Scan|Aggregate' <<<"$plan" | sed 's/^ *//' | tr '\n' '/')" ;; + esac +} + +SEQ="$NO_COLUMNAR SET enable_indexscan=off; SET enable_bitmapscan=off; SET enable_indexonlyscan=off;" +check "premise: the sequential arm really plans a Seq Scan" \ + "$(path_plan "$SEQ" "SELECT count(*) FROM t WHERE v >= 0" "Seq Scan")" yes check "sequential scan sees the live set (row-emitting path)" \ - "$(last 'SET enable_indexscan=off; SET enable_bitmapscan=off; SET enable_indexonlyscan=off; SELECT count(*) FROM t WHERE v >= 0;')" "$LIVE" + "$(last "$SEQ SELECT count(*) FROM t WHERE v >= 0;")" "$LIVE" + +IDX="$NO_COLUMNAR SET enable_seqscan=off; SET enable_indexonlyscan=off;" +check "premise: the index arm really plans an Index Scan" \ + "$(path_plan "$IDX" "SELECT count(*) FROM t WHERE id >= 0" "Index Scan")" yes check "index / bitmap scan sees the live set" \ - "$(last 'SET enable_seqscan=off; SET enable_indexonlyscan=off; SELECT count(*) FROM t WHERE id >= 0;')" "$LIVE" + "$(last "$IDX SELECT count(*) FROM t WHERE id >= 0;")" "$LIVE" + +IOS="$NO_COLUMNAR SET enable_seqscan=off; SET enable_indexonlyscan=on;" +check "premise: the index-only arm really plans an Index Only Scan" \ + "$(path_plan "$IOS" "SELECT count(id) FROM t WHERE id >= 0" "Index Only Scan")" yes check "index-only scan sees the live set" \ - "$(last 'SET enable_seqscan=off; SET enable_indexonlyscan=on; SELECT count(id) FROM t WHERE id >= 0;')" "$LIVE" + "$(last "$IOS SELECT count(id) FROM t WHERE id >= 0;")" "$LIVE" + +# The columnar path, left to the planner: a different consumer of the same fold. +check "premise: the whole-table aggregate stays on the columnar path" \ + "$(path_plan "" "SELECT count(*) FROM t" "Custom Scan")" yes check "aggregate over the whole table sees the live set" \ "$(q 'SELECT count(*) FROM t;')" "$LIVE" # A specific deleted row is invisible, and its live neighbour visible, via the -# index fetch (which reaches the same bit test through a different path). +# index (which reaches the same bit test through a different path). check "a deleted row is invisible via the index" \ - "$(last 'SET enable_seqscan=off; SELECT count(*) FROM t WHERE id = 700;')" "0" + "$(last "$NO_COLUMNAR SET enable_seqscan=off; SELECT count(*) FROM t WHERE id = 700;")" "0" check "a live row is visible via the index" \ - "$(last 'SET enable_seqscan=off; SELECT count(*) FROM t WHERE id = 701;')" "1" + "$(last "$NO_COLUMNAR SET enable_seqscan=off; SELECT count(*) FROM t WHERE id = 701;")" "1" # The buffered (same-transaction) path: delete more inside a txn, then read it back. check "a row deleted in-transaction is invisible in the same transaction" \ "$(q 'BEGIN; DELETE FROM t WHERE id = 8; SELECT count(*) FROM t WHERE id = 8; ROLLBACK;' | grep -E '^[0-9]+$' | head -1)" "0" +# and the rollback really rolled back, so the arm above is not a delete that persisted +check "the rolled-back delete left the row in place" "$(q 'SELECT count(*) FROM t WHERE id = 8;')" "1" # A DELETE whose qual deletes another row of the same table (#1394). The inner DELETE's # scan start writes the outer statement's buffered marks, and its statement end writes diff --git a/test/native_index.sh b/test/native_index.sh index 2c5908863..efee68c8e 100755 --- a/test/native_index.sh +++ b/test/native_index.sh @@ -23,15 +23,31 @@ psql_run "INSERT INTO n $GEN;" check "row count" "$(q 'SELECT count(*) FROM n;')" "8000" -# Force the index path (seqscan off) in a single session (two -c, quiet so the SET -# tag is not printed), and confirm scalar results match the heap oracle. Filtered -# aggregates fall back from the zone-map path to an index scan that fetches each -# row via PgColumnarReadRowByNumber. +# Force the index path in a single session (two -c, quiet so the SET tag is not +# printed), and confirm scalar results match the heap oracle. Each lookup fetches +# its rows by number through the index. +# +# enable_seqscan=off was all this set, and it leaves the columnar custom scan on. +# Measured (#1341): the range aggregate planned as Custom Scan (PgColumnarScan), +# so it fetched nothing by row number, and with the index fetch reading the +# neighbouring row it stayed green. The custom scan is off now, and index-only +# and bitmap scans with it, so every lookup is an Index Scan that fetches; each +# asserts that plan first. +FORCE_INDEX="SET enable_seqscan=off; SET enable_bitmapscan=off; + SET enable_indexonlyscan=off; SET pgcolumnar.enable_custom_scan=off;" iscan() { env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ - -d "$PGC_DB" -q -At -c "SET enable_seqscan=off" -c "$1" 2>/dev/null + -d "$PGC_DB" -q -At -c "$FORCE_INDEX" -c "$1" 2>/dev/null } +for _ni in "point lookup:SELECT v FROM n WHERE id = 4242" \ + "range aggregate:SELECT count(*), sum(k), min(v), max(v) FROM n WHERE id BETWEEN 100 AND 400" \ + "IN lookup:SELECT count(*), sum(k) FROM n WHERE id IN (1,2000,4000,6000,8000)" \ + "lookup miss:SELECT count(*) FROM n WHERE id = 999999"; do + check "premise: the index ${_ni%%:*} fetches through n_pkey" \ + "$(pgc_uses_row_fetch "$FORCE_INDEX" "${_ni#*:}")" yes +done + check "index point lookup" \ "$(iscan 'SELECT v FROM n WHERE id = 4242;')" \ "$(q 'SELECT v FROM h WHERE id = 4242;')" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 4b984cccc..a82be591b 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -216,6 +216,8 @@ behaviour, the source of that number is named. - [168. test_native_fetch_cache.py: the fetch-by-row-number cache](#168-test_native_fetch_cachepy-the-fetch-by-row-number-cache) - [169. test_native_parquet_pushdown.py: the Parquet FDW skips row groups its statistics rule out](#169-test_native_parquet_pushdownpy-the-parquet-fdw-skips-row-groups-its-statistics-rule-out) - [170. test_native_parquet_flba.py: uuid and DECIMAL read from fixed-length byte arrays](#170-test_native_parquet_flbapy-uuid-and-decimal-read-from-fixed-length-byte-arrays) +- [171. test_native_parquet_multifile.py: directories and globs read as one relation](#171-test_native_parquet_multifilepy-directories-and-globs-read-as-one-relation) +- [172. test_native_index.py: index lookups and unique enforcement fetch by row number](#172-test_native_indexpy-index-lookups-and-unique-enforcement-fetch-by-row-number) ## 1. How to read a test in here @@ -7746,3 +7748,28 @@ The pytest twin of `test/native_parquet_flba.sh` (ported for #1341), every check | `test_our_export_reads_back_through_every_surface` | uuid, numeric and whole rows round-trip through `read_parquet`, `import_parquet` and the FDW | | `test_pyarrow_written_flba_reads_exactly` | both pyarrow uuids read in order, decimal128 values are exact, and a crafted scale is refused at bind with the backend alive | | `test_a_decimal_held_in_an_integer_reads_as_numeric` | INT32 and INT64 DECIMALs read at their scale with their null, `parquet_schema` advises numeric, and bigint still binds to the unscaled integer | + +## 171. test_native_parquet_multifile.py: directories and globs read as one relation + +The pytest twin of `test/native_parquet_multifile.sh` (ported for #1341), every check by the shell's name. A directory reads every `*.parquet` file below it as one relation, and a glob expands the same way. The union must equal a single-file oracle through `read_parquet`, `import_parquet` and the FDW. Pushdown and the EXPLAIN counters aggregate across files. + +Several inputs must fail: an empty directory, a glob matching nothing, a schema-mismatched file, a dangling symlink, and a tree past the depth bound. A FIFO and a directory named `*.parquet` are skipped. The walk skips names that start with `_` or `.`, and it does not follow a symlinked directory, so a loop terminates. It still follows a symlinked file, and it orders files by full path. Reads that could block are bounded, as the shell's `timeout` bounds them. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_directories_and_globs_read_as_one_relation` | a directory and a glob equal the oracle on every surface, pushdown and counters aggregate, and every error case fails | +| `test_a_fifo_named_parquet_is_skipped_not_opened` | a FIFO named `*.parquet` is skipped rather than opened, within a bound; declined as "the FIFO case" where mkfifo is unavailable | +| `test_the_recursive_walk` | a nested tree reads as one relation in sorted path order, `_`/`.` names and symlinked directories are not walked, a symlinked file is read, and the depth bound errors | + +## 172. test_native_index.py: index lookups and unique enforcement fetch by row number + +The pytest twin of `test/native_index.sh` (ported for #1341), every check by the shell's name. An index scan, unique enforcement and primary-key enforcement all fetch rows by number (Phase D6a). Before that, an index scan on a native table returned no rows, and unique was silently unenforced. Lookups are compared with a heap mirror under a forced index path. The columnar custom scan is off as well as seqscan, because with only seqscan off the range aggregate planned as the custom scan. Each lookup first asserts that its plan is an Index Scan through `n_pkey`. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_index_lookups_match_the_heap` | four lookups plan as Index Scans and match the heap, and a duplicate primary key is refused with the row count unchanged | +| `test_a_standalone_unique_index_enforces` | a standalone unique index refuses a duplicate and admits a new key | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 007a993de..fdbaa4406 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -803,4 +803,8 @@ guard_tests 438 # four tests, by collection over the cluster files on this tree. # cluster_tests 734 -> 737 for #1341's test_native_parquet_flba.py, the twin of native_parquet_flba.sh: # three tests, by collection over the cluster files on this tree. -cluster_tests 737 +# cluster_tests 737 -> 740 for #1341's test_native_parquet_multifile.py, the twin of native_parquet_multifile.sh: +# three tests, by collection over the cluster files on this tree. +# cluster_tests 740 -> 742 for #1341's test_native_index.py, the twin of native_index.sh: +# two tests, by collection over the cluster files on this tree. +cluster_tests 742 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 4af427309..6708c3f05 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -135,12 +135,14 @@ "native_fetch_projection", "native_groupagg", "native_groupagg_batch", + "native_index", "native_index_fetch_stripe_cost", "native_index_projection", "native_join_vector_agg", "native_ownership", "native_parquet_dict_oob", "native_parquet_flba", + "native_parquet_multifile", "native_parquet_pushdown", "native_parquet_schema", "native_projection", diff --git a/test/pytest/test_native_delete_visibility_paths.py b/test/pytest/test_native_delete_visibility_paths.py index 28d39f414..3e443f4ae 100644 --- a/test/pytest/test_native_delete_visibility_paths.py +++ b/test/pytest/test_native_delete_visibility_paths.py @@ -33,8 +33,7 @@ index-only Index Only Scan 4286 So this port forces the path each arm is named for and ASSERTS THE NODE before reading -the count. The shell suite is unchanged -- reported rather than edited, since this pull -request adds no bash check. +the count. The shell suite now does the same, with the same premise names (#1339). """ import psycopg @@ -85,7 +84,7 @@ def test_native_delete_visibility_paths(pgc_conn, expect): cur.execute(f"INSERT INTO t SELECT g, g FROM generate_series(1,{ROWS}) g") cur.execute("CREATE UNIQUE INDEX t_id ON t (id)") cur.execute("SELECT count(*) FROM t") - expect.num(cur.fetchone()[0], ROWS, f"premise: the table holds all {ROWS} rows") + expect.num(cur.fetchone()[0], ROWS, "premise: the table holds all 5000 rows") cur.execute("DELETE FROM t WHERE id % 7 = 0") cur.execute( "SELECT count(*) FROM pgcolumnar.storage s " diff --git a/test/pytest/test_native_index.py b/test/pytest/test_native_index.py new file mode 100644 index 000000000..0364bf40e --- /dev/null +++ b/test/pytest/test_native_index.py @@ -0,0 +1,101 @@ +"""Native fetch-by-row-number through an index (Phase D6a, #1341). The pytest twin of +`test/native_index.sh`, every check by the shell's name. + +An index scan, and unique and primary-key enforcement, all route through the fetch by +row number. Before D6a an index scan on a native table returned no rows, and unique +was silently unenforced. Lookups are compared with a heap mirror under a forced index +path. The columnar custom scan is off as well as seqscan, because seqscan alone left +the range aggregate planned as the custom scan, and each lookup asserts first that its +plan is an Index Scan through n_pkey. +""" +import psycopg + +FORCE_INDEX = ("SET enable_seqscan=off", "SET enable_bitmapscan=off", "SET enable_indexonlyscan=off", + "SET pgcolumnar.enable_custom_scan=off") + + +def _one(cur, sql): + cur.execute(sql) + row = cur.fetchone() + return "|".join("" if v is None else str(v) for v in row) if row else "" + + +def _forced(cur, sql): + """-> `sql` under the forced index path, as the shell's iscan; an error is a value.""" + try: + for s in FORCE_INDEX: + cur.execute(s) + return _one(cur, sql) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + finally: + for g in ("enable_seqscan", "enable_bitmapscan", "enable_indexonlyscan", "pgcolumnar.enable_custom_scan"): + cur.execute(f"RESET {g}") + + +def _plan_fetches(cur, sql): + """-> 'yes' when `sql` plans an Index Scan under the forced path, as pgc_uses_row_fetch.""" + try: + for s in FORCE_INDEX: + cur.execute(s) + cur.execute(f"EXPLAIN (COSTS OFF) {sql}") + return "yes" if any("Index Scan using" in r[0] for r in cur.fetchall()) else "no" + finally: + for g in ("enable_seqscan", "enable_bitmapscan", "enable_indexonlyscan", "pgcolumnar.enable_custom_scan"): + cur.execute(f"RESET {g}") + + +def _refused(cur, sql): + """-> 1 when `sql` is refused as a duplicate, as the shell's grep -c counts it.""" + try: + cur.execute(sql) + return 0 + except psycopg.errors.UniqueViolation: + return 1 + except psycopg.Error: + return 0 + + +def test_index_lookups_match_the_heap(pgc_conn, expect): + with pgc_conn.cursor() as cur: + cur.execute("CREATE TABLE h (id int PRIMARY KEY, k bigint, v text)") + cur.execute("CREATE TABLE n (id int PRIMARY KEY, k bigint, v text) USING pgcolumnar") + cur.execute("SELECT pgcolumnar.set_options('n', stripe_row_limit => 2048)") + gen = "SELECT g, (g * 10)::bigint, 'row-' || g FROM generate_series(1, 8000) g" + cur.execute(f"INSERT INTO h {gen}") + cur.execute(f"INSERT INTO n {gen}") + expect.num(int(_one(cur, "SELECT count(*) FROM n")), 8000, "row count") + for label, sql in ( + ("point lookup", "SELECT v FROM n WHERE id = 4242"), + ("range aggregate", "SELECT count(*), sum(k), min(v), max(v) FROM n WHERE id BETWEEN 100 AND 400"), + ("IN lookup", "SELECT count(*), sum(k) FROM n WHERE id IN (1,2000,4000,6000,8000)"), + ("lookup miss", "SELECT count(*) FROM n WHERE id = 999999"), + ): + expect.text(_plan_fetches(cur, sql), "yes", f"premise: the index {label} fetches through n_pkey") + expect.text(_forced(cur, "SELECT v FROM n WHERE id = 4242"), _one(cur, "SELECT v FROM h WHERE id = 4242"), + "index point lookup") + expect.text(_forced(cur, "SELECT count(*), sum(k), min(v), max(v) FROM n WHERE id BETWEEN 100 AND 400"), + _one(cur, "SELECT count(*), sum(k), min(v), max(v) FROM h WHERE id BETWEEN 100 AND 400"), + "index range aggregate") + expect.text(_forced(cur, "SELECT count(*), sum(k) FROM n WHERE id IN (1,2000,4000,6000,8000)"), + _one(cur, "SELECT count(*), sum(k) FROM h WHERE id IN (1,2000,4000,6000,8000)"), + "index IN lookup") + expect.text(_forced(cur, "SELECT count(*) FROM n WHERE id = 999999"), "0", "index lookup miss returns nothing") + + # the fetch is what finds the conflicting row: before D6a duplicates went in + expect.num(_refused(cur, "INSERT INTO n VALUES (4242, 1, 'dup')"), 1, "duplicate id rejected") + expect.num(int(_one(cur, "SELECT count(*) FROM n")), 8000, "row count unchanged after rejected dup") + + +def test_a_standalone_unique_index_enforces(pgc_conn, expect): + with pgc_conn.cursor() as cur: + cur.execute("CREATE TABLE nu (id int, k bigint) USING pgcolumnar") + cur.execute("CREATE UNIQUE INDEX nu_k ON nu (k)") + cur.execute("INSERT INTO nu SELECT g, g FROM generate_series(1,1000) g") + expect.num(_refused(cur, "INSERT INTO nu VALUES (99999, 500)"), 1, "unique index rejects duplicate") + try: + cur.execute("INSERT INTO nu VALUES (99999, 100000)") + errors = 0 + except psycopg.Error: + errors = 1 + expect.num(errors, 0, "unique index admits a new key") diff --git a/test/pytest/test_native_parquet_multifile.py b/test/pytest/test_native_parquet_multifile.py new file mode 100644 index 000000000..ea3babc61 --- /dev/null +++ b/test/pytest/test_native_parquet_multifile.py @@ -0,0 +1,288 @@ +"""Multi-file and directory Parquet reads (Phase G, #1341). The pytest twin of +`test/native_parquet_multifile.sh`, every check by the shell's name. + +A directory reads every *.parquet below it as one relation, a glob expands the same +way, and the union must equal a single-file oracle through read_parquet, +import_parquet and the FDW. Per-file pushdown and the EXPLAIN counters aggregate +across files. An empty directory, a glob matching nothing, a schema-mismatched file, +a dangling symlink and a tree past the depth bound must each fail; a FIFO and a +directory named *.parquet are skipped; the walk skips _ and . names, does not follow +a symlinked directory (so a loop terminates), still follows a symlinked file, and +orders files by full path. +""" +import os +import shutil +import threading + +import psycopg + +COLS = "id int, v int8" +HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) " + "FROM (SELECT _row::text AS t FROM ({}) _row) _s") + + +def _hash(cur, sql): + try: + cur.execute(HASH.format(sql)) + return cur.fetchone()[0] + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary} [{sql}]" + + +def _one(cur, sql): + try: + cur.execute(sql) + return str(cur.fetchone()[0]) + except psycopg.Error as exc: + return f"error: {exc.diag.message_primary}" + + +def _errs(cur, sql): + """-> OK when `sql` raises, as the shell's errs counts any ERROR.""" + try: + cur.execute(sql) + cur.fetchall() + return "NO ERROR" + except psycopg.Error: + return "OK" + + +def _plan(cur, sql): + """-> the EXPLAIN ANALYZE lines, or the error as the one line: a broken read names its red.""" + try: + cur.execute(f"EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) {sql}") + return [r[0] for r in cur.fetchall()] + except psycopg.Error as exc: + return [f"error: {exc.diag.message_primary}"] + + +def _counter(cur, where, label): + for line in _plan(cur, f"SELECT count(*) FROM ftd WHERE {where}"): + if label in line: + digits = "".join(ch if ch.isdigit() else " " for ch in line.split(label, 1)[1]).split() + return digits[0] if digits else "" + return "" + + +def _cols_read(cur, sql): + for line in _plan(cur, sql): + if "Columns Read:" in line: + return "".join(ch if ch.isdigit() else " " for ch in line.split("Columns Read:", 1)[1]).split()[0] + return "" + + +def _rp(path): + return f"SELECT * FROM pgcolumnar.read_parquet('{path}') AS t({COLS})" + + +def _count(path): + return f"SELECT count(*) FROM pgcolumnar.read_parquet('{path}') AS t({COLS})" + + +def _bounded(cluster, sql, seconds, fifo=None): + """-> the first value of `sql` on its own connection, or TIMED OUT after `seconds`. + + The shell runs psql under `timeout`; a cancel is no substitute, because a backend + blocked in open(2) on a FIFO resumes the open after a signal (SA_RESTART) rather + than failing. So the query runs in a thread and is waited for. When it times out + on a FIFO, opening the FIFO for writing releases the blocked reader, so the + backend and the cluster's teardown are not left hanging. + """ + out = [] + + def run(): + try: + with psycopg.connect(cluster.dsn(), autocommit=True) as c: + out.append(str(c.execute(sql).fetchone()[0])) + except psycopg.Error as exc: + out.append(f"error: {exc.diag.message_primary}") + + t = threading.Thread(target=run, daemon=True) + t.start() + t.join(seconds) + if t.is_alive(): + if fifo is not None: + fd = os.open(fifo, os.O_WRONLY | os.O_NONBLOCK) + os.close(fd) + t.join(30) + return "TIMED OUT" + return out[0] if out else "no result" + + +def test_directories_and_globs_read_as_one_relation(pgc_conn, pgc_cluster, server_dir, expect): + pa = expect.require_module("pyarrow", name="pyarrow not available; multi-file suite needs it") + if pa is None: + return + pq = expect.require_module("pyarrow.parquet", name="pyarrow not available; multi-file suite needs it") + w = server_dir + parts = w / "parts" + parts.mkdir() + + def tbl(lo, hi): + return pa.table({"id": pa.array(range(lo, hi), pa.int32()), + "v": pa.array([i * 2 for i in range(lo, hi)], pa.int64())}) + + for i in range(3): + pq.write_table(tbl(i * 1000, i * 1000 + 1000), str(parts / f"part-{i}.parquet"), row_group_size=1000) + (parts / "README.txt").write_text("not parquet\n") + pq.write_table(tbl(0, 3000), str(w / "all.parquet"), row_group_size=1000) + for p in list(parts.iterdir()) + [w / "all.parquet"]: + p.chmod(0o644) + + with pgc_conn.cursor() as cur: + cur.execute("CREATE SERVER IF NOT EXISTS pq_multi FOREIGN DATA WRAPPER pgcolumnar_parquet") + oracle = _hash(cur, _rp(w / "all.parquet")) + expect.text(_hash(cur, _rp(parts)), oracle, "read_parquet(directory) == oracle") + cur.execute(f"CREATE TABLE imp ({COLS})") + _one(cur, f"SELECT pgcolumnar.import_parquet('imp'::regclass, '{parts}')") + expect.text(_hash(cur, "SELECT * FROM imp"), oracle, "import_parquet(directory) == oracle") + expect.text(_one(cur, "SELECT count(*) FROM imp"), "3000", "import_parquet(directory) row count") + cur.execute(f"CREATE FOREIGN TABLE ftd ({COLS}) SERVER pq_multi OPTIONS (path '{parts}')") + expect.text(_hash(cur, "SELECT * FROM ftd"), oracle, "FDW(directory) == oracle") + expect.text(_one(cur, "SELECT count(*) FROM ftd"), "3000", "directory read ignores non-parquet files") + expect.text(_hash(cur, _rp(f"{parts}/part-[01].parquet")), + _hash(cur, _rp(w / "all.parquet") + " WHERE id < 2000"), "glob part-[01] == first two files") + + # id >= 2500 lies only in part-2: the other two files' groups skip + expect.text(_counter(cur, "true", "Files:"), "3", "EXPLAIN reports 3 files") + expect.text(_counter(cur, "true", "Row Groups:"), "3", "EXPLAIN sums row groups across files") + expect.text(_counter(cur, "id >= 2500", "Row Groups Skipped"), "2", + "predicate skips the two non-matching files' groups") + expect.text(_hash(cur, "SELECT * FROM ftd WHERE id >= 2500"), + _hash(cur, _rp(w / "all.parquet") + " WHERE id >= 2500"), "cross-file predicate result == oracle") + expect.text(_cols_read(cur, "SELECT count(*) FROM ftd"), "0", "count(*) over directory reads 0 columns") + expect.text(_cols_read(cur, "SELECT id FROM ftd"), "1", "one column over directory reads 1") + expect.text(_one(cur, "SELECT count(*) FROM ftd"), "3000", "count(*) over directory is correct") + + # ---- error and skip cases + (w / "empty").mkdir() + expect.text(_errs(cur, _rp(w / "empty")), "OK", "empty directory errors") + expect.text(_errs(cur, _rp(f"{parts}/nope-*.parquet")), "OK", "non-matching glob errors") + mixed = w / "mixed" + mixed.mkdir() + shutil.copy(parts / "part-0.parquet", mixed / "a.parquet") + pq.write_table(pa.table({"x": pa.array(["a", "b"])}), str(mixed / "b.parquet")) + for p in mixed.iterdir(): + p.chmod(0o644) + expect.text(_errs(cur, _rp(mixed)), "OK", "schema-mismatched file in directory errors") + + (parts / "sub.parquet").mkdir() + expect.text(_one(cur, "SELECT count(*) FROM ftd"), "3000", "directory ignores a subdir named *.parquet") + (parts / "sub.parquet").rmdir() + (parts / "part-9.parquet").mkdir() + expect.text(_hash(cur, _rp(f"{parts}/part-*.parquet")), oracle, "glob skips a directory match, reads the files") + (parts / "part-9.parquet").rmdir() + + onlydirs = w / "onlydirs" + (onlydirs / "a.parquet").mkdir(parents=True) + (onlydirs / "b.parquet").mkdir() + expect.text(_errs(cur, _rp(f"{onlydirs}/*.parquet")), "OK", "glob matching only directories errors") + expect.text(_errs(cur, _rp(onlydirs)), "OK", "directory of only *.parquet subdirs errors") + + # a path that cannot be stat'ed is reported, never dropped + os.symlink(parts / "does-not-exist.parquet", parts / "broken.parquet") + expect.text(_errs(cur, _count(parts)), "OK", "dangling symlink in directory errors, not skipped") + expect.text(_errs(cur, _count(f"{parts}/*.parquet")), "OK", "dangling symlink in a glob errors, not skipped") + (parts / "broken.parquet").unlink() + + + +def test_a_fifo_named_parquet_is_skipped_not_opened(pgc_cluster, server_dir, expect): + """Its own test because the shell declines this one check when mkfifo is unavailable, + and cannot_run declines a whole test.""" + pa = expect.require_module("pyarrow", name="pyarrow not available; multi-file suite needs it") + if pa is None: + return + pq = expect.require_module("pyarrow.parquet", name="pyarrow not available; multi-file suite needs it") + parts = server_dir / "fifoparts" + parts.mkdir() + for i in range(3): + path = parts / f"part-{i}.parquet" + pq.write_table(pa.table({"id": pa.array(range(i * 1000, i * 1000 + 1000), pa.int32()), + "v": pa.array([j * 2 for j in range(i * 1000, i * 1000 + 1000)], pa.int64())}), + str(path), row_group_size=1000) + path.chmod(0o644) + try: + os.mkfifo(parts / "pipe.parquet") + except OSError as exc: + expect.cannot_run("MISSING_DEPENDENCY", f"mkfifo unavailable ({exc})", name="the FIFO case") + return + (parts / "pipe.parquet").chmod(0o644) + # a FIFO blocks open(2) until a writer appears; the bound is the assertion + expect.text(_bounded(pgc_cluster, _count(parts), 30, fifo=parts / "pipe.parquet"), "3000", + "FIFO named *.parquet is skipped, does not block") + + +def test_the_recursive_walk(pgc_conn, pgc_cluster, server_dir, expect): + pa = expect.require_module("pyarrow", name="pyarrow not available; multi-file suite needs it") + if pa is None: + return + pq = expect.require_module("pyarrow.parquet", name="pyarrow not available; multi-file suite needs it") + w = server_dir + + def part(i, path): + pq.write_table(pa.table({"id": pa.array(range(i * 1000, i * 1000 + 1000), pa.int32()), + "v": pa.array([j * 2 for j in range(i * 1000, i * 1000 + 1000)], pa.int64())}), + str(path), row_group_size=1000) + path.chmod(0o644) + + tree = w / "tree" + (tree / "a" / "b").mkdir(parents=True) + (tree / "c").mkdir() + part(0, tree / "a" / "part-0.parquet") + part(1, tree / "a" / "b" / "part-1.parquet") + part(2, tree / "c" / "part-2.parquet") + (tree / "a" / "b" / "notes.txt").write_text("not parquet\n") + part(0, w / "p0.parquet") + with pgc_conn.cursor() as cur: + oracle = _hash(cur, " UNION ALL ".join( + f"SELECT g AS id, (g * 2)::int8 AS v FROM generate_series({lo}, {lo + 999}) g" for lo in (0, 1000, 2000))) + expect.text(_hash(cur, _rp(tree)), oracle, "nested tree reads every level as one relation") + expect.text(_one(cur, _count(tree)), "3000", "nested tree row count") + + # _temporary and . names are not walked; the file below _temporary is real parquet + (tree / "_temporary" / "task-0").mkdir(parents=True) + (tree / ".hidden").mkdir() + shutil.copy(w / "p0.parquet", tree / "_temporary" / "task-0" / "in-progress.parquet") + shutil.copy(w / "p0.parquet", tree / ".hidden" / "hidden.parquet") + (tree / "_SUCCESS").touch() + expect.text(_one(cur, _count(tree)), "3000", "_temporary and dot directories are not walked") + shutil.copy(w / "p0.parquet", tree / "c" / "_part.parquet") + expect.text(_one(cur, _count(tree)), "3000", "a file whose name starts with _ is skipped too") + for p in (tree / "_temporary", tree / ".hidden"): + shutil.rmtree(p) + for p in (tree / "_SUCCESS", tree / "c" / "_part.parquet"): + p.unlink() + + # sorted by full path: a/b/part-1, a/part-0, c/part-2, so part-1's rows come first + try: + cur.execute(f"SELECT id FROM pgcolumnar.read_parquet('{tree}') AS t({COLS}) LIMIT 3") + first = " ".join(str(r[0]) for r in cur.fetchall()) + " " + except psycopg.Error as exc: + first = f"error: {exc.diag.message_primary}" + expect.text(first, "1000 1001 1002 ", + "tree read order follows the sorted full path") + + # a symlink to an ancestor: the bound is the assertion + os.symlink(tree, tree / "a" / "loop") + expect.text(_bounded(pgc_cluster, _count(tree), 60), "3000", + "a symlink loop terminates and does not multiply rows") + (tree / "a" / "loop").unlink() + + elsewhere = w / "elsewhere" + elsewhere.mkdir() + shutil.copy(w / "p0.parquet", elsewhere / "extra.parquet") + os.symlink(elsewhere, tree / "linkdir") + expect.text(_one(cur, _count(tree)), "3000", "a symlinked directory is not descended") + (tree / "linkdir").unlink() + os.symlink(elsewhere / "extra.parquet", tree / "c" / "linked.parquet") + expect.text(_one(cur, _count(tree)), "4000", "a symlinked file is still read") + (tree / "c" / "linked.parquet").unlink() + + # past the bound it errors rather than reading the part it walked + d = w / "deep" + for i in range(1, 41): + d = d / f"l{i}" + d.mkdir(parents=True) + shutil.copy(w / "p0.parquet", d / "part-0.parquet") + expect.text(_errs(cur, _count(w / "deep")), "OK", "a tree deeper than the bound is rejected, not truncated")