diff --git a/test/approx_aggregates.sh b/test/approx_aggregates.sh index a2b5b88a6..3318a652c 100755 --- a/test/approx_aggregates.sh +++ b/test/approx_aggregates.sh @@ -16,8 +16,10 @@ # its default precision (log2m 11) has a standard error of 1.04/sqrt(2048), so # three of them, 6.9%; tdigest at compression 100 within 0.1% at p99, about # four times what phase 0 measured (0.025%). -# - each type stored in a columnar table comes back usable: an hll byte for byte, -# a tdigest as a digest whose union still answers within the bound. +# - each type stored in a columnar table comes back byte for byte, and its union +# still answers. For tdigest that union's bound is a premise only because the +# byte-for-byte arm comes first: a storage change that altered a digest reddens +# the arm before it can move the percentile. # - the aggregate reads through the columnar scan, so the arms exercise our reader. # That is an arm, not a premise: turning the custom scan off by default reddens it # (#1236: a check a repository change can redden is not a premise). @@ -69,8 +71,16 @@ TD_H="$(qv "$P0 SELECT tdigest_percentile(v, 100, 0.99) FROM ax_heap;")" EXACT99="$(q "SELECT percentile_cont(0.99) WITHIN GROUP (ORDER BY v) FROM ax_heap;")" check "tdigest over a columnar table gives exactly what it gives over heap" "$TD_C" "$TD_H" check "premise: and its p99 is within 0.1% of the exact percentile" "$(within "$TD_C" "$EXACT99" 0.001)" "ok" -psql_run "CREATE TABLE ax_td (day int, t tdigest) USING pgcolumnar; - INSERT INTO ax_td SELECT day, tdigest(v, 100) FROM ax GROUP BY day;" +# The digests are built ONCE, into a heap table, and that table's values are stored. +# A tdigest depends on the order its inputs arrive in, so digests rebuilt from heap +# could differ from these for reasons that are not storage. Copied from one source, +# any difference is the columnar round trip's (#1339). +psql_run "CREATE TABLE ax_td_src USING heap AS SELECT day, tdigest(v, 100) AS t FROM ax GROUP BY day; + CREATE TABLE ax_td (day int, t tdigest) USING pgcolumnar; + INSERT INTO ax_td SELECT day, t FROM ax_td_src;" +check "a tdigest stored in a columnar table comes back byte for byte" \ + "$(q "SELECT (SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td) + = (SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td_src);")" "t" check "premise: a tdigest stored in a columnar table still answers: the union of daily digests is within 0.1% at p99" \ "$(within "$(q "SELECT tdigest_percentile(t, 0.99) FROM ax_td;")" "$EXACT99" 0.001)" "ok" diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index d10894222..89adaa237 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -284,6 +284,7 @@ analyze_stats analyze_stats the sampled tuples carry a valid item pointer 15;16; analyze_stats analyze_stats the server survived ANALYZE 15;16;17;18;19 never - analyze_stats analyze_stats without the fetch penalty an unclustered ORDER BY takes the index (#355 premise) 15;16;17;18;19 never - analyze_stats analyze_stats without the penalty a selective scattered condition takes the index (#362 premise) 15;16;17;18;19 never - +approx_aggregates approx_aggregates a tdigest stored in a columnar table comes back byte for byte 15;16;17;18;19 2026-10-22 columnar_reader.c: a long value (over 512 bytes) reads as the previous long value read, so each stored digest comes back as another day's approx_aggregates approx_aggregates an hll stored in a columnar table comes back byte for byte 15;16;17;18;19 2026-10-13 the custom scan off by default, and the access-method scan skips every row whose row number ends in 999 approx_aggregates approx_aggregates and the union of the stored sketches counts what one sketch over all rows counts 15;16;17;18;19 2026-10-13 the custom scan off by default, and the access-method scan skips every row whose row number ends in 999 approx_aggregates approx_aggregates hll over a columnar table builds the same sketch, byte for byte, as over heap 15;16;17;18;19 2026-10-13 the custom scan off by default, and the access-method scan skips every row whose row number ends in 999 @@ -4721,6 +4722,19 @@ objstore_crlf objstore_crlf a CR/LF in the URL path is rejected before the reque objstore_crlf objstore_crlf backend alive 15;16;17;18;19 never - objstore_crlf objstore_crlf backend alive after the REST attempt 15;16;17;18;19 never - objstore_crlf objstore_crlf no injected line reached the listener (no request splitting) 15;16;17;18;19 2026-10-14 the request-line guard on the URL path removed +parallel parallel par count 15;16;17;18;19 2026-10-22 columnar_reader.c: a parallel custom scan claims row groups from its own rowGroupIndex, not the shared counter, so every participant reads every group (D6e) +parallel parallel par filtered scan 15;16;17;18;19 2026-10-22 columnar_reader.c: a parallel custom scan claims row groups from its own rowGroupIndex, not the shared counter, so every participant reads every group (D6e) +parallel parallel par no-filter 15;16;17;18;19 2026-10-22 columnar_reader.c: a parallel custom scan claims row groups from its own rowGroupIndex, not the shared counter, so every participant reads every group (D6e) +parallel parallel par queries all plan a Gather 15;16;17;18;19 2026-10-22 columnar_customscan.c: the parallel PgColumnarScan path is never added (add_partial_path dropped);columnar_customscan.c: the scan methods' CustomName is ColumnarScan, as before #428 +parallel parallel par range scan 15;16;17;18;19 2026-10-22 columnar_reader.c: a parallel custom scan claims row groups from its own rowGroupIndex, not the shared counter, so every participant reads every group (D6e) +parallel parallel par table has 25 row groups to claim 15;16;17;18;19 2026-10-22 columnar_write_state.c: the writer ignores the per-table stripe_row_limit (if (false && opts.stripeRowLimitSet)), so 50000 rows land in 1 row group +parallel parallel par whole sample 15;16;17;18;19 2026-10-22 columnar_reader.c: a parallel custom scan claims row groups from its own rowGroupIndex, not the shared counter, so every participant reads every group (D6e) +parallel parallel parallel plan has Gather 15;16;17;18;19 2026-10-22 columnar_customscan.c: the parallel PgColumnarScan path is never added (add_partial_path dropped) +parallel parallel parallel plan has PgColumnarScan 15;16;17;18;19 2026-10-22 columnar_customscan.c: the scan methods' CustomName is ColumnarScan, as before #428 +parallel parallel serial count 15;16;17;18;19 2026-10-22 columnar_reader.c: a serial scan advances rowGroupIndex by 2, so it reads every other row group +parallel parallel serial filtered scan 15;16;17;18;19 2026-10-22 columnar_reader.c: a serial scan advances rowGroupIndex by 2, so it reads every other row group +parallel parallel serial queries plan no Gather 15;16;17;18;19 2026-10-22 columnar_customscan.c: the scan methods' CustomName is ColumnarScan, as before #428 +parallel parallel serial range scan 15;16;17;18;19 2026-10-22 columnar_reader.c: a serial scan advances rowGroupIndex by 2, so it reads every other row group parallel_am_scan parallel_am_scan a parallel index build indexes every row of the table 15;16;17;18;19 never - parallel_am_scan parallel_am_scan a parallel table-AM scan returns the same row count as serial 15;16;17;18;19 never - parallel_am_scan parallel_am_scan premise: ANALYZE printed a rows= line per launched worker 15;16;17;18;19 never - @@ -4735,6 +4749,104 @@ parallel_am_scan parallel_am_scan premise: the serial plan is not a columnar cus parallel_am_scan parallel_am_scan premise: the table holds every inserted row 15;16;17;18;19 never - parallel_am_scan parallel_am_scan premise: with the custom scan off the serial plan is a Seq Scan 15;16;17;18;19 never - parallel_am_scan parallel_am_scan workers share the table-AM scan, it is not a single claimer 15;16;17;18;19 never - +parallel_copy parallel_copy 1 row / 4 workers (edge smoke): offsets well-formed and newline-aligned 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it +parallel_copy parallel_copy 1 row / 4 workers (edge smoke): per-range loads reconstruct the whole file 15;16;17;18;19 2026-10-22 columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy 5 rows / 8 workers (empty ranges): per-range loads reconstruct the whole file 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it;columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy 5 rows / 8 workers: offsets well-formed and newline-aligned 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it +parallel_copy parallel_copy 5000 rows / 16: offsets well-formed and newline-aligned 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it +parallel_copy parallel_copy 5000 rows / 16: per-range loads reconstruct the whole file 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it;columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy 5000 rows / 16: split is real (interior offsets distinct, ranges non-empty) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the splitter degrades to one range (every interior offset is the file size) +parallel_copy parallel_copy 5000 rows / 1: offsets well-formed and newline-aligned 15;16;17;18;19 never - +parallel_copy parallel_copy 5000 rows / 1: per-range loads reconstruct the whole file 15;16;17;18;19 2026-10-22 columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy 5000 rows / 2: offsets well-formed and newline-aligned 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it +parallel_copy parallel_copy 5000 rows / 2: per-range loads reconstruct the whole file 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it;columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy 5000 rows / 2: split is real (interior offsets distinct, ranges non-empty) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the splitter degrades to one range (every interior offset is the file size) +parallel_copy parallel_copy 5000 rows / 3: offsets well-formed and newline-aligned 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it +parallel_copy parallel_copy 5000 rows / 3: per-range loads reconstruct the whole file 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it;columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy 5000 rows / 3: split is real (interior offsets distinct, ranges non-empty) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the splitter degrades to one range (every interior offset is the file size) +parallel_copy parallel_copy 5000 rows / 8: offsets well-formed and newline-aligned 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it +parallel_copy parallel_copy 5000 rows / 8: per-range loads reconstruct the whole file 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it;columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy 5000 rows / 8: split is real (interior offsets distinct, ranges non-empty) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the splitter degrades to one range (every interior offset is the file size) +parallel_copy parallel_copy DEFAULT partition target is rejected 15;16;17;18;19 never - +parallel_copy parallel_copy atomic: a bad partition-key value is rejected 15;16;17;18;19 never - +parallel_copy parallel_copy atomic: a loader failure fails the whole load 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: a failed loader records no message, so the load reports 'loader exited without reporting a result' +parallel_copy parallel_copy atomic: bad-key load leaves the target empty 15;16;17;18;19 never - +parallel_copy parallel_copy atomic: loader-failure leaves the target empty (siblings rolled back) 15;16;17;18;19 never - +parallel_copy parallel_copy atomic: no prepared-transaction leak after bad-key 15;16;17;18;19 never - +parallel_copy parallel_copy atomic: no prepared-transaction leak after loader failure 15;16;17;18;19 never - +parallel_copy parallel_copy atomic: server still up after a failed load 15;16;17;18;19 2026-10-22 test/parallel_copy.sh: a backend killed by SIGSEGV, and the server recovered, before each survival check +parallel_copy parallel_copy backend still up after the parallel_copy FIFO refusal 15;16;17;18;19 2026-10-22 test/parallel_copy.sh: a backend killed by SIGSEGV, and the server recovered, before each survival check +parallel_copy parallel_copy backend still up after the partitioned-coordinator FIFO refusal 15;16;17;18;19 2026-10-22 test/parallel_copy.sh: a backend killed by SIGSEGV, and the server recovered, before each survival check +parallel_copy parallel_copy bulk_parallel_writer GUC defaults off (core write path unchanged) 15;16;17;18;19 2026-10-22 columnar_tableam.c: pgcolumnar.bulk_parallel_writer boots on +parallel_copy parallel_copy file_split_offsets: a directory is rejected (not an 8-exabyte file) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the single-file opener's S_ISREG refusal removed +parallel_copy parallel_copy guard rejects before loading anything 15;16;17;18;19 never - +parallel_copy parallel_copy key not in column 1 (generated col before it): rows = 3000 15;16;17;18;19 never - +parallel_copy parallel_copy key not in column 1: result == oracle 15;16;17;18;19 never - +parallel_copy parallel_copy long lines >64kB / 4 workers: offsets well-formed and newline-aligned 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it +parallel_copy parallel_copy long lines >64kB / 4 workers: per-range loads reconstruct the whole file 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it;columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy long lines >64kB / 4 workers: split is real (interior offsets distinct, ranges non-empty) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the splitter degrades to one range (every interior offset is the file size) +parallel_copy parallel_copy max_prepared_transactions guard fires 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the max_prepared_transactions guard removed +parallel_copy parallel_copy missing file: errors, not crashes 15;16;17;18;19 never - +parallel_copy parallel_copy missing file: nothing loaded 15;16;17;18;19 never - +parallel_copy parallel_copy missing file: server still up (no worker crash) 15;16;17;18;19 2026-10-22 test/parallel_copy.sh: a backend killed by SIGSEGV, and the server recovered, before each survival check +parallel_copy parallel_copy no trailing newline / 4 workers: offsets well-formed and newline-aligned 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it +parallel_copy parallel_copy no trailing newline / 4 workers: per-range loads reconstruct the whole file 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it;columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy non-columnar target is rejected 15;16;17;18;19 never - +parallel_copy parallel_copy parallel_copy on a FIFO is refused, not a hang (42809) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the single-file opener drops O_NONBLOCK, so a FIFO blocks open(2) (#686);columnar_parallel_copy.c: the single-file opener's S_ISREG refusal removed +parallel_copy parallel_copy parallel_copy on a FIFO via the partitioned coordinator is refused, not a hang (42809) 15;16;17;18;19 never - +parallel_copy parallel_copy parallel_copy(partitioned, 1 workers): no prepared-transaction leak 15;16;17;18;19 never - +parallel_copy parallel_copy parallel_copy(partitioned, 1 workers): result == single-COPY oracle 15;16;17;18;19 2026-10-22 columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy parallel_copy(partitioned, 1 workers): rows returned = 5000 15;16;17;18;19 never - +parallel_copy parallel_copy parallel_copy(partitioned, 2 workers): no prepared-transaction leak 15;16;17;18;19 never - +parallel_copy parallel_copy parallel_copy(partitioned, 2 workers): result == single-COPY oracle 15;16;17;18;19 2026-10-22 columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy parallel_copy(partitioned, 2 workers): rows returned = 5000 15;16;17;18;19 never - +parallel_copy parallel_copy parallel_copy(partitioned, 4 workers): no prepared-transaction leak 15;16;17;18;19 never - +parallel_copy parallel_copy parallel_copy(partitioned, 4 workers): result == single-COPY oracle 15;16;17;18;19 2026-10-22 columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy parallel_copy(partitioned, 4 workers): rows returned = 5000 15;16;17;18;19 never - +parallel_copy parallel_copy parallel_copy(single table, 1 workers): no prepared-transaction leak 15;16;17;18;19 never - +parallel_copy parallel_copy parallel_copy(single table, 1 workers): result == single-COPY oracle 15;16;17;18;19 2026-10-22 columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy parallel_copy(single table, 1 workers): rows returned = 5000 15;16;17;18;19 never - +parallel_copy parallel_copy parallel_copy(single table, 2 workers): no prepared-transaction leak 15;16;17;18;19 never - +parallel_copy parallel_copy parallel_copy(single table, 2 workers): result == single-COPY oracle 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it;columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy parallel_copy(single table, 2 workers): rows returned = 5000 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it +parallel_copy parallel_copy parallel_copy(single table, 4 workers): no prepared-transaction leak 15;16;17;18;19 never - +parallel_copy parallel_copy parallel_copy(single table, 4 workers): result == single-COPY oracle 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it;columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy parallel_copy(single table, 4 workers): rows returned = 5000 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it +parallel_copy parallel_copy signed keys straddling 0: no prepared-transaction leak 15;16;17;18;19 never - +parallel_copy parallel_copy signed keys straddling 0: result == oracle (no misbucketing/deadlock) 15;16;17;18;19 2026-10-22 columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy signed keys straddling 0: rows returned = 500 15;16;17;18;19 never - +parallel_copy parallel_copy single table: a bad row fails the whole load 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: a failed loader records no message, so the load reports 'loader exited without reporting a result' +parallel_copy parallel_copy single table: failed load leaves the target empty 15;16;17;18;19 never - +parallel_copy parallel_copy single table: no prepared-transaction leak after failure 15;16;17;18;19 never - +parallel_copy parallel_copy single-COPY heap and columnar agree 15;16;17;18;19 2026-10-22 columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy text partition key is rejected (not a crash) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the numeric or date/time partition-key restriction removed +parallel_copy parallel_copy text partition key: server still up (no crash) 15;16;17;18;19 2026-10-22 test/parallel_copy.sh: a backend killed by SIGSEGV, and the server recovered, before each survival check +parallel_copy parallel_copy unsorted input is rejected 15;16;17;18;19 never - +parallel_copy parallel_copy unsorted rejection loads nothing 15;16;17;18;19 never - +parallel_copy parallel_copy witness: >4 distinct stripe ids (each worker reserved several stripes) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it +parallel_copy parallel_copy witness: no prepared-transaction leak 15;16;17;18;19 never - +parallel_copy parallel_copy witness: no stripe byte-range overlaps another (no on-disk collision) 15;16;17;18;19 never - +parallel_copy parallel_copy witness: parallel_copy(single table, 4 workers, 40000 rows) loads every row 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it +parallel_copy parallel_copy witness: result == single-COPY oracle 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it;columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy witness: stripe rowcounts sum to every input row (complete coverage) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: an interior split offset lands on the newline, not after it;columnar_tableam.c: multi_insert drops the row whose first int4 column is 1 +parallel_copy parallel_copy workers < 1 is rejected 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: file_split_offsets accepts workers < 1 +parallel_copy_dedup parallel_copy_dedup a changed file at the same path is loaded, not skipped (#403 item 7) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: pcopy_fingerprint_file hashes only the first 256 KiB read (break after the first pg_cryptohash_update);columnar_parallel_copy.c: the refused load commits its ranges anyway (pcopy_finish_prepared(gid, true) in place of the rollback) +parallel_copy_dedup parallel_copy_dedup and each of those loads reported its rows 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: dedup always on, whatever the caller passed (dedup = true) +parallel_copy_dedup parallel_copy_dedup and reports 0 rather than claiming it loaded them 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the refused load reports the rows it discarded (total_rows summed, not 0) +parallel_copy_dedup parallel_copy_dedup and reports its rows 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarLoadFingerprintSeen matches the fingerprint in any table (heap scan on the fingerprint key alone) +parallel_copy_dedup parallel_copy_dedup and reports the rows it loaded 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: pcopy_fingerprint_file hashes only the first 256 KiB read (break after the first pg_cryptohash_update) +parallel_copy_dedup parallel_copy_dedup and reports them 15;16;17;18;19 never - +parallel_copy_dedup parallel_copy_dedup and reports those rows too 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: dedup always on, whatever the caller passed (dedup = true) +parallel_copy_dedup parallel_copy_dedup and says in a NOTICE that it skipped the load, rather than returning a silent 0 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the skip's NOTICE never raised (if (false && skipped)) +parallel_copy_dedup parallel_copy_dedup asking for no dedup loads again even though the file is on record 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarLoadFingerprintSeen matches the fingerprint in any table (heap scan on the fingerprint key alone);columnar_parallel_copy.c: dedup always on, whatever the caller passed (dedup = true) +parallel_copy_dedup parallel_copy_dedup no prepared transaction leaked through any of it 15;16;17;18;19 never - +parallel_copy_dedup parallel_copy_dedup premise: the changed row lies past the first 256 KiB of the file 15;16;17;18;19 never - +parallel_copy_dedup parallel_copy_dedup the first load stores its rows normally (#403 item 7) 15;16;17;18;19 never - +parallel_copy_dedup parallel_copy_dedup the records hold the SHA-256 of each loaded file's bytes 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: pcopy_fingerprint_file hashes only the first 256 KiB read (break after the first pg_cryptohash_update) +parallel_copy_dedup parallel_copy_dedup the same file loads into a DIFFERENT table (the record is per table) 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarLoadFingerprintSeen matches the fingerprint in any table (heap scan on the fingerprint key alone) +parallel_copy_dedup parallel_copy_dedup the second load of the same file stores nothing (#403 item 7) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the refused load commits its ranges anyway (pcopy_finish_prepared(gid, true) in place of the rollback) +parallel_copy_dedup parallel_copy_dedup the skipped load left exactly one record for that table and file 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: pcopy_fingerprint_file hashes only the first 256 KiB read (break after the first pg_cryptohash_update) +parallel_copy_dedup parallel_copy_dedup without dedup a repeated load doubles the table, which is the defect 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: dedup always on, whatever the caller passed (dedup = true) parallel_degree parallel_degree both tables hold the same rows 15;16;17;18;19 2026-10-21 columnar_tableam.c: tuple_insert drops the row whose first column is 1 parallel_degree parallel_degree identical data is granted the same parallel degree 15;16;17;18;19 2026-10-21 columnar_customscan.c: the #451 estimate over-asks into the cap, pages * 243;columnar_customscan.c: the #451 estimate over-asks, pages * 9;columnar_customscan.c: the #451 uncompressed-page estimate never applied, so the scan is sized from compressed relpages;columnar_customscan.c: the parallel columnar partial path never added (if (false && workers > 0)) parallel_degree parallel_degree premise: the columnar table really is smaller on disk 15;16;17;18;19 never - @@ -5212,11 +5324,22 @@ prepared_writes prepared_writes no backend was terminated by a signal 15;16;17;1 prepared_writes prepared_writes premise: prepared transactions are enabled 15;16;17;18;19 never - prepared_writes prepared_writes premise: the fetch is an index scan, which is what reaches the fetch cache 15;16;17;18;19 never - prepared_writes prepared_writes premise: the server log is where a crash would be recorded 15;16;17;18;19 never - +projection_drop_column projection_drop_column a non-owner is refused before it waits for the table lock 15;16;17;18;19 2026-10-22 columnar_tableam.c: both DROP COLUMN ownership checks deleted, so a stranger gets 2BP01 for a projected column and 42501 for another;columnar_tableam.c: the unlocked DROP COLUMN ownership check deleted, so a stranger queues for ShareUpdateExclusiveLock before it is refused +projection_drop_column projection_drop_column a non-owner learns nothing from a projected column 15;16;17;18;19 2026-10-22 columnar_tableam.c: both DROP COLUMN ownership checks deleted, so a stranger gets 2BP01 for a projected column and 42501 for another +projection_drop_column projection_drop_column a parent DROP sees projections on columnar partitions 15;16;17;18;19 never - +projection_drop_column projection_drop_column and the projection still receives the row 15;16;17;18;19 never - +projection_drop_column projection_drop_column and the two refusals are the same error, so neither leaks 15;16;17;18;19 2026-10-22 columnar_tableam.c: both DROP COLUMN ownership checks deleted, so a stranger gets 2BP01 for a projected column and 42501 for another +projection_drop_column projection_drop_column dropping a projected column is refused as a dependency 15;16;17;18;19 never - +projection_drop_column projection_drop_column dropping an unrelated column remains allowed 15;16;17;18;19 never - +projection_drop_column projection_drop_column premise: the owner holds SHARE on pdc before the stranger asks 15;16;17;18;19 never - +projection_drop_column projection_drop_column the rejected DDL leaves the table writable 15;16;17;18;19 never - +projection_drop_column projection_drop_column the rejected parent DDL leaves its partition writable 15;16;17;18;19 never - +projection_drop_column projection_drop_column the same non-owner error is returned for an unprojected column 15;16;17;18;19 never - +projection_parallel projection_parallel ANALYZE printed a rows= line per launched worker 15;16;17;18;19 2026-10-22 columnar_customscan.c: the parallel projection path's parallel_workers set to 1 instead of workers +projection_parallel projection_parallel EXPLAIN ANALYZE launched two workers 15;16;17;18;19 2026-10-22 columnar_customscan.c: the parallel projection path's parallel_workers set to 1 instead of workers projection_parallel projection_parallel a covering projection can be a parallel scan 15;16;17;18;19 2026-09-18 - projection_parallel projection_parallel a page cost that makes I/O dominate costs the parallel covering path 15;16;17;18;19 2026-09-23 the partial covering total amortises I/O as well as CPU: (ioRunProj + cpuRunProj) / divisor projection_parallel projection_parallel a parallel covering projection returns the covering rows once 15;16;17;18;19 2026-10-13 a parallel worker never wires the shared row-group counter, so every worker scans every group -projection_parallel projection_parallel premise: ANALYZE printed a rows= line per launched worker 15;16;17;18;19 never - -projection_parallel projection_parallel premise: EXPLAIN ANALYZE launched two workers 15;16;17;18;19 never - projection_parallel projection_parallel premise: a covering projection exists 15;16;17;18;19 never - projection_parallel projection_parallel premise: a parallel base scan is available when the projection is off 15;16;17;18;19 never - projection_parallel projection_parallel premise: a serial covering query uses the projection 15;16;17;18;19 never - @@ -5224,18 +5347,18 @@ projection_parallel projection_parallel premise: the covering plan is parallel a projection_parallel projection_parallel premise: the table holds every inserted row 15;16;17;18;19 never - projection_parallel projection_parallel the parallel count is a parallel covering projection scan 15;16;17;18;19 never - projection_parallel projection_parallel the workers' rows add up to the covering rows, each returned once 15;16;17;18;19 2026-10-13 a parallel worker never wires the shared row-group counter, so every worker scans every group -projection_parallel projection_parallel workers share the covering projection scan, it is not a single claimer 15;16;17;18;19 2026-10-13 first-wins-claimer;only the first parallel worker claims row groups, so one worker scans the whole table (#1336) -projection_privilege projection_privilege a caller with no SELECT is told a privilege is missing, not that RLS is in force 15;16;17;18;19 never - +projection_parallel projection_parallel workers share the covering projection scan, it is not a single claimer 15;16;17;18;19 2026-10-22 columnar_customscan.c: the parallel projection path's parallel_workers set to 1 instead of workers;first-wins-claimer;only the first parallel worker claims row groups, so one worker scans the whole table (#1336) +projection_privilege projection_privilege a caller with no SELECT is told a privilege is missing, not that RLS is in force 15;16;17;18;19 2026-10-22 columnar_projection.c: read_projection's aclcheck_error replaced by a plain elog, so its refusal is XX000 projection_privilege projection_privilege a role WITH SELECT still reads the projection 15;16;17;18;19 2026-10-14 read_projection stores no rows: its tuplestore_putvalues call removed, so every caller gets zero rows projection_privilege projection_privilege a role with EXECUTE but no SELECT is refused read_projection 15;16;17;18;19 never - projection_privilege projection_privilege a role with only schema USAGE is refused read_projection 15;16;17;18;19 never - -projection_privilege projection_privilege and THAT refusal is the C check, which raises 42501 from aclcheck_error 15;16;17;18;19 never - +projection_privilege projection_privilege and THAT refusal is the C check, which raises 42501 from aclcheck_error 15;16;17;18;19 2026-10-22 columnar_projection.c: read_projection's aclcheck_error replaced by a plain elog, so its refusal is XX000 projection_privilege projection_privilege and is refused reconstruct_via_projection 15;16;17;18;19 never - -projection_privilege projection_privilege and is refused reconstruct_via_projection, which leaks non-covered columns 15;16;17;18;19 never - +projection_privilege projection_privilege and is refused reconstruct_via_projection, which leaks non-covered columns 15;16;17;18;19 2026-10-22 columnar_projection.c: reconstruct_via_projection's aclcheck_error replaced by a plain elog, so its refusal is XX000 projection_privilege projection_privilege and read_projection does NOT, so the two differ as the issue says 15;16;17;18;19 2026-10-14 read_projection bound to the reconstruct_via_projection C symbol in the extension SQL, so it returns the non-covered columns too;read_projection stores no rows: its tuplestore_putvalues call removed, so every caller gets zero rows -projection_privilege projection_privilege and so does reconstruct_via_projection 15;16;17;18;19 never - +projection_privilege projection_privilege and so does reconstruct_via_projection 15;16;17;18;19 2026-10-22 columnar_vacuum.c: PgColumnarRequireNoRowSecurity raises 42501 instead of 0A000 projection_privilege projection_privilege and still reconstructs 15;16;17;18;19 2026-10-14 reconstruct_via_projection stores no rows: its tuplestore_putvalues call removed, so every caller gets zero rows -projection_privilege projection_privilege and the refusal says the feature is not supported, not that a privilege is missing 15;16;17;18;19 never - +projection_privilege projection_privilege and the refusal says the feature is not supported, not that a privilege is missing 15;16;17;18;19 2026-10-22 columnar_vacuum.c: PgColumnarRequireNoRowSecurity raises 42501 instead of 0A000 projection_privilege projection_privilege premise: t_prjexec can open a session, so a refusal below is a refusal 15;16;17;18;19 never - projection_privilege projection_privilege premise: t_prjexec really does hold EXECUTE, so it reaches the C gate 15;16;17;18;19 never - projection_privilege projection_privilege premise: t_prjsel CAN read the table, so its arm below tests the bar 15;16;17;18;19 never - @@ -5243,11 +5366,11 @@ projection_privilege projection_privilege premise: the test roles are not superu projection_privilege projection_privilege premise: the unprivileged role cannot read the table by any ordinary route 15;16;17;18;19 never - projection_privilege projection_privilege read_projection now refuses a policy-restricted caller (#563) 15;16;17;18;19 never - projection_privilege projection_privilege reconstruct really does return the NON-COVERED column 15;16;17;18;19 2026-10-14 reconstruct_via_projection renders every column the projection does not cover as NULL instead of the base value;reconstruct_via_projection stores no rows: its tuplestore_putvalues call removed, so every caller gets zero rows -projection_privilege projection_privilege reconstruct: a caller with no SELECT is told a privilege is missing, not that RLS is in force 15;16;17;18;19 never - +projection_privilege projection_privilege reconstruct: a caller with no SELECT is told a privilege is missing, not that RLS is in force 15;16;17;18;19 2026-10-22 columnar_projection.c: reconstruct_via_projection's aclcheck_error replaced by a plain elog, so its refusal is XX000 projection_privilege projection_privilege reconstruct: the EXECUTE role reaches the projection lookup on the same call 15;16;17;18;19 never - -projection_privilege projection_privilege reconstruct: the base ACL is checked before the projection name is looked up 15;16;17;18;19 never - +projection_privilege projection_privilege reconstruct: the base ACL is checked before the projection name is looked up 15;16;17;18;19 2026-10-22 columnar_projection.c: reconstruct_via_projection's aclcheck_error replaced by a plain elog, so its refusal is XX000 projection_privilege projection_privilege reconstruct: the no-EXECUTE role never reaches the body, so the grant is what stopped it 15;16;17;18;19 never - -projection_privilege projection_privilege the base ACL is checked before the projection name is looked up 15;16;17;18;19 never - +projection_privilege projection_privilege the base ACL is checked before the projection name is looked up 15;16;17;18;19 2026-10-22 columnar_projection.c: read_projection's aclcheck_error replaced by a plain elog, so its refusal is XX000 projection_privilege projection_privilege the no-EXECUTE role never reaches the body, so the grant is what stopped it 15;16;17;18;19 never - projection_privilege projection_privilege the owner reads the projection, and gets rows 15;16;17;18;19 2026-10-14 read_projection stores no rows: its tuplestore_putvalues call removed, so every caller gets zero rows projection_privilege projection_privilege the owner reconstructs, and gets rows 15;16;17;18;19 2026-10-14 reconstruct_via_projection stores no rows: its tuplestore_putvalues call removed, so every caller gets zero rows @@ -5365,6 +5488,7 @@ projection_scan_cost projection_scan_cost while a plain range on the sort key st projection_scan_io projection_scan_io a covering projection is not priced from the base table's pages 15;16;17;18;19 2026-09-19 rel->pages projection_scan_io projection_scan_io a covering projection whose storage cannot be found is not priced as one page 15;16;17;18;19 2026-09-21 return 1 projection_scan_io projection_scan_io an empty covering projection is still priced as one page, so it is chosen 15;16;17;18;19 2026-09-23 pages = 997 on the third return of pgcolumnar_projection_pages +projection_scan_io projection_scan_io and its scan is not priced at zero pages 15;16;17;18;19 2026-10-22 columnar_customscan.c: the one-page floor in pgcolumnar_projection_pages deleted, so an empty projection is priced at 0 pages projection_scan_io projection_scan_io premise: a covering path is still offered when projection storage cannot be found 15;16;17;18;19 never - projection_scan_io projection_scan_io premise: a covering projection exists 15;16;17;18;19 never - projection_scan_io projection_scan_io premise: the covering projection occupies a minority of the relation 15;16;17;18;19 never - @@ -5413,6 +5537,7 @@ projections projections drop unknown rejected 15;16;17;18;19 never - projections projections duplicate column rejected 15;16;17;18;19 never - projections projections duplicate name rejected 15;16;17;18;19 never - projections projections empty columns rejected 15;16;17;18;19 never - +projections projections executing it reads the projection, so the filter sees fewer rows than the base holds 15;16;17;18;19 2026-10-22 columnar_customscan.c: projection pages multiplied by 1000000, so the planner chooses the base;columnar_customscan.c: the executor never opens the chosen projection (if (false && projName)) while EXPLAIN still names it projections projections fp chunks carry min/max skip metadata 15;16;17;18;19 never - projections projections fp count after delete matches base 15;16;17;18;19 never - projections projections fp fan-out matches base (a,c) 15;16;17;18;19 never - @@ -5424,7 +5549,7 @@ projections projections fq fan-out matches base (b) 15;16;17;18;19 never - projections projections new snapshot projection scan sees both batches 15;16;17;18;19 never - projections projections no projection rows before first add 15;16;17;18;19 never - projections projections non-covering query (references b) uses the base 15;16;17;18;19 never - -projections projections old snapshot projection scan does not see post-snapshot rows 15;16;17;18;19 never - +projections projections old snapshot projection scan does not see post-snapshot rows 15;16;17;18;19 2026-10-22 columnar_customscan.c: the projection scan and its base liveness cache both read GetLatestSnapshot() instead of the query snapshot projections projections p1 columns 15;16;17;18;19 never - projections projections p1 gone after drop 15;16;17;18;19 never - projections projections p1 has its own storage id 15;16;17;18;19 never - @@ -5433,9 +5558,9 @@ projections projections p1 sort_key 15;16;17;18;19 never - projections projections p2 columns 15;16;17;18;19 never - projections projections p2 id is 2 15;16;17;18;19 never - projections projections p2 sort_key empty 15;16;17;18;19 never - -projections projections planner still uses projection after vacuum 15;16;17;18;19 never - +projections projections planner still uses projection after vacuum 15;16;17;18;19 2026-10-22 columnar_customscan.c: projection pages multiplied by 1000000, so the planner chooses the base projections projections premise: the base table holds every committed row 15;16;17;18;19 never - -projections projections projection chosen for covering + sort-key query 15;16;17;18;19 never - +projections projections projection chosen for covering + sort-key query 15;16;17;18;19 2026-10-22 columnar_customscan.c: projection pages multiplied by 1000000, so the planner chooses the base projections projections projection matches base after second vacuum 15;16;17;18;19 never - projections projections projection scan matches oracle after delete 15;16;17;18;19 never - projections projections projection survives vacuum 15;16;17;18;19 never - @@ -5454,6 +5579,7 @@ projections projections storage id resolves 15;16;17;18;19 never - projections projections table populated 15;16;17;18;19 never - projections projections table still readable after DDL 15;16;17;18;19 never - projections projections the drop really removed the projection 15;16;17;18;19 never - +projections projections the old-snapshot count is planned as a projection scan 15;16;17;18;19 2026-10-22 columnar_customscan.c: projection pages multiplied by 1000000, so the planner chooses the base projections projections the rebuild does not abort on it 15;16;17;18;19 never - projections projections two declared projections to start 15;16;17;18;19 never - projections projections two rows after first add (base + p1) 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 7e51bae74..6f7b81990 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 75 +suites_not_covered 71 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -5957,4 +5957,114 @@ suites_not_covered 75 # never 4129 + 8 = 4137 # dated 2791 + 7 = 2798 # 4137 + 2798 == 6935 -checks_never_observed_red 4137 +# +# #1339, a batch of projection checks (jdatcmd): 4137 -> 4132, and suites_not_covered +# 75 -> 74. Seven findings, each shown green on main under its mutation in both +# twins and red by name after the fix. projection_drop_column is seeded: it had no +# rows, and it enters with 11 checks from five-major runs (11/11 on each of 15-19, +# twin green on each). Its stranger arms read only 42501, so deleting the unlocked +# ownership check passed; a stranger is now asked while the owner holds SHARE, and +# queues to 55P03 under that mutation. Its "two refusals are the same" check lost its +# premise: prefix, joined the shell suite, and moved ahead of the two constant arms +# so it can fail. projection_parallel's two worker-count checks lost their premise: +# prefix: parallel_workers = 1 reddens both. projection_privilege's read_projection +# arms each asserted one SQLSTATE twice; the first now asks only "refused", as the +# shell suite does, and the shell's reconstruct arms read the SQLSTATE, as the twin +# does. projection_scan_io gains a check that an empty projection is not priced at +# zero pages. projections gains two: an EXPLAIN ANALYZE reading that the executor +# opened the projection, and that the old-snapshot count plans it. Its twin's two +# session-A timeout arms now read whether A answered, not the count again, and the +# shell's poll no longer takes an ERROR that echoes the token for an answer. 19 rows +# are dated by ten mutations, each reddening its target by the same name in both +# twins (the pytest twin stops at a test's first red): the unlocked ownership check +# deleted, both deleted, parallel_workers = 1, each function's aclcheck_error +# replaced by elog, the RLS refusal raised as 42501, the one-page floor deleted, the +# executor never opening the projection, projection pages multiplied by 1000000, +# and the projection scan and liveness cache both on GetLatestSnapshot(). 8 arrive +# never: one premise and seven drop_column checks no mutation here reached. +# +# ledger rows 6935 + 14 = 6949 +# never 4137 + 8 - 13 = 4132 +# dated 2798 + 6 + 13 = 2817 +# 4132 + 2817 == 6949 +# +# #1339, approx_aggregates gains one arm (jdatcmd): 4132 -> 4132, unchanged. "a tdigest stored in +# a columnar table comes back byte for byte" is dated by a reader that returns each long +# value as the previous long value read: main's suite passed every tdigest check under it, +# and only the hll arm went red. suites_not_covered does not move. +# +# ledger rows 6949 + 1 = 6950 +# never 4132 + 0 = 4132 +# dated 2817 + 1 = 2818 +# 4132 + 2818 == 6950 +# +# #1341, parallel_copy_dedup seeded, with its twin (jdatcmd): 3853 -> 3857, and +# suites_not_covered 105 -> 104. The suite had no rows. It enters with 17 checks from +# five-major runs (17/17 on each of 15-19, shell and twin alike), after three fixes. +# The changed file differed from the first in its first bytes and fitted in one +# 256 KiB read, so a fingerprint of the first read alone still loaded it and main's +# shell passed 14/14; the changed file now differs only in its last row, past the +# first read, and a new check compares the table's records with the server's +# sha256() of each file. The skip's NOTICE, promised in the header, was never read: +# with it removed main's shell passed 14/14; a new check reads it. And the two +# control checks lost their premise: prefix under #1339: dedup is pgcolumnar's own +# default, and forcing it on reddens them. 13 are dated by six mutations, each +# reddening the same names in both twins: the NOTICE never raised, the first read +# alone fingerprinted, dedup forced on, the refused load committed anyway, a +# fingerprint matched in any table, and the refused load reporting the rows it +# discarded. 4 arrive never: the fixture-size premise, the first load's two checks, +# and the prepared-transaction leak. +# +# ledger rows 6950 + 17 = 6967 +# never 4132 + 4 = 4136 +# dated 2818 + 13 = 2831 +# 4136 + 2831 == 6967 +# +# #1341, parallel_copy seeded, with its twin (jdatcmd): 3853 -> 3885, and +# suites_not_covered 105 -> 104. The suite had no rows. It enters with 81 checks from +# five-major runs (81/81 on each of 15-19, shell and twin alike), after three fixes. +# Five survival checks read "SELECT 1", which answers after crash recovery: with a +# backend killed by SIGSEGV before each, all five passed; they now also need the +# server log's count of killed backends unmoved. Two failure checks took any message +# with "failed" in it, and every worker failure reads "worker N failed: ...": with +# the loader's message dropped they passed; they now need the loader's own cause. +# The nine reconstruct checks compared the per-range loads with a single columnar +# COPY of the same file, which shares the COPY path's defects: with multi_insert +# dropping the row whose id is 1, all nine passed; they now compare with the +# generator that wrote the file. Names built from loop variables are literal. 49 are +# dated by ten product mutations and the crash edit, each reddening the same names in +# both twins: the loader's message dropped, a row dropped by multi_insert, the +# max_prepared_transactions guard removed, bulk_parallel_writer booting on, the FIFO +# opener blocking, a split offset on the newline, a split degraded to one range, +# workers < 1 accepted, the S_ISREG refusal removed, and the key-type restriction +# removed. 32 arrive never: the leak, empty-target and rows-returned checks of loads +# no mutation reached, the refusals of a bad key, an unsorted file, a DEFAULT +# partition, a heap target and a missing file, the generated-column load, the +# partitioned FIFO refusal, the stripe-overlap check, and the one-worker split. +# +# ledger rows 6967 + 81 = 7048 +# never 4136 + 32 = 4168 +# dated 2831 + 49 = 2880 +# 4168 + 2880 == 7048 +# +# #1341, parallel seeded, with its twin (jdatcmd): 3853 -> 3853, and +# suites_not_covered 105 -> 104. The suite had no rows. It enters with 13 checks from +# five-major runs (13/13 on each of 15-19, shell and twin alike), after three fixes and +# three new checks. "par no-filter" was a bare count(*), which the vectorized aggregate +# answers with no Gather, so with every participant reading every row group it stayed +# green; it now returns rows, and "par queries all plan a Gather" holds all five par +# queries to a Gather. The table was assumed to have 25 row groups: with the writer +# ignoring stripe_row_limit it had 1, and a claim that skips every other group passed +# main's shell 10/10; "par table has 25 row groups to claim" now counts them. And setg +# swallows its error, so a serial setting that did not take ran the serial checks in +# parallel; "serial queries plan no Gather" now says so. All 13 are dated by five +# mutations, each reddening the same names in both twins: the writer ignoring +# stripe_row_limit, every participant reading every row group, the partial path never +# added, the scan named ColumnarScan as before #428, and a serial scan reading every +# other row group. +# +# ledger rows 7048 + 13 = 7061 +# never 4168 + 0 = 4168 +# dated 2880 + 13 = 2893 +# 4168 + 2893 == 7061 +checks_never_observed_red 4168 diff --git a/test/parallel.sh b/test/parallel.sh index b229c0901..931578cea 100755 --- a/test/parallel.sh +++ b/test/parallel.sh @@ -22,6 +22,11 @@ pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" make_pair "id int, k int, v bigint, t text" q "SELECT pgcolumnar.set_options('t_col', chunk_group_row_limit => 1000, stripe_row_limit => 2000);" >/dev/null load_pair "SELECT g, g%100, g::bigint*2, 'r'||g FROM generate_series(1,50000) g" +# 50000 rows at 2000 a row group is 25 groups. With one group there is nothing to +# share: a claim that skips every other group returns the same rows as a correct one, +# and with the writer ignoring stripe_row_limit main's shell passed 10 of 10 under +# exactly that claim. So the count is asserted, not assumed. +check_num "par table has 25 row groups to claim" "$(stripe_count)" "25" setg() { q "ALTER DATABASE $PGC_DB SET $1 = $2;" >/dev/null; } @@ -40,18 +45,55 @@ check "parallel plan has Gather" "$(echo "$plan" | grep -c -i 'Gather')" check "parallel plan has PgColumnarScan" \ "$(echo "$plan" | grep -c 'Custom Scan (PgColumnarScan)')" "1" +# The queries, named once, so the plan checks below read the queries the +# comparisons ran and not a copy that can drift from them. +Q_FILTERED="SELECT id, v, t FROM %T WHERE k = 7" +Q_RANGE="SELECT id FROM %T WHERE k < 30" +Q_SAMPLE="SELECT * FROM %T WHERE id % 997 = 0" +Q_COUNT="SELECT count(*), sum(v) FROM %T WHERE k < 50" +# Row-returning, not count(*). A bare count(*) is answered by the vectorized +# aggregate, which plans no Gather, so this check ran serially and stayed green +# with every participant reading every row group. +Q_NOFILTER="SELECT id FROM %T" + +# plans_with GATHER|NOGATHER QUERY... -> how many of the queries, on t_col, plan a +# PgColumnarScan with (or without) a Gather. A query whose EXPLAIN fails counts +# for neither, so a dead server cannot satisfy either reading. +plans_with() { + local want="$1" n=0 tmpl plan + shift + for tmpl in "$@"; do + plan="$(q "EXPLAIN (COSTS off) ${tmpl//%T/t_col}")" + [[ "$plan" == *'Custom Scan (PgColumnarScan)'* ]] || continue + if [[ "$plan" == *Gather* ]]; then + [ "$want" = GATHER ] && n=$((n + 1)) + else + [ "$want" = NOGATHER ] && n=$((n + 1)) + fi + done + echo "$n" +} + # Results must match the heap oracle under the parallel plan. -diff_query "par filtered scan" "SELECT id, v, t FROM %T WHERE k = 7" -diff_query "par range scan" "SELECT id FROM %T WHERE k < 30" -diff_query "par whole sample" "SELECT * FROM %T WHERE id % 997 = 0" -diff_query "par count" "SELECT count(*), sum(v) FROM %T WHERE k < 50" -diff_query "par no-filter" "SELECT count(*) FROM %T" +diff_query "par filtered scan" "$Q_FILTERED" +diff_query "par range scan" "$Q_RANGE" +diff_query "par whole sample" "$Q_SAMPLE" +diff_query "par count" "$Q_COUNT" +diff_query "par no-filter" "$Q_NOFILTER" +# A "par" check proves nothing about the parallel scan unless its query ran under +# one. Measured on main: "par no-filter" planned no Gather. +check_num "par queries all plan a Gather" \ + "$(plans_with GATHER "$Q_FILTERED" "$Q_RANGE" "$Q_SAMPLE" "$Q_COUNT" "$Q_NOFILTER")" "5" # The same queries, forced serial, must give the same answers. setg max_parallel_workers_per_gather 0 -diff_query "serial filtered scan" "SELECT id, v, t FROM %T WHERE k = 7" -diff_query "serial range scan" "SELECT id FROM %T WHERE k < 30" -diff_query "serial count" "SELECT count(*), sum(v) FROM %T WHERE k < 50" +diff_query "serial filtered scan" "$Q_FILTERED" +diff_query "serial range scan" "$Q_RANGE" +diff_query "serial count" "$Q_COUNT" +# setg swallows its error, so a setting that did not take leaves these three +# running in parallel, and they pass as "serial" without a serial scan. +check_num "serial queries plan no Gather" \ + "$(plans_with NOGATHER "$Q_FILTERED" "$Q_RANGE" "$Q_COUNT")" "3" q "ALTER DATABASE $PGC_DB RESET ALL;" >/dev/null diff --git a/test/parallel_copy.sh b/test/parallel_copy.sh index f2d592716..81a02d06d 100755 --- a/test/parallel_copy.sh +++ b/test/parallel_copy.sh @@ -51,17 +51,37 @@ offsets_of() { # path workers -> space-separated offsets q "SELECT array_to_string(pgcolumnar.file_split_offsets('$1', $2), ' ')" } +# A backend that died does not pass as one that survived. A fresh connection answers +# again after crash recovery, so "SELECT 1" alone passed with a backend killed by +# SIGSEGV right before each of this suite's five survival checks (measured on PG17, +# #1341). survived() also requires the server log's count of killed backends to be +# unmoved since the suite began, and fails when that count cannot be read. +crashes() { + local _n + _n="$(pgc_pg "grep -c 'terminated by signal' '$PGC_LOGFILE'" 2>/dev/null)" + case "$_n" in '' | *[!0-9]*) ;; *) echo "$_n" ;; esac +} +survived() { # survived BEFORE -> "1" when the server answers and no backend died since + local _up _now + _up="$(q 'SELECT 1')"; _now="$(crashes)" + if ! [[ "$1" =~ ^[0-9]+$ && "$_now" =~ ^[0-9]+$ ]]; then echo "no (crash count unreadable)" + elif [ "$_up" = 1 ] && [ "$_now" = "$1" ]; then echo 1 + else echo "no ([$_up], $(( _now - $1 )) backend crashes)"; fi +} +CRASHES_BEFORE="$(crashes)" + # ---- structural checks: boundaries land after a newline, cover the file ------- +# Each helper below echoes one reading; the check, with its literal name, is at the +# call site. -check_structure() { # path workers label - local f="$1" w="$2" label="$3" +structure_of() { # path workers -> 1 when the offsets are well-formed and aligned + local f="$1" w="$2" local size off arr ok prev i b size=$(stat -c %s "$f") read -r -a arr <<<"$(offsets_of "$f" "$w")" ok=1 if [ "${#arr[@]}" != "$((w + 1))" ]; then - check "$label: offsets well-formed and newline-aligned" \ - "got ${#arr[@]} offsets" "want $((w + 1))" + echo "got ${#arr[@]} offsets, want $((w + 1))" return fi [ "${arr[0]}" = "0" ] || ok=0 @@ -77,13 +97,17 @@ check_structure() { # path workers label [ "$b" = "0a" ] || ok=0 fi done - check "$label: offsets well-formed and newline-aligned" "$ok" 1 + echo "$ok" } # ---- end-to-end: per-range loads reconstruct the whole file ------------------- +# The oracle is the generator that wrote the file, not a single columnar COPY of it. +# A single COPY into pgcolumnar shares every defect of pgcolumnar's COPY path with +# the per-range loads: with multi_insert dropping the row whose id is 1, both lost +# the same row and all nine reconstruct checks passed (measured on PG17, #1341). -check_reconstruct() { # path workers label - local f="$1" w="$2" label="$3" +split_load_hash() { # path workers -> set hash of the per-range loads + local f="$1" w="$2" local arr i off len rf read -r -a arr <<<"$(offsets_of "$f" "$w")" psql_run "DROP TABLE IF EXISTS t_split; @@ -96,23 +120,19 @@ check_reconstruct() { # path workers label [ "$(id -u)" = "0" ] && chown postgres "$rf" psql_run "\copy t_split FROM '$rf' WITH (FORMAT text)" >/dev/null done - # t_col holds the single-COPY load of the same file -> oracle for this file - local h_whole h_split - h_whole="$(pgc_set_hash "SELECT * FROM t_col")" - h_split="$(pgc_set_hash "SELECT * FROM t_split")" - check "$label: per-range loads reconstruct the whole file" "$h_split" "$h_whole" + pgc_set_hash "SELECT * FROM t_split" } # ---- the split must actually split ------------------------------------------- -# check_structure and check_reconstruct both pass on a splitter degraded to a +# structure_of and split_load_hash both pass on a splitter degraded to a # single range ({0, size, size, ..., size}): the offsets are ascending and # newline-aligned, and one range covering the whole file reconstructs it. That is # not hypothetical -- a bare-CR file and phase 5's "fall back to a single COPY" # path both produce exactly that shape. So when the file has far more records than # workers, assert every range is non-empty: the W-1 interior offsets must be # strictly increasing and strictly inside (0, size). A degraded split fails here. -check_split_happened() { # path workers label - local f="$1" w="$2" label="$3" +split_real_of() { # path workers -> 1 when every range is non-empty + local f="$1" w="$2" local size arr ok prev i off size=$(stat -c %s "$f") read -r -a arr <<<"$(offsets_of "$f" "$w")" @@ -123,11 +143,12 @@ check_split_happened() { # path workers label { [ "$off" -gt "$prev" ] && [ "$off" -lt "$size" ]; } 2>/dev/null || ok=0 prev=$off done - check "$label: split is real (interior offsets distinct, ranges non-empty)" "$ok" 1 + echo "$ok" } # ---- the main file: 5000 rows ----------------------------------------------- +GEN_F="SELECT g AS id, 'host_'||(g%97)||'_x'||g AS txt FROM generate_series(1, 5000) g" make_pair "id int, txt text" F=$(make_file 5000) psql_run "\copy t_heap FROM '$F' WITH (FORMAT text)" >/dev/null @@ -135,55 +156,76 @@ psql_run "\copy t_col FROM '$F' WITH (FORMAT text)" >/dev/null check "single-COPY heap and columnar agree" \ "$(pgc_set_hash "SELECT * FROM t_col")" "$(pgc_set_hash "SELECT * FROM t_heap")" -for W in 1 2 3 8 16; do - check_structure "$F" "$W" "5000 rows / $W" - check_reconstruct "$F" "$W" "5000 rows / $W" - # 5000 rows >> W, so every range must be non-empty: catches a degraded split. - [ "$W" -ge 2 ] && check_split_happened "$F" "$W" "5000 rows / $W" -done +# 5000 rows >> W, so every range must be non-empty: the split-is-real checks catch a +# degraded split (W=1 has no interior offset, so it has none). +check "5000 rows / 1: offsets well-formed and newline-aligned" "$(structure_of "$F" 1)" 1 +check "5000 rows / 1: per-range loads reconstruct the whole file" \ + "$(split_load_hash "$F" 1)" "$(pgc_set_hash "$GEN_F")" +check "5000 rows / 2: offsets well-formed and newline-aligned" "$(structure_of "$F" 2)" 1 +check "5000 rows / 2: per-range loads reconstruct the whole file" \ + "$(split_load_hash "$F" 2)" "$(pgc_set_hash "$GEN_F")" +check "5000 rows / 2: split is real (interior offsets distinct, ranges non-empty)" \ + "$(split_real_of "$F" 2)" 1 +check "5000 rows / 3: offsets well-formed and newline-aligned" "$(structure_of "$F" 3)" 1 +check "5000 rows / 3: per-range loads reconstruct the whole file" \ + "$(split_load_hash "$F" 3)" "$(pgc_set_hash "$GEN_F")" +check "5000 rows / 3: split is real (interior offsets distinct, ranges non-empty)" \ + "$(split_real_of "$F" 3)" 1 +check "5000 rows / 8: offsets well-formed and newline-aligned" "$(structure_of "$F" 8)" 1 +check "5000 rows / 8: per-range loads reconstruct the whole file" \ + "$(split_load_hash "$F" 8)" "$(pgc_set_hash "$GEN_F")" +check "5000 rows / 8: split is real (interior offsets distinct, ranges non-empty)" \ + "$(split_real_of "$F" 8)" 1 +check "5000 rows / 16: offsets well-formed and newline-aligned" "$(structure_of "$F" 16)" 1 +check "5000 rows / 16: per-range loads reconstruct the whole file" \ + "$(split_load_hash "$F" 16)" "$(pgc_set_hash "$GEN_F")" +check "5000 rows / 16: split is real (interior offsets distinct, ranges non-empty)" \ + "$(split_real_of "$F" 16)" 1 # a line longer than the 64 kB scan chunk: the boundary for a mid-file split lands # only after crossing a read-chunk boundary, exercising the multi-chunk read (the # only part of the scan with carried-over state). F_BIG="$DATADIR/biglines.txt" -psql_run "COPY (SELECT g AS id, 'h'||g||'_'||repeat('x', 70000) AS txt - FROM generate_series(1, 20) g) - TO '$F_BIG' WITH (FORMAT text);" >/dev/null +GEN_BIG="SELECT g AS id, 'h'||g||'_'||repeat('x', 70000) AS txt FROM generate_series(1, 20) g" +psql_run "COPY ($GEN_BIG) TO '$F_BIG' WITH (FORMAT text);" >/dev/null [ "$(id -u)" = "0" ] && chown postgres "$F_BIG" -psql_run "TRUNCATE t_col;" >/dev/null -psql_run "\copy t_col FROM '$F_BIG' WITH (FORMAT text)" >/dev/null -check_structure "$F_BIG" 4 "long lines >64kB / 4 workers" -check_reconstruct "$F_BIG" 4 "long lines >64kB / 4 workers" -check_split_happened "$F_BIG" 4 "long lines >64kB / 4 workers" +check "long lines >64kB / 4 workers: offsets well-formed and newline-aligned" \ + "$(structure_of "$F_BIG" 4)" 1 +check "long lines >64kB / 4 workers: per-range loads reconstruct the whole file" \ + "$(split_load_hash "$F_BIG" 4)" "$(pgc_set_hash "$GEN_BIG")" +check "long lines >64kB / 4 workers: split is real (interior offsets distinct, ranges non-empty)" \ + "$(split_real_of "$F_BIG" 4)" 1 # more workers than rows: extra ranges must be empty, load still exact F_TINY=$(make_file 5) -psql_run "TRUNCATE t_col;" >/dev/null -psql_run "\copy t_col FROM '$F_TINY' WITH (FORMAT text)" >/dev/null -check_structure "$F_TINY" 8 "5 rows / 8 workers" -check_reconstruct "$F_TINY" 8 "5 rows / 8 workers (empty ranges)" +check "5 rows / 8 workers: offsets well-formed and newline-aligned" "$(structure_of "$F_TINY" 8)" 1 +check "5 rows / 8 workers (empty ranges): per-range loads reconstruct the whole file" \ + "$(split_load_hash "$F_TINY" 8)" \ + "$(pgc_set_hash "SELECT g AS id, 'host_'||(g%97)||'_x'||g AS txt FROM generate_series(1, 5) g")" # single row: an edge smoke test only. With one record and 4 workers the correct # result is {0, size, size, size} -- the degenerate single-range shape -- so this # case cannot distinguish a working splitter from a degraded one (that is what -# check_split_happened on the 5000-row file is for); it just confirms one record +# the split-is-real checks on the 5000-row file are for); it just confirms one record # does not crash or drop. F_ONE=$(make_file 1) -psql_run "TRUNCATE t_col;" >/dev/null -psql_run "\copy t_col FROM '$F_ONE' WITH (FORMAT text)" >/dev/null -check_structure "$F_ONE" 4 "1 row / 4 workers (edge smoke)" -check_reconstruct "$F_ONE" 4 "1 row / 4 workers (edge smoke)" +check "1 row / 4 workers (edge smoke): offsets well-formed and newline-aligned" \ + "$(structure_of "$F_ONE" 4)" 1 +check "1 row / 4 workers (edge smoke): per-range loads reconstruct the whole file" \ + "$(split_load_hash "$F_ONE" 4)" \ + "$(pgc_set_hash "SELECT g AS id, 'host_'||(g%97)||'_x'||g AS txt FROM generate_series(1, 1) g")" # file with no trailing newline: last range must still run to EOF F_NONL="$DATADIR/nonl.txt" -psql_run "COPY (SELECT g, 'r'||g FROM generate_series(1,1000) g) TO '$F_NONL' WITH (FORMAT text);" >/dev/null +GEN_NONL="SELECT g AS id, 'r'||g AS txt FROM generate_series(1,1000) g" +psql_run "COPY ($GEN_NONL) TO '$F_NONL' WITH (FORMAT text);" >/dev/null # strip the trailing newline truncate -s -1 "$F_NONL" [ "$(id -u)" = "0" ] && chown postgres "$F_NONL" -psql_run "TRUNCATE t_col;" >/dev/null -psql_run "\copy t_col FROM '$F_NONL' WITH (FORMAT text)" >/dev/null -check_structure "$F_NONL" 4 "no trailing newline / 4 workers" -check_reconstruct "$F_NONL" 4 "no trailing newline / 4 workers" +check "no trailing newline / 4 workers: offsets well-formed and newline-aligned" \ + "$(structure_of "$F_NONL" 4)" 1 +check "no trailing newline / 4 workers: per-range loads reconstruct the whole file" \ + "$(split_load_hash "$F_NONL" 4)" "$(pgc_set_hash "$GEN_NONL")" # workers < 1 is rejected (capture stderr into a var first: q() hides stderr, and # a `psql | grep` pipeline would trip set -o pipefail on psql's error exit). @@ -221,15 +263,27 @@ mkpart() { # name nparts -> range-partitioned columnar table over id in [1,5000 psql_run "$ddl" >/dev/null } -for W in 1 2 4; do - mkpart t_pcp 4 - check "parallel_copy(partitioned, $W workers): rows returned = 5000" \ - "$(q "SELECT pgcolumnar.parallel_copy('t_pcp'::regclass, '$F', $W)")" 5000 - check "parallel_copy(partitioned, $W workers): result == single-COPY oracle" \ - "$(pgc_set_hash "SELECT * FROM t_pcp")" "$(pgc_set_hash "SELECT * FROM t_heap")" - check "parallel_copy(partitioned, $W workers): no prepared-transaction leak" \ - "$(q "SELECT count(*) FROM pg_prepared_xacts")" 0 -done +mkpart t_pcp 4 +check "parallel_copy(partitioned, 1 workers): rows returned = 5000" \ + "$(q "SELECT pgcolumnar.parallel_copy('t_pcp'::regclass, '$F', 1)")" 5000 +check "parallel_copy(partitioned, 1 workers): result == single-COPY oracle" \ + "$(pgc_set_hash "SELECT * FROM t_pcp")" "$(pgc_set_hash "SELECT * FROM t_heap")" +check "parallel_copy(partitioned, 1 workers): no prepared-transaction leak" \ + "$(q "SELECT count(*) FROM pg_prepared_xacts")" 0 +mkpart t_pcp 4 +check "parallel_copy(partitioned, 2 workers): rows returned = 5000" \ + "$(q "SELECT pgcolumnar.parallel_copy('t_pcp'::regclass, '$F', 2)")" 5000 +check "parallel_copy(partitioned, 2 workers): result == single-COPY oracle" \ + "$(pgc_set_hash "SELECT * FROM t_pcp")" "$(pgc_set_hash "SELECT * FROM t_heap")" +check "parallel_copy(partitioned, 2 workers): no prepared-transaction leak" \ + "$(q "SELECT count(*) FROM pg_prepared_xacts")" 0 +mkpart t_pcp 4 +check "parallel_copy(partitioned, 4 workers): rows returned = 5000" \ + "$(q "SELECT pgcolumnar.parallel_copy('t_pcp'::regclass, '$F', 4)")" 5000 +check "parallel_copy(partitioned, 4 workers): result == single-COPY oracle" \ + "$(pgc_set_hash "SELECT * FROM t_pcp")" "$(pgc_set_hash "SELECT * FROM t_heap")" +check "parallel_copy(partitioned, 4 workers): no prepared-transaction leak" \ + "$(q "SELECT count(*) FROM pg_prepared_xacts")" 0 # ---- signed keys straddling 0 with an unbounded first partition -------------- # Regression for the bucket miscompare that ignored MINVALUE/MAXVALUE bound kinds: @@ -266,7 +320,7 @@ psql_run "DROP TABLE IF EXISTS t_txtkey CASCADE; tk_err="$(err_of "SELECT pgcolumnar.parallel_copy('t_txtkey'::regclass, '$F', 2)")" check "text partition key is rejected (not a crash)" \ "$(grep -qi "numeric or date/time" <<<"$tk_err" && echo ok || echo no)" ok -check "text partition key: server still up (no crash)" "$(q "SELECT 1")" 1 +check "text partition key: server still up (no crash)" "$(survived "$CRASHES_BEFORE")" 1 # ---- key NOT in column 1, with a generated column before it ------------------- # COPY's default column list skips generated columns, so the splitter's key-field @@ -313,13 +367,16 @@ psql_run "DROP TABLE IF EXISTS t_gap CASCADE; CREATE TABLE t_gap_a PARTITION OF t_gap FOR VALUES FROM (MINVALUE) TO (2000) USING pgcolumnar; CREATE TABLE t_gap_b PARTITION OF t_gap FOR VALUES FROM (3000) TO (MAXVALUE) USING pgcolumnar;" >/dev/null gap_err="$(err_of "SELECT pgcolumnar.parallel_copy('t_gap'::regclass, '$F', 4)")" +# The loader's own error, not any message with "failed" in it: every worker failure +# reads "worker N failed: ", so with the loader's message dropped the old +# pattern still matched "failed: loader exited without reporting a result" (#1341). check "atomic: a loader failure fails the whole load" \ - "$(grep -qiE 'no partition of relation|failed' <<<"$gap_err" && echo ok || echo no)" ok + "$(grep -qi 'failed: no partition of relation' <<<"$gap_err" && echo ok || echo no)" ok check "atomic: loader-failure leaves the target empty (siblings rolled back)" \ "$(q "SELECT count(*) FROM t_gap")" 0 check "atomic: no prepared-transaction leak after loader failure" \ "$(q "SELECT count(*) FROM pg_prepared_xacts")" 0 -check "atomic: server still up after a failed load" "$(q "SELECT 1")" 1 +check "atomic: server still up after a failed load" "$(survived "$CRASHES_BEFORE")" 1 # ---- input must be sorted by the partition key ------------------------------- F_SHUF="$DATADIR/shuf.txt" @@ -353,7 +410,7 @@ mkpart t_pcp 4 mf_err="$(err_of "SELECT pgcolumnar.parallel_copy('t_pcp'::regclass, '$DATADIR/nope.txt', 2)")" check "missing file: errors, not crashes" \ "$(grep -qiE "could not (open|stat)|no such file|not a regular file" <<<"$mf_err" && echo ok || echo no)" ok -check "missing file: server still up (no worker crash)" "$(q "SELECT 1")" 1 +check "missing file: server still up (no worker crash)" "$(survived "$CRASHES_BEFORE")" 1 check "missing file: nothing loaded" "$(q "SELECT count(*) FROM t_pcp")" 0 # ---- a non-columnar (heap) target is rejected -------------------------------- @@ -370,15 +427,27 @@ check "non-columnar target is rejected" \ # skip must default off, so an ordinary write never takes that path. check "bulk_parallel_writer GUC defaults off (core write path unchanged)" \ "$(q "SHOW pgcolumnar.bulk_parallel_writer")" off -for W in 1 2 4; do - psql_run "DROP TABLE IF EXISTS t_single; CREATE TABLE t_single (id int, txt text) USING pgcolumnar;" >/dev/null - check "parallel_copy(single table, $W workers): rows returned = 5000" \ - "$(q "SELECT pgcolumnar.parallel_copy('t_single'::regclass, '$F', $W)")" 5000 - check "parallel_copy(single table, $W workers): result == single-COPY oracle" \ - "$(pgc_set_hash "SELECT * FROM t_single")" "$(pgc_set_hash "SELECT * FROM t_heap")" - check "parallel_copy(single table, $W workers): no prepared-transaction leak" \ - "$(q "SELECT count(*) FROM pg_prepared_xacts")" 0 -done +psql_run "DROP TABLE IF EXISTS t_single; CREATE TABLE t_single (id int, txt text) USING pgcolumnar;" >/dev/null +check "parallel_copy(single table, 1 workers): rows returned = 5000" \ + "$(q "SELECT pgcolumnar.parallel_copy('t_single'::regclass, '$F', 1)")" 5000 +check "parallel_copy(single table, 1 workers): result == single-COPY oracle" \ + "$(pgc_set_hash "SELECT * FROM t_single")" "$(pgc_set_hash "SELECT * FROM t_heap")" +check "parallel_copy(single table, 1 workers): no prepared-transaction leak" \ + "$(q "SELECT count(*) FROM pg_prepared_xacts")" 0 +psql_run "DROP TABLE IF EXISTS t_single; CREATE TABLE t_single (id int, txt text) USING pgcolumnar;" >/dev/null +check "parallel_copy(single table, 2 workers): rows returned = 5000" \ + "$(q "SELECT pgcolumnar.parallel_copy('t_single'::regclass, '$F', 2)")" 5000 +check "parallel_copy(single table, 2 workers): result == single-COPY oracle" \ + "$(pgc_set_hash "SELECT * FROM t_single")" "$(pgc_set_hash "SELECT * FROM t_heap")" +check "parallel_copy(single table, 2 workers): no prepared-transaction leak" \ + "$(q "SELECT count(*) FROM pg_prepared_xacts")" 0 +psql_run "DROP TABLE IF EXISTS t_single; CREATE TABLE t_single (id int, txt text) USING pgcolumnar;" >/dev/null +check "parallel_copy(single table, 4 workers): rows returned = 5000" \ + "$(q "SELECT pgcolumnar.parallel_copy('t_single'::regclass, '$F', 4)")" 5000 +check "parallel_copy(single table, 4 workers): result == single-COPY oracle" \ + "$(pgc_set_hash "SELECT * FROM t_single")" "$(pgc_set_hash "SELECT * FROM t_heap")" +check "parallel_copy(single table, 4 workers): no prepared-transaction leak" \ + "$(q "SELECT count(*) FROM pg_prepared_xacts")" 0 # ---- concurrency witness: prove the writers actually shared the storage ------- # The oracle-equality checks above pass even if the loaders serialised by accident, @@ -405,14 +474,13 @@ psql_run "DROP TABLE IF EXISTS t_wit; CREATE TABLE t_wit (id int, txt text) USIN SELECT pgcolumnar.set_options('t_wit'::regclass, chunk_group_row_limit => 1000, stripe_row_limit => 2000);" >/dev/null -WN=4 -check "witness: parallel_copy(single table, $WN workers, 40000 rows) loads every row" \ - "$(q "SELECT pgcolumnar.parallel_copy('t_wit'::regclass, '$F_W', $WN)")" 40000 +check "witness: parallel_copy(single table, 4 workers, 40000 rows) loads every row" \ + "$(q "SELECT pgcolumnar.parallel_copy('t_wit'::regclass, '$F_W', 4)")" 40000 check "witness: result == single-COPY oracle" \ "$(pgc_set_hash "SELECT * FROM t_wit")" "$(pgc_set_hash "SELECT * FROM t_wit_heap")" # (a) more distinct stripes than workers: concurrent reservations, >1 group per worker -check "witness: >$WN distinct stripe ids (each worker reserved several stripes)" \ - "$(q "SELECT count(DISTINCT stripeid) > $WN FROM pgcolumnar.stats('t_wit'::regclass)")" t +check "witness: >4 distinct stripe ids (each worker reserved several stripes)" \ + "$(q "SELECT count(DISTINCT stripeid) > 4 FROM pgcolumnar.stats('t_wit'::regclass)")" t # (b) no two stripe byte ranges overlap (each overlap counted twice; want 0) check "witness: no stripe byte-range overlaps another (no on-disk collision)" \ "$(q "WITH s AS (SELECT stripeid, fileoffset AS lo, fileoffset+datalength AS hi @@ -428,8 +496,9 @@ check "witness: no prepared-transaction leak" \ # single-table atomicity: a bad row rolls back the whole load, no leak psql_run "DROP TABLE IF EXISTS t_single; CREATE TABLE t_single (id int, txt text) USING pgcolumnar;" >/dev/null st_bad="$(err_of "SELECT pgcolumnar.parallel_copy('t_single'::regclass, '$F_BADKEY', 4)")" +# The loader's own error, for the reason given at the loader-failure check above. check "single table: a bad row fails the whole load" \ - "$(grep -qiE 'invalid input syntax|failed' <<<"$st_bad" && echo ok || echo no)" ok + "$(grep -qi 'failed: invalid input syntax' <<<"$st_bad" && echo ok || echo no)" ok check "single table: failed load leaves the target empty" "$(q "SELECT count(*) FROM t_single")" 0 check "single table: no prepared-transaction leak after failure" "$(q "SELECT count(*) FROM pg_prepared_xacts")" 0 @@ -456,7 +525,7 @@ check "parallel_copy on a FIFO is refused, not a hang (42809)" \ "$(sqlstate_or_hang "SELECT pgcolumnar.parallel_copy('t_fifo'::regclass, '$PCFIFO', 2)")" \ "42809" exec 9<>"$PCFIFO" 2>/dev/null; exec 9>&- 2>/dev/null # release any blocked opener -check "backend still up after the parallel_copy FIFO refusal" "$(q 'SELECT 1')" "1" +check "backend still up after the parallel_copy FIFO refusal" "$(survived "$CRASHES_BEFORE")" "1" # The partition-aligned coordinator (pcopy_partition_aligned_offsets, reached only # for a PARTITIONED target) opens the file via getline() and had the same @@ -468,6 +537,6 @@ check "parallel_copy on a FIFO via the partitioned coordinator is refused, not a "$(sqlstate_or_hang "SELECT pgcolumnar.parallel_copy('t_fifo_part'::regclass, '$PCFIFO2', 3)")" \ "42809" exec 9<>"$PCFIFO2" 2>/dev/null; exec 9>&- 2>/dev/null -check "backend still up after the partitioned-coordinator FIFO refusal" "$(q 'SELECT 1')" "1" +check "backend still up after the partitioned-coordinator FIFO refusal" "$(survived "$CRASHES_BEFORE")" "1" pgc_summary diff --git a/test/parallel_copy_dedup.sh b/test/parallel_copy_dedup.sh index 9fdc44390..ba768168d 100755 --- a/test/parallel_copy_dedup.sh +++ b/test/parallel_copy_dedup.sh @@ -49,35 +49,53 @@ DATADIR="$PGC_WORKDIR/pcdedup" mkdir -p "$DATADIR"; chmod 777 "$DATADIR" if [ "$(id -u)" = "0" ]; then chown postgres "$DATADIR"; fi -ROWS=20000 +# The coordinator reads the file 256 KiB at a time (PCOPY_FINGERPRINT_BUFSZ). The +# changed file G differs from F ONLY in its last row, and 25,000 rows put that row +# past the first read. When G differed from its first bytes and the file fitted in +# one read, a fingerprint of the first read alone loaded G as a change and this +# suite passed 14 of 14. +ROWS=25000 F="$DATADIR/load.txt" G="$DATADIR/other.txt" psql_run "COPY (SELECT g AS id, 'v'||g AS txt FROM generate_series(1,$ROWS) g) TO '$F' WITH (FORMAT text);" >/dev/null -psql_run "COPY (SELECT g AS id, 'w'||g AS txt FROM generate_series(1,$ROWS) g) - TO '$G' WITH (FORMAT text);" >/dev/null +psql_run "COPY (SELECT g AS id, CASE WHEN g = $ROWS THEN 'changed' ELSE 'v'||g END AS txt + FROM generate_series(1,$ROWS) g) TO '$G' WITH (FORMAT text);" >/dev/null q() { env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -Atq \ -c "$1" 2>&1 | tail -1 } +# The SHA-256 of each file's bytes, computed by the server from the file itself. +sha() { q "SELECT encode(sha256(pg_read_binary_file('$1')), 'hex')"; } +sha_f="$(sha "$F")" +sha_g="$(sha "$G")" +check_num "premise: the changed row lies past the first 256 KiB of the file" \ + "$(q "SELECT ((SELECT size FROM pg_stat_file('$G')) + - octet_length('$ROWS'||E'\tchanged\n') >= 262144)::int")" 1 + mk() { psql_run "DROP TABLE IF EXISTS $1; CREATE TABLE $1 (id int, txt text) USING pgcolumnar;" >/dev/null; } # ---- the control: without dedup the retry doubles the table ---------------- mk pcd_plain r1="$(q "SELECT pgcolumnar.parallel_copy('pcd_plain','$F',4)")" r2="$(q "SELECT pgcolumnar.parallel_copy('pcd_plain','$F',4)")" -check "premise: without dedup a repeated load doubles the table, which is the defect" \ +# Not premises (#1339): dedup is pgcolumnar's own default, and forcing it on reddens +# both of these. +check "without dedup a repeated load doubles the table, which is the defect" \ "$(q "SELECT count(*) FROM pcd_plain")" "$(( ROWS * 2 ))" -check "premise: and each of those loads reported its rows" "$r1/$r2" "$ROWS/$ROWS" +check "and each of those loads reported its rows" "$r1/$r2" "$ROWS/$ROWS" # ---- the same load twice, with dedup -------------------------------------- mk pcd d1="$(q "SELECT pgcolumnar.parallel_copy('pcd','$F',4,true)")" c1="$(q "SELECT count(*) FROM pcd")" -d2="$(q "SELECT pgcolumnar.parallel_copy('pcd','$F',4,true)")" +# All of psql's output, NOTICE included: the header promises the skip says so. +o2="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -Atq \ + -c "SELECT pgcolumnar.parallel_copy('pcd','$F',4,true)" 2>&1)" +d2="$(printf '%s\n' "$o2" | tail -1)" c2="$(q "SELECT count(*) FROM pcd")" echo "-- dedup on: load 1 returned $d1 (table $c1), load 2 returned $d2 (table $c2)" @@ -85,6 +103,8 @@ check "the first load stores its rows normally (#403 item 7)" "$c1" "$ROWS" check "and reports them" "$d1" "$ROWS" check "the second load of the same file stores nothing (#403 item 7)" "$c2" "$ROWS" check "and reports 0 rather than claiming it loaded them" "$d2" "0" +check_num "and says in a NOTICE that it skipped the load, rather than returning a silent 0" \ + "$(printf '%s\n' "$o2" | grep -c 'NOTICE: pgcolumnar.parallel_copy skipped a load already applied to "pcd"')" 1 # ---- a changed file at the same path is a different load ------------------- cp "$G" "$F" @@ -112,6 +132,13 @@ check "the skipped load left exactly one record for that table and file" \ "$(q "SELECT count(*) FROM pgcolumnar.load_fingerprint WHERE relation_oid = 'pcd'::regclass")" "2" +# The SHA-256 of the file's bytes, not of some part of them: the two records are +# exactly the two files' digests, in load order. +check_text "the records hold the SHA-256 of each loaded file's bytes" \ + "$(q "SELECT string_agg(encode(fingerprint, 'hex'), ',' ORDER BY loaded_at) + FROM pgcolumnar.load_fingerprint WHERE relation_oid = 'pcd'::regclass")" \ + "$sha_f,$sha_g" + check "no prepared transaction leaked through any of it" \ "$(q "SELECT count(*) FROM pg_prepared_xacts")" "0" diff --git a/test/projection_drop_column.sh b/test/projection_drop_column.sh index 724631665..8500ba707 100755 --- a/test/projection_drop_column.sh +++ b/test/projection_drop_column.sh @@ -39,10 +39,39 @@ psql_run "INSERT INTO pdc # The pre-statement dependency check must not tell a stranger which column the # projection covers, or let that role take the lock retained by the owner path. psql_run "CREATE ROLE pdc_nobody LOGIN;" -check "a non-owner learns nothing from a projected column" \ - "$(sqlstate_as pdc_nobody 'ALTER TABLE pdc DROP COLUMN sort_key;')" "42501" +proj_state="$(sqlstate_as pdc_nobody 'ALTER TABLE pdc DROP COLUMN sort_key;')" +plain_state="$(sqlstate_as pdc_nobody 'ALTER TABLE pdc DROP COLUMN payload;')" +# The property is that the two refusals are INDISTINGUISHABLE, so it is compared +# first: with both ownership checks deleted, the stranger gets 2BP01 for the +# projected column and 42501 for the other, and this is the check that names it. +check "and the two refusals are the same error, so neither leaks" \ + "$proj_state" "$plain_state" +check "a non-owner learns nothing from a projected column" "$proj_state" "42501" check "the same non-owner error is returned for an unprojected column" \ - "$(sqlstate_as pdc_nobody 'ALTER TABLE pdc DROP COLUMN payload;')" "42501" + "$plain_state" "42501" + +# THE LOCK HALF OF THE COMMENT ABOVE (#1339). The ownership check runs twice: +# once with no lock, then again after ShareUpdateExclusiveLock. The two checks +# above cannot tell them apart, because both raise 42501. Contention can: while +# the owner holds SHARE on pdc, which conflicts with ShareUpdateExclusiveLock, a +# stranger refused by the unlocked check gets 42501 at once, and one that reaches +# the lock first waits out its lock_timeout (55P03). +holder_log="$PGC_WORKDIR/pdc_holder.log" +env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -Atq -v ON_ERROR_STOP=0 -c "BEGIN;" -c "LOCK TABLE pdc IN SHARE MODE;" \ + -c "SELECT pg_sleep(3600) /* pdc_holder */;" -c "ROLLBACK;" > "$holder_log" 2>&1 & +holder_pid=$! +held=0 +for _ in $(seq 1 40); do + held="$(q "SELECT count(*) FROM pg_locks WHERE relation = 'pdc'::regclass AND mode = 'ShareLock' AND granted;")" + [ "${held:-0}" -ge 1 ] && break + sleep 0.25 +done +check "premise: the owner holds SHARE on pdc before the stranger asks" "${held:-0}" "1" +check "a non-owner is refused before it waits for the table lock" \ + "$(sqlstate_as pdc_nobody "SET lock_timeout = '2s'; ALTER TABLE pdc DROP COLUMN sort_key;")" "42501" +psql_run "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE query LIKE '%pdc_holder%' AND pid <> pg_backend_pid();" >/dev/null +wait "$holder_pid" 2>/dev/null || true # Red before the fix: PostgreSQL accepts this (00000), and the next INSERT # reaches the projection writer with a type OID of zero. diff --git a/test/projection_parallel.sh b/test/projection_parallel.sh index 010951c60..7d041d14f 100755 --- a/test/projection_parallel.sh +++ b/test/projection_parallel.sh @@ -277,10 +277,13 @@ check "premise: the covering plan is parallel at the default page cost" \ check "a page cost that makes I/O dominate costs the parallel covering path" \ "$(shape "$(explain_cov_spc $SPC_HI)")" "projection-only" -check "premise: EXPLAIN ANALYZE launched two workers" \ +# NOT PREMISES (#1339, #1236). The worker count is the projection path's own +# parallel_workers: with it set to 1, Workers Launched reads 1 and both of these +# go red. A check a pgcolumnar change can redden is an arm. +check "EXPLAIN ANALYZE launched two workers" \ "$(echo "$par_on_ana" | grep -oE 'Workers Launched: [0-9]+' | head -1 | grep -oE '[0-9]+')" "2" -check "premise: ANALYZE printed a rows= line per launched worker" \ +check "ANALYZE printed a rows= line per launched worker" \ "$n_lines" "2" # THE DEFECT the plan-shape arms cannot see: Gather is present and the count diff --git a/test/projection_privilege.sh b/test/projection_privilege.sh index 579903195..f369c9249 100755 --- a/test/projection_privilege.sh +++ b/test/projection_privilege.sh @@ -191,8 +191,12 @@ check "premise: t_prjexec can open a session, so a refusal below is a refusal" \ "$(as t_prjexec 'SELECT 1;' | head -1)" "1" check "a role with EXECUTE but no SELECT is refused read_projection" \ "$(count_as t_prjexec read_projection)" "refused" +# The SQLSTATE, not merely "refused", as the pytest twin asserts (#1339): this +# function has no second arm reading the code, so "refused" alone accepted any +# error at all, a login FATAL included. check "and is refused reconstruct_via_projection, which leaks non-covered columns" \ - "$(count_as t_prjexec reconstruct_via_projection)" "refused" + "$(state_as t_prjexec "SELECT count(*) FROM pgcolumnar.reconstruct_via_projection('secret','p1');")" \ + "42501" check "and THAT refusal is the C check, which raises 42501 from aclcheck_error" \ "$(state_as t_prjexec "SELECT count(*) FROM pgcolumnar.read_projection('secret','p1');")" \ "42501" @@ -241,7 +245,8 @@ check_num "the policy is in force for ordinary SQL" \ check "read_projection now refuses a policy-restricted caller (#563)" \ "$(count_as t_prjsel read_projection)" "refused" check "and so does reconstruct_via_projection" \ - "$(count_as t_prjsel reconstruct_via_projection)" "refused" + "$(state_as t_prjsel "SELECT count(*) FROM pgcolumnar.reconstruct_via_projection('secret','p1');")" \ + "0A000" # 0A000, not a phrase. PgColumnarRequireNoRowSecurity raises # ERRCODE_FEATURE_NOT_SUPPORTED, which is a different SQLSTATE CLASS from either # ACL refusal -- 0A against 42 -- so no rewording of any message can confuse them. diff --git a/test/projection_scan_io.sh b/test/projection_scan_io.sh index 9aaefc3e6..6dff914db 100755 --- a/test/projection_scan_io.sh +++ b/test/projection_scan_io.sh @@ -200,8 +200,21 @@ check "premise: the empty projection is found and its storage holds no row group GROUP BY p.proj_storage_id;")" \ "true/0" +psio_e_plan="$(explain_scan on "SELECT ik FROM psio_e WHERE ik BETWEEN 1 AND 50")" check "an empty covering projection is still priced as one page, so it is chosen" \ - "$(explain_scan on "SELECT ik FROM psio_e WHERE ik BETWEEN 1 AND 50" | grep -c 'Columnar Projection: byik')" \ + "$(printf '%s\n' "$psio_e_plan" | grep -c 'Columnar Projection: byik')" \ "1" +# THE OTHER DIRECTION (#1339). The arm above catches over-pricing only: priced +# at 0 pages instead of 1, the projection is cheaper still and stays chosen. +# With every CPU cost set to 0 by explain_scan, the projection scan's whole cost +# is its page cost, so a total of zero means it was priced at zero pages. +# Measured on PG17: 5.00 with the clamp, 0.00 with `if (pages < 1) pages = 1;` +# deleted, and the projection chosen both times. +psio_e_cost="$(printf '%s\n' "$psio_e_plan" | grep -m1 'Custom Scan' | + sed -n 's/.*cost=[0-9.]*\.\.\([0-9.]*\) .*/\1/p')" +check "and its scan is not priced at zero pages" \ + "$(awk -v c="${psio_e_cost:-none}" 'BEGIN { if (c ~ /^[0-9]+(\.[0-9]+)?$/) print (c + 0 > 0 ? "priced" : "free at " c); else print "unread: " c }')" \ + "priced" + pgc_summary diff --git a/test/projections.sh b/test/projections.sh index 5a546d47b..54313063f 100755 --- a/test/projections.sh +++ b/test/projections.sh @@ -194,6 +194,19 @@ check "projection-scan results match heap oracle" \ check "aggregate over projection scan matches oracle" \ "$(q "SELECT count(*), sum(a) FROM ps WHERE c BETWEEN 100 AND 200;")" \ "$(q "SELECT count(*), sum(a) FROM ps_h WHERE c BETWEEN 100 AND 200;")" +# THE EXECUTOR, NOT THE PLANNER (#1339). The first arm reads the name from +# EXPLAIN, which takes it before the executor decides what to open, and the two +# oracle arms pass whichever storage is read. So a scan that EXPLAINs as `pc` and +# reads the base passed all three. Executed, the sorted projection lets the +# reader skip the vectors outside 100..200 and the filter sees only part of the +# table; the base, unsorted on c, sends every row to the filter. +pj_removed="$(q "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) SELECT a, c FROM ps WHERE c BETWEEN 100 AND 200;" | + grep -m1 'Rows Removed by Filter:' | grep -oE '[0-9.]+$')" +pj_outside="$(q "SELECT count(*) FROM ps_h WHERE NOT (c BETWEEN 100 AND 200);")" +check "executing it reads the projection, so the filter sees fewer rows than the base holds" \ + "$(if pgc_is_number "$pj_removed" && pgc_is_number "$pj_outside"; then + awk -v r="$pj_removed" -v o="$pj_outside" 'BEGIN { print (r + 0 < o + 0 ? "fewer" : "all " r " of " o) }' + else echo "unread: [$pj_removed] [$pj_outside]"; fi)" "fewer" check "GUC off: no projection scan" \ "$(q "SET pgcolumnar.enable_projection_scan=off; EXPLAIN (COSTS OFF) SELECT a, c FROM ps WHERE c BETWEEN 100 AND 200;" | grep -c 'Columnar Projection')" "0" @@ -266,6 +279,12 @@ psql_run "CREATE TABLE pm (a int, c int) USING pgcolumnar;" psql_run "SELECT pgcolumnar.add_projection('pm', 'pmp', ARRAY['a','c'], ARRAY['c']);" psql_run "INSERT INTO pm SELECT g, g FROM generate_series(1,10000) g;" # batch 1 +# THE PLAN THE ARMS BELOW NAME (#1339). Four of them say "via projection" or +# "projection scan", and nothing asked how this count plans. Planned on the base, +# every one of them passes whatever the projection scan does with the snapshot. +check "the old-snapshot count is planned as a projection scan" \ + "$(q "SET pgcolumnar.enable_projection_scan = on; EXPLAIN (COSTS OFF) SELECT count(a) FROM pm WHERE c BETWEEN 1 AND 20000;" | grep -c 'Columnar Projection: pmp')" "1" + A_IN="$PGC_WORKDIR/pmA.in"; A_OUT="$PGC_WORKDIR/pmA.out" mkfifo "$A_IN"; : > "$A_OUT" env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ @@ -273,7 +292,10 @@ env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ A_PID=$! exec 9> "$A_IN" pa_send() { printf '%s\n' "$*" >&9; } -pa_wait() { local t="$1" i; for i in $(seq 1 200); do grep -q "$t" "$A_OUT" && return 0; sleep 0.1; done; return 1; } +# The token must carry a count. An ERROR echoes the statement, token included +# ("LINE 1: SELECT 'PMA_' || ..."), so matching the bare token read a failed +# query as an answer and the timeout arms below could not fire on it (#1339). +pa_wait() { local t="$1" i; for i in $(seq 1 200); do grep -qE "${t}[0-9]+" "$A_OUT" && return 0; sleep 0.1; done; return 1; } # A opens a snapshot that sees batch 1, using a projection scan. pa_send "SET pgcolumnar.enable_projection_scan = on;" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index a7fb24ab9..e5b59a5e6 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -341,6 +341,9 @@ behaviour, the source of that number is named. - [293. test_native_fetch_sort_context.py: a merge join over sorted, index-fetched columnar rows decodes outside the sort's context](#293-test_native_fetch_sort_contextpy-a-merge-join-over-sorted-index-fetched-columnar-rows-decodes-outside-the-sorts-context) - [294. test_native_fold_skipguard.py: the batch fold is reachable for a shape that skips vectors, and accounts for the skipping](#294-test_native_fold_skipguardpy-the-batch-fold-is-reachable-for-a-shape-that-skips-vectors-and-accounts-for-the-skipping) - [295. test_native_fetch_group_memo.py: the by-row-number fetch resolves its row group from a memo, and the memo never serves a group it should not](#295-test_native_fetch_group_memopy-the-by-row-number-fetch-resolves-its-row-group-from-a-memo-and-the-memo-never-serves-a-group-it-should-not) +- [296. test_parallel_copy_dedup.py: pgcolumnar.parallel_copy can refuse a load it has already done](#296-test_parallel_copy_deduppy-pgcolumnarparallel_copy-can-refuse-a-load-it-has-already-done) +- [297. test_parallel_copy.py: parallel_copy splits a file on record boundaries and loads it atomically](#297-test_parallel_copypy-parallel_copy-splits-a-file-on-record-boundaries-and-loads-it-atomically) +- [298. test_parallel.py: a parallel columnar scan returns what the heap returns](#298-test_parallelpy-a-parallel-columnar-scan-returns-what-the-heap-returns) ## 1. How to read a test in here @@ -4357,7 +4360,7 @@ the tool grades THIS tree. | `test_the_grader_itself_refuses_the_suite_it_cannot_read` | `main` exits 2 and prints no verdict, with a readable suite as the control | | `test_a_helper_reaching_only_the_primitive_is_found` | the closure is seeded from `pgc_record`, not the `check` family; four suites turn on it | | `test_a_longer_helper_name_is_not_shadowed_by_a_shorter_one` | `check_ratio` must not eat `check_ratio_needs_quiet_machine` | -| `test_the_suite_local_helpers_are_known_and_excluded` | the four suite-local helpers, and that none of their suites is graded | +| `test_the_suite_local_helpers_are_known_and_excluded` | no suite defines its own check_ helper, which the grader would not read | | `test_every_pair_in_the_tree_is_declared` | the declaration is asserted BOTH ways, so a new pair cannot be silently ungraded | | `test_every_registered_suite_is_ported_or_declared_unported` | every registered suite with no twin is listed in `unported_suites.txt`, a ported one leaves it, and no listed stem is phantom (#1341); the parity goal (#432) is that file empty | | `test_the_ported_suites_in_this_tree_are_graded_one_for_one` | the standing arm: every pair in the tree, graded | @@ -6946,11 +6949,11 @@ Removal proofs, each red in both twins: the prototype's zone handling; `CREATE T Port of `approx_aggregates.sh` (#1296 phase 2). Phase 0 on #1296 decided that approximate distinct counts and percentiles come from the `hll` and `tdigest` extensions for 1.0, so this pair is the tested compatibility statement for them; docs/how-to.md names the versions. -Independent of the shell suite: its own generator (other multipliers, the same 200,000 rows and 50,000 devices, because the check names state them), its own SQL. `compare_to_bash` grades the pair 14 literal matches, 0 missing. +Independent of the shell suite: its own generator (other multipliers, the same 200,000 rows and 50,000 devices, because the check names state them), its own SQL. `compare_to_bash` grades the pair 15 literal matches, 0 missing. THE HEAP MIRROR IS WRITTEN FROM THE GENERATOR, never copied from the columnar table. The shell suite first built it by `CREATE TABLE ... AS SELECT * FROM ax`, which reads through the scan under test: with a mutation dropping one row in a thousand, every equality stayed green on all five majors. Writing both sides from the generator turned seven checks red under the same mutation. -The bounds against the exact answer are premises, as pg_partman's are: they state the extension's own error, and a change here that broke them would break the equalities first. +The bounds against the exact answer are premises, as pg_partman's are: they state the extension's own error, and a change here that broke them would break the equalities first. That holds for the stored tdigest only since #1339 added "a tdigest stored in a columnar table comes back byte for byte": before it, nothing compared the stored digests, and a reader returning each long value as the previous one passed every tdigest check. The digests are built once into a heap table and stored from there, because a tdigest depends on the order its inputs arrive in. hll 2.19 does not build against PostgreSQL 19, so `test_hll_over_columnar` is `UNSUPPORTED_MAJOR` there and listed in `expected_unrunnable.txt`. CI installs both extensions from PGDG for 15 to 18. @@ -11352,3 +11355,108 @@ Every test makes one check under the shell's name, on a private database built o | `test_fgm_delete_visible_to_the_next_fetch` | a DELETE is visible to the very next fetch (delete vectors uncached) | | `test_fgm_same_transaction_truncate_then_fetch` | same-transaction TRUNCATE then fetch sees only the new rows | | `test_fgm_backend_alive` | backend alive | + +## 296. test_parallel_copy_dedup.py: pgcolumnar.parallel_copy can refuse a load it has already done + +The pytest twin of `test/parallel_copy_dedup.sh` (ported for #1341), every check by the shell's name. With `dedup => true`, `pgcolumnar.parallel_copy` refuses a second load of a file whose bytes it already loaded into the same table (#403 item 7): it stores nothing, returns 0 and says so in a NOTICE. Without dedup the same load doubles the table; a file changed only in its last row, past the coordinator's first 256 KiB read, loads; the record is per table; and the table's records are exactly the server's `sha256()` of the two files it loaded. On a private database, with the files in `server_dir`. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_pcdedup_the_changed_row_lies_past_the_first_read` | premise: the changed row lies past the first 256 KiB of the file | +| `test_pcdedup_without_dedup_a_repeated_load_doubles_the_table` | without dedup a repeated load doubles the table, which is the defect | +| `test_pcdedup_each_plain_load_reported_its_rows` | and each of those loads reported its rows | +| `test_pcdedup_the_first_load_stores_its_rows` | the first load stores its rows normally (#403 item 7) | +| `test_pcdedup_the_first_load_reports_them` | and reports them | +| `test_pcdedup_the_second_load_of_the_same_file_stores_nothing` | the second load of the same file stores nothing (#403 item 7) | +| `test_pcdedup_the_second_load_reports_0` | and reports 0 rather than claiming it loaded them | +| `test_pcdedup_the_skip_says_so_in_a_notice` | and says in a NOTICE that it skipped the load, rather than returning a silent 0 | +| `test_pcdedup_a_changed_file_at_the_same_path_is_loaded` | a changed file at the same path is loaded, not skipped (#403 item 7) | +| `test_pcdedup_the_changed_file_reports_its_rows` | and reports the rows it loaded | +| `test_pcdedup_the_same_file_loads_into_a_different_table` | the same file loads into a DIFFERENT table (the record is per table) | +| `test_pcdedup_the_other_table_load_reports_its_rows` | and reports its rows | +| `test_pcdedup_dedup_off_loads_again` | asking for no dedup loads again even though the file is on record | +| `test_pcdedup_dedup_off_reports_those_rows` | and reports those rows too | +| `test_pcdedup_the_skipped_load_left_no_record` | the skipped load left exactly one record for that table and file | +| `test_pcdedup_the_records_hold_each_files_sha256` | the records hold the SHA-256 of each loaded file's bytes | +| `test_pcdedup_no_prepared_transaction_leaked` | no prepared transaction leaked through any of it | + +## 297. test_parallel_copy.py: parallel_copy splits a file on record boundaries and loads it atomically + +The pytest twin of `test/parallel_copy.sh` (ported for #1341), every check by the shell's name. `pgcolumnar.file_split_offsets` cuts a text file into record-aligned ranges that cover it, and loading the ranges one by one gives back the rows the file was generated from. `pgcolumnar.parallel_copy` loads a partitioned or a single columnar target with the rows a single COPY gives, leaves no prepared transaction, rolls every worker back when one fails, and refuses a text key, an unsorted file, a DEFAULT partition, too many workers, a missing file, a heap target and a FIFO. A survival check reads the server log's count of killed backends as well as `SELECT 1`. On a private database. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_parallel_copy_single_copy_heap_and_columnar_agree` | a single COPY of the 5000-row file into heap and into pgcolumnar gives the same rows | +| `test_parallel_copy_split_offsets_are_newline_aligned` | `file_split_offsets` returns workers+1 offsets from 0 to the file size, never decreasing, each interior one right after a newline: 9 tests, one per file and worker count | +| `test_parallel_copy_range_loads_give_back_the_generated_rows` | loading each range on its own gives back the rows the file was generated from: 9 tests, one per file and worker count | +| `test_parallel_copy_every_range_of_a_large_file_is_non_empty` | on a file with far more records than workers, every range is non-empty, so a split degraded to one range fails: 5 tests | +| `test_parallel_copy_zero_workers_is_rejected` | workers < 1 is refused ("at least 1") | +| `test_parallel_copy_split_of_a_directory_is_rejected` | a directory is refused as not a regular file, not split as an 8-exabyte file | +| `test_parallel_copy_returns_every_row_loaded` | `parallel_copy` into a partitioned and into a single table, with 1, 2 and 4 workers, returns 5000: 6 tests | +| `test_parallel_copy_load_equals_the_single_copy` | each of those loads holds the rows of a single heap COPY of the file: 6 tests | +| `test_parallel_copy_load_leaves_no_prepared_transaction` | each of those loads leaves no prepared transaction: 6 tests | +| `test_parallel_copy_signed_keys_return_every_row` | keys from -100 to 399 with an unbounded first partition: 500 rows returned | +| `test_parallel_copy_signed_keys_equal_the_oracle` | that load equals its heap oracle (no misbucketing, no deadlock) | +| `test_parallel_copy_signed_keys_leave_no_prepared_transaction` | that load leaves no prepared transaction | +| `test_parallel_copy_text_partition_key_is_rejected` | a text partition key is refused ("numeric or date/time") | +| `test_parallel_copy_text_partition_key_leaves_the_server_up` | the server answers and no backend was killed since the suite began | +| `test_parallel_copy_key_after_a_generated_column_returns_every_row` | a key after a generated column: 3000 rows returned | +| `test_parallel_copy_key_after_a_generated_column_equals_the_oracle` | that load equals its heap oracle | +| `test_parallel_copy_bad_partition_key_value_is_rejected` | a non-integer key is refused by the splitter ("invalid input syntax") | +| `test_parallel_copy_bad_partition_key_loads_nothing` | the refused load leaves the target empty | +| `test_parallel_copy_bad_partition_key_leaves_no_prepared_transaction` | the refused load leaves no prepared transaction | +| `test_parallel_copy_loader_failure_fails_the_load_with_its_cause` | rows routed into a partition gap fail the load with the loader's own message ("failed: no partition of relation") | +| `test_parallel_copy_loader_failure_rolls_back_the_siblings` | the failed load leaves the target empty: the prepared siblings were rolled back | +| `test_parallel_copy_loader_failure_leaves_no_prepared_transaction` | the failed load leaves no prepared transaction | +| `test_parallel_copy_loader_failure_leaves_the_server_up` | the server answers and no backend was killed | +| `test_parallel_copy_unsorted_input_is_rejected` | a shuffled file is refused ("not sorted") | +| `test_parallel_copy_unsorted_input_loads_nothing` | the refused load leaves the target empty | +| `test_parallel_copy_default_partition_is_rejected` | a target with a DEFAULT partition is refused | +| `test_parallel_copy_prepared_transaction_guard_fires` | 10 workers against max_prepared_transactions = 8 are refused up front, naming the setting | +| `test_parallel_copy_prepared_transaction_guard_loads_nothing` | the refused load leaves the target empty | +| `test_parallel_copy_missing_file_errors` | a missing file is refused with a file error | +| `test_parallel_copy_missing_file_leaves_the_server_up` | the server answers and no backend was killed | +| `test_parallel_copy_missing_file_loads_nothing` | the refused load leaves the target empty | +| `test_parallel_copy_heap_target_is_rejected` | a heap target is refused ("not a pgcolumnar table") | +| `test_parallel_copy_bulk_parallel_writer_is_off_by_default` | `pgcolumnar.bulk_parallel_writer` is off by default | +| `test_parallel_copy_witness_loads_every_row` | 4 workers load 40000 rows into one table with small stripes: 40000 returned | +| `test_parallel_copy_witness_equals_the_oracle` | that load equals its heap oracle | +| `test_parallel_copy_witness_has_more_stripes_than_workers` | more than 4 distinct stripes: each worker reserved several | +| `test_parallel_copy_witness_stripes_do_not_overlap` | no two stripes' byte ranges overlap | +| `test_parallel_copy_witness_stripes_cover_every_row` | the stripes' row counts sum to 40000 | +| `test_parallel_copy_witness_leaves_no_prepared_transaction` | that load leaves no prepared transaction | +| `test_parallel_copy_single_table_bad_row_fails_with_its_cause` | a bad row fails a single-table load with the loader's own message ("failed: invalid input syntax") | +| `test_parallel_copy_single_table_failure_loads_nothing` | the failed load leaves the target empty | +| `test_parallel_copy_single_table_failure_leaves_no_prepared_transaction` | the failed load leaves no prepared transaction | +| `test_parallel_copy_fifo_is_refused_not_a_hang` | a FIFO given to a single-table load is refused with 42809 within 5 seconds (#686) | +| `test_parallel_copy_fifo_refusal_leaves_the_backend_up` | the server answers and no backend was killed | +| `test_parallel_copy_fifo_on_a_partitioned_target_is_refused_not_a_hang` | a FIFO given to a partitioned load is refused with 42809 within 5 seconds | +| `test_parallel_copy_partitioned_fifo_refusal_leaves_the_backend_up` | the server answers and no backend was killed | + +## 298. test_parallel.py: a parallel columnar scan returns what the heap returns + +The pytest twin of `test/parallel.sh` (ported for #1341), every check by the shell's name. A 50000-row table is written in 25 row groups of 2000 rows, and the planner settings force a parallel plan, so several backends claim distinct row groups from one shared counter. Each "par" query must plan a Gather over a PgColumnarScan and return the heap's rows; forced serial, the same queries plan no Gather and still return the heap's rows. On a private database. + +### Every test + +The table must hold 25 row groups: with one, a claim that skips every other group returns the right rows. "par no-filter" returns rows rather than count(*), which the vectorized aggregate answered without a Gather. Two plan counts say every "par" query planned a Gather and every "serial" one did not. + +| test | what it holds | +| --- | --- | +| `test_parallel_table_has_row_groups_to_claim` | par table has 25 row groups to claim | +| `test_parallel_plan_has_a_gather` | parallel plan has Gather | +| `test_parallel_plan_has_the_columnar_scan` | parallel plan has PgColumnarScan | +| `test_parallel_filtered_scan_matches_the_heap` | par filtered scan | +| `test_parallel_range_scan_matches_the_heap` | par range scan | +| `test_parallel_whole_row_sample_matches_the_heap` | par whole sample | +| `test_parallel_count_and_sum_match_the_heap` | par count | +| `test_parallel_unfiltered_scan_matches_the_heap` | par no-filter | +| `test_parallel_every_par_query_plans_a_gather` | par queries all plan a Gather | +| `test_serial_filtered_scan_matches_the_heap` | serial filtered scan | +| `test_serial_range_scan_matches_the_heap` | serial range scan | +| `test_serial_count_and_sum_match_the_heap` | serial count | +| `test_serial_queries_plan_no_gather` | serial queries plan no Gather | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 44008c150..9e0b812bc 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -1094,4 +1094,10 @@ guard_tests 804 # cluster_tests 2670 -> 2677 for #1637's test_catalog_filter_reads.py, the twin of catalog_filter_reads.sh: # 7 tests, one per check, by collection over the cluster files. # cluster_tests 2748 -> 2753 for #1706's five timestamp-flag tests in test_native_parquet_schema.py, by collection. -cluster_tests 3222 +# cluster_tests 3222 -> 3239 for #1341's test_parallel_copy_dedup.py, the twin of parallel_copy_dedup.sh: +# 17 tests, one per check, by collection over the cluster files. +# cluster_tests 3239 -> 3320 for #1341's test_parallel_copy.py, the twin of parallel_copy.sh: +# 81 tests, one per check, by collection over the cluster files. +# cluster_tests 3320 -> 3333 for #1341's test_parallel.py, the twin of parallel.sh: +# 13 tests, one per check, by collection over the cluster files. +cluster_tests 3333 diff --git a/test/pytest/test_approx_aggregates.py b/test/pytest/test_approx_aggregates.py index 29e01f79d..c7c73e191 100644 --- a/test/pytest/test_approx_aggregates.py +++ b/test/pytest/test_approx_aggregates.py @@ -74,8 +74,14 @@ def test_tdigest_over_columnar(pgc_conn, expect): exact = _one(conn, "SELECT percentile_cont(0.99) WITHIN GROUP (ORDER BY v) FROM ax_heap") expect.text(repr(col), repr(heap), "tdigest over a columnar table gives exactly what it gives over heap") expect.text(_within(col, exact, 0.001), "ok", "premise: and its p99 is within 0.1% of the exact percentile") + # Built once into heap and stored from there, as the shell does: a tdigest depends on + # the order its inputs arrive in, so only a copy isolates the round trip (#1339). + conn.execute("CREATE TABLE ax_td_src USING heap AS SELECT day, tdigest(v, 100) AS t FROM ax GROUP BY day") conn.execute("CREATE TABLE ax_td (day int, t tdigest) USING pgcolumnar") - conn.execute("INSERT INTO ax_td SELECT day, tdigest(v, 100) FROM ax GROUP BY day") + conn.execute("INSERT INTO ax_td SELECT day, t FROM ax_td_src") + expect.text(_one(conn, "SELECT ((SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td) " + "= (SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td_src))::text"), "true", + "a tdigest stored in a columnar table comes back byte for byte") expect.text(_within(_one(conn, "SELECT tdigest_percentile(t, 0.99) FROM ax_td"), exact, 0.001), "ok", "premise: a tdigest stored in a columnar table still answers: " "the union of daily digests is within 0.1% at p99") diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index a6f0954c5..ea74eccfc 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -92,6 +92,9 @@ "smoke", "sort_status", "sort_status_privilege", + "parallel", + "parallel_copy", + "parallel_copy_dedup", "parquet_export_stats", "parallel_flush_optin", "parallel_degree", @@ -1358,16 +1361,15 @@ def test_a_longer_helper_name_is_not_shadowed_by_a_shorter_one(expect): def test_the_suite_local_helpers_are_known_and_excluded(expect): - """Three helpers are defined by one suite, `parallel_copy.sh`, and the extractor - does not read them. That is a scope decision and it is asserted rather than left - implicit. + """No suite defines its own `check_` helper, which the extractor does + not read. That is a scope decision and it is asserted rather than left implicit. - `compare_to_bash.py` grades a `test/.sh` against a - `test/pytest/test_.py`. `parallel_copy.sh` has no pytest twin, so it is not - graded and the exclusion costs nothing TODAY. - The day one of them is ported, this arm is what says the grader cannot see it. - It said so for parquet_export_stats.sh (#1341), whose check_float went when the - suite was ported: its two NaN-fixture checks and eight f4 bounds call `check`. + `compare_to_bash.py` reads the checks a suite records through `lib.sh`; a helper + a suite defines for itself, recording under a name it builds, is invisible to it. + There were three such helpers, all in parallel_copy.sh, and they went when that + suite was ported (#1341): its checks now carry literal names at the call site. + parquet_export_stats.sh's check_float went the same way before it. A suite that + adds one again reddens this arm, which is what says the grader cannot see it. The population is `check_`, which is not the same as "starts with check": `checks_in` in `decode_interrupts.sh` is a COUNTING utility returning a @@ -1381,15 +1383,8 @@ def test_the_suite_local_helpers_are_known_and_excluded(expect): continue for h in re.findall(r'^(check_[a-z_]+)\(\)\s*\{', sh.read_text(), re.M): local.setdefault(h, sh.name) - expect.text(", ".join(f"{h} ({f})" for h, f in sorted(local.items())), - "check_reconstruct (parallel_copy.sh), " - "check_split_happened (parallel_copy.sh), " - "check_structure (parallel_copy.sh)", - "the suite-local helpers are exactly these three") - twinned = [f for h, f in local.items() - if (HERE / f"test_{f[:-3]}.py").exists()] - expect.num(len(twinned), 0, - "and none of their suites has a pytest twin, so none is graded today") + expect.text(", ".join(f"{h} ({f})" for h, f in sorted(local.items())) or "none", "none", + "no suite defines its own check_ helper, which the grader would not read") def test_a_suites_own_forwarding_wrapper_is_read(expect): diff --git a/test/pytest/test_parallel.py b/test/pytest/test_parallel.py new file mode 100644 index 000000000..6789f3fdc --- /dev/null +++ b/test/pytest/test_parallel.py @@ -0,0 +1,194 @@ +"""A parallel columnar scan returns what the heap oracle returns. The pytest twin of +`test/parallel.sh`, every check by the shell's name (#1341). + +The planner settings force a parallel plan, so several backends claim distinct row +groups from one shared counter. Asserted: the table really has row groups to share, +the planner puts a Gather over a parallel PgColumnarScan, every "par" query really +plans one, and its rows match the heap. Then the same queries, forced serial, plan no +Gather and still match. + +EVERY READING IS A MEASUREMENT. A failed query is a query_error(), so two failures +never compare equal, and a failed EXPLAIN counts for neither plan reading. The heap +is loaded from the generator and the columnar table from the heap, as the shell does. + +ON A PRIVATE DATABASE, the planner settings set on it as the shell does, built by a +module fixture; each test asserts one reading, so one red cannot stop the next. +""" +import re + +import psycopg +import pytest + +from pgc_vacuity import QUERY_ERROR, query_error + +DB = "pgc_parallel" +SET_HASH = ("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), $e$EMPTY$e$) " + "FROM (SELECT _row::text AS t FROM ( {q} ) _row) _s") +# The shell's queries, named once, so the plan readings read the queries the +# comparisons ran. +Q_FILTERED = "SELECT id, v, t FROM %T WHERE k = 7" +Q_RANGE = "SELECT id FROM %T WHERE k < 30" +Q_SAMPLE = "SELECT * FROM %T WHERE id % 997 = 0" +Q_COUNT = "SELECT count(*), sum(v) FROM %T WHERE k < 50" +# Row-returning: a bare count(*) is answered by the vectorized aggregate, which plans +# no Gather, so it never reached the parallel scan. +Q_NOFILTER = "SELECT id FROM %T" +PAR = { + "par filtered scan": Q_FILTERED, + "par range scan": Q_RANGE, + "par whole sample": Q_SAMPLE, + "par count": Q_COUNT, + "par no-filter": Q_NOFILTER, +} +SERIAL = { + "serial filtered scan": Q_FILTERED, + "serial range scan": Q_RANGE, + "serial count": Q_COUNT, +} + + +def _failed(exc): + return query_error(str(exc).splitlines()[0] if str(exc) else type(exc).__name__) + + +class _Db: + def __init__(self, dsn): + self.dsn = dsn + + def run(self, sql): + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + conn.execute(sql) + except psycopg.Error as exc: + print(f"-- step failed: {str(exc).splitlines()[0]}") + + def value(self, sql): + """-> the first column of the first row as text, or a query_error().""" + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + row = conn.execute(sql).fetchone() + except psycopg.Error as exc: + return _failed(exc) + if row is None or row[0] is None: + return query_error("no row") + return row[0] + + def plan(self, sql): + """-> the EXPLAIN (COSTS off) text, or a query_error().""" + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + return "\n".join(r[0] for r in conn.execute(f"EXPLAIN (COSTS off) {sql}")) + except psycopg.Error as exc: + return _failed(exc) + + def diff(self, tmpl): + """-> (columnar hash, heap hash), each a query_error() when its query failed.""" + return (self.value(SET_HASH.format(q=tmpl.replace("%T", "t_col"))), + self.value(SET_HASH.format(q=tmpl.replace("%T", "t_heap")))) + + def plans_with(self, gather, tmpls): + """-> how many of the queries, on t_col, plan a PgColumnarScan with (or without) + a Gather. A failed EXPLAIN counts for neither.""" + n = 0 + for tmpl in tmpls: + p = self.plan(tmpl.replace("%T", "t_col")) + if p.startswith(QUERY_ERROR) or "Custom Scan (PgColumnarScan)" not in p: + continue + if ("Gather" in p) == gather: + n += 1 + return n + + +def _lines(plan, pattern, flags=0): + """-> how many plan lines match, as `grep -c` counts them, or the query_error().""" + if plan.startswith(QUERY_ERROR): + return plan + return sum(1 for line in plan.splitlines() if re.search(pattern, line, flags)) + + +@pytest.fixture(scope="module") +def par(pgc_cluster): + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {DB} WITH (FORCE)") + a.execute(f"CREATE DATABASE {DB}") + try: + d = _Db(pgc_cluster.dsn(dbname=DB)) + d.run("CREATE EXTENSION IF NOT EXISTS pgcolumnar") + d.run("CREATE TABLE t_heap (id int, k int, v bigint, t text) USING heap") + d.run("CREATE TABLE t_col (id int, k int, v bigint, t text) USING pgcolumnar") + d.run("SELECT pgcolumnar.set_options('t_col', chunk_group_row_limit => 1000, " + "stripe_row_limit => 2000)") + d.run("INSERT INTO t_heap SELECT g, g%100, g::bigint*2, 'r'||g FROM generate_series(1,50000) g") + d.run("INSERT INTO t_col SELECT * FROM t_heap") + r = {"row groups": d.value("SELECT count(*) FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('t_col')")} + for guc in ("max_parallel_workers_per_gather = 4", "parallel_setup_cost = 0", + "parallel_tuple_cost = 0", "min_parallel_table_scan_size = 0"): + d.run(f"ALTER DATABASE {DB} SET {guc}") + plan = d.plan("SELECT id, v FROM t_col WHERE k = 5") + r["gather lines"] = _lines(plan, r"gather", re.IGNORECASE) + r["scan lines"] = _lines(plan, r"Custom Scan \(PgColumnarScan\)") + r["diffs"] = {name: d.diff(t) for name, t in PAR.items()} + r["par plans"] = d.plans_with(True, PAR.values()) + d.run(f"ALTER DATABASE {DB} SET max_parallel_workers_per_gather = 0") + r["diffs"].update({name: d.diff(t) for name, t in SERIAL.items()}) + r["serial plans"] = d.plans_with(False, SERIAL.values()) + print(f"row groups={r['row groups']} par plans={r['par plans']} serial plans={r['serial plans']}") + yield r + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {DB} WITH (FORCE)") + + +# Not a premise (#1339): the writer honoring stripe_row_limit is pgcolumnar's own +# work, and a writer that ignores it writes 1 group and reddens this. +def test_parallel_table_has_row_groups_to_claim(par, expect): + expect.num(par["row groups"], 25, "par table has 25 row groups to claim") + + +def test_parallel_plan_has_a_gather(par, expect): + expect.num(par["gather lines"], 1, "parallel plan has Gather") + + +def test_parallel_plan_has_the_columnar_scan(par, expect): + expect.num(par["scan lines"], 1, "parallel plan has PgColumnarScan") + + +def test_parallel_filtered_scan_matches_the_heap(par, expect): + expect.text(*par["diffs"]["par filtered scan"], "par filtered scan") + + +def test_parallel_range_scan_matches_the_heap(par, expect): + expect.text(*par["diffs"]["par range scan"], "par range scan") + + +def test_parallel_whole_row_sample_matches_the_heap(par, expect): + expect.text(*par["diffs"]["par whole sample"], "par whole sample") + + +def test_parallel_count_and_sum_match_the_heap(par, expect): + expect.text(*par["diffs"]["par count"], "par count") + + +def test_parallel_unfiltered_scan_matches_the_heap(par, expect): + expect.text(*par["diffs"]["par no-filter"], "par no-filter") + + +def test_parallel_every_par_query_plans_a_gather(par, expect): + expect.num(par["par plans"], 5, "par queries all plan a Gather") + + +def test_serial_filtered_scan_matches_the_heap(par, expect): + expect.text(*par["diffs"]["serial filtered scan"], "serial filtered scan") + + +def test_serial_range_scan_matches_the_heap(par, expect): + expect.text(*par["diffs"]["serial range scan"], "serial range scan") + + +def test_serial_count_and_sum_match_the_heap(par, expect): + expect.text(*par["diffs"]["serial count"], "serial count") + + +def test_serial_queries_plan_no_gather(par, expect): + expect.num(par["serial plans"], 3, "serial queries plan no Gather") diff --git a/test/pytest/test_parallel_copy.py b/test/pytest/test_parallel_copy.py new file mode 100644 index 000000000..99191e14b --- /dev/null +++ b/test/pytest/test_parallel_copy.py @@ -0,0 +1,651 @@ +"""pgcolumnar splits a text file into record-aligned ranges and loads it in parallel, +atomically. The pytest twin of `test/parallel_copy.sh`, every check by the shell's +name (#1341). + +`pgcolumnar.file_split_offsets` must cut a file into ranges that start right after a +newline, cover the file, and, on a file with far more records than workers, are all +non-empty; loading the ranges one by one must give back the rows the file was written +from. `pgcolumnar.parallel_copy` must load a partitioned or a single columnar target +with the rows a single COPY gives, leave no prepared transaction behind, roll every +worker back when one fails, and refuse what it cannot load: a text partition key, an +unsorted file, a DEFAULT partition, more workers than prepared-transaction slots, a +missing file, a heap target, and a FIFO, which it must refuse rather than hang on. + +THREE FIXES THE SHELL SHARES. The per-range loads are compared with the generator +that wrote the file, not with a single columnar COPY of it, which shares every defect +of the COPY path. A failed load must carry the loader's own message, not any message +with "failed" in it. And a backend that died does not pass as one that survived: a +fresh connection answers after crash recovery, so survival also reads the server log's +count of killed backends. + +ON A PRIVATE DATABASE, built in the shell's order by a module fixture. Prepared +transactions are counted in that database only, so another file's leftover cannot +redden this one. +""" +import os +import random +import threading + +import psycopg +import pytest + +from pgc_vacuity import query_error + +DB = "pgc_parallel_copy" +GEN_F = "SELECT g AS id, 'host_'||(g%97)||'_x'||g AS txt FROM generate_series(1, {n}) g" +GEN_BIG = "SELECT g AS id, 'h'||g||'_'||repeat('x', 70000) AS txt FROM generate_series(1, 20) g" +GEN_NONL = "SELECT g AS id, 'r'||g AS txt FROM generate_series(1,1000) g" +LEAK = "SELECT count(*) FROM pg_prepared_xacts WHERE database = current_database()" + +# (reading key, the shell's check name) +STRUCTURE = ( + ("5000/1", "5000 rows / 1: offsets well-formed and newline-aligned"), + ("5000/2", "5000 rows / 2: offsets well-formed and newline-aligned"), + ("5000/3", "5000 rows / 3: offsets well-formed and newline-aligned"), + ("5000/8", "5000 rows / 8: offsets well-formed and newline-aligned"), + ("5000/16", "5000 rows / 16: offsets well-formed and newline-aligned"), + ("big/4", "long lines >64kB / 4 workers: offsets well-formed and newline-aligned"), + ("tiny/8", "5 rows / 8 workers: offsets well-formed and newline-aligned"), + ("one/4", "1 row / 4 workers (edge smoke): offsets well-formed and newline-aligned"), + ("nonl/4", "no trailing newline / 4 workers: offsets well-formed and newline-aligned"), +) +RECONSTRUCT = ( + ("5000/1", "5000 rows / 1: per-range loads reconstruct the whole file"), + ("5000/2", "5000 rows / 2: per-range loads reconstruct the whole file"), + ("5000/3", "5000 rows / 3: per-range loads reconstruct the whole file"), + ("5000/8", "5000 rows / 8: per-range loads reconstruct the whole file"), + ("5000/16", "5000 rows / 16: per-range loads reconstruct the whole file"), + ("big/4", "long lines >64kB / 4 workers: per-range loads reconstruct the whole file"), + ("tiny/8", "5 rows / 8 workers (empty ranges): per-range loads reconstruct the whole file"), + ("one/4", "1 row / 4 workers (edge smoke): per-range loads reconstruct the whole file"), + ("nonl/4", "no trailing newline / 4 workers: per-range loads reconstruct the whole file"), +) +SPLIT_REAL = ( + ("5000/2", "5000 rows / 2: split is real (interior offsets distinct, ranges non-empty)"), + ("5000/3", "5000 rows / 3: split is real (interior offsets distinct, ranges non-empty)"), + ("5000/8", "5000 rows / 8: split is real (interior offsets distinct, ranges non-empty)"), + ("5000/16", "5000 rows / 16: split is real (interior offsets distinct, ranges non-empty)"), + ("big/4", "long lines >64kB / 4 workers: split is real (interior offsets distinct, ranges non-empty)"), +) +LOAD_ROWS = ( + ("partitioned/1", "parallel_copy(partitioned, 1 workers): rows returned = 5000"), + ("partitioned/2", "parallel_copy(partitioned, 2 workers): rows returned = 5000"), + ("partitioned/4", "parallel_copy(partitioned, 4 workers): rows returned = 5000"), + ("single/1", "parallel_copy(single table, 1 workers): rows returned = 5000"), + ("single/2", "parallel_copy(single table, 2 workers): rows returned = 5000"), + ("single/4", "parallel_copy(single table, 4 workers): rows returned = 5000"), +) +LOAD_ORACLE = ( + ("partitioned/1", "parallel_copy(partitioned, 1 workers): result == single-COPY oracle"), + ("partitioned/2", "parallel_copy(partitioned, 2 workers): result == single-COPY oracle"), + ("partitioned/4", "parallel_copy(partitioned, 4 workers): result == single-COPY oracle"), + ("single/1", "parallel_copy(single table, 1 workers): result == single-COPY oracle"), + ("single/2", "parallel_copy(single table, 2 workers): result == single-COPY oracle"), + ("single/4", "parallel_copy(single table, 4 workers): result == single-COPY oracle"), +) +LOAD_LEAK = ( + ("partitioned/1", "parallel_copy(partitioned, 1 workers): no prepared-transaction leak"), + ("partitioned/2", "parallel_copy(partitioned, 2 workers): no prepared-transaction leak"), + ("partitioned/4", "parallel_copy(partitioned, 4 workers): no prepared-transaction leak"), + ("single/1", "parallel_copy(single table, 1 workers): no prepared-transaction leak"), + ("single/2", "parallel_copy(single table, 2 workers): no prepared-transaction leak"), + ("single/4", "parallel_copy(single table, 4 workers): no prepared-transaction leak"), +) + + +def _message(exc): + return str(exc).splitlines()[0] if str(exc) else type(exc).__name__ + + +class _Db: + def __init__(self, dsn): + self.dsn = dsn + + def run(self, sql): + """A setup step. A failure is printed; the readings that need the step fail.""" + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + conn.execute(sql) + except psycopg.Error as exc: + print(f"-- step failed: {_message(exc)}") + + def value(self, sql): + """-> the first column of the first row, or the failure as a query_error().""" + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + row = conn.execute(sql).fetchone() + except psycopg.Error as exc: + return query_error(_message(exc)) + if row is None or row[0] is None: + return query_error(f"no value from {sql}") + return row[0] + + def set_hash(self, sql): + """lib.sh's pgc_set_hash: md5 over the sorted rendered rows, EMPTY for none.""" + return self.value("SELECT coalesce(md5(string_agg(t, chr(10) ORDER BY t)), 'EMPTY') " + f"FROM (SELECT _row::text AS t FROM ( {sql} ) _row) _s") + + def refused(self, sql, phrase, *alternatives): + """-> "refused: " when `sql` raised with `phrase`, or one of the + alternatives, in its message (any case); the message when it has none of them; + "noerror" when it did not raise; a query_error() when the server never answered.""" + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + conn.execute(sql).fetchall() + except psycopg.Error as exc: + if not exc.sqlstate: + return query_error(_message(exc)) + said = str(exc).lower() + hit = any(p.lower() in said for p in (phrase,) + alternatives) + return f"refused: {phrase}" if hit else str(exc) + return "noerror" + + def copy_in(self, table, data): + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + with conn.cursor().copy(f"COPY {table} FROM STDIN WITH (FORMAT text)") as cp: + cp.write(data) + except psycopg.Error as exc: + print(f"-- copy into {table} failed: {_message(exc)}") + + def offsets(self, path, workers): + return self.value(f"SELECT pgcolumnar.file_split_offsets('{path}', {workers})") + + def sqlstate_or_hang(self, sql, fifo): + """-> the SQLSTATE `sql` raised, "HANG" when it did not return in 5 s, or + "noerror". A hang is released by opening the FIFO for writing, as the shell does.""" + out = {} + + def go(): + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + conn.execute(sql).fetchall() + out["r"] = "noerror" + except psycopg.Error as exc: + out["r"] = exc.sqlstate or query_error(_message(exc)) + + t = threading.Thread(target=go, daemon=True) + t.start() + t.join(5) + if t.is_alive(): + try: + fd = os.open(fifo, os.O_RDWR | os.O_NONBLOCK) + os.close(fd) + except OSError: + pass + t.join(10) + return "HANG" + return out["r"] + + +def _crashes(cluster): + try: + return (cluster.datadir / "server.log").read_text(errors="replace").count("terminated by signal") + except OSError as exc: + return query_error(f"crash count unreadable: {exc}") + + +def _survived(db, cluster, before): + up, now = db.value("SELECT 1"), _crashes(cluster) + if not isinstance(before, int) or not isinstance(now, int): + return query_error(f"crash count unreadable: {before} / {now}") + return "1" if up == 1 and now == before else f"no ([{up}], {now - before} backend crashes)" + + +def _structure(data, offs, w): + """-> 1 when the offsets are w+1, start at 0, end at the size, never decrease, and + every interior one sits right after a newline; else 0 or what was read.""" + if not isinstance(offs, list): + return offs + if len(offs) != w + 1: + return f"got {len(offs)} offsets, want {w + 1}" + size = len(data) + ok = offs[0] == 0 and offs[w] == size + for i in range(1, w + 1): + ok = ok and offs[i] >= offs[i - 1] + for i in range(1, w): + if 0 < offs[i] < size and data[offs[i] - 1] != 0x0a: + ok = False + return 1 if ok else 0 + + +def _split_real(data, offs, w): + """-> 1 when the w-1 interior offsets strictly increase inside (0, size).""" + if not isinstance(offs, list): + return offs + if len(offs) != w + 1: + return f"got {len(offs)} offsets, want {w + 1}" + prev, ok = 0, True + for i in range(1, w): + ok = ok and prev < offs[i] < len(data) + prev = offs[i] + return 1 if ok else 0 + + +def _split_load_hash(db, data, offs, w): + """Load each range into a fresh columnar t_split -> the set hash of t_split.""" + if not isinstance(offs, list): + return offs + db.run("DROP TABLE IF EXISTS t_split; CREATE TABLE t_split (id int, txt text) USING pgcolumnar") + for i in range(w): + lo = offs[i] if i < len(offs) else 0 + hi = offs[i + 1] if i + 1 < len(offs) else 0 + if hi - lo > 0: + db.copy_in("t_split", data[lo:hi]) + return db.set_hash("SELECT * FROM t_split") + + +def _mkpart(db, name, n): + """A RANGE-partitioned columnar table over id in [1,5000], n partitions.""" + step = (5000 + n) // n + ddl = f"DROP TABLE IF EXISTS {name} CASCADE; CREATE TABLE {name} (id int, txt text) PARTITION BY RANGE (id);" + for i in range(n): + lo = "MINVALUE" if i == 0 else str(i * step) + hi = "MAXVALUE" if i == n - 1 else str((i + 1) * step) + ddl += f" CREATE TABLE {name}_p{i} PARTITION OF {name} FOR VALUES FROM ({lo}) TO ({hi}) USING pgcolumnar;" + db.run(ddl) + + +def _server_file(db, d, name, sql): + """The server writes `sql`'s rows to d/name -> the path.""" + path = d / name + db.run(f"COPY ({sql}) TO '{path}' WITH (FORMAT text)") + return path + + +def _read(path): + try: + return path.read_bytes() + except OSError as exc: + return query_error(f"cannot read {path}: {exc}") + + +def _split_readings(r, db, key, path, w, gen, *, real=False): + data = _read(path) + offs = db.offsets(path, w) + if not isinstance(data, bytes): + offs = data + r["structure " + key] = _structure(data, offs, w) + r["reconstruct " + key] = _split_load_hash(db, data, offs, w) + r["reconstruct want " + key] = db.set_hash(gen) + if real: + r["split real " + key] = _split_real(data, offs, w) + + +@pytest.fixture(scope="module") +def pcopy(pgc_cluster, server_dir): + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {DB} WITH (FORCE)") + a.execute(f"CREATE DATABASE {DB}") + d = server_dir + db = _Db(pgc_cluster.dsn(dbname=DB)) + try: + db.run("CREATE EXTENSION IF NOT EXISTS pgcolumnar") + r = {} + before = _crashes(pgc_cluster) + + # ---- the splitter ------------------------------------------------------- + db.run("CREATE TABLE t_heap (id int, txt text) USING heap; " + "CREATE TABLE t_col (id int, txt text) USING pgcolumnar") + f = _server_file(db, d, "data_5000.txt", GEN_F.format(n=5000)) + db.run(f"COPY t_heap FROM '{f}' WITH (FORMAT text)") + db.run(f"COPY t_col FROM '{f}' WITH (FORMAT text)") + r["single col"] = db.set_hash("SELECT * FROM t_col") + r["single heap"] = db.set_hash("SELECT * FROM t_heap") + for w in (1, 2, 3, 8, 16): + _split_readings(r, db, f"5000/{w}", f, w, GEN_F.format(n=5000), real=w >= 2) + big = _server_file(db, d, "biglines.txt", GEN_BIG) + _split_readings(r, db, "big/4", big, 4, GEN_BIG, real=True) + tiny = _server_file(db, d, "data_5.txt", GEN_F.format(n=5)) + _split_readings(r, db, "tiny/8", tiny, 8, GEN_F.format(n=5)) + one = _server_file(db, d, "data_1.txt", GEN_F.format(n=1)) + _split_readings(r, db, "one/4", one, 4, GEN_F.format(n=1)) + nonl = _server_file(db, d, "nonl.txt", GEN_NONL) + try: + os.truncate(nonl, nonl.stat().st_size - 1) + except OSError as exc: + print(f"-- truncate failed: {exc}") + _split_readings(r, db, "nonl/4", nonl, 4, GEN_NONL) + + r["workers 0"] = db.refused(f"SELECT pgcolumnar.file_split_offsets('{f}', 0)", "at least 1") + r["directory"] = db.refused(f"SELECT pgcolumnar.file_split_offsets('{d}', 1)", "not a regular file") + + # ---- the coordinator: partitioned target -------------------------------- + for w in (1, 2, 4): + _mkpart(db, "t_pcp", 4) + r[f"rows partitioned/{w}"] = db.value(f"SELECT pgcolumnar.parallel_copy('t_pcp'::regclass, '{f}', {w})") + r[f"oracle partitioned/{w}"] = db.set_hash("SELECT * FROM t_pcp") + r[f"oracle want partitioned/{w}"] = db.set_hash("SELECT * FROM t_heap") + r[f"leak partitioned/{w}"] = db.value(LEAK) + + signed = _server_file(db, d, "signed.txt", + "SELECT g AS id, 'h'||g AS txt FROM generate_series(-100, 399) g") + db.run("CREATE TABLE t_heap_s (id int, txt text)") + db.run(f"COPY t_heap_s FROM '{signed}' WITH (FORMAT text)") + db.run("CREATE TABLE t_signed (id int, txt text) PARTITION BY RANGE (id); " + "CREATE TABLE t_signed_a PARTITION OF t_signed FOR VALUES FROM (MINVALUE) TO (100) USING pgcolumnar; " + "CREATE TABLE t_signed_b PARTITION OF t_signed FOR VALUES FROM (100) TO (300) USING pgcolumnar; " + "CREATE TABLE t_signed_c PARTITION OF t_signed FOR VALUES FROM (300) TO (MAXVALUE) USING pgcolumnar") + r["signed rows"] = db.value(f"SELECT pgcolumnar.parallel_copy('t_signed'::regclass, '{signed}', 3)") + r["signed"] = db.set_hash("SELECT * FROM t_signed") + r["signed want"] = db.set_hash("SELECT * FROM t_heap_s") + r["signed leak"] = db.value(LEAK) + + db.run("CREATE TABLE t_txtkey (k text, v int) PARTITION BY RANGE (k); " + "CREATE TABLE t_txtkey_a PARTITION OF t_txtkey FOR VALUES FROM (MINVALUE) TO ('m') USING pgcolumnar; " + "CREATE TABLE t_txtkey_b PARTITION OF t_txtkey FOR VALUES FROM ('m') TO (MAXVALUE) USING pgcolumnar") + r["text key"] = db.refused(f"SELECT pgcolumnar.parallel_copy('t_txtkey'::regclass, '{f}', 2)", + "numeric or date/time") + r["text key up"] = _survived(db, pgc_cluster, before) + + gen = _server_file(db, d, "genkey.txt", "SELECT 'lbl'||g AS label, g AS id FROM generate_series(1,3000) g") + schema = "label text, gcol int GENERATED ALWAYS AS (id*2) STORED, id int" + db.run(f"CREATE TABLE t_heap_g ({schema})") + db.run(f"COPY t_heap_g (label, id) FROM '{gen}' WITH (FORMAT text)") + db.run(f"CREATE TABLE t_gen ({schema}) PARTITION BY RANGE (id); " + "CREATE TABLE t_gen_a PARTITION OF t_gen FOR VALUES FROM (MINVALUE) TO (1000) USING pgcolumnar; " + "CREATE TABLE t_gen_b PARTITION OF t_gen FOR VALUES FROM (1000) TO (2000) USING pgcolumnar; " + "CREATE TABLE t_gen_c PARTITION OF t_gen FOR VALUES FROM (2000) TO (MAXVALUE) USING pgcolumnar") + r["gen rows"] = db.value(f"SELECT pgcolumnar.parallel_copy('t_gen'::regclass, '{gen}', 3)") + r["gen"] = db.set_hash("SELECT * FROM t_gen") + r["gen want"] = db.set_hash("SELECT * FROM t_heap_g") + + badkey = d / "badkey.txt" + badkey.write_text("".join(f"{i}\thost_{i}\n" for i in range(1, 401)) + "notanint\tboom\n" + + "".join(f"{i}\thost_{i}\n" for i in range(401, 801))) + badkey.chmod(0o644) + _mkpart(db, "t_pcp", 4) + r["bad key"] = db.refused(f"SELECT pgcolumnar.parallel_copy('t_pcp'::regclass, '{badkey}', 4)", + "invalid input syntax") + r["bad key empty"] = db.value("SELECT count(*) FROM t_pcp") + r["bad key leak"] = db.value(LEAK) + + db.run("CREATE TABLE t_gap (id int, txt text) PARTITION BY RANGE (id); " + "CREATE TABLE t_gap_a PARTITION OF t_gap FOR VALUES FROM (MINVALUE) TO (2000) USING pgcolumnar; " + "CREATE TABLE t_gap_b PARTITION OF t_gap FOR VALUES FROM (3000) TO (MAXVALUE) USING pgcolumnar") + r["gap"] = db.refused(f"SELECT pgcolumnar.parallel_copy('t_gap'::regclass, '{f}', 4)", + "failed: no partition of relation") + r["gap empty"] = db.value("SELECT count(*) FROM t_gap") + r["gap leak"] = db.value(LEAK) + r["gap up"] = _survived(db, pgc_cluster, before) + + lines = f.read_bytes().splitlines(keepends=True) + random.Random(1341).shuffle(lines) + shuf = d / "shuf.txt" + shuf.write_bytes(b"".join(lines)) + shuf.chmod(0o644) + _mkpart(db, "t_pcp", 4) + r["unsorted"] = db.refused(f"SELECT pgcolumnar.parallel_copy('t_pcp'::regclass, '{shuf}', 4)", "not sorted") + r["unsorted empty"] = db.value("SELECT count(*) FROM t_pcp") + + db.run("CREATE TABLE t_def (id int, txt text) PARTITION BY RANGE (id); " + "CREATE TABLE t_def_a PARTITION OF t_def FOR VALUES FROM (MINVALUE) TO (2500) USING pgcolumnar; " + "CREATE TABLE t_def_d PARTITION OF t_def DEFAULT USING pgcolumnar") + r["default"] = db.refused(f"SELECT pgcolumnar.parallel_copy('t_def'::regclass, '{f}', 4)", "DEFAULT partition") + + _mkpart(db, "t_pcp10", 10) + r["guard"] = db.refused(f"SELECT pgcolumnar.parallel_copy('t_pcp10'::regclass, '{f}', 10)", + "max_prepared_transactions") + r["guard empty"] = db.value("SELECT count(*) FROM t_pcp10") + + _mkpart(db, "t_pcp", 4) + r["missing"] = db.refused(f"SELECT pgcolumnar.parallel_copy('t_pcp'::regclass, '{d / 'nope.txt'}', 2)", + "could not open", "could not stat", "no such file", "not a regular file") + r["missing up"] = _survived(db, pgc_cluster, before) + r["missing empty"] = db.value("SELECT count(*) FROM t_pcp") + + db.run("CREATE TABLE t_heaptgt (id int, txt text)") + r["heap target"] = db.refused(f"SELECT pgcolumnar.parallel_copy('t_heaptgt'::regclass, '{f}', 2)", + "not a pgcolumnar table") + + # ---- the coordinator: one columnar table --------------------------------- + r["guc"] = db.value("SHOW pgcolumnar.bulk_parallel_writer") + for w in (1, 2, 4): + db.run("DROP TABLE IF EXISTS t_single; CREATE TABLE t_single (id int, txt text) USING pgcolumnar") + r[f"rows single/{w}"] = db.value(f"SELECT pgcolumnar.parallel_copy('t_single'::regclass, '{f}', {w})") + r[f"oracle single/{w}"] = db.set_hash("SELECT * FROM t_single") + r[f"oracle want single/{w}"] = db.set_hash("SELECT * FROM t_heap") + r[f"leak single/{w}"] = db.value(LEAK) + + fw = _server_file(db, d, "data_40000.txt", GEN_F.format(n=40000)) + db.run("CREATE TABLE t_wit_heap (id int, txt text)") + db.run(f"COPY t_wit_heap FROM '{fw}' WITH (FORMAT text)") + db.run("CREATE TABLE t_wit (id int, txt text) USING pgcolumnar; " + "SELECT pgcolumnar.set_options('t_wit'::regclass, chunk_group_row_limit => 1000, " + "stripe_row_limit => 2000)") + r["wit rows"] = db.value(f"SELECT pgcolumnar.parallel_copy('t_wit'::regclass, '{fw}', 4)") + r["wit"] = db.set_hash("SELECT * FROM t_wit") + r["wit want"] = db.set_hash("SELECT * FROM t_wit_heap") + r["wit stripes"] = db.value("SELECT (count(DISTINCT stripeid) > 4)::text FROM pgcolumnar.stats('t_wit'::regclass)") + r["wit overlaps"] = db.value( + "WITH s AS (SELECT stripeid, fileoffset AS lo, fileoffset+datalength AS hi " + "FROM pgcolumnar.stats('t_wit'::regclass)) " + "SELECT count(*) FROM s a JOIN s b ON a.stripeid <> b.stripeid AND a.lo < b.hi AND b.lo < a.hi") + r["wit coverage"] = db.value("SELECT sum(rowcount)::bigint FROM pgcolumnar.stats('t_wit'::regclass)") + r["wit leak"] = db.value(LEAK) + + db.run("DROP TABLE IF EXISTS t_single; CREATE TABLE t_single (id int, txt text) USING pgcolumnar") + r["single bad"] = db.refused(f"SELECT pgcolumnar.parallel_copy('t_single'::regclass, '{badkey}', 4)", + "failed: invalid input syntax") + r["single bad empty"] = db.value("SELECT count(*) FROM t_single") + r["single bad leak"] = db.value(LEAK) + + # ---- #686: a FIFO is refused, not a hang -------------------------------- + db.run("CREATE TABLE t_fifo (id int, txt text) USING pgcolumnar") + fifo = d / "pc.fifo" + os.mkfifo(fifo, 0o666) + r["fifo"] = db.sqlstate_or_hang(f"SELECT pgcolumnar.parallel_copy('t_fifo'::regclass, '{fifo}', 2)", fifo) + r["fifo up"] = _survived(db, pgc_cluster, before) + _mkpart(db, "t_fifo_part", 3) + fifo2 = d / "pc2.fifo" + os.mkfifo(fifo2, 0o666) + r["fifo part"] = db.sqlstate_or_hang( + f"SELECT pgcolumnar.parallel_copy('t_fifo_part'::regclass, '{fifo2}', 3)", fifo2) + r["fifo part up"] = _survived(db, pgc_cluster, before) + yield r + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {DB} WITH (FORCE)") + + +# ---- the splitter --------------------------------------------------------------- + +def test_parallel_copy_single_copy_heap_and_columnar_agree(pcopy, expect): + expect.hash(pcopy["single col"], pcopy["single heap"], "single-COPY heap and columnar agree") + + +@pytest.mark.parametrize("key,name", STRUCTURE, ids=[k for k, _ in STRUCTURE]) +def test_parallel_copy_split_offsets_are_newline_aligned(pcopy, expect, key, name): + expect.num(pcopy["structure " + key], 1, name) + + +@pytest.mark.parametrize("key,name", RECONSTRUCT, ids=[k for k, _ in RECONSTRUCT]) +def test_parallel_copy_range_loads_give_back_the_generated_rows(pcopy, expect, key, name): + expect.hash(pcopy["reconstruct " + key], pcopy["reconstruct want " + key], name) + + +@pytest.mark.parametrize("key,name", SPLIT_REAL, ids=[k for k, _ in SPLIT_REAL]) +def test_parallel_copy_every_range_of_a_large_file_is_non_empty(pcopy, expect, key, name): + expect.num(pcopy["split real " + key], 1, name) + + +def test_parallel_copy_zero_workers_is_rejected(pcopy, expect): + expect.text(pcopy["workers 0"], "refused: at least 1", "workers < 1 is rejected") + + +def test_parallel_copy_split_of_a_directory_is_rejected(pcopy, expect): + expect.text(pcopy["directory"], "refused: not a regular file", + "file_split_offsets: a directory is rejected (not an 8-exabyte file)") + + +# ---- the coordinator -------------------------------------------------------------- + +@pytest.mark.parametrize("key,name", LOAD_ROWS, ids=[k for k, _ in LOAD_ROWS]) +def test_parallel_copy_returns_every_row_loaded(pcopy, expect, key, name): + expect.num(pcopy["rows " + key], 5000, name) + + +@pytest.mark.parametrize("key,name", LOAD_ORACLE, ids=[k for k, _ in LOAD_ORACLE]) +def test_parallel_copy_load_equals_the_single_copy(pcopy, expect, key, name): + expect.hash(pcopy["oracle " + key], pcopy["oracle want " + key], name) + + +@pytest.mark.parametrize("key,name", LOAD_LEAK, ids=[k for k, _ in LOAD_LEAK]) +def test_parallel_copy_load_leaves_no_prepared_transaction(pcopy, expect, key, name): + expect.num(pcopy["leak " + key], 0, name) + + +def test_parallel_copy_signed_keys_return_every_row(pcopy, expect): + expect.num(pcopy["signed rows"], 500, "signed keys straddling 0: rows returned = 500") + + +def test_parallel_copy_signed_keys_equal_the_oracle(pcopy, expect): + expect.hash(pcopy["signed"], pcopy["signed want"], + "signed keys straddling 0: result == oracle (no misbucketing/deadlock)") + + +def test_parallel_copy_signed_keys_leave_no_prepared_transaction(pcopy, expect): + expect.num(pcopy["signed leak"], 0, "signed keys straddling 0: no prepared-transaction leak") + + +def test_parallel_copy_text_partition_key_is_rejected(pcopy, expect): + expect.text(pcopy["text key"], "refused: numeric or date/time", "text partition key is rejected (not a crash)") + + +def test_parallel_copy_text_partition_key_leaves_the_server_up(pcopy, expect): + expect.text(pcopy["text key up"], "1", "text partition key: server still up (no crash)") + + +def test_parallel_copy_key_after_a_generated_column_returns_every_row(pcopy, expect): + expect.num(pcopy["gen rows"], 3000, "key not in column 1 (generated col before it): rows = 3000") + + +def test_parallel_copy_key_after_a_generated_column_equals_the_oracle(pcopy, expect): + expect.hash(pcopy["gen"], pcopy["gen want"], "key not in column 1: result == oracle") + + +def test_parallel_copy_bad_partition_key_value_is_rejected(pcopy, expect): + expect.text(pcopy["bad key"], "refused: invalid input syntax", "atomic: a bad partition-key value is rejected") + + +def test_parallel_copy_bad_partition_key_loads_nothing(pcopy, expect): + expect.num(pcopy["bad key empty"], 0, "atomic: bad-key load leaves the target empty") + + +def test_parallel_copy_bad_partition_key_leaves_no_prepared_transaction(pcopy, expect): + expect.num(pcopy["bad key leak"], 0, "atomic: no prepared-transaction leak after bad-key") + + +def test_parallel_copy_loader_failure_fails_the_load_with_its_cause(pcopy, expect): + expect.text(pcopy["gap"], "refused: failed: no partition of relation", + "atomic: a loader failure fails the whole load") + + +def test_parallel_copy_loader_failure_rolls_back_the_siblings(pcopy, expect): + expect.num(pcopy["gap empty"], 0, "atomic: loader-failure leaves the target empty (siblings rolled back)") + + +def test_parallel_copy_loader_failure_leaves_no_prepared_transaction(pcopy, expect): + expect.num(pcopy["gap leak"], 0, "atomic: no prepared-transaction leak after loader failure") + + +def test_parallel_copy_loader_failure_leaves_the_server_up(pcopy, expect): + expect.text(pcopy["gap up"], "1", "atomic: server still up after a failed load") + + +def test_parallel_copy_unsorted_input_is_rejected(pcopy, expect): + expect.text(pcopy["unsorted"], "refused: not sorted", "unsorted input is rejected") + + +def test_parallel_copy_unsorted_input_loads_nothing(pcopy, expect): + expect.num(pcopy["unsorted empty"], 0, "unsorted rejection loads nothing") + + +def test_parallel_copy_default_partition_is_rejected(pcopy, expect): + expect.text(pcopy["default"], "refused: DEFAULT partition", "DEFAULT partition target is rejected") + + +def test_parallel_copy_prepared_transaction_guard_fires(pcopy, expect): + expect.text(pcopy["guard"], "refused: max_prepared_transactions", "max_prepared_transactions guard fires") + + +def test_parallel_copy_prepared_transaction_guard_loads_nothing(pcopy, expect): + expect.num(pcopy["guard empty"], 0, "guard rejects before loading anything") + + +def test_parallel_copy_missing_file_errors(pcopy, expect): + expect.text(pcopy["missing"], "refused: could not open", "missing file: errors, not crashes") + + +def test_parallel_copy_missing_file_leaves_the_server_up(pcopy, expect): + expect.text(pcopy["missing up"], "1", "missing file: server still up (no worker crash)") + + +def test_parallel_copy_missing_file_loads_nothing(pcopy, expect): + expect.num(pcopy["missing empty"], 0, "missing file: nothing loaded") + + +def test_parallel_copy_heap_target_is_rejected(pcopy, expect): + expect.text(pcopy["heap target"], "refused: not a pgcolumnar table", "non-columnar target is rejected") + + +def test_parallel_copy_bulk_parallel_writer_is_off_by_default(pcopy, expect): + expect.text(pcopy["guc"], "off", "bulk_parallel_writer GUC defaults off (core write path unchanged)") + + +# ---- the concurrency witness ------------------------------------------------------- + +def test_parallel_copy_witness_loads_every_row(pcopy, expect): + expect.num(pcopy["wit rows"], 40000, "witness: parallel_copy(single table, 4 workers, 40000 rows) loads every row") + + +def test_parallel_copy_witness_equals_the_oracle(pcopy, expect): + expect.hash(pcopy["wit"], pcopy["wit want"], "witness: result == single-COPY oracle") + + +def test_parallel_copy_witness_has_more_stripes_than_workers(pcopy, expect): + expect.text(pcopy["wit stripes"], "true", "witness: >4 distinct stripe ids (each worker reserved several stripes)") + + +def test_parallel_copy_witness_stripes_do_not_overlap(pcopy, expect): + expect.num(pcopy["wit overlaps"], 0, "witness: no stripe byte-range overlaps another (no on-disk collision)") + + +def test_parallel_copy_witness_stripes_cover_every_row(pcopy, expect): + expect.num(pcopy["wit coverage"], 40000, "witness: stripe rowcounts sum to every input row (complete coverage)") + + +def test_parallel_copy_witness_leaves_no_prepared_transaction(pcopy, expect): + expect.num(pcopy["wit leak"], 0, "witness: no prepared-transaction leak") + + +def test_parallel_copy_single_table_bad_row_fails_with_its_cause(pcopy, expect): + expect.text(pcopy["single bad"], "refused: failed: invalid input syntax", + "single table: a bad row fails the whole load") + + +def test_parallel_copy_single_table_failure_loads_nothing(pcopy, expect): + expect.num(pcopy["single bad empty"], 0, "single table: failed load leaves the target empty") + + +def test_parallel_copy_single_table_failure_leaves_no_prepared_transaction(pcopy, expect): + expect.num(pcopy["single bad leak"], 0, "single table: no prepared-transaction leak after failure") + + +# ---- #686: a FIFO is refused, not a hang -------------------------------------------- + +def test_parallel_copy_fifo_is_refused_not_a_hang(pcopy, expect): + expect.text(pcopy["fifo"], "42809", "parallel_copy on a FIFO is refused, not a hang (42809)") + + +def test_parallel_copy_fifo_refusal_leaves_the_backend_up(pcopy, expect): + expect.text(pcopy["fifo up"], "1", "backend still up after the parallel_copy FIFO refusal") + + +def test_parallel_copy_fifo_on_a_partitioned_target_is_refused_not_a_hang(pcopy, expect): + expect.text(pcopy["fifo part"], "42809", + "parallel_copy on a FIFO via the partitioned coordinator is refused, not a hang (42809)") + + +def test_parallel_copy_partitioned_fifo_refusal_leaves_the_backend_up(pcopy, expect): + expect.text(pcopy["fifo part up"], "1", "backend still up after the partitioned-coordinator FIFO refusal") diff --git a/test/pytest/test_parallel_copy_dedup.py b/test/pytest/test_parallel_copy_dedup.py new file mode 100644 index 000000000..48c47eb54 --- /dev/null +++ b/test/pytest/test_parallel_copy_dedup.py @@ -0,0 +1,212 @@ +"""pgcolumnar.parallel_copy can refuse a load it has already done (#403 item 7). The +pytest twin of `test/parallel_copy_dedup.sh`, every check by the shell's name (#1341). + +Asserted: without dedup a repeated load doubles the table; with dedup the second load +of the same file stores nothing, returns 0 and says so in a NOTICE; a file changed at +the same path loads; the record is per table; dedup off loads again; the table's +records are exactly the SHA-256 digests of the two files it loaded; and no prepared +transaction is left behind. + +THREE FIXES THE SHELL SHARES. The changed file now differs from the first ONLY in its +last row, and that row lies past the coordinator's first 256 KiB read, so a fingerprint +of the first read alone refuses it; the records are compared with the server's own +sha256() of each file. The skip's NOTICE is read, not only the 0. And the two control +checks lose their "premise:" prefix under #1339: dedup is pgcolumnar's own default, and +forcing it on reddens them. + +ON A PRIVATE DATABASE, built in the shell's order by a module fixture. The files live +in `server_dir`, because the backend writes and reads them. +""" +import shutil + +import psycopg +import pytest + +from pgc_vacuity import QUERY_ERROR, query_error + +DB = "pgc_parallel_copy_dedup" +ROWS = 25000 +FIRST_READ = 262144 # PCOPY_FINGERPRINT_BUFSZ, the coordinator's read size +SKIP_NOTICE = 'pgcolumnar.parallel_copy skipped a load already applied to "pcd"' + + +def _failed(v): + return str(v).startswith(QUERY_ERROR) + + +def _err(exc): + return query_error(str(exc).splitlines()[0] if str(exc) else type(exc).__name__) + + +def _int(v): + return int(v) if isinstance(v, str) and v.lstrip("-").isdigit() else v + + +class _Db: + def __init__(self, dsn): + self.dsn = dsn + + def run(self, sql): + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + conn.execute(sql) + except psycopg.Error as exc: + print(f"-- step failed: {str(exc).splitlines()[0]}") + + def q(self, sql): + """-> the first column as text, "" for NULL, or a query_error().""" + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + row = conn.execute(sql).fetchone() + except psycopg.Error as exc: + return _err(exc) + return "" if row is None or row[0] is None else str(row[0]) + + def copy_noticed(self, sql): + """-> (the first column as text, the number of skip NOTICEs), or two query_error()s.""" + notes = [] + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + conn.add_notice_handler(lambda d: notes.append((d.severity_nonlocalized or d.severity, + d.message_primary or ""))) + row = conn.execute(sql).fetchone() + except psycopg.Error as exc: + return _err(exc), _err(exc) + got = "" if row is None or row[0] is None else str(row[0]) + return got, str(sum(1 for sev, msg in notes if sev == "NOTICE" and msg == SKIP_NOTICE)) + + def mk(self, table): + self.run(f"DROP TABLE IF EXISTS {table}; CREATE TABLE {table} (id int, txt text) USING pgcolumnar") + + +@pytest.fixture(scope="module") +def pcd(pgc_cluster, server_dir): + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {DB} WITH (FORCE)") + a.execute(f"CREATE DATABASE {DB}") + try: + d = _Db(pgc_cluster.dsn(dbname=DB)) + d.run("CREATE EXTENSION IF NOT EXISTS pgcolumnar") + f = server_dir / "pcd_load.txt" + g = server_dir / "pcd_other.txt" + d.run(f"COPY (SELECT g AS id, 'v'||g AS txt FROM generate_series(1,{ROWS}) g) " + f"TO '{f}' WITH (FORMAT text)") + # differs from f ONLY in its last row, past the first read + d.run(f"COPY (SELECT g AS id, CASE WHEN g = {ROWS} THEN 'changed' ELSE 'v'||g END AS txt " + f"FROM generate_series(1,{ROWS}) g) TO '{g}' WITH (FORMAT text)") + sha = "SELECT encode(sha256(pg_read_binary_file('{p}')), 'hex')" + r = {"sha f": d.q(sha.format(p=f)), "sha g": d.q(sha.format(p=g))} + r["past first read"] = d.q(f"SELECT ((SELECT size FROM pg_stat_file('{g}')) " + f"- octet_length('{ROWS}'||E'\\tchanged\\n') >= {FIRST_READ})::int") + + load = "SELECT pgcolumnar.parallel_copy('{t}', '{p}', 4{dd})" + d.mk("pcd_plain") + r["r1"] = d.q(load.format(t="pcd_plain", p=f, dd="")) + r["r2"] = d.q(load.format(t="pcd_plain", p=f, dd="")) + r["plain count"] = d.q("SELECT count(*) FROM pcd_plain") + + d.mk("pcd") + r["d1"] = d.q(load.format(t="pcd", p=f, dd=", true")) + r["c1"] = d.q("SELECT count(*) FROM pcd") + r["d2"], r["skip notices"] = d.copy_noticed(load.format(t="pcd", p=f, dd=", true")) + r["c2"] = d.q("SELECT count(*) FROM pcd") + + shutil.copyfile(g, f) + r["d3"] = d.q(load.format(t="pcd", p=f, dd=", true")) + r["c3"] = d.q("SELECT count(*) FROM pcd") + + d.mk("pcd_other") + r["d4"] = d.q(load.format(t="pcd_other", p=f, dd=", true")) + r["other count 1"] = d.q("SELECT count(*) FROM pcd_other") + r["d5"] = d.q(load.format(t="pcd_other", p=f, dd=", false")) + r["other count 2"] = d.q("SELECT count(*) FROM pcd_other") + + r["records"] = d.q("SELECT count(*) FROM pgcolumnar.load_fingerprint " + "WHERE relation_oid = 'pcd'::regclass") + r["digests"] = d.q("SELECT string_agg(encode(fingerprint, 'hex'), ',' ORDER BY loaded_at) " + "FROM pgcolumnar.load_fingerprint WHERE relation_oid = 'pcd'::regclass") + # this database's own: other suites in the session prepare transactions too + r["prepared"] = d.q("SELECT count(*) FROM pg_prepared_xacts WHERE database = current_database()") + print(f"-- dedup on: load 1 returned {r['d1']} (table {r['c1']}), " + f"load 2 returned {r['d2']} (table {r['c2']}); changed file: {r['d3']} (table {r['c3']})") + yield r + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {DB} WITH (FORCE)") + + +def test_pcdedup_the_changed_row_lies_past_the_first_read(pcd, expect): + expect.num(_int(pcd["past first read"]), 1, "premise: the changed row lies past the first 256 KiB of the file") + + +# Not premises (#1339): dedup is pgcolumnar's own default, and forcing it on reddens both. +def test_pcdedup_without_dedup_a_repeated_load_doubles_the_table(pcd, expect): + expect.num(_int(pcd["plain count"]), ROWS * 2, + "without dedup a repeated load doubles the table, which is the defect") + + +def test_pcdedup_each_plain_load_reported_its_rows(pcd, expect): + r1, r2 = pcd["r1"], pcd["r2"] + got = r1 if _failed(r1) else r2 if _failed(r2) else f"{r1}/{r2}" + expect.text(got, f"{ROWS}/{ROWS}", "and each of those loads reported its rows") + + +def test_pcdedup_the_first_load_stores_its_rows(pcd, expect): + expect.num(_int(pcd["c1"]), ROWS, "the first load stores its rows normally (#403 item 7)") + + +def test_pcdedup_the_first_load_reports_them(pcd, expect): + expect.num(_int(pcd["d1"]), ROWS, "and reports them") + + +def test_pcdedup_the_second_load_of_the_same_file_stores_nothing(pcd, expect): + expect.num(_int(pcd["c2"]), ROWS, "the second load of the same file stores nothing (#403 item 7)") + + +def test_pcdedup_the_second_load_reports_0(pcd, expect): + expect.num(_int(pcd["d2"]), 0, "and reports 0 rather than claiming it loaded them") + + +def test_pcdedup_the_skip_says_so_in_a_notice(pcd, expect): + expect.num(_int(pcd["skip notices"]), 1, + "and says in a NOTICE that it skipped the load, rather than returning a silent 0") + + +def test_pcdedup_a_changed_file_at_the_same_path_is_loaded(pcd, expect): + expect.num(_int(pcd["c3"]), ROWS * 2, "a changed file at the same path is loaded, not skipped (#403 item 7)") + + +def test_pcdedup_the_changed_file_reports_its_rows(pcd, expect): + expect.num(_int(pcd["d3"]), ROWS, "and reports the rows it loaded") + + +def test_pcdedup_the_same_file_loads_into_a_different_table(pcd, expect): + expect.num(_int(pcd["other count 1"]), ROWS, + "the same file loads into a DIFFERENT table (the record is per table)") + + +def test_pcdedup_the_other_table_load_reports_its_rows(pcd, expect): + expect.num(_int(pcd["d4"]), ROWS, "and reports its rows") + + +def test_pcdedup_dedup_off_loads_again(pcd, expect): + expect.num(_int(pcd["other count 2"]), ROWS * 2, + "asking for no dedup loads again even though the file is on record") + + +def test_pcdedup_dedup_off_reports_those_rows(pcd, expect): + expect.num(_int(pcd["d5"]), ROWS, "and reports those rows too") + + +def test_pcdedup_the_skipped_load_left_no_record(pcd, expect): + expect.num(_int(pcd["records"]), 2, "the skipped load left exactly one record for that table and file") + + +def test_pcdedup_the_records_hold_each_files_sha256(pcd, expect): + sf, sg = pcd["sha f"], pcd["sha g"] + want = sf if _failed(sf) else sg if _failed(sg) else f"{sf},{sg}" + expect.text(pcd["digests"], want, "the records hold the SHA-256 of each loaded file's bytes") + + +def test_pcdedup_no_prepared_transaction_leaked(pcd, expect): + expect.num(_int(pcd["prepared"]), 0, "no prepared transaction leaked through any of it") diff --git a/test/pytest/test_projection_drop_column.py b/test/pytest/test_projection_drop_column.py index 9768cb38e..26e25302d 100644 --- a/test/pytest/test_projection_drop_column.py +++ b/test/pytest/test_projection_drop_column.py @@ -31,11 +31,14 @@ premise below fails first and names reachability instead of leaving `42501 != 42P01` to be interpreted. -ONE THING THIS PORT ASSERTS THAT THE BASH SUITE DOES NOT. The two non-owner arms exist -to show that a stranger cannot tell a projected column from an unprojected one. The bash -suite asserts each against the literal `42501` and leaves the reader to notice they are -the same string. Here they are also compared to each other, so the property -- these two -are INDISTINGUISHABLE -- is asserted rather than implied by two separate constants. +THE TWO NON-OWNER REFUSALS ARE COMPARED TO EACH OTHER, in both suites (#1339). The arms +exist to show that a stranger cannot tell a projected column from an unprojected one, so +the property -- these two are INDISTINGUISHABLE -- is asserted directly rather than +implied by two separate constants, and it is asserted before them, so it can fail. + +**The lock is tested by contention.** The ownership check runs before the lock and again +after it, and both raise 42501, so a SQLSTATE alone cannot tell which one refused. While +the owner holds SHARE on the table, only the unlocked check refuses a stranger at once. """ import psycopg @@ -110,15 +113,43 @@ def test_drop_column_is_refused_while_a_projection_depends_on_it(pgc_conn, expec proj_err = _as_nobody(cur, "ALTER TABLE pdc DROP COLUMN sort_key") plain_err = _as_nobody(cur, "ALTER TABLE pdc DROP COLUMN payload") + # The property is that the two are INDISTINGUISHABLE, not that each happens to + # equal a constant, so it is compared FIRST. Placed after the two constant arms + # it could never run red: `expect` stops at the first failure, and if both + # constants hold the two are equal. With both ownership checks deleted, the + # stranger gets 2BP01 for the projected column and 42501 for the other, and + # this is the check that names it (#1339). + expect.text(proj_err.sqlstate if proj_err else "no error", + plain_err.sqlstate if plain_err else "no error", + "and the two refusals are the same error, so neither leaks") expect.sqlstate(proj_err, "42501", "a non-owner learns nothing from a projected column") expect.sqlstate(plain_err, "42501", "the same non-owner error is returned for an unprojected column") - # EXTRA: the property is that the two are INDISTINGUISHABLE, not that each - # happens to equal a constant. - expect.text(proj_err.sqlstate if proj_err else "no error", - plain_err.sqlstate if plain_err else "no error", - "premise: and the two refusals are the same error, so neither leaks") + + # THE LOCK HALF OF THE PROPERTY (#1339). The ownership check runs twice: once + # with no lock, then again after ShareUpdateExclusiveLock. The arms above + # cannot tell them apart, because both raise 42501. Contention can: while the + # owner holds SHARE on pdc, which conflicts with ShareUpdateExclusiveLock, a + # stranger refused by the unlocked check gets 42501 at once, and one that + # reaches the lock first waits out its lock_timeout (55P03). + holder = psycopg.connect(pgc_conn.info.dsn, autocommit=False) + try: + holder.execute(f'SET search_path TO "{schema}"') + holder.execute("LOCK TABLE pdc IN SHARE MODE") + held = _one(cur, "SELECT count(*) FROM pg_locks WHERE relation = 'pdc'::regclass " + "AND mode = 'ShareLock' AND granted") + expect.num(held, 1, "premise: the owner holds SHARE on pdc before the stranger asks") + cur.execute("SET lock_timeout = '2s'") + try: + waited_err = _as_nobody(cur, "ALTER TABLE pdc DROP COLUMN sort_key") + finally: + cur.execute("RESET lock_timeout") + expect.sqlstate(waited_err, "42501", + "a non-owner is refused before it waits for the table lock") + finally: + holder.rollback() + holder.close() # --- the owner is refused as a dependency --------------------------- # diff --git a/test/pytest/test_projection_parallel.py b/test/pytest/test_projection_parallel.py index c1b673552..56524235d 100644 --- a/test/pytest/test_projection_parallel.py +++ b/test/pytest/test_projection_parallel.py @@ -232,15 +232,18 @@ def test_projection_parallel(pgc_conn, expect): want, "a parallel covering projection returns the covering rows once", ) + # NOT PREMISES (#1339, #1236). The worker count is the projection path's own + # parallel_workers: with it set to 1, Workers Launched reads 1 and these go red. + # A check a pgcolumnar change can redden is an arm. expect.num( launched, 2, - "premise: EXPLAIN ANALYZE launched two workers", + "EXPLAIN ANALYZE launched two workers", ) expect.num( len(worker_rows), 2, - "premise: ANALYZE printed a rows= line per launched worker", + "ANALYZE printed a rows= line per launched worker", ) expect.num( n_busy, diff --git a/test/pytest/test_projection_privilege.py b/test/pytest/test_projection_privilege.py index 40e8b986c..eb84a8682 100644 --- a/test/pytest/test_projection_privilege.py +++ b/test/pytest/test_projection_privilege.py @@ -254,11 +254,18 @@ def test_a_role_with_execute_but_no_select_is_refused(pgc_cluster, pgc_conn, exp schema = _fixture(cur) _, err = _as(pgc_cluster, EXEC, f"SELECT count(*) FROM pgcolumnar.{func}('prj_secret','p1')", schema) - expect.sqlstate(err, "42501", name) if func == "read_projection": + # TWO PROPERTIES, TWO ARMS, as in `projection_privilege.sh` (#1339). The first + # asks only that the call was refused; the second asks which code refused it. + # Both used to assert 42501 on the same error, so the second could never fail + # alone: with read_projection's aclcheck_error replaced by a plain elog + # (XX000), the first arm went red under the second's name. + expect.text("refused" if err is not None else "noerror", "refused", name) expect.sqlstate( err, "42501", "and THAT refusal is the C check, which raises 42501 from aclcheck_error") + else: + expect.sqlstate(err, "42501", name) def test_which_layer_refused_without_reading_the_message(pgc_cluster, pgc_conn, expect): @@ -372,12 +379,16 @@ def test_a_policy_restricted_caller_is_refused(pgc_cluster, pgc_conn, expect, fu _, err = _as(pgc_cluster, SEL, f"SELECT count(*) FROM pgcolumnar.{func}('prj_secret','p1')", schema) - expect.sqlstate(err, "0A000", name) if func == "read_projection": + # Refused, then WHICH refusal, as in the shell suite (#1339): asserting 0A000 + # twice on one error left the second arm unable to fail on its own. + expect.text("refused" if err is not None else "noerror", "refused", name) expect.sqlstate( err, "0A000", "and the refusal says the feature is not supported, not that a privilege " "is missing") + else: + expect.sqlstate(err, "0A000", name) def test_the_acl_is_checked_before_rls_so_no_privilege_discloses_no_rls_state( diff --git a/test/pytest/test_projection_scan_io.py b/test/pytest/test_projection_scan_io.py index 13a2a20aa..5d58cbc18 100644 --- a/test/pytest/test_projection_scan_io.py +++ b/test/pytest/test_projection_scan_io.py @@ -263,3 +263,14 @@ def test_an_empty_covering_projection_is_priced_as_one_page(pgc_conn, expect): "onck", "an empty covering projection is still priced as one page, so it is chosen", ) + # THE OTHER DIRECTION (#1339). The arm above catches over-pricing only: priced at + # 0 pages instead of 1, the projection is cheaper still and stays chosen. `_plan` + # sets every CPU cost to 0, so the scan's whole cost is its page cost, and a total + # of zero means it was priced at zero pages. Measured on PG17: 5.0 with the clamp, + # 0.0 with `if (pages < 1) pages = 1;` deleted, and the projection chosen both times. + cost = (node or {}).get("Total Cost") + expect.text( + "unread" if cost is None else ("priced" if cost > 0 else f"free at {cost}"), + "priced", + "and its scan is not priced at zero pages", + ) diff --git a/test/pytest/test_projections.py b/test/pytest/test_projections.py index 29894df6b..3d5ffde13 100644 --- a/test/pytest/test_projections.py +++ b/test/pytest/test_projections.py @@ -136,6 +136,20 @@ def walk(node): return found[0] if found else None +def _plan_field(plan, key): + """-> the first value of `key` anywhere in a JSON plan, or None.""" + if isinstance(plan, dict): + if key in plan: + return plan[key] + plan = list(plan.values()) + if isinstance(plan, list): + for v in plan: + found = _plan_field(v, key) + if found is not None: + return found + return None + + def _read_projection(cur, table, name): """The projection's rows as the server renders them: columns joined by '|'.""" return _rows(cur, "SELECT pgcolumnar.read_projection(%s, %s)", (table, name)) @@ -492,6 +506,23 @@ def test_a_covering_sort_key_query_reads_the_projection(planner, expect): _rows(cur, "SELECT count(*), sum(a) FROM ps_h " "WHERE c BETWEEN 100 AND 200"), "aggregate over projection scan matches oracle") + # THE EXECUTOR, NOT THE PLANNER (#1339). The first arm reads the name from + # EXPLAIN, which takes it before the executor decides what to open, and the + # two oracle arms pass whichever storage is read. So a scan that EXPLAINs as + # `pc` and reads the base passed all three. Executed, the sorted projection + # lets the reader skip the vectors outside 100..200 and the filter sees only + # part of the table; the base, unsorted on c, sends every row to the filter. + # Measured on PG17: 5980 removed against 17980 for the base. + analyzed = _one(cur, "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF, " + "FORMAT JSON) " + COVERING) + removed = _plan_field(analyzed, "Rows Removed by Filter") + outside = _one(cur, "SELECT count(*) FROM ps_h WHERE NOT (c BETWEEN 100 AND 200)") + expect.text( + "unread" if removed is None + else ("fewer" if removed < outside else f"all {removed} of {outside}"), + "fewer", + "executing it reads the projection, so the filter sees fewer rows than " + "the base holds") def test_the_guc_is_an_off_switch(planner, expect): @@ -620,6 +651,32 @@ def test_an_old_snapshot_never_sees_rows_committed_after_it(pgc_conn, expect): dsn = pgc_conn.info.dsn COUNT = "SELECT count(a) FROM pm WHERE c BETWEEN 1 AND 20000" + + # THE PLAN THE ARMS BELOW NAME (#1339). Four of them say "via projection" or + # "projection scan", and nothing asked how COUNT plans. Planned on the base, every + # one of them passes whatever the projection scan does with the snapshot. + with pgc_conn.cursor() as cur: + cur.execute("SET pgcolumnar.enable_projection_scan = on") + expect.text(_projection_in_plan(_plan(cur, COUNT)) or "none", "pmp", + "the old-snapshot count is planned as a projection scan") + + def answer(ac): + """-> session A's count, or the error that stopped it. + + The shell suite's two timeout arms fail when session A gives no count. Here a + statement_timeout stands in for its 20-second poll, and an error is caught + rather than raised, so those two arms read what they say: that A answered. + """ + try: + return _one(ac, COUNT) + except psycopg.Error as exc: + ac.connection.rollback() + return exc + + def answered(reading): + return ("answered" if not isinstance(reading, psycopg.Error) + else f"no answer: {reading.sqlstate}") + a = psycopg.connect(dsn, autocommit=False) try: with a.cursor() as ac: @@ -627,10 +684,11 @@ def test_an_old_snapshot_never_sees_rows_committed_after_it(pgc_conn, expect): # the same one or it would read a different (absent) table. ac.execute(f'SET search_path TO "{schema}", public') ac.execute("SET pgcolumnar.enable_projection_scan = on") + ac.execute("SET statement_timeout = '20s'") ac.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") - base = _one(ac, COUNT) - expect.num(base, 10000, "session A opened snapshot") + base = answer(ac) + expect.text(answered(base), "answered", "session A opened snapshot") expect.num(base, 10000, "session A baseline via projection sees batch 1") @@ -638,8 +696,8 @@ def test_an_old_snapshot_never_sees_rows_committed_after_it(pgc_conn, expect): cur.execute("INSERT INTO pm SELECT g, g " "FROM generate_series(10001,20000) g") - after = _one(ac, COUNT) - expect.num(after, 10000, "session A responded post-commit") + after = answer(ac) + expect.text(answered(after), "answered", "session A responded post-commit") expect.num(after, 10000, "old snapshot projection scan does not see post-snapshot rows") a.commit() diff --git a/test/pytest/unported_suites.txt b/test/pytest/unported_suites.txt index 256c4fe3b..0cc618016 100644 --- a/test/pytest/unported_suites.txt +++ b/test/pytest/unported_suites.txt @@ -44,9 +44,6 @@ objstore_s3_read objstore_sink_write objstore_stash_recovery objstore_tls_read -parallel -parallel_copy -parallel_copy_dedup parallel_export_parquet parallel_vector_agg parquet_export