diff --git a/test/approx_aggregates.sh b/test/approx_aggregates.sh index a2b5b88a6..3318a652c 100755 --- a/test/approx_aggregates.sh +++ b/test/approx_aggregates.sh @@ -16,8 +16,10 @@ # its default precision (log2m 11) has a standard error of 1.04/sqrt(2048), so # three of them, 6.9%; tdigest at compression 100 within 0.1% at p99, about # four times what phase 0 measured (0.025%). -# - each type stored in a columnar table comes back usable: an hll byte for byte, -# a tdigest as a digest whose union still answers within the bound. +# - each type stored in a columnar table comes back byte for byte, and its union +# still answers. For tdigest that union's bound is a premise only because the +# byte-for-byte arm comes first: a storage change that altered a digest reddens +# the arm before it can move the percentile. # - the aggregate reads through the columnar scan, so the arms exercise our reader. # That is an arm, not a premise: turning the custom scan off by default reddens it # (#1236: a check a repository change can redden is not a premise). @@ -69,8 +71,16 @@ TD_H="$(qv "$P0 SELECT tdigest_percentile(v, 100, 0.99) FROM ax_heap;")" EXACT99="$(q "SELECT percentile_cont(0.99) WITHIN GROUP (ORDER BY v) FROM ax_heap;")" check "tdigest over a columnar table gives exactly what it gives over heap" "$TD_C" "$TD_H" check "premise: and its p99 is within 0.1% of the exact percentile" "$(within "$TD_C" "$EXACT99" 0.001)" "ok" -psql_run "CREATE TABLE ax_td (day int, t tdigest) USING pgcolumnar; - INSERT INTO ax_td SELECT day, tdigest(v, 100) FROM ax GROUP BY day;" +# The digests are built ONCE, into a heap table, and that table's values are stored. +# A tdigest depends on the order its inputs arrive in, so digests rebuilt from heap +# could differ from these for reasons that are not storage. Copied from one source, +# any difference is the columnar round trip's (#1339). +psql_run "CREATE TABLE ax_td_src USING heap AS SELECT day, tdigest(v, 100) AS t FROM ax GROUP BY day; + CREATE TABLE ax_td (day int, t tdigest) USING pgcolumnar; + INSERT INTO ax_td SELECT day, t FROM ax_td_src;" +check "a tdigest stored in a columnar table comes back byte for byte" \ + "$(q "SELECT (SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td) + = (SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td_src);")" "t" check "premise: a tdigest stored in a columnar table still answers: the union of daily digests is within 0.1% at p99" \ "$(within "$(q "SELECT tdigest_percentile(t, 0.99) FROM ax_td;")" "$EXACT99" 0.001)" "ok" diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index d10894222..51e0b9e7e 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -284,6 +284,7 @@ analyze_stats analyze_stats the sampled tuples carry a valid item pointer 15;16; analyze_stats analyze_stats the server survived ANALYZE 15;16;17;18;19 never - analyze_stats analyze_stats without the fetch penalty an unclustered ORDER BY takes the index (#355 premise) 15;16;17;18;19 never - analyze_stats analyze_stats without the penalty a selective scattered condition takes the index (#362 premise) 15;16;17;18;19 never - +approx_aggregates approx_aggregates a tdigest stored in a columnar table comes back byte for byte 15;16;17;18;19 2026-10-22 columnar_reader.c: a long value (over 512 bytes) reads as the previous long value read, so each stored digest comes back as another day's approx_aggregates approx_aggregates an hll stored in a columnar table comes back byte for byte 15;16;17;18;19 2026-10-13 the custom scan off by default, and the access-method scan skips every row whose row number ends in 999 approx_aggregates approx_aggregates and the union of the stored sketches counts what one sketch over all rows counts 15;16;17;18;19 2026-10-13 the custom scan off by default, and the access-method scan skips every row whose row number ends in 999 approx_aggregates approx_aggregates hll over a columnar table builds the same sketch, byte for byte, as over heap 15;16;17;18;19 2026-10-13 the custom scan off by default, and the access-method scan skips every row whose row number ends in 999 @@ -4735,6 +4736,23 @@ parallel_am_scan parallel_am_scan premise: the serial plan is not a columnar cus parallel_am_scan parallel_am_scan premise: the table holds every inserted row 15;16;17;18;19 never - parallel_am_scan parallel_am_scan premise: with the custom scan off the serial plan is a Seq Scan 15;16;17;18;19 never - parallel_am_scan parallel_am_scan workers share the table-AM scan, it is not a single claimer 15;16;17;18;19 never - +parallel_copy_dedup parallel_copy_dedup a changed file at the same path is loaded, not skipped (#403 item 7) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: pcopy_fingerprint_file hashes only the first 256 KiB read (break after the first pg_cryptohash_update);columnar_parallel_copy.c: the refused load commits its ranges anyway (pcopy_finish_prepared(gid, true) in place of the rollback) +parallel_copy_dedup parallel_copy_dedup and each of those loads reported its rows 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: dedup always on, whatever the caller passed (dedup = true) +parallel_copy_dedup parallel_copy_dedup and reports 0 rather than claiming it loaded them 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the refused load reports the rows it discarded (total_rows summed, not 0) +parallel_copy_dedup parallel_copy_dedup and reports its rows 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarLoadFingerprintSeen matches the fingerprint in any table (heap scan on the fingerprint key alone) +parallel_copy_dedup parallel_copy_dedup and reports the rows it loaded 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: pcopy_fingerprint_file hashes only the first 256 KiB read (break after the first pg_cryptohash_update) +parallel_copy_dedup parallel_copy_dedup and reports them 15;16;17;18;19 never - +parallel_copy_dedup parallel_copy_dedup and reports those rows too 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: dedup always on, whatever the caller passed (dedup = true) +parallel_copy_dedup parallel_copy_dedup and says in a NOTICE that it skipped the load, rather than returning a silent 0 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the skip's NOTICE never raised (if (false && skipped)) +parallel_copy_dedup parallel_copy_dedup asking for no dedup loads again even though the file is on record 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarLoadFingerprintSeen matches the fingerprint in any table (heap scan on the fingerprint key alone);columnar_parallel_copy.c: dedup always on, whatever the caller passed (dedup = true) +parallel_copy_dedup parallel_copy_dedup no prepared transaction leaked through any of it 15;16;17;18;19 never - +parallel_copy_dedup parallel_copy_dedup premise: the changed row lies past the first 256 KiB of the file 15;16;17;18;19 never - +parallel_copy_dedup parallel_copy_dedup the first load stores its rows normally (#403 item 7) 15;16;17;18;19 never - +parallel_copy_dedup parallel_copy_dedup the records hold the SHA-256 of each loaded file's bytes 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: pcopy_fingerprint_file hashes only the first 256 KiB read (break after the first pg_cryptohash_update) +parallel_copy_dedup parallel_copy_dedup the same file loads into a DIFFERENT table (the record is per table) 15;16;17;18;19 2026-10-22 columnar_metadata.c: PgColumnarLoadFingerprintSeen matches the fingerprint in any table (heap scan on the fingerprint key alone) +parallel_copy_dedup parallel_copy_dedup the second load of the same file stores nothing (#403 item 7) 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: the refused load commits its ranges anyway (pcopy_finish_prepared(gid, true) in place of the rollback) +parallel_copy_dedup parallel_copy_dedup the skipped load left exactly one record for that table and file 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: pcopy_fingerprint_file hashes only the first 256 KiB read (break after the first pg_cryptohash_update) +parallel_copy_dedup parallel_copy_dedup without dedup a repeated load doubles the table, which is the defect 15;16;17;18;19 2026-10-22 columnar_parallel_copy.c: dedup always on, whatever the caller passed (dedup = true) parallel_degree parallel_degree both tables hold the same rows 15;16;17;18;19 2026-10-21 columnar_tableam.c: tuple_insert drops the row whose first column is 1 parallel_degree parallel_degree identical data is granted the same parallel degree 15;16;17;18;19 2026-10-21 columnar_customscan.c: the #451 estimate over-asks into the cap, pages * 243;columnar_customscan.c: the #451 estimate over-asks, pages * 9;columnar_customscan.c: the #451 uncompressed-page estimate never applied, so the scan is sized from compressed relpages;columnar_customscan.c: the parallel columnar partial path never added (if (false && workers > 0)) parallel_degree parallel_degree premise: the columnar table really is smaller on disk 15;16;17;18;19 never - @@ -5212,11 +5230,22 @@ prepared_writes prepared_writes no backend was terminated by a signal 15;16;17;1 prepared_writes prepared_writes premise: prepared transactions are enabled 15;16;17;18;19 never - prepared_writes prepared_writes premise: the fetch is an index scan, which is what reaches the fetch cache 15;16;17;18;19 never - prepared_writes prepared_writes premise: the server log is where a crash would be recorded 15;16;17;18;19 never - +projection_drop_column projection_drop_column a non-owner is refused before it waits for the table lock 15;16;17;18;19 2026-10-22 columnar_tableam.c: both DROP COLUMN ownership checks deleted, so a stranger gets 2BP01 for a projected column and 42501 for another;columnar_tableam.c: the unlocked DROP COLUMN ownership check deleted, so a stranger queues for ShareUpdateExclusiveLock before it is refused +projection_drop_column projection_drop_column a non-owner learns nothing from a projected column 15;16;17;18;19 2026-10-22 columnar_tableam.c: both DROP COLUMN ownership checks deleted, so a stranger gets 2BP01 for a projected column and 42501 for another +projection_drop_column projection_drop_column a parent DROP sees projections on columnar partitions 15;16;17;18;19 never - +projection_drop_column projection_drop_column and the projection still receives the row 15;16;17;18;19 never - +projection_drop_column projection_drop_column and the two refusals are the same error, so neither leaks 15;16;17;18;19 2026-10-22 columnar_tableam.c: both DROP COLUMN ownership checks deleted, so a stranger gets 2BP01 for a projected column and 42501 for another +projection_drop_column projection_drop_column dropping a projected column is refused as a dependency 15;16;17;18;19 never - +projection_drop_column projection_drop_column dropping an unrelated column remains allowed 15;16;17;18;19 never - +projection_drop_column projection_drop_column premise: the owner holds SHARE on pdc before the stranger asks 15;16;17;18;19 never - +projection_drop_column projection_drop_column the rejected DDL leaves the table writable 15;16;17;18;19 never - +projection_drop_column projection_drop_column the rejected parent DDL leaves its partition writable 15;16;17;18;19 never - +projection_drop_column projection_drop_column the same non-owner error is returned for an unprojected column 15;16;17;18;19 never - +projection_parallel projection_parallel ANALYZE printed a rows= line per launched worker 15;16;17;18;19 2026-10-22 columnar_customscan.c: the parallel projection path's parallel_workers set to 1 instead of workers +projection_parallel projection_parallel EXPLAIN ANALYZE launched two workers 15;16;17;18;19 2026-10-22 columnar_customscan.c: the parallel projection path's parallel_workers set to 1 instead of workers projection_parallel projection_parallel a covering projection can be a parallel scan 15;16;17;18;19 2026-09-18 - projection_parallel projection_parallel a page cost that makes I/O dominate costs the parallel covering path 15;16;17;18;19 2026-09-23 the partial covering total amortises I/O as well as CPU: (ioRunProj + cpuRunProj) / divisor projection_parallel projection_parallel a parallel covering projection returns the covering rows once 15;16;17;18;19 2026-10-13 a parallel worker never wires the shared row-group counter, so every worker scans every group -projection_parallel projection_parallel premise: ANALYZE printed a rows= line per launched worker 15;16;17;18;19 never - -projection_parallel projection_parallel premise: EXPLAIN ANALYZE launched two workers 15;16;17;18;19 never - projection_parallel projection_parallel premise: a covering projection exists 15;16;17;18;19 never - projection_parallel projection_parallel premise: a parallel base scan is available when the projection is off 15;16;17;18;19 never - projection_parallel projection_parallel premise: a serial covering query uses the projection 15;16;17;18;19 never - @@ -5224,18 +5253,18 @@ projection_parallel projection_parallel premise: the covering plan is parallel a projection_parallel projection_parallel premise: the table holds every inserted row 15;16;17;18;19 never - projection_parallel projection_parallel the parallel count is a parallel covering projection scan 15;16;17;18;19 never - projection_parallel projection_parallel the workers' rows add up to the covering rows, each returned once 15;16;17;18;19 2026-10-13 a parallel worker never wires the shared row-group counter, so every worker scans every group -projection_parallel projection_parallel workers share the covering projection scan, it is not a single claimer 15;16;17;18;19 2026-10-13 first-wins-claimer;only the first parallel worker claims row groups, so one worker scans the whole table (#1336) -projection_privilege projection_privilege a caller with no SELECT is told a privilege is missing, not that RLS is in force 15;16;17;18;19 never - +projection_parallel projection_parallel workers share the covering projection scan, it is not a single claimer 15;16;17;18;19 2026-10-22 columnar_customscan.c: the parallel projection path's parallel_workers set to 1 instead of workers;first-wins-claimer;only the first parallel worker claims row groups, so one worker scans the whole table (#1336) +projection_privilege projection_privilege a caller with no SELECT is told a privilege is missing, not that RLS is in force 15;16;17;18;19 2026-10-22 columnar_projection.c: read_projection's aclcheck_error replaced by a plain elog, so its refusal is XX000 projection_privilege projection_privilege a role WITH SELECT still reads the projection 15;16;17;18;19 2026-10-14 read_projection stores no rows: its tuplestore_putvalues call removed, so every caller gets zero rows projection_privilege projection_privilege a role with EXECUTE but no SELECT is refused read_projection 15;16;17;18;19 never - projection_privilege projection_privilege a role with only schema USAGE is refused read_projection 15;16;17;18;19 never - -projection_privilege projection_privilege and THAT refusal is the C check, which raises 42501 from aclcheck_error 15;16;17;18;19 never - +projection_privilege projection_privilege and THAT refusal is the C check, which raises 42501 from aclcheck_error 15;16;17;18;19 2026-10-22 columnar_projection.c: read_projection's aclcheck_error replaced by a plain elog, so its refusal is XX000 projection_privilege projection_privilege and is refused reconstruct_via_projection 15;16;17;18;19 never - -projection_privilege projection_privilege and is refused reconstruct_via_projection, which leaks non-covered columns 15;16;17;18;19 never - +projection_privilege projection_privilege and is refused reconstruct_via_projection, which leaks non-covered columns 15;16;17;18;19 2026-10-22 columnar_projection.c: reconstruct_via_projection's aclcheck_error replaced by a plain elog, so its refusal is XX000 projection_privilege projection_privilege and read_projection does NOT, so the two differ as the issue says 15;16;17;18;19 2026-10-14 read_projection bound to the reconstruct_via_projection C symbol in the extension SQL, so it returns the non-covered columns too;read_projection stores no rows: its tuplestore_putvalues call removed, so every caller gets zero rows -projection_privilege projection_privilege and so does reconstruct_via_projection 15;16;17;18;19 never - +projection_privilege projection_privilege and so does reconstruct_via_projection 15;16;17;18;19 2026-10-22 columnar_vacuum.c: PgColumnarRequireNoRowSecurity raises 42501 instead of 0A000 projection_privilege projection_privilege and still reconstructs 15;16;17;18;19 2026-10-14 reconstruct_via_projection stores no rows: its tuplestore_putvalues call removed, so every caller gets zero rows -projection_privilege projection_privilege and the refusal says the feature is not supported, not that a privilege is missing 15;16;17;18;19 never - +projection_privilege projection_privilege and the refusal says the feature is not supported, not that a privilege is missing 15;16;17;18;19 2026-10-22 columnar_vacuum.c: PgColumnarRequireNoRowSecurity raises 42501 instead of 0A000 projection_privilege projection_privilege premise: t_prjexec can open a session, so a refusal below is a refusal 15;16;17;18;19 never - projection_privilege projection_privilege premise: t_prjexec really does hold EXECUTE, so it reaches the C gate 15;16;17;18;19 never - projection_privilege projection_privilege premise: t_prjsel CAN read the table, so its arm below tests the bar 15;16;17;18;19 never - @@ -5243,11 +5272,11 @@ projection_privilege projection_privilege premise: the test roles are not superu projection_privilege projection_privilege premise: the unprivileged role cannot read the table by any ordinary route 15;16;17;18;19 never - projection_privilege projection_privilege read_projection now refuses a policy-restricted caller (#563) 15;16;17;18;19 never - projection_privilege projection_privilege reconstruct really does return the NON-COVERED column 15;16;17;18;19 2026-10-14 reconstruct_via_projection renders every column the projection does not cover as NULL instead of the base value;reconstruct_via_projection stores no rows: its tuplestore_putvalues call removed, so every caller gets zero rows -projection_privilege projection_privilege reconstruct: a caller with no SELECT is told a privilege is missing, not that RLS is in force 15;16;17;18;19 never - +projection_privilege projection_privilege reconstruct: a caller with no SELECT is told a privilege is missing, not that RLS is in force 15;16;17;18;19 2026-10-22 columnar_projection.c: reconstruct_via_projection's aclcheck_error replaced by a plain elog, so its refusal is XX000 projection_privilege projection_privilege reconstruct: the EXECUTE role reaches the projection lookup on the same call 15;16;17;18;19 never - -projection_privilege projection_privilege reconstruct: the base ACL is checked before the projection name is looked up 15;16;17;18;19 never - +projection_privilege projection_privilege reconstruct: the base ACL is checked before the projection name is looked up 15;16;17;18;19 2026-10-22 columnar_projection.c: reconstruct_via_projection's aclcheck_error replaced by a plain elog, so its refusal is XX000 projection_privilege projection_privilege reconstruct: the no-EXECUTE role never reaches the body, so the grant is what stopped it 15;16;17;18;19 never - -projection_privilege projection_privilege the base ACL is checked before the projection name is looked up 15;16;17;18;19 never - +projection_privilege projection_privilege the base ACL is checked before the projection name is looked up 15;16;17;18;19 2026-10-22 columnar_projection.c: read_projection's aclcheck_error replaced by a plain elog, so its refusal is XX000 projection_privilege projection_privilege the no-EXECUTE role never reaches the body, so the grant is what stopped it 15;16;17;18;19 never - projection_privilege projection_privilege the owner reads the projection, and gets rows 15;16;17;18;19 2026-10-14 read_projection stores no rows: its tuplestore_putvalues call removed, so every caller gets zero rows projection_privilege projection_privilege the owner reconstructs, and gets rows 15;16;17;18;19 2026-10-14 reconstruct_via_projection stores no rows: its tuplestore_putvalues call removed, so every caller gets zero rows @@ -5365,6 +5394,7 @@ projection_scan_cost projection_scan_cost while a plain range on the sort key st projection_scan_io projection_scan_io a covering projection is not priced from the base table's pages 15;16;17;18;19 2026-09-19 rel->pages projection_scan_io projection_scan_io a covering projection whose storage cannot be found is not priced as one page 15;16;17;18;19 2026-09-21 return 1 projection_scan_io projection_scan_io an empty covering projection is still priced as one page, so it is chosen 15;16;17;18;19 2026-09-23 pages = 997 on the third return of pgcolumnar_projection_pages +projection_scan_io projection_scan_io and its scan is not priced at zero pages 15;16;17;18;19 2026-10-22 columnar_customscan.c: the one-page floor in pgcolumnar_projection_pages deleted, so an empty projection is priced at 0 pages projection_scan_io projection_scan_io premise: a covering path is still offered when projection storage cannot be found 15;16;17;18;19 never - projection_scan_io projection_scan_io premise: a covering projection exists 15;16;17;18;19 never - projection_scan_io projection_scan_io premise: the covering projection occupies a minority of the relation 15;16;17;18;19 never - @@ -5413,6 +5443,7 @@ projections projections drop unknown rejected 15;16;17;18;19 never - projections projections duplicate column rejected 15;16;17;18;19 never - projections projections duplicate name rejected 15;16;17;18;19 never - projections projections empty columns rejected 15;16;17;18;19 never - +projections projections executing it reads the projection, so the filter sees fewer rows than the base holds 15;16;17;18;19 2026-10-22 columnar_customscan.c: projection pages multiplied by 1000000, so the planner chooses the base;columnar_customscan.c: the executor never opens the chosen projection (if (false && projName)) while EXPLAIN still names it projections projections fp chunks carry min/max skip metadata 15;16;17;18;19 never - projections projections fp count after delete matches base 15;16;17;18;19 never - projections projections fp fan-out matches base (a,c) 15;16;17;18;19 never - @@ -5424,7 +5455,7 @@ projections projections fq fan-out matches base (b) 15;16;17;18;19 never - projections projections new snapshot projection scan sees both batches 15;16;17;18;19 never - projections projections no projection rows before first add 15;16;17;18;19 never - projections projections non-covering query (references b) uses the base 15;16;17;18;19 never - -projections projections old snapshot projection scan does not see post-snapshot rows 15;16;17;18;19 never - +projections projections old snapshot projection scan does not see post-snapshot rows 15;16;17;18;19 2026-10-22 columnar_customscan.c: the projection scan and its base liveness cache both read GetLatestSnapshot() instead of the query snapshot projections projections p1 columns 15;16;17;18;19 never - projections projections p1 gone after drop 15;16;17;18;19 never - projections projections p1 has its own storage id 15;16;17;18;19 never - @@ -5433,9 +5464,9 @@ projections projections p1 sort_key 15;16;17;18;19 never - projections projections p2 columns 15;16;17;18;19 never - projections projections p2 id is 2 15;16;17;18;19 never - projections projections p2 sort_key empty 15;16;17;18;19 never - -projections projections planner still uses projection after vacuum 15;16;17;18;19 never - +projections projections planner still uses projection after vacuum 15;16;17;18;19 2026-10-22 columnar_customscan.c: projection pages multiplied by 1000000, so the planner chooses the base projections projections premise: the base table holds every committed row 15;16;17;18;19 never - -projections projections projection chosen for covering + sort-key query 15;16;17;18;19 never - +projections projections projection chosen for covering + sort-key query 15;16;17;18;19 2026-10-22 columnar_customscan.c: projection pages multiplied by 1000000, so the planner chooses the base projections projections projection matches base after second vacuum 15;16;17;18;19 never - projections projections projection scan matches oracle after delete 15;16;17;18;19 never - projections projections projection survives vacuum 15;16;17;18;19 never - @@ -5454,6 +5485,7 @@ projections projections storage id resolves 15;16;17;18;19 never - projections projections table populated 15;16;17;18;19 never - projections projections table still readable after DDL 15;16;17;18;19 never - projections projections the drop really removed the projection 15;16;17;18;19 never - +projections projections the old-snapshot count is planned as a projection scan 15;16;17;18;19 2026-10-22 columnar_customscan.c: projection pages multiplied by 1000000, so the planner chooses the base projections projections the rebuild does not abort on it 15;16;17;18;19 never - projections projections two declared projections to start 15;16;17;18;19 never - projections projections two rows after first add (base + p1) 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 7e51bae74..7b08d4811 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -15,7 +15,7 @@ # # Adding a check to a suite that is already covered does not move it, which is # what makes it safe to bound. -suites_not_covered 75 +suites_not_covered 73 # # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become # one. @@ -5957,4 +5957,66 @@ suites_not_covered 75 # never 4129 + 8 = 4137 # dated 2791 + 7 = 2798 # 4137 + 2798 == 6935 -checks_never_observed_red 4137 +# +# #1339, a batch of projection checks (jdatcmd): 4137 -> 4132, and suites_not_covered +# 75 -> 74. Seven findings, each shown green on main under its mutation in both +# twins and red by name after the fix. projection_drop_column is seeded: it had no +# rows, and it enters with 11 checks from five-major runs (11/11 on each of 15-19, +# twin green on each). Its stranger arms read only 42501, so deleting the unlocked +# ownership check passed; a stranger is now asked while the owner holds SHARE, and +# queues to 55P03 under that mutation. Its "two refusals are the same" check lost its +# premise: prefix, joined the shell suite, and moved ahead of the two constant arms +# so it can fail. projection_parallel's two worker-count checks lost their premise: +# prefix: parallel_workers = 1 reddens both. projection_privilege's read_projection +# arms each asserted one SQLSTATE twice; the first now asks only "refused", as the +# shell suite does, and the shell's reconstruct arms read the SQLSTATE, as the twin +# does. projection_scan_io gains a check that an empty projection is not priced at +# zero pages. projections gains two: an EXPLAIN ANALYZE reading that the executor +# opened the projection, and that the old-snapshot count plans it. Its twin's two +# session-A timeout arms now read whether A answered, not the count again, and the +# shell's poll no longer takes an ERROR that echoes the token for an answer. 19 rows +# are dated by ten mutations, each reddening its target by the same name in both +# twins (the pytest twin stops at a test's first red): the unlocked ownership check +# deleted, both deleted, parallel_workers = 1, each function's aclcheck_error +# replaced by elog, the RLS refusal raised as 42501, the one-page floor deleted, the +# executor never opening the projection, projection pages multiplied by 1000000, +# and the projection scan and liveness cache both on GetLatestSnapshot(). 8 arrive +# never: one premise and seven drop_column checks no mutation here reached. +# +# ledger rows 6935 + 14 = 6949 +# never 4137 + 8 - 13 = 4132 +# dated 2798 + 6 + 13 = 2817 +# 4132 + 2817 == 6949 +# +# #1339, approx_aggregates gains one arm (jdatcmd): 4132 -> 4132, unchanged. "a tdigest stored in +# a columnar table comes back byte for byte" is dated by a reader that returns each long +# value as the previous long value read: main's suite passed every tdigest check under it, +# and only the hll arm went red. suites_not_covered does not move. +# +# ledger rows 6949 + 1 = 6950 +# never 4132 + 0 = 4132 +# dated 2817 + 1 = 2818 +# 4132 + 2818 == 6950 +# +# #1341, parallel_copy_dedup seeded, with its twin (jdatcmd): 3853 -> 3857, and +# suites_not_covered 105 -> 104. The suite had no rows. It enters with 17 checks from +# five-major runs (17/17 on each of 15-19, shell and twin alike), after three fixes. +# The changed file differed from the first in its first bytes and fitted in one +# 256 KiB read, so a fingerprint of the first read alone still loaded it and main's +# shell passed 14/14; the changed file now differs only in its last row, past the +# first read, and a new check compares the table's records with the server's +# sha256() of each file. The skip's NOTICE, promised in the header, was never read: +# with it removed main's shell passed 14/14; a new check reads it. And the two +# control checks lost their premise: prefix under #1339: dedup is pgcolumnar's own +# default, and forcing it on reddens them. 13 are dated by six mutations, each +# reddening the same names in both twins: the NOTICE never raised, the first read +# alone fingerprinted, dedup forced on, the refused load committed anyway, a +# fingerprint matched in any table, and the refused load reporting the rows it +# discarded. 4 arrive never: the fixture-size premise, the first load's two checks, +# and the prepared-transaction leak. +# +# ledger rows 6950 + 17 = 6967 +# never 4132 + 4 = 4136 +# dated 2818 + 13 = 2831 +# 4136 + 2831 == 6967 +checks_never_observed_red 4136 diff --git a/test/parallel_copy_dedup.sh b/test/parallel_copy_dedup.sh index 9fdc44390..ba768168d 100755 --- a/test/parallel_copy_dedup.sh +++ b/test/parallel_copy_dedup.sh @@ -49,35 +49,53 @@ DATADIR="$PGC_WORKDIR/pcdedup" mkdir -p "$DATADIR"; chmod 777 "$DATADIR" if [ "$(id -u)" = "0" ]; then chown postgres "$DATADIR"; fi -ROWS=20000 +# The coordinator reads the file 256 KiB at a time (PCOPY_FINGERPRINT_BUFSZ). The +# changed file G differs from F ONLY in its last row, and 25,000 rows put that row +# past the first read. When G differed from its first bytes and the file fitted in +# one read, a fingerprint of the first read alone loaded G as a change and this +# suite passed 14 of 14. +ROWS=25000 F="$DATADIR/load.txt" G="$DATADIR/other.txt" psql_run "COPY (SELECT g AS id, 'v'||g AS txt FROM generate_series(1,$ROWS) g) TO '$F' WITH (FORMAT text);" >/dev/null -psql_run "COPY (SELECT g AS id, 'w'||g AS txt FROM generate_series(1,$ROWS) g) - TO '$G' WITH (FORMAT text);" >/dev/null +psql_run "COPY (SELECT g AS id, CASE WHEN g = $ROWS THEN 'changed' ELSE 'v'||g END AS txt + FROM generate_series(1,$ROWS) g) TO '$G' WITH (FORMAT text);" >/dev/null q() { env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -Atq \ -c "$1" 2>&1 | tail -1 } +# The SHA-256 of each file's bytes, computed by the server from the file itself. +sha() { q "SELECT encode(sha256(pg_read_binary_file('$1')), 'hex')"; } +sha_f="$(sha "$F")" +sha_g="$(sha "$G")" +check_num "premise: the changed row lies past the first 256 KiB of the file" \ + "$(q "SELECT ((SELECT size FROM pg_stat_file('$G')) + - octet_length('$ROWS'||E'\tchanged\n') >= 262144)::int")" 1 + mk() { psql_run "DROP TABLE IF EXISTS $1; CREATE TABLE $1 (id int, txt text) USING pgcolumnar;" >/dev/null; } # ---- the control: without dedup the retry doubles the table ---------------- mk pcd_plain r1="$(q "SELECT pgcolumnar.parallel_copy('pcd_plain','$F',4)")" r2="$(q "SELECT pgcolumnar.parallel_copy('pcd_plain','$F',4)")" -check "premise: without dedup a repeated load doubles the table, which is the defect" \ +# Not premises (#1339): dedup is pgcolumnar's own default, and forcing it on reddens +# both of these. +check "without dedup a repeated load doubles the table, which is the defect" \ "$(q "SELECT count(*) FROM pcd_plain")" "$(( ROWS * 2 ))" -check "premise: and each of those loads reported its rows" "$r1/$r2" "$ROWS/$ROWS" +check "and each of those loads reported its rows" "$r1/$r2" "$ROWS/$ROWS" # ---- the same load twice, with dedup -------------------------------------- mk pcd d1="$(q "SELECT pgcolumnar.parallel_copy('pcd','$F',4,true)")" c1="$(q "SELECT count(*) FROM pcd")" -d2="$(q "SELECT pgcolumnar.parallel_copy('pcd','$F',4,true)")" +# All of psql's output, NOTICE included: the header promises the skip says so. +o2="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -Atq \ + -c "SELECT pgcolumnar.parallel_copy('pcd','$F',4,true)" 2>&1)" +d2="$(printf '%s\n' "$o2" | tail -1)" c2="$(q "SELECT count(*) FROM pcd")" echo "-- dedup on: load 1 returned $d1 (table $c1), load 2 returned $d2 (table $c2)" @@ -85,6 +103,8 @@ check "the first load stores its rows normally (#403 item 7)" "$c1" "$ROWS" check "and reports them" "$d1" "$ROWS" check "the second load of the same file stores nothing (#403 item 7)" "$c2" "$ROWS" check "and reports 0 rather than claiming it loaded them" "$d2" "0" +check_num "and says in a NOTICE that it skipped the load, rather than returning a silent 0" \ + "$(printf '%s\n' "$o2" | grep -c 'NOTICE: pgcolumnar.parallel_copy skipped a load already applied to "pcd"')" 1 # ---- a changed file at the same path is a different load ------------------- cp "$G" "$F" @@ -112,6 +132,13 @@ check "the skipped load left exactly one record for that table and file" \ "$(q "SELECT count(*) FROM pgcolumnar.load_fingerprint WHERE relation_oid = 'pcd'::regclass")" "2" +# The SHA-256 of the file's bytes, not of some part of them: the two records are +# exactly the two files' digests, in load order. +check_text "the records hold the SHA-256 of each loaded file's bytes" \ + "$(q "SELECT string_agg(encode(fingerprint, 'hex'), ',' ORDER BY loaded_at) + FROM pgcolumnar.load_fingerprint WHERE relation_oid = 'pcd'::regclass")" \ + "$sha_f,$sha_g" + check "no prepared transaction leaked through any of it" \ "$(q "SELECT count(*) FROM pg_prepared_xacts")" "0" diff --git a/test/projection_drop_column.sh b/test/projection_drop_column.sh index 724631665..8500ba707 100755 --- a/test/projection_drop_column.sh +++ b/test/projection_drop_column.sh @@ -39,10 +39,39 @@ psql_run "INSERT INTO pdc # The pre-statement dependency check must not tell a stranger which column the # projection covers, or let that role take the lock retained by the owner path. psql_run "CREATE ROLE pdc_nobody LOGIN;" -check "a non-owner learns nothing from a projected column" \ - "$(sqlstate_as pdc_nobody 'ALTER TABLE pdc DROP COLUMN sort_key;')" "42501" +proj_state="$(sqlstate_as pdc_nobody 'ALTER TABLE pdc DROP COLUMN sort_key;')" +plain_state="$(sqlstate_as pdc_nobody 'ALTER TABLE pdc DROP COLUMN payload;')" +# The property is that the two refusals are INDISTINGUISHABLE, so it is compared +# first: with both ownership checks deleted, the stranger gets 2BP01 for the +# projected column and 42501 for the other, and this is the check that names it. +check "and the two refusals are the same error, so neither leaks" \ + "$proj_state" "$plain_state" +check "a non-owner learns nothing from a projected column" "$proj_state" "42501" check "the same non-owner error is returned for an unprojected column" \ - "$(sqlstate_as pdc_nobody 'ALTER TABLE pdc DROP COLUMN payload;')" "42501" + "$plain_state" "42501" + +# THE LOCK HALF OF THE COMMENT ABOVE (#1339). The ownership check runs twice: +# once with no lock, then again after ShareUpdateExclusiveLock. The two checks +# above cannot tell them apart, because both raise 42501. Contention can: while +# the owner holds SHARE on pdc, which conflicts with ShareUpdateExclusiveLock, a +# stranger refused by the unlocked check gets 42501 at once, and one that reaches +# the lock first waits out its lock_timeout (55P03). +holder_log="$PGC_WORKDIR/pdc_holder.log" +env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \ + -Atq -v ON_ERROR_STOP=0 -c "BEGIN;" -c "LOCK TABLE pdc IN SHARE MODE;" \ + -c "SELECT pg_sleep(3600) /* pdc_holder */;" -c "ROLLBACK;" > "$holder_log" 2>&1 & +holder_pid=$! +held=0 +for _ in $(seq 1 40); do + held="$(q "SELECT count(*) FROM pg_locks WHERE relation = 'pdc'::regclass AND mode = 'ShareLock' AND granted;")" + [ "${held:-0}" -ge 1 ] && break + sleep 0.25 +done +check "premise: the owner holds SHARE on pdc before the stranger asks" "${held:-0}" "1" +check "a non-owner is refused before it waits for the table lock" \ + "$(sqlstate_as pdc_nobody "SET lock_timeout = '2s'; ALTER TABLE pdc DROP COLUMN sort_key;")" "42501" +psql_run "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE query LIKE '%pdc_holder%' AND pid <> pg_backend_pid();" >/dev/null +wait "$holder_pid" 2>/dev/null || true # Red before the fix: PostgreSQL accepts this (00000), and the next INSERT # reaches the projection writer with a type OID of zero. diff --git a/test/projection_parallel.sh b/test/projection_parallel.sh index 010951c60..7d041d14f 100755 --- a/test/projection_parallel.sh +++ b/test/projection_parallel.sh @@ -277,10 +277,13 @@ check "premise: the covering plan is parallel at the default page cost" \ check "a page cost that makes I/O dominate costs the parallel covering path" \ "$(shape "$(explain_cov_spc $SPC_HI)")" "projection-only" -check "premise: EXPLAIN ANALYZE launched two workers" \ +# NOT PREMISES (#1339, #1236). The worker count is the projection path's own +# parallel_workers: with it set to 1, Workers Launched reads 1 and both of these +# go red. A check a pgcolumnar change can redden is an arm. +check "EXPLAIN ANALYZE launched two workers" \ "$(echo "$par_on_ana" | grep -oE 'Workers Launched: [0-9]+' | head -1 | grep -oE '[0-9]+')" "2" -check "premise: ANALYZE printed a rows= line per launched worker" \ +check "ANALYZE printed a rows= line per launched worker" \ "$n_lines" "2" # THE DEFECT the plan-shape arms cannot see: Gather is present and the count diff --git a/test/projection_privilege.sh b/test/projection_privilege.sh index 579903195..f369c9249 100755 --- a/test/projection_privilege.sh +++ b/test/projection_privilege.sh @@ -191,8 +191,12 @@ check "premise: t_prjexec can open a session, so a refusal below is a refusal" \ "$(as t_prjexec 'SELECT 1;' | head -1)" "1" check "a role with EXECUTE but no SELECT is refused read_projection" \ "$(count_as t_prjexec read_projection)" "refused" +# The SQLSTATE, not merely "refused", as the pytest twin asserts (#1339): this +# function has no second arm reading the code, so "refused" alone accepted any +# error at all, a login FATAL included. check "and is refused reconstruct_via_projection, which leaks non-covered columns" \ - "$(count_as t_prjexec reconstruct_via_projection)" "refused" + "$(state_as t_prjexec "SELECT count(*) FROM pgcolumnar.reconstruct_via_projection('secret','p1');")" \ + "42501" check "and THAT refusal is the C check, which raises 42501 from aclcheck_error" \ "$(state_as t_prjexec "SELECT count(*) FROM pgcolumnar.read_projection('secret','p1');")" \ "42501" @@ -241,7 +245,8 @@ check_num "the policy is in force for ordinary SQL" \ check "read_projection now refuses a policy-restricted caller (#563)" \ "$(count_as t_prjsel read_projection)" "refused" check "and so does reconstruct_via_projection" \ - "$(count_as t_prjsel reconstruct_via_projection)" "refused" + "$(state_as t_prjsel "SELECT count(*) FROM pgcolumnar.reconstruct_via_projection('secret','p1');")" \ + "0A000" # 0A000, not a phrase. PgColumnarRequireNoRowSecurity raises # ERRCODE_FEATURE_NOT_SUPPORTED, which is a different SQLSTATE CLASS from either # ACL refusal -- 0A against 42 -- so no rewording of any message can confuse them. diff --git a/test/projection_scan_io.sh b/test/projection_scan_io.sh index 9aaefc3e6..6dff914db 100755 --- a/test/projection_scan_io.sh +++ b/test/projection_scan_io.sh @@ -200,8 +200,21 @@ check "premise: the empty projection is found and its storage holds no row group GROUP BY p.proj_storage_id;")" \ "true/0" +psio_e_plan="$(explain_scan on "SELECT ik FROM psio_e WHERE ik BETWEEN 1 AND 50")" check "an empty covering projection is still priced as one page, so it is chosen" \ - "$(explain_scan on "SELECT ik FROM psio_e WHERE ik BETWEEN 1 AND 50" | grep -c 'Columnar Projection: byik')" \ + "$(printf '%s\n' "$psio_e_plan" | grep -c 'Columnar Projection: byik')" \ "1" +# THE OTHER DIRECTION (#1339). The arm above catches over-pricing only: priced +# at 0 pages instead of 1, the projection is cheaper still and stays chosen. +# With every CPU cost set to 0 by explain_scan, the projection scan's whole cost +# is its page cost, so a total of zero means it was priced at zero pages. +# Measured on PG17: 5.00 with the clamp, 0.00 with `if (pages < 1) pages = 1;` +# deleted, and the projection chosen both times. +psio_e_cost="$(printf '%s\n' "$psio_e_plan" | grep -m1 'Custom Scan' | + sed -n 's/.*cost=[0-9.]*\.\.\([0-9.]*\) .*/\1/p')" +check "and its scan is not priced at zero pages" \ + "$(awk -v c="${psio_e_cost:-none}" 'BEGIN { if (c ~ /^[0-9]+(\.[0-9]+)?$/) print (c + 0 > 0 ? "priced" : "free at " c); else print "unread: " c }')" \ + "priced" + pgc_summary diff --git a/test/projections.sh b/test/projections.sh index 5a546d47b..54313063f 100755 --- a/test/projections.sh +++ b/test/projections.sh @@ -194,6 +194,19 @@ check "projection-scan results match heap oracle" \ check "aggregate over projection scan matches oracle" \ "$(q "SELECT count(*), sum(a) FROM ps WHERE c BETWEEN 100 AND 200;")" \ "$(q "SELECT count(*), sum(a) FROM ps_h WHERE c BETWEEN 100 AND 200;")" +# THE EXECUTOR, NOT THE PLANNER (#1339). The first arm reads the name from +# EXPLAIN, which takes it before the executor decides what to open, and the two +# oracle arms pass whichever storage is read. So a scan that EXPLAINs as `pc` and +# reads the base passed all three. Executed, the sorted projection lets the +# reader skip the vectors outside 100..200 and the filter sees only part of the +# table; the base, unsorted on c, sends every row to the filter. +pj_removed="$(q "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) SELECT a, c FROM ps WHERE c BETWEEN 100 AND 200;" | + grep -m1 'Rows Removed by Filter:' | grep -oE '[0-9.]+$')" +pj_outside="$(q "SELECT count(*) FROM ps_h WHERE NOT (c BETWEEN 100 AND 200);")" +check "executing it reads the projection, so the filter sees fewer rows than the base holds" \ + "$(if pgc_is_number "$pj_removed" && pgc_is_number "$pj_outside"; then + awk -v r="$pj_removed" -v o="$pj_outside" 'BEGIN { print (r + 0 < o + 0 ? "fewer" : "all " r " of " o) }' + else echo "unread: [$pj_removed] [$pj_outside]"; fi)" "fewer" check "GUC off: no projection scan" \ "$(q "SET pgcolumnar.enable_projection_scan=off; EXPLAIN (COSTS OFF) SELECT a, c FROM ps WHERE c BETWEEN 100 AND 200;" | grep -c 'Columnar Projection')" "0" @@ -266,6 +279,12 @@ psql_run "CREATE TABLE pm (a int, c int) USING pgcolumnar;" psql_run "SELECT pgcolumnar.add_projection('pm', 'pmp', ARRAY['a','c'], ARRAY['c']);" psql_run "INSERT INTO pm SELECT g, g FROM generate_series(1,10000) g;" # batch 1 +# THE PLAN THE ARMS BELOW NAME (#1339). Four of them say "via projection" or +# "projection scan", and nothing asked how this count plans. Planned on the base, +# every one of them passes whatever the projection scan does with the snapshot. +check "the old-snapshot count is planned as a projection scan" \ + "$(q "SET pgcolumnar.enable_projection_scan = on; EXPLAIN (COSTS OFF) SELECT count(a) FROM pm WHERE c BETWEEN 1 AND 20000;" | grep -c 'Columnar Projection: pmp')" "1" + A_IN="$PGC_WORKDIR/pmA.in"; A_OUT="$PGC_WORKDIR/pmA.out" mkfifo "$A_IN"; : > "$A_OUT" env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ @@ -273,7 +292,10 @@ env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \ A_PID=$! exec 9> "$A_IN" pa_send() { printf '%s\n' "$*" >&9; } -pa_wait() { local t="$1" i; for i in $(seq 1 200); do grep -q "$t" "$A_OUT" && return 0; sleep 0.1; done; return 1; } +# The token must carry a count. An ERROR echoes the statement, token included +# ("LINE 1: SELECT 'PMA_' || ..."), so matching the bare token read a failed +# query as an answer and the timeout arms below could not fire on it (#1339). +pa_wait() { local t="$1" i; for i in $(seq 1 200); do grep -qE "${t}[0-9]+" "$A_OUT" && return 0; sleep 0.1; done; return 1; } # A opens a snapshot that sees batch 1, using a projection scan. pa_send "SET pgcolumnar.enable_projection_scan = on;" diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index a7fb24ab9..e48f1983a 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -341,6 +341,7 @@ behaviour, the source of that number is named. - [293. test_native_fetch_sort_context.py: a merge join over sorted, index-fetched columnar rows decodes outside the sort's context](#293-test_native_fetch_sort_contextpy-a-merge-join-over-sorted-index-fetched-columnar-rows-decodes-outside-the-sorts-context) - [294. test_native_fold_skipguard.py: the batch fold is reachable for a shape that skips vectors, and accounts for the skipping](#294-test_native_fold_skipguardpy-the-batch-fold-is-reachable-for-a-shape-that-skips-vectors-and-accounts-for-the-skipping) - [295. test_native_fetch_group_memo.py: the by-row-number fetch resolves its row group from a memo, and the memo never serves a group it should not](#295-test_native_fetch_group_memopy-the-by-row-number-fetch-resolves-its-row-group-from-a-memo-and-the-memo-never-serves-a-group-it-should-not) +- [296. test_parallel_copy_dedup.py: pgcolumnar.parallel_copy can refuse a load it has already done](#296-test_parallel_copy_deduppy-pgcolumnarparallel_copy-can-refuse-a-load-it-has-already-done) ## 1. How to read a test in here @@ -6946,11 +6947,11 @@ Removal proofs, each red in both twins: the prototype's zone handling; `CREATE T Port of `approx_aggregates.sh` (#1296 phase 2). Phase 0 on #1296 decided that approximate distinct counts and percentiles come from the `hll` and `tdigest` extensions for 1.0, so this pair is the tested compatibility statement for them; docs/how-to.md names the versions. -Independent of the shell suite: its own generator (other multipliers, the same 200,000 rows and 50,000 devices, because the check names state them), its own SQL. `compare_to_bash` grades the pair 14 literal matches, 0 missing. +Independent of the shell suite: its own generator (other multipliers, the same 200,000 rows and 50,000 devices, because the check names state them), its own SQL. `compare_to_bash` grades the pair 15 literal matches, 0 missing. THE HEAP MIRROR IS WRITTEN FROM THE GENERATOR, never copied from the columnar table. The shell suite first built it by `CREATE TABLE ... AS SELECT * FROM ax`, which reads through the scan under test: with a mutation dropping one row in a thousand, every equality stayed green on all five majors. Writing both sides from the generator turned seven checks red under the same mutation. -The bounds against the exact answer are premises, as pg_partman's are: they state the extension's own error, and a change here that broke them would break the equalities first. +The bounds against the exact answer are premises, as pg_partman's are: they state the extension's own error, and a change here that broke them would break the equalities first. That holds for the stored tdigest only since #1339 added "a tdigest stored in a columnar table comes back byte for byte": before it, nothing compared the stored digests, and a reader returning each long value as the previous one passed every tdigest check. The digests are built once into a heap table and stored from there, because a tdigest depends on the order its inputs arrive in. hll 2.19 does not build against PostgreSQL 19, so `test_hll_over_columnar` is `UNSUPPORTED_MAJOR` there and listed in `expected_unrunnable.txt`. CI installs both extensions from PGDG for 15 to 18. @@ -11352,3 +11353,29 @@ Every test makes one check under the shell's name, on a private database built o | `test_fgm_delete_visible_to_the_next_fetch` | a DELETE is visible to the very next fetch (delete vectors uncached) | | `test_fgm_same_transaction_truncate_then_fetch` | same-transaction TRUNCATE then fetch sees only the new rows | | `test_fgm_backend_alive` | backend alive | + +## 296. test_parallel_copy_dedup.py: pgcolumnar.parallel_copy can refuse a load it has already done + +The pytest twin of `test/parallel_copy_dedup.sh` (ported for #1341), every check by the shell's name. With `dedup => true`, `pgcolumnar.parallel_copy` refuses a second load of a file whose bytes it already loaded into the same table (#403 item 7): it stores nothing, returns 0 and says so in a NOTICE. Without dedup the same load doubles the table; a file changed only in its last row, past the coordinator's first 256 KiB read, loads; the record is per table; and the table's records are exactly the server's `sha256()` of the two files it loaded. On a private database, with the files in `server_dir`. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_pcdedup_the_changed_row_lies_past_the_first_read` | premise: the changed row lies past the first 256 KiB of the file | +| `test_pcdedup_without_dedup_a_repeated_load_doubles_the_table` | without dedup a repeated load doubles the table, which is the defect | +| `test_pcdedup_each_plain_load_reported_its_rows` | and each of those loads reported its rows | +| `test_pcdedup_the_first_load_stores_its_rows` | the first load stores its rows normally (#403 item 7) | +| `test_pcdedup_the_first_load_reports_them` | and reports them | +| `test_pcdedup_the_second_load_of_the_same_file_stores_nothing` | the second load of the same file stores nothing (#403 item 7) | +| `test_pcdedup_the_second_load_reports_0` | and reports 0 rather than claiming it loaded them | +| `test_pcdedup_the_skip_says_so_in_a_notice` | and says in a NOTICE that it skipped the load, rather than returning a silent 0 | +| `test_pcdedup_a_changed_file_at_the_same_path_is_loaded` | a changed file at the same path is loaded, not skipped (#403 item 7) | +| `test_pcdedup_the_changed_file_reports_its_rows` | and reports the rows it loaded | +| `test_pcdedup_the_same_file_loads_into_a_different_table` | the same file loads into a DIFFERENT table (the record is per table) | +| `test_pcdedup_the_other_table_load_reports_its_rows` | and reports its rows | +| `test_pcdedup_dedup_off_loads_again` | asking for no dedup loads again even though the file is on record | +| `test_pcdedup_dedup_off_reports_those_rows` | and reports those rows too | +| `test_pcdedup_the_skipped_load_left_no_record` | the skipped load left exactly one record for that table and file | +| `test_pcdedup_the_records_hold_each_files_sha256` | the records hold the SHA-256 of each loaded file's bytes | +| `test_pcdedup_no_prepared_transaction_leaked` | no prepared transaction leaked through any of it | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 44008c150..2e0892cde 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -1094,4 +1094,6 @@ guard_tests 804 # cluster_tests 2670 -> 2677 for #1637's test_catalog_filter_reads.py, the twin of catalog_filter_reads.sh: # 7 tests, one per check, by collection over the cluster files. # cluster_tests 2748 -> 2753 for #1706's five timestamp-flag tests in test_native_parquet_schema.py, by collection. -cluster_tests 3222 +# cluster_tests 3222 -> 3239 for #1341's test_parallel_copy_dedup.py, the twin of parallel_copy_dedup.sh: +# 17 tests, one per check, by collection over the cluster files. +cluster_tests 3239 diff --git a/test/pytest/test_approx_aggregates.py b/test/pytest/test_approx_aggregates.py index 29e01f79d..c7c73e191 100644 --- a/test/pytest/test_approx_aggregates.py +++ b/test/pytest/test_approx_aggregates.py @@ -74,8 +74,14 @@ def test_tdigest_over_columnar(pgc_conn, expect): exact = _one(conn, "SELECT percentile_cont(0.99) WITHIN GROUP (ORDER BY v) FROM ax_heap") expect.text(repr(col), repr(heap), "tdigest over a columnar table gives exactly what it gives over heap") expect.text(_within(col, exact, 0.001), "ok", "premise: and its p99 is within 0.1% of the exact percentile") + # Built once into heap and stored from there, as the shell does: a tdigest depends on + # the order its inputs arrive in, so only a copy isolates the round trip (#1339). + conn.execute("CREATE TABLE ax_td_src USING heap AS SELECT day, tdigest(v, 100) AS t FROM ax GROUP BY day") conn.execute("CREATE TABLE ax_td (day int, t tdigest) USING pgcolumnar") - conn.execute("INSERT INTO ax_td SELECT day, tdigest(v, 100) FROM ax GROUP BY day") + conn.execute("INSERT INTO ax_td SELECT day, t FROM ax_td_src") + expect.text(_one(conn, "SELECT ((SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td) " + "= (SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td_src))::text"), "true", + "a tdigest stored in a columnar table comes back byte for byte") expect.text(_within(_one(conn, "SELECT tdigest_percentile(t, 0.99) FROM ax_td"), exact, 0.001), "ok", "premise: a tdigest stored in a columnar table still answers: " "the union of daily digests is within 0.1% at p99") diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index a6f0954c5..48cb7ead3 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -92,6 +92,7 @@ "smoke", "sort_status", "sort_status_privilege", + "parallel_copy_dedup", "parquet_export_stats", "parallel_flush_optin", "parallel_degree", diff --git a/test/pytest/test_parallel_copy_dedup.py b/test/pytest/test_parallel_copy_dedup.py new file mode 100644 index 000000000..48c47eb54 --- /dev/null +++ b/test/pytest/test_parallel_copy_dedup.py @@ -0,0 +1,212 @@ +"""pgcolumnar.parallel_copy can refuse a load it has already done (#403 item 7). The +pytest twin of `test/parallel_copy_dedup.sh`, every check by the shell's name (#1341). + +Asserted: without dedup a repeated load doubles the table; with dedup the second load +of the same file stores nothing, returns 0 and says so in a NOTICE; a file changed at +the same path loads; the record is per table; dedup off loads again; the table's +records are exactly the SHA-256 digests of the two files it loaded; and no prepared +transaction is left behind. + +THREE FIXES THE SHELL SHARES. The changed file now differs from the first ONLY in its +last row, and that row lies past the coordinator's first 256 KiB read, so a fingerprint +of the first read alone refuses it; the records are compared with the server's own +sha256() of each file. The skip's NOTICE is read, not only the 0. And the two control +checks lose their "premise:" prefix under #1339: dedup is pgcolumnar's own default, and +forcing it on reddens them. + +ON A PRIVATE DATABASE, built in the shell's order by a module fixture. The files live +in `server_dir`, because the backend writes and reads them. +""" +import shutil + +import psycopg +import pytest + +from pgc_vacuity import QUERY_ERROR, query_error + +DB = "pgc_parallel_copy_dedup" +ROWS = 25000 +FIRST_READ = 262144 # PCOPY_FINGERPRINT_BUFSZ, the coordinator's read size +SKIP_NOTICE = 'pgcolumnar.parallel_copy skipped a load already applied to "pcd"' + + +def _failed(v): + return str(v).startswith(QUERY_ERROR) + + +def _err(exc): + return query_error(str(exc).splitlines()[0] if str(exc) else type(exc).__name__) + + +def _int(v): + return int(v) if isinstance(v, str) and v.lstrip("-").isdigit() else v + + +class _Db: + def __init__(self, dsn): + self.dsn = dsn + + def run(self, sql): + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + conn.execute(sql) + except psycopg.Error as exc: + print(f"-- step failed: {str(exc).splitlines()[0]}") + + def q(self, sql): + """-> the first column as text, "" for NULL, or a query_error().""" + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + row = conn.execute(sql).fetchone() + except psycopg.Error as exc: + return _err(exc) + return "" if row is None or row[0] is None else str(row[0]) + + def copy_noticed(self, sql): + """-> (the first column as text, the number of skip NOTICEs), or two query_error()s.""" + notes = [] + try: + with psycopg.connect(self.dsn, autocommit=True) as conn: + conn.add_notice_handler(lambda d: notes.append((d.severity_nonlocalized or d.severity, + d.message_primary or ""))) + row = conn.execute(sql).fetchone() + except psycopg.Error as exc: + return _err(exc), _err(exc) + got = "" if row is None or row[0] is None else str(row[0]) + return got, str(sum(1 for sev, msg in notes if sev == "NOTICE" and msg == SKIP_NOTICE)) + + def mk(self, table): + self.run(f"DROP TABLE IF EXISTS {table}; CREATE TABLE {table} (id int, txt text) USING pgcolumnar") + + +@pytest.fixture(scope="module") +def pcd(pgc_cluster, server_dir): + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {DB} WITH (FORCE)") + a.execute(f"CREATE DATABASE {DB}") + try: + d = _Db(pgc_cluster.dsn(dbname=DB)) + d.run("CREATE EXTENSION IF NOT EXISTS pgcolumnar") + f = server_dir / "pcd_load.txt" + g = server_dir / "pcd_other.txt" + d.run(f"COPY (SELECT g AS id, 'v'||g AS txt FROM generate_series(1,{ROWS}) g) " + f"TO '{f}' WITH (FORMAT text)") + # differs from f ONLY in its last row, past the first read + d.run(f"COPY (SELECT g AS id, CASE WHEN g = {ROWS} THEN 'changed' ELSE 'v'||g END AS txt " + f"FROM generate_series(1,{ROWS}) g) TO '{g}' WITH (FORMAT text)") + sha = "SELECT encode(sha256(pg_read_binary_file('{p}')), 'hex')" + r = {"sha f": d.q(sha.format(p=f)), "sha g": d.q(sha.format(p=g))} + r["past first read"] = d.q(f"SELECT ((SELECT size FROM pg_stat_file('{g}')) " + f"- octet_length('{ROWS}'||E'\\tchanged\\n') >= {FIRST_READ})::int") + + load = "SELECT pgcolumnar.parallel_copy('{t}', '{p}', 4{dd})" + d.mk("pcd_plain") + r["r1"] = d.q(load.format(t="pcd_plain", p=f, dd="")) + r["r2"] = d.q(load.format(t="pcd_plain", p=f, dd="")) + r["plain count"] = d.q("SELECT count(*) FROM pcd_plain") + + d.mk("pcd") + r["d1"] = d.q(load.format(t="pcd", p=f, dd=", true")) + r["c1"] = d.q("SELECT count(*) FROM pcd") + r["d2"], r["skip notices"] = d.copy_noticed(load.format(t="pcd", p=f, dd=", true")) + r["c2"] = d.q("SELECT count(*) FROM pcd") + + shutil.copyfile(g, f) + r["d3"] = d.q(load.format(t="pcd", p=f, dd=", true")) + r["c3"] = d.q("SELECT count(*) FROM pcd") + + d.mk("pcd_other") + r["d4"] = d.q(load.format(t="pcd_other", p=f, dd=", true")) + r["other count 1"] = d.q("SELECT count(*) FROM pcd_other") + r["d5"] = d.q(load.format(t="pcd_other", p=f, dd=", false")) + r["other count 2"] = d.q("SELECT count(*) FROM pcd_other") + + r["records"] = d.q("SELECT count(*) FROM pgcolumnar.load_fingerprint " + "WHERE relation_oid = 'pcd'::regclass") + r["digests"] = d.q("SELECT string_agg(encode(fingerprint, 'hex'), ',' ORDER BY loaded_at) " + "FROM pgcolumnar.load_fingerprint WHERE relation_oid = 'pcd'::regclass") + # this database's own: other suites in the session prepare transactions too + r["prepared"] = d.q("SELECT count(*) FROM pg_prepared_xacts WHERE database = current_database()") + print(f"-- dedup on: load 1 returned {r['d1']} (table {r['c1']}), " + f"load 2 returned {r['d2']} (table {r['c2']}); changed file: {r['d3']} (table {r['c3']})") + yield r + finally: + with psycopg.connect(pgc_cluster.dsn(), autocommit=True) as a: + a.execute(f"DROP DATABASE IF EXISTS {DB} WITH (FORCE)") + + +def test_pcdedup_the_changed_row_lies_past_the_first_read(pcd, expect): + expect.num(_int(pcd["past first read"]), 1, "premise: the changed row lies past the first 256 KiB of the file") + + +# Not premises (#1339): dedup is pgcolumnar's own default, and forcing it on reddens both. +def test_pcdedup_without_dedup_a_repeated_load_doubles_the_table(pcd, expect): + expect.num(_int(pcd["plain count"]), ROWS * 2, + "without dedup a repeated load doubles the table, which is the defect") + + +def test_pcdedup_each_plain_load_reported_its_rows(pcd, expect): + r1, r2 = pcd["r1"], pcd["r2"] + got = r1 if _failed(r1) else r2 if _failed(r2) else f"{r1}/{r2}" + expect.text(got, f"{ROWS}/{ROWS}", "and each of those loads reported its rows") + + +def test_pcdedup_the_first_load_stores_its_rows(pcd, expect): + expect.num(_int(pcd["c1"]), ROWS, "the first load stores its rows normally (#403 item 7)") + + +def test_pcdedup_the_first_load_reports_them(pcd, expect): + expect.num(_int(pcd["d1"]), ROWS, "and reports them") + + +def test_pcdedup_the_second_load_of_the_same_file_stores_nothing(pcd, expect): + expect.num(_int(pcd["c2"]), ROWS, "the second load of the same file stores nothing (#403 item 7)") + + +def test_pcdedup_the_second_load_reports_0(pcd, expect): + expect.num(_int(pcd["d2"]), 0, "and reports 0 rather than claiming it loaded them") + + +def test_pcdedup_the_skip_says_so_in_a_notice(pcd, expect): + expect.num(_int(pcd["skip notices"]), 1, + "and says in a NOTICE that it skipped the load, rather than returning a silent 0") + + +def test_pcdedup_a_changed_file_at_the_same_path_is_loaded(pcd, expect): + expect.num(_int(pcd["c3"]), ROWS * 2, "a changed file at the same path is loaded, not skipped (#403 item 7)") + + +def test_pcdedup_the_changed_file_reports_its_rows(pcd, expect): + expect.num(_int(pcd["d3"]), ROWS, "and reports the rows it loaded") + + +def test_pcdedup_the_same_file_loads_into_a_different_table(pcd, expect): + expect.num(_int(pcd["other count 1"]), ROWS, + "the same file loads into a DIFFERENT table (the record is per table)") + + +def test_pcdedup_the_other_table_load_reports_its_rows(pcd, expect): + expect.num(_int(pcd["d4"]), ROWS, "and reports its rows") + + +def test_pcdedup_dedup_off_loads_again(pcd, expect): + expect.num(_int(pcd["other count 2"]), ROWS * 2, + "asking for no dedup loads again even though the file is on record") + + +def test_pcdedup_dedup_off_reports_those_rows(pcd, expect): + expect.num(_int(pcd["d5"]), ROWS, "and reports those rows too") + + +def test_pcdedup_the_skipped_load_left_no_record(pcd, expect): + expect.num(_int(pcd["records"]), 2, "the skipped load left exactly one record for that table and file") + + +def test_pcdedup_the_records_hold_each_files_sha256(pcd, expect): + sf, sg = pcd["sha f"], pcd["sha g"] + want = sf if _failed(sf) else sg if _failed(sg) else f"{sf},{sg}" + expect.text(pcd["digests"], want, "the records hold the SHA-256 of each loaded file's bytes") + + +def test_pcdedup_no_prepared_transaction_leaked(pcd, expect): + expect.num(_int(pcd["prepared"]), 0, "no prepared transaction leaked through any of it") diff --git a/test/pytest/test_projection_drop_column.py b/test/pytest/test_projection_drop_column.py index 9768cb38e..26e25302d 100644 --- a/test/pytest/test_projection_drop_column.py +++ b/test/pytest/test_projection_drop_column.py @@ -31,11 +31,14 @@ premise below fails first and names reachability instead of leaving `42501 != 42P01` to be interpreted. -ONE THING THIS PORT ASSERTS THAT THE BASH SUITE DOES NOT. The two non-owner arms exist -to show that a stranger cannot tell a projected column from an unprojected one. The bash -suite asserts each against the literal `42501` and leaves the reader to notice they are -the same string. Here they are also compared to each other, so the property -- these two -are INDISTINGUISHABLE -- is asserted rather than implied by two separate constants. +THE TWO NON-OWNER REFUSALS ARE COMPARED TO EACH OTHER, in both suites (#1339). The arms +exist to show that a stranger cannot tell a projected column from an unprojected one, so +the property -- these two are INDISTINGUISHABLE -- is asserted directly rather than +implied by two separate constants, and it is asserted before them, so it can fail. + +**The lock is tested by contention.** The ownership check runs before the lock and again +after it, and both raise 42501, so a SQLSTATE alone cannot tell which one refused. While +the owner holds SHARE on the table, only the unlocked check refuses a stranger at once. """ import psycopg @@ -110,15 +113,43 @@ def test_drop_column_is_refused_while_a_projection_depends_on_it(pgc_conn, expec proj_err = _as_nobody(cur, "ALTER TABLE pdc DROP COLUMN sort_key") plain_err = _as_nobody(cur, "ALTER TABLE pdc DROP COLUMN payload") + # The property is that the two are INDISTINGUISHABLE, not that each happens to + # equal a constant, so it is compared FIRST. Placed after the two constant arms + # it could never run red: `expect` stops at the first failure, and if both + # constants hold the two are equal. With both ownership checks deleted, the + # stranger gets 2BP01 for the projected column and 42501 for the other, and + # this is the check that names it (#1339). + expect.text(proj_err.sqlstate if proj_err else "no error", + plain_err.sqlstate if plain_err else "no error", + "and the two refusals are the same error, so neither leaks") expect.sqlstate(proj_err, "42501", "a non-owner learns nothing from a projected column") expect.sqlstate(plain_err, "42501", "the same non-owner error is returned for an unprojected column") - # EXTRA: the property is that the two are INDISTINGUISHABLE, not that each - # happens to equal a constant. - expect.text(proj_err.sqlstate if proj_err else "no error", - plain_err.sqlstate if plain_err else "no error", - "premise: and the two refusals are the same error, so neither leaks") + + # THE LOCK HALF OF THE PROPERTY (#1339). The ownership check runs twice: once + # with no lock, then again after ShareUpdateExclusiveLock. The arms above + # cannot tell them apart, because both raise 42501. Contention can: while the + # owner holds SHARE on pdc, which conflicts with ShareUpdateExclusiveLock, a + # stranger refused by the unlocked check gets 42501 at once, and one that + # reaches the lock first waits out its lock_timeout (55P03). + holder = psycopg.connect(pgc_conn.info.dsn, autocommit=False) + try: + holder.execute(f'SET search_path TO "{schema}"') + holder.execute("LOCK TABLE pdc IN SHARE MODE") + held = _one(cur, "SELECT count(*) FROM pg_locks WHERE relation = 'pdc'::regclass " + "AND mode = 'ShareLock' AND granted") + expect.num(held, 1, "premise: the owner holds SHARE on pdc before the stranger asks") + cur.execute("SET lock_timeout = '2s'") + try: + waited_err = _as_nobody(cur, "ALTER TABLE pdc DROP COLUMN sort_key") + finally: + cur.execute("RESET lock_timeout") + expect.sqlstate(waited_err, "42501", + "a non-owner is refused before it waits for the table lock") + finally: + holder.rollback() + holder.close() # --- the owner is refused as a dependency --------------------------- # diff --git a/test/pytest/test_projection_parallel.py b/test/pytest/test_projection_parallel.py index c1b673552..56524235d 100644 --- a/test/pytest/test_projection_parallel.py +++ b/test/pytest/test_projection_parallel.py @@ -232,15 +232,18 @@ def test_projection_parallel(pgc_conn, expect): want, "a parallel covering projection returns the covering rows once", ) + # NOT PREMISES (#1339, #1236). The worker count is the projection path's own + # parallel_workers: with it set to 1, Workers Launched reads 1 and these go red. + # A check a pgcolumnar change can redden is an arm. expect.num( launched, 2, - "premise: EXPLAIN ANALYZE launched two workers", + "EXPLAIN ANALYZE launched two workers", ) expect.num( len(worker_rows), 2, - "premise: ANALYZE printed a rows= line per launched worker", + "ANALYZE printed a rows= line per launched worker", ) expect.num( n_busy, diff --git a/test/pytest/test_projection_privilege.py b/test/pytest/test_projection_privilege.py index 40e8b986c..eb84a8682 100644 --- a/test/pytest/test_projection_privilege.py +++ b/test/pytest/test_projection_privilege.py @@ -254,11 +254,18 @@ def test_a_role_with_execute_but_no_select_is_refused(pgc_cluster, pgc_conn, exp schema = _fixture(cur) _, err = _as(pgc_cluster, EXEC, f"SELECT count(*) FROM pgcolumnar.{func}('prj_secret','p1')", schema) - expect.sqlstate(err, "42501", name) if func == "read_projection": + # TWO PROPERTIES, TWO ARMS, as in `projection_privilege.sh` (#1339). The first + # asks only that the call was refused; the second asks which code refused it. + # Both used to assert 42501 on the same error, so the second could never fail + # alone: with read_projection's aclcheck_error replaced by a plain elog + # (XX000), the first arm went red under the second's name. + expect.text("refused" if err is not None else "noerror", "refused", name) expect.sqlstate( err, "42501", "and THAT refusal is the C check, which raises 42501 from aclcheck_error") + else: + expect.sqlstate(err, "42501", name) def test_which_layer_refused_without_reading_the_message(pgc_cluster, pgc_conn, expect): @@ -372,12 +379,16 @@ def test_a_policy_restricted_caller_is_refused(pgc_cluster, pgc_conn, expect, fu _, err = _as(pgc_cluster, SEL, f"SELECT count(*) FROM pgcolumnar.{func}('prj_secret','p1')", schema) - expect.sqlstate(err, "0A000", name) if func == "read_projection": + # Refused, then WHICH refusal, as in the shell suite (#1339): asserting 0A000 + # twice on one error left the second arm unable to fail on its own. + expect.text("refused" if err is not None else "noerror", "refused", name) expect.sqlstate( err, "0A000", "and the refusal says the feature is not supported, not that a privilege " "is missing") + else: + expect.sqlstate(err, "0A000", name) def test_the_acl_is_checked_before_rls_so_no_privilege_discloses_no_rls_state( diff --git a/test/pytest/test_projection_scan_io.py b/test/pytest/test_projection_scan_io.py index 13a2a20aa..5d58cbc18 100644 --- a/test/pytest/test_projection_scan_io.py +++ b/test/pytest/test_projection_scan_io.py @@ -263,3 +263,14 @@ def test_an_empty_covering_projection_is_priced_as_one_page(pgc_conn, expect): "onck", "an empty covering projection is still priced as one page, so it is chosen", ) + # THE OTHER DIRECTION (#1339). The arm above catches over-pricing only: priced at + # 0 pages instead of 1, the projection is cheaper still and stays chosen. `_plan` + # sets every CPU cost to 0, so the scan's whole cost is its page cost, and a total + # of zero means it was priced at zero pages. Measured on PG17: 5.0 with the clamp, + # 0.0 with `if (pages < 1) pages = 1;` deleted, and the projection chosen both times. + cost = (node or {}).get("Total Cost") + expect.text( + "unread" if cost is None else ("priced" if cost > 0 else f"free at {cost}"), + "priced", + "and its scan is not priced at zero pages", + ) diff --git a/test/pytest/test_projections.py b/test/pytest/test_projections.py index 29894df6b..3d5ffde13 100644 --- a/test/pytest/test_projections.py +++ b/test/pytest/test_projections.py @@ -136,6 +136,20 @@ def walk(node): return found[0] if found else None +def _plan_field(plan, key): + """-> the first value of `key` anywhere in a JSON plan, or None.""" + if isinstance(plan, dict): + if key in plan: + return plan[key] + plan = list(plan.values()) + if isinstance(plan, list): + for v in plan: + found = _plan_field(v, key) + if found is not None: + return found + return None + + def _read_projection(cur, table, name): """The projection's rows as the server renders them: columns joined by '|'.""" return _rows(cur, "SELECT pgcolumnar.read_projection(%s, %s)", (table, name)) @@ -492,6 +506,23 @@ def test_a_covering_sort_key_query_reads_the_projection(planner, expect): _rows(cur, "SELECT count(*), sum(a) FROM ps_h " "WHERE c BETWEEN 100 AND 200"), "aggregate over projection scan matches oracle") + # THE EXECUTOR, NOT THE PLANNER (#1339). The first arm reads the name from + # EXPLAIN, which takes it before the executor decides what to open, and the + # two oracle arms pass whichever storage is read. So a scan that EXPLAINs as + # `pc` and reads the base passed all three. Executed, the sorted projection + # lets the reader skip the vectors outside 100..200 and the filter sees only + # part of the table; the base, unsorted on c, sends every row to the filter. + # Measured on PG17: 5980 removed against 17980 for the base. + analyzed = _one(cur, "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF, " + "FORMAT JSON) " + COVERING) + removed = _plan_field(analyzed, "Rows Removed by Filter") + outside = _one(cur, "SELECT count(*) FROM ps_h WHERE NOT (c BETWEEN 100 AND 200)") + expect.text( + "unread" if removed is None + else ("fewer" if removed < outside else f"all {removed} of {outside}"), + "fewer", + "executing it reads the projection, so the filter sees fewer rows than " + "the base holds") def test_the_guc_is_an_off_switch(planner, expect): @@ -620,6 +651,32 @@ def test_an_old_snapshot_never_sees_rows_committed_after_it(pgc_conn, expect): dsn = pgc_conn.info.dsn COUNT = "SELECT count(a) FROM pm WHERE c BETWEEN 1 AND 20000" + + # THE PLAN THE ARMS BELOW NAME (#1339). Four of them say "via projection" or + # "projection scan", and nothing asked how COUNT plans. Planned on the base, every + # one of them passes whatever the projection scan does with the snapshot. + with pgc_conn.cursor() as cur: + cur.execute("SET pgcolumnar.enable_projection_scan = on") + expect.text(_projection_in_plan(_plan(cur, COUNT)) or "none", "pmp", + "the old-snapshot count is planned as a projection scan") + + def answer(ac): + """-> session A's count, or the error that stopped it. + + The shell suite's two timeout arms fail when session A gives no count. Here a + statement_timeout stands in for its 20-second poll, and an error is caught + rather than raised, so those two arms read what they say: that A answered. + """ + try: + return _one(ac, COUNT) + except psycopg.Error as exc: + ac.connection.rollback() + return exc + + def answered(reading): + return ("answered" if not isinstance(reading, psycopg.Error) + else f"no answer: {reading.sqlstate}") + a = psycopg.connect(dsn, autocommit=False) try: with a.cursor() as ac: @@ -627,10 +684,11 @@ def test_an_old_snapshot_never_sees_rows_committed_after_it(pgc_conn, expect): # the same one or it would read a different (absent) table. ac.execute(f'SET search_path TO "{schema}", public') ac.execute("SET pgcolumnar.enable_projection_scan = on") + ac.execute("SET statement_timeout = '20s'") ac.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ") - base = _one(ac, COUNT) - expect.num(base, 10000, "session A opened snapshot") + base = answer(ac) + expect.text(answered(base), "answered", "session A opened snapshot") expect.num(base, 10000, "session A baseline via projection sees batch 1") @@ -638,8 +696,8 @@ def test_an_old_snapshot_never_sees_rows_committed_after_it(pgc_conn, expect): cur.execute("INSERT INTO pm SELECT g, g " "FROM generate_series(10001,20000) g") - after = _one(ac, COUNT) - expect.num(after, 10000, "session A responded post-commit") + after = answer(ac) + expect.text(answered(after), "answered", "session A responded post-commit") expect.num(after, 10000, "old snapshot projection scan does not see post-snapshot rows") a.commit() diff --git a/test/pytest/unported_suites.txt b/test/pytest/unported_suites.txt index 256c4fe3b..c0c304b05 100644 --- a/test/pytest/unported_suites.txt +++ b/test/pytest/unported_suites.txt @@ -46,7 +46,6 @@ objstore_stash_recovery objstore_tls_read parallel parallel_copy -parallel_copy_dedup parallel_export_parquet parallel_vector_agg parquet_export