test: parallel_export_parquet gains its pytest twin, and an export that loses a row no longer passes (#1341)

test/approx_aggregates.sh

@@ -16,8 +16,10 @@
 #     its default precision (log2m 11) has a standard error of 1.04/sqrt(2048), so
 #     three of them, 6.9%; tdigest at compression 100 within 0.1% at p99, about
 #     four times what phase 0 measured (0.025%).
-#   - each type stored in a columnar table comes back usable: an hll byte for byte,
-#     a tdigest as a digest whose union still answers within the bound.
+#   - each type stored in a columnar table comes back byte for byte, and its union
+#     still answers. For tdigest that union's bound is a premise only because the
+#     byte-for-byte arm comes first: a storage change that altered a digest reddens
+#     the arm before it can move the percentile.
 #   - the aggregate reads through the columnar scan, so the arms exercise our reader.
 #     That is an arm, not a premise: turning the custom scan off by default reddens it
 #     (#1236: a check a repository change can redden is not a premise).
@@ -69,8 +71,16 @@ TD_H="$(qv "$P0 SELECT tdigest_percentile(v, 100, 0.99) FROM ax_heap;")"
 EXACT99="$(q "SELECT percentile_cont(0.99) WITHIN GROUP (ORDER BY v) FROM ax_heap;")"
 check "tdigest over a columnar table gives exactly what it gives over heap" "$TD_C" "$TD_H"
 check "premise: and its p99 is within 0.1% of the exact percentile" "$(within "$TD_C" "$EXACT99" 0.001)" "ok"
-psql_run "CREATE TABLE ax_td (day int, t tdigest) USING pgcolumnar;
-	INSERT INTO ax_td SELECT day, tdigest(v, 100) FROM ax GROUP BY day;"
+# The digests are built ONCE, into a heap table, and that table's values are stored.
+# A tdigest depends on the order its inputs arrive in, so digests rebuilt from heap
+# could differ from these for reasons that are not storage. Copied from one source,
+# any difference is the columnar round trip's (#1339).
+psql_run "CREATE TABLE ax_td_src USING heap AS SELECT day, tdigest(v, 100) AS t FROM ax GROUP BY day;
+	CREATE TABLE ax_td (day int, t tdigest) USING pgcolumnar;
+	INSERT INTO ax_td SELECT day, t FROM ax_td_src;"
+check "a tdigest stored in a columnar table comes back byte for byte" \
+	"$(q "SELECT (SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td)
+	        = (SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td_src);")" "t"
 check "premise: a tdigest stored in a columnar table still answers: the union of daily digests is within 0.1% at p99" \
 	"$(within "$(q "SELECT tdigest_percentile(t, 0.99) FROM ax_td;")" "$EXACT99" 0.001)" "ok"
 

test/check_ledger_budget.txt

@@ -15,7 +15,7 @@
 #
 #   Adding a check to a suite that is already covered does not move it, which is
 #   what makes it safe to bound.
-suites_not_covered 75
+suites_not_covered 69
 #
 # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become
 #   one.
@@ -5957,4 +5957,159 @@ suites_not_covered 75
 #     never         4129 + 8 = 4137
 #     dated         2791 + 7 = 2798
 #     4137 + 2798 == 6935
-checks_never_observed_red 4137
+#
+# #1339, a batch of projection checks (jdatcmd): 4137 -> 4132, and suites_not_covered
+# 75 -> 74. Seven findings, each shown green on main under its mutation in both
+# twins and red by name after the fix. projection_drop_column is seeded: it had no
+# rows, and it enters with 11 checks from five-major runs (11/11 on each of 15-19,
+# twin green on each). Its stranger arms read only 42501, so deleting the unlocked
+# ownership check passed; a stranger is now asked while the owner holds SHARE, and
+# queues to 55P03 under that mutation. Its "two refusals are the same" check lost its
+# premise: prefix, joined the shell suite, and moved ahead of the two constant arms
+# so it can fail. projection_parallel's two worker-count checks lost their premise:
+# prefix: parallel_workers = 1 reddens both. projection_privilege's read_projection
+# arms each asserted one SQLSTATE twice; the first now asks only "refused", as the
+# shell suite does, and the shell's reconstruct arms read the SQLSTATE, as the twin
+# does. projection_scan_io gains a check that an empty projection is not priced at
+# zero pages. projections gains two: an EXPLAIN ANALYZE reading that the executor
+# opened the projection, and that the old-snapshot count plans it. Its twin's two
+# session-A timeout arms now read whether A answered, not the count again, and the
+# shell's poll no longer takes an ERROR that echoes the token for an answer. 19 rows
+# are dated by ten mutations, each reddening its target by the same name in both
+# twins (the pytest twin stops at a test's first red): the unlocked ownership check
+# deleted, both deleted, parallel_workers = 1, each function's aclcheck_error
+# replaced by elog, the RLS refusal raised as 42501, the one-page floor deleted, the
+# executor never opening the projection, projection pages multiplied by 1000000,
+# and the projection scan and liveness cache both on GetLatestSnapshot(). 8 arrive
+# never: one premise and seven drop_column checks no mutation here reached.
+#
+#     ledger rows   6935 + 14 = 6949
+#     never         4137 + 8 - 13 = 4132
+#     dated         2798 + 6 + 13 = 2817
+#     4132 + 2817 == 6949
+#
+# #1339, approx_aggregates gains one arm (jdatcmd): 4132 -> 4132, unchanged. "a tdigest stored in
+# a columnar table comes back byte for byte" is dated by a reader that returns each long
+# value as the previous long value read: main's suite passed every tdigest check under it,
+# and only the hll arm went red. suites_not_covered does not move.
+#
+#     ledger rows   6949 + 1 = 6950
+#     never         4132 + 0 = 4132
+#     dated         2817 + 1 = 2818
+#     4132 + 2818 == 6950
+#
+# #1341, parallel_copy_dedup seeded, with its twin (jdatcmd): 3853 -> 3857, and
+# suites_not_covered 105 -> 104. The suite had no rows. It enters with 17 checks from
+# five-major runs (17/17 on each of 15-19, shell and twin alike), after three fixes.
+# The changed file differed from the first in its first bytes and fitted in one
+# 256 KiB read, so a fingerprint of the first read alone still loaded it and main's
+# shell passed 14/14; the changed file now differs only in its last row, past the
+# first read, and a new check compares the table's records with the server's
+# sha256() of each file. The skip's NOTICE, promised in the header, was never read:
+# with it removed main's shell passed 14/14; a new check reads it. And the two
+# control checks lost their premise: prefix under #1339: dedup is pgcolumnar's own
+# default, and forcing it on reddens them. 13 are dated by six mutations, each
+# reddening the same names in both twins: the NOTICE never raised, the first read
+# alone fingerprinted, dedup forced on, the refused load committed anyway, a
+# fingerprint matched in any table, and the refused load reporting the rows it
+# discarded. 4 arrive never: the fixture-size premise, the first load's two checks,
+# and the prepared-transaction leak.
+#
+#     ledger rows   6950 + 17 = 6967
+#     never         4132 + 4 = 4136
+#     dated         2818 + 13 = 2831
+#     4136 + 2831 == 6967
+#
+# #1341, parallel_copy seeded, with its twin (jdatcmd): 3853 -> 3885, and
+# suites_not_covered 105 -> 104. The suite had no rows. It enters with 81 checks from
+# five-major runs (81/81 on each of 15-19, shell and twin alike), after three fixes.
+# Five survival checks read "SELECT 1", which answers after crash recovery: with a
+# backend killed by SIGSEGV before each, all five passed; they now also need the
+# server log's count of killed backends unmoved. Two failure checks took any message
+# with "failed" in it, and every worker failure reads "worker N failed: ...": with
+# the loader's message dropped they passed; they now need the loader's own cause.
+# The nine reconstruct checks compared the per-range loads with a single columnar
+# COPY of the same file, which shares the COPY path's defects: with multi_insert
+# dropping the row whose id is 1, all nine passed; they now compare with the
+# generator that wrote the file. Names built from loop variables are literal. 49 are
+# dated by ten product mutations and the crash edit, each reddening the same names in
+# both twins: the loader's message dropped, a row dropped by multi_insert, the
+# max_prepared_transactions guard removed, bulk_parallel_writer booting on, the FIFO
+# opener blocking, a split offset on the newline, a split degraded to one range,
+# workers < 1 accepted, the S_ISREG refusal removed, and the key-type restriction
+# removed. 32 arrive never: the leak, empty-target and rows-returned checks of loads
+# no mutation reached, the refusals of a bad key, an unsorted file, a DEFAULT
+# partition, a heap target and a missing file, the generated-column load, the
+# partitioned FIFO refusal, the stripe-overlap check, and the one-worker split.
+#
+#     ledger rows   6967 + 81 = 7048
+#     never         4136 + 32 = 4168
+#     dated         2831 + 49 = 2880
+#     4168 + 2880 == 7048
+#
+# #1341, parallel seeded, with its twin (jdatcmd): 3853 -> 3853, and
+# suites_not_covered 105 -> 104. The suite had no rows. It enters with 13 checks from
+# five-major runs (13/13 on each of 15-19, shell and twin alike), after three fixes and
+# three new checks. "par no-filter" was a bare count(*), which the vectorized aggregate
+# answers with no Gather, so with every participant reading every row group it stayed
+# green; it now returns rows, and "par queries all plan a Gather" holds all five par
+# queries to a Gather. The table was assumed to have 25 row groups: with the writer
+# ignoring stripe_row_limit it had 1, and a claim that skips every other group passed
+# main's shell 10/10; "par table has 25 row groups to claim" now counts them. And setg
+# swallows its error, so a serial setting that did not take ran the serial checks in
+# parallel; "serial queries plan no Gather" now says so. All 13 are dated by five
+# mutations, each reddening the same names in both twins: the writer ignoring
+# stripe_row_limit, every participant reading every row group, the partial path never
+# added, the scan named ColumnarScan as before #428, and a serial scan reading every
+# other row group.
+#
+#     ledger rows   7048 + 13 = 7061
+#     never         4168 + 0 = 4168
+#     dated         2880 + 13 = 2893
+#     4168 + 2893 == 7061
+#
+# #1341, parallel_vector_agg seeded, with its twin (jdatcmd): 3853 -> 3861, and
+# suites_not_covered 105 -> 104. The suite had no rows. It enters with 29 checks from
+# five-major runs (29/29 on each of 15-19, shell and twin alike), after four fixes.
+# With the server stopped after the first table, main's shell passed 11 of 29: each
+# comparison read psql's error text, the same on both arms. q now reads psql's exit
+# status and prints nothing on failure, and every two-arm comparison is check_text.
+# The H2 sessions read the exit status and need both SELECTs: with both parallel
+# InitializeDSM callbacks raising an ERROR in a writing transaction, main's shell
+# passed 29/29. The grouped oracles are core's aggregate, not pgcolumnar's grouped
+# fold run serially: with the grouped row path skipping its WHERE recheck, main's
+# shell passed 29/29. "server still up" also needs the log's "terminated by signal"
+# count unmoved: main's passed after a worker's SIGSEGV. The partial-node check now
+# needs the Parallel Custom Scan, and nine checks lost their premise: prefix under
+# #1339. 21 are dated by nine mutations, each reddening the same names in both twins;
+# 8 arrive never: three float and all-null averages, the two empty reads, the
+# grouped float, and the Gather and Finalize row counts.
+#
+#     ledger rows   7061 + 29 = 7090
+#     never         4168 + 8 = 4176
+#     dated         2893 + 21 = 2914
+#     4176 + 2914 == 7090
+#
+# #1341, parallel_export_parquet seeded, with its twin (jdatcmd): 3878 -> 3894, and
+# suites_not_covered 103 -> 102. The suite had no rows. It enters with 54 checks from
+# five-major runs (54/54 on each of 15-19, shell and twin alike), after four fixes. The
+# source oracles read the columnar table under test, so with tuple_insert dropping one
+# row all 54 passed; they now read the generator. The two refusals grepped for "error",
+# which a dead server's psql prints, and took any error as the refusal; they now read
+# psql's exit status and match their own messages. "a 984-byte destination still
+# exports" read a dead server as "none" (accepted), and the two "not created" checks
+# read it as absent; each now needs the statement to have reached the server. "server
+# still up" took SELECT 1, which answers after crash recovery; it now also needs the
+# log's count of backends killed by a signal unmoved. Loop names are now literal. 38
+# are dated by seven mutations, each reddening the same names in both twins: an insert
+# that drops one row, a non-columnar target not refused, the cancel cleanup not
+# removing files, no _SUCCESS marker, the length guard probing only the part name, a
+# non-empty directory not refused, and every worker writing part-0000. 16 arrive never,
+# among them the premises, the empty-input pair, the per-partition file counts and the
+# survival check, which a backend killed by SIGSEGV reddens as a harness edit.
+#
+#     ledger rows   7090 + 54 = 7144
+#     never         4176 + 16 = 4192
+#     dated         2914 + 38 = 2952
+#     4192 + 2952 == 7144
+checks_never_observed_red 4192

test/parallel.sh

@@ -22,6 +22,11 @@ pgc_setup "${1:-/usr/local/pg17/bin/pg_config}"
 make_pair "id int, k int, v bigint, t text"
 q "SELECT pgcolumnar.set_options('t_col', chunk_group_row_limit => 1000, stripe_row_limit => 2000);" >/dev/null
 load_pair "SELECT g, g%100, g::bigint*2, 'r'||g FROM generate_series(1,50000) g"
+# 50000 rows at 2000 a row group is 25 groups. With one group there is nothing to
+# share: a claim that skips every other group returns the same rows as a correct one,
+# and with the writer ignoring stripe_row_limit main's shell passed 10 of 10 under
+# exactly that claim. So the count is asserted, not assumed.
+check_num "par table has 25 row groups to claim" "$(stripe_count)" "25"
 
 setg() { q "ALTER DATABASE $PGC_DB SET $1 = $2;" >/dev/null; }
 
@@ -40,18 +45,55 @@ check "parallel plan has Gather"        "$(echo "$plan" | grep -c -i 'Gather')"
 check "parallel plan has PgColumnarScan" \
 	"$(echo "$plan" | grep -c 'Custom Scan (PgColumnarScan)')" "1"
 
+# The queries, named once, so the plan checks below read the queries the
+# comparisons ran and not a copy that can drift from them.
+Q_FILTERED="SELECT id, v, t FROM %T WHERE k = 7"
+Q_RANGE="SELECT id FROM %T WHERE k < 30"
+Q_SAMPLE="SELECT * FROM %T WHERE id % 997 = 0"
+Q_COUNT="SELECT count(*), sum(v) FROM %T WHERE k < 50"
+# Row-returning, not count(*). A bare count(*) is answered by the vectorized
+# aggregate, which plans no Gather, so this check ran serially and stayed green
+# with every participant reading every row group.
+Q_NOFILTER="SELECT id FROM %T"
+
+# plans_with GATHER|NOGATHER QUERY... -> how many of the queries, on t_col, plan a
+# PgColumnarScan with (or without) a Gather. A query whose EXPLAIN fails counts
+# for neither, so a dead server cannot satisfy either reading.
+plans_with() {
+	local want="$1" n=0 tmpl plan
+	shift
+	for tmpl in "$@"; do
+		plan="$(q "EXPLAIN (COSTS off) ${tmpl//%T/t_col}")"
+		[[ "$plan" == *'Custom Scan (PgColumnarScan)'* ]] || continue
+		if [[ "$plan" == *Gather* ]]; then
+			[ "$want" = GATHER ] && n=$((n + 1))
+		else
+			[ "$want" = NOGATHER ] && n=$((n + 1))
+		fi
+	done
+	echo "$n"
+}
+
 # Results must match the heap oracle under the parallel plan.
-diff_query "par filtered scan"  "SELECT id, v, t FROM %T WHERE k = 7"
-diff_query "par range scan"     "SELECT id FROM %T WHERE k < 30"
-diff_query "par whole sample"   "SELECT * FROM %T WHERE id % 997 = 0"
-diff_query "par count"          "SELECT count(*), sum(v) FROM %T WHERE k < 50"
-diff_query "par no-filter"      "SELECT count(*) FROM %T"
+diff_query "par filtered scan"  "$Q_FILTERED"
+diff_query "par range scan"     "$Q_RANGE"
+diff_query "par whole sample"   "$Q_SAMPLE"
+diff_query "par count"          "$Q_COUNT"
+diff_query "par no-filter"      "$Q_NOFILTER"
+# A "par" check proves nothing about the parallel scan unless its query ran under
+# one. Measured on main: "par no-filter" planned no Gather.
+check_num "par queries all plan a Gather" \
+	"$(plans_with GATHER "$Q_FILTERED" "$Q_RANGE" "$Q_SAMPLE" "$Q_COUNT" "$Q_NOFILTER")" "5"
 
 # The same queries, forced serial, must give the same answers.
 setg max_parallel_workers_per_gather 0
-diff_query "serial filtered scan" "SELECT id, v, t FROM %T WHERE k = 7"
-diff_query "serial range scan"    "SELECT id FROM %T WHERE k < 30"
-diff_query "serial count"         "SELECT count(*), sum(v) FROM %T WHERE k < 50"
+diff_query "serial filtered scan" "$Q_FILTERED"
+diff_query "serial range scan"    "$Q_RANGE"
+diff_query "serial count"         "$Q_COUNT"
+# setg swallows its error, so a setting that did not take leaves these three
+# running in parallel, and they pass as "serial" without a serial scan.
+check_num "serial queries plan no Gather" \
+	"$(plans_with NOGATHER "$Q_FILTERED" "$Q_RANGE" "$Q_COUNT")" "3"
 
 q "ALTER DATABASE $PGC_DB RESET ALL;" >/dev/null