test: approx_aggregates compares a stored tdigest byte for byte (#1339)

test/approx_aggregates.sh

@@ -16,8 +16,10 @@
 #     its default precision (log2m 11) has a standard error of 1.04/sqrt(2048), so
 #     three of them, 6.9%; tdigest at compression 100 within 0.1% at p99, about
 #     four times what phase 0 measured (0.025%).
-#   - each type stored in a columnar table comes back usable: an hll byte for byte,
-#     a tdigest as a digest whose union still answers within the bound.
+#   - each type stored in a columnar table comes back byte for byte, and its union
+#     still answers. For tdigest that union's bound is a premise only because the
+#     byte-for-byte arm comes first: a storage change that altered a digest reddens
+#     the arm before it can move the percentile.
 #   - the aggregate reads through the columnar scan, so the arms exercise our reader.
 #     That is an arm, not a premise: turning the custom scan off by default reddens it
 #     (#1236: a check a repository change can redden is not a premise).
@@ -69,8 +71,16 @@ TD_H="$(qv "$P0 SELECT tdigest_percentile(v, 100, 0.99) FROM ax_heap;")"
 EXACT99="$(q "SELECT percentile_cont(0.99) WITHIN GROUP (ORDER BY v) FROM ax_heap;")"
 check "tdigest over a columnar table gives exactly what it gives over heap" "$TD_C" "$TD_H"
 check "premise: and its p99 is within 0.1% of the exact percentile" "$(within "$TD_C" "$EXACT99" 0.001)" "ok"
-psql_run "CREATE TABLE ax_td (day int, t tdigest) USING pgcolumnar;
-	INSERT INTO ax_td SELECT day, tdigest(v, 100) FROM ax GROUP BY day;"
+# The digests are built ONCE, into a heap table, and that table's values are stored.
+# A tdigest depends on the order its inputs arrive in, so digests rebuilt from heap
+# could differ from these for reasons that are not storage. Copied from one source,
+# any difference is the columnar round trip's (#1339).
+psql_run "CREATE TABLE ax_td_src USING heap AS SELECT day, tdigest(v, 100) AS t FROM ax GROUP BY day;
+	CREATE TABLE ax_td (day int, t tdigest) USING pgcolumnar;
+	INSERT INTO ax_td SELECT day, t FROM ax_td_src;"
+check "a tdigest stored in a columnar table comes back byte for byte" \
+	"$(q "SELECT (SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td)
+	        = (SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td_src);")" "t"
 check "premise: a tdigest stored in a columnar table still answers: the union of daily digests is within 0.1% at p99" \
 	"$(within "$(q "SELECT tdigest_percentile(t, 0.99) FROM ax_td;")" "$EXACT99" 0.001)" "ok"
 

test/check_ledger_budget.txt

@@ -15,7 +15,7 @@
 #
 #   Adding a check to a suite that is already covered does not move it, which is
 #   what makes it safe to bound.
-suites_not_covered 75
+suites_not_covered 74
 #
 # checks_never_observed_red -- A CENSUS. NOT a ceiling, and it must not become
 #   one.
@@ -5957,4 +5957,44 @@ suites_not_covered 75
 #     never         4129 + 8 = 4137
 #     dated         2791 + 7 = 2798
 #     4137 + 2798 == 6935
-checks_never_observed_red 4137
+#
+# #1339, a batch of projection checks (jdatcmd): 4137 -> 4132, and suites_not_covered
+# 75 -> 74. Seven findings, each shown green on main under its mutation in both
+# twins and red by name after the fix. projection_drop_column is seeded: it had no
+# rows, and it enters with 11 checks from five-major runs (11/11 on each of 15-19,
+# twin green on each). Its stranger arms read only 42501, so deleting the unlocked
+# ownership check passed; a stranger is now asked while the owner holds SHARE, and
+# queues to 55P03 under that mutation. Its "two refusals are the same" check lost its
+# premise: prefix, joined the shell suite, and moved ahead of the two constant arms
+# so it can fail. projection_parallel's two worker-count checks lost their premise:
+# prefix: parallel_workers = 1 reddens both. projection_privilege's read_projection
+# arms each asserted one SQLSTATE twice; the first now asks only "refused", as the
+# shell suite does, and the shell's reconstruct arms read the SQLSTATE, as the twin
+# does. projection_scan_io gains a check that an empty projection is not priced at
+# zero pages. projections gains two: an EXPLAIN ANALYZE reading that the executor
+# opened the projection, and that the old-snapshot count plans it. Its twin's two
+# session-A timeout arms now read whether A answered, not the count again, and the
+# shell's poll no longer takes an ERROR that echoes the token for an answer. 19 rows
+# are dated by ten mutations, each reddening its target by the same name in both
+# twins (the pytest twin stops at a test's first red): the unlocked ownership check
+# deleted, both deleted, parallel_workers = 1, each function's aclcheck_error
+# replaced by elog, the RLS refusal raised as 42501, the one-page floor deleted, the
+# executor never opening the projection, projection pages multiplied by 1000000,
+# and the projection scan and liveness cache both on GetLatestSnapshot(). 8 arrive
+# never: one premise and seven drop_column checks no mutation here reached.
+#
+#     ledger rows   6935 + 14 = 6949
+#     never         4137 + 8 - 13 = 4132
+#     dated         2798 + 6 + 13 = 2817
+#     4132 + 2817 == 6949
+#
+# #1339, approx_aggregates gains one arm (jdatcmd): 4132 -> 4132, unchanged. "a tdigest stored in
+# a columnar table comes back byte for byte" is dated by a reader that returns each long
+# value as the previous long value read: main's suite passed every tdigest check under it,
+# and only the hll arm went red. suites_not_covered does not move.
+#
+#     ledger rows   6949 + 1 = 6950
+#     never         4132 + 0 = 4132
+#     dated         2817 + 1 = 2818
+#     4132 + 2818 == 6950
+checks_never_observed_red 4132

test/projection_drop_column.sh

@@ -39,10 +39,39 @@ psql_run "INSERT INTO pdc
 # The pre-statement dependency check must not tell a stranger which column the
 # projection covers, or let that role take the lock retained by the owner path.
 psql_run "CREATE ROLE pdc_nobody LOGIN;"
-check "a non-owner learns nothing from a projected column" \
-	"$(sqlstate_as pdc_nobody 'ALTER TABLE pdc DROP COLUMN sort_key;')" "42501"
+proj_state="$(sqlstate_as pdc_nobody 'ALTER TABLE pdc DROP COLUMN sort_key;')"
+plain_state="$(sqlstate_as pdc_nobody 'ALTER TABLE pdc DROP COLUMN payload;')"
+# The property is that the two refusals are INDISTINGUISHABLE, so it is compared
+# first: with both ownership checks deleted, the stranger gets 2BP01 for the
+# projected column and 42501 for the other, and this is the check that names it.
+check "and the two refusals are the same error, so neither leaks" \
+	"$proj_state" "$plain_state"
+check "a non-owner learns nothing from a projected column" "$proj_state" "42501"
 check "the same non-owner error is returned for an unprojected column" \
-	"$(sqlstate_as pdc_nobody 'ALTER TABLE pdc DROP COLUMN payload;')" "42501"
+	"$plain_state" "42501"
+
+# THE LOCK HALF OF THE COMMENT ABOVE (#1339). The ownership check runs twice:
+# once with no lock, then again after ShareUpdateExclusiveLock. The two checks
+# above cannot tell them apart, because both raise 42501. Contention can: while
+# the owner holds SHARE on pdc, which conflicts with ShareUpdateExclusiveLock, a
+# stranger refused by the unlocked check gets 42501 at once, and one that reaches
+# the lock first waits out its lock_timeout (55P03).
+holder_log="$PGC_WORKDIR/pdc_holder.log"
+env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" \
+	-Atq -v ON_ERROR_STOP=0 -c "BEGIN;" -c "LOCK TABLE pdc IN SHARE MODE;" \
+	-c "SELECT pg_sleep(3600) /* pdc_holder */;" -c "ROLLBACK;" > "$holder_log" 2>&1 &
+holder_pid=$!
+held=0
+for _ in $(seq 1 40); do
+	held="$(q "SELECT count(*) FROM pg_locks WHERE relation = 'pdc'::regclass AND mode = 'ShareLock' AND granted;")"
+	[ "${held:-0}" -ge 1 ] && break
+	sleep 0.25
+done
+check "premise: the owner holds SHARE on pdc before the stranger asks" "${held:-0}" "1"
+check "a non-owner is refused before it waits for the table lock" \
+	"$(sqlstate_as pdc_nobody "SET lock_timeout = '2s'; ALTER TABLE pdc DROP COLUMN sort_key;")" "42501"
+psql_run "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE query LIKE '%pdc_holder%' AND pid <> pg_backend_pid();" >/dev/null
+wait "$holder_pid" 2>/dev/null || true
 
 # Red before the fix: PostgreSQL accepts this (00000), and the next INSERT
 # reaches the projection writer with a type OID of zero.

test/projection_parallel.sh

@@ -277,10 +277,13 @@ check "premise: the covering plan is parallel at the default page cost" \
 check "a page cost that makes I/O dominate costs the parallel covering path" \
 	"$(shape "$(explain_cov_spc $SPC_HI)")" "projection-only"
 
-check "premise: EXPLAIN ANALYZE launched two workers" \
+# NOT PREMISES (#1339, #1236). The worker count is the projection path's own
+# parallel_workers: with it set to 1, Workers Launched reads 1 and both of these
+# go red. A check a pgcolumnar change can redden is an arm.
+check "EXPLAIN ANALYZE launched two workers" \
 	"$(echo "$par_on_ana" | grep -oE 'Workers Launched: [0-9]+' | head -1 | grep -oE '[0-9]+')" "2"
 
-check "premise: ANALYZE printed a rows= line per launched worker" \
+check "ANALYZE printed a rows= line per launched worker" \
 	"$n_lines" "2"
 
 # THE DEFECT the plan-shape arms cannot see: Gather is present and the count

test/projection_privilege.sh

@@ -191,8 +191,12 @@ check "premise: t_prjexec can open a session, so a refusal below is a refusal" \
 	"$(as t_prjexec 'SELECT 1;' | head -1)" "1"
 check "a role with EXECUTE but no SELECT is refused read_projection" \
 	"$(count_as t_prjexec read_projection)" "refused"
+# The SQLSTATE, not merely "refused", as the pytest twin asserts (#1339): this
+# function has no second arm reading the code, so "refused" alone accepted any
+# error at all, a login FATAL included.
 check "and is refused reconstruct_via_projection, which leaks non-covered columns" \
-	"$(count_as t_prjexec reconstruct_via_projection)" "refused"
+	"$(state_as t_prjexec "SELECT count(*) FROM pgcolumnar.reconstruct_via_projection('secret','p1');")" \
+	"42501"
 check "and THAT refusal is the C check, which raises 42501 from aclcheck_error" \
 	"$(state_as t_prjexec "SELECT count(*) FROM pgcolumnar.read_projection('secret','p1');")" \
 	"42501"
@@ -241,7 +245,8 @@ check_num "the policy is in force for ordinary SQL" \
 check "read_projection now refuses a policy-restricted caller (#563)" \
 	"$(count_as t_prjsel read_projection)" "refused"
 check "and so does reconstruct_via_projection" \
-	"$(count_as t_prjsel reconstruct_via_projection)" "refused"
+	"$(state_as t_prjsel "SELECT count(*) FROM pgcolumnar.reconstruct_via_projection('secret','p1');")" \
+	"0A000"
 # 0A000, not a phrase. PgColumnarRequireNoRowSecurity raises
 # ERRCODE_FEATURE_NOT_SUPPORTED, which is a different SQLSTATE CLASS from either
 # ACL refusal -- 0A against 42 -- so no rewording of any message can confuse them.

test/projection_scan_io.sh

@@ -200,8 +200,21 @@ check "premise: the empty projection is found and its storage holds no row group
 	      GROUP BY p.proj_storage_id;")" \
 	"true/0"
 
+psio_e_plan="$(explain_scan on "SELECT ik FROM psio_e WHERE ik BETWEEN 1 AND 50")"
 check "an empty covering projection is still priced as one page, so it is chosen" \
-	"$(explain_scan on "SELECT ik FROM psio_e WHERE ik BETWEEN 1 AND 50" | grep -c 'Columnar Projection: byik')" \
+	"$(printf '%s\n' "$psio_e_plan" | grep -c 'Columnar Projection: byik')" \
 	"1"
 
+# THE OTHER DIRECTION (#1339). The arm above catches over-pricing only: priced
+# at 0 pages instead of 1, the projection is cheaper still and stays chosen.
+# With every CPU cost set to 0 by explain_scan, the projection scan's whole cost
+# is its page cost, so a total of zero means it was priced at zero pages.
+# Measured on PG17: 5.00 with the clamp, 0.00 with `if (pages < 1) pages = 1;`
+# deleted, and the projection chosen both times.
+psio_e_cost="$(printf '%s\n' "$psio_e_plan" | grep -m1 'Custom Scan' |
+	sed -n 's/.*cost=[0-9.]*\.\.\([0-9.]*\) .*/\1/p')"
+check "and its scan is not priced at zero pages" \
+	"$(awk -v c="${psio_e_cost:-none}" 'BEGIN { if (c ~ /^[0-9]+(\.[0-9]+)?$/) print (c + 0 > 0 ? "priced" : "free at " c); else print "unread: " c }')" \
+	"priced"
+
 pgc_summary

test/projections.sh

@@ -194,6 +194,19 @@ check "projection-scan results match heap oracle" \
 check "aggregate over projection scan matches oracle" \
 	"$(q "SELECT count(*), sum(a) FROM ps WHERE c BETWEEN 100 AND 200;")" \
 	"$(q "SELECT count(*), sum(a) FROM ps_h WHERE c BETWEEN 100 AND 200;")"
+# THE EXECUTOR, NOT THE PLANNER (#1339). The first arm reads the name from
+# EXPLAIN, which takes it before the executor decides what to open, and the two
+# oracle arms pass whichever storage is read. So a scan that EXPLAINs as `pc` and
+# reads the base passed all three. Executed, the sorted projection lets the
+# reader skip the vectors outside 100..200 and the filter sees only part of the
+# table; the base, unsorted on c, sends every row to the filter.
+pj_removed="$(q "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) SELECT a, c FROM ps WHERE c BETWEEN 100 AND 200;" |
+	grep -m1 'Rows Removed by Filter:' | grep -oE '[0-9.]+$')"
+pj_outside="$(q "SELECT count(*) FROM ps_h WHERE NOT (c BETWEEN 100 AND 200);")"
+check "executing it reads the projection, so the filter sees fewer rows than the base holds" \
+	"$(if pgc_is_number "$pj_removed" && pgc_is_number "$pj_outside"; then
+		awk -v r="$pj_removed" -v o="$pj_outside" 'BEGIN { print (r + 0 < o + 0 ? "fewer" : "all " r " of " o) }'
+	else echo "unread: [$pj_removed] [$pj_outside]"; fi)" "fewer"
 
 check "GUC off: no projection scan" \
 	"$(q "SET pgcolumnar.enable_projection_scan=off; EXPLAIN (COSTS OFF) SELECT a, c FROM ps WHERE c BETWEEN 100 AND 200;" | grep -c 'Columnar Projection')" "0"
@@ -266,14 +279,23 @@ psql_run "CREATE TABLE pm (a int, c int) USING pgcolumnar;"
 psql_run "SELECT pgcolumnar.add_projection('pm', 'pmp', ARRAY['a','c'], ARRAY['c']);"
 psql_run "INSERT INTO pm SELECT g, g FROM generate_series(1,10000) g;"   # batch 1
 
+# THE PLAN THE ARMS BELOW NAME (#1339). Four of them say "via projection" or
+# "projection scan", and nothing asked how this count plans. Planned on the base,
+# every one of them passes whatever the projection scan does with the snapshot.
+check "the old-snapshot count is planned as a projection scan" \
+	"$(q "SET pgcolumnar.enable_projection_scan = on; EXPLAIN (COSTS OFF) SELECT count(a) FROM pm WHERE c BETWEEN 1 AND 20000;" | grep -c 'Columnar Projection: pmp')" "1"
+
 A_IN="$PGC_WORKDIR/pmA.in"; A_OUT="$PGC_WORKDIR/pmA.out"
 mkfifo "$A_IN"; : > "$A_OUT"
 env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres \
 	-d "$PGC_DB" -At -f "$A_IN" >> "$A_OUT" 2>&1 &
 A_PID=$!
 exec 9> "$A_IN"
 pa_send() { printf '%s\n' "$*" >&9; }
-pa_wait() { local t="$1" i; for i in $(seq 1 200); do grep -q "$t" "$A_OUT" && return 0; sleep 0.1; done; return 1; }
+# The token must carry a count. An ERROR echoes the statement, token included
+# ("LINE 1: SELECT 'PMA_' || ..."), so matching the bare token read a failed
+# query as an answer and the timeout arms below could not fire on it (#1339).
+pa_wait() { local t="$1" i; for i in $(seq 1 200); do grep -qE "${t}[0-9]+" "$A_OUT" && return 0; sleep 0.1; done; return 1; }
 
 # A opens a snapshot that sees batch 1, using a projection scan.
 pa_send "SET pgcolumnar.enable_projection_scan = on;"

test/pytest/TESTS.md

@@ -6946,11 +6946,11 @@ Removal proofs, each red in both twins: the prototype's zone handling; `CREATE T
 
 Port of `approx_aggregates.sh` (#1296 phase 2). Phase 0 on #1296 decided that approximate distinct counts and percentiles come from the `hll` and `tdigest` extensions for 1.0, so this pair is the tested compatibility statement for them; docs/how-to.md names the versions.
 
-Independent of the shell suite: its own generator (other multipliers, the same 200,000 rows and 50,000 devices, because the check names state them), its own SQL. `compare_to_bash` grades the pair 14 literal matches, 0 missing.
+Independent of the shell suite: its own generator (other multipliers, the same 200,000 rows and 50,000 devices, because the check names state them), its own SQL. `compare_to_bash` grades the pair 15 literal matches, 0 missing.
 
 THE HEAP MIRROR IS WRITTEN FROM THE GENERATOR, never copied from the columnar table. The shell suite first built it by `CREATE TABLE ... AS SELECT * FROM ax`, which reads through the scan under test: with a mutation dropping one row in a thousand, every equality stayed green on all five majors. Writing both sides from the generator turned seven checks red under the same mutation.
 
-The bounds against the exact answer are premises, as pg_partman's are: they state the extension's own error, and a change here that broke them would break the equalities first.
+The bounds against the exact answer are premises, as pg_partman's are: they state the extension's own error, and a change here that broke them would break the equalities first. That holds for the stored tdigest only since #1339 added "a tdigest stored in a columnar table comes back byte for byte": before it, nothing compared the stored digests, and a reader returning each long value as the previous one passed every tdigest check. The digests are built once into a heap table and stored from there, because a tdigest depends on the order its inputs arrive in.
 
 hll 2.19 does not build against PostgreSQL 19, so `test_hll_over_columnar` is `UNSUPPORTED_MAJOR` there and listed in `expected_unrunnable.txt`. CI installs both extensions from PGDG for 15 to 18.
 

test/pytest/test_approx_aggregates.py

@@ -74,8 +74,14 @@ def test_tdigest_over_columnar(pgc_conn, expect):
     exact = _one(conn, "SELECT percentile_cont(0.99) WITHIN GROUP (ORDER BY v) FROM ax_heap")
     expect.text(repr(col), repr(heap), "tdigest over a columnar table gives exactly what it gives over heap")
     expect.text(_within(col, exact, 0.001), "ok", "premise: and its p99 is within 0.1% of the exact percentile")
+    # Built once into heap and stored from there, as the shell does: a tdigest depends on
+    # the order its inputs arrive in, so only a copy isolates the round trip (#1339).
+    conn.execute("CREATE TABLE ax_td_src USING heap AS SELECT day, tdigest(v, 100) AS t FROM ax GROUP BY day")
     conn.execute("CREATE TABLE ax_td (day int, t tdigest) USING pgcolumnar")
-    conn.execute("INSERT INTO ax_td SELECT day, tdigest(v, 100) FROM ax GROUP BY day")
+    conn.execute("INSERT INTO ax_td SELECT day, t FROM ax_td_src")
+    expect.text(_one(conn, "SELECT ((SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td) "
+                           "= (SELECT string_agg(t::text, ',' ORDER BY day) FROM ax_td_src))::text"), "true",
+                "a tdigest stored in a columnar table comes back byte for byte")
     expect.text(_within(_one(conn, "SELECT tdigest_percentile(t, 0.99) FROM ax_td"), exact, 0.001), "ok",
                 "premise: a tdigest stored in a columnar table still answers: "
                 "the union of daily digests is within 0.1% at p99")