From 92fe6948ef690a1d9587e5f07fd232a998ea705e Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 06:07:02 +0000 Subject: [PATCH 01/12] test: row-level security holds on every columnar read path (#1341) A new suite, rls_query_paths, and its pytest twin. A columnar table and a heap mirror hold the same 20000 rows, half owned by each of two roles, under a policy that shows a role only its own rows. One role, on its own login, queries both by the custom scan, the vectorized and grouped aggregates, with vectorization and the custom scan off, by a parallel scan, an index-only scan and a covering projection. Each answer must be heap's, and plan premises check each path was taken. A cheap function that is not leakproof must see only that role's rows. 14/14 on each of PG15-19, and the twin on each. Three checks are dated on PG17 by a mutant whose custom-scan plan qual drops the policy: the projection answered with the other role's rows, and the leak checks failed. The aggregate arm by the custom scan did not redden under it. cluster_tests 565 -> 566. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019VvWuX1zUwk6Y513waRd4N --- test/pytest/test_rls_query_paths.py | 109 ++++++++++++++++++++++++++++ test/rls_query_paths.sh | 95 ++++++++++++++++++++++++ test/run_all_versions.sh | 1 + 3 files changed, 205 insertions(+) create mode 100644 test/pytest/test_rls_query_paths.py create mode 100755 test/rls_query_paths.sh diff --git a/test/pytest/test_rls_query_paths.py b/test/pytest/test_rls_query_paths.py new file mode 100644 index 000000000..266dff929 --- /dev/null +++ b/test/pytest/test_rls_query_paths.py @@ -0,0 +1,109 @@ +"""Row-level security on every columnar read path (#1341). The pytest twin of +`test/rls_query_paths.sh`, every check by the shell's name. + +A columnar table and a heap mirror hold the same 20000 rows, half owned by each of two +roles, under a policy that shows a role only its own rows. One role, on its own login, +queries both by the custom scan, the vectorized and grouped aggregates, a parallel +scan, an index-only scan and a covering projection, and each answer must be heap's; the +plan premises check each path was taken. A cheap function that is not leakproof must +see only that role's rows: its NOTICEs are counted by a notice handler. +""" +import psycopg + +ALICE, BOB = "rqp_alice", "rqp_bob" +AGG = ("SELECT count(*) || '/' || sum(v) || '/' || min(v) || '/' || max(v) || '/' || " + "count(*) FILTER (WHERE owner <> current_user) FROM %t") +GRP = ("SELECT string_agg(grp || ':' || n || ':' || s, ',' ORDER BY grp) " + "FROM (SELECT grp, count(*) n, sum(v) s FROM %t GROUP BY grp) x") +OFF = ("SET pgcolumnar.enable_vectorization = off", "SET pgcolumnar.enable_group_vectorization = off", + "SET pgcolumnar.enable_custom_scan = off") +PAR = ("SET parallel_setup_cost = 0", "SET parallel_tuple_cost = 0", "SET min_parallel_table_scan_size = 0", + "SET max_parallel_workers_per_gather = 2") +IOS = ("SET enable_seqscan = off", "SET enable_bitmapscan = off", "SET pgcolumnar.enable_custom_scan = off") +IQ = "SELECT count(*) || '/' || sum(id) FROM %t WHERE id BETWEEN 100 AND 15000" +PQ = "SELECT count(*) || '/' || sum(v) FROM %t WHERE id BETWEEN 100 AND 15000" + + +def test_row_level_security_holds_on_every_read_path(pgc_cluster, pgc_conn, expect): + with pgc_conn.cursor() as cur: + cur.execute("SELECT current_schema()") + schema = cur.fetchone()[0] + for r in (ALICE, BOB): + cur.execute(f"SELECT 1 FROM pg_roles WHERE rolname = '{r}'") + if cur.fetchone() is None: + cur.execute(f"CREATE ROLE {r} LOGIN NOSUPERUSER") + cur.execute(f'GRANT USAGE ON SCHEMA "{schema}", pgcolumnar TO {r}') + cur.execute("CREATE FUNCTION rq_leak(text) RETURNS bool LANGUAGE plpgsql COST 0.0001 " + "AS $$ BEGIN RAISE NOTICE 'seen:%', $1; RETURN true; END $$") + for t, am in (("rq", "pgcolumnar"), ("rh", "heap")): + cur.execute(f"CREATE TABLE {t} (id int, owner name, grp int, v int, secret text) USING {am}") + cur.execute(f"INSERT INTO {t} SELECT g, CASE WHEN g % 2 = 0 THEN '{ALICE}' ELSE '{BOB}' END, g % 10, g, " + f"'S-' || CASE WHEN g % 2 = 0 THEN '{ALICE}' ELSE '{BOB}' END || '-' || g " + f"FROM generate_series(1, 20000) g") + cur.execute(f"CREATE INDEX {t}_io ON {t} (id, owner)") + cur.execute(f"ALTER TABLE {t} ENABLE ROW LEVEL SECURITY") + cur.execute(f"CREATE POLICY {t}_own ON {t} USING (owner = current_user)") + cur.execute(f"GRANT SELECT ON {t} TO {ALICE}, {BOB}") + cur.execute(f"VACUUM ANALYZE {t}") + cur.execute("SELECT pgcolumnar.add_projection('rq', 'rq_p', ARRAY['id', 'owner', 'v'], ARRAY['id'])") + cur.execute("SELECT count(*) FROM rq") + total = cur.fetchone()[0] + + alice = psycopg.connect(pgc_cluster.dsn().replace("user=postgres", f"user={ALICE}"), autocommit=True) + notices = [] + alice.add_notice_handler(lambda d: notices.append(d.message_primary or "")) + try: + alice.execute(f'SET search_path TO "{schema}", public, pgcolumnar') + + def run(setup, sql): + try: + for s in setup: + alice.execute(s) + return alice.execute(sql).fetchone()[0] + except psycopg.Error as exc: + return f"ERROR: {exc.sqlstate}" + finally: + alice.execute("RESET ALL") + alice.execute(f'SET search_path TO "{schema}", public, pgcolumnar') + + def same(setup, sql): + c, h = run(setup, sql.replace("%t", "rq")), run(setup, sql.replace("%t", "rh")) + ok = c == h and c is not None and not str(c).startswith("ERROR") + return "same" if ok else f"columnar=[{c}] heap=[{h}]" + + def plan_has(setup, sql, needle): + for s in setup: + alice.execute(s) + plan = "\n".join(r[0] for r in alice.execute("EXPLAIN (COSTS OFF) " + sql.replace("%t", "rq")).fetchall()) + alice.execute("RESET ALL") + alice.execute(f'SET search_path TO "{schema}", public, pgcolumnar') + return sum(1 for line in plan.splitlines() if needle in line) + + def leaks(table, setup): + notices.clear() + run(setup, f"SELECT count(*) FROM {table} WHERE rq_leak(secret)") + seen = [n for n in notices if n.startswith("seen:")] + return f"{len(seen)}|{sum(1 for n in seen if n.startswith(f'seen:S-{BOB}'))}" + + expect.text(f"{total}|{run((), 'SELECT count(*) FROM rq')}|{run((), 'SELECT count(*) FROM rh')}", + "20000|10000|10000", + "premise: a superuser sees all 20000 rows, and rq_alice her 10000 on both tables") + expect.num(plan_has((), AGG, "Custom Scan (PgColumnarScan)"), 1, + "premise: the aggregate is answered by the columnar custom scan") + expect.text(same((), AGG), "same", "count, sum, min and max under the policy, by the custom scan, are heap's") + expect.text(same((), GRP), "same", "a GROUP BY under the policy, with vectorization on, is heap's") + expect.text(f"{same(OFF, GRP)}|{same(OFF, AGG)}", "same|same", "and with vectorization and the custom scan off") + expect.num(plan_has(PAR, AGG, "Gather"), 1, "premise: the parallel arm's plan gathers from workers") + expect.text(f"{same(PAR, AGG)}|{same(PAR, GRP)}", "same|same", + "count, sum, min and max under the policy, by a parallel scan, are heap's") + expect.num(plan_has(IOS, IQ, "Index Only Scan"), 1, "premise: the index-only arm's plan is an index-only scan") + expect.text(same(IOS, IQ), "same", "an index-only scan under the policy is heap's") + expect.num(plan_has((), PQ, "Columnar Projection: rq_p"), 1, + "premise: the projection arm's plan reads the covering projection") + expect.text(same((), PQ), "same", "a covering projection scan under the policy is heap's") + expect.text(f"{leaks('rq', ())}|{leaks('rh', ())}", "10000|0|10000|0", + "a cheap function that is not leakproof sees only rq_alice's rows on columnar, as on heap") + expect.text(leaks("rq", OFF), "10000|0", "and the same with vectorization and the custom scan off") + expect.text(leaks("rq", PAR), "10000|0", "and through a parallel scan") + finally: + alice.close() diff --git a/test/rls_query_paths.sh b/test/rls_query_paths.sh new file mode 100755 index 000000000..6c29cf857 --- /dev/null +++ b/test/rls_query_paths.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# +# Row-level security on every columnar read path (#1341). The columnar custom scan +# builds its quals from the restriction list, the vectorized and grouped aggregates +# and the covering projection scan answer from their own paths, and an index-only +# scan and a parallel scan read the table differently again. A policy must hold on +# each, and a function that is not leakproof must never see a row the policy hides. +# +# A table and a heap mirror hold the same 20000 rows, half owned by each of two +# roles, with a policy that shows a role only its own rows. Each arm logs in as one of +# the roles (a real login, not SET ROLE) and must answer as heap does, on the path the +# arm names: the plan premises check that path was taken. +# +# Usage: test/rls_query_paths.sh [PG_CONFIG] +# Written fresh for pgColumnar. + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +psql_run "DROP ROLE IF EXISTS rq_alice; DROP ROLE IF EXISTS rq_bob; + CREATE ROLE rq_alice LOGIN NOSUPERUSER; CREATE ROLE rq_bob LOGIN NOSUPERUSER; + GRANT USAGE ON SCHEMA pgcolumnar TO rq_alice; + CREATE FUNCTION rq_leak(text) RETURNS bool LANGUAGE plpgsql COST 0.0001 + AS \$\$ BEGIN RAISE NOTICE 'seen:%', \$1; RETURN true; END \$\$;" +for t in rq rh; do + am=pgcolumnar; [ $t = rh ] && am=heap + psql_run "CREATE TABLE $t (id int, owner name, grp int, v int, secret text) USING $am; + INSERT INTO $t SELECT g, CASE WHEN g % 2 = 0 THEN 'rq_alice' ELSE 'rq_bob' END, g % 10, g, + 'S-' || CASE WHEN g % 2 = 0 THEN 'rq_alice' ELSE 'rq_bob' END || '-' || g + FROM generate_series(1, 20000) g; + CREATE INDEX ${t}_io ON $t (id, owner); + ALTER TABLE $t ENABLE ROW LEVEL SECURITY; + CREATE POLICY ${t}_own ON $t USING (owner = current_user); + GRANT SELECT ON $t TO rq_alice, rq_bob;" + psql_run "VACUUM ANALYZE $t;" # not in a transaction block +done +psql_run "SELECT pgcolumnar.add_projection('rq', 'rq_p', ARRAY['id', 'owner', 'v'], ARRAY['id']);" >/dev/null + +as_alice() { # as_alice SQL -> its output, as rq_alice on her own login + env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U rq_alice -d "$PGC_DB" -XAtq \ + -c "SET search_path TO public, pgcolumnar" -c "$1" 2>&1 +} +same() { # same SETUP QUERY -> "same", or both answers; %t is the table + local c h + c="$(as_alice "$1 ${2//%t/rq}" | tail -1)"; h="$(as_alice "$1 ${2//%t/rh}" | tail -1)" + # an error on both sides is not agreement + case "$c" in *ERROR*) echo "columnar=[$c] heap=[$h]"; return ;; esac + [ "$c" = "$h" ] && [ -n "$c" ] && echo same || echo "columnar=[$c] heap=[$h]" +} +plan_has() { # plan_has SETUP QUERY PATTERN -> how many plan lines match, on the columnar table + as_alice "$1 EXPLAIN (COSTS OFF) ${2//%t/rq}" | grep -cE "$3" +} + +AGG="SELECT count(*) || '/' || sum(v) || '/' || min(v) || '/' || max(v) || '/' || count(*) FILTER (WHERE owner <> current_user) FROM %t" +GRP="SELECT string_agg(grp || ':' || n || ':' || s, ',' ORDER BY grp) FROM (SELECT grp, count(*) n, sum(v) s FROM %t GROUP BY grp) x" + +check "premise: a superuser sees all 20000 rows, and rq_alice her 10000 on both tables" \ + "$(q "SELECT count(*) FROM rq")|$(as_alice "SELECT count(*) FROM rq")|$(as_alice "SELECT count(*) FROM rh")" "20000|10000|10000" + +# ---- the custom scan and the vectorized aggregates ----------------------------------- +check "premise: the aggregate is answered by the columnar custom scan" "$(plan_has "" "$AGG" 'Custom Scan \(PgColumnarScan\)')" "1" +check "count, sum, min and max under the policy, by the custom scan, are heap's" "$(same "" "$AGG")" "same" +check "a GROUP BY under the policy, with vectorization on, is heap's" "$(same "" "$GRP")" "same" +OFF="SET pgcolumnar.enable_vectorization = off; SET pgcolumnar.enable_group_vectorization = off; SET pgcolumnar.enable_custom_scan = off;" +check "and with vectorization and the custom scan off" "$(same "$OFF" "$GRP")|$(same "$OFF" "$AGG")" "same|same" + +# ---- a parallel scan ----------------------------------------------------------------- +PAR="SET parallel_setup_cost = 0; SET parallel_tuple_cost = 0; SET min_parallel_table_scan_size = 0; SET max_parallel_workers_per_gather = 2;" +check "premise: the parallel arm's plan gathers from workers" "$(plan_has "$PAR" "$AGG" 'Gather')" "1" +check "count, sum, min and max under the policy, by a parallel scan, are heap's" "$(same "$PAR" "$AGG")|$(same "$PAR" "$GRP")" "same|same" + +# ---- an index-only scan -------------------------------------------------------------- +IOS="SET enable_seqscan = off; SET enable_bitmapscan = off; SET pgcolumnar.enable_custom_scan = off;" +IQ="SELECT count(*) || '/' || sum(id) FROM %t WHERE id BETWEEN 100 AND 15000" +check "premise: the index-only arm's plan is an index-only scan" "$(plan_has "$IOS" "$IQ" 'Index Only Scan')" "1" +check "an index-only scan under the policy is heap's" "$(same "$IOS" "$IQ")" "same" + +# ---- a covering projection ----------------------------------------------------------- +PQ="SELECT count(*) || '/' || sum(v) FROM %t WHERE id BETWEEN 100 AND 15000" +check "premise: the projection arm's plan reads the covering projection" "$(plan_has "" "$PQ" 'Columnar Projection: rq_p')" "1" +check "a covering projection scan under the policy is heap's" "$(same "" "$PQ")" "same" + +# ---- a function that is not leakproof sees only visible rows -------------------------- +leaks() { # leaks TABLE SETUP -> "|" + local out + out="$(as_alice "$2 SELECT count(*) FROM $1 WHERE rq_leak(secret)")" + echo "$(grep -c 'seen:' <<<"$out")|$(grep -c 'seen:S-rq_bob' <<<"$out")" +} +check "a cheap function that is not leakproof sees only rq_alice's rows on columnar, as on heap" \ + "$(leaks rq "")|$(leaks rh "")" "10000|0|10000|0" +check "and the same with vectorization and the custom scan off" "$(leaks rq "$OFF")" "10000|0" +check "and through a parallel scan" "$(leaks rq "$PAR")" "10000|0" + +pgc_summary diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index f3961c4bf..dbd03a1c3 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -294,6 +294,7 @@ SUITES=( rewrite_group_scan rewrite_storage_oid rls_direct_storage + rls_query_paths rollup row_triggers savepoint_flush From 5a26beb7eb5da69b8aab7e4328dbaab63d72a719 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 17:06:13 +0000 Subject: [PATCH 02/12] test: rls_query_paths' parallel leak arm really runs in parallel, with a premise that says so (#1341) rq_leak had no parallel label, so the planner kept it out of the workers and the 'through a parallel scan' arm planned no Gather, repeating the first leak arm. It is PARALLEL SAFE now, in both twins, and a premise asserts the Gather. Found by OffgridwithJD in review of #1429. Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_rls_query_paths.py | 6 +++++- test/rls_query_paths.sh | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/test/pytest/test_rls_query_paths.py b/test/pytest/test_rls_query_paths.py index 266dff929..efde5a30a 100644 --- a/test/pytest/test_rls_query_paths.py +++ b/test/pytest/test_rls_query_paths.py @@ -33,7 +33,7 @@ def test_row_level_security_holds_on_every_read_path(pgc_cluster, pgc_conn, expe if cur.fetchone() is None: cur.execute(f"CREATE ROLE {r} LOGIN NOSUPERUSER") cur.execute(f'GRANT USAGE ON SCHEMA "{schema}", pgcolumnar TO {r}') - cur.execute("CREATE FUNCTION rq_leak(text) RETURNS bool LANGUAGE plpgsql COST 0.0001 " + cur.execute("CREATE FUNCTION rq_leak(text) RETURNS bool LANGUAGE plpgsql COST 0.0001 PARALLEL SAFE " "AS $$ BEGIN RAISE NOTICE 'seen:%', $1; RETURN true; END $$") for t, am in (("rq", "pgcolumnar"), ("rh", "heap")): cur.execute(f"CREATE TABLE {t} (id int, owner name, grp int, v int, secret text) USING {am}") @@ -104,6 +104,10 @@ def leaks(table, setup): expect.text(f"{leaks('rq', ())}|{leaks('rh', ())}", "10000|0|10000|0", "a cheap function that is not leakproof sees only rq_alice's rows on columnar, as on heap") expect.text(leaks("rq", OFF), "10000|0", "and the same with vectorization and the custom scan off") + # PARALLEL SAFE, or the planner keeps the function out of the workers and + # this arm repeats the first one. + expect.num(plan_has(PAR, "SELECT count(*) FROM %t WHERE rq_leak(secret)", "Gather"), 1, + "premise: the parallel leak arm's plan gathers from workers") expect.text(leaks("rq", PAR), "10000|0", "and through a parallel scan") finally: alice.close() diff --git a/test/rls_query_paths.sh b/test/rls_query_paths.sh index 6c29cf857..518baeaf7 100755 --- a/test/rls_query_paths.sh +++ b/test/rls_query_paths.sh @@ -21,7 +21,7 @@ pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" psql_run "DROP ROLE IF EXISTS rq_alice; DROP ROLE IF EXISTS rq_bob; CREATE ROLE rq_alice LOGIN NOSUPERUSER; CREATE ROLE rq_bob LOGIN NOSUPERUSER; GRANT USAGE ON SCHEMA pgcolumnar TO rq_alice; - CREATE FUNCTION rq_leak(text) RETURNS bool LANGUAGE plpgsql COST 0.0001 + CREATE FUNCTION rq_leak(text) RETURNS bool LANGUAGE plpgsql COST 0.0001 PARALLEL SAFE AS \$\$ BEGIN RAISE NOTICE 'seen:%', \$1; RETURN true; END \$\$;" for t in rq rh; do am=pgcolumnar; [ $t = rh ] && am=heap @@ -90,6 +90,10 @@ leaks() { # leaks TABLE SETUP -> "| check "a cheap function that is not leakproof sees only rq_alice's rows on columnar, as on heap" \ "$(leaks rq "")|$(leaks rh "")" "10000|0|10000|0" check "and the same with vectorization and the custom scan off" "$(leaks rq "$OFF")" "10000|0" +# PARALLEL SAFE, or the planner keeps the function out of the workers and this arm +# repeats the first one (OffgridwithJD, review of #1429). +check "premise: the parallel leak arm's plan gathers from workers" \ + "$(plan_has "$PAR" "SELECT count(*) FROM %t WHERE rq_leak(secret)" 'Gather')" "1" check "and through a parallel scan" "$(leaks rq "$PAR")" "10000|0" pgc_summary From 96130438ca818963203519aba15fa074b0fbeb96 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 17:12:31 +0000 Subject: [PATCH 03/12] test: rls_query_paths is dated by the mutants that drop the policy, and the reddened premise becomes an arm (#1341) Review of #1429 by OffgridwithJD: the old mutant kept the security_level 0 clauses, so it kept the policy and dropped the user's quals. Dated now by two mutants that keep only clauses above level 0: one in the custom scan's plan qual (6 checks), one in the grouped vectorized node's own quals, which it builds from baserestrictinfo and which the first cannot reach (the GROUP BY arm). The superuser/rq_alice check, reddened by the first, loses its premise prefix, in both twins. Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_rls_query_paths.py | 2 +- test/rls_query_paths.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pytest/test_rls_query_paths.py b/test/pytest/test_rls_query_paths.py index efde5a30a..46c4c87d6 100644 --- a/test/pytest/test_rls_query_paths.py +++ b/test/pytest/test_rls_query_paths.py @@ -87,7 +87,7 @@ def leaks(table, setup): expect.text(f"{total}|{run((), 'SELECT count(*) FROM rq')}|{run((), 'SELECT count(*) FROM rh')}", "20000|10000|10000", - "premise: a superuser sees all 20000 rows, and rq_alice her 10000 on both tables") + "a superuser sees all 20000 rows, and rq_alice her 10000 on both tables") expect.num(plan_has((), AGG, "Custom Scan (PgColumnarScan)"), 1, "premise: the aggregate is answered by the columnar custom scan") expect.text(same((), AGG), "same", "count, sum, min and max under the policy, by the custom scan, are heap's") diff --git a/test/rls_query_paths.sh b/test/rls_query_paths.sh index 518baeaf7..b74331731 100755 --- a/test/rls_query_paths.sh +++ b/test/rls_query_paths.sh @@ -55,7 +55,7 @@ plan_has() { # plan_has SETUP QUERY PATTERN -> how many plan lines match, on the AGG="SELECT count(*) || '/' || sum(v) || '/' || min(v) || '/' || max(v) || '/' || count(*) FILTER (WHERE owner <> current_user) FROM %t" GRP="SELECT string_agg(grp || ':' || n || ':' || s, ',' ORDER BY grp) FROM (SELECT grp, count(*) n, sum(v) s FROM %t GROUP BY grp) x" -check "premise: a superuser sees all 20000 rows, and rq_alice her 10000 on both tables" \ +check "a superuser sees all 20000 rows, and rq_alice her 10000 on both tables" \ "$(q "SELECT count(*) FROM rq")|$(as_alice "SELECT count(*) FROM rq")|$(as_alice "SELECT count(*) FROM rh")" "20000|10000|10000" # ---- the custom scan and the vectorized aggregates ----------------------------------- From ed6ad3653d33a609cb05bed937d024366695d4aa Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 18:07:26 +0000 Subject: [PATCH 04/12] test: rls_query_paths covers the ungrouped vectorized aggregate, and premises pin both vectorized plans (#1341) Review of #1429 by OffgridwithJD: the ungrouped vectorized aggregate builds its own quals from baserestrictinfo (columnar_vector.c:1532), and a user can turn it on; no arm reached it on its own. New arm, with that setting on and a premise on 'Columnar Vectorized Aggregates', and a premise that the GROUP BY arm is the grouped vectorized node. Both twins. Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_rls_query_paths.py | 11 +++++++++++ test/rls_query_paths.sh | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/test/pytest/test_rls_query_paths.py b/test/pytest/test_rls_query_paths.py index 46c4c87d6..9b644026e 100644 --- a/test/pytest/test_rls_query_paths.py +++ b/test/pytest/test_rls_query_paths.py @@ -91,7 +91,18 @@ def leaks(table, setup): expect.num(plan_has((), AGG, "Custom Scan (PgColumnarScan)"), 1, "premise: the aggregate is answered by the columnar custom scan") expect.text(same((), AGG), "same", "count, sum, min and max under the policy, by the custom scan, are heap's") + expect.num(plan_has((), GRP, "Columnar Vectorized Group Keys"), 1, + "premise: the GROUP BY arm's plan is the grouped vectorized node") expect.text(same((), GRP), "same", "a GROUP BY under the policy, with vectorization on, is heap's") + # The ungrouped vectorized aggregate builds its own quals too, and any user can + # turn it on. With no qual left, count(*) alone comes from metadata, so a sum + # makes it scan. + uva = ("SET pgcolumnar.enable_ungrouped_vector_agg = on",) + uagg = "SELECT count(*) || '/' || sum(v) FROM %t" + expect.num(plan_has(uva, uagg, "Columnar Vectorized Aggregates"), 1, + "premise: the ungrouped vectorized arm's plan folds the aggregates in the scan") + expect.text(same(uva, uagg), "same", + "count and sum under the policy, by the ungrouped vectorized aggregate, are heap's") expect.text(f"{same(OFF, GRP)}|{same(OFF, AGG)}", "same|same", "and with vectorization and the custom scan off") expect.num(plan_has(PAR, AGG, "Gather"), 1, "premise: the parallel arm's plan gathers from workers") expect.text(f"{same(PAR, AGG)}|{same(PAR, GRP)}", "same|same", diff --git a/test/rls_query_paths.sh b/test/rls_query_paths.sh index b74331731..e69b00577 100755 --- a/test/rls_query_paths.sh +++ b/test/rls_query_paths.sh @@ -61,7 +61,17 @@ check "a superuser sees all 20000 rows, and rq_alice her 10000 on both tables" \ # ---- the custom scan and the vectorized aggregates ----------------------------------- check "premise: the aggregate is answered by the columnar custom scan" "$(plan_has "" "$AGG" 'Custom Scan \(PgColumnarScan\)')" "1" check "count, sum, min and max under the policy, by the custom scan, are heap's" "$(same "" "$AGG")" "same" +check "premise: the GROUP BY arm's plan is the grouped vectorized node" \ + "$(plan_has "" "$GRP" 'Columnar Vectorized Group Keys')" "1" check "a GROUP BY under the policy, with vectorization on, is heap's" "$(same "" "$GRP")" "same" +# The ungrouped vectorized aggregate builds its own quals from baserestrictinfo too, +# and any user can turn it on (PGC_USERSET). With no qual left, count(*) alone is +# answered from metadata, so a sum makes it scan (OffgridwithJD, review of #1429). +UVA="SET pgcolumnar.enable_ungrouped_vector_agg = on;" +UAGG="SELECT count(*) || '/' || sum(v) FROM %t" +check "premise: the ungrouped vectorized arm's plan folds the aggregates in the scan" \ + "$(plan_has "$UVA" "$UAGG" 'Columnar Vectorized Aggregates')" "1" +check "count and sum under the policy, by the ungrouped vectorized aggregate, are heap's" "$(same "$UVA" "$UAGG")" "same" OFF="SET pgcolumnar.enable_vectorization = off; SET pgcolumnar.enable_group_vectorization = off; SET pgcolumnar.enable_custom_scan = off;" check "and with vectorization and the custom scan off" "$(same "$OFF" "$GRP")|$(same "$OFF" "$AGG")" "same|same" From 460be18f53c70b90cbb608887bca37a26977ea72 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 20:53:58 +0000 Subject: [PATCH 05/12] test: rls_query_paths holds both vectorized nodes to a non-leakproof function, red first (#1341) Review of #1429 by OffgridwithJD. Both vectorized aggregate nodes hand a non-leakproof function rows the policy hides: on PG17 at d343a50, rq_leak saw 20000 rows, 10000 of them rq_bob's, through the grouped node (the default) and through the ungrouped aggregate with enable_ungrouped_vector_agg on; heap shows it 10000 and 0. Two leak arms, each with a premise that the same query shape with a leakproof filter is answered by that node. Both twins. Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_rls_query_paths.py | 18 ++++++++++++++++-- test/rls_query_paths.sh | 17 ++++++++++++++--- 2 files changed, 30 insertions(+), 5 deletions(-) diff --git a/test/pytest/test_rls_query_paths.py b/test/pytest/test_rls_query_paths.py index 9b644026e..1bf09f7bd 100644 --- a/test/pytest/test_rls_query_paths.py +++ b/test/pytest/test_rls_query_paths.py @@ -79,9 +79,9 @@ def plan_has(setup, sql, needle): alice.execute(f'SET search_path TO "{schema}", public, pgcolumnar') return sum(1 for line in plan.splitlines() if needle in line) - def leaks(table, setup): + def leaks(table, setup, sql="SELECT count(*) FROM %t WHERE rq_leak(secret)"): notices.clear() - run(setup, f"SELECT count(*) FROM {table} WHERE rq_leak(secret)") + run(setup, sql.replace("%t", table)) seen = [n for n in notices if n.startswith("seen:")] return f"{len(seen)}|{sum(1 for n in seen if n.startswith(f'seen:S-{BOB}'))}" @@ -120,5 +120,19 @@ def leaks(table, setup): expect.num(plan_has(PAR, "SELECT count(*) FROM %t WHERE rq_leak(secret)", "Gather"), 1, "premise: the parallel leak arm's plan gathers from workers") expect.text(leaks("rq", PAR), "10000|0", "and through a parallel scan") + # Both vectorized nodes build their own quals, so each must also keep a + # non-leakproof function from rows the policy hides. + expect.num(plan_has((), "SELECT grp, count(*) FROM %t WHERE secret IS NOT NULL GROUP BY grp", + "Columnar Vectorized Group Keys"), 1, + "premise: a GROUP BY with a leakproof filter under the policy is the grouped vectorized node") + lgrp = "SELECT grp, count(*) FROM %t WHERE rq_leak(secret) GROUP BY grp" + expect.text(f"{leaks('rq', (), lgrp)}|{leaks('rh', (), lgrp)}", "10000|0|10000|0", + "a non-leakproof function in a GROUP BY under the policy sees only rq_alice's rows, as on heap") + expect.num(plan_has(uva, "SELECT count(*), sum(v) FROM %t WHERE secret IS NOT NULL", + "Columnar Vectorized Aggregates"), 1, + "premise: an ungrouped count and sum with a leakproof filter, with the setting on, " + "is the ungrouped vectorized aggregate") + expect.text(leaks("rq", uva, "SELECT count(*), sum(v) FROM %t WHERE rq_leak(secret)"), "10000|0", + "and in the ungrouped vectorized aggregate") finally: alice.close() diff --git a/test/rls_query_paths.sh b/test/rls_query_paths.sh index e69b00577..47ff4f691 100755 --- a/test/rls_query_paths.sh +++ b/test/rls_query_paths.sh @@ -92,9 +92,9 @@ check "premise: the projection arm's plan reads the covering projection" "$(plan check "a covering projection scan under the policy is heap's" "$(same "" "$PQ")" "same" # ---- a function that is not leakproof sees only visible rows -------------------------- -leaks() { # leaks TABLE SETUP -> "|" - local out - out="$(as_alice "$2 SELECT count(*) FROM $1 WHERE rq_leak(secret)")" +leaks() { # leaks TABLE SETUP [QUERY] -> "|" + local out sql="${3:-SELECT count(*) FROM %t WHERE rq_leak(secret)}" + out="$(as_alice "$2 ${sql//%t/$1}")" echo "$(grep -c 'seen:' <<<"$out")|$(grep -c 'seen:S-rq_bob' <<<"$out")" } check "a cheap function that is not leakproof sees only rq_alice's rows on columnar, as on heap" \ @@ -105,5 +105,16 @@ check "and the same with vectorization and the custom scan off" "$(leaks rq "$OF check "premise: the parallel leak arm's plan gathers from workers" \ "$(plan_has "$PAR" "SELECT count(*) FROM %t WHERE rq_leak(secret)" 'Gather')" "1" check "and through a parallel scan" "$(leaks rq "$PAR")" "10000|0" +# Both vectorized nodes build their own quals (above), so each must also keep a +# function that is not leakproof from rows the policy hides. Each premise shows the +# arm's query shape, with a leakproof filter in place of the function, is answered by +# that node (OffgridwithJD, review of #1429). +check "premise: a GROUP BY with a leakproof filter under the policy is the grouped vectorized node" \ + "$(plan_has "" "SELECT grp, count(*) FROM %t WHERE secret IS NOT NULL GROUP BY grp" 'Columnar Vectorized Group Keys')" "1" +check "a non-leakproof function in a GROUP BY under the policy sees only rq_alice's rows, as on heap" \ + "$(leaks rq "" "SELECT grp, count(*) FROM %t WHERE rq_leak(secret) GROUP BY grp")|$(leaks rh "" "SELECT grp, count(*) FROM %t WHERE rq_leak(secret) GROUP BY grp")" "10000|0|10000|0" +check "premise: an ungrouped count and sum with a leakproof filter, with the setting on, is the ungrouped vectorized aggregate" \ + "$(plan_has "$UVA" "SELECT count(*), sum(v) FROM %t WHERE secret IS NOT NULL" 'Columnar Vectorized Aggregates')" "1" +check "and in the ungrouped vectorized aggregate" "$(leaks rq "$UVA" "SELECT count(*), sum(v) FROM %t WHERE rq_leak(secret)")" "10000|0" pgc_summary From c73a0cce01322a5f082eeecc69a452e0c1c42c43 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 20:53:58 +0000 Subject: [PATCH 06/12] fix: the vectorized aggregates decline a WHERE that could leak past a row-level security policy (#1341) Row-level security puts the policy's quals at a lower security level than the user's, and a clause may run before a lower-level one only if it is leakproof. The grouped and ungrouped vectorized nodes build their own quals from baserestrictinfo and evaluate them in their own order, so a non-leakproof function in the user's WHERE saw rows the policy hides. Both nodes now decline such a query (pgcolumnar_quals_could_leak), leaving it to the custom scan, which orders its quals as order_qual_clauses does. A WHERE with only leakproof clauses, or with no policy, vectorizes as before. Co-Authored-By: Claude Opus 5.5 --- src/columnar_vector.c | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/columnar_vector.c b/src/columnar_vector.c index 7444f2e8c..3eb72b5b9 100644 --- a/src/columnar_vector.c +++ b/src/columnar_vector.c @@ -63,6 +63,7 @@ #include "nodes/nodeFuncs.h" #include "nodes/pathnodes.h" #include "nodes/plannodes.h" +#include "optimizer/clauses.h" #include "optimizer/optimizer.h" #include "optimizer/pathnode.h" #include "optimizer/paths.h" @@ -820,6 +821,38 @@ static bool pgcolumnar_batch_shape_eligible(PgColumnarAggScanState *state, * vectorized aggregate: planning * ------------------------------------------------------------------------- */ + +/* + * pgcolumnar_quals_could_leak + * True when a clause above the lowest security level could leak, so a node that + * evaluates the WHERE itself must not take the query. + * + * Row-level security puts the policy's quals at a lower security level than the + * user's. PostgreSQL lets a clause run before a lower-level one only if it is + * leakproof, and order_qual_clauses enforces that for an ordinary plan. The + * vectorized nodes build their own quals from baserestrictinfo and evaluate them + * in their own order, so a non-leakproof function in the user's WHERE was handed + * rows the policy hides (OffgridwithJD, review of #1429). Declining leaves such a + * query to the custom scan, which orders its quals correctly. + */ +static bool +pgcolumnar_quals_could_leak(List *rinfos) +{ + Index minLevel = UINT_MAX; + ListCell *lc; + + foreach(lc, rinfos) + minLevel = Min(minLevel, lfirst_node(RestrictInfo, lc)->security_level); + foreach(lc, rinfos) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + + if (rinfo->security_level > minLevel && + contain_leaked_vars((Node *) rinfo->clause)) + return true; + } + return false; +} static Plan * PgColumnarPlanAggPath(PlannerInfo *root, RelOptInfo *rel, CustomPath *best_path, List *tlist, List *clauses, List *custom_plans) @@ -1529,6 +1562,8 @@ PgColumnarCreateUpperPaths(PlannerInfo *root, UpperRelationKind stage, * zone-map answerable. A filter, or a sum/avg over int8/float/numeric, needs * the scan-fold path instead (#289). */ + if (pgcolumnar_quals_could_leak(factRel->baserestrictinfo)) + return; quals = extract_actual_clauses(factRel->baserestrictinfo, false); needsScan = (quals != NIL); @@ -2586,8 +2621,12 @@ PgColumnarTryGroupAggPath(PlannerInfo *root, RelOptInfo *input_rel, /* * WHERE is carried whole and rechecked per row, so correctness never depends - * on which clauses become scan keys; the scan keys only prune groups. + * on which clauses become scan keys; the scan keys only prune groups. A WHERE + * whose clauses could leak past a row-level security policy is left to the + * custom scan (pgcolumnar_quals_could_leak). */ + if (pgcolumnar_quals_could_leak(input_rel->baserestrictinfo)) + return; quals = extract_actual_clauses(input_rel->baserestrictinfo, false); /* From 47dbc22e97b858639e5ac17a3be73bc1e8e566e3 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 20:58:08 +0000 Subject: [PATCH 07/12] test: rls_query_paths' rows re-dated with the leak fix, restated on main 2d1a77f0, and the CHANGELOG entry (#1341) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 803430e91..32339204e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -401,6 +401,16 @@ true until the next version shipped. ### Fixed +- **A vectorized aggregate no longer shows a non-leakproof function rows that row-level + security hides.** Under a policy, a query such as `SELECT grp, count(*) FROM t WHERE + f(secret) GROUP BY grp` ran `f` on every row, the other owners' included, through the + grouped vectorized node (the default for a GROUP BY) and the ungrouped one + (`pgcolumnar.enable_ungrouped_vector_agg`). The answers were right; only the function + saw too much. Both nodes now leave a WHERE whose clauses could leak past the policy to + the custom scan, which orders its quals as PostgreSQL does. A WHERE with only leakproof + clauses, or a table with no policy, vectorizes as before. Found by OffgridwithJD in + review. `test/rls_query_paths.sh` and `test_rls_query_paths.py`. + - **A concurrent `UPDATE` of the same row now fails with 40001 on a table with a unique index, as documented, not 23505 (#1421).** Two sessions updating one row of a table with a PRIMARY KEY or UNIQUE index: the second used to wait on the first's From 7671c00e1ce58350446a7960a9061ff50227405d Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 22:41:51 +0000 Subject: [PATCH 08/12] test: column_extent refuses a caller a row-level security policy restricts, red first (#1341) Review of #1429 by OffgridwithJD: pgcolumnar.column_extent checked only the table ACL and returned the stored extent, which covers every row, so rq_alice read min and max over rq_bob's rows too. At 7d5b26b8 the new arm is red in both twins (not refused). A control checks the owner still reads the extent. Co-Authored-By: Claude Opus 5.5 --- test/pytest/test_rls_query_paths.py | 10 ++++++++++ test/rls_query_paths.sh | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/test/pytest/test_rls_query_paths.py b/test/pytest/test_rls_query_paths.py index 1bf09f7bd..b817fe60a 100644 --- a/test/pytest/test_rls_query_paths.py +++ b/test/pytest/test_rls_query_paths.py @@ -134,5 +134,15 @@ def leaks(table, setup, sql="SELECT count(*) FROM %t WHERE rq_leak(secret)"): "is the ungrouped vectorized aggregate") expect.text(leaks("rq", uva, "SELECT count(*), sum(v) FROM %t WHERE rq_leak(secret)"), "10000|0", "and in the ungrouped vectorized aggregate") + # column_extent reads the stored extent, which covers every row, so under a + # policy it would show values from rows the caller cannot see. It refuses, as + # the other direct-storage entry points do (#563); the owner still reads it. + ce = run((), "SELECT count(*) FROM pgcolumnar.column_extent('rq', 'v')") + expect.text("refused" if ce == "ERROR: 0A000" else f"not refused ({ce})", "refused", + "column_extent refuses a caller a policy restricts") + with pgc_conn.cursor() as cur: + cur.execute("SELECT count(*) FROM pgcolumnar.column_extent('rq', 'v')") + owner = cur.fetchone()[0] + expect.num(owner, 1, "and the owner still reads the extent") finally: alice.close() diff --git a/test/rls_query_paths.sh b/test/rls_query_paths.sh index 47ff4f691..e52c62a62 100755 --- a/test/rls_query_paths.sh +++ b/test/rls_query_paths.sh @@ -116,5 +116,13 @@ check "a non-leakproof function in a GROUP BY under the policy sees only rq_alic check "premise: an ungrouped count and sum with a leakproof filter, with the setting on, is the ungrouped vectorized aggregate" \ "$(plan_has "$UVA" "SELECT count(*), sum(v) FROM %t WHERE secret IS NOT NULL" 'Columnar Vectorized Aggregates')" "1" check "and in the ungrouped vectorized aggregate" "$(leaks rq "$UVA" "SELECT count(*), sum(v) FROM %t WHERE rq_leak(secret)")" "10000|0" +# pgcolumnar.column_extent reads the stored extent, which covers every row, so under a +# policy it would show values from rows the caller cannot see. It refuses, as the +# other direct-storage entry points do (#563), and the owner still reads it +# (OffgridwithJD, review of #1429). +ce="$(as_alice "SELECT count(*) FROM pgcolumnar.column_extent('rq', 'v')")" +check "column_extent refuses a caller a policy restricts" \ + "$(grep -q 'row-level security is in force' <<<"$ce" && echo refused || echo "not refused (${ce//$'\n'/ })")" "refused" +check "and the owner still reads the extent" "$(q "SELECT count(*) FROM pgcolumnar.column_extent('rq', 'v')")" "1" pgc_summary From f3dfc9794f44a9c3f033d60c66060a33fe2cdd71 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 22:41:51 +0000 Subject: [PATCH 09/12] fix: column_extent refuses a table whose row-level security policies apply to the caller (#1341) It calls PgColumnarRequireNoRowSecurity after its ACL check, as the other direct-storage entry points do (#563). The limitations page lists it with them, and the reference says it needs SELECT and refuses under a policy. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 ++++++ docs/limitations.md | 4 +++- docs/sql-reference.md | 3 +++ src/columnar_extent.c | 6 ++++++ 4 files changed, 18 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 32339204e..7db731a92 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -411,6 +411,12 @@ true until the next version shipped. clauses, or a table with no policy, vectorizes as before. Found by OffgridwithJD in review. `test/rls_query_paths.sh` and `test_rls_query_paths.py`. +- **`pgcolumnar.column_extent` refuses a table whose row-level security policies + apply to the caller.** It returned the column's stored smallest and largest value + over every row, so a caller limited by a policy could read values from rows it + cannot see. It now raises `row-level security is in force`, as the other functions + that read storage directly do. Found by OffgridwithJD in review. + - **A concurrent `UPDATE` of the same row now fails with 40001 on a table with a unique index, as documented, not 23505 (#1421).** Two sessions updating one row of a table with a PRIMARY KEY or UNIQUE index: the second used to wait on the first's diff --git a/docs/limitations.md b/docs/limitations.md index 7b343fd3b..d64ba6fe0 100644 --- a/docs/limitations.md +++ b/docs/limitations.md @@ -173,7 +173,9 @@ caller every row, so the call is refused instead. The functions affected are `read_projection`, `reconstruct_via_projection`, `export_arrow`, `export_parquet`, `parallel_export_parquet`, `import_arrow`, -`import_parquet` and `parallel_copy`. +`import_parquet`, `parallel_copy` and `column_extent`. `column_extent` reports a +column's stored smallest and largest value over every row. Under a policy, that +would show values from rows the caller cannot see. A superuser, a role with `BYPASSRLS`, and a table's owner without `FORCE ROW LEVEL SECURITY` are not refused. Each already reads every row by other means. diff --git a/docs/sql-reference.md b/docs/sql-reference.md index 085275157..d16b81067 100644 --- a/docs/sql-reference.md +++ b/docs/sql-reference.md @@ -468,6 +468,9 @@ commit. Every operation that drops or rewrites row groups recomputes it before committing: `expire`, `compact_rewrite`, `VACUUM`, `recluster`. So it can be wider than the data only by rows that are deleted but still stored. +It needs `SELECT` on the table, and it refuses a table whose row-level security +policies apply to the caller (see [limitations](limitations.md#row-level-security)). + No row means the planner reads every row group's zone map instead, which gives the same answer at a cost that grows with the table. That happens for: diff --git a/src/columnar_extent.c b/src/columnar_extent.c index 1a43395e3..e47066724 100644 --- a/src/columnar_extent.c +++ b/src/columnar_extent.c @@ -527,6 +527,12 @@ pgcolumnar_column_extent(PG_FUNCTION_ARGS) ac = pg_class_aclcheck(relid, GetUserId(), ACL_SELECT); if (ac != ACLCHECK_OK) aclcheck_error(ac, OBJECT_TABLE, get_rel_name(relid)); + /* + * The stored extent covers every row, so under a policy it would show values + * from rows the caller cannot see. Refused as the other direct-storage entry + * points are (#563), after the ACL check (OffgridwithJD, review of #1429). + */ + PgColumnarRequireNoRowSecurity(relid); rel = relation_open(relid, AccessShareLock); if (!PgColumnarIsColumnarRelation(relid)) ereport(ERROR, From 12503181c517d912dd5172d4e86ae2817c0abfc4 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 22:47:14 +0000 Subject: [PATCH 10/12] test: rls_query_paths' rows re-dated with the column_extent arm, restated on main 6f0bd12e (#1341) Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 24 ++++++++++++++++++++++++ test/check_ledger_budget.txt | 21 ++++++++++++++++++++- test/pytest/TESTS.md | 11 +++++++++++ test/pytest/expected_tests.txt | 4 +++- test/pytest/test_compare_to_bash.py | 1 + 5 files changed, 59 insertions(+), 2 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 40f578002..560957880 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -2861,6 +2861,30 @@ rls_direct_storage rls_direct_storage the policy is in force for ordinary SQL 15 rls_direct_storage rls_direct_storage the source files the import arms need exist 15;16;17;18;19 2026-10-14 export_parquet writes no file (total = 0) rls_direct_storage rls_direct_storage under FORCE the non-superuser owner IS refused, so owner-only is no substitute 15;16;17;18;19 2026-10-14 PgColumnarRequireNoRowSecurity never refuses (the RLS guard deleted) rls_direct_storage rls_direct_storage without FORCE the owner is allowed, because RLS_NONE_ENV is not RLS_ENABLED 15;16;17;18;19 2026-10-14 the RLS guard refuses anything but RLS_NONE so the owner and superuser (RLS_NONE_ENV) are refused too +rls_query_paths rls_query_paths a GROUP BY under the policy, with vectorization on, is heap's 15;16;17;18;19 2026-10-15 the grouped vectorized node's quals keep only clauses with security_level above 0, dropping the policy (src/columnar_vector.c, its baserestrictinfo walk) +rls_query_paths rls_query_paths a cheap function that is not leakproof sees only rq_alice's rows on columnar, as on heap 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath) +rls_query_paths rls_query_paths a covering projection scan under the policy is heap's 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath) +rls_query_paths rls_query_paths a non-leakproof function in a GROUP BY under the policy sees only rq_alice's rows, as on heap 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath);the vectorized nodes take a WHERE whose non-leakproof clause would run before the policy (the tree before pgcolumnar_quals_could_leak) +rls_query_paths rls_query_paths a superuser sees all 20000 rows, and rq_alice her 10000 on both tables 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath);the ungrouped vectorized aggregate's quals keep only clauses with security_level above 0, dropping the policy (src/columnar_vector.c, factRel->baserestrictinfo) +rls_query_paths rls_query_paths an index-only scan under the policy is heap's 15;16;17;18;19 never - +rls_query_paths rls_query_paths and in the ungrouped vectorized aggregate 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath);the vectorized nodes take a WHERE whose non-leakproof clause would run before the policy (the tree before pgcolumnar_quals_could_leak) +rls_query_paths rls_query_paths and the owner still reads the extent 15;16;17;18;19 never - +rls_query_paths rls_query_paths and the same with vectorization and the custom scan off 15;16;17;18;19 never - +rls_query_paths rls_query_paths and through a parallel scan 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath) +rls_query_paths rls_query_paths and with vectorization and the custom scan off 15;16;17;18;19 never - +rls_query_paths rls_query_paths column_extent refuses a caller a policy restricts 15;16;17;18;19 2026-10-15 column_extent without PgColumnarRequireNoRowSecurity, so it returns the extent of rows a policy hides +rls_query_paths rls_query_paths count and sum under the policy, by the ungrouped vectorized aggregate, are heap's 15;16;17;18;19 2026-10-15 the ungrouped vectorized aggregate's quals keep only clauses with security_level above 0, dropping the policy (src/columnar_vector.c, factRel->baserestrictinfo) +rls_query_paths rls_query_paths count, sum, min and max under the policy, by a parallel scan, are heap's 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath) +rls_query_paths rls_query_paths count, sum, min and max under the policy, by the custom scan, are heap's 15;16;17;18;19 2026-10-15 the custom scan's plan qual keeps only clauses with security_level above 0, dropping the policy (src/columnar_customscan.c PgColumnarPlanCustomPath) +rls_query_paths rls_query_paths premise: a GROUP BY with a leakproof filter under the policy is the grouped vectorized node 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: an ungrouped count and sum with a leakproof filter, with the setting on, is the ungrouped vectorized aggregate 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the GROUP BY arm's plan is the grouped vectorized node 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the aggregate is answered by the columnar custom scan 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the index-only arm's plan is an index-only scan 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the parallel arm's plan gathers from workers 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the parallel leak arm's plan gathers from workers 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the projection arm's plan reads the covering projection 15;16;17;18;19 never - +rls_query_paths rls_query_paths premise: the ungrouped vectorized arm's plan folds the aggregates in the scan 15;16;17;18;19 never - rollup rollup * with anything but count is refused 15;16;17;18;19 2026-10-13 create_rollup allowing * with any function rollup rollup DROP TABLE of a rollup's source is refused while the rollup exists (2BP01, #1376) 15;16;17;18;19 2026-10-14 a rollup's view, partials and source with no dependency between them, and no catalog cleanup (main before #1376's fix) rollup rollup DROP VIEW of a rollup takes its partials and its catalog rows with it (#1376) 15;16;17;18;19 2026-10-14 a rollup's view, partials and source with no dependency between them, and no catalog cleanup (main before #1376's fix) diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 60dbfb047..a3960011a 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -2741,4 +2741,23 @@ suites_not_covered 224 # never 2528 + 16 = 2544 # dated 822 + 14 = 836 # 2544 + 836 == 3380 -checks_never_observed_red 2544 +# +# #1341, row-level security holds on every columnar read path (jdatcmd): 2544 -> +# 2557. rls_query_paths enters with all 24 checks from five-major runs (24/24 on each +# of 15-19). 11 are dated on PG17 by five mutants (OffgridwithJD, reviews of +# #1429; the first three reproduce on PG18 too): +# - three that drop the policy, keeping only clauses with security_level above 0, +# one per place it is applied: the custom scan's plan qual (the counts, the +# aggregates, the projection and every leak arm, the vectorized nodes' included, +# since those fall back to it), the grouped vectorized node's quals (the GROUP BY +# arm), and the ungrouped vectorized aggregate's (its arm and the counts); +# - the tree before pgcolumnar_quals_could_leak: the two vectorized-node leak arms; +# - column_extent without its row-security guard: the column_extent arm. +# The rest arrive never: the plan premises, the owner's extent control, and arms on +# paths these mutants miss. +# +# ledger rows 3380 + 24 = 3404 +# never 2544 + 13 = 2557 +# dated 836 + 11 = 847 +# 2557 + 847 == 3404 +checks_never_observed_red 2557 diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 87009bb81..2e32bcdcb 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -175,6 +175,7 @@ behaviour, the source of that number is named. - [127. test_objstore_allowlist.py: the endpoint allow-list, empty by default](#127-test_objstore_allowlistpy-the-endpoint-allow-list-empty-by-default) - [128. test_objstore_userinfo.py: userinfo in a URL is refused on every entry point](#128-test_objstore_userinfopy-userinfo-in-a-url-is-refused-on-every-entry-point) - [129. test_savepoint_flush.py: a savepoint rolled back after a flush](#129-test_savepoint_flushpy-a-savepoint-rolled-back-after-a-flush) +- [130. test_rls_query_paths.py: row-level security holds on every read path](#130-test_rls_query_pathspy-row-level-security-holds-on-every-read-path) ## 1. How to read a test in here @@ -7164,3 +7165,13 @@ The pytest twin of `test/savepoint_flush.sh` (#1341), every check by the shell's | test | what it holds | | --- | --- | | `test_a_savepoint_rolled_back_after_a_flush` | the reader paths, the flushes, and every shape answering as heap does inside the transaction and after it | + +## 130. test_rls_query_paths.py: row-level security holds on every read path + +The pytest twin of `test/rls_query_paths.sh` (#1341), every check by the shell's name. A columnar table and a heap mirror hold the same 20000 rows, half owned by each of two roles, under a policy that shows a role only its own rows. One role, on its own login, queries both by the custom scan, the vectorized and grouped aggregates, with vectorization and the custom scan off, by a parallel scan, an index-only scan and a covering projection; each answer must be heap's, and plan premises check each path was taken. A cheap function that is not leakproof must see only that role's rows, by the custom scan, with it off, and in parallel; a notice handler counts what it saw. Each vectorized node also gets a leak arm, with a premise that the same query with a leakproof filter is answered by that node, and `column_extent`, which reads the stored extent of every row, must refuse a caller the policy restricts. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_row_level_security_holds_on_every_read_path` | the read paths against heap (the custom scan, both vectorized aggregates, parallel, index-only, the covering projection), their plan premises, and the leak arms, including one through each vectorized node | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 59af69adc..e230ed616 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -722,4 +722,6 @@ guard_tests 438 # cluster_tests 598 -> 599 for #1341's test_min_and_max_keep_each_columns_collation in # test_rollup.py, by collection over the cluster files on this tree. # cluster_tests 599 -> 600 for #1341's test_savepoint_flush.py, one test, by collection -cluster_tests 600 +# cluster_tests 600 -> 601 for #1341's test_rls_query_paths.py, one test, by collection +# over the cluster files on this tree. +cluster_tests 601 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index ffa87bfdf..267066828 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -149,6 +149,7 @@ "recovery_minimal_wal", "rewrite_storage_oid", "rls_direct_storage", + "rls_query_paths", "rollup", "savepoint_flush", "scan_decode_cost", From 996413c03e35c91d61203a77e448e577fe840ca5 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 22:57:27 +0000 Subject: [PATCH 11/12] test: types the differential matrix never stored answer as heap (#1341) The encoder has fast paths for fixed widths 1, 2, 4 and 8, and every other width and every varlena takes a generic path. A new suite, type_matrix, and its pytest twin store what no test had: - by-reference fixed widths 6, 8, 12, 16 and 64: macaddr, macaddr8, timetz, point, name; - the 1-byte "char"; - time, money, inet, cidr, bit, varbit, tsvector, xml, json; - an enum, a composite, a domain; - numeric NaN and +-Infinity, timestamp and date +-infinity. 12000 rows in row groups of 1000, each column NULL on its own pattern, are generated once into heap and copied. Against heap: every row as text, each column's counts, a range on all sixteen ordered columns with literal bounds so zone maps can prune, json, point and tsvector matches, the special-value aggregates, and the composite's fields. 24/24 on each of PG15-19, and the twin on each. Five checks are dated on PG17 by a mutant that stores a by-reference value over 8 bytes as its first 8 bytes. cluster_tests 565 -> 566. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019VvWuX1zUwk6Y513waRd4N Squashed for the rebase onto main; it replaces: - test: type_matrix's pinned cidr arm names #1433, the cause OffgridwithJD found (#1341) - test: type_matrix's rows re-dated, restated on main (#1341) - test: type_matrix varies timetz's zone, asserts each range arm's pruning, and names a missing libxml (#1341) - test: types the differential matrix never stored answer as heap (#1341) Squashed for the rebase; it replaces: - test: type_matrix's rows re-dated, restated on main 6f0bd12e (#1341) - test: type_matrix's bit and varbit columns increase with the row, its bounds keep their typmod, and vb must skip (#1341) - test: types the differential matrix never stored answer as heap (#1341) --- test/check_ledger.tsv | 41 ++++++++ test/check_ledger_budget.txt | 19 +++- test/pytest/TESTS.md | 11 +++ test/pytest/expected_tests.txt | 3 +- test/pytest/test_compare_to_bash.py | 1 + test/pytest/test_type_matrix.py | 140 ++++++++++++++++++++++++++++ test/run_all_versions.sh | 1 + test/type_matrix.sh | 128 +++++++++++++++++++++++++ 8 files changed, 342 insertions(+), 2 deletions(-) create mode 100644 test/pytest/test_type_matrix.py create mode 100755 test/type_matrix.sh diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 560957880..25931eddd 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -3325,6 +3325,47 @@ ttl_expire ttl_expire the retention is recorded where the other options live 15; ttl_expire ttl_expire with the extent, expire retires the same groups 15;16;17;18;19 2026-10-13 the early exit compares the extent's maximum to the cutoff instead of its minimum ttl_expire ttl_expire without the extent the same expire probes every group's zone map 15;16;17;18;19 never - ttl_expire ttl_expire without the extent, expire retires the aged groups 15;16;17;18;19 2026-10-14 expire returning 0 whatever it retired +type_matrix type_matrix a range on bt finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on bt lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on cd finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on cd skips no chunk groups: a cidr key is treated as cross-type against inet and dropped (#1433) 15;16;17;18;19 never - +type_matrix type_matrix a range on ch finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on ch skips no chunk groups: every row group holds every value, since the column cycles through 26 letters 15;16;17;18;19 never - +type_matrix type_matrix a range on d finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on d lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on dom finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on dom lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on e finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on e skips no chunk groups: every row group holds every value, since the column cycles through the 4 labels 15;16;17;18;19 never - +type_matrix type_matrix a range on ip finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on ip lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on mac finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on mac lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on mac8 finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on mac8 lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on mo finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on mo lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on n finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on n skips no chunk groups: every row group holds NaN and both infinities, so each zone map spans every value 15;16;17;18;19 never - +type_matrix type_matrix a range on nm finds heap's rows 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen) +type_matrix type_matrix a range on nm lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen);the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on tm finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on tm lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on tsi finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on tsi lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on tz finds heap's rows 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen) +type_matrix type_matrix a range on tz lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a range on vb finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on vb lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) +type_matrix type_matrix a tsvector match finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix each column's count and NULL count is heap's 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen) +type_matrix type_matrix equality on json, as text, finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix every row, as text, is heap's 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen) +type_matrix type_matrix point equality (~=) finds heap's rows 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen) +type_matrix type_matrix premise: both tables hold 12000 rows, the columnar one in at least twelve row groups 15;16;17;18;19 never - +type_matrix type_matrix premise: the server supports xml (built with libxml) 15;16;17;18;19 never - +type_matrix type_matrix the aggregates over numeric NaN and Infinity and timestamp and date infinity are heap's 15;16;17;18;19 never - +type_matrix type_matrix the composite's fields and the enum's order read back as heap's 15;16;17;18;19 never - vacuum_lock_privilege vacuum_lock_privilege add_projection refuses a non-owner before taking its lock (#749) 15;16;17;18;19 2026-10-14 every maintenance entry point checks ownership after taking its lock vacuum_lock_privilege vacuum_lock_privilege cluster refuses a non-owner before taking the exclusive lock 15;16;17;18;19 2026-10-14 every maintenance entry point checks ownership after taking its lock vacuum_lock_privilege vacuum_lock_privilege compact does the same, which is the behaviour drop_projection now matches 15;16;17;18;19 never - diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index a3960011a..ad80824b3 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -2760,4 +2760,21 @@ suites_not_covered 224 # never 2544 + 13 = 2557 # dated 836 + 11 = 847 # 2557 + 847 == 3404 -checks_never_observed_red 2557 +# +# #1341, types the differential matrix never stored answer as heap (jdatcmd): +# 2557 -> 2581. type_matrix enters with all 41 checks from five-major runs (41/41 on +# each of 15-19). 17 are dated on PG17 by two mutants (OffgridwithJD, reviews of +# #1426): +# - a by-reference fixed-width value over 8 bytes stored as its first 8 bytes and +# zero padding: every row, the counts, the tz range (its zone offset varies, so +# bytes 8-11 matter), the nm range and its pruning, and point equality; +# - the scan never skipping a row group by zone map: the 12 range arms that must +# skip chunk groups, varbit's included now that its values increase with the row. +# The rest arrive never, among them the four range arms pinned at 0 skipped groups, +# with the reason in each name (cidr's names #1433), and the libxml premise. +# +# ledger rows 3404 + 41 = 3445 +# never 2557 + 24 = 2581 +# dated 847 + 17 = 864 +# 2581 + 864 == 3445 +checks_never_observed_red 2581 diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index 2e32bcdcb..e3fb22797 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -176,6 +176,7 @@ behaviour, the source of that number is named. - [128. test_objstore_userinfo.py: userinfo in a URL is refused on every entry point](#128-test_objstore_userinfopy-userinfo-in-a-url-is-refused-on-every-entry-point) - [129. test_savepoint_flush.py: a savepoint rolled back after a flush](#129-test_savepoint_flushpy-a-savepoint-rolled-back-after-a-flush) - [130. test_rls_query_paths.py: row-level security holds on every read path](#130-test_rls_query_pathspy-row-level-security-holds-on-every-read-path) +- [131. test_type_matrix.py: types the differential matrix never stored answer as heap](#131-test_type_matrixpy-types-the-differential-matrix-never-stored-answer-as-heap) ## 1. How to read a test in here @@ -7175,3 +7176,13 @@ The pytest twin of `test/rls_query_paths.sh` (#1341), every check by the shell's | test | what it holds | | --- | --- | | `test_row_level_security_holds_on_every_read_path` | the read paths against heap (the custom scan, both vectorized aggregates, parallel, index-only, the covering projection), their plan premises, and the leak arms, including one through each vectorized node | + +## 131. test_type_matrix.py: types the differential matrix never stored answer as heap + +The pytest twin of `test/type_matrix.sh` (#1341), every check by the shell's name. The encoder has fast paths for fixed widths 1, 2, 4 and 8, and every other width and every varlena takes a generic path. This matrix stores what no test had: by-reference fixed widths 6, 8, 12, 16 and 64 (macaddr, macaddr8, timetz, point, name), the 1-byte `"char"`, time, money, inet, cidr, bit, varbit, tsvector, xml, json, an enum, a composite, a domain, and numeric NaN and ±Infinity with timestamp and date ±infinity. 12000 rows in row groups of 1000, each column NULL on its own pattern, are generated once into heap and copied. Every row as text, each column's counts, a range on every ordered column with literal bounds read from heap, json, point and tsvector matches, the special-value aggregates, and the composite's fields must all equal heap's. + +### Every test + +| test | what it holds | +| --- | --- | +| `test_types_the_matrix_never_stored_answer_as_heap` | the 21 columns against heap: rows, counts, sixteen ranges, the three non-btree matches, and the special values | diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index e230ed616..968ada983 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -724,4 +724,5 @@ guard_tests 438 # cluster_tests 599 -> 600 for #1341's test_savepoint_flush.py, one test, by collection # cluster_tests 600 -> 601 for #1341's test_rls_query_paths.py, one test, by collection # over the cluster files on this tree. -cluster_tests 601 +# cluster_tests 601 -> 602 for #1341's test_type_matrix.py, one test, by collection +cluster_tests 602 diff --git a/test/pytest/test_compare_to_bash.py b/test/pytest/test_compare_to_bash.py index 267066828..43943a462 100644 --- a/test/pytest/test_compare_to_bash.py +++ b/test/pytest/test_compare_to_bash.py @@ -164,6 +164,7 @@ "tier_partition", "timeseries_recipes", "ttl_expire", + "type_matrix", "vacuum_lock_privilege", "validity_elision", "vm_privilege", diff --git a/test/pytest/test_type_matrix.py b/test/pytest/test_type_matrix.py new file mode 100644 index 000000000..9b72e510e --- /dev/null +++ b/test/pytest/test_type_matrix.py @@ -0,0 +1,140 @@ +"""Types the differential matrix never stored, against heap (#1341). The pytest twin +of `test/type_matrix.sh`, every check by the shell's name. + +The encoder has fast paths for fixed widths 1, 2, 4 and 8; every other width and every +varlena takes a generic path, and a zone map is built for each column whose type has a +btree ordering. These columns cover widths 6, 12, 16 and 64 by reference, the 1-byte +"char", time, money, inet, cidr, bit, varbit, tsvector, xml, json, an enum, a composite, +a domain, and numeric, timestamp and date special values. 12000 rows in row groups of +1000, each column NULL on its own pattern, generated once into heap and copied. + +Each comparison is of the rows themselves (`expect.row_set`), so a failure names the rows +that differ. The range bounds are the values heap holds at ids 3005 and 9005, written as +literals so the columnar scan can prune row groups by zone map (not 3001, which holds n's +NaN). Each range arm also says whether the zone maps skipped chunk groups, from EXPLAIN +ANALYZE; four that measure 0 are pinned at 0 with the reason. timetz's zone offset varies +with the row, so its 12-byte width is used past byte 8. +""" +import re + +import psycopg + +COLS = """id int, tz timetz, mac macaddr, mac8 macaddr8, nm name, pt point, ch "char", + tm time, mo money, ip inet, cd cidr, bt bit(16), vb varbit, tsv tsvector, x xml, + j json, e tm_mood, cmp tm_pair, dom tm_pos, n numeric, tsi timestamp, d date""" + +LOAD = """SELECT g, + CASE WHEN g % 7 = 0 THEN NULL ELSE ((('00:00'::time + (g * 7) * interval '1 s')::text || '+' || lpad((g % 13)::text, 2, '0'))::timetz) END, + CASE WHEN g % 11 = 0 THEN NULL ELSE ('08:00:2b:' || lpad(to_hex(g / 256 % 256), 2, '0') || ':' || lpad(to_hex(g % 256), 2, '0') || ':01')::macaddr END, + CASE WHEN g % 13 = 0 THEN NULL ELSE ('08:00:2b:01:02:03:' || lpad(to_hex(g / 256 % 256), 2, '0') || ':' || lpad(to_hex(g % 256), 2, '0'))::macaddr8 END, + CASE WHEN g % 17 = 0 THEN NULL ELSE ('name_' || lpad(g::text, 6, '0'))::name END, + CASE WHEN g % 19 = 0 THEN NULL ELSE point(g, -g) END, + CASE WHEN g % 23 = 0 THEN NULL ELSE chr(65 + g % 26)::"char" END, + CASE WHEN g % 29 = 0 THEN NULL ELSE ('00:00'::time + g * interval '7 s') END, + CASE WHEN g % 31 = 0 THEN NULL ELSE (g * 1.25)::numeric::money END, + CASE WHEN g % 37 = 0 THEN NULL ELSE ('10.' || g / 256 % 256 || '.' || g % 256 || '.' || g % 7 || '/32')::inet END, + CASE WHEN g % 41 = 0 THEN NULL ELSE ('10.' || g / 256 % 256 || '.' || g % 256 || '.0/24')::cidr END, + CASE WHEN g % 43 = 0 THEN NULL ELSE g::bit(16) END, + CASE WHEN g % 47 = 0 THEN NULL ELSE (g::bit(16) || (g % 7)::bit(3))::varbit END, + CASE WHEN g % 53 = 0 THEN NULL ELSE to_tsvector('simple', 'word' || g || ' common term' || g % 9) END, + CASE WHEN g % 59 = 0 THEN NULL ELSE ('' || md5(g::text) || '')::xml END, + CASE WHEN g % 61 = 0 THEN NULL ELSE json_build_object('g', g, 's', md5(g::text)) END, + CASE WHEN g % 67 = 0 THEN NULL ELSE (enum_range(NULL::tm_mood))[1 + g % 4] END, + CASE WHEN g % 71 = 0 THEN NULL ELSE ROW(g, 'p' || g)::tm_pair END, + CASE WHEN g % 73 = 0 THEN NULL ELSE (g)::tm_pos END, + CASE WHEN g % 79 = 0 THEN NULL WHEN g % 1000 = 1 THEN 'NaN'::numeric WHEN g % 1000 = 2 THEN 'Infinity'::numeric + WHEN g % 1000 = 3 THEN '-Infinity'::numeric ELSE g / 7.0 END, + CASE WHEN g % 83 = 0 THEN NULL WHEN g = 500 THEN 'infinity'::timestamp WHEN g = 501 THEN '-infinity'::timestamp + ELSE '2026-01-01'::timestamp + g * interval '1 min' END, + CASE WHEN g % 89 = 0 THEN NULL WHEN g = 600 THEN 'infinity'::date WHEN g = 601 THEN '-infinity'::date + ELSE '2026-01-01'::date + g END + FROM generate_series(1, 12000) g""" + +ALL = ("id, tz::text, mac::text, mac8::text, nm::text, pt::text, ch::text, tm::text, mo::text, ip::text, " + "cd::text, bt::text, vb::text, tsv::text, x::text, j::text, e::text, cmp::text, dom::text, n::text, " + "tsi::text, d::text") + + +def _rows(conn, sql): + with conn.cursor() as cur: + cur.execute(sql) + return cur.fetchall() + + +def _both(conn, sql): + return _rows(conn, sql.replace("%T", "t_col")), _rows(conn, sql.replace("%T", "t_heap")) + + +NO_PRUNE = { + "ch": "every row group holds every value, since the column cycles through 26 letters", + "e": "every row group holds every value, since the column cycles through the 4 labels", + "n": "every row group holds NaN and both infinities, so each zone map spans every value", + "cd": "a cidr key is treated as cross-type against inet and dropped (#1433)", +} + + +def _removed(conn, sql): + plan = "\n".join(r[0] for r in _rows(conn, "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) " + + sql.replace("%T", "t_col"))) + m = re.search(r"Columnar Chunk Groups Removed by Filter: (\d+)", plan) + return m.group(1) if m else "none" + + +def test_types_the_matrix_never_stored_answer_as_heap(pgc_conn, expect): + # xml needs a server built with libxml; without it every arm after the load would + # fail for that reason, so this says so first. + try: + xml_ok = "yes" if _rows(pgc_conn, "SELECT ''::xml IS NOT NULL")[0][0] else "no" + except psycopg.Error as exc: + xml_ok = f"no ({str(exc).splitlines()[0]})" + expect.text(xml_ok, "yes", "premise: the server supports xml (built with libxml)") + with pgc_conn.cursor() as cur: + cur.execute("CREATE TYPE tm_mood AS ENUM ('sad', 'ok', 'happy', 'elated')") + cur.execute("CREATE TYPE tm_pair AS (a int, b text)") + cur.execute("CREATE DOMAIN tm_pos AS int CHECK (VALUE > 0)") + cur.execute(f"CREATE TABLE t_heap ({COLS})") + cur.execute(f"CREATE TABLE t_col ({COLS}) USING pgcolumnar") + cur.execute("SELECT pgcolumnar.set_options('t_col', stripe_row_limit => 1000)") + cur.execute(f"INSERT INTO t_heap {LOAD}") + cur.execute("INSERT INTO t_col SELECT * FROM t_heap") + groups = _rows(pgc_conn, "SELECT count(*) FROM pgcolumnar.row_group " + "WHERE storage_id = pgcolumnar.get_storage_id('t_col')")[0][0] + n_heap = _rows(pgc_conn, "SELECT count(*) FROM t_heap")[0][0] + n_col = _rows(pgc_conn, "SELECT count(*) FROM t_col")[0][0] + expect.text(f"{n_heap}|{n_col}|{'many' if groups >= 12 else groups}", "12000|12000|many", + "premise: both tables hold 12000 rows, the columnar one in at least twelve row groups") + + got, want = _both(pgc_conn, f"SELECT {ALL} FROM %T") + expect.row_set(got, want, "every row, as text, is heap's") + counts = ", ".join(f"count({c}), count(*) FILTER (WHERE {c} IS NULL)" + for c in "tz mac mac8 nm pt ch tm mo ip cd bt vb tsv x j e cmp dom n tsi d".split()) + got, want = _both(pgc_conn, f"SELECT {counts} FROM %T") + expect.row_set(got, want, "each column's count and NULL count is heap's") + + for c in ("tz", "mac", "mac8", "nm", "ch", "tm", "mo", "ip", "cd", "bt", "vb", "e", "dom", "n", "tsi", "d"): + # format_type keeps the typmod: pg_typeof would make bt's bounds bit(1). + ty = _rows(pgc_conn, f"SELECT format_type(atttypid, atttypmod) FROM pg_attribute " + f"WHERE attrelid = 't_heap'::regclass AND attname = '{c}'")[0][0] + lo, hi = (_rows(pgc_conn, f"SELECT quote_literal({c}::text) || '::' || '{ty}' " + f"FROM t_heap WHERE id = {i}")[0][0] for i in (3005, 9005)) + sql = f"SELECT id FROM %T WHERE {c} BETWEEN least({lo}, {hi}) AND greatest({lo}, {hi})" + got, want = _both(pgc_conn, sql) + expect.row_set(got, want, f"a range on {c} finds heap's rows") + r = _removed(pgc_conn, sql) + if c in NO_PRUNE: + expect.text(r, "0", f"a range on {c} skips no chunk groups: {NO_PRUNE[c]}") + else: + expect.text("skips" if r.isdigit() and int(r) >= 1 else f"removed {r}", "skips", + f"a range on {c} lets the zone maps skip chunk groups") + got, want = _both(pgc_conn, "SELECT id FROM %T WHERE j::text = (SELECT j::text FROM t_heap WHERE id = 4242)") + expect.row_set(got, want, "equality on json, as text, finds heap's rows") + got, want = _both(pgc_conn, "SELECT id FROM %T WHERE pt ~= point(4242, -4242)") + expect.row_set(got, want, "point equality (~=) finds heap's rows") + got, want = _both(pgc_conn, "SELECT id FROM %T WHERE tsv @@ to_tsquery('simple', 'term3')") + expect.row_set(got, want, "a tsvector match finds heap's rows") + got, want = _both(pgc_conn, "SELECT min(n)::text, max(n)::text, count(*) FILTER (WHERE n = 'NaN'), " + "count(*) FILTER (WHERE n = 'Infinity'), count(*) FILTER (WHERE n = '-Infinity'), " + "min(tsi)::text, max(tsi)::text, min(d)::text, max(d)::text FROM %T") + expect.row_set(got, want, "the aggregates over numeric NaN and Infinity and timestamp and date infinity are heap's") + got, want = _both(pgc_conn, "SELECT (cmp).a, (cmp).b, e::text, e > 'ok' FROM %T WHERE id % 97 = 0") + expect.row_set(got, want, "the composite's fields and the enum's order read back as heap's") diff --git a/test/run_all_versions.sh b/test/run_all_versions.sh index dbd03a1c3..4466359f9 100755 --- a/test/run_all_versions.sh +++ b/test/run_all_versions.sh @@ -319,6 +319,7 @@ SUITES=( timeseries_recipes truncate_cleanup ttl_expire + type_matrix ungrouped_vector_agg unique_conc update_conc diff --git a/test/type_matrix.sh b/test/type_matrix.sh new file mode 100755 index 000000000..3c660af2b --- /dev/null +++ b/test/type_matrix.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# +# Types the differential matrix never stored, against heap (#1341). The encoder has +# fast paths for fixed widths 1, 2, 4 and 8; every other width and every varlena +# takes a generic path, and a zone map is built for each column whose type has a +# btree ordering. None of these types had been written to a columnar table: +# +# - fixed width, by reference, at unusual widths: timetz (12), macaddr (6), +# macaddr8 (8), name (64), point (16, no btree ordering); +# - the 1-byte "char"; +# - time, money, inet, cidr, bit, varbit, tsvector, xml, json (no equality); +# - an enum, a composite, a domain over int; +# - numeric NaN and +-Infinity, timestamp and date +-infinity. +# +# 12000 rows in row groups of 1000, each column NULL on its own pattern. The same +# rows go into heap and are copied to columnar. Then: every row, as text; each +# column's count and NULL count; a range on every ordered column, with constants +# read from heap so the zone maps can prune; equality on json (as text) and point +# (~=); and the aggregates over the special values. +# +# Usage: test/type_matrix.sh [PG_CONFIG] +# Written fresh for pgColumnar. + +set -uo pipefail +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +pgc_setup "${1:-/usr/local/pg17/bin/pg_config}" + +psql_run "CREATE TYPE tm_mood AS ENUM ('sad', 'ok', 'happy', 'elated'); + CREATE TYPE tm_pair AS (a int, b text); + CREATE DOMAIN tm_pos AS int CHECK (VALUE > 0);" +COLS="id int, tz timetz, mac macaddr, mac8 macaddr8, nm name, pt point, ch \"char\", + tm time, mo money, ip inet, cd cidr, bt bit(16), vb varbit, tsv tsvector, x xml, + j json, e tm_mood, cmp tm_pair, dom tm_pos, n numeric, tsi timestamp, d date" +# xml needs a server built with libxml. Without it the load below fails and every +# arm after it would fail for that reason, so the suite says so and stops +# (OffgridwithJD, review of #1426). +xml_ok="$(env PATH="$PGC_BINDIR:$PATH" psql -h 127.0.0.1 -p "$PGC_PORT" -U postgres -d "$PGC_DB" -XAtq \ + -c "SELECT ''::xml IS NOT NULL" 2>&1 | tail -1)" # 2>&1, so a missing libxml is named +check "premise: the server supports xml (built with libxml)" "$([ "$xml_ok" = t ] && echo yes || echo "no ($xml_ok)")" "yes" +[ "$xml_ok" = t ] || pgc_summary +make_pair "$COLS" +q "SELECT pgcolumnar.set_options('t_col', stripe_row_limit => 1000)" >/dev/null +# g % k = 0 makes the column NULL; each column has its own k. +load_pair "SELECT g, + CASE WHEN g % 7 = 0 THEN NULL ELSE ((('00:00'::time + (g * 7) * interval '1 s')::text || '+' || lpad((g % 13)::text, 2, '0'))::timetz) END, + CASE WHEN g % 11 = 0 THEN NULL ELSE ('08:00:2b:' || lpad(to_hex(g / 256 % 256), 2, '0') || ':' || lpad(to_hex(g % 256), 2, '0') || ':01')::macaddr END, + CASE WHEN g % 13 = 0 THEN NULL ELSE ('08:00:2b:01:02:03:' || lpad(to_hex(g / 256 % 256), 2, '0') || ':' || lpad(to_hex(g % 256), 2, '0'))::macaddr8 END, + CASE WHEN g % 17 = 0 THEN NULL ELSE ('name_' || lpad(g::text, 6, '0'))::name END, + CASE WHEN g % 19 = 0 THEN NULL ELSE point(g, -g) END, + CASE WHEN g % 23 = 0 THEN NULL ELSE chr(65 + g % 26)::\"char\" END, + CASE WHEN g % 29 = 0 THEN NULL ELSE ('00:00'::time + g * interval '7 s') END, + CASE WHEN g % 31 = 0 THEN NULL ELSE (g * 1.25)::numeric::money END, + CASE WHEN g % 37 = 0 THEN NULL ELSE ('10.' || g / 256 % 256 || '.' || g % 256 || '.' || g % 7 || '/32')::inet END, + CASE WHEN g % 41 = 0 THEN NULL ELSE ('10.' || g / 256 % 256 || '.' || g % 256 || '.0/24')::cidr END, + CASE WHEN g % 43 = 0 THEN NULL ELSE g::bit(16) END, + CASE WHEN g % 47 = 0 THEN NULL ELSE (g::bit(16) || (g % 7)::bit(3))::varbit END, + CASE WHEN g % 53 = 0 THEN NULL ELSE to_tsvector('simple', 'word' || g || ' common term' || g % 9) END, + CASE WHEN g % 59 = 0 THEN NULL ELSE ('' || md5(g::text) || '')::xml END, + CASE WHEN g % 61 = 0 THEN NULL ELSE json_build_object('g', g, 's', md5(g::text)) END, + CASE WHEN g % 67 = 0 THEN NULL ELSE (enum_range(NULL::tm_mood))[1 + g % 4] END, + CASE WHEN g % 71 = 0 THEN NULL ELSE ROW(g, 'p' || g)::tm_pair END, + CASE WHEN g % 73 = 0 THEN NULL ELSE (g)::tm_pos END, + CASE WHEN g % 79 = 0 THEN NULL WHEN g % 1000 = 1 THEN 'NaN'::numeric WHEN g % 1000 = 2 THEN 'Infinity'::numeric + WHEN g % 1000 = 3 THEN '-Infinity'::numeric ELSE g / 7.0 END, + CASE WHEN g % 83 = 0 THEN NULL WHEN g = 500 THEN 'infinity'::timestamp WHEN g = 501 THEN '-infinity'::timestamp + ELSE '2026-01-01'::timestamp + g * interval '1 min' END, + CASE WHEN g % 89 = 0 THEN NULL WHEN g = 600 THEN 'infinity'::date WHEN g = 601 THEN '-infinity'::date + ELSE '2026-01-01'::date + g END + FROM generate_series(1, 12000) g" + +check "premise: both tables hold 12000 rows, the columnar one in at least twelve row groups" \ + "$(q "SELECT count(*) FROM t_heap")|$(q "SELECT count(*) FROM t_col")|$(q "SELECT CASE WHEN count(*) >= 12 THEN 'many' ELSE count(*)::text END + FROM pgcolumnar.row_group WHERE storage_id = pgcolumnar.get_storage_id('t_col')" 2>/dev/null | tail -1)" "12000|12000|many" + +ALL="id, tz::text, mac::text, mac8::text, nm::text, pt::text, ch::text, tm::text, mo::text, ip::text, cd::text, + bt::text, vb::text, tsv::text, x::text, j::text, e::text, cmp::text, dom::text, n::text, tsi::text, d::text" +diff_query "every row, as text, is heap's" "SELECT $ALL FROM %T" +parts=() +for c in tz mac mac8 nm pt ch tm mo ip cd bt vb tsv x j e cmp dom n tsi d; do + parts+=("count($c), count(*) FILTER (WHERE $c IS NULL)") +done +COUNTS="$(IFS=,; echo "${parts[*]}")" +diff_query "each column's count and NULL count is heap's" "SELECT $COUNTS FROM %T" + +# A range on every ordered column. The bounds are the values heap holds at ids 3005 +# and 9005, as literals, so the columnar scan can prune row groups by zone map. Not +# 3001 and 9001: id 3001 holds n's NaN, which made n's "range" NaN..NaN (OffgridwithJD, +# review of #1426). Neither id is NULL in any column. +# +# Each arm also says whether the zone maps skipped chunk groups, from EXPLAIN ANALYZE, +# so an arm that should prune cannot pass on a scan that read everything. Four measure +# 0 of 12 on PG17 and PG18, and are pinned at 0 with the reason: +declare -A NO_PRUNE=( + [ch]="every row group holds every value, since the column cycles through 26 letters" + [e]="every row group holds every value, since the column cycles through the 4 labels" + [n]="every row group holds NaN and both infinities, so each zone map spans every value" + [cd]="a cidr key is treated as cross-type against inet and dropped (#1433)") +removed() { # removed SQL -> the columnar scan's "Chunk Groups Removed by Filter" + q "EXPLAIN (ANALYZE, COSTS OFF, TIMING OFF, SUMMARY OFF) ${1//%T/t_col}" | + sed -n 's/.*Columnar Chunk Groups Removed by Filter: \([0-9]*\).*/\1/p' | head -1 +} +for c in tz mac mac8 nm ch tm mo ip cd bt vb e dom n tsi d; do + # format_type keeps the typmod: pg_typeof would make bt's bounds bit(1) + # (OffgridwithJD, review of #1426). + ty="$(q "SELECT format_type(atttypid, atttypmod) FROM pg_attribute WHERE attrelid = 't_heap'::regclass AND attname = '$c'" | tail -1)" + lo="$(q "SELECT quote_literal($c::text) || '::' || '$ty' FROM t_heap WHERE id = 3005" | tail -1)" + hi="$(q "SELECT quote_literal($c::text) || '::' || '$ty' FROM t_heap WHERE id = 9005" | tail -1)" + sql="SELECT id FROM %T WHERE $c BETWEEN least($lo, $hi) AND greatest($lo, $hi)" + diff_query "a range on $c finds heap's rows" "$sql" + r="$(removed "$sql")" + if [ -n "${NO_PRUNE[$c]:-}" ]; then + check "a range on $c skips no chunk groups: ${NO_PRUNE[$c]}" "${r:-none}" "0" + else + check "a range on $c lets the zone maps skip chunk groups" \ + "$([ -n "$r" ] && [ "$r" -ge 1 ] && echo skips || echo "removed ${r:-none}")" "skips" + fi +done +diff_query "equality on json, as text, finds heap's rows" \ + "SELECT id FROM %T WHERE j::text = (SELECT j::text FROM t_heap WHERE id = 4242)" +diff_query "point equality (~=) finds heap's rows" "SELECT id FROM %T WHERE pt ~= point(4242, -4242)" +diff_query "a tsvector match finds heap's rows" "SELECT id FROM %T WHERE tsv @@ to_tsquery('simple', 'term3')" +diff_query "the aggregates over numeric NaN and Infinity and timestamp and date infinity are heap's" \ + "SELECT min(n)::text, max(n)::text, count(*) FILTER (WHERE n = 'NaN'), count(*) FILTER (WHERE n = 'Infinity'), + count(*) FILTER (WHERE n = '-Infinity'), min(tsi)::text, max(tsi)::text, min(d)::text, max(d)::text FROM %T" +diff_query "the composite's fields and the enum's order read back as heap's" \ + "SELECT (cmp).a, (cmp).b, e, e > 'ok' FROM %T WHERE id % 97 = 0" + +pgc_summary From c1441b4352b8bec546361eb329953772ba643aed Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Thu, 15 Oct 2026 23:07:03 +0000 Subject: [PATCH 12/12] test: each type_matrix range arm requires heap's answer to hold rows, so bounds that come out wrong cannot pass on nothing (#1341) Review of #1426 by OffgridwithJD: with the old pg_typeof bounds put back, bt's range became BETWEEN '0' AND '0', found no rows on either table, and both its arms passed on two empty answers. Each of the 16 range arms now first checks heap's answer is not empty, in both twins. That mutation turns bt's new arm red (0 rows), and nothing else. Co-Authored-By: Claude Opus 5.5 --- test/check_ledger.tsv | 16 ++++++++++++++++ test/check_ledger_budget.txt | 15 +++++++++------ test/pytest/test_type_matrix.py | 4 ++++ test/type_matrix.sh | 6 ++++++ 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 25931eddd..b69d6584f 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -3326,36 +3326,52 @@ ttl_expire ttl_expire with the extent, expire retires the same groups 15;16;17;1 ttl_expire ttl_expire without the extent the same expire probes every group's zone map 15;16;17;18;19 never - ttl_expire ttl_expire without the extent, expire retires the aged groups 15;16;17;18;19 2026-10-14 expire returning 0 whatever it retired type_matrix type_matrix a range on bt finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on bt holds rows on heap, so its comparison reads something 15;16;17;18;19 2026-10-15 the range bounds cast with pg_typeof, which drops the typmod, so bt's become bit(1) and its range finds no rows type_matrix type_matrix a range on bt lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) type_matrix type_matrix a range on cd finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on cd holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on cd skips no chunk groups: a cidr key is treated as cross-type against inet and dropped (#1433) 15;16;17;18;19 never - type_matrix type_matrix a range on ch finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on ch holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on ch skips no chunk groups: every row group holds every value, since the column cycles through 26 letters 15;16;17;18;19 never - type_matrix type_matrix a range on d finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on d holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on d lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) type_matrix type_matrix a range on dom finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on dom holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on dom lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) type_matrix type_matrix a range on e finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on e holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on e skips no chunk groups: every row group holds every value, since the column cycles through the 4 labels 15;16;17;18;19 never - type_matrix type_matrix a range on ip finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on ip holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on ip lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) type_matrix type_matrix a range on mac finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on mac holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on mac lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) type_matrix type_matrix a range on mac8 finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on mac8 holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on mac8 lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) type_matrix type_matrix a range on mo finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on mo holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on mo lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) type_matrix type_matrix a range on n finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on n holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on n skips no chunk groups: every row group holds NaN and both infinities, so each zone map spans every value 15;16;17;18;19 never - type_matrix type_matrix a range on nm finds heap's rows 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen) +type_matrix type_matrix a range on nm holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on nm lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen);the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) type_matrix type_matrix a range on tm finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on tm holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on tm lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) type_matrix type_matrix a range on tsi finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on tsi holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on tsi lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) type_matrix type_matrix a range on tz finds heap's rows 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen) +type_matrix type_matrix a range on tz holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on tz lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) type_matrix type_matrix a range on vb finds heap's rows 15;16;17;18;19 never - +type_matrix type_matrix a range on vb holds rows on heap, so its comparison reads something 15;16;17;18;19 never - type_matrix type_matrix a range on vb lets the zone maps skip chunk groups 15;16;17;18;19 2026-10-15 the scan never skips a row group by zone map (pgcolumnar_native_group_can_match ignored) type_matrix type_matrix a tsvector match finds heap's rows 15;16;17;18;19 never - type_matrix type_matrix each column's count and NULL count is heap's 15;16;17;18;19 2026-10-15 a by-reference fixed-width value over 8 bytes is stored as its first 8 bytes and zero padding (PgColumnarEncodeValueByLen) diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index ad80824b3..86492b7ca 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -2762,7 +2762,7 @@ suites_not_covered 224 # 2557 + 847 == 3404 # # #1341, types the differential matrix never stored answer as heap (jdatcmd): -# 2557 -> 2581. type_matrix enters with all 41 checks from five-major runs (41/41 on +# 2557 -> 2596. type_matrix enters with all 57 checks from five-major runs (57/57 on # each of 15-19). 17 are dated on PG17 by two mutants (OffgridwithJD, reviews of # #1426): # - a by-reference fixed-width value over 8 bytes stored as its first 8 bytes and @@ -2770,11 +2770,14 @@ suites_not_covered 224 # bytes 8-11 matter), the nm range and its pruning, and point equality; # - the scan never skipping a row group by zone map: the 12 range arms that must # skip chunk groups, varbit's included now that its values increase with the row. +# A third, the range bounds cast with pg_typeof (which drops the typmod), dates bt's +# heap-not-empty arm: its range then finds no rows. Each of the 16 range arms has +# one, so no comparison can pass on two empty answers (review of #1426). # The rest arrive never, among them the four range arms pinned at 0 skipped groups, # with the reason in each name (cidr's names #1433), and the libxml premise. # -# ledger rows 3404 + 41 = 3445 -# never 2557 + 24 = 2581 -# dated 847 + 17 = 864 -# 2581 + 864 == 3445 -checks_never_observed_red 2581 +# ledger rows 3404 + 57 = 3461 +# never 2557 + 39 = 2596 +# dated 847 + 18 = 865 +# 2596 + 865 == 3461 +checks_never_observed_red 2596 diff --git a/test/pytest/test_type_matrix.py b/test/pytest/test_type_matrix.py index 9b72e510e..999c7bd23 100644 --- a/test/pytest/test_type_matrix.py +++ b/test/pytest/test_type_matrix.py @@ -118,6 +118,10 @@ def test_types_the_matrix_never_stored_answer_as_heap(pgc_conn, expect): lo, hi = (_rows(pgc_conn, f"SELECT quote_literal({c}::text) || '::' || '{ty}' " f"FROM t_heap WHERE id = {i}")[0][0] for i in (3005, 9005)) sql = f"SELECT id FROM %T WHERE {c} BETWEEN least({lo}, {hi}) AND greatest({lo}, {hi})" + # Heap's answer must not be empty: wrong bounds would make both answers empty. + n = _rows(pgc_conn, f"SELECT count(*) FROM ({sql.replace('%T', 't_heap')}) x")[0][0] + expect.text("yes" if n > 0 else f"no ({n} rows)", "yes", + f"a range on {c} holds rows on heap, so its comparison reads something") got, want = _both(pgc_conn, sql) expect.row_set(got, want, f"a range on {c} finds heap's rows") r = _removed(pgc_conn, sql) diff --git a/test/type_matrix.sh b/test/type_matrix.sh index 3c660af2b..392f2dc4e 100755 --- a/test/type_matrix.sh +++ b/test/type_matrix.sh @@ -106,6 +106,12 @@ for c in tz mac mac8 nm ch tm mo ip cd bt vb e dom n tsi d; do lo="$(q "SELECT quote_literal($c::text) || '::' || '$ty' FROM t_heap WHERE id = 3005" | tail -1)" hi="$(q "SELECT quote_literal($c::text) || '::' || '$ty' FROM t_heap WHERE id = 9005" | tail -1)" sql="SELECT id FROM %T WHERE $c BETWEEN least($lo, $hi) AND greatest($lo, $hi)" + # Heap's answer must not be empty: bounds that came out wrong (a lost typmod made + # bt's BETWEEN '0' AND '0') would make both answers empty, and the comparison and + # the skip arm would pass on nothing (OffgridwithJD, review of #1426). + n="$(q "SELECT count(*) FROM (${sql//%T/t_heap}) x" | tail -1)" + check "a range on $c holds rows on heap, so its comparison reads something" \ + "$([ -n "$n" ] && [ "$n" -gt 0 ] && echo yes || echo "no (${n:-none} rows)")" "yes" diff_query "a range on $c finds heap's rows" "$sql" r="$(removed "$sql")" if [ -n "${NO_PRUNE[$c]:-}" ]; then