From b2c6de5e76be69c54862b1d082225c069c3f2e26 Mon Sep 17 00:00:00 2001 From: "Joshua D. Drake" Date: Fri, 16 Oct 2026 01:24:45 +0000 Subject: [PATCH] fix: a join whose runtime-filter build is empty no longer reads the fact table (#1447) With a large dimension filtered to no row, the hash join fetches its first outer row before it builds the hash table, so the fact scan ran. The runtime filter coordinator had already found the build empty, and the Bloom check rejected every fact row one at a time. Every chunk group was read for an answer of no rows: 5 of 5 groups and 5000 rows rejected, where without the check the scan stopped after one row. The coordinator wraps only inner joins, and an inner join with an empty build has no rows. So it now returns end-of-data as soon as it finds the build empty. A rescan resets prepared, so each execution decides again. Red first, both twins, PG17: on main the large-dimension empty build reads every fact group (shell 5, pytest 6, where 0 are required). A new rescan arm, whose first outer row builds nothing and second builds every key, is red when prepared is kept across a rescan: an internal error in the shell, no rows in pytest. It guards the reset the early return relies on. Found while verifying #1339's weak empty-build entry; the old small-dimension arm can only fail on a crash, because the hash join never runs its outer side, and its comment now says so. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 +++ src/columnar_runtime_filter.c | 11 ++++ test/check_ledger.tsv | 7 +++ test/check_ledger_budget.txt | 15 ++++- test/native_join_runtime_filter.sh | 31 +++++++++- test/pytest/TESTS.md | 15 +++++ test/pytest/expected_tests.txt | 4 +- test/pytest/test_join_runtime_filter.py | 81 +++++++++++++++++++++++++ 8 files changed, 169 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7620decce..d2b779a6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -401,6 +401,14 @@ true until the next version shipped. ### Fixed +- **A join whose runtime-filter build is empty no longer reads the fact table.** When + a large dimension filtered to no row was joined to a columnar table, the hash join + fetched its first fact row before building, and the runtime filter then rejected + every fact row one at a time: every chunk group was read for an answer of no rows. + The coordinator now ends the join as soon as it finds the build empty, since an + inner join with an empty build has no rows (#1447). `test/native_join_runtime_filter.sh` + and `test_join_runtime_filter.py`. + - **A vectorized aggregate no longer shows a non-leakproof function rows that row-level security hides.** Under a policy, a query such as `SELECT grp, count(*) FROM t WHERE f(secret) GROUP BY grp` ran `f` on every row, the other owners' included, through the diff --git a/src/columnar_runtime_filter.c b/src/columnar_runtime_filter.c index ef16c5716..08022ec37 100644 --- a/src/columnar_runtime_filter.c +++ b/src/columnar_runtime_filter.c @@ -898,6 +898,17 @@ PgColumnarExecRuntimeFilter(CustomScanState *node) if (!state->prepared) PgColumnarPrepareRuntimeFilter(state); + /* + * AN EMPTY BUILD HAS NO JOIN ROWS (#1447). The coordinator wraps only inner + * joins, so once the build is drained and found empty, the answer is already + * known. Running the join anyway let the hash join fetch its first outer row + * before building, and the Bloom check then rejected every fact row one at a + * time: the whole fact table read for no row. A rescan resets prepared, so each + * execution decides again. + */ + if (state->buildEmpty) + return NULL; + return ExecProcNode(state->joinState); } diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 56255ed35..dbcc327ba 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -3562,3 +3562,10 @@ vm_privilege vm_privilege vm entry point is covered by this boundary test: vm_se vm_privilege vm_privilege vm_is_visible refuses a heap relation even for its owner (#748) 15;16;17;18;19 never - vm_privilege vm_privilege vm_selftest did not report success to the unprivileged role 15;16;17;18;19 2026-10-14 vm_selftest without its owner check vm_privilege vm_privilege vm_selftest refuses a heap relation even for its owner (#748) 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: the large-dimension empty build runs under the coordinator 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: and its build is empty 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter an empty build reads no fact chunk group 15;16;17;18;19 2026-10-16 an inner join whose build was empty ran on, and the fact scan rejected every row one at a time (the tree before #1447) +native_join_runtime_filter native_join_runtime_filter an empty build from a large dimension answers as heap 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: the empty-then-full rescan runs under the coordinator 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: heap's empty-then-full rescan answer holds the 200 keys 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter an empty build on one rescan does not empty the next 15;16;17;18;19 2026-10-16 the rescan kept the first execution's runtime filter state: prepared not reset on rescan (mutation, #1447) diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 7570954e8..e4c69dd22 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -2873,4 +2873,17 @@ suites_not_covered 221 # never 2621 + 18 = 2639 # dated 914 + 11 = 925 # 2639 + 925 == 3564 -checks_never_observed_red 2639 +# +# #1447, an empty runtime-filter build made the fact scan read every chunk group +# (jdatcmd): 2639 -> 2644. native_join_runtime_filter gains seven checks from +# five-major runs (62/62 on each of 15-19). 'an empty build reads no fact chunk +# group' is dated on PG17 by the tree before the fix (5 of 5 groups read). 'an +# empty build on one rescan does not empty the next' is dated by the mutation that +# keeps prepared across a rescan (an internal error on the second execution). The +# five others arrive never. +# +# ledger rows 3564 + 7 = 3571 +# never 2639 + 5 = 2644 +# dated 925 + 2 = 927 +# 2644 + 927 == 3571 +checks_never_observed_red 2644 diff --git a/test/native_join_runtime_filter.sh b/test/native_join_runtime_filter.sh index 71cba4fc1..ca1fde0a1 100755 --- a/test/native_join_runtime_filter.sh +++ b/test/native_join_runtime_filter.sh @@ -174,6 +174,16 @@ ORDER BY 1 SQL ) check "rescan answers equal heap" "$(pc "${rf_on}SET max_parallel_workers_per_gather=0;$SQLR" | grep -v '^SET$')" "$(q "$SQLRH")" +# An empty build on one rescan must not carry into the next (#1447). An empty +# build now ends the join at once, so a state left over from the first outer row +# (99999 keeps no key) would answer the second (0 keeps all 200) with no rows. +SQLRE="${SQLR//(VALUES (0),(8000))/(VALUES (99999),(0))}" +SQLRHE="${SQLRH//(VALUES (0),(8000))/(VALUES (99999),(0))}" +check "premise: the empty-then-full rescan runs under the coordinator" \ + "$(pc "${rf_on}SET max_parallel_workers_per_gather=0;EXPLAIN (COSTS OFF) $SQLRE" | grep -c 'Columnar Runtime Filter Coordinator')" 1 +check "premise: heap's empty-then-full rescan answer holds the 200 keys" "$(q "$SQLRHE" | head -1)" "0|200" +check "an empty build on one rescan does not empty the next" \ + "$(pc "${rf_on}SET max_parallel_workers_per_gather=0;$SQLRE" | grep -v '^SET$')" "$(q "$SQLRHE")" # 220000 distinct build keys exceed d*10 vs 2^21, so Bloom must refuse rather # than emit a saturated filter. Fact is larger so it remains the hash outer. @@ -239,7 +249,9 @@ check "3-table join order matches filter-off" "$(printf '%s\n' "$on3" | rel_orde check "3-table answer equals filter-off" "$(pc "${rf_on}SET max_parallel_workers_per_gather=0;SET enable_nestloop=off;SET enable_mergejoin=off;$SQL3"|tail -1)" "$(pc "${rf_off}SET max_parallel_workers_per_gather=0;SET enable_nestloop=off;SET enable_mergejoin=off;$SQL3"|tail -1)" # Empty build: Hash Join still exists, the coordinator still wraps it, and the -# answer is zero rather than a leftover hull from a previous execution. +# answer is zero. With a dimension this small the hash join never runs its outer +# side at all, so this answer cannot fail on what the coordinator does with an +# empty build (#1339); the large-dimension arms below can. q "$(cat <<'SQL' CREATE TABLE dim_none(k int); CREATE TABLE fact_none(k int, payload text) USING pgcolumnar; @@ -254,6 +266,23 @@ SQLN="SELECT count(*),sum(fact_none.k) FROM fact_none JOIN dim_none ON fact_none onn="$(pc "${rf_on}SET max_parallel_workers_per_gather=0;SET enable_nestloop=off;SET enable_mergejoin=off;EXPLAIN(ANALYZE,TIMING off,SUMMARY off)$SQLN")" check "empty-build plan has coordinator" "$(grep -c 'Columnar Runtime Filter Coordinator' <<<"$onn")" 1 check "empty-build answer equals heap" "$(pc "${rf_on}$SQLN"|tail -1)" "$(q "SELECT count(*),sum(heap_none.k) FROM heap_none JOIN dim_none ON heap_none.k=dim_none.k")" +# An empty build from a LARGE dimension filtered to nothing (#1447). Here the hash +# join fetches its first outer row before it builds, so the fact scan runs. The +# coordinator found the build empty, and an inner join with an empty build has no +# rows, so the fact table must not be read. It was read whole: every fact row was +# rejected one at a time (5 of 5 groups read, 5000 rows rejected). +q "$(cat <<'SQL' +CREATE TABLE dim_gone(k int, v int); +INSERT INTO dim_gone SELECT g, g FROM generate_series(1,200000) g; +ANALYZE dim_gone; +SQL +)" >/dev/null +SQLE="SELECT count(*),sum(fact_none.k) FROM fact_none JOIN dim_gone ON fact_none.k=dim_gone.k WHERE dim_gone.v < 0" +one="$(pc "${rf_on}SET max_parallel_workers_per_gather=0;SET enable_nestloop=off;SET enable_mergejoin=off;EXPLAIN(ANALYZE,TIMING off,SUMMARY off)$SQLE")" +check "premise: the large-dimension empty build runs under the coordinator" "$(grep -c 'Columnar Runtime Filter Coordinator' <<<"$one")" 1 +check "premise: and its build is empty" "$(val 'Runtime Filter Build Rows' "$one")" 0 +check "an empty build reads no fact chunk group" "$(val 'Columnar Chunk Groups Read' "$one")" 0 +check "an empty build from a large dimension answers as heap" "$(pc "${rf_on}$SQLE"|tail -1)" "$(q "SELECT count(*),sum(heap_none.k) FROM heap_none JOIN dim_gone ON heap_none.k=dim_gone.k WHERE dim_gone.v < 0")" # Projection-backed outer scans are excluded: a covering projection with a # sort-key restriction is cheaper than the base scan, so Hash Join would wrap diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index ebc40c29d..0c8cd1f85 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -3037,6 +3037,21 @@ ERROR and an assert build aborted the backend. Each answer must be heap's, with premises that the same join with no gate takes the coordinator and that heap's true-gate answer holds rows. Independent of the shell arms. +### `test_an_empty_build_reads_no_fact_group` + +An empty build from a large dimension filtered to nothing (#1447). With a dimension +that large, the hash join fetches its first outer row before it builds, so the fact +scan runs. An inner join with an empty build has no rows, so the fact table must not +be read; it was read whole, every row rejected one at a time. Premises check that the +coordinator ran and that the build was empty, and the answer must be heap's. + +### `test_an_empty_build_on_one_rescan_does_not_empty_the_next` + +A LATERAL whose first outer row builds nothing and whose second builds every key +(#1447). An empty build now ends the join at once, so state left over from the first +execution would answer the second with no rows. Premises check the coordinator and +that heap's answer holds the keys. + ## 26. test_check_records.py: every counted assertion is a record #937, first phase. The shell harness makes counting and recording the same call, diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 335fed30b..76ea8a89f 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -739,4 +739,6 @@ guard_tests 438 # eleven tests, by collection over the cluster files on this tree. # cluster_tests 638 -> 641 for #1341's test_recovery.py, the twin of recovery.sh: three # tests, by collection over the cluster files on this tree. -cluster_tests 641 +# cluster_tests 641 -> 643 for #1447's two tests in test_join_runtime_filter.py, by +# collection over the cluster files on this tree. +cluster_tests 643 diff --git a/test/pytest/test_join_runtime_filter.py b/test/pytest/test_join_runtime_filter.py index 40b822741..e76f30b35 100644 --- a/test/pytest/test_join_runtime_filter.py +++ b/test/pytest/test_join_runtime_filter.py @@ -769,3 +769,84 @@ def test_a_gating_qual_over_the_join_matches_heap(pgc_conn, expect): heap = cur.fetchone() expect.rows([got], [heap], f"a (SELECT {gate}) gate over a {kind} dimension equals heap") + + +def test_an_empty_build_reads_no_fact_group(pgc_conn, expect): + """An empty build from a large dimension filtered to nothing reads no fact group (#1447). + + With a large dimension the hash join fetches its first outer row before it builds, + so the fact scan runs. The coordinator has found the build empty, and an inner join + with an empty build has no rows, so the fact table must not be read. It was read + whole, every fact row rejected one at a time. Independently of + native_join_runtime_filter.sh: different names, sizes and filter. + """ + with pgc_conn.cursor() as cur: + cur.execute( + """ + CREATE TABLE dbig(k int, w int); + INSERT INTO dbig SELECT g, -g FROM generate_series(1, 150000) g; + CREATE TABLE fz(k int, payload text) USING pgcolumnar; + SELECT pgcolumnar.set_options($t$fz$t$, stripe_row_limit => 1000); + INSERT INTO fz SELECT g, repeat(md5(g::text), 3) FROM generate_series(1, 6000) g; + CREATE TABLE hz(k int, payload text); + INSERT INTO hz SELECT g, repeat(md5(g::text), 3) FROM generate_series(1, 6000) g; + ANALYZE dbig; + ANALYZE fz + """ + ) + sql = ("SELECT count(*), sum(fz.k) FROM fz JOIN dbig ON fz.k = dbig.k " + "WHERE dbig.w > 0") + plan = _plan(pgc_conn, sql, on=True) + expect.num(plan.count("Columnar Runtime Filter Coordinator"), 1, + "premise: the large-dimension empty build runs under the coordinator") + expect.num(_v(plan, "Runtime Filter Build Rows"), 0, "premise: and its build is empty") + expect.num(_v(plan, "Columnar Chunk Groups Read"), 0, "an empty build reads no fact chunk group") + with pgc_conn.cursor() as cur: + cur.execute(sql) + got = cur.fetchone() + cur.execute("SELECT count(*), sum(hz.k) FROM hz JOIN dbig ON hz.k = dbig.k WHERE dbig.w > 0") + heap = cur.fetchone() + expect.rows([got], [heap], "an empty build from a large dimension answers as heap") + + +def test_an_empty_build_on_one_rescan_does_not_empty_the_next(pgc_conn, expect): + """A LATERAL whose first outer row builds nothing and second builds every key (#1447). + + An empty build now ends the join at once, so state left over from the first + execution would answer the second with no rows. Independently of + native_join_runtime_filter.sh: different names and keys. + """ + with pgc_conn.cursor() as cur: + cur.execute( + """ + CREATE TABLE dre(k int); + INSERT INTO dre SELECT 40 + 100 * g FROM generate_series(0, 149) g; + CREATE TABLE fre(k int, payload text) USING pgcolumnar; + SELECT pgcolumnar.set_options($t$fre$t$, stripe_row_limit => 1000); + INSERT INTO fre SELECT g, repeat(md5(g::text), 4) FROM generate_series(1, 16000) g; + CREATE TABLE hre(k int, payload text); + INSERT INTO hre SELECT g, repeat(md5(g::text), 4) FROM generate_series(1, 16000) g; + ANALYZE dre; + ANALYZE fre + """ + ) + if _has_runtime_filter(pgc_conn): + cur.execute("SET pgcolumnar.enable_join_runtime_filter=on") + cur.execute("SET max_parallel_workers_per_gather=0") + lateral = """ + SELECT v.x, s.c FROM (VALUES (88888),(0)) v(x) + CROSS JOIN LATERAL ( + SELECT count(*) c FROM {t} JOIN dre ON {t}.k = dre.k WHERE dre.k > v.x + ) s ORDER BY 1 + """ + cur.execute("EXPLAIN (COSTS OFF) " + lateral.format(t="fre")) + plan = "\n".join(r[0] for r in cur.fetchall()) + cur.execute(lateral.format(t="fre")) + got = cur.fetchall() + cur.execute(lateral.format(t="hre")) + heap = cur.fetchall() + expect.num(plan.count("Columnar Runtime Filter Coordinator"), 1, + "premise: the empty-then-full rescan runs under the coordinator") + expect.text(f"{heap[0][0]}|{heap[0][1]}", "0|150", + "premise: heap's empty-then-full rescan answer holds the 150 keys") + expect.rows(got, heap, "an empty build on one rescan does not empty the next")