diff --git a/CHANGELOG.md b/CHANGELOG.md index 7620decce..d2b779a6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -401,6 +401,14 @@ true until the next version shipped. ### Fixed +- **A join whose runtime-filter build is empty no longer reads the fact table.** When + a large dimension filtered to no row was joined to a columnar table, the hash join + fetched its first fact row before building, and the runtime filter then rejected + every fact row one at a time: every chunk group was read for an answer of no rows. + The coordinator now ends the join as soon as it finds the build empty, since an + inner join with an empty build has no rows (#1447). `test/native_join_runtime_filter.sh` + and `test_join_runtime_filter.py`. + - **A vectorized aggregate no longer shows a non-leakproof function rows that row-level security hides.** Under a policy, a query such as `SELECT grp, count(*) FROM t WHERE f(secret) GROUP BY grp` ran `f` on every row, the other owners' included, through the diff --git a/src/columnar_runtime_filter.c b/src/columnar_runtime_filter.c index ef16c5716..08022ec37 100644 --- a/src/columnar_runtime_filter.c +++ b/src/columnar_runtime_filter.c @@ -898,6 +898,17 @@ PgColumnarExecRuntimeFilter(CustomScanState *node) if (!state->prepared) PgColumnarPrepareRuntimeFilter(state); + /* + * AN EMPTY BUILD HAS NO JOIN ROWS (#1447). The coordinator wraps only inner + * joins, so once the build is drained and found empty, the answer is already + * known. Running the join anyway let the hash join fetch its first outer row + * before building, and the Bloom check then rejected every fact row one at a + * time: the whole fact table read for no row. A rescan resets prepared, so each + * execution decides again. + */ + if (state->buildEmpty) + return NULL; + return ExecProcNode(state->joinState); } diff --git a/test/check_ledger.tsv b/test/check_ledger.tsv index 56255ed35..dbcc327ba 100644 --- a/test/check_ledger.tsv +++ b/test/check_ledger.tsv @@ -3562,3 +3562,10 @@ vm_privilege vm_privilege vm entry point is covered by this boundary test: vm_se vm_privilege vm_privilege vm_is_visible refuses a heap relation even for its owner (#748) 15;16;17;18;19 never - vm_privilege vm_privilege vm_selftest did not report success to the unprivileged role 15;16;17;18;19 2026-10-14 vm_selftest without its owner check vm_privilege vm_privilege vm_selftest refuses a heap relation even for its owner (#748) 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: the large-dimension empty build runs under the coordinator 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: and its build is empty 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter an empty build reads no fact chunk group 15;16;17;18;19 2026-10-16 an inner join whose build was empty ran on, and the fact scan rejected every row one at a time (the tree before #1447) +native_join_runtime_filter native_join_runtime_filter an empty build from a large dimension answers as heap 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: the empty-then-full rescan runs under the coordinator 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter premise: heap's empty-then-full rescan answer holds the 200 keys 15;16;17;18;19 never - +native_join_runtime_filter native_join_runtime_filter an empty build on one rescan does not empty the next 15;16;17;18;19 2026-10-16 the rescan kept the first execution's runtime filter state: prepared not reset on rescan (mutation, #1447) diff --git a/test/check_ledger_budget.txt b/test/check_ledger_budget.txt index 7570954e8..e4c69dd22 100644 --- a/test/check_ledger_budget.txt +++ b/test/check_ledger_budget.txt @@ -2873,4 +2873,17 @@ suites_not_covered 221 # never 2621 + 18 = 2639 # dated 914 + 11 = 925 # 2639 + 925 == 3564 -checks_never_observed_red 2639 +# +# #1447, an empty runtime-filter build made the fact scan read every chunk group +# (jdatcmd): 2639 -> 2644. native_join_runtime_filter gains seven checks from +# five-major runs (62/62 on each of 15-19). 'an empty build reads no fact chunk +# group' is dated on PG17 by the tree before the fix (5 of 5 groups read). 'an +# empty build on one rescan does not empty the next' is dated by the mutation that +# keeps prepared across a rescan (an internal error on the second execution). The +# five others arrive never. +# +# ledger rows 3564 + 7 = 3571 +# never 2639 + 5 = 2644 +# dated 925 + 2 = 927 +# 2644 + 927 == 3571 +checks_never_observed_red 2644 diff --git a/test/native_join_runtime_filter.sh b/test/native_join_runtime_filter.sh index 71cba4fc1..ca1fde0a1 100755 --- a/test/native_join_runtime_filter.sh +++ b/test/native_join_runtime_filter.sh @@ -174,6 +174,16 @@ ORDER BY 1 SQL ) check "rescan answers equal heap" "$(pc "${rf_on}SET max_parallel_workers_per_gather=0;$SQLR" | grep -v '^SET$')" "$(q "$SQLRH")" +# An empty build on one rescan must not carry into the next (#1447). An empty +# build now ends the join at once, so a state left over from the first outer row +# (99999 keeps no key) would answer the second (0 keeps all 200) with no rows. +SQLRE="${SQLR//(VALUES (0),(8000))/(VALUES (99999),(0))}" +SQLRHE="${SQLRH//(VALUES (0),(8000))/(VALUES (99999),(0))}" +check "premise: the empty-then-full rescan runs under the coordinator" \ + "$(pc "${rf_on}SET max_parallel_workers_per_gather=0;EXPLAIN (COSTS OFF) $SQLRE" | grep -c 'Columnar Runtime Filter Coordinator')" 1 +check "premise: heap's empty-then-full rescan answer holds the 200 keys" "$(q "$SQLRHE" | head -1)" "0|200" +check "an empty build on one rescan does not empty the next" \ + "$(pc "${rf_on}SET max_parallel_workers_per_gather=0;$SQLRE" | grep -v '^SET$')" "$(q "$SQLRHE")" # 220000 distinct build keys exceed d*10 vs 2^21, so Bloom must refuse rather # than emit a saturated filter. Fact is larger so it remains the hash outer. @@ -239,7 +249,9 @@ check "3-table join order matches filter-off" "$(printf '%s\n' "$on3" | rel_orde check "3-table answer equals filter-off" "$(pc "${rf_on}SET max_parallel_workers_per_gather=0;SET enable_nestloop=off;SET enable_mergejoin=off;$SQL3"|tail -1)" "$(pc "${rf_off}SET max_parallel_workers_per_gather=0;SET enable_nestloop=off;SET enable_mergejoin=off;$SQL3"|tail -1)" # Empty build: Hash Join still exists, the coordinator still wraps it, and the -# answer is zero rather than a leftover hull from a previous execution. +# answer is zero. With a dimension this small the hash join never runs its outer +# side at all, so this answer cannot fail on what the coordinator does with an +# empty build (#1339); the large-dimension arms below can. q "$(cat <<'SQL' CREATE TABLE dim_none(k int); CREATE TABLE fact_none(k int, payload text) USING pgcolumnar; @@ -254,6 +266,23 @@ SQLN="SELECT count(*),sum(fact_none.k) FROM fact_none JOIN dim_none ON fact_none onn="$(pc "${rf_on}SET max_parallel_workers_per_gather=0;SET enable_nestloop=off;SET enable_mergejoin=off;EXPLAIN(ANALYZE,TIMING off,SUMMARY off)$SQLN")" check "empty-build plan has coordinator" "$(grep -c 'Columnar Runtime Filter Coordinator' <<<"$onn")" 1 check "empty-build answer equals heap" "$(pc "${rf_on}$SQLN"|tail -1)" "$(q "SELECT count(*),sum(heap_none.k) FROM heap_none JOIN dim_none ON heap_none.k=dim_none.k")" +# An empty build from a LARGE dimension filtered to nothing (#1447). Here the hash +# join fetches its first outer row before it builds, so the fact scan runs. The +# coordinator found the build empty, and an inner join with an empty build has no +# rows, so the fact table must not be read. It was read whole: every fact row was +# rejected one at a time (5 of 5 groups read, 5000 rows rejected). +q "$(cat <<'SQL' +CREATE TABLE dim_gone(k int, v int); +INSERT INTO dim_gone SELECT g, g FROM generate_series(1,200000) g; +ANALYZE dim_gone; +SQL +)" >/dev/null +SQLE="SELECT count(*),sum(fact_none.k) FROM fact_none JOIN dim_gone ON fact_none.k=dim_gone.k WHERE dim_gone.v < 0" +one="$(pc "${rf_on}SET max_parallel_workers_per_gather=0;SET enable_nestloop=off;SET enable_mergejoin=off;EXPLAIN(ANALYZE,TIMING off,SUMMARY off)$SQLE")" +check "premise: the large-dimension empty build runs under the coordinator" "$(grep -c 'Columnar Runtime Filter Coordinator' <<<"$one")" 1 +check "premise: and its build is empty" "$(val 'Runtime Filter Build Rows' "$one")" 0 +check "an empty build reads no fact chunk group" "$(val 'Columnar Chunk Groups Read' "$one")" 0 +check "an empty build from a large dimension answers as heap" "$(pc "${rf_on}$SQLE"|tail -1)" "$(q "SELECT count(*),sum(heap_none.k) FROM heap_none JOIN dim_gone ON heap_none.k=dim_gone.k WHERE dim_gone.v < 0")" # Projection-backed outer scans are excluded: a covering projection with a # sort-key restriction is cheaper than the base scan, so Hash Join would wrap diff --git a/test/pytest/TESTS.md b/test/pytest/TESTS.md index ebc40c29d..0c8cd1f85 100644 --- a/test/pytest/TESTS.md +++ b/test/pytest/TESTS.md @@ -3037,6 +3037,21 @@ ERROR and an assert build aborted the backend. Each answer must be heap's, with premises that the same join with no gate takes the coordinator and that heap's true-gate answer holds rows. Independent of the shell arms. +### `test_an_empty_build_reads_no_fact_group` + +An empty build from a large dimension filtered to nothing (#1447). With a dimension +that large, the hash join fetches its first outer row before it builds, so the fact +scan runs. An inner join with an empty build has no rows, so the fact table must not +be read; it was read whole, every row rejected one at a time. Premises check that the +coordinator ran and that the build was empty, and the answer must be heap's. + +### `test_an_empty_build_on_one_rescan_does_not_empty_the_next` + +A LATERAL whose first outer row builds nothing and whose second builds every key +(#1447). An empty build now ends the join at once, so state left over from the first +execution would answer the second with no rows. Premises check the coordinator and +that heap's answer holds the keys. + ## 26. test_check_records.py: every counted assertion is a record #937, first phase. The shell harness makes counting and recording the same call, diff --git a/test/pytest/expected_tests.txt b/test/pytest/expected_tests.txt index 335fed30b..76ea8a89f 100644 --- a/test/pytest/expected_tests.txt +++ b/test/pytest/expected_tests.txt @@ -739,4 +739,6 @@ guard_tests 438 # eleven tests, by collection over the cluster files on this tree. # cluster_tests 638 -> 641 for #1341's test_recovery.py, the twin of recovery.sh: three # tests, by collection over the cluster files on this tree. -cluster_tests 641 +# cluster_tests 641 -> 643 for #1447's two tests in test_join_runtime_filter.py, by +# collection over the cluster files on this tree. +cluster_tests 643 diff --git a/test/pytest/test_join_runtime_filter.py b/test/pytest/test_join_runtime_filter.py index 40b822741..e76f30b35 100644 --- a/test/pytest/test_join_runtime_filter.py +++ b/test/pytest/test_join_runtime_filter.py @@ -769,3 +769,84 @@ def test_a_gating_qual_over_the_join_matches_heap(pgc_conn, expect): heap = cur.fetchone() expect.rows([got], [heap], f"a (SELECT {gate}) gate over a {kind} dimension equals heap") + + +def test_an_empty_build_reads_no_fact_group(pgc_conn, expect): + """An empty build from a large dimension filtered to nothing reads no fact group (#1447). + + With a large dimension the hash join fetches its first outer row before it builds, + so the fact scan runs. The coordinator has found the build empty, and an inner join + with an empty build has no rows, so the fact table must not be read. It was read + whole, every fact row rejected one at a time. Independently of + native_join_runtime_filter.sh: different names, sizes and filter. + """ + with pgc_conn.cursor() as cur: + cur.execute( + """ + CREATE TABLE dbig(k int, w int); + INSERT INTO dbig SELECT g, -g FROM generate_series(1, 150000) g; + CREATE TABLE fz(k int, payload text) USING pgcolumnar; + SELECT pgcolumnar.set_options($t$fz$t$, stripe_row_limit => 1000); + INSERT INTO fz SELECT g, repeat(md5(g::text), 3) FROM generate_series(1, 6000) g; + CREATE TABLE hz(k int, payload text); + INSERT INTO hz SELECT g, repeat(md5(g::text), 3) FROM generate_series(1, 6000) g; + ANALYZE dbig; + ANALYZE fz + """ + ) + sql = ("SELECT count(*), sum(fz.k) FROM fz JOIN dbig ON fz.k = dbig.k " + "WHERE dbig.w > 0") + plan = _plan(pgc_conn, sql, on=True) + expect.num(plan.count("Columnar Runtime Filter Coordinator"), 1, + "premise: the large-dimension empty build runs under the coordinator") + expect.num(_v(plan, "Runtime Filter Build Rows"), 0, "premise: and its build is empty") + expect.num(_v(plan, "Columnar Chunk Groups Read"), 0, "an empty build reads no fact chunk group") + with pgc_conn.cursor() as cur: + cur.execute(sql) + got = cur.fetchone() + cur.execute("SELECT count(*), sum(hz.k) FROM hz JOIN dbig ON hz.k = dbig.k WHERE dbig.w > 0") + heap = cur.fetchone() + expect.rows([got], [heap], "an empty build from a large dimension answers as heap") + + +def test_an_empty_build_on_one_rescan_does_not_empty_the_next(pgc_conn, expect): + """A LATERAL whose first outer row builds nothing and second builds every key (#1447). + + An empty build now ends the join at once, so state left over from the first + execution would answer the second with no rows. Independently of + native_join_runtime_filter.sh: different names and keys. + """ + with pgc_conn.cursor() as cur: + cur.execute( + """ + CREATE TABLE dre(k int); + INSERT INTO dre SELECT 40 + 100 * g FROM generate_series(0, 149) g; + CREATE TABLE fre(k int, payload text) USING pgcolumnar; + SELECT pgcolumnar.set_options($t$fre$t$, stripe_row_limit => 1000); + INSERT INTO fre SELECT g, repeat(md5(g::text), 4) FROM generate_series(1, 16000) g; + CREATE TABLE hre(k int, payload text); + INSERT INTO hre SELECT g, repeat(md5(g::text), 4) FROM generate_series(1, 16000) g; + ANALYZE dre; + ANALYZE fre + """ + ) + if _has_runtime_filter(pgc_conn): + cur.execute("SET pgcolumnar.enable_join_runtime_filter=on") + cur.execute("SET max_parallel_workers_per_gather=0") + lateral = """ + SELECT v.x, s.c FROM (VALUES (88888),(0)) v(x) + CROSS JOIN LATERAL ( + SELECT count(*) c FROM {t} JOIN dre ON {t}.k = dre.k WHERE dre.k > v.x + ) s ORDER BY 1 + """ + cur.execute("EXPLAIN (COSTS OFF) " + lateral.format(t="fre")) + plan = "\n".join(r[0] for r in cur.fetchall()) + cur.execute(lateral.format(t="fre")) + got = cur.fetchall() + cur.execute(lateral.format(t="hre")) + heap = cur.fetchall() + expect.num(plan.count("Columnar Runtime Filter Coordinator"), 1, + "premise: the empty-then-full rescan runs under the coordinator") + expect.text(f"{heap[0][0]}|{heap[0][1]}", "0|150", + "premise: heap's empty-then-full rescan answer holds the 150 keys") + expect.rows(got, heap, "an empty build on one rescan does not empty the next")