From afcce56873f95a3a8fc788f5fd551accea4e86a0 Mon Sep 17 00:00:00 2001 From: Rishabh Gupta Date: Sun, 1 Nov 2026 03:42:30 +0000 Subject: [PATCH 1/9] fix: avoid pruning timestamp-to-time casts (#26452) ## Which issue does this PR close? - Closes #26451. ## Rationale for this change Casting timestamp statistics to time-of-day values does not preserve ordering across midnight. This can invert the minimum and maximum used by statistics pruning and incorrectly discard files or row groups that contain matching rows. ## What changes are included in this PR? - Prevent timestamp-to-`Time32` and timestamp-to-`Time64` casts from being used for statistics pruning. - Add pruning-expression coverage for both `CAST` and `TRY_CAST` across all supported time units. - Add a Parquet regression test whose timestamp bounds cross midnight. ## What is the testing strategy for this PR? The regression test verifies equality and range predicates for `Time32` and `Time64`, including the expected row counts and pruning metrics. The full extended workspace test suite also passes. ## Are there any user-facing changes? Queries that cast timestamps to time-of-day values may perform less statistics pruning, but they no longer risk incorrect results when timestamp bounds cross a day boundary. (backport of apache/datafusion#26452, 6110f74f5f4c311e55e76d5b02bf3f951c1e4084) Signed-off-by: discord9 (cherry picked from commit 17f5b0ba54232e76d6bb45d1e4b77b9ade691fd7) --- .../core/tests/parquet/row_group_pruning.rs | 49 +++++- datafusion/pruning/src/pruning_predicate.rs | 60 +++++++ .../test_files/push_down_filter_parquet.slt | 163 ++++++++++++++++++ 3 files changed, 270 insertions(+), 2 deletions(-) diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rs index 0721715921909..91195ae04c79c 100644 --- a/datafusion/core/tests/parquet/row_group_pruning.rs +++ b/datafusion/core/tests/parquet/row_group_pruning.rs @@ -20,8 +20,8 @@ //! expected. use std::sync::Arc; -use arrow::array::{ArrayRef, Int32Array, RecordBatch}; -use arrow_schema::{DataType, Field, Schema}; +use arrow::array::{ArrayRef, Int32Array, RecordBatch, TimestampSecondArray}; +use arrow_schema::{DataType, Field, Schema, TimeUnit}; use datafusion::prelude::SessionConfig; use datafusion_common::{DataFusionError, ScalarValue}; use itertools::Itertools; @@ -306,6 +306,51 @@ async fn prune_timestamps_seconds() { .await; } +#[tokio::test] +async fn timestamp_to_time_cast_across_midnight_is_not_pruned() { + let schema = Arc::new(Schema::new(vec![Field::new( + "ts", + DataType::Timestamp(TimeUnit::Second, None), + false, + )])); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(TimestampSecondArray::from(vec![ + 82_800, 86_400, 90_000, + ]))], + ) + .unwrap(); + + for (time_type, op, literal, expected_rows) in [ + ("Time32(Second)", "=", "00:00:00", 1), + ("Time32(Second)", "<", "02:00:00", 2), + ("Time64(Nanosecond)", "=", "00:00:00", 1), + ("Time64(Nanosecond)", "<", "02:00:00", 2), + ] { + let query = format!( + "SELECT ts FROM t WHERE arrow_cast(ts, '{time_type}') {op} \ + arrow_cast('{literal}', '{time_type}')" + ); + + RowGroupPruningTest::new() + .with_scenario(Scenario::Timestamps) + .with_query(&query) + .with_expected_errors(Some(0)) + .with_matched_by_stats(Some(1)) + .with_fully_matched_by_stats(Some(0)) + .with_pruned_by_stats(Some(0)) + .with_pruned_files(Some(0)) + .with_limit_pruned_row_groups(Some(0)) + .with_expected_rows(expected_rows) + .test_row_group_prune_with_custom_data( + Arc::clone(&schema), + vec![batch.clone()], + 3, + ) + .await; + } +} + #[tokio::test] async fn prune_date32() { RowGroupPruningTest::new() diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs index 4735fa375ced8..8f176426fcc70 100644 --- a/datafusion/pruning/src/pruning_predicate.rs +++ b/datafusion/pruning/src/pruning_predicate.rs @@ -1418,6 +1418,16 @@ fn verify_support_type_for_prune(from_type: &DataType, to_type: &DataType) -> Re } _ => to_type, }; + // Casting a timestamp to a time-of-day wraps at midnight and does not + // preserve ordering across day boundaries. Casting timestamp min/max + // statistics could therefore produce invalid bounds and prune matches. + if matches!(from_type, DataType::Timestamp(_, _)) + && matches!(to_type, DataType::Time32(_) | DataType::Time64(_)) + { + return plan_err!( + "Try Cast/Cast with from type {from_type} to type {to_type} is not supported" + ); + } // If both types are strings or both are not strings (number, timestamp, etc) // then we can compare them. // PruningPredicate does not support casting of strings to numbers and such. @@ -5132,6 +5142,56 @@ mod tests { Ok(()) } + #[test] + fn row_group_predicate_cast_timestamp_to_time() { + // A timezone shifts where the day boundary falls but does not remove + // it, so a timezone-aware timestamp is just as unsafe to prune on. + for timestamp_type in [ + DataType::Timestamp(TimeUnit::Second, None), + DataType::Timestamp(TimeUnit::Second, Some("+05:00".into())), + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + ] { + assert_timestamp_to_time_cast_is_not_prunable(timestamp_type); + } + } + + /// Asserts that no comparison of `CAST`/`TRY_CAST` from `timestamp_type` to + /// a time-of-day produces a pruning expression. + fn assert_timestamp_to_time_cast_is_not_prunable(timestamp_type: DataType) { + let schema = Schema::new(vec![Field::new("ts", timestamp_type, false)]); + + for (time_type, literal) in [ + ( + DataType::Time32(TimeUnit::Second), + ScalarValue::Time32Second(Some(0)), + ), + ( + DataType::Time32(TimeUnit::Millisecond), + ScalarValue::Time32Millisecond(Some(0)), + ), + ( + DataType::Time64(TimeUnit::Microsecond), + ScalarValue::Time64Microsecond(Some(0)), + ), + ( + DataType::Time64(TimeUnit::Nanosecond), + ScalarValue::Time64Nanosecond(Some(0)), + ), + ] { + for expr in [ + cast(col("ts"), time_type.clone()).eq(lit(literal.clone())), + try_cast(col("ts"), time_type).eq(lit(literal)), + ] { + let predicate_expr = test_build_predicate_expression( + &expr, + &schema, + &mut RequiredColumns::new(), + ); + assert_eq!(predicate_expr.to_string(), "true"); + } + } + } + #[test] fn row_group_predicate_cast_string_string() -> Result<()> { let schema = Schema::new(vec![Field::new("c1", DataType::Utf8View, false)]); diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt index 72d034067663e..1dd691d9b8967 100644 --- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt +++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt @@ -1336,3 +1336,166 @@ RESET datafusion.execution.parquet.pushdown_filters; statement ok drop table t; + +########## +# Casting a timestamp to a time-of-day wraps at midnight +# +# https://github.com/apache/datafusion/issues/26451 +########## + +# Pruning must never change a query's answer. A cast whose output does not +# preserve the ordering of its input can invert the bounds derived from +# statistics, which makes pruning discard row groups that do contain matching +# rows. Every predicate below is therefore run twice: once against an +# in-memory table, which is never pruned and so gives the ground truth, and +# once against the same rows stored as Parquet. The two must agree. + +# Three timestamps at 23:00, 00:00 and 01:00, written as a single row group so +# that the row group's min/max bracket midnight. Casting those bounds to +# time-of-day turns them into 23:00:00 and 01:00:00, which excludes every value +# that actually lies between them. +statement ok +CREATE TABLE source_data(ts TIMESTAMP) AS VALUES + (TIMESTAMP '2024-03-10 23:00:00'), + (TIMESTAMP '2024-03-11 00:00:00'), + (TIMESTAMP '2024-03-11 01:00:00'); + +statement ok +COPY (SELECT ts FROM source_data) +TO 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps.parquet' +STORED AS PARQUET +OPTIONS ('format.max_row_group_size' '3'); + +statement ok +CREATE EXTERNAL TABLE timestamps +STORED AS PARQUET +LOCATION 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps.parquet'; + +# Equality against the row at midnight. +query P +SELECT ts FROM source_data WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00 + +query P +SELECT ts FROM timestamps WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00 + +# Range predicate spanning the wrap point. +query P +SELECT ts FROM source_data WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00 +2024-03-11T01:00:00 + +query P +SELECT ts FROM timestamps WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00 +2024-03-11T01:00:00 + +# The same predicates spelled with arrow_cast, which also reaches Time32. +query P +SELECT ts FROM source_data +WHERE arrow_cast(ts, 'Time32(Second)') = arrow_cast('00:00:00', 'Time32(Second)') +ORDER BY ts; +---- +2024-03-11T00:00:00 + +query P +SELECT ts FROM timestamps +WHERE arrow_cast(ts, 'Time32(Second)') = arrow_cast('00:00:00', 'Time32(Second)') +ORDER BY ts; +---- +2024-03-11T00:00:00 + +query P +SELECT ts FROM source_data +WHERE arrow_cast(ts, 'Time32(Second)') < arrow_cast('02:00:00', 'Time32(Second)') +ORDER BY ts; +---- +2024-03-11T00:00:00 +2024-03-11T01:00:00 + +query P +SELECT ts FROM timestamps +WHERE arrow_cast(ts, 'Time32(Second)') < arrow_cast('02:00:00', 'Time32(Second)') +ORDER BY ts; +---- +2024-03-11T00:00:00 +2024-03-11T01:00:00 + +statement ok +DROP TABLE timestamps; + +########## +# The same wrap happens at the local midnight of a timezone-aware column +########## + +# The same three local wall-clock times, tagged +05:00, so the instants +# actually stored are 18:00, 19:00 and 20:00 UTC on 2024-03-10. Those are +# monotonic and fall inside a single UTC day, so the min/max bounds only +# invert if the cast honours the column's timezone. +statement ok +CREATE TABLE source_data_tz AS +SELECT arrow_cast(ts, 'Timestamp(Nanosecond, Some("+05:00"))') AS ts +FROM source_data; + +statement ok +DROP TABLE source_data; + +statement ok +COPY (SELECT ts FROM source_data_tz) +TO 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps_tz.parquet' +STORED AS PARQUET +OPTIONS ('format.max_row_group_size' '3'); + +statement ok +CREATE EXTERNAL TABLE timestamps_tz +STORED AS PARQUET +LOCATION 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps_tz.parquet'; + +# Pin the shape of the data this section depends on: the local times have to +# straddle a local midnight while the stored instants stay monotonic within a +# single UTC day. Without this, a change to how a timezone is attached would +# leave the queries below passing while no longer exercising the wrap at all. +query TIT +SELECT + ts::string AS local_time, + to_unixtime(ts) AS stored_instant, + CAST(ts AS TIME)::string AS time_of_day +FROM timestamps_tz ORDER BY ts; +---- +2024-03-10T23:00:00+05:00 1710093600 23:00:00 +2024-03-11T00:00:00+05:00 1710097200 00:00:00 +2024-03-11T01:00:00+05:00 1710100800 01:00:00 + +query P +SELECT ts FROM source_data_tz WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00+05:00 + +query P +SELECT ts FROM timestamps_tz WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00+05:00 + +query P +SELECT ts FROM source_data_tz WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00+05:00 +2024-03-11T01:00:00+05:00 + +query P +SELECT ts FROM timestamps_tz WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00+05:00 +2024-03-11T01:00:00+05:00 + +statement ok +DROP TABLE timestamps_tz; + +statement ok +DROP TABLE source_data_tz; + From a710e344db16af5b308a7c1416dcb35ab7ba255a Mon Sep 17 00:00:00 2001 From: ranflarion Date: Fri, 28 Aug 2026 03:33:45 +0000 Subject: [PATCH 2/9] fix: refuse memory-limited NestedLoopJoin fallback for left-emission joins with a multi-partition probe side (#24675) The memory-limited fallback returns wrong rows for join types whose final emission reads the visited-left bitmap (LEFT, LEFT SEMI, LEFT ANTI, LEFT MARK) when the probe side has more than one partition. The fallback rebuilds a per-partition `JoinLeftData` per left chunk with `probe_threads_counter == 1` and a per-partition bitmap, so every partition believes it is the last to finish probing and emits its own unmatched set, from a bitmap that only saw its own right rows. Unmatched rows are emitted once per partition, and rows matched only in another partition are emitted as unmatched. The existing gate covers only FULL. This extends it to every `need_produce_result_in_final` type, so an over-budget build side fails with `ResourcesExhausted` instead of returning wrong rows. Right-side emission types (RIGHT, RIGHT SEMI, RIGHT ANTI, RIGHT MARK) keep the fallback: each partition owns its right rows exclusively, so their bitmaps are complete per partition. Proper cross-partition coordination of the left bitmap, which would re-enable the fallback for these types, is apache/datafusion#22038. Changes: - `NestedLoopJoinExec::execute`: `full_join_multi_partition` becomes `left_emission_multi_partition = need_produce_result_in_final(join_type) && right_partition_count > 1`, with the comment updated to describe the failure mode. - `test_overallocation` moves LEFT/LEFT SEMI/LEFT ANTI/LEFT MARK from the succeed-via-fallback group into the must-OOM group alongside FULL, and adds single-partition-probe regression tests for LEFT SEMI, LEFT ANTI and LEFT MARK (spill still allowed there). Backport notes: applied by hand because `git cherry-pick -x` is not usable here (the worktree's git directory is not writable from this sandbox). The only structural difference from the upstream diff is that this fork computes `spill_state` inline in `execute` instead of through a `can_spill` local (the fork's fallback is initiated lazily from the stream, so it has no eager `left_spill_manager`); the gate condition and the comments are carried over verbatim. All other hunks (tests) applied byte-identically. (cherry picked from commit c4910e0764f3b02cc27d347bfc117f85a4edb46d) (backport of apache/datafusion#24675, c4910e0764f3b02cc27d347bfc117f85a4edb46d) Signed-off-by: discord9 (cherry picked from commit 3d17c338d4aee595d8b8d074f9c93a2df7693784) --- .../src/joins/nested_loop_join.rs | 173 ++++++++++++++---- 1 file changed, 136 insertions(+), 37 deletions(-) diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs index eb1df638c7dc5..c87760ebe6db2 100644 --- a/datafusion/physical-plan/src/joins/nested_loop_join.rs +++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs @@ -681,22 +681,23 @@ impl ExecutionPlan for NestedLoopJoinExec { // Determine if OOM fallback to memory-limited mode is possible. // Conditions: // 1. Disk manager supports temp files (needed for spilling). - // 2. FULL join with multiple right partitions is not yet supported - // in the fallback path. FULL join needs to track BOTH left-side - // matches (for unmatched left rows) AND right-side matches (for - // unmatched right rows). The fallback path builds a per-partition - // `JoinLeftData` with `probe_threads_counter == 1`, so each - // partition emits unmatched left rows based only on its own - // right-side matches, producing incorrect duplicate output for - // left rows that match in another partition. Other join types - // that need only one-sided final emission (LEFT, LEFT SEMI, - // LEFT ANTI, LEFT MARK) have a similar latent issue in the - // fallback path which predates this change; tracking is out of - // scope for this PR. - let full_join_multi_partition = - matches!(self.join_type, JoinType::Full) && right_partition_count > 1; + // 2. Join types whose final emission reads the visited-left bitmap + // (LEFT, LEFT SEMI, LEFT ANTI, LEFT MARK, FULL) need that bitmap + // complete across every probe partition. The memory-limited path + // builds a per-partition `JoinLeftData` with + // `probe_threads_counter == 1`, so with more than one right + // partition each partition emits from a bitmap that only saw its + // own right rows: unmatched rows are emitted once per partition, + // and rows matched only in another partition are emitted as + // unmatched. Refusing the fallback turns those wrong results into + // a plain ResourcesExhausted error. Right-side emission types are + // safe because each partition owns its right rows exclusively. + // Cross-partition coordination of the left bitmap is tracked in + // https://github.com/apache/datafusion/issues/22038. + let left_emission_multi_partition = + need_produce_result_in_final(self.join_type) && right_partition_count > 1; let spill_state = if context.runtime_env().disk_manager.tmp_files_enabled() - && !full_join_multi_partition + && !left_emission_multi_partition { SpillState::Pending { left_plan: Arc::clone(&self.left), @@ -3728,12 +3729,10 @@ pub(crate) mod tests { // Join types that support memory-limited fallback should succeed // even under tight memory limits (they spill to disk instead of OOM). + // Right-side emission types are safe with multiple right partitions + // because each partition owns its right rows exclusively. let fallback_join_types = vec![ JoinType::Inner, - JoinType::Left, - JoinType::LeftSemi, - JoinType::LeftAnti, - JoinType::LeftMark, JoinType::Right, JoinType::RightSemi, JoinType::RightAnti, @@ -3758,24 +3757,34 @@ pub(crate) mod tests { .await?; } - // FULL JOIN with multiple right partitions is intentionally not - // supported in the fallback path yet (cross-partition left-bitmap - // coordination is missing). It should still OOM under tight memory. - let runtime = RuntimeEnvBuilder::new() - .with_memory_limit(100, 1.0) - .build_arc()?; - let task_ctx = TaskContext::default().with_runtime(runtime); - let task_ctx = Arc::new(task_ctx); - let err = multi_partitioned_join_collect( - Arc::clone(&left), - Arc::clone(&right), - &JoinType::Full, - Some(filter.clone()), - task_ctx, - ) - .await - .unwrap_err(); - assert_contains!(err.to_string(), "Resources exhausted"); + // Join types whose final emission reads the visited-left bitmap are + // intentionally not supported in the fallback path with multiple right + // partitions (cross-partition left-bitmap coordination is missing; + // per-partition bitmaps emit wrong rows). They must OOM cleanly instead. + let gated_join_types = vec![ + JoinType::Left, + JoinType::LeftSemi, + JoinType::LeftAnti, + JoinType::LeftMark, + JoinType::Full, + ]; + for join_type in &gated_join_types { + let runtime = RuntimeEnvBuilder::new() + .with_memory_limit(100, 1.0) + .build_arc()?; + let task_ctx = TaskContext::default().with_runtime(runtime); + let task_ctx = Arc::new(task_ctx); + let err = multi_partitioned_join_collect( + Arc::clone(&left), + Arc::clone(&right), + join_type, + Some(filter.clone()), + task_ctx, + ) + .await + .unwrap_err(); + assert_contains!(err.to_string(), "Resources exhausted"); + } Ok(()) } @@ -3888,6 +3897,96 @@ pub(crate) mod tests { Ok(()) } + #[tokio::test] + async fn test_nlj_memory_limited_left_semi_join() -> Result<()> { + let task_ctx = task_ctx_with_memory_limit(50, 16)?; + let left = build_left_table(); + let right = build_right_table(); + let filter = prepare_join_filter(); + + let (columns, batches, metrics) = + join_collect(left, right, &JoinType::LeftSemi, Some(filter), task_ctx) + .await?; + + assert_eq!(columns, vec!["a1", "b1", "c1"]); + + assert!( + metrics.spill_count().unwrap_or(0) > 0, + "Expected spilling to occur under tight memory limit" + ); + + // Left semi: only left rows that matched at least one right row. + allow_duplicates!(assert_snapshot!(batches_to_sort_string(&batches), @r" + +----+----+----+ + | a1 | b1 | c1 | + +----+----+----+ + | 5 | 5 | 50 | + +----+----+----+ + ")); + Ok(()) + } + + #[tokio::test] + async fn test_nlj_memory_limited_left_anti_join() -> Result<()> { + let task_ctx = task_ctx_with_memory_limit(50, 16)?; + let left = build_left_table(); + let right = build_right_table(); + let filter = prepare_join_filter(); + + let (columns, batches, metrics) = + join_collect(left, right, &JoinType::LeftAnti, Some(filter), task_ctx) + .await?; + + assert_eq!(columns, vec!["a1", "b1", "c1"]); + + assert!( + metrics.spill_count().unwrap_or(0) > 0, + "Expected spilling to occur under tight memory limit" + ); + + // Left anti: left rows that did NOT match any right row. + allow_duplicates!(assert_snapshot!(batches_to_sort_string(&batches), @r" + +----+----+-----+ + | a1 | b1 | c1 | + +----+----+-----+ + | 11 | 8 | 110 | + | 9 | 8 | 90 | + +----+----+-----+ + ")); + Ok(()) + } + + #[tokio::test] + async fn test_nlj_memory_limited_left_mark_join() -> Result<()> { + let task_ctx = task_ctx_with_memory_limit(50, 16)?; + let left = build_left_table(); + let right = build_right_table(); + let filter = prepare_join_filter(); + + let (columns, batches, metrics) = + join_collect(left, right, &JoinType::LeftMark, Some(filter), task_ctx) + .await?; + + assert_eq!(columns, vec!["a1", "b1", "c1", "mark"]); + + assert!( + metrics.spill_count().unwrap_or(0) > 0, + "Expected spilling to occur under tight memory limit" + ); + + // Left mark: all left rows with a bool column indicating match. + allow_duplicates!(assert_snapshot!(batches_to_sort_string(&batches), @r" + +----+----+-----+-------+ + | a1 | b1 | c1 | mark | + +----+----+-----+-------+ + | 11 | 8 | 110 | false | + | 5 | 5 | 50 | true | + | 9 | 8 | 90 | false | + +----+----+-----+-------+ + ")); + Ok(()) + } + #[tokio::test] async fn test_nlj_fits_in_memory_no_spill() -> Result<()> { // Use a large memory limit — everything fits, no spilling needed. From 68a0148831eadafc1be8d9a009acb73b906e0fd8 Mon Sep 17 00:00:00 2001 From: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com> Date: Fri, 4 Sep 2026 16:06:19 +0000 Subject: [PATCH 3/9] fix: RepartitionExec deadlock when its spill pool holds two open files (#24891) Closes https://github.com/apache/datafusion/issues/24883 Under a memory limit, a grouped aggregation on two or more partitions can stop and never complete: when `RepartitionExec` cannot reserve memory for a batch it writes the batch to its spill pool and sends a marker, and the reader then blocks on the pool until it gets a batch. When two input tasks spill at the same time the pool holds two open files. The reader read only the oldest file and, when that file had no unread batch, waited for it, while the batch it needed was in the newer file. Once every channel had data the gate closed and the writers waited too, so nothing could wake the reader. `SpillPoolReader` now keeps every file it received (oldest first) and returns the first available batch. A file that has no unread batch is skipped; the reader still waits on the oldest file that has unread batches whose read is not complete, so a single writer keeps FIFO order. The writer side is unchanged. Also backported: - `test: reproduce the RepartitionExec spill pool deadlock with a SQL query` (`datafusion/core/tests/memory_limit/repartition_mem_limit.rs`), which sums the `spill_count` metric and fails if no attempt spilled. - `test: add a scenario fuzzer for the spill pool contract`, including `test_read_error_is_reported_to_the_reader`, a test that a second writer's batch is readable while the first writer is held, and `spill_pool_scenario_fuzz` (default 50 seeds). - the extended-CI job sets `DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS: 1000`. Backport notes: applied by hand because `git cherry-pick -x` is not usable here (the worktree's git directory is not writable from this sandbox). All three files applied byte-identically; the only fork-specific deltas in the touched regions are pre-existing cosmetic differences (`#[deprecated]` without `since`, `sort_unstable` vs `sort`), which this commit does not touch. (cherry picked from commit 09a2aff72d88c16499ae3266a943a4cd9841b618) (backport of apache/datafusion#24891, 09a2aff72d88c16499ae3266a943a4cd9841b618) Signed-off-by: discord9 (cherry picked from commit ac60580950ee42b7358a74fdf1203a0dc5e84992) --- .github/workflows/extended.yml | 3 + .../memory_limit/repartition_mem_limit.rs | 154 ++++ .../physical-plan/src/spill/spill_pool.rs | 810 ++++++++++++++++-- 3 files changed, 874 insertions(+), 93 deletions(-) diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index a6e303e3d6ff4..ae7408ae771fa 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -88,6 +88,9 @@ jobs: - name: Run tests (excluding doctests) env: RUST_BACKTRACE: 1 + # Run more random scenarios of the spill pool fuzzer than the + # default suite does (about a minute). + DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS: 1000 run: | cargo test \ --profile ci \ diff --git a/datafusion/core/tests/memory_limit/repartition_mem_limit.rs b/datafusion/core/tests/memory_limit/repartition_mem_limit.rs index 27bcd33926e99..21a0eaec5473d 100644 --- a/datafusion/core/tests/memory_limit/repartition_mem_limit.rs +++ b/datafusion/core/tests/memory_limit/repartition_mem_limit.rs @@ -117,3 +117,157 @@ async fn test_repartition_memory_limit() { ]; assert_batches_sorted_eq!(expected, &all_batches); } + +/// Regression test for . +/// +/// In non-preserve-order mode every input task of a `RepartitionExec` shares one +/// spill pool per output partition. When two input tasks spilled concurrently the +/// pool could end up with two open spill files while the reader only ever drained +/// the head file: it parked on a drained-but-unfinished head file even though the +/// batch it was waiting for had been written to the second file. Once every +/// distributor channel was non-empty the gate closed, both input tasks parked in +/// `send`, no sink was dropped, and nothing could wake anybody. +/// +/// This module is not compiled with `force_hash_collisions`. That feature makes +/// every hash 0, so the hash `RepartitionExec` sends all rows to one output +/// partition. The gate closes only when every channel holds data, so the +/// deadlock cannot occur and the test cannot test anything. The feature also +/// makes the group-by hash table degenerate, which made one attempt of this +/// query approximately 36 times slower and thus longer than the limit below. +#[cfg(not(feature = "force_hash_collisions"))] +mod spill_pool_deadlock { + use super::*; + + use std::time::Duration; + + use datafusion::execution::session_state::SessionStateBuilder; + use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; + use datafusion_execution::memory_pool::{GreedyMemoryPool, MemoryPool}; + use datafusion_physical_plan::collect; + + /// Memory limit that puts `RepartitionExec` into its spilling path for this + /// query without failing the aggregation outright. + const MEMORY_LIMIT: usize = 4 * 1024 * 1024; + + /// Number of attempts. On an unfixed tree the first attempt deadlocks; the + /// budget is only there so a fix cannot pass by luck. + const ATTEMPTS: usize = 12; + + /// Generous per-attempt budget: a healthy run of this query takes well under + /// a second. + const PER_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(20); + + /// Runs the query once. Returns the number of output rows and the number + /// of spills the `RepartitionExec`s did. + async fn run_once() -> datafusion::error::Result<(usize, usize)> { + let pool: Arc = Arc::new(GreedyMemoryPool::new(MEMORY_LIMIT)); + let runtime = RuntimeEnvBuilder::new() + // Spilling must be possible: with the disk manager disabled the query + // fails with a resource error instead of deadlocking. + .with_disk_manager_builder( + DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory), + ) + .with_memory_pool(pool) + .build_arc()?; + + let config = SessionConfig::new() + // Two input partitions feeding one hash RepartitionExec is the + // smallest configuration with two concurrent writers per spill pool. + .with_target_partitions(2) + // Small batches so many small batches reach the repartition spill + // path while the spill file stays far below + // `max_spill_file_size_bytes` and so never rotates. + .with_batch_size(64); + + let state = SessionStateBuilder::new() + .with_config(config) + .with_runtime_env(runtime) + .with_default_features() + .build(); + let ctx = SessionContext::new_with_state(state); + + ctx.sql( + "create table trace_events as + select v % 13 as g, + case when v % 29 = 0 then null + else md5(cast(v % 337 as varchar)) end as trace_id + from generate_series(1, 40000) as t(v)", + ) + .await? + .collect() + .await?; + + ctx.sql( + "create view tv as + select g, arrow_cast(trace_id, 'Utf8View') as trace_id from trace_events", + ) + .await? + .collect() + .await?; + + let logical = ctx + .state() + .create_logical_plan( + "select g, count(distinct trace_id) as n from tv group by g", + ) + .await?; + let plan = ctx.state().create_physical_plan(&logical).await?; + let batches = collect(Arc::clone(&plan), ctx.task_ctx()).await?; + + // Count the spills of the `RepartitionExec`s. The caller uses this to + // make sure the query still goes through the spill path: if a change + // stops it spilling, this test keeps passing but no longer covers the + // deadlock, and that must fail loudly instead. + let mut spills = 0; + plan.transform_down(|node| { + if node.is::() { + spills += node.metrics().and_then(|m| m.spill_count()).unwrap_or(0); + } + Ok(Transformed::no(node)) + })?; + + Ok((batches.iter().map(|b| b.num_rows()).sum(), spills)) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 8)] + async fn repartition_spill_pool_does_not_deadlock() { + let mut attempts_that_spilled = 0; + for attempt in 0..ATTEMPTS { + match tokio::time::timeout(PER_ATTEMPT_TIMEOUT, run_once()).await { + Err(elapsed) => panic!( + "attempt {attempt}: grouped COUNT(DISTINCT Utf8View) never completed \ + within {PER_ATTEMPT_TIMEOUT:?} ({elapsed}); RepartitionExec spill \ + pool deadlock" + ), + Ok(Ok((rows, spills))) => { + assert_eq!(rows, 13, "attempt {attempt}: wrong row count"); + if spills > 0 { + attempts_that_spilled += 1; + } + } + // The budget is deliberately far too small for the query, and the + // greedy pool hands memory out first come first served, so once in + // a few thousand attempts an aggregate's allocation is refused + // while the repartition reservations hold the pool. That is a + // legitimate outcome of the limit, not the hang this test guards + // against. + Ok(Err(e)) + if matches!( + e.find_root(), + datafusion::error::DataFusionError::ResourcesExhausted(_) + ) => {} + Ok(Err(e)) => panic!("attempt {attempt}: query failed: {e}"), + } + } + + // The memory limit must be tight enough that `RepartitionExec` spills, + // because the deadlock is in its spill pool. Timing decides how many + // attempts spill, but if none of them do, the limit no longer forces + // the spill path and this test has stopped testing anything. + assert!( + attempts_that_spilled > 0, + "no attempt reached the RepartitionExec spill path in {ATTEMPTS} \ + attempts; the memory limit no longer forces a spill" + ); + } +} diff --git a/datafusion/physical-plan/src/spill/spill_pool.rs b/datafusion/physical-plan/src/spill/spill_pool.rs index 6e964d7a6497b..86e3e3b6cd8da 100644 --- a/datafusion/physical-plan/src/spill/spill_pool.rs +++ b/datafusion/physical-plan/src/spill/spill_pool.rs @@ -48,10 +48,11 @@ use super::spill_manager::SpillManager; /// **Lock ordering discipline**: Never hold both locks simultaneously to prevent deadlock. /// Always: acquire outer lock → release outer lock → acquire inner lock (if needed). struct SpillPoolShared { - /// Queue of ALL files (including the current write files if any exist). - /// Readers always read from the front of this queue (FIFO). - /// Each file has its own lock to enable concurrent reader/writer access. - files: VecDeque>>, + /// Files created by writers that the reader has not picked up yet, in creation + /// order. The reader moves them into its own list on every poll (see + /// [`SpillPoolReader`]), so this queue only ever holds files the reader has not + /// seen. Each file has its own lock to enable concurrent reader/writer access. + new_files: VecDeque>>, /// SpillManager for creating files and tracking metrics spill_manager: Arc, /// Pool-level waker to notify when new files are available (single reader) @@ -70,7 +71,7 @@ impl SpillPoolShared { /// Creates a new shared pool state fn new(spill_manager: Arc) -> Self { Self { - files: VecDeque::new(), + new_files: VecDeque::new(), spill_manager, waker: None, open_write_files: VecDeque::new(), @@ -237,7 +238,7 @@ impl SpillPoolSink { // Re-acquire lock and push to shared queue shared = self.shared.lock(); - shared.files.push_back(Arc::clone(&file_shared)); + shared.new_files.push_back(Arc::clone(&file_shared)); shared.wake(); // Wake readers waiting for new files file_shared }; @@ -570,17 +571,34 @@ struct SpillPoolFile { spill_manager: Arc, } -impl Stream for SpillPoolFile { - type Item = Result; +/// Outcome of polling a single file of the pool, see [`SpillPoolFile::poll_file`]. +enum FilePoll { + /// A batch was read from the file, or reading it failed. + Item(Result), + /// The file's stream ended: every batch written to it has been read and the + /// writer has finished it (the caller verifies the latter). + Done, + /// Every batch written so far has been read, but the writer has not finished + /// the file. The file's waker is registered, so the task is woken when the + /// writer appends another batch or finishes the file. + CaughtUp, + /// The file has unread batches, but the underlying I/O is not ready yet. The + /// I/O has registered the task's waker. + Pending, +} - fn poll_next( - mut self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { +impl SpillPoolFile { + /// Polls the file for its next batch. + /// + /// Unlike a plain stream poll this tells the caller *why* no batch is + /// available, so [`SpillPoolReader`] can move on to a newer file when this + /// one is only waiting for its writer, but keeps waiting on it while its + /// data is still being read from disk (which keeps batches in file order). + fn poll_file(&mut self, cx: &mut std::task::Context<'_>) -> FilePoll { use std::task::Poll; // Step 1: Lock shared state and check coordination - let (should_read, file) = { + let file = { let mut shared = self.shared.lock(); // Determine if we can read @@ -588,24 +606,23 @@ impl Stream for SpillPoolFile { if batches_read < shared.batches_written { // More data available to read - take the file if we don't have a reader yet - let file = if self.reader.is_none() { + if self.reader.is_none() { shared.file.take() } else { None - }; - (true, file) + } } else if shared.writer_finished { // No more data and writer is done - EOF - return Poll::Ready(None); + return FilePoll::Done; } else { // Caught up to writer, but writer still active - register waker and wait shared.register_waker(cx.waker().clone()); - return Poll::Pending; + return FilePoll::CaughtUp; } }; // Lock released here // Step 2: Lazy-create reader stream if needed - if self.reader.is_none() && should_read { + if self.reader.is_none() { if let Some(file) = file { // we want this unbuffered because files are actively being written to match self @@ -618,36 +635,35 @@ impl Stream for SpillPoolFile { batches_read: 0, }); } - Err(e) => return Poll::Ready(Some(Err(e))), + Err(e) => return FilePoll::Item(Err(e)), } } else { // File not available yet (writer hasn't finished or already taken) // Register waker and wait for file to be ready let mut shared = self.shared.lock(); shared.register_waker(cx.waker().clone()); - return Poll::Pending; + return FilePoll::CaughtUp; } } // Step 3: Poll the reader stream (no lock held) - if let Some(reader) = &mut self.reader { - match reader.stream.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(batch))) => { - // Successfully read a batch - increment counter - reader.batches_read += 1; - Poll::Ready(Some(Ok(batch))) - } - Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))), - Poll::Ready(None) => { - // Stream exhausted unexpectedly - // This shouldn't happen if coordination is correct, but handle gracefully - Poll::Ready(None) - } - Poll::Pending => Poll::Pending, - } - } else { + let Some(reader) = &mut self.reader else { // Should not reach here, but handle gracefully - Poll::Ready(None) + return FilePoll::Done; + }; + match reader.stream.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(batch))) => { + // Successfully read a batch - increment counter + reader.batches_read += 1; + FilePoll::Item(Ok(batch)) + } + Poll::Ready(Some(Err(e))) => FilePoll::Item(Err(e)), + Poll::Ready(None) => { + // Stream exhausted unexpectedly + // This shouldn't happen if coordination is correct, but handle gracefully + FilePoll::Done + } + Poll::Pending => FilePoll::Pending, } } } @@ -669,11 +685,31 @@ impl Stream for SpillPoolFile { /// /// This makes it suitable for continuous streaming scenarios where the writer may /// produce data intermittently. +/// +/// # Reading from several open files +/// +/// With a single writer ([`spsc_channel`]) at most one file is open for writing at +/// a time and every earlier file is finished, so reading the files oldest first is +/// exactly FIFO. With several writers ([`mpsc_channel`]) concurrent pushes can +/// leave more than one file open, and a batch may land in a newer file while the +/// oldest file is drained but not finished. The reader therefore polls every file +/// it knows about, oldest first, and returns the first batch that is available. +/// A file that is merely caught up with its writer is skipped; the reader only +/// waits on a file while that file has unread batches whose bytes are still being +/// read from disk, which keeps batches in file order (strict FIFO for one writer). +/// +/// This matters for [`RepartitionExec`](crate::repartition::RepartitionExec), which +/// sends a "spilled" marker through its channel after every `push_batch` and then +/// blocks on this stream until it yields a batch. If the reader only ever drained the +/// oldest file, it could wait for a batch that had been written to a newer file; with +/// the channel gate closed the writers could not push anything to wake it, so the +/// query would deadlock (). pub struct SpillPoolReader { /// Shared reference to the spill pool shared: Arc>, - /// Current SpillPoolFile we're reading from - current_file: Option, + /// Files this reader has picked up from the pool and not fully consumed yet, in + /// creation order. Each carries its own lazily-created stream and read position. + files: VecDeque, /// Schema of the spilled data schema: SchemaRef, } @@ -689,7 +725,7 @@ impl SpillPoolReader { fn new(shared: Arc>, schema: SchemaRef) -> Self { Self { shared, - current_file: None, + files: VecDeque::new(), schema, } } @@ -704,75 +740,81 @@ impl Stream for SpillPoolReader { ) -> std::task::Poll> { use std::task::Poll; + // `Self: Unpin`; reborrow once so `files` and `shared` can be used together + let this = &mut *self; + loop { - // If we have a current file, try to read from it - if let Some(ref mut file) = self.current_file { - match file.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(batch))) => { - // Got a batch, return it - return Poll::Ready(Some(Ok(batch))); - } - Poll::Ready(Some(Err(e))) => { - // Error reading batch - return Poll::Ready(Some(Err(e))); + // Pick up the files writers have created since the last poll, in + // creation order. + { + let mut shared = this.shared.lock(); + while let Some(file_shared) = shared.new_files.pop_front() { + let spill_manager = Arc::clone(&shared.spill_manager); + this.files.push_back(SpillPoolFile { + shared: file_shared, + reader: None, + spill_manager, + }); + } + } // Lock released here + + // Poll the files oldest first and return the first available batch. + // A file that is finished and fully read is dropped, which releases + // its disk space. A file that is only waiting for its writer has + // registered its own waker, so we move on to the next file instead + // of waiting on it. + let mut idx = 0; + while idx < this.files.len() { + match this.files[idx].poll_file(cx) { + FilePoll::Item(item) => { + // Got a batch (or an error), return it + return Poll::Ready(Some(item)); } - Poll::Ready(None) => { - // Current file stream exhausted + FilePoll::Done => { + // File stream exhausted // Check if this file is marked as writer_finished - let writer_finished = { file.shared.lock().writer_finished }; + let writer_finished = + { this.files[idx].shared.lock().writer_finished }; if writer_finished { - // File is complete, pop it from the queue and move to next - let mut shared = self.shared.lock(); - shared.files.pop_front(); - drop(shared); // Release lock - - // Clear current file and continue loop to get next file - self.current_file = None; - continue; + // File is complete, drop it and move on to the next + this.files.remove(idx); } else { // Stream exhausted but writer not finished - unexpected // This shouldn't happen with proper coordination return Poll::Ready(None); } } - Poll::Pending => { - // File not ready yet (waiting for writer) - // Register waker so we get notified when writer adds more batches - let mut shared = self.shared.lock(); - shared.register_waker(cx.waker().clone()); + FilePoll::Pending => { + // The oldest file with unread batches is still being read + // from disk. Wait for it rather than skipping ahead, so + // batches come back in file order (FIFO for one writer). return Poll::Pending; } + FilePoll::CaughtUp => { + // Nothing to read here until its writer appends more; a + // newer file may have unread batches, so try the next one + idx += 1; + } } } - // No current file, need to get the next one - let mut shared = self.shared.lock(); - - // Peek at the front of the queue (don't pop yet) - if let Some(file_shared) = shared.files.front() { - // Create a SpillPoolFile from the shared state - let spill_manager = Arc::clone(&shared.spill_manager); - let file_shared = Arc::clone(file_shared); - drop(shared); // Release lock before creating SpillPoolFile - - self.current_file = Some(SpillPoolFile { - shared: file_shared, - reader: None, - spill_manager, - }); + // No known file has an unread batch + let mut shared = this.shared.lock(); - // Continue loop to poll the new file + if !shared.new_files.is_empty() { + // A writer created a file while we were polling; pick it up continue; } - // No files in queue - check if writer is done - if shared.remaining_writer_count == 0 { + if this.files.is_empty() && shared.remaining_writer_count == 0 { // Writer is done and no more files will be added - EOF return Poll::Ready(None); } - // Writer still active, register waker that will get notified when new files are added + // Writer still active: register the pool-level waker so we are + // notified when a new file is created or the last writer is dropped. + // Each pending file has registered its own waker for new batches. shared.register_waker(cx.waker().clone()); return Poll::Pending; } @@ -791,8 +833,20 @@ mod tests { use crate::metrics::{ExecutionPlanMetricsSet, SpillMetrics}; use arrow::array::{ArrayRef, Int32Array}; use arrow::datatypes::{DataType, Field, Schema}; + use datafusion_common::instant::Instant; + use datafusion_common::{DataFusionError, exec_datafusion_err}; use datafusion_common_runtime::{JoinSet, SpawnedTask}; + use datafusion_execution::disk_manager::{ + DiskManager, DiskManagerBuilder, DiskManagerMode, + }; use datafusion_execution::runtime_env::RuntimeEnv; + use datafusion_execution::runtime_env::RuntimeEnvBuilder; + use datafusion_execution::{SpillFile, SpillWriter, TempFileFactory}; + use std::pin::Pin; + use std::sync::Barrier; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::mpsc::{self, Sender}; + use std::time::Duration; fn create_test_schema() -> SchemaRef { Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])) @@ -919,8 +973,7 @@ mod tests { // Reader should pend since no batches were written let mut reader = reader; let result = - tokio::time::timeout(std::time::Duration::from_millis(100), reader.next()) - .await; + tokio::time::timeout(Duration::from_millis(100), reader.next()).await; assert!(result.is_err(), "Reader should timeout on empty writer"); @@ -1230,7 +1283,7 @@ mod tests { let batch = create_test_batch(i * 10, 10); writer.push_batch(&batch).unwrap(); // Small delay to simulate real concurrent work - tokio::time::sleep(std::time::Duration::from_millis(5)).await; + tokio::time::sleep(Duration::from_millis(5)).await; } }); @@ -1523,10 +1576,9 @@ mod tests { proceed_tx.send(()).unwrap(); // The reader should wait (Pending) for writer2's data, not EOF. - let batch2 = - tokio::time::timeout(std::time::Duration::from_secs(5), reader.next()) - .await - .expect("Reader timed out — should not hang"); + let batch2 = tokio::time::timeout(Duration::from_secs(5), reader.next()) + .await + .expect("Reader timed out — should not hang"); assert!( batch2.is_some(), @@ -1645,4 +1697,576 @@ mod tests { Ok(()) } + + type WriteHook = Arc; + /// What a read of a spill file does. The two failure modes reach the two + /// different error paths of [`SpillPoolFile::poll_file`]: one fails while it + /// builds the stream, the other fails while it polls a stream that is + /// already built. + enum ReadBehavior { + /// Read the real bytes, after this delay. + Delay(Duration), + /// Fail to open the file, so the stream is never built. + FailOpen(DataFusionError), + /// Open the file, then give this error as the first item of the stream. + FailFirstItem(DataFusionError), + } + + type ReadHook = Arc ReadBehavior + Send + Sync>; + + /// Test double for a spill file that runs `write_hook` before every disk + /// write or flush, and consults `read_hook` before every read. Writers write + /// while holding their file's lock, so a hook that blocks pauses a + /// `push_batch` at exactly the point where it has a file checked out for + /// writing; a read delay makes an older file's bytes arrive after a newer + /// file's, and a read error fails the read. Data still goes to real + /// temporary files. + struct IoHookFactory { + inner: Arc, + write_hook: WriteHook, + read_hook: ReadHook, + } + + impl TempFileFactory for IoHookFactory { + fn create_temp_file(&self, description: &str) -> Result> { + Ok(Arc::new(IoHookFile { + inner: self.inner.create_tmp_file(description)?, + write_hook: Arc::clone(&self.write_hook), + read_hook: Arc::clone(&self.read_hook), + })) + } + } + + struct IoHookFile { + inner: Arc, + write_hook: WriteHook, + read_hook: ReadHook, + } + + impl SpillFile for IoHookFile { + fn path(&self) -> Option<&std::path::Path> { + self.inner.path() + } + + fn size(&self) -> Option { + self.inner.size() + } + + fn read_stream( + &self, + ) -> Result> + Send>>> { + let delay = match (self.read_hook)() { + ReadBehavior::Delay(delay) => delay, + ReadBehavior::FailOpen(e) => return Err(e), + ReadBehavior::FailFirstItem(e) => { + return Ok(Box::pin(futures::stream::once(async move { Err(e) }))); + } + }; + let mut inner = Some(self.inner.read_stream()?); + Ok(Box::pin( + futures::stream::once(tokio::time::sleep(delay)) + .flat_map(move |_| inner.take().expect("polled once")), + )) + } + + fn open_writer(&self) -> Result> { + Ok(Box::new(IoHookWriter { + inner: self.inner.open_writer()?, + hook: Arc::clone(&self.write_hook), + })) + } + } + + struct IoHookWriter { + inner: Box, + hook: WriteHook, + } + + impl std::io::Write for IoHookWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + (self.hook)(); + self.inner.write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + (self.hook)(); + self.inner.flush() + } + } + + impl SpillWriter for IoHookWriter { + fn finish(&mut self) -> Result<()> { + self.inner.finish() + } + } + + /// A `SpillManager` whose files run `write_hook` before every disk write and + /// consult `read_hook` before every read, plus the `DiskManager` that owns + /// the files so tests can check disk usage. + fn spill_manager_with_io_hooks( + write_hook: WriteHook, + read_hook: ReadHook, + ) -> Result<(Arc, Arc)> { + let disk_manager = Arc::new(DiskManagerBuilder::default().build()?); + let runtime = RuntimeEnvBuilder::new() + .with_disk_manager_builder(DiskManagerBuilder::default().with_mode( + DiskManagerMode::Custom(Arc::new(IoHookFactory { + inner: Arc::clone(&disk_manager), + write_hook, + read_hook, + })), + )) + .build_arc()?; + let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0); + let spill_manager = + Arc::new(SpillManager::new(runtime, metrics, create_test_schema())); + Ok((spill_manager, disk_manager)) + } + + /// Holds the first disk write on the pool until the test releases it, and + /// reports every other write so a test can wait for a second writer to + /// reach a file of its own instead of sleeping. + struct WriteGate { + taken: AtomicBool, + entered: Barrier, + release: Barrier, + other_writes: Sender<()>, + } + + impl WriteGate { + fn hold_if_first(&self) { + if self.taken.swap(true, Ordering::SeqCst) { + // A write only gets here once its writer holds a file of its + // own: a writer that had to wait for the held writer's file + // lock would not reach a write of its own at all. + let _ = self.other_writes.send(()); + } else { + self.entered.wait(); + self.release.wait(); + } + } + } + + /// Regression test for . + /// + /// Two writers push concurrently. The first writer is held inside its first + /// write to disk (holding its file's lock) while the second writer pushes. + /// On the unfixed pool that leaves two open files with one batch each, and + /// the reader, which only ever drained the oldest file, parked on that file + /// once it had read its single batch even though the second batch was on + /// disk. `RepartitionExec` blocks on this stream once per pushed batch and, + /// with its channel gate closed, cannot push anything else to wake it. + /// + /// The reader must yield both batches while both writers are still alive. + #[tokio::test] + async fn test_reader_does_not_wait_on_drained_file_while_another_has_data() + -> Result<()> { + let (other_writes_tx, other_writes) = mpsc::channel(); + let gate = Arc::new(WriteGate { + taken: AtomicBool::new(false), + entered: Barrier::new(2), + release: Barrier::new(2), + other_writes: other_writes_tx, + }); + let write_hook: WriteHook = { + let gate = Arc::clone(&gate); + Arc::new(move || gate.hold_if_first()) + }; + let (spill_manager, _disk_manager) = spill_manager_with_io_hooks( + write_hook, + Arc::new(|| ReadBehavior::Delay(Duration::ZERO)), + )?; + + let (writer1, mut reader) = mpsc_channel(1024 * 1024, Arc::clone(&spill_manager)); + let writer2 = writer1.clone(); + + // Writer 1 pushes and is held inside its first disk write, holding its + // file's lock. The writers run on plain threads because `push_batch` + // blocks. + let writer1 = std::thread::spawn(move || { + writer1.push_batch(&create_test_batch(0, 10)).unwrap(); + writer1 + }); + gate.entered.wait(); + + // Writer 2 pushes while writer 1 is held. + let writer2 = std::thread::spawn(move || { + writer2.push_batch(&create_test_batch(10, 10)).unwrap(); + writer2 + }); + // Wait for writer 2 to reach a write of its own, so that writer 1 is + // only released once the pool really holds two open files. Sleeping + // instead would let a loaded runner schedule writer 2 after writer 1 + // was released, and writer 2 would then reuse writer 1's returned file: + // a single-file run that says nothing about the case under test. + other_writes + .recv_timeout(Duration::from_secs(30)) + .expect("writer 2 must write to a file of its own while writer 1 is held"); + gate.release.wait(); + let writer1 = writer1.join().unwrap(); + let writer2 = writer2.join().unwrap(); + assert_eq!( + spill_manager.metrics.spill_file_count.value(), + 2, + "the writers must have written to two different files" + ); + + // Both batches are on disk and both writers are still alive, so no file + // is finished. The reader must yield both batches without waiting for a + // writer. + let mut values = vec![]; + for _ in 0..2 { + let batch = tokio::time::timeout(Duration::from_secs(5), reader.next()) + .await + .expect( + "reader waited on a drained file while another file had an unread batch", + ) + .expect("reader must not signal EOF while writers are alive")?; + assert_eq!(batch.num_rows(), 10); + values.push(id_of(&batch)); + } + // Multiple writers: the order between their batches is not guaranteed. + values.sort_unstable(); + assert_eq!(values, vec![0, 10]); + + // Only once every writer is gone does the reader signal EOF. + drop(writer1); + drop(writer2); + assert!(reader.next().await.is_none()); + + Ok(()) + } + + /// A spill file that cannot be read must fail the reader rather than stall + /// it. `RepartitionExec` blocks on this stream after every spilled marker, + /// so an error that is swallowed here would hang the query exactly like the + /// deadlock this module guards against. + /// + /// Both error paths of `SpillPoolFile::poll_file` are covered: the file that + /// does not open, so the failure comes from the construction of the stream, + /// and the file that opens and then gives an error, so the failure comes + /// from a poll of a stream that the reader already holds. + #[tokio::test] + async fn test_read_errors_are_reported_to_the_reader() -> Result<()> { + async fn assert_reader_reports(behavior: fn() -> ReadBehavior) -> Result<()> { + let (spill_manager, _disk_manager) = + spill_manager_with_io_hooks(Arc::new(|| {}), Arc::new(behavior))?; + let (writer, mut reader) = spsc_channel(1024 * 1024, spill_manager); + + // Nothing reads the file until the reader does, so every read that + // the pool makes is a read of the batch below. + writer.push_batch(&create_test_batch(0, 10))?; + + let item = tokio::time::timeout(Duration::from_secs(5), reader.next()) + .await + .expect("reader must report the read error instead of waiting") + .expect("reader must report the read error instead of signalling EOF"); + let err = item.expect_err("a failed read must not produce a batch"); + assert!( + err.to_string().contains("injected spill read failure"), + "unexpected error: {err}" + ); + Ok(()) + } + + assert_reader_reports(|| { + ReadBehavior::FailOpen(exec_datafusion_err!("injected spill read failure")) + }) + .await?; + assert_reader_reports(|| { + ReadBehavior::FailFirstItem(exec_datafusion_err!( + "injected spill read failure" + )) + }) + .await?; + + Ok(()) + } + + /// Pauses every writer of a "phase" inside its first disk write, so that + /// several `push_batch` calls overlap at the point where each holds a file + /// checked out for writing. A writer that blocks earlier (for example on + /// another writer's file lock) never arrives; `release_phase` stops waiting + /// for it after a short while. + #[derive(Default)] + struct PhasePauser { + state: Mutex, + changed: parking_lot::Condvar, + } + + #[derive(Default)] + struct PhaseState { + phase: u64, + expected: usize, + held: usize, + released: bool, + } + + thread_local! { + /// The phase in which the current writer thread has already been held. + static HELD_IN_PHASE: std::cell::Cell = const { std::cell::Cell::new(0) }; + } + + impl PhasePauser { + fn begin_phase(&self, expected: usize) { + let mut state = self.state.lock(); + state.phase += 1; + state.expected = expected; + state.held = 0; + state.released = false; + } + + /// Called by writer threads before every disk write. + fn on_io(&self) { + let mut state = self.state.lock(); + if state.released + || state.held >= state.expected + || HELD_IN_PHASE.get() == state.phase + { + return; + } + HELD_IN_PHASE.set(state.phase); + state.held += 1; + self.changed.notify_all(); + while !state.released { + self.changed.wait(&mut state); + } + } + + /// Waits until every writer of the phase is held (or 300 ms have + /// passed), then releases them all at once. + fn release_phase(&self) { + let mut state = self.state.lock(); + let deadline = Instant::now() + Duration::from_millis(300); + while state.held < state.expected { + if self.changed.wait_until(&mut state, deadline).timed_out() { + break; + } + } + state.released = true; + self.changed.notify_all(); + } + } + + fn batch_with_id(id: i32, rows: usize) -> RecordBatch { + let a: ArrayRef = Arc::new(Int32Array::from(vec![id; rows])); + RecordBatch::try_new(create_test_schema(), vec![a]).unwrap() + } + + fn id_of(batch: &RecordBatch) -> i32 { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + } + + /// One randomly generated scenario, see [`spill_pool_scenario_fuzz`]. + async fn run_spill_pool_scenario(seed: u64) -> Result<()> { + use rand::rngs::StdRng; + use rand::{Rng, SeedableRng}; + + const READ_TIMEOUT: Duration = Duration::from_secs(5); + + let mut rng = StdRng::seed_from_u64(seed); + let n_writers = rng.random_range(1..=4); + let batch_bytes = batch_with_id(0, 10).get_array_memory_size(); + // Rotate after every batch, after a few batches, or never. + let max_file_size = match rng.random_range(0..3) { + 0 => batch_bytes / 2, + 1 => batch_bytes * 3, + _ => usize::MAX / 2, + }; + let n_phases = rng.random_range(1..=6); + let context = format!( + "seed {seed}: {n_writers} writers, max_file_size {max_file_size}, {n_phases} phases" + ); + + let pauser = Arc::new(PhasePauser::default()); + let write_hook: WriteHook = { + let pauser = Arc::clone(&pauser); + Arc::new(move || pauser.on_io()) + }; + // Some files are slow to read back, so an older file's bytes can arrive + // after a newer file's. Batches must still come back in file order for a + // single writer. + let read_hook: ReadHook = { + let rng = Mutex::new(StdRng::seed_from_u64(seed ^ 0x5EED_4EAD)); + Arc::new(move || { + let mut rng = rng.lock(); + if rng.random_bool(0.5) { + ReadBehavior::Delay(Duration::ZERO) + } else { + ReadBehavior::Delay(Duration::from_millis(rng.random_range(1..=20))) + } + }) + }; + let (spill_manager, disk_manager) = + spill_manager_with_io_hooks(write_hook, read_hook)?; + let (writer, mut reader) = mpsc_channel(max_file_size, spill_manager); + let mut sinks: Vec> = + (0..n_writers).map(|_| Some(writer.new_sink())).collect(); + drop(writer); + + let mut pushed: Vec = vec![]; + let mut read: Vec = vec![]; + + for phase in 0..n_phases { + let alive: Vec = + (0..n_writers).filter(|w| sinks[*w].is_some()).collect(); + if alive.is_empty() { + break; + } + + // A random non-empty subset of the live writers each pushes a few + // batches, concurrently, held so that their pushes overlap. + let mut pushing: Vec = alive + .iter() + .copied() + .filter(|_| rng.random_bool(0.6)) + .collect(); + if pushing.is_empty() { + pushing.push(alive[rng.random_range(0..alive.len())]); + } + pauser.begin_phase(pushing.len()); + let mut threads = Vec::with_capacity(pushing.len()); + for w in pushing { + let sink = sinks[w].take().unwrap(); + let batches: Vec = (0..rng.random_range(1..=3)) + .map(|_| { + let id = pushed.len() as i32 + 1; + pushed.push(id); + batch_with_id(id, rng.random_range(1..=40)) + }) + .collect(); + threads.push(( + w, + std::thread::spawn(move || { + for batch in &batches { + sink.push_batch(batch).unwrap(); + } + sink + }), + )); + } + pauser.release_phase(); + for (w, thread) in threads { + sinks[w] = Some(thread.join().unwrap()); + } + + // Quiescent point: no push is in progress and none will happen until + // the reader is done. Every batch pushed so far must be readable now, + // in any amount. `RepartitionExec` relies on exactly this: it blocks + // on the reader once per pushed batch while its channel gate keeps + // the writers parked. Sometimes leave a backlog for later phases. + let read_now = rng.random_range(0..=pushed.len() - read.len()); + for _ in 0..read_now { + let next = tokio::time::timeout(READ_TIMEOUT, reader.next()) + .await + .unwrap_or_else(|_| { + panic!( + "{context}, phase {phase}: reader stalled with {} of {} \ + batches unread and {} writers alive", + pushed.len() - read.len(), + pushed.len(), + alive.len() + ) + }); + let batch = next.unwrap_or_else(|| { + panic!("{context}, phase {phase}: EOF while writers are alive") + })?; + read.push(id_of(&batch)); + } + + // With nothing left to read the reader must wait, not signal EOF. + if read.len() == pushed.len() && rng.random_bool(0.3) { + let probe = + tokio::time::timeout(Duration::from_millis(50), reader.next()).await; + assert!( + probe.is_err(), + "{context}, phase {phase}: reader yielded {probe:?} with nothing \ + pushed and writers alive" + ); + } + + // Dropping one of several writers must not disturb anything. + if alive.len() > 1 && rng.random_bool(0.3) { + sinks[alive[rng.random_range(0..alive.len())]] = None; + } + } + + // Once the last writer is gone the reader must hand out the backlog and + // then signal EOF. + sinks.clear(); + while read.len() < pushed.len() { + let next = tokio::time::timeout(READ_TIMEOUT, reader.next()) + .await + .unwrap_or_else(|_| { + panic!( + "{context}: reader stalled with {} of {} batches unread after \ + all writers were dropped", + pushed.len() - read.len(), + pushed.len() + ) + }); + let batch = next.unwrap_or_else(|| { + panic!( + "{context}: EOF with {} batches unread", + pushed.len() - read.len() + ) + })?; + read.push(id_of(&batch)); + } + let eof = tokio::time::timeout(READ_TIMEOUT, reader.next()) + .await + .unwrap_or_else(|_| { + panic!("{context}: reader stalled instead of signalling EOF after all writers were dropped") + }); + assert!(eof.is_none(), "{context}: batch after everything was read"); + + let mut expected = pushed.clone(); + expected.sort_unstable(); + let mut actual = read.clone(); + actual.sort_unstable(); + assert_eq!(actual, expected, "{context}: batches lost or duplicated"); + if n_writers == 1 { + assert_eq!(read, pushed, "{context}: single-writer pool must be FIFO"); + } + + drop(reader); + assert_eq!( + disk_manager.used_disk_space(), + 0, + "{context}: spill files not released" + ); + Ok(()) + } + + /// Randomized scenarios against the pool's contract: after any set of + /// overlapping pushes the reader can drain everything without further + /// writer activity (no lost wakeups, no parking on one file while another + /// has data), it never signals EOF while a writer is alive, it returns + /// exactly the pushed batches (in order for a single writer), and the + /// files are released when it is dropped. + /// + /// `DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS` and + /// `DATAFUSION_SPILL_POOL_FUZZ_SEED` select how many scenarios run and + /// from which seed; a failure names its seed so it can be replayed. + #[tokio::test] + async fn spill_pool_scenario_fuzz() -> Result<()> { + let env_u64 = |name: &str, default: u64| { + std::env::var(name) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) + }; + let first_seed = env_u64("DATAFUSION_SPILL_POOL_FUZZ_SEED", 0); + let iterations = env_u64("DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS", 50); + // `saturating_add` so a replay seed near `u64::MAX` cannot overflow. + for seed in first_seed..first_seed.saturating_add(iterations) { + run_spill_pool_scenario(seed).await?; + } + Ok(()) + } } From ce6434c228258e6234cc0247e90e4db771813ce2 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Nov 2026 15:02:04 +0800 Subject: [PATCH 4/9] adapt: return the null-aware probe-key NULL check to logical nulls The null-aware probe phase decides whether the probe key saw a NULL with `Array::null_count()`, which is 0 for a dictionary key that points at a NULL dictionary value (a logical NULL). Upstream changed this check to `logical_null_count()` in apache/datafusion#23957 ("feat: null aware RightAnti hash join execution + planning support", 0a4e9305226767b336b78b631fa455dd1feec5d6); this fork's base predates it, so a dictionary-encoded `NOT IN` probe side is currently treated as NULL-free. Carry just that one line. The null-aware fixes that follow rely on it: their dictionary-NULL coverage on the probe side is meaningless otherwise. >>> patch: .backport-artifacts/p0-adapt-logical-null.patch Adaptation, not a backport: upstream commit 0a4e93052 also brings the RightAnti executor, which is out of scope here. Signed-off-by: discord9 (cherry picked from commit b44ec5964eff95be09722bb3b894c8e86618efb9) --- datafusion/physical-plan/src/joins/hash_join/stream.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rs index 3b5185909c7c7..9e0ca26d7ab89 100644 --- a/datafusion/physical-plan/src/joins/hash_join/stream.rs +++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs @@ -769,7 +769,7 @@ impl HashJoinStream { // Check if probe side (RIGHT) contains NULL // Since null_aware validation ensures single column join, we only check the first column let probe_key_column = &state.values[0]; - if probe_key_column.null_count() > 0 { + if probe_key_column.logical_null_count() > 0 { // Found NULL in probe side - set shared flag to prevent any output build_side .left_data From 1360562efeb248fdc05f0756b5372e6a0c975414 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Nov 2026 15:02:04 +0800 Subject: [PATCH 5/9] fix: null-aware anti join could emit rows after a sibling probe partition saw NULL RE-IMPLEMENTATION, not a faithful cherry-pick. apache/datafusion#25076 (5747e87e95f3a224b9bf98a8c157c9e069a7d24c) does not apply here: this fork's base predates the null-aware branching refactor (#24957, `NullAwareMode`) and the chunked final emission (#25028, `EmitUnmatchedBuildRowsState`), and `git show 5747e87e | git apply --check` fails on `exec.rs`, `stream.rs` and the three manifests. The semantics below are the upstream ones; the shape is this fork's. The bug: the final stage read the shared "probe side saw NULL" flag and only then decremented the probe-partition counter, so a partition could read `false`, let a sibling record a NULL and finish, then become the last partition itself and emit build rows that `NOT IN` must suppress. The counter also decremented with `Relaxed`, so nothing ordered a sibling's store before the last partition's load. Changes: - New `hash_join/probe_completion.rs`, ported nearly verbatim: `ProbeCompletion` owns the probe-partition counter and the two shared flags, `report_completed()` decrements with `AcqRel` and returns a `ProbeSideSummary` only to the last partition, so the final stage cannot read the flags before its own decrement. Its sequential unit tests and the `--cfg datafusion_loom` model-checked tests come along with the workspace `loom` dependency and the `cfg(datafusion_loom)` check-cfg registration. - `JoinLeftData` holds one `ProbeCompletion` instead of `probe_threads_counter`/`probe_side_non_empty`/`probe_side_has_null`. The probe phase records through `record_probe_batch()` and keeps a best-effort `probe_side_has_null_hint()` early exit. - `process_unmatched_build_batch` decrements first and hands the resulting summary to the new `null_aware_left_anti_final_indices`, which holds the `LeftAnti` rules: a probe NULL seen by any partition suppresses every build row; otherwise a NULL build key is dropped unless the probe side was empty. - This fork has no null-aware `LeftMark`, so the mark half of the upstream PR has no counterpart here. Tests (`hash_join/exec.rs`): - `test_null_aware_anti_join_probe_null_in_other_partition`: two probe partitions, only partition 0 sees the NULL, drained in both finishing orders; no rows may be emitted. - `test_null_aware_anti_join_probe_logical_null_in_other_partition`: the same with a dictionary key whose NULL is logical only. - `test_null_aware_anti_join_empty_probe_keeps_null_build_key`: pins `NULL NOT IN (empty) = TRUE` (added here; upstream covers it via slt only). (backport of apache/datafusion#25076, 5747e87e95f3a224b9bf98a8c157c9e069a7d24c) Signed-off-by: discord9 (cherry picked from commit 3fb9a3084e39781b90d5dbffa01f2664f64dcc2d) --- Cargo.lock | 48 +++ Cargo.toml | 6 + .../physical-optimizer/src/join_selection.rs | 7 +- datafusion/physical-plan/Cargo.toml | 19 + .../physical-plan/src/joins/hash_join/exec.rs | 346 ++++++++++++++++- .../physical-plan/src/joins/hash_join/mod.rs | 1 + .../src/joins/hash_join/probe_completion.rs | 350 ++++++++++++++++++ .../src/joins/hash_join/stream.rs | 133 +++---- 8 files changed, 820 insertions(+), 90 deletions(-) create mode 100644 datafusion/physical-plan/src/joins/hash_join/probe_completion.rs diff --git a/Cargo.lock b/Cargo.lock index 9f0fa41743af7..6bc415c8825db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2513,6 +2513,7 @@ dependencies = [ "insta", "itertools 0.15.0", "log", + "loom", "num-traits", "parking_lot", "pin-project-lite", @@ -3216,6 +3217,21 @@ dependencies = [ "prost-build", ] +[[package]] +name = "generator" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ade96dc9003043bce7c035c85a9df5a858bfb2039c5a2e6fdf00f324f6c551" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -4072,6 +4088,19 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -4087,6 +4116,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.8.4" @@ -5529,6 +5567,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -6566,10 +6610,14 @@ version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex-automata", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", ] diff --git a/Cargo.toml b/Cargo.toml index 7060b1df93f26..a2ae6695a712e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -174,6 +174,7 @@ itertools = "0.15" itoa = "1.0" liblzma = { version = "0.4.6", features = ["static"] } log = "^0.4" +loom = "0.7" memchr = "2.8.1" num-traits = { version = "0.2" } object_store = { version = "0.13.2", default-features = false } @@ -230,6 +231,11 @@ unexpected_cfgs = { level = "warn", check-cfg = [ 'cfg(datafusion_coop, values("tokio", "tokio_fallback", "per_stream"))', "cfg(coverage)", "cfg(coverage_nightly)", + # Enables the loom model-checked concurrency tests; see + # datafusion/physical-plan/src/joins/hash_join/probe_completion.rs. + # Namespaced rather than a bare `loom`, which tokio also reads and which + # would switch tokio itself into its loom build. + "cfg(datafusion_loom)", ] } unused_qualifications = "deny" diff --git a/datafusion/physical-optimizer/src/join_selection.rs b/datafusion/physical-optimizer/src/join_selection.rs index 42736f8205089..2f88385781037 100644 --- a/datafusion/physical-optimizer/src/join_selection.rs +++ b/datafusion/physical-optimizer/src/join_selection.rs @@ -283,9 +283,10 @@ pub(crate) fn partitioned_hash_join( hash_join.swap_inputs(PartitionMode::Partitioned) } else { // Null-aware anti joins must use CollectLeft mode because they track probe-side state - // (probe_side_non_empty, probe_side_has_null) per-partition, but need global knowledge - // for correct null handling. With partitioning, a partition might not see probe rows - // even if the probe side is globally non-empty, leading to incorrect NULL row handling. + // (did any probe partition see a row, did any see a NULL key) that has to be shared + // between partitions for correct null handling. With partitioning, a partition might + // not see probe rows even if the probe side is globally non-empty, leading to + // incorrect NULL row handling. let partition_mode = if hash_join.null_aware { PartitionMode::CollectLeft } else { diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml index 50d02162cfee9..b84940b5b5feb 100644 --- a/datafusion/physical-plan/Cargo.toml +++ b/datafusion/physical-plan/Cargo.toml @@ -105,6 +105,25 @@ tokio = { workspace = true, features = [ "parking_lot", ] } +# Model checker for the hash join probe-completion protocol; see +# joins/hash_join/probe_completion.rs. +# +# Gated on `--cfg datafusion_loom` rather than declared a plain dev-dependency: loom pulls +# in tracing-subscriber, which turns on `regex-automata`'s DFA features for +# every crate in the workspace build. That is not inert. It changes which +# engine the shared `regex` picks and costs enough extra stack to overflow +# unrelated deep-recursion tests (`sql::unparser::test_tpch_unparser_roundtrip` +# aborts in CI). Keeping loom behind its own cfg leaves the ordinary build +# untouched. +# +# Run the model checks with: +# RUSTFLAGS="--cfg datafusion_loom" cargo test -p datafusion-physical-plan --lib loom_tests +# +# The cfg is namespaced because a bare `loom` is also read by tokio, which +# would switch tokio into its own loom build and drop `tokio::fs`. +[target.'cfg(datafusion_loom)'.dev-dependencies] +loom = { workspace = true } + [[bench]] harness = false name = "partial_ordering" diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 3a30bf5f7053b..a305bb0f78725 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -18,7 +18,6 @@ use std::collections::HashSet; use std::fmt; use std::mem::size_of; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, OnceLock}; use std::vec; @@ -33,6 +32,7 @@ use crate::filter_pushdown::{ use crate::joins::Map; use crate::joins::array_map::ArrayMap; use crate::joins::hash_join::inlist_builder::build_struct_inlist_values; +use crate::joins::hash_join::probe_completion::{ProbeCompletion, ProbeSideSummary}; use crate::joins::hash_join::shared_bounds::{ ColumnBounds, PartitionBounds, PushdownStrategy, SharedBuildAccumulator, }; @@ -216,12 +216,9 @@ pub(super) struct JoinBuildData { pub(super) struct JoinLeftData { /// Shared bitmap for visited build indices within this join execution. visited_indices_bitmap: SharedBitmapBuilder, - /// Counter of running probe partitions that may update the bitmap. - probe_threads_counter: AtomicUsize, - /// Whether any probe partition saw data (for null-aware anti joins). - pub(super) probe_side_non_empty: AtomicBool, - /// Whether any probe partition saw a NULL key (for null-aware anti joins). - pub(super) probe_side_has_null: AtomicBool, + /// Tracks which probe partition finishes last and what the partitions + /// collectively saw. See [`ProbeCompletion`] for the invariant it upholds. + probe_completion: ProbeCompletion, /// IN-list values or a hash-table reference used for filter pushdown. membership: PushdownStrategy, /// The mutable visited bitmap belongs to this execution, not the build owner. @@ -281,10 +278,29 @@ impl JoinLeftData { self.build.bounds.as_ref() } - /// Decrements the counter of running threads, and returns `true` - /// if caller is the last running thread - pub(super) fn report_probe_completed(&self) -> bool { - self.probe_threads_counter.fetch_sub(1, Ordering::Relaxed) == 1 + /// Records what a probe partition saw in one batch, for the null-aware + /// rules evaluated in the final stage. + pub(super) fn record_probe_batch(&self, non_empty: bool, has_null: bool) { + self.probe_completion.record_batch(non_empty, has_null); + } + + /// Whether some probe partition has already recorded a NULL join key. + /// + /// Only an early-exit hint for the probe phase: it may lag behind sibling + /// partitions. Final-stage decisions must use the [`ProbeSideSummary`] + /// returned by [`Self::report_probe_completed`] instead. + pub(super) fn probe_side_has_null_hint(&self) -> bool { + self.probe_completion.saw_null_key_hint() + } + + /// Marks this probe partition as finished, returning `Some` for the last + /// one together with what every partition saw. + /// + /// The summary is reachable only through this call, which is what stops + /// the final stage from reading the shared flags before its own + /// decrement. See [`ProbeCompletion`] for why that ordering matters. + pub(super) fn report_probe_completed(&self) -> Option { + self.probe_completion.report_completed() } } @@ -2674,9 +2690,7 @@ async fn collect_left_input( let data = JoinLeftData { visited_indices_bitmap: Mutex::new(visited_indices_bitmap), - probe_threads_counter: AtomicUsize::new(probe_threads_count), - probe_side_non_empty: AtomicBool::new(false), - probe_side_has_null: AtomicBool::new(false), + probe_completion: ProbeCompletion::new(probe_threads_count), membership, _visited_reservation: visited_reservation, build: Arc::new(JoinBuildData { @@ -2755,6 +2769,7 @@ mod tests { ScalarValue, assert_batches_eq, assert_batches_sorted_eq, assert_contains, exec_err, internal_err, }; + use futures::StreamExt; use datafusion_execution::config::SessionConfig; use datafusion_execution::memory_pool::{ GreedyMemoryPool, MemoryPool, UnboundedMemoryPool, @@ -8579,6 +8594,309 @@ mod tests { Ok(()) } + /// Builds a two-partition probe side for the cross-partition null-aware + /// tests: partition 0 holds the only NULL key, partition 1 holds none. + fn build_two_partition_probe_with_null_in_partition_0() -> Arc { + let schema = Arc::new(Schema::new(vec![ + Field::new("c2", DataType::Int32, true), + Field::new("dummy", DataType::Int32, true), + ])); + let partition_0 = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int32Array::from(vec![Some(1), None])), + Arc::new(Int32Array::from(vec![Some(100), Some(400)])), + ], + ) + .unwrap(); + let partition_1 = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int32Array::from(vec![Some(2)])), + Arc::new(Int32Array::from(vec![Some(200)])), + ], + ) + .unwrap(); + TestMemoryExec::try_new_exec( + &[vec![partition_0], vec![partition_1]], + schema, + None, + ) + .unwrap() + } + + /// The dictionary counterpart of + /// [`build_two_partition_probe_with_null_in_partition_0`]: the same two + /// probe partitions, but the join key is a dictionary and partition 0's + /// NULL exists only logically. + /// + /// Every dictionary key is a physically valid index; one of them points at + /// a NULL dictionary value. The array therefore reports + /// `null_count() == 0` while `logical_null_count() > 0`, which is the case + /// that a physical-NULL check silently misses. + fn build_two_partition_dict_probe_with_logical_null_in_partition_0() + -> Arc { + let dict_type = + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Int32)); + let schema = Arc::new(Schema::new(vec![ + Field::new("c2", dict_type, true), + Field::new("dummy", DataType::Int32, true), + ])); + + // Dictionary values: [1, NULL]; keys: [0, 1] => logical [1, NULL]. + let with_logical_null: ArrayRef = Arc::new(DictionaryArray::new( + Int32Array::from(vec![0, 1]), + Arc::new(Int32Array::from(vec![Some(1), None])), + )); + assert_eq!( + with_logical_null.null_count(), + 0, + "the probe NULL must be logical only, or this test stops covering \ + the logical_null_count path" + ); + assert!( + with_logical_null.logical_null_count() > 0, + "the probe key must carry a logical NULL" + ); + + let partition_0 = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + with_logical_null, + Arc::new(Int32Array::from(vec![Some(100), Some(400)])), + ], + ) + .unwrap(); + + // Dictionary values: [2]; keys: [0] => logical [2], no NULL anywhere. + let partition_1 = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(DictionaryArray::new( + Int32Array::from(vec![0]), + Arc::new(Int32Array::from(vec![Some(2)])), + )), + Arc::new(Int32Array::from(vec![Some(200)])), + ], + ) + .unwrap(); + + TestMemoryExec::try_new_exec( + &[vec![partition_0], vec![partition_1]], + schema, + None, + ) + .unwrap() + } + + /// Builds a table whose join key is dictionary encoded, so a key can point + /// at a NULL dictionary value and be logically, but not physically, NULL. + fn build_table_dict_key( + key_name: &str, + dict_values: Vec>, + keys: Vec, + dummy_name: &str, + dummy: Vec>, + ) -> Arc { + let dict_type = + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Int32)); + let schema = Arc::new(Schema::new(vec![ + Field::new(key_name, dict_type, true), + Field::new(dummy_name, DataType::Int32, true), + ])); + let values = Int32Array::from(dict_values); + let key_array = Int32Array::from(keys); + let dict: ArrayRef = Arc::new(DictionaryArray::new(key_array, Arc::new(values))); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![dict, Arc::new(Int32Array::from(dummy))], + ) + .unwrap(); + TestMemoryExec::try_new_exec(&[vec![batch]], schema, None).unwrap() + } + + /// Drains the probe partitions of `join` one after another in the given + /// order and returns everything they emitted. + async fn collect_partitions_in_order( + join: &HashJoinExec, + order: &[usize], + task_ctx: &Arc, + ) -> Result> { + let mut batches = vec![]; + for &partition in order { + let stream = join.execute(partition, Arc::clone(task_ctx))?; + batches.extend(common::collect(stream).await?); + } + Ok(batches) + } + + /// A NULL probe key silences a null-aware anti join even when the + /// partition that saw it is not the one emitting the final build rows. + /// + /// `CollectLeft` with several probe partitions is what the planner + /// produces for `NOT IN`, and only the last partition to finish emits the + /// build rows, reading what its siblings recorded. Both finishing orders + /// must produce no rows. + /// + /// These partitions are drained sequentially, so this covers the + /// cross-partition plumbing, not the concurrent race that motivated + /// [`ProbeCompletion`]. + #[apply(hash_join_exec_configs)] + #[tokio::test] + async fn test_null_aware_anti_join_probe_null_in_other_partition( + batch_size: usize, + ) -> Result<()> { + let task_ctx = prepare_task_ctx(batch_size, false); + + for order in [[0, 1], [1, 0]] { + let left = build_table_two_cols( + ("c1", &vec![Some(1), Some(2), Some(3), Some(4)]), + ("dummy", &vec![Some(10), Some(20), Some(30), Some(40)]), + ); + let right = build_two_partition_probe_with_null_in_partition_0(); + + let on = vec![( + Arc::new(Column::new_with_schema("c1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("c2", &right.schema())?) as _, + )]; + + let join = HashJoinExec::try_new( + left, + right, + on, + None, + &JoinType::LeftAnti, + None, + PartitionMode::CollectLeft, + NullEquality::NullEqualsNothing, + true, // null_aware = true + )?; + + let batches = collect_partitions_in_order(&join, &order, &task_ctx).await?; + let rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!( + rows, 0, + "probe order {order:?} emitted rows although a probe partition saw NULL" + ); + } + Ok(()) + } + + /// The dictionary counterpart of + /// [`test_null_aware_anti_join_probe_null_in_other_partition`], where the + /// silencing probe NULL exists only logically. + /// + /// A dictionary key that points at a NULL dictionary value has + /// `null_count() == 0`, so it takes the logical-null check to record it. + /// This pins that check to the cross-partition summary: the partition + /// holding the logical NULL is not the one that emits the build rows, in + /// either drain order. + #[apply(hash_join_exec_configs)] + #[tokio::test] + async fn test_null_aware_anti_join_probe_logical_null_in_other_partition( + batch_size: usize, + ) -> Result<()> { + let task_ctx = prepare_task_ctx(batch_size, false); + + for order in [[0, 1], [1, 0]] { + // Build keys are dictionary-encoded too, and NULL-free. + let left = build_table_dict_key( + "c1", + vec![Some(1), Some(2), Some(3), Some(4)], + vec![0, 1, 2, 3], + "dummy", + vec![Some(10), Some(20), Some(30), Some(40)], + ); + let right = build_two_partition_dict_probe_with_logical_null_in_partition_0(); + + let on = vec![( + Arc::new(Column::new_with_schema("c1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("c2", &right.schema())?) as _, + )]; + + let join = HashJoinExec::try_new( + left, + right, + on, + None, + &JoinType::LeftAnti, + None, + PartitionMode::CollectLeft, + NullEquality::NullEqualsNothing, + true, // null_aware = true + )?; + + let batches = collect_partitions_in_order(&join, &order, &task_ctx).await?; + let rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!( + rows, 0, + "probe order {order:?} emitted rows although a probe partition saw a logical NULL" + ); + } + Ok(()) + } + + /// `NULL NOT IN (empty)` is TRUE: with an empty probe side the NULL build + /// key must be kept, even though the same build side emits nothing once + /// the probe side has a row. + #[apply(hash_join_exec_configs)] + #[tokio::test] + async fn test_null_aware_anti_join_empty_probe_keeps_null_build_key( + batch_size: usize, + ) -> Result<()> { + let task_ctx = prepare_task_ctx(batch_size, false); + + let left = build_table_two_cols( + ("c1", &vec![Some(1), None]), + ("dummy", &vec![Some(10), Some(0)]), + ); + // Genuinely empty probe side (not an all-NULL probe side, which is the + // different `NOT IN` UNKNOWN case). + let empty_right_schema = Arc::new(Schema::new(vec![ + Field::new("c2", DataType::Int32, true), + Field::new("dummy", DataType::Int32, true), + ])); + let empty_right = TestMemoryExec::try_new_exec( + &[vec![RecordBatch::new_empty(Arc::clone( + &empty_right_schema, + ))]], + empty_right_schema, + None, + )?; + + let on = vec![( + Arc::new(Column::new_with_schema("c1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("c2", &empty_right.schema())?) as _, + )]; + + let join = HashJoinExec::try_new( + left, + empty_right, + on, + None, + &JoinType::LeftAnti, + None, + PartitionMode::CollectLeft, + NullEquality::NullEqualsNothing, + true, // null_aware = true + )?; + + let stream = join.execute(0, task_ctx)?; + let batches = common::collect(stream).await?; + + allow_duplicates! { + assert_snapshot!(batches_to_sort_string(&batches), @r" + +----+-------+ + | c1 | dummy | + +----+-------+ + | 1 | 10 | + | | 0 | + +----+-------+ + "); + } + Ok(()) + } + /// Test that null_aware validation rejects non-LeftAnti join types #[tokio::test] async fn test_null_aware_validation_wrong_join_type() { diff --git a/datafusion/physical-plan/src/joins/hash_join/mod.rs b/datafusion/physical-plan/src/joins/hash_join/mod.rs index 4687d8545ec69..7b50435e18792 100644 --- a/datafusion/physical-plan/src/joins/hash_join/mod.rs +++ b/datafusion/physical-plan/src/joins/hash_join/mod.rs @@ -23,5 +23,6 @@ pub use partitioned_hash_eval::{HashExpr, HashTableLookupExpr, SeededRandomState mod exec; mod inlist_builder; mod partitioned_hash_eval; +mod probe_completion; mod shared_bounds; mod stream; diff --git a/datafusion/physical-plan/src/joins/hash_join/probe_completion.rs b/datafusion/physical-plan/src/joins/hash_join/probe_completion.rs new file mode 100644 index 0000000000000..11fb72d352c6a --- /dev/null +++ b/datafusion/physical-plan/src/joins/hash_join/probe_completion.rs @@ -0,0 +1,350 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Completion protocol shared by the probe partitions of a hash join. +//! +//! The probe partitions of one `HashJoinExec` run concurrently over a shared +//! build side. Two facts have to cross partition boundaries: +//! +//! - which partition emits the build-side rows in the final stage, namely the +//! last one to finish, and +//! - what the partitions collectively saw, which the null-aware `NOT IN` rules +//! need: did any partition see a row, and did any see a NULL join key. +//! +//! [`ProbeCompletion`] owns both, because reading the second without ordering +//! it against the first produced wrong results. A partition that read the NULL +//! flag *before* decrementing the counter could observe `false`, have a +//! sibling record a NULL and finish, then become the last partition itself and +//! emit build rows that `NOT IN` must suppress. +//! +//! The invariant is therefore: **whichever caller observes itself to be last +//! sees every fact recorded by every other partition.** It rests on two +//! properties, and the type is shaped to keep both. +//! +//! 1. *Order.* The facts are reachable only through the value returned by +//! [`ProbeCompletion::report_completed`], so no caller can read them before +//! its own decrement. +//! 2. *Visibility.* That decrement is `AcqRel`, so the last partition +//! synchronizes with the release of every partition that finished before +//! it. Were the counter `Relaxed`, nothing would order a sibling's store +//! before the final load even when the decrement happened first. +//! +//! Neither property can be pinned down by an ordinary concurrent test: the +//! window is a few instructions wide, and on x86 a `Relaxed` decrement lowers +//! to the same instruction as an `AcqRel` one. Both are model-checked in +//! `loom_tests` instead. + +use std::sync::atomic::Ordering; + +/// What every probe partition together saw, as observed by the last one to +/// finish. +/// +/// Obtainable only from [`ProbeCompletion::report_completed`], which is what +/// stops the final stage from reading the shared state too early. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct ProbeSideSummary { + /// Some probe partition saw a row. An entirely empty probe side is not the + /// same as one that matched nothing: `NULL NOT IN (empty)` is TRUE. + pub(super) non_empty: bool, + /// Some probe partition saw a NULL join key. + pub(super) has_null: bool, +} + +/// Defines [`ProbeCompletion`] over the given atomic types. +/// +/// [loom] can only explore its own atomics, so the protocol is written once +/// here and instantiated twice: over [`std::sync::atomic`] for the join, and +/// over `loom::sync::atomic` for the model in `loom_tests`. Both share this +/// single copy of the orderings and the call sequence, which is what lets the +/// model speak for the real thing. Keep it that way: a second copy of these +/// orderings would not be covered. +/// +/// [loom]: https://docs.rs/loom +macro_rules! define_probe_completion { + (counter: $counter:ty, flag: $flag:ty) => { + /// Counts the probe partitions that are still running. + type Counter = $counter; + /// One fact, set by any partition and read by the last one. + type Flag = $flag; + + /// Tracks how many probe partitions are still running, together with + /// the null-aware facts they contribute. + /// + /// See the [module docs](self) for the invariant this upholds. + #[derive(Debug)] + pub(super) struct ProbeCompletion { + /// Probe partitions that have not finished yet. + running: Counter, + /// Set once any probe partition has seen a row. + saw_row: Flag, + /// Set once any probe partition has seen a NULL join key. + saw_null_key: Flag, + } + + impl ProbeCompletion { + /// Creates the protocol state for `probe_threads` partitions. + pub(super) fn new(probe_threads: usize) -> Self { + Self { + running: Counter::new(probe_threads), + saw_row: Flag::new(false), + saw_null_key: Flag::new(false), + } + } + + /// Records what one probe partition saw in one batch. + /// + /// `Relaxed` suffices: the `AcqRel` decrement in + /// [`Self::report_completed`] publishes these stores to the last + /// partition. + pub(super) fn record_batch(&self, non_empty: bool, has_null: bool) { + if non_empty { + self.saw_row.store(true, Ordering::Relaxed); + } + if has_null { + self.saw_null_key.store(true, Ordering::Relaxed); + } + } + + /// Whether some partition has already recorded a NULL join key. + /// + /// A hint for skipping work during the probe phase, which may lag + /// behind sibling partitions. It must never decide what the final + /// stage emits; use the [`ProbeSideSummary`] from + /// [`Self::report_completed`] for that. + pub(super) fn saw_null_key_hint(&self) -> bool { + self.saw_null_key.load(Ordering::Relaxed) + } + + /// Marks the calling partition finished, returning `Some` only for + /// the last one, together with what all partitions saw. + /// + /// The `AcqRel` decrement publishes this partition's own stores + /// and acquires those of the partitions that finished earlier, so + /// the summary handed to the last caller is complete. + pub(super) fn report_completed(&self) -> Option { + // `fetch_sub` returns the value from *before* the decrement, + // so `1` means this call is the one that took it to zero. + let was_last = self.running.fetch_sub(1, Ordering::AcqRel) == 1; + was_last.then(|| ProbeSideSummary { + non_empty: self.saw_row.load(Ordering::Relaxed), + has_null: self.saw_null_key.load(Ordering::Relaxed), + }) + } + } + }; +} + +define_probe_completion!( + counter: std::sync::atomic::AtomicUsize, + flag: std::sync::atomic::AtomicBool +); + +#[cfg(test)] +mod tests { + use super::*; + + /// The sequential contract: only the last caller is handed the summary, + /// and it carries what the earlier partitions recorded. + /// + /// The concurrent invariant is covered by `loom_tests` instead. + #[test] + fn only_the_last_partition_receives_the_summary() { + let completion = ProbeCompletion::new(3); + + completion.record_batch(true, false); + assert_eq!(completion.report_completed(), None); + + completion.record_batch(true, true); + assert_eq!(completion.report_completed(), None); + + assert_eq!( + completion.report_completed(), + Some(ProbeSideSummary { + non_empty: true, + has_null: true, + }) + ); + } + + /// A probe side that never saw a row leaves both facts clear, which is how + /// `NULL NOT IN (empty)` stays TRUE. + #[test] + fn an_untouched_probe_side_reports_nothing_seen() { + let completion = ProbeCompletion::new(1); + + assert_eq!( + completion.report_completed(), + Some(ProbeSideSummary { + non_empty: false, + has_null: false, + }) + ); + } + + /// The hint may lag, but it must never report a NULL that was not + /// recorded. + #[test] + fn the_hint_reflects_recorded_nulls() { + let completion = ProbeCompletion::new(1); + assert!(!completion.saw_null_key_hint()); + + completion.record_batch(true, false); + assert!(!completion.saw_null_key_hint()); + + completion.record_batch(true, true); + assert!(completion.saw_null_key_hint()); + } +} + +/// Model-checked concurrency tests for the protocol. +/// +/// These instantiate the protocol over loom's atomics (see the +/// [module docs](self)) and let loom enumerate the interleavings and store +/// visibility the memory model allows. They fail if the decrement in +/// `report_completed` is weakened to `Relaxed`. +/// +/// Compiled only under `--cfg datafusion_loom`, which is also what pulls the `loom` +/// dev-dependency into the build (see `datafusion-physical-plan`'s +/// `Cargo.toml` for why it is kept out of the ordinary one). Run them with: +/// +/// ```text +/// RUSTFLAGS="--cfg datafusion_loom" cargo test -p datafusion-physical-plan --lib loom_tests +/// ``` +#[cfg(all(test, datafusion_loom))] +mod loom_tests { + use super::ProbeSideSummary; + use loom::sync::Arc; + use std::sync::atomic::Ordering; + + define_probe_completion!( + counter: loom::sync::atomic::AtomicUsize, + flag: loom::sync::atomic::AtomicBool + ); + + /// The invariant behind the fix: whichever partition observes itself to be + /// last sees the NULL its sibling recorded. + /// + /// This is the regression test for the wrong-results bug. One partition + /// records a NULL and finishes while the other finishes having recorded + /// nothing, and the last one must not conclude that no NULL was seen. + #[test] + fn the_last_partition_observes_a_sibling_null() { + loom::model(|| { + let completion = Arc::new(ProbeCompletion::new(2)); + + let recorder = { + let completion = Arc::clone(&completion); + loom::thread::spawn(move || { + completion.record_batch(true, true); + completion.report_completed() + }) + }; + + let observer = completion.report_completed(); + let recorded = recorder.join().unwrap(); + + let summary = match (recorded, observer) { + (Some(summary), None) | (None, Some(summary)) => summary, + (Some(_), Some(_)) => panic!("two partitions both finished last"), + (None, None) => panic!("no partition finished last"), + }; + + assert!( + summary.has_null, + "the last partition missed the NULL its sibling recorded" + ); + assert!( + summary.non_empty, + "the last partition missed the row its sibling recorded" + ); + }); + } + + /// With both partitions recording, the summary must be the union of what + /// they saw, whichever finishes last. + #[test] + fn the_summary_unions_what_every_partition_recorded() { + loom::model(|| { + let completion = Arc::new(ProbeCompletion::new(2)); + + let other = { + let completion = Arc::clone(&completion); + loom::thread::spawn(move || { + // Rows, but no NULL key. + completion.record_batch(true, false); + completion.report_completed() + }) + }; + + // A NULL key in an otherwise empty batch. + completion.record_batch(false, true); + let here = completion.report_completed(); + let there = other.join().unwrap(); + + let summary = here.or(there).expect("some partition must finish last"); + assert_eq!( + summary, + ProbeSideSummary { + non_empty: true, + has_null: true, + }, + "the summary lost a fact one of the partitions recorded" + ); + }); + } + + /// Exactly one partition may take the final stage. Two would emit the + /// build side twice; none would drop it. + #[test] + fn exactly_one_partition_finishes_last() { + loom::model(|| { + let completion = Arc::new(ProbeCompletion::new(2)); + + let other = { + let completion = Arc::clone(&completion); + loom::thread::spawn(move || completion.report_completed().is_some()) + }; + + let here = completion.report_completed().is_some(); + let there = other.join().unwrap(); + + assert!(here ^ there, "exactly one partition must finish last"); + }); + } + + /// The probe-phase hint never invents a NULL: if no partition recorded + /// one, no observation of the hint may report one. + #[test] + fn the_hint_never_reports_an_unrecorded_null() { + loom::model(|| { + let completion = Arc::new(ProbeCompletion::new(2)); + + let other = { + let completion = Arc::clone(&completion); + loom::thread::spawn(move || { + completion.record_batch(true, false); + completion.saw_null_key_hint() + }) + }; + + let here = completion.saw_null_key_hint(); + let there = other.join().unwrap(); + + assert!(!here && !there, "the hint reported an unrecorded NULL"); + }); + } +} diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rs index 9e0ca26d7ab89..1cae2eb4f12d2 100644 --- a/datafusion/physical-plan/src/joins/hash_join/stream.rs +++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs @@ -21,7 +21,6 @@ //! [`super::HashJoinExec`]. See comments in [`HashJoinStream`] for more details. use std::sync::Arc; -use std::sync::atomic::Ordering; use std::task::Poll; use crate::coalesce::{LimitedBatchCoalescer, PushBatchStatus}; @@ -29,6 +28,7 @@ use crate::joins::Map; use crate::joins::MapOffset; use crate::joins::PartitionMode; use crate::joins::hash_join::exec::JoinLeftData; +use crate::joins::hash_join::probe_completion::ProbeSideSummary; use crate::joins::hash_join::shared_bounds::{ PartitionBounds, PartitionBuildData, SharedBuildAccumulator, }; @@ -674,7 +674,7 @@ impl HashJoinStream { // Note: For null-aware anti join, we need to check the probe side (right) for NULLs, // not the build side (left). The probe-side NULL check happens during process_probe_batch. - // The probe_side_has_null flag will be set there if any probe batch contains NULL. + // The shared probe-completion state records it there if any probe batch contains NULL. self.state = self.transition_after_build_collected(&left_data); @@ -756,33 +756,19 @@ impl HashJoinStream { // 1. If RIGHT (probe) contains NULL in any batch, no LEFT rows should be output // 2. LEFT rows with NULL keys should not be output (handled in final stage) if self.null_aware { - // Mark that we've seen a probe batch with actual rows (probe side is non-empty) - // Only set this if batch has rows - empty batches don't count - // Use shared atomic state so all partitions can see this global information - if state.batch.num_rows() > 0 { - build_side - .left_data - .probe_side_non_empty - .store(true, Ordering::Relaxed); - } - - // Check if probe side (RIGHT) contains NULL // Since null_aware validation ensures single column join, we only check the first column let probe_key_column = &state.values[0]; - if probe_key_column.logical_null_count() > 0 { - // Found NULL in probe side - set shared flag to prevent any output - build_side - .left_data - .probe_side_has_null - .store(true, Ordering::Relaxed); - } - - // If probe side has NULL (detected in this or any other partition), return empty result - if build_side - .left_data - .probe_side_has_null - .load(Ordering::Relaxed) - { + // Only batches with rows count: `NULL NOT IN (empty)` is TRUE. + build_side.left_data.record_probe_batch( + state.batch.num_rows() > 0, + probe_key_column.logical_null_count() > 0, + ); + + // If probe side has NULL (detected in this or any other + // partition so far), return empty result. This is a best-effort + // early exit: the final stage re-checks the flag through + // `report_probe_completed`. + if build_side.left_data.probe_side_has_null_hint() { timer.done(); self.state = HashJoinStreamState::FetchProbeBatch; return Ok(StatefulStreamResult::Continue); @@ -970,22 +956,15 @@ impl HashJoinStream { let build_side = self.build_side.try_as_ready()?; - // For null-aware anti join, if probe side had NULL, no rows should be output - // Check shared atomic state to get global knowledge across all partitions - if self.null_aware - && build_side - .left_data - .probe_side_has_null - .load(Ordering::Relaxed) - { + // Only the last probe partition to finish emits the build-side rows, + // and only it receives the shared probe-side summary (see + // `JoinLeftData::report_probe_completed` for why the flags are not + // readable any other way here). + let Some(probe_summary) = build_side.left_data.report_probe_completed() else { timer.done(); self.state = HashJoinStreamState::Completed; return Ok(StatefulStreamResult::Continue); - } - if !build_side.left_data.report_probe_completed() { - self.state = HashJoinStreamState::Completed; - return Ok(StatefulStreamResult::Continue); - } + }; // use the global left bitmap to produce the left indices and right indices let (mut left_side, mut right_side) = get_final_indices_from_shared_bitmap( @@ -994,39 +973,17 @@ impl HashJoinStream { true, ); - // For null-aware anti join, filter out LEFT rows with NULL in join keys - // BUT only if the probe side (RIGHT) was non-empty. If probe side is empty, - // NULL NOT IN (empty) = TRUE, so NULL rows should be returned. - // Use shared atomic state to get global knowledge across all partitions - if self.null_aware - && self.join_type == JoinType::LeftAnti - && build_side - .left_data - .probe_side_non_empty - .load(Ordering::Relaxed) - { - // Since null_aware validation ensures single column join, we only check the first column - let build_key_column = &build_side.left_data.values()[0]; - - // Filter out indices where the key is NULL - let filtered_indices: Vec = left_side - .iter() - .filter_map(|idx| { - let idx_usize = idx.unwrap() as usize; - if build_key_column.is_null(idx_usize) { - None // Skip rows with NULL keys - } else { - Some(idx.unwrap()) - } - }) - .collect(); - - left_side = UInt64Array::from(filtered_indices); - - // Update right_side to match the new length - let mut builder = arrow::array::UInt32Builder::with_capacity(left_side.len()); - builder.append_nulls(left_side.len()); - right_side = builder.finish(); + // Null-aware joins post-process the build rows under SQL three-valued + // logic; see the helper for the rules. + if self.null_aware && self.join_type == JoinType::LeftAnti { + let (null_aware_left, null_aware_right) = null_aware_left_anti_final_indices( + &build_side.left_data, + probe_summary, + left_side, + right_side, + ); + left_side = null_aware_left; + right_side = null_aware_right; } self.join_metrics.input_batches.add(1); @@ -1061,6 +1018,36 @@ impl HashJoinStream { } } +/// Final-stage rules of a null-aware `LeftAnti` join, evaluated by the last +/// probe partition from what every partition together saw: +/// - a NULL probe key seen by any partition makes `build.key NOT IN (probe)` +/// UNKNOWN for every build row, so nothing is emitted; +/// - otherwise a NULL build key means `NULL NOT IN (probe)`, which is UNKNOWN +/// (row dropped) unless the probe side was empty, where it is TRUE (row +/// kept). +fn null_aware_left_anti_final_indices( + left_data: &JoinLeftData, + probe_summary: ProbeSideSummary, + left_side: UInt64Array, + right_side: UInt32Array, +) -> (UInt64Array, UInt32Array) { + if probe_summary.has_null { + return (UInt64Array::new_null(0), UInt32Array::new_null(0)); + } + if !probe_summary.non_empty { + return (left_side, right_side); + } + // null_aware validation ensures a single join key + let build_key_column = &left_data.values()[0]; + let left_side = UInt64Array::from_iter_values(left_side.iter().filter_map(|idx| { + let idx = + idx.expect("LeftAnti final indices should always contain build-side rows"); + (!build_key_column.is_null(idx as usize)).then_some(idx) + })); + let right_side = UInt32Array::new_null(left_side.len()); + (left_side, right_side) +} + impl Stream for HashJoinStream { type Item = Result; From b30bec9a34ac7a3a023d16b2e9cb6970bedbcb82 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Nov 2026 15:02:04 +0800 Subject: [PATCH 6/9] fix: correlated NOT IN with a non-equality correlation returns wrong results RE-IMPLEMENTATION, not a faithful cherry-pick. apache/datafusion#25339 (95bb0a0dfa48ca20d017faa55b2fc96ee2f300fe) builds on the correlated null-aware machinery of #21585/#24957 (correlation scope maps, `NullAwareMode`, `null_indices_bitmap`, shared null-mark buffers), none of which exists here; `git show 95bb0a0d | git apply --check` fails on `exec.rs`, `stream.rs` and `join_selection.rs`, and it also edits benchmark files this fork does not have. The bug: a correlated `NOT IN` whose correlation cannot become an equi-join key leaves a residual join filter, and the executor ignored that filter when deciding whether a NULL on the subquery side makes `NOT IN` UNKNOWN, so a NULL the filter excludes still poisoned every outer row. Same semantics for the shapes this fork can plan (`on[0]` is the `NOT IN` value key and the residual correlation lives in the join filter; the null-aware validation still requires a single join key, where upstream also accepted `on[1..]` scope keys): - `JoinLeftData` gains the shared `null_indices_bitmap` of build rows whose `NOT IN` is UNKNOWN, allocated with the build side for correlated joins. - A correlated null-aware join (one with a join filter) no longer lets a probe NULL poison the whole join: `mark_null_candidates_for_probe_batch` records, per build row, whether a NULL that the residual filter keeps makes `NOT IN` UNKNOWN - case 1, a NULL build key with a probe row that passes the filter; case 2, a probe row with a NULL key that passes the filter. Without scope keys both cases are cross products, already-marked build rows are skipped, and the filter is not re-evaluated for them. - The final stage drops the marked build rows instead of consulting the probe-side flags, so outer rows whose subquery is empty keep `NOT IN (empty) = TRUE`. - The uncorrelated probe-side NULL check now uses logical nulls too, so dictionary keys are covered on both paths. Tests: - `hash_join/exec.rs`: `test_null_aware_left_anti_filter_only` (a `LeftAnti` with a second-column `>` filter and no scope keys; only the rows with an empty subquery result are emitted), from the upstream PR. - `sqllogictest/test_files/null_aware_anti_join.slt`: new correlated-residual section - the query from the upstream issue, an `EXPLAIN` assertion that the shape stays `LeftAnti ... filter=..., null_aware` on a single equi key, and the case where the residual leaves the subquery empty for every outer row. (backport of apache/datafusion#25339, 95bb0a0dfa48ca20d017faa55b2fc96ee2f300fe) Signed-off-by: discord9 (cherry picked from commit 163cc1d5fb8a8565825a13ebfc82c010bc8750fa) --- .../physical-plan/src/joins/hash_join/exec.rs | 144 ++++++++++ .../src/joins/hash_join/stream.rs | 253 ++++++++++++++++-- .../test_files/null_aware_anti_join.slt | 56 ++++ 3 files changed, 437 insertions(+), 16 deletions(-) diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index a305bb0f78725..af8ec65ecf40b 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -216,6 +216,12 @@ pub(super) struct JoinBuildData { pub(super) struct JoinLeftData { /// Shared bitmap for visited build indices within this join execution. visited_indices_bitmap: SharedBitmapBuilder, + /// Bitmap of the build rows whose `NOT IN` is UNKNOWN instead of TRUE. + /// + /// Filled by the probe phase of a correlated null-aware join, whose + /// per-build-row decision replaces the shared probe-side flags (see + /// [`HashJoinExec::null_aware`]). + null_indices_bitmap: SharedBitmapBuilder, /// Tracks which probe partition finishes last and what the partitions /// collectively saw. See [`ProbeCompletion`] for the invariant it upholds. probe_completion: ProbeCompletion, @@ -264,6 +270,12 @@ impl JoinLeftData { &self.visited_indices_bitmap } + /// returns a reference to the bitmap of build rows whose `NOT IN` is + /// UNKNOWN + pub(super) fn null_indices_bitmap(&self) -> &SharedBitmapBuilder { + &self.null_indices_bitmap + } + /// returns a reference to the InList values for filter pushdown pub(super) fn membership(&self) -> &PushdownStrategy { &self.membership @@ -822,6 +834,15 @@ pub struct HashJoinExec { /// The equality null-handling behavior of the join algorithm. pub null_equality: NullEquality, /// Flag to indicate if this is a null-aware anti join + /// + /// Set for the physical lowering of scalar `NOT IN` subqueries + /// (`JoinType::LeftAnti`). When `true`, NULLs in the join keys follow SQL + /// `NOT IN` three-valued logic rather than ordinary equi-join semantics. + /// + /// A join filter holds the non-equality part of a correlated subquery, and + /// only the build rows it keeps take part in the three-valued logic: such + /// a join records the UNKNOWN decision per build row in the null-indices + /// bitmap instead of in shared probe-side flags. pub null_aware: bool, /// Cache holding plan properties like equivalences, output partitioning etc. cache: Arc, @@ -1572,6 +1593,8 @@ impl ExecutionPlan for HashJoinExec { join_metrics.clone(), reservation, need_produce_result_in_final(self.join_type), + // Correlated null-aware joins record UNKNOWN per build row. + self.null_aware && self.filter.is_some(), right_partitions, enable_dynamic_filter_pushdown, Arc::clone(context.session_config().options()), @@ -1593,6 +1616,7 @@ impl ExecutionPlan for HashJoinExec { join_metrics.clone(), reservation, need_produce_result_in_final(self.join_type), + self.null_aware && self.filter.is_some(), 1, enable_dynamic_filter_pushdown, Arc::clone(context.session_config().options()), @@ -2390,6 +2414,8 @@ fn concat_build_batches( /// * `metrics` - Metrics collector for tracking memory usage and row counts /// * `reservation` - Memory reservation tracker for the hash table and data /// * `with_visited_indices_bitmap` - Whether to track visited indices (for outer joins) +/// * `with_null_aware_row_state` - Whether to track the per-build-row UNKNOWN +/// decision of a correlated null-aware join /// * `probe_threads_count` - Number of threads that will probe this hash table /// * `should_compute_dynamic_filters` - Whether to compute min/max bounds for dynamic filtering /// @@ -2420,6 +2446,7 @@ async fn collect_left_input( metrics: BuildProbeJoinMetrics, reservation: MemoryReservation, with_visited_indices_bitmap: bool, + with_null_aware_row_state: bool, probe_threads_count: usize, should_compute_dynamic_filters: bool, config: Arc, @@ -2614,6 +2641,20 @@ async fn collect_left_input( (BooleanBufferBuilder::new(0), reservation.new_empty()) }; + // Shared by all probe partitions of a correlated null-aware join, which + // record the per-build-row UNKNOWN decision here. Charged to the build + // reservation, which stays alive as long as this data is reachable. + let null_indices_bitmap = if with_null_aware_row_state { + let bitmap_size = bit_util::ceil(batch.num_rows(), 8); + reservation.try_grow(bitmap_size)?; + metrics.build_mem_used.add(bitmap_size); + let mut bitmap_buffer = BooleanBufferBuilder::new(batch.num_rows()); + bitmap_buffer.append_n(num_rows, false); + bitmap_buffer + } else { + BooleanBufferBuilder::new(0) + }; + let map = Arc::new(join_hash_map); // For an ordinary build, nothing reads the strategy unless this join's own @@ -2690,6 +2731,7 @@ async fn collect_left_input( let data = JoinLeftData { visited_indices_bitmap: Mutex::new(visited_indices_bitmap), + null_indices_bitmap: Mutex::new(null_indices_bitmap), probe_completion: ProbeCompletion::new(probe_threads_count), membership, _visited_reservation: visited_reservation, @@ -8897,6 +8939,108 @@ mod tests { Ok(()) } + /// `left.z > right.z` over the second column of two-column tables: the + /// non-equality correlation of + /// `id NOT IN (SELECT r.id FROM r WHERE r.z < l.z)`. + fn prepare_second_column_gt_filter() -> JoinFilter { + let column_indices = vec![ + ColumnIndex { + index: 1, + side: JoinSide::Left, + }, + ColumnIndex { + index: 1, + side: JoinSide::Right, + }, + ]; + let intermediate_schema = Schema::new(vec![ + Field::new("z", DataType::Int32, true), + Field::new("z", DataType::Int32, true), + ]); + let filter_expression = Arc::new(BinaryExpr::new( + Arc::new(Column::new("z", 0)), + Operator::Gt, + Arc::new(Column::new("z", 1)), + )) as Arc; + + JoinFilter::new( + filter_expression, + column_indices, + Arc::new(intermediate_schema), + ) + } + + /// Build and probe sides of a null-aware join whose only correlation is + /// the non-equality filter from [`prepare_second_column_gt_filter`]. + /// + /// For each build row, the probe rows with a smaller `z` form its + /// subquery result: + /// - `(1, 10)` and `(2, 20)`: `{1, NULL}` + /// - `(NULL, 30)`: `{1, NULL}` + /// - `(4, 40)`: `{1, 4, NULL}` + /// - `(NULL, 1)` and `(5, 1)`: empty + /// + /// The probe row `(NULL, 50)` never passes the filter. + fn build_null_aware_filter_only_inputs() + -> (Arc, Arc, JoinOn) { + let left = build_table_two_cols( + ("id", &vec![Some(1), Some(2), None, Some(4), None, Some(5)]), + ( + "z", + &vec![Some(10), Some(20), Some(30), Some(40), Some(1), Some(1)], + ), + ); + let right = build_table_two_cols( + ("id", &vec![Some(1), None, Some(4), None]), + ("z", &vec![Some(5), Some(50), Some(35), Some(2)]), + ); + let on = vec![( + Arc::new(Column::new_with_schema("id", &left.schema()).unwrap()) as _, + Arc::new(Column::new_with_schema("id", &right.schema()).unwrap()) as _, + )]; + (left, right, on) + } + + /// Null-aware `LeftAnti` with a join filter and no correlation scope keys. + /// + /// A NULL on either side only makes `NOT IN` UNKNOWN for the build rows + /// where the filter keeps the NULL, so the NULLs must not remove every row. + #[apply(hash_join_exec_configs)] + #[tokio::test] + async fn test_null_aware_left_anti_filter_only(batch_size: usize) -> Result<()> { + let task_ctx = prepare_task_ctx(batch_size, false); + let (left, right, on) = build_null_aware_filter_only_inputs(); + + let join = HashJoinExec::try_new( + left, + right, + on, + Some(prepare_second_column_gt_filter()), + &JoinType::LeftAnti, + None, + PartitionMode::CollectLeft, + NullEquality::NullEqualsNothing, + true, + )?; + + let stream = join.execute(0, task_ctx)?; + let batches = common::collect(stream).await?; + + // Only the rows with an empty subquery result are TRUE. + allow_duplicates! { + assert_snapshot!(batches_to_sort_string(&batches), @r" + +----+---+ + | id | z | + +----+---+ + | | 1 | + | 5 | 1 | + +----+---+ + "); + } + + Ok(()) + } + /// Test that null_aware validation rejects non-LeftAnti join types #[tokio::test] async fn test_null_aware_validation_wrong_join_type() { diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rs index 1cae2eb4f12d2..14cafc3ad2c59 100644 --- a/datafusion/physical-plan/src/joins/hash_join/stream.rs +++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs @@ -755,23 +755,45 @@ impl HashJoinStream { // For LeftAnti: output LEFT (build) rows where LEFT.key NOT IN RIGHT.key // 1. If RIGHT (probe) contains NULL in any batch, no LEFT rows should be output // 2. LEFT rows with NULL keys should not be output (handled in final stage) + // + // A correlated join (one with a join filter) narrows rule 1 to the build + // rows whose correlation scope and filter keep that NULL; see + // `mark_null_candidates_for_probe_batch`. if self.null_aware { - // Since null_aware validation ensures single column join, we only check the first column - let probe_key_column = &state.values[0]; - // Only batches with rows count: `NULL NOT IN (empty)` is TRUE. - build_side.left_data.record_probe_batch( - state.batch.num_rows() > 0, - probe_key_column.logical_null_count() > 0, - ); - - // If probe side has NULL (detected in this or any other - // partition so far), return empty result. This is a best-effort - // early exit: the final stage re-checks the flag through - // `report_probe_completed`. - if build_side.left_data.probe_side_has_null_hint() { - timer.done(); - self.state = HashJoinStreamState::FetchProbeBatch; - return Ok(StatefulStreamResult::Continue); + // A correlated null-aware join (its join filter is the + // non-equality part of the subquery) decides UNKNOWN per build row + // instead of poisoning the whole join. + if self.filter.is_some() { + // Must precede the empty-build-map return below: an all-NULL-key + // build side has an empty full-key map but still needs UNKNOWN + // marks. + if state.offset == (0, None) { + mark_null_candidates_for_probe_batch( + build_side, + state, + self.filter.as_ref(), + self.join_type, + self.batch_size, + )?; + } + } else { + // Since null_aware validation ensures single column join, we only check the first column + let probe_key_column = &state.values[0]; + // Only batches with rows count: `NULL NOT IN (empty)` is TRUE. + build_side.left_data.record_probe_batch( + state.batch.num_rows() > 0, + probe_key_column.logical_null_count() > 0, + ); + + // If probe side has NULL (detected in this or any other + // partition so far), return empty result. This is a + // best-effort early exit: the final stage re-checks the flag + // through `report_probe_completed`. + if build_side.left_data.probe_side_has_null_hint() { + timer.done(); + self.state = HashJoinStreamState::FetchProbeBatch; + return Ok(StatefulStreamResult::Continue); + } } } @@ -978,6 +1000,9 @@ impl HashJoinStream { if self.null_aware && self.join_type == JoinType::LeftAnti { let (null_aware_left, null_aware_right) = null_aware_left_anti_final_indices( &build_side.left_data, + // A join filter makes the join correlated, so the UNKNOWN + // decision was recorded per build row by the probe phase. + self.filter.is_some(), probe_summary, left_side, right_side, @@ -1018,6 +1043,185 @@ impl HashJoinStream { } } +/// Records which build rows of a correlated null-aware join are UNKNOWN +/// candidates for this probe batch. +/// +/// An unmatched build row's `NOT IN` is UNKNOWN instead of TRUE (its row is +/// dropped instead of emitted) when either: +/// 1. its value key is NULL and some probe row passes the join filter, or +/// 2. some probe row with a NULL value key passes the join filter. +/// +/// The correlation scope of this fork's null-aware joins is carried by the +/// join filter alone (the null-aware validation allows a single join key), so +/// every pair on the relevant side is a candidate and the filter decides which +/// of them count. +/// +/// A build row stays UNKNOWN once it is marked, so candidates whose build row +/// is already marked are skipped, and the join filter is not evaluated for +/// them. This also ends the pairing as soon as no unmarked build row is left. +fn mark_null_candidates_for_probe_batch( + build_side: &BuildSideReadyState, + state: &ProcessProbeBatchState, + filter: Option<&JoinFilter>, + join_type: JoinType, + batch_size: usize, +) -> Result<()> { + let left_data = &build_side.left_data; + let probe_value_key = &state.values[0]; + let probe_has_null_values = probe_value_key.logical_null_count() > 0; + let build_value_key = &left_data.values()[0]; + let build_has_null_values = build_value_key.logical_null_count() > 0; + if !build_has_null_values && !probe_has_null_values { + return Ok(()); + } + + debug_assert_eq!( + left_data.values().len(), + state.values.len(), + "build/probe key counts must match" + ); + + // Keeps the candidate pairs that pass the join filter and marks their + // build rows as UNKNOWN. + let mut mark = |build_indices: UInt64Array, probe_indices: UInt32Array| { + let (build_indices, probe_indices) = + retain_unmarked(left_data, build_indices, probe_indices); + if build_indices.is_empty() { + return Ok(()); + } + let build_indices = match filter { + Some(filter) => { + apply_join_filter_to_indices( + left_data.batch(), + &state.batch, + build_indices, + probe_indices, + filter, + JoinSide::Left, + None, + join_type, + )? + .0 + } + None => build_indices, + }; + if !build_indices.is_empty() { + let mut null_bitmap = left_data.null_indices_bitmap().lock(); + for build_idx in build_indices.values() { + null_bitmap.set_bit(*build_idx as usize, true); + } + } + Ok(()) + }; + + // Case 1: build rows with a NULL value key are UNKNOWN as soon as any + // probe row passes the filter. + if build_has_null_values { + let null_mask = arrow::compute::is_null(build_value_key.as_ref())?; + let null_build_rows = UInt64Array::from_iter_values( + null_mask.values().set_indices().map(|idx| idx as u64), + ); + for_each_unmarked_cross_product( + left_data, + null_build_rows.values().iter().copied(), + 0..state.batch.num_rows() as u32, + batch_size, + &mut mark, + )?; + } + + // Case 2: NULL-valued probe rows make every build row that passes the + // filter an UNKNOWN candidate. + if probe_has_null_values { + let null_mask = arrow::compute::is_null(probe_value_key.as_ref())?; + let null_probe_rows = UInt32Array::from_iter_values( + null_mask.values().set_indices().map(|idx| idx as u32), + ); + for_each_unmarked_cross_product( + left_data, + 0..left_data.batch().num_rows() as u64, + null_probe_rows.values().iter().copied(), + batch_size, + &mut mark, + )?; + } + + Ok(()) +} + +/// Removes the candidate pairs whose build row is already marked UNKNOWN. +fn retain_unmarked( + left_data: &JoinLeftData, + build_indices: UInt64Array, + probe_indices: UInt32Array, +) -> (UInt64Array, UInt32Array) { + let bitmap = left_data.null_indices_bitmap().lock(); + let is_unmarked = |build_idx: &u64| !bitmap.get_bit(*build_idx as usize); + if build_indices.values().iter().all(is_unmarked) { + return (build_indices, probe_indices); + } + let (build, probe): (Vec, Vec) = build_indices + .values() + .iter() + .zip(probe_indices.values().iter()) + .filter(|(build_idx, _)| is_unmarked(build_idx)) + .unzip(); + (build.into(), probe.into()) +} + +/// Calls `f` with the pairs of `build_rows` x `probe_rows` whose build row is +/// not marked UNKNOWN, as chunks of at most `batch_size` pairs. +/// +/// `f` marks build rows, so the unmarked build rows are found again after each +/// chunk. The pairing stops when no unmarked build row is left. +fn for_each_unmarked_cross_product( + left_data: &JoinLeftData, + build_rows: impl Iterator, + probe_rows: impl Iterator, + batch_size: usize, + mut f: impl FnMut(UInt64Array, UInt32Array) -> Result<()>, +) -> Result<()> { + let retain_unmarked_rows = |rows: &mut Vec| { + let bitmap = left_data.null_indices_bitmap().lock(); + rows.retain(|idx| !bitmap.get_bit(*idx as usize)); + }; + + let mut build_rows: Vec = build_rows.collect(); + retain_unmarked_rows(&mut build_rows); + + let mut build_chunk = Vec::with_capacity(batch_size); + let mut probe_chunk = Vec::with_capacity(batch_size); + let mut marks_since_refresh = false; + for probe_row in probe_rows { + // Refresh only after a chunk was sent, so the cost of the refresh + // stays proportional to the pairs already evaluated. + if marks_since_refresh { + retain_unmarked_rows(&mut build_rows); + marks_since_refresh = false; + } + if build_rows.is_empty() { + break; + } + for build_row in &build_rows { + build_chunk.push(*build_row); + probe_chunk.push(probe_row); + if build_chunk.len() == batch_size { + f( + std::mem::replace(&mut build_chunk, Vec::with_capacity(batch_size)) + .into(), + std::mem::replace(&mut probe_chunk, Vec::with_capacity(batch_size)) + .into(), + )?; + marks_since_refresh = true; + } + } + } + if !build_chunk.is_empty() { + f(build_chunk.into(), probe_chunk.into())?; + } + Ok(()) +} + /// Final-stage rules of a null-aware `LeftAnti` join, evaluated by the last /// probe partition from what every partition together saw: /// - a NULL probe key seen by any partition makes `build.key NOT IN (probe)` @@ -1027,10 +1231,27 @@ impl HashJoinStream { /// kept). fn null_aware_left_anti_final_indices( left_data: &JoinLeftData, + correlated: bool, probe_summary: ProbeSideSummary, left_side: UInt64Array, right_side: UInt32Array, ) -> (UInt64Array, UInt32Array) { + if correlated { + // The probe phase recorded, per build row, whether the residual filter + // kept a NULL that makes its `NOT IN` UNKNOWN. Rows outside every + // correlation scope keep TRUE, which is what makes the empty-scope + // case (`NOT IN (empty)`) return them. + let null_indices_bitmap = left_data.null_indices_bitmap().lock(); + let left_side = UInt64Array::from_iter_values( + left_side + .values() + .iter() + .copied() + .filter(|idx| !null_indices_bitmap.get_bit(*idx as usize)), + ); + let right_side = UInt32Array::new_null(left_side.len()); + return (left_side, right_side); + } if probe_summary.has_null { return (UInt64Array::new_null(0), UInt32Array::new_null(0)); } diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt index bdb56cf22045a..7120458956b55 100644 --- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt +++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt @@ -414,6 +414,62 @@ SELECT * FROM test_table WHERE (c1 NOT IN (SELECT c2 FROM test_table)) = true; # This is a known limitation - currently only supports single column null-aware anti joins. # This will be addressed in next Phase (multi-column support). +############# +## Correlated NOT IN with a non-equality (residual) correlation +############# +## The correlation `residual_outer.g > 0` cannot become an equi-join key, so it +## stays a join filter. A NULL in the subquery then only makes `NOT IN` +## UNKNOWN for the outer rows the filter keeps: the rows with `g = 0` or +## `g IS NULL` see an empty subquery, and `NOT IN (empty)` is TRUE for them. + +statement ok +CREATE TABLE residual_outer(id INT, g INT) AS VALUES (1,5),(2,5),(3,0),(4,NULL),(NULL,5),(NULL,0); + +statement ok +CREATE TABLE residual_inner(id INT) AS VALUES (1),(NULL); + +query II rowsort +SELECT id, g FROM residual_outer +WHERE id NOT IN (SELECT id FROM residual_inner WHERE residual_outer.g > 0); +---- +3 0 +4 NULL +NULL 0 + +# The shape the executor fix covers: a single equi key plus the residual +# `> 0` correlation as the join filter, on a null-aware `LeftAnti`. +query TT +EXPLAIN SELECT id, g FROM residual_outer +WHERE id NOT IN (SELECT id FROM residual_inner WHERE residual_outer.g > 0); +---- +logical_plan +01)LeftAnti Join: residual_outer.id = __correlated_sq_1.id Filter: residual_outer.g > Int32(0) null_aware +02)--TableScan: residual_outer projection=[id, g] +03)--SubqueryAlias: __correlated_sq_1 +04)----TableScan: residual_inner projection=[id] +physical_plan +01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], filter=g@0 > 0, null_aware +02)--DataSourceExec: partitions=1, partition_sizes=[1] +03)--DataSourceExec: partitions=1, partition_sizes=[1] + +# Subquery left empty by the residual: NULL NOT IN (empty) is TRUE. +query II rowsort +SELECT id, g FROM residual_outer +WHERE id NOT IN (SELECT id FROM residual_inner WHERE residual_outer.g > 100); +---- +1 5 +2 5 +3 0 +4 NULL +NULL 0 +NULL 5 + +statement ok +DROP TABLE residual_outer; + +statement ok +DROP TABLE residual_inner; + ############# ## Cleanup ############# From 0b9bda87f725c05a9762fc31162aa9d8f2501670 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Nov 2026 15:02:04 +0800 Subject: [PATCH 7/9] fix: drop build rows with a dictionary NULL key in null-aware anti joins RE-IMPLEMENTATION, not a faithful cherry-pick. apache/datafusion#26578 (a79a694887a17fd5d78882974cd9645d14defbd3) reuses `null_aware_build_key_nulls`, which #25693 introduced for null-aware `LeftMark` - a join type this fork does not have - and `git show a79a6948 | git apply --check` fails on both touched files. The bug: the final stage kept a build row whose key is a dictionary entry pointing at a NULL value, although `NULL NOT IN (5, 6)` is UNKNOWN and the row should be dropped exactly as when the key is a physical NULL. Changes: - `JoinLeftData` stores the logical nulls of the scalar build key for an uncorrelated null-aware join, computed once with the build side (`logical_nulls()`), charged to the memory reservation only when it materializes a new buffer instead of reusing the input's. - `null_aware_left_anti_final_indices` filters the final build rows by those logical nulls instead of `Array::is_null`. Test: `test_null_aware_left_anti_build_logical_null` (`NULL NOT IN (5, 6)` with a dictionary key that points at a NULL dictionary value), from the upstream PR. (backport of apache/datafusion#26578, a79a694887a17fd5d78882974cd9645d14defbd3) Signed-off-by: discord9 (cherry picked from commit 44a08c06981fd370993073cd9c4f75a9fd91e5eb) --- .../physical-plan/src/joins/hash_join/exec.rs | 117 +++++++++++++++++- .../src/joins/hash_join/stream.rs | 9 +- 2 files changed, 120 insertions(+), 6 deletions(-) diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index af8ec65ecf40b..0007fac78246a 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -71,6 +71,7 @@ use crate::{ }; use arrow::array::{Array, ArrayRef, BooleanBufferBuilder}; +use arrow::buffer::NullBuffer; use arrow::compute::concat_batches; use arrow::datatypes::SchemaRef; use arrow::record_batch::RecordBatch; @@ -216,6 +217,12 @@ pub(super) struct JoinBuildData { pub(super) struct JoinLeftData { /// Shared bitmap for visited build indices within this join execution. visited_indices_bitmap: SharedBitmapBuilder, + /// Logical nulls of the scalar build key for an uncorrelated null-aware + /// `LeftAnti` join. + /// + /// Computed once while collecting the build side because dictionary + /// logical-null discovery can scan nested dictionary values. + null_aware_build_key_nulls: Option, /// Bitmap of the build rows whose `NOT IN` is UNKNOWN instead of TRUE. /// /// Filled by the probe phase of a correlated null-aware join, whose @@ -276,6 +283,12 @@ impl JoinLeftData { &self.null_indices_bitmap } + /// returns the logical nulls of the scalar build key, when they were + /// precomputed + pub(super) fn null_aware_build_key_nulls(&self) -> Option<&NullBuffer> { + self.null_aware_build_key_nulls.as_ref() + } + /// returns a reference to the InList values for filter pushdown pub(super) fn membership(&self) -> &PushdownStrategy { &self.membership @@ -1593,8 +1606,11 @@ impl ExecutionPlan for HashJoinExec { join_metrics.clone(), reservation, need_produce_result_in_final(self.join_type), - // Correlated null-aware joins record UNKNOWN per build row. + // Correlated null-aware joins record UNKNOWN per build row, + // uncorrelated ones precompute the logical nulls of the + // scalar build key for the final stage. self.null_aware && self.filter.is_some(), + self.null_aware && self.filter.is_none(), right_partitions, enable_dynamic_filter_pushdown, Arc::clone(context.session_config().options()), @@ -1617,6 +1633,7 @@ impl ExecutionPlan for HashJoinExec { reservation, need_produce_result_in_final(self.join_type), self.null_aware && self.filter.is_some(), + self.null_aware && self.filter.is_none(), 1, enable_dynamic_filter_pushdown, Arc::clone(context.session_config().options()), @@ -2416,6 +2433,8 @@ fn concat_build_batches( /// * `with_visited_indices_bitmap` - Whether to track visited indices (for outer joins) /// * `with_null_aware_row_state` - Whether to track the per-build-row UNKNOWN /// decision of a correlated null-aware join +/// * `with_null_aware_build_key_nulls` - Whether to precompute the logical +/// nulls of the scalar build key for an uncorrelated null-aware join /// * `probe_threads_count` - Number of threads that will probe this hash table /// * `should_compute_dynamic_filters` - Whether to compute min/max bounds for dynamic filtering /// @@ -2447,6 +2466,7 @@ async fn collect_left_input( reservation: MemoryReservation, with_visited_indices_bitmap: bool, with_null_aware_row_state: bool, + with_null_aware_build_key_nulls: bool, probe_threads_count: usize, should_compute_dynamic_filters: bool, config: Arc, @@ -2655,6 +2675,30 @@ async fn collect_left_input( BooleanBufferBuilder::new(0) }; + // Logical nulls, so that a dictionary key pointing at a NULL dictionary + // value counts as a NULL build key in the final stage. + let null_aware_build_key_nulls = + if with_null_aware_build_key_nulls && !left_values.is_empty() { + left_values[0] + .logical_nulls() + .filter(|nulls| nulls.null_count() > 0) + } else { + None + }; + if let Some(nulls) = &null_aware_build_key_nulls { + // `logical_nulls` reuses an array's physical null buffer when it can. + // Only charge newly materialized buffers, such as dictionary logical + // nulls, because the build batch's buffers were reserved above. + let reuses_input_buffer = left_values[0] + .nulls() + .is_some_and(|input| input.inner().ptr_eq(nulls.inner())); + if !reuses_input_buffer { + let retained_size = nulls.buffer().capacity(); + reservation.try_grow(retained_size)?; + metrics.build_mem_used.add(retained_size); + } + } + let map = Arc::new(join_hash_map); // For an ordinary build, nothing reads the strategy unless this join's own @@ -2732,6 +2776,7 @@ async fn collect_left_input( let data = JoinLeftData { visited_indices_bitmap: Mutex::new(visited_indices_bitmap), null_indices_bitmap: Mutex::new(null_indices_bitmap), + null_aware_build_key_nulls, probe_completion: ProbeCompletion::new(probe_threads_count), membership, _visited_reservation: visited_reservation, @@ -2811,7 +2856,6 @@ mod tests { ScalarValue, assert_batches_eq, assert_batches_sorted_eq, assert_contains, exec_err, internal_err, }; - use futures::StreamExt; use datafusion_execution::config::SessionConfig; use datafusion_execution::memory_pool::{ GreedyMemoryPool, MemoryPool, UnboundedMemoryPool, @@ -2822,6 +2866,7 @@ mod tests { use datafusion_physical_expr::{ EquivalenceProperties, PhysicalSortExpr, RangePartitioning, SplitPoint, }; + use futures::StreamExt; use hashbrown::HashTable; use insta::{allow_duplicates, assert_snapshot}; use rstest::*; @@ -8931,8 +8976,8 @@ mod tests { +----+-------+ | c1 | dummy | +----+-------+ - | 1 | 10 | | | 0 | + | 1 | 10 | +----+-------+ "); } @@ -9041,6 +9086,72 @@ mod tests { Ok(()) } + /// `NULL NOT IN (5, 6)` is UNKNOWN, so the build row whose key points at a + /// NULL dictionary value must be dropped, as it is when the key is a + /// physical NULL. + #[apply(hash_join_exec_configs)] + #[tokio::test] + async fn test_null_aware_left_anti_build_logical_null( + batch_size: usize, + ) -> Result<()> { + let task_ctx = prepare_task_ctx(batch_size, false); + + let left = build_table_dict_key( + "c1", + vec![Some(1), None, Some(2)], + vec![0, 1, 2], + "dummy", + vec![Some(10), Some(0), Some(20)], + ); + let left_key = left + .execute(0, Arc::clone(&task_ctx))? + .next() + .await + .unwrap()?; + assert_eq!(left_key.column(0).null_count(), 0); + assert_eq!(left_key.column(0).logical_null_count(), 1); + + let right = build_table_dict_key( + "c2", + vec![Some(5), Some(6)], + vec![0, 1], + "dummy", + vec![Some(50), Some(60)], + ); + + let on = vec![( + Arc::new(Column::new_with_schema("c1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("c2", &right.schema())?) as _, + )]; + + let join = HashJoinExec::try_new( + left, + right, + on, + None, + &JoinType::LeftAnti, + None, + PartitionMode::CollectLeft, + NullEquality::NullEqualsNothing, + true, + )?; + + let stream = join.execute(0, task_ctx)?; + let batches = common::collect(stream).await?; + + allow_duplicates! { + assert_snapshot!(batches_to_sort_string(&batches), @r" + +----+-------+ + | c1 | dummy | + +----+-------+ + | 1 | 10 | + | 2 | 20 | + +----+-------+ + "); + } + Ok(()) + } + /// Test that null_aware validation rejects non-LeftAnti join types #[tokio::test] async fn test_null_aware_validation_wrong_join_type() { diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rs index 14cafc3ad2c59..79a59b3126747 100644 --- a/datafusion/physical-plan/src/joins/hash_join/stream.rs +++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs @@ -1258,12 +1258,15 @@ fn null_aware_left_anti_final_indices( if !probe_summary.non_empty { return (left_side, right_side); } - // null_aware validation ensures a single join key - let build_key_column = &left_data.values()[0]; + // Logical nulls, so that a dictionary key pointing at a NULL value counts + // as a NULL build key. Computed once with the build side. + let Some(build_key_nulls) = left_data.null_aware_build_key_nulls() else { + return (left_side, right_side); + }; let left_side = UInt64Array::from_iter_values(left_side.iter().filter_map(|idx| { let idx = idx.expect("LeftAnti final indices should always contain build-side rows"); - (!build_key_column.is_null(idx as usize)).then_some(idx) + (!build_key_nulls.is_null(idx as usize)).then_some(idx) })); let right_side = UInt32Array::new_null(left_side.len()); (left_side, right_side) From cd9d9eaec3e3b5185508ec2ac6e992cad15a934a Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 9 Nov 2026 15:40:44 +0800 Subject: [PATCH 8/9] perf: stop building unmatched indices once a probe NULL forces an empty result Follow-up to the #25076 backport. When any probe partition saw a NULL join key, `build.key NOT IN (probe)` is UNKNOWN for every build row, so a null-aware LeftAnti emits nothing. This fork still materialises the whole final index set in `get_final_indices_from_shared_bitmap` before `null_aware_left_anti_final_indices` discards it, which costs an `O(build rows)` scan plus `UInt64Array(N)` and `UInt32Array(N)` (~12 bytes per build row) that are dropped immediately. The base code returned before building those indices. The #25076 backport moved the decision behind them, because the completion protocol has to run first: only the last probe partition gets the final `ProbeSideSummary`. Restore the early return after the summary is obtained and before the indices are built. The `AcqRel` decrement and the ordering guarantee are untouched; correlated joins keep going because their UNKNOWN decision is per build row and already recorded by the probe phase. Signed-off-by: discord9 --- .../physical-plan/src/joins/hash_join/stream.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rs index 79a59b3126747..9341b51a0e0f7 100644 --- a/datafusion/physical-plan/src/joins/hash_join/stream.rs +++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs @@ -988,6 +988,23 @@ impl HashJoinStream { return Ok(StatefulStreamResult::Continue); }; + // A NULL probe key seen by any partition makes `build.key NOT IN (probe)` + // UNKNOWN for every build row, so nothing is emitted. Decide that before + // materialising the unmatched indices: an uncorrelated join would build + // the whole index set only to have `null_aware_left_anti_final_indices` + // discard it, which costs an `O(build rows)` scan and two temporary arrays. + // Correlated joins keep going, because their UNKNOWN decision is per build + // row and already recorded by the probe phase. + if self.null_aware + && self.join_type == JoinType::LeftAnti + && self.filter.is_none() + && probe_summary.has_null + { + timer.done(); + self.state = HashJoinStreamState::Completed; + return Ok(StatefulStreamResult::Continue); + } + // use the global left bitmap to produce the left indices and right indices let (mut left_side, mut right_side) = get_final_indices_from_shared_bitmap( build_side.left_data.visited_indices_bitmap(), From 1da34f97699c6ae8dd8af6782048704b8d3137cb Mon Sep 17 00:00:00 2001 From: discord9 Date: Tue, 10 Nov 2026 17:00:49 +0800 Subject: [PATCH 9/9] adapt: keep prepared hash-join builds in step with the probe-completion state #39 introduced `PreparedHashJoinBuild::probe_data`, which builds a fresh `JoinLeftData` per consuming join. It still created the `probe_threads_counter` and the two probe-side atomics that the null-aware backports replace with `ProbeCompletion`, and the `collect_left_input` call it makes predates the two null-aware flags. A prepared build is only reachable for ordinary joins: `prepared_key_indices` requires `INNER`, `CollectLeft` and `!null_aware`, and both `prepare_build` and `execute` re-check it. The per-join null bitmaps therefore start empty and `null_aware_build_key_nulls` stays `None`. The correlation bitmap of a correlated null-aware join is mutable per-execution state, so it is charged to the same `_visited_reservation` lease as the visited bitmap rather than to the build reservation. #39 hands that build data out through `build_lease`, and a dynamic filter can retain it after this join's own state is dropped; charging the bitmap there would keep paying for memory that is already released. `correlated_bitmap_reservation_is_execution_local` pins that down: with the correlated flag on, dropping `JoinLeftData` returns the pool to the producer's charge alone, and dropping the retained lease returns it to zero. Signed-off-by: discord9 --- .../physical-plan/src/joins/hash_join/exec.rs | 72 +++++++++++++++++-- .../src/joins/hash_join/exec/prepared.rs | 10 ++- 2 files changed, 75 insertions(+), 7 deletions(-) diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 0007fac78246a..664b99141e4b5 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -234,7 +234,10 @@ pub(super) struct JoinLeftData { probe_completion: ProbeCompletion, /// IN-list values or a hash-table reference used for filter pushdown. membership: PushdownStrategy, - /// The mutable visited bitmap belongs to this execution, not the build owner. + /// The mutable probe-side bitmaps belong to this execution, not the build + /// owner: the visited bitmap and, for a correlated null-aware join, the + /// correlation bitmap. Charging them here keeps a leased build from + /// carrying the cost of state that belongs to one consuming join. _visited_reservation: MemoryReservation, build: Arc, } @@ -2662,11 +2665,13 @@ async fn collect_left_input( }; // Shared by all probe partitions of a correlated null-aware join, which - // record the per-build-row UNKNOWN decision here. Charged to the build - // reservation, which stays alive as long as this data is reachable. + // record the per-build-row UNKNOWN decision here. It is per-execution + // state, so it is charged to the same lease as the visited bitmap: a + // dynamic filter can retain the build data after this join's own state is + // dropped, and that retained lease must not carry this bitmap's cost. let null_indices_bitmap = if with_null_aware_row_state { let bitmap_size = bit_util::ceil(batch.num_rows(), 8); - reservation.try_grow(bitmap_size)?; + visited_reservation.try_grow(bitmap_size)?; metrics.build_mem_used.add(bitmap_size); let mut bitmap_buffer = BooleanBufferBuilder::new(batch.num_rows()); bitmap_buffer.append_n(num_rows, false); @@ -7594,6 +7599,8 @@ mod tests { BuildProbeJoinMetrics::new(0, &metrics_set), MemoryConsumer::new("build quota").register(&unbounded), false, + false, + false, 1, false, Arc::clone(&config), @@ -7623,6 +7630,8 @@ mod tests { BuildProbeJoinMetrics::new(0, &metrics_set), MemoryConsumer::new("build quota").register(&pool), false, + false, + false, 1, false, Arc::clone(&config), @@ -7815,6 +7824,8 @@ mod tests { BuildProbeJoinMetrics::new(0, &metrics_set), MemoryConsumer::new("visited lease").register(&pool), true, + false, + false, 1, true, Arc::new(ConfigOptions::default()), @@ -7846,6 +7857,57 @@ mod tests { Ok(()) } + /// The correlation bitmap of a correlated null-aware join is mutable + /// per-execution state too: a retained build lease must not keep paying + /// for it once the join's own state is gone. + #[tokio::test] + async fn correlated_bitmap_reservation_is_execution_local() -> Result<()> { + use crate::memory::MemoryStream; + let values: Vec = (0..1024).collect(); + let batch = build_table_i32(("a1", &values), ("b1", &values), ("c1", &values)); + let schema = batch.schema(); + let pool: Arc = Arc::new(UnboundedMemoryPool::default()); + let metrics_set = ExecutionPlanMetricsSet::new(); + let data = collect_left_input( + HASH_JOIN_SEED.random_state().clone(), + Box::pin(MemoryStream::try_new(vec![batch], schema, None)?), + vec![Arc::new(Column::new("a1", 0))], + BuildProbeJoinMetrics::new(0, &metrics_set), + MemoryConsumer::new("correlated lease").register(&pool), + true, + true, + false, + 1, + true, + Arc::new(ConfigOptions::default()), + NullEquality::NullEqualsNothing, + MetricBuilder::new(&metrics_set) + .counter(ARRAY_MAP_CREATED_COUNT_METRIC_NAME, 0), + BuildMode::Ordinary, + ) + .await?; + let producer_bytes = data.build.reservation.size(); + #[expect(clippy::used_underscore_binding)] + let visited_bytes = data._visited_reservation.size(); + let bitmap_bytes = data.membership.bitmap_reserved_bytes(); + // Visited bitmap plus the correlation bitmap, one bit per build row each. + assert_eq!(visited_bytes, 2 * (1024 / 8)); + assert_eq!( + pool.reserved(), + producer_bytes + visited_bytes + bitmap_bytes + ); + let producer = Arc::clone(&data.build); + drop(data); + assert_eq!( + pool.reserved(), + producer_bytes, + "both mutable bitmaps must release with execution, not with the lease" + ); + drop(producer); + assert_eq!(pool.reserved(), 0); + Ok(()) + } + /// A build lease retains the batch, computed keys, map, and their reservation. #[rstest] #[tokio::test] @@ -7880,6 +7942,8 @@ mod tests { metrics, reservation, false, + false, + false, 1, false, config, diff --git a/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs b/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs index 9ec405e919bab..b02c3ce92f3ee 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs @@ -73,9 +73,11 @@ impl PreparedHashJoinBuild { JoinLeftData { build: Arc::clone(&self.build), visited_indices_bitmap: Mutex::new(BooleanBufferBuilder::new(0)), - probe_threads_counter: AtomicUsize::new(probe_threads), - probe_side_non_empty: AtomicBool::new(false), - probe_side_has_null: AtomicBool::new(false), + probe_completion: ProbeCompletion::new(probe_threads), + // A prepared build serves ordinary joins only: correlated + // null-aware joins are rejected before they can consume one. + null_indices_bitmap: Mutex::new(BooleanBufferBuilder::new(0)), + null_aware_build_key_nulls: None, membership: self.membership.clone(), _visited_reservation: self.build.reservation.new_empty(), } @@ -193,6 +195,8 @@ impl HashJoinExec { metrics, reservation, false, + false, + false, 0, !byte_keys, config,