diff --git a/.github/workflows/extended.yml b/.github/workflows/extended.yml index a6e303e3d6ff4..ae7408ae771fa 100644 --- a/.github/workflows/extended.yml +++ b/.github/workflows/extended.yml @@ -88,6 +88,9 @@ jobs: - name: Run tests (excluding doctests) env: RUST_BACKTRACE: 1 + # Run more random scenarios of the spill pool fuzzer than the + # default suite does (about a minute). + DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS: 1000 run: | cargo test \ --profile ci \ diff --git a/Cargo.lock b/Cargo.lock index 9f0fa41743af7..6bc415c8825db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2513,6 +2513,7 @@ dependencies = [ "insta", "itertools 0.15.0", "log", + "loom", "num-traits", "parking_lot", "pin-project-lite", @@ -3216,6 +3217,21 @@ dependencies = [ "prost-build", ] +[[package]] +name = "generator" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ade96dc9003043bce7c035c85a9df5a858bfb2039c5a2e6fdf00f324f6c551" +dependencies = [ + "cc", + "cfg-if", + "libc", + "log", + "rustversion", + "windows-link", + "windows-result", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -4072,6 +4088,19 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +[[package]] +name = "loom" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" +dependencies = [ + "cfg-if", + "generator", + "scoped-tls", + "tracing", + "tracing-subscriber", +] + [[package]] name = "lru-slab" version = "0.1.2" @@ -4087,6 +4116,15 @@ dependencies = [ "twox-hash", ] +[[package]] +name = "matchers" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" +dependencies = [ + "regex-automata", +] + [[package]] name = "matchit" version = "0.8.4" @@ -5529,6 +5567,12 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -6566,10 +6610,14 @@ version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex-automata", "sharded-slab", "smallvec", "thread_local", + "tracing", "tracing-core", "tracing-log", ] diff --git a/Cargo.toml b/Cargo.toml index 7060b1df93f26..a2ae6695a712e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -174,6 +174,7 @@ itertools = "0.15" itoa = "1.0" liblzma = { version = "0.4.6", features = ["static"] } log = "^0.4" +loom = "0.7" memchr = "2.8.1" num-traits = { version = "0.2" } object_store = { version = "0.13.2", default-features = false } @@ -230,6 +231,11 @@ unexpected_cfgs = { level = "warn", check-cfg = [ 'cfg(datafusion_coop, values("tokio", "tokio_fallback", "per_stream"))', "cfg(coverage)", "cfg(coverage_nightly)", + # Enables the loom model-checked concurrency tests; see + # datafusion/physical-plan/src/joins/hash_join/probe_completion.rs. + # Namespaced rather than a bare `loom`, which tokio also reads and which + # would switch tokio itself into its loom build. + "cfg(datafusion_loom)", ] } unused_qualifications = "deny" diff --git a/datafusion/core/tests/memory_limit/repartition_mem_limit.rs b/datafusion/core/tests/memory_limit/repartition_mem_limit.rs index 27bcd33926e99..21a0eaec5473d 100644 --- a/datafusion/core/tests/memory_limit/repartition_mem_limit.rs +++ b/datafusion/core/tests/memory_limit/repartition_mem_limit.rs @@ -117,3 +117,157 @@ async fn test_repartition_memory_limit() { ]; assert_batches_sorted_eq!(expected, &all_batches); } + +/// Regression test for . +/// +/// In non-preserve-order mode every input task of a `RepartitionExec` shares one +/// spill pool per output partition. When two input tasks spilled concurrently the +/// pool could end up with two open spill files while the reader only ever drained +/// the head file: it parked on a drained-but-unfinished head file even though the +/// batch it was waiting for had been written to the second file. Once every +/// distributor channel was non-empty the gate closed, both input tasks parked in +/// `send`, no sink was dropped, and nothing could wake anybody. +/// +/// This module is not compiled with `force_hash_collisions`. That feature makes +/// every hash 0, so the hash `RepartitionExec` sends all rows to one output +/// partition. The gate closes only when every channel holds data, so the +/// deadlock cannot occur and the test cannot test anything. The feature also +/// makes the group-by hash table degenerate, which made one attempt of this +/// query approximately 36 times slower and thus longer than the limit below. +#[cfg(not(feature = "force_hash_collisions"))] +mod spill_pool_deadlock { + use super::*; + + use std::time::Duration; + + use datafusion::execution::session_state::SessionStateBuilder; + use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; + use datafusion_execution::memory_pool::{GreedyMemoryPool, MemoryPool}; + use datafusion_physical_plan::collect; + + /// Memory limit that puts `RepartitionExec` into its spilling path for this + /// query without failing the aggregation outright. + const MEMORY_LIMIT: usize = 4 * 1024 * 1024; + + /// Number of attempts. On an unfixed tree the first attempt deadlocks; the + /// budget is only there so a fix cannot pass by luck. + const ATTEMPTS: usize = 12; + + /// Generous per-attempt budget: a healthy run of this query takes well under + /// a second. + const PER_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(20); + + /// Runs the query once. Returns the number of output rows and the number + /// of spills the `RepartitionExec`s did. + async fn run_once() -> datafusion::error::Result<(usize, usize)> { + let pool: Arc = Arc::new(GreedyMemoryPool::new(MEMORY_LIMIT)); + let runtime = RuntimeEnvBuilder::new() + // Spilling must be possible: with the disk manager disabled the query + // fails with a resource error instead of deadlocking. + .with_disk_manager_builder( + DiskManagerBuilder::default().with_mode(DiskManagerMode::OsTmpDirectory), + ) + .with_memory_pool(pool) + .build_arc()?; + + let config = SessionConfig::new() + // Two input partitions feeding one hash RepartitionExec is the + // smallest configuration with two concurrent writers per spill pool. + .with_target_partitions(2) + // Small batches so many small batches reach the repartition spill + // path while the spill file stays far below + // `max_spill_file_size_bytes` and so never rotates. + .with_batch_size(64); + + let state = SessionStateBuilder::new() + .with_config(config) + .with_runtime_env(runtime) + .with_default_features() + .build(); + let ctx = SessionContext::new_with_state(state); + + ctx.sql( + "create table trace_events as + select v % 13 as g, + case when v % 29 = 0 then null + else md5(cast(v % 337 as varchar)) end as trace_id + from generate_series(1, 40000) as t(v)", + ) + .await? + .collect() + .await?; + + ctx.sql( + "create view tv as + select g, arrow_cast(trace_id, 'Utf8View') as trace_id from trace_events", + ) + .await? + .collect() + .await?; + + let logical = ctx + .state() + .create_logical_plan( + "select g, count(distinct trace_id) as n from tv group by g", + ) + .await?; + let plan = ctx.state().create_physical_plan(&logical).await?; + let batches = collect(Arc::clone(&plan), ctx.task_ctx()).await?; + + // Count the spills of the `RepartitionExec`s. The caller uses this to + // make sure the query still goes through the spill path: if a change + // stops it spilling, this test keeps passing but no longer covers the + // deadlock, and that must fail loudly instead. + let mut spills = 0; + plan.transform_down(|node| { + if node.is::() { + spills += node.metrics().and_then(|m| m.spill_count()).unwrap_or(0); + } + Ok(Transformed::no(node)) + })?; + + Ok((batches.iter().map(|b| b.num_rows()).sum(), spills)) + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 8)] + async fn repartition_spill_pool_does_not_deadlock() { + let mut attempts_that_spilled = 0; + for attempt in 0..ATTEMPTS { + match tokio::time::timeout(PER_ATTEMPT_TIMEOUT, run_once()).await { + Err(elapsed) => panic!( + "attempt {attempt}: grouped COUNT(DISTINCT Utf8View) never completed \ + within {PER_ATTEMPT_TIMEOUT:?} ({elapsed}); RepartitionExec spill \ + pool deadlock" + ), + Ok(Ok((rows, spills))) => { + assert_eq!(rows, 13, "attempt {attempt}: wrong row count"); + if spills > 0 { + attempts_that_spilled += 1; + } + } + // The budget is deliberately far too small for the query, and the + // greedy pool hands memory out first come first served, so once in + // a few thousand attempts an aggregate's allocation is refused + // while the repartition reservations hold the pool. That is a + // legitimate outcome of the limit, not the hang this test guards + // against. + Ok(Err(e)) + if matches!( + e.find_root(), + datafusion::error::DataFusionError::ResourcesExhausted(_) + ) => {} + Ok(Err(e)) => panic!("attempt {attempt}: query failed: {e}"), + } + } + + // The memory limit must be tight enough that `RepartitionExec` spills, + // because the deadlock is in its spill pool. Timing decides how many + // attempts spill, but if none of them do, the limit no longer forces + // the spill path and this test has stopped testing anything. + assert!( + attempts_that_spilled > 0, + "no attempt reached the RepartitionExec spill path in {ATTEMPTS} \ + attempts; the memory limit no longer forces a spill" + ); + } +} diff --git a/datafusion/core/tests/parquet/row_group_pruning.rs b/datafusion/core/tests/parquet/row_group_pruning.rs index 0721715921909..91195ae04c79c 100644 --- a/datafusion/core/tests/parquet/row_group_pruning.rs +++ b/datafusion/core/tests/parquet/row_group_pruning.rs @@ -20,8 +20,8 @@ //! expected. use std::sync::Arc; -use arrow::array::{ArrayRef, Int32Array, RecordBatch}; -use arrow_schema::{DataType, Field, Schema}; +use arrow::array::{ArrayRef, Int32Array, RecordBatch, TimestampSecondArray}; +use arrow_schema::{DataType, Field, Schema, TimeUnit}; use datafusion::prelude::SessionConfig; use datafusion_common::{DataFusionError, ScalarValue}; use itertools::Itertools; @@ -306,6 +306,51 @@ async fn prune_timestamps_seconds() { .await; } +#[tokio::test] +async fn timestamp_to_time_cast_across_midnight_is_not_pruned() { + let schema = Arc::new(Schema::new(vec![Field::new( + "ts", + DataType::Timestamp(TimeUnit::Second, None), + false, + )])); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(TimestampSecondArray::from(vec![ + 82_800, 86_400, 90_000, + ]))], + ) + .unwrap(); + + for (time_type, op, literal, expected_rows) in [ + ("Time32(Second)", "=", "00:00:00", 1), + ("Time32(Second)", "<", "02:00:00", 2), + ("Time64(Nanosecond)", "=", "00:00:00", 1), + ("Time64(Nanosecond)", "<", "02:00:00", 2), + ] { + let query = format!( + "SELECT ts FROM t WHERE arrow_cast(ts, '{time_type}') {op} \ + arrow_cast('{literal}', '{time_type}')" + ); + + RowGroupPruningTest::new() + .with_scenario(Scenario::Timestamps) + .with_query(&query) + .with_expected_errors(Some(0)) + .with_matched_by_stats(Some(1)) + .with_fully_matched_by_stats(Some(0)) + .with_pruned_by_stats(Some(0)) + .with_pruned_files(Some(0)) + .with_limit_pruned_row_groups(Some(0)) + .with_expected_rows(expected_rows) + .test_row_group_prune_with_custom_data( + Arc::clone(&schema), + vec![batch.clone()], + 3, + ) + .await; + } +} + #[tokio::test] async fn prune_date32() { RowGroupPruningTest::new() diff --git a/datafusion/physical-optimizer/src/join_selection.rs b/datafusion/physical-optimizer/src/join_selection.rs index 42736f8205089..2f88385781037 100644 --- a/datafusion/physical-optimizer/src/join_selection.rs +++ b/datafusion/physical-optimizer/src/join_selection.rs @@ -283,9 +283,10 @@ pub(crate) fn partitioned_hash_join( hash_join.swap_inputs(PartitionMode::Partitioned) } else { // Null-aware anti joins must use CollectLeft mode because they track probe-side state - // (probe_side_non_empty, probe_side_has_null) per-partition, but need global knowledge - // for correct null handling. With partitioning, a partition might not see probe rows - // even if the probe side is globally non-empty, leading to incorrect NULL row handling. + // (did any probe partition see a row, did any see a NULL key) that has to be shared + // between partitions for correct null handling. With partitioning, a partition might + // not see probe rows even if the probe side is globally non-empty, leading to + // incorrect NULL row handling. let partition_mode = if hash_join.null_aware { PartitionMode::CollectLeft } else { diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml index 50d02162cfee9..b84940b5b5feb 100644 --- a/datafusion/physical-plan/Cargo.toml +++ b/datafusion/physical-plan/Cargo.toml @@ -105,6 +105,25 @@ tokio = { workspace = true, features = [ "parking_lot", ] } +# Model checker for the hash join probe-completion protocol; see +# joins/hash_join/probe_completion.rs. +# +# Gated on `--cfg datafusion_loom` rather than declared a plain dev-dependency: loom pulls +# in tracing-subscriber, which turns on `regex-automata`'s DFA features for +# every crate in the workspace build. That is not inert. It changes which +# engine the shared `regex` picks and costs enough extra stack to overflow +# unrelated deep-recursion tests (`sql::unparser::test_tpch_unparser_roundtrip` +# aborts in CI). Keeping loom behind its own cfg leaves the ordinary build +# untouched. +# +# Run the model checks with: +# RUSTFLAGS="--cfg datafusion_loom" cargo test -p datafusion-physical-plan --lib loom_tests +# +# The cfg is namespaced because a bare `loom` is also read by tokio, which +# would switch tokio into its own loom build and drop `tokio::fs`. +[target.'cfg(datafusion_loom)'.dev-dependencies] +loom = { workspace = true } + [[bench]] harness = false name = "partial_ordering" diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 3a30bf5f7053b..664b99141e4b5 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -18,7 +18,6 @@ use std::collections::HashSet; use std::fmt; use std::mem::size_of; -use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, OnceLock}; use std::vec; @@ -33,6 +32,7 @@ use crate::filter_pushdown::{ use crate::joins::Map; use crate::joins::array_map::ArrayMap; use crate::joins::hash_join::inlist_builder::build_struct_inlist_values; +use crate::joins::hash_join::probe_completion::{ProbeCompletion, ProbeSideSummary}; use crate::joins::hash_join::shared_bounds::{ ColumnBounds, PartitionBounds, PushdownStrategy, SharedBuildAccumulator, }; @@ -71,6 +71,7 @@ use crate::{ }; use arrow::array::{Array, ArrayRef, BooleanBufferBuilder}; +use arrow::buffer::NullBuffer; use arrow::compute::concat_batches; use arrow::datatypes::SchemaRef; use arrow::record_batch::RecordBatch; @@ -216,15 +217,27 @@ pub(super) struct JoinBuildData { pub(super) struct JoinLeftData { /// Shared bitmap for visited build indices within this join execution. visited_indices_bitmap: SharedBitmapBuilder, - /// Counter of running probe partitions that may update the bitmap. - probe_threads_counter: AtomicUsize, - /// Whether any probe partition saw data (for null-aware anti joins). - pub(super) probe_side_non_empty: AtomicBool, - /// Whether any probe partition saw a NULL key (for null-aware anti joins). - pub(super) probe_side_has_null: AtomicBool, + /// Logical nulls of the scalar build key for an uncorrelated null-aware + /// `LeftAnti` join. + /// + /// Computed once while collecting the build side because dictionary + /// logical-null discovery can scan nested dictionary values. + null_aware_build_key_nulls: Option, + /// Bitmap of the build rows whose `NOT IN` is UNKNOWN instead of TRUE. + /// + /// Filled by the probe phase of a correlated null-aware join, whose + /// per-build-row decision replaces the shared probe-side flags (see + /// [`HashJoinExec::null_aware`]). + null_indices_bitmap: SharedBitmapBuilder, + /// Tracks which probe partition finishes last and what the partitions + /// collectively saw. See [`ProbeCompletion`] for the invariant it upholds. + probe_completion: ProbeCompletion, /// IN-list values or a hash-table reference used for filter pushdown. membership: PushdownStrategy, - /// The mutable visited bitmap belongs to this execution, not the build owner. + /// The mutable probe-side bitmaps belong to this execution, not the build + /// owner: the visited bitmap and, for a correlated null-aware join, the + /// correlation bitmap. Charging them here keeps a leased build from + /// carrying the cost of state that belongs to one consuming join. _visited_reservation: MemoryReservation, build: Arc, } @@ -267,6 +280,18 @@ impl JoinLeftData { &self.visited_indices_bitmap } + /// returns a reference to the bitmap of build rows whose `NOT IN` is + /// UNKNOWN + pub(super) fn null_indices_bitmap(&self) -> &SharedBitmapBuilder { + &self.null_indices_bitmap + } + + /// returns the logical nulls of the scalar build key, when they were + /// precomputed + pub(super) fn null_aware_build_key_nulls(&self) -> Option<&NullBuffer> { + self.null_aware_build_key_nulls.as_ref() + } + /// returns a reference to the InList values for filter pushdown pub(super) fn membership(&self) -> &PushdownStrategy { &self.membership @@ -281,10 +306,29 @@ impl JoinLeftData { self.build.bounds.as_ref() } - /// Decrements the counter of running threads, and returns `true` - /// if caller is the last running thread - pub(super) fn report_probe_completed(&self) -> bool { - self.probe_threads_counter.fetch_sub(1, Ordering::Relaxed) == 1 + /// Records what a probe partition saw in one batch, for the null-aware + /// rules evaluated in the final stage. + pub(super) fn record_probe_batch(&self, non_empty: bool, has_null: bool) { + self.probe_completion.record_batch(non_empty, has_null); + } + + /// Whether some probe partition has already recorded a NULL join key. + /// + /// Only an early-exit hint for the probe phase: it may lag behind sibling + /// partitions. Final-stage decisions must use the [`ProbeSideSummary`] + /// returned by [`Self::report_probe_completed`] instead. + pub(super) fn probe_side_has_null_hint(&self) -> bool { + self.probe_completion.saw_null_key_hint() + } + + /// Marks this probe partition as finished, returning `Some` for the last + /// one together with what every partition saw. + /// + /// The summary is reachable only through this call, which is what stops + /// the final stage from reading the shared flags before its own + /// decrement. See [`ProbeCompletion`] for why that ordering matters. + pub(super) fn report_probe_completed(&self) -> Option { + self.probe_completion.report_completed() } } @@ -806,6 +850,15 @@ pub struct HashJoinExec { /// The equality null-handling behavior of the join algorithm. pub null_equality: NullEquality, /// Flag to indicate if this is a null-aware anti join + /// + /// Set for the physical lowering of scalar `NOT IN` subqueries + /// (`JoinType::LeftAnti`). When `true`, NULLs in the join keys follow SQL + /// `NOT IN` three-valued logic rather than ordinary equi-join semantics. + /// + /// A join filter holds the non-equality part of a correlated subquery, and + /// only the build rows it keeps take part in the three-valued logic: such + /// a join records the UNKNOWN decision per build row in the null-indices + /// bitmap instead of in shared probe-side flags. pub null_aware: bool, /// Cache holding plan properties like equivalences, output partitioning etc. cache: Arc, @@ -1556,6 +1609,11 @@ impl ExecutionPlan for HashJoinExec { join_metrics.clone(), reservation, need_produce_result_in_final(self.join_type), + // Correlated null-aware joins record UNKNOWN per build row, + // uncorrelated ones precompute the logical nulls of the + // scalar build key for the final stage. + self.null_aware && self.filter.is_some(), + self.null_aware && self.filter.is_none(), right_partitions, enable_dynamic_filter_pushdown, Arc::clone(context.session_config().options()), @@ -1577,6 +1635,8 @@ impl ExecutionPlan for HashJoinExec { join_metrics.clone(), reservation, need_produce_result_in_final(self.join_type), + self.null_aware && self.filter.is_some(), + self.null_aware && self.filter.is_none(), 1, enable_dynamic_filter_pushdown, Arc::clone(context.session_config().options()), @@ -2374,6 +2434,10 @@ fn concat_build_batches( /// * `metrics` - Metrics collector for tracking memory usage and row counts /// * `reservation` - Memory reservation tracker for the hash table and data /// * `with_visited_indices_bitmap` - Whether to track visited indices (for outer joins) +/// * `with_null_aware_row_state` - Whether to track the per-build-row UNKNOWN +/// decision of a correlated null-aware join +/// * `with_null_aware_build_key_nulls` - Whether to precompute the logical +/// nulls of the scalar build key for an uncorrelated null-aware join /// * `probe_threads_count` - Number of threads that will probe this hash table /// * `should_compute_dynamic_filters` - Whether to compute min/max bounds for dynamic filtering /// @@ -2404,6 +2468,8 @@ async fn collect_left_input( metrics: BuildProbeJoinMetrics, reservation: MemoryReservation, with_visited_indices_bitmap: bool, + with_null_aware_row_state: bool, + with_null_aware_build_key_nulls: bool, probe_threads_count: usize, should_compute_dynamic_filters: bool, config: Arc, @@ -2598,6 +2664,46 @@ async fn collect_left_input( (BooleanBufferBuilder::new(0), reservation.new_empty()) }; + // Shared by all probe partitions of a correlated null-aware join, which + // record the per-build-row UNKNOWN decision here. It is per-execution + // state, so it is charged to the same lease as the visited bitmap: a + // dynamic filter can retain the build data after this join's own state is + // dropped, and that retained lease must not carry this bitmap's cost. + let null_indices_bitmap = if with_null_aware_row_state { + let bitmap_size = bit_util::ceil(batch.num_rows(), 8); + visited_reservation.try_grow(bitmap_size)?; + metrics.build_mem_used.add(bitmap_size); + let mut bitmap_buffer = BooleanBufferBuilder::new(batch.num_rows()); + bitmap_buffer.append_n(num_rows, false); + bitmap_buffer + } else { + BooleanBufferBuilder::new(0) + }; + + // Logical nulls, so that a dictionary key pointing at a NULL dictionary + // value counts as a NULL build key in the final stage. + let null_aware_build_key_nulls = + if with_null_aware_build_key_nulls && !left_values.is_empty() { + left_values[0] + .logical_nulls() + .filter(|nulls| nulls.null_count() > 0) + } else { + None + }; + if let Some(nulls) = &null_aware_build_key_nulls { + // `logical_nulls` reuses an array's physical null buffer when it can. + // Only charge newly materialized buffers, such as dictionary logical + // nulls, because the build batch's buffers were reserved above. + let reuses_input_buffer = left_values[0] + .nulls() + .is_some_and(|input| input.inner().ptr_eq(nulls.inner())); + if !reuses_input_buffer { + let retained_size = nulls.buffer().capacity(); + reservation.try_grow(retained_size)?; + metrics.build_mem_used.add(retained_size); + } + } + let map = Arc::new(join_hash_map); // For an ordinary build, nothing reads the strategy unless this join's own @@ -2674,9 +2780,9 @@ async fn collect_left_input( let data = JoinLeftData { visited_indices_bitmap: Mutex::new(visited_indices_bitmap), - probe_threads_counter: AtomicUsize::new(probe_threads_count), - probe_side_non_empty: AtomicBool::new(false), - probe_side_has_null: AtomicBool::new(false), + null_indices_bitmap: Mutex::new(null_indices_bitmap), + null_aware_build_key_nulls, + probe_completion: ProbeCompletion::new(probe_threads_count), membership, _visited_reservation: visited_reservation, build: Arc::new(JoinBuildData { @@ -2765,6 +2871,7 @@ mod tests { use datafusion_physical_expr::{ EquivalenceProperties, PhysicalSortExpr, RangePartitioning, SplitPoint, }; + use futures::StreamExt; use hashbrown::HashTable; use insta::{allow_duplicates, assert_snapshot}; use rstest::*; @@ -7492,6 +7599,8 @@ mod tests { BuildProbeJoinMetrics::new(0, &metrics_set), MemoryConsumer::new("build quota").register(&unbounded), false, + false, + false, 1, false, Arc::clone(&config), @@ -7521,6 +7630,8 @@ mod tests { BuildProbeJoinMetrics::new(0, &metrics_set), MemoryConsumer::new("build quota").register(&pool), false, + false, + false, 1, false, Arc::clone(&config), @@ -7713,6 +7824,8 @@ mod tests { BuildProbeJoinMetrics::new(0, &metrics_set), MemoryConsumer::new("visited lease").register(&pool), true, + false, + false, 1, true, Arc::new(ConfigOptions::default()), @@ -7744,6 +7857,57 @@ mod tests { Ok(()) } + /// The correlation bitmap of a correlated null-aware join is mutable + /// per-execution state too: a retained build lease must not keep paying + /// for it once the join's own state is gone. + #[tokio::test] + async fn correlated_bitmap_reservation_is_execution_local() -> Result<()> { + use crate::memory::MemoryStream; + let values: Vec = (0..1024).collect(); + let batch = build_table_i32(("a1", &values), ("b1", &values), ("c1", &values)); + let schema = batch.schema(); + let pool: Arc = Arc::new(UnboundedMemoryPool::default()); + let metrics_set = ExecutionPlanMetricsSet::new(); + let data = collect_left_input( + HASH_JOIN_SEED.random_state().clone(), + Box::pin(MemoryStream::try_new(vec![batch], schema, None)?), + vec![Arc::new(Column::new("a1", 0))], + BuildProbeJoinMetrics::new(0, &metrics_set), + MemoryConsumer::new("correlated lease").register(&pool), + true, + true, + false, + 1, + true, + Arc::new(ConfigOptions::default()), + NullEquality::NullEqualsNothing, + MetricBuilder::new(&metrics_set) + .counter(ARRAY_MAP_CREATED_COUNT_METRIC_NAME, 0), + BuildMode::Ordinary, + ) + .await?; + let producer_bytes = data.build.reservation.size(); + #[expect(clippy::used_underscore_binding)] + let visited_bytes = data._visited_reservation.size(); + let bitmap_bytes = data.membership.bitmap_reserved_bytes(); + // Visited bitmap plus the correlation bitmap, one bit per build row each. + assert_eq!(visited_bytes, 2 * (1024 / 8)); + assert_eq!( + pool.reserved(), + producer_bytes + visited_bytes + bitmap_bytes + ); + let producer = Arc::clone(&data.build); + drop(data); + assert_eq!( + pool.reserved(), + producer_bytes, + "both mutable bitmaps must release with execution, not with the lease" + ); + drop(producer); + assert_eq!(pool.reserved(), 0); + Ok(()) + } + /// A build lease retains the batch, computed keys, map, and their reservation. #[rstest] #[tokio::test] @@ -7778,6 +7942,8 @@ mod tests { metrics, reservation, false, + false, + false, 1, false, config, @@ -8579,6 +8745,477 @@ mod tests { Ok(()) } + /// Builds a two-partition probe side for the cross-partition null-aware + /// tests: partition 0 holds the only NULL key, partition 1 holds none. + fn build_two_partition_probe_with_null_in_partition_0() -> Arc { + let schema = Arc::new(Schema::new(vec![ + Field::new("c2", DataType::Int32, true), + Field::new("dummy", DataType::Int32, true), + ])); + let partition_0 = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int32Array::from(vec![Some(1), None])), + Arc::new(Int32Array::from(vec![Some(100), Some(400)])), + ], + ) + .unwrap(); + let partition_1 = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(Int32Array::from(vec![Some(2)])), + Arc::new(Int32Array::from(vec![Some(200)])), + ], + ) + .unwrap(); + TestMemoryExec::try_new_exec( + &[vec![partition_0], vec![partition_1]], + schema, + None, + ) + .unwrap() + } + + /// The dictionary counterpart of + /// [`build_two_partition_probe_with_null_in_partition_0`]: the same two + /// probe partitions, but the join key is a dictionary and partition 0's + /// NULL exists only logically. + /// + /// Every dictionary key is a physically valid index; one of them points at + /// a NULL dictionary value. The array therefore reports + /// `null_count() == 0` while `logical_null_count() > 0`, which is the case + /// that a physical-NULL check silently misses. + fn build_two_partition_dict_probe_with_logical_null_in_partition_0() + -> Arc { + let dict_type = + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Int32)); + let schema = Arc::new(Schema::new(vec![ + Field::new("c2", dict_type, true), + Field::new("dummy", DataType::Int32, true), + ])); + + // Dictionary values: [1, NULL]; keys: [0, 1] => logical [1, NULL]. + let with_logical_null: ArrayRef = Arc::new(DictionaryArray::new( + Int32Array::from(vec![0, 1]), + Arc::new(Int32Array::from(vec![Some(1), None])), + )); + assert_eq!( + with_logical_null.null_count(), + 0, + "the probe NULL must be logical only, or this test stops covering \ + the logical_null_count path" + ); + assert!( + with_logical_null.logical_null_count() > 0, + "the probe key must carry a logical NULL" + ); + + let partition_0 = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + with_logical_null, + Arc::new(Int32Array::from(vec![Some(100), Some(400)])), + ], + ) + .unwrap(); + + // Dictionary values: [2]; keys: [0] => logical [2], no NULL anywhere. + let partition_1 = RecordBatch::try_new( + Arc::clone(&schema), + vec![ + Arc::new(DictionaryArray::new( + Int32Array::from(vec![0]), + Arc::new(Int32Array::from(vec![Some(2)])), + )), + Arc::new(Int32Array::from(vec![Some(200)])), + ], + ) + .unwrap(); + + TestMemoryExec::try_new_exec( + &[vec![partition_0], vec![partition_1]], + schema, + None, + ) + .unwrap() + } + + /// Builds a table whose join key is dictionary encoded, so a key can point + /// at a NULL dictionary value and be logically, but not physically, NULL. + fn build_table_dict_key( + key_name: &str, + dict_values: Vec>, + keys: Vec, + dummy_name: &str, + dummy: Vec>, + ) -> Arc { + let dict_type = + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Int32)); + let schema = Arc::new(Schema::new(vec![ + Field::new(key_name, dict_type, true), + Field::new(dummy_name, DataType::Int32, true), + ])); + let values = Int32Array::from(dict_values); + let key_array = Int32Array::from(keys); + let dict: ArrayRef = Arc::new(DictionaryArray::new(key_array, Arc::new(values))); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![dict, Arc::new(Int32Array::from(dummy))], + ) + .unwrap(); + TestMemoryExec::try_new_exec(&[vec![batch]], schema, None).unwrap() + } + + /// Drains the probe partitions of `join` one after another in the given + /// order and returns everything they emitted. + async fn collect_partitions_in_order( + join: &HashJoinExec, + order: &[usize], + task_ctx: &Arc, + ) -> Result> { + let mut batches = vec![]; + for &partition in order { + let stream = join.execute(partition, Arc::clone(task_ctx))?; + batches.extend(common::collect(stream).await?); + } + Ok(batches) + } + + /// A NULL probe key silences a null-aware anti join even when the + /// partition that saw it is not the one emitting the final build rows. + /// + /// `CollectLeft` with several probe partitions is what the planner + /// produces for `NOT IN`, and only the last partition to finish emits the + /// build rows, reading what its siblings recorded. Both finishing orders + /// must produce no rows. + /// + /// These partitions are drained sequentially, so this covers the + /// cross-partition plumbing, not the concurrent race that motivated + /// [`ProbeCompletion`]. + #[apply(hash_join_exec_configs)] + #[tokio::test] + async fn test_null_aware_anti_join_probe_null_in_other_partition( + batch_size: usize, + ) -> Result<()> { + let task_ctx = prepare_task_ctx(batch_size, false); + + for order in [[0, 1], [1, 0]] { + let left = build_table_two_cols( + ("c1", &vec![Some(1), Some(2), Some(3), Some(4)]), + ("dummy", &vec![Some(10), Some(20), Some(30), Some(40)]), + ); + let right = build_two_partition_probe_with_null_in_partition_0(); + + let on = vec![( + Arc::new(Column::new_with_schema("c1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("c2", &right.schema())?) as _, + )]; + + let join = HashJoinExec::try_new( + left, + right, + on, + None, + &JoinType::LeftAnti, + None, + PartitionMode::CollectLeft, + NullEquality::NullEqualsNothing, + true, // null_aware = true + )?; + + let batches = collect_partitions_in_order(&join, &order, &task_ctx).await?; + let rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!( + rows, 0, + "probe order {order:?} emitted rows although a probe partition saw NULL" + ); + } + Ok(()) + } + + /// The dictionary counterpart of + /// [`test_null_aware_anti_join_probe_null_in_other_partition`], where the + /// silencing probe NULL exists only logically. + /// + /// A dictionary key that points at a NULL dictionary value has + /// `null_count() == 0`, so it takes the logical-null check to record it. + /// This pins that check to the cross-partition summary: the partition + /// holding the logical NULL is not the one that emits the build rows, in + /// either drain order. + #[apply(hash_join_exec_configs)] + #[tokio::test] + async fn test_null_aware_anti_join_probe_logical_null_in_other_partition( + batch_size: usize, + ) -> Result<()> { + let task_ctx = prepare_task_ctx(batch_size, false); + + for order in [[0, 1], [1, 0]] { + // Build keys are dictionary-encoded too, and NULL-free. + let left = build_table_dict_key( + "c1", + vec![Some(1), Some(2), Some(3), Some(4)], + vec![0, 1, 2, 3], + "dummy", + vec![Some(10), Some(20), Some(30), Some(40)], + ); + let right = build_two_partition_dict_probe_with_logical_null_in_partition_0(); + + let on = vec![( + Arc::new(Column::new_with_schema("c1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("c2", &right.schema())?) as _, + )]; + + let join = HashJoinExec::try_new( + left, + right, + on, + None, + &JoinType::LeftAnti, + None, + PartitionMode::CollectLeft, + NullEquality::NullEqualsNothing, + true, // null_aware = true + )?; + + let batches = collect_partitions_in_order(&join, &order, &task_ctx).await?; + let rows: usize = batches.iter().map(|b| b.num_rows()).sum(); + assert_eq!( + rows, 0, + "probe order {order:?} emitted rows although a probe partition saw a logical NULL" + ); + } + Ok(()) + } + + /// `NULL NOT IN (empty)` is TRUE: with an empty probe side the NULL build + /// key must be kept, even though the same build side emits nothing once + /// the probe side has a row. + #[apply(hash_join_exec_configs)] + #[tokio::test] + async fn test_null_aware_anti_join_empty_probe_keeps_null_build_key( + batch_size: usize, + ) -> Result<()> { + let task_ctx = prepare_task_ctx(batch_size, false); + + let left = build_table_two_cols( + ("c1", &vec![Some(1), None]), + ("dummy", &vec![Some(10), Some(0)]), + ); + // Genuinely empty probe side (not an all-NULL probe side, which is the + // different `NOT IN` UNKNOWN case). + let empty_right_schema = Arc::new(Schema::new(vec![ + Field::new("c2", DataType::Int32, true), + Field::new("dummy", DataType::Int32, true), + ])); + let empty_right = TestMemoryExec::try_new_exec( + &[vec![RecordBatch::new_empty(Arc::clone( + &empty_right_schema, + ))]], + empty_right_schema, + None, + )?; + + let on = vec![( + Arc::new(Column::new_with_schema("c1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("c2", &empty_right.schema())?) as _, + )]; + + let join = HashJoinExec::try_new( + left, + empty_right, + on, + None, + &JoinType::LeftAnti, + None, + PartitionMode::CollectLeft, + NullEquality::NullEqualsNothing, + true, // null_aware = true + )?; + + let stream = join.execute(0, task_ctx)?; + let batches = common::collect(stream).await?; + + allow_duplicates! { + assert_snapshot!(batches_to_sort_string(&batches), @r" + +----+-------+ + | c1 | dummy | + +----+-------+ + | | 0 | + | 1 | 10 | + +----+-------+ + "); + } + Ok(()) + } + + /// `left.z > right.z` over the second column of two-column tables: the + /// non-equality correlation of + /// `id NOT IN (SELECT r.id FROM r WHERE r.z < l.z)`. + fn prepare_second_column_gt_filter() -> JoinFilter { + let column_indices = vec![ + ColumnIndex { + index: 1, + side: JoinSide::Left, + }, + ColumnIndex { + index: 1, + side: JoinSide::Right, + }, + ]; + let intermediate_schema = Schema::new(vec![ + Field::new("z", DataType::Int32, true), + Field::new("z", DataType::Int32, true), + ]); + let filter_expression = Arc::new(BinaryExpr::new( + Arc::new(Column::new("z", 0)), + Operator::Gt, + Arc::new(Column::new("z", 1)), + )) as Arc; + + JoinFilter::new( + filter_expression, + column_indices, + Arc::new(intermediate_schema), + ) + } + + /// Build and probe sides of a null-aware join whose only correlation is + /// the non-equality filter from [`prepare_second_column_gt_filter`]. + /// + /// For each build row, the probe rows with a smaller `z` form its + /// subquery result: + /// - `(1, 10)` and `(2, 20)`: `{1, NULL}` + /// - `(NULL, 30)`: `{1, NULL}` + /// - `(4, 40)`: `{1, 4, NULL}` + /// - `(NULL, 1)` and `(5, 1)`: empty + /// + /// The probe row `(NULL, 50)` never passes the filter. + fn build_null_aware_filter_only_inputs() + -> (Arc, Arc, JoinOn) { + let left = build_table_two_cols( + ("id", &vec![Some(1), Some(2), None, Some(4), None, Some(5)]), + ( + "z", + &vec![Some(10), Some(20), Some(30), Some(40), Some(1), Some(1)], + ), + ); + let right = build_table_two_cols( + ("id", &vec![Some(1), None, Some(4), None]), + ("z", &vec![Some(5), Some(50), Some(35), Some(2)]), + ); + let on = vec![( + Arc::new(Column::new_with_schema("id", &left.schema()).unwrap()) as _, + Arc::new(Column::new_with_schema("id", &right.schema()).unwrap()) as _, + )]; + (left, right, on) + } + + /// Null-aware `LeftAnti` with a join filter and no correlation scope keys. + /// + /// A NULL on either side only makes `NOT IN` UNKNOWN for the build rows + /// where the filter keeps the NULL, so the NULLs must not remove every row. + #[apply(hash_join_exec_configs)] + #[tokio::test] + async fn test_null_aware_left_anti_filter_only(batch_size: usize) -> Result<()> { + let task_ctx = prepare_task_ctx(batch_size, false); + let (left, right, on) = build_null_aware_filter_only_inputs(); + + let join = HashJoinExec::try_new( + left, + right, + on, + Some(prepare_second_column_gt_filter()), + &JoinType::LeftAnti, + None, + PartitionMode::CollectLeft, + NullEquality::NullEqualsNothing, + true, + )?; + + let stream = join.execute(0, task_ctx)?; + let batches = common::collect(stream).await?; + + // Only the rows with an empty subquery result are TRUE. + allow_duplicates! { + assert_snapshot!(batches_to_sort_string(&batches), @r" + +----+---+ + | id | z | + +----+---+ + | | 1 | + | 5 | 1 | + +----+---+ + "); + } + + Ok(()) + } + + /// `NULL NOT IN (5, 6)` is UNKNOWN, so the build row whose key points at a + /// NULL dictionary value must be dropped, as it is when the key is a + /// physical NULL. + #[apply(hash_join_exec_configs)] + #[tokio::test] + async fn test_null_aware_left_anti_build_logical_null( + batch_size: usize, + ) -> Result<()> { + let task_ctx = prepare_task_ctx(batch_size, false); + + let left = build_table_dict_key( + "c1", + vec![Some(1), None, Some(2)], + vec![0, 1, 2], + "dummy", + vec![Some(10), Some(0), Some(20)], + ); + let left_key = left + .execute(0, Arc::clone(&task_ctx))? + .next() + .await + .unwrap()?; + assert_eq!(left_key.column(0).null_count(), 0); + assert_eq!(left_key.column(0).logical_null_count(), 1); + + let right = build_table_dict_key( + "c2", + vec![Some(5), Some(6)], + vec![0, 1], + "dummy", + vec![Some(50), Some(60)], + ); + + let on = vec![( + Arc::new(Column::new_with_schema("c1", &left.schema())?) as _, + Arc::new(Column::new_with_schema("c2", &right.schema())?) as _, + )]; + + let join = HashJoinExec::try_new( + left, + right, + on, + None, + &JoinType::LeftAnti, + None, + PartitionMode::CollectLeft, + NullEquality::NullEqualsNothing, + true, + )?; + + let stream = join.execute(0, task_ctx)?; + let batches = common::collect(stream).await?; + + allow_duplicates! { + assert_snapshot!(batches_to_sort_string(&batches), @r" + +----+-------+ + | c1 | dummy | + +----+-------+ + | 1 | 10 | + | 2 | 20 | + +----+-------+ + "); + } + Ok(()) + } + /// Test that null_aware validation rejects non-LeftAnti join types #[tokio::test] async fn test_null_aware_validation_wrong_join_type() { diff --git a/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs b/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs index 9ec405e919bab..b02c3ce92f3ee 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec/prepared.rs @@ -73,9 +73,11 @@ impl PreparedHashJoinBuild { JoinLeftData { build: Arc::clone(&self.build), visited_indices_bitmap: Mutex::new(BooleanBufferBuilder::new(0)), - probe_threads_counter: AtomicUsize::new(probe_threads), - probe_side_non_empty: AtomicBool::new(false), - probe_side_has_null: AtomicBool::new(false), + probe_completion: ProbeCompletion::new(probe_threads), + // A prepared build serves ordinary joins only: correlated + // null-aware joins are rejected before they can consume one. + null_indices_bitmap: Mutex::new(BooleanBufferBuilder::new(0)), + null_aware_build_key_nulls: None, membership: self.membership.clone(), _visited_reservation: self.build.reservation.new_empty(), } @@ -193,6 +195,8 @@ impl HashJoinExec { metrics, reservation, false, + false, + false, 0, !byte_keys, config, diff --git a/datafusion/physical-plan/src/joins/hash_join/mod.rs b/datafusion/physical-plan/src/joins/hash_join/mod.rs index 4687d8545ec69..7b50435e18792 100644 --- a/datafusion/physical-plan/src/joins/hash_join/mod.rs +++ b/datafusion/physical-plan/src/joins/hash_join/mod.rs @@ -23,5 +23,6 @@ pub use partitioned_hash_eval::{HashExpr, HashTableLookupExpr, SeededRandomState mod exec; mod inlist_builder; mod partitioned_hash_eval; +mod probe_completion; mod shared_bounds; mod stream; diff --git a/datafusion/physical-plan/src/joins/hash_join/probe_completion.rs b/datafusion/physical-plan/src/joins/hash_join/probe_completion.rs new file mode 100644 index 0000000000000..11fb72d352c6a --- /dev/null +++ b/datafusion/physical-plan/src/joins/hash_join/probe_completion.rs @@ -0,0 +1,350 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Completion protocol shared by the probe partitions of a hash join. +//! +//! The probe partitions of one `HashJoinExec` run concurrently over a shared +//! build side. Two facts have to cross partition boundaries: +//! +//! - which partition emits the build-side rows in the final stage, namely the +//! last one to finish, and +//! - what the partitions collectively saw, which the null-aware `NOT IN` rules +//! need: did any partition see a row, and did any see a NULL join key. +//! +//! [`ProbeCompletion`] owns both, because reading the second without ordering +//! it against the first produced wrong results. A partition that read the NULL +//! flag *before* decrementing the counter could observe `false`, have a +//! sibling record a NULL and finish, then become the last partition itself and +//! emit build rows that `NOT IN` must suppress. +//! +//! The invariant is therefore: **whichever caller observes itself to be last +//! sees every fact recorded by every other partition.** It rests on two +//! properties, and the type is shaped to keep both. +//! +//! 1. *Order.* The facts are reachable only through the value returned by +//! [`ProbeCompletion::report_completed`], so no caller can read them before +//! its own decrement. +//! 2. *Visibility.* That decrement is `AcqRel`, so the last partition +//! synchronizes with the release of every partition that finished before +//! it. Were the counter `Relaxed`, nothing would order a sibling's store +//! before the final load even when the decrement happened first. +//! +//! Neither property can be pinned down by an ordinary concurrent test: the +//! window is a few instructions wide, and on x86 a `Relaxed` decrement lowers +//! to the same instruction as an `AcqRel` one. Both are model-checked in +//! `loom_tests` instead. + +use std::sync::atomic::Ordering; + +/// What every probe partition together saw, as observed by the last one to +/// finish. +/// +/// Obtainable only from [`ProbeCompletion::report_completed`], which is what +/// stops the final stage from reading the shared state too early. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct ProbeSideSummary { + /// Some probe partition saw a row. An entirely empty probe side is not the + /// same as one that matched nothing: `NULL NOT IN (empty)` is TRUE. + pub(super) non_empty: bool, + /// Some probe partition saw a NULL join key. + pub(super) has_null: bool, +} + +/// Defines [`ProbeCompletion`] over the given atomic types. +/// +/// [loom] can only explore its own atomics, so the protocol is written once +/// here and instantiated twice: over [`std::sync::atomic`] for the join, and +/// over `loom::sync::atomic` for the model in `loom_tests`. Both share this +/// single copy of the orderings and the call sequence, which is what lets the +/// model speak for the real thing. Keep it that way: a second copy of these +/// orderings would not be covered. +/// +/// [loom]: https://docs.rs/loom +macro_rules! define_probe_completion { + (counter: $counter:ty, flag: $flag:ty) => { + /// Counts the probe partitions that are still running. + type Counter = $counter; + /// One fact, set by any partition and read by the last one. + type Flag = $flag; + + /// Tracks how many probe partitions are still running, together with + /// the null-aware facts they contribute. + /// + /// See the [module docs](self) for the invariant this upholds. + #[derive(Debug)] + pub(super) struct ProbeCompletion { + /// Probe partitions that have not finished yet. + running: Counter, + /// Set once any probe partition has seen a row. + saw_row: Flag, + /// Set once any probe partition has seen a NULL join key. + saw_null_key: Flag, + } + + impl ProbeCompletion { + /// Creates the protocol state for `probe_threads` partitions. + pub(super) fn new(probe_threads: usize) -> Self { + Self { + running: Counter::new(probe_threads), + saw_row: Flag::new(false), + saw_null_key: Flag::new(false), + } + } + + /// Records what one probe partition saw in one batch. + /// + /// `Relaxed` suffices: the `AcqRel` decrement in + /// [`Self::report_completed`] publishes these stores to the last + /// partition. + pub(super) fn record_batch(&self, non_empty: bool, has_null: bool) { + if non_empty { + self.saw_row.store(true, Ordering::Relaxed); + } + if has_null { + self.saw_null_key.store(true, Ordering::Relaxed); + } + } + + /// Whether some partition has already recorded a NULL join key. + /// + /// A hint for skipping work during the probe phase, which may lag + /// behind sibling partitions. It must never decide what the final + /// stage emits; use the [`ProbeSideSummary`] from + /// [`Self::report_completed`] for that. + pub(super) fn saw_null_key_hint(&self) -> bool { + self.saw_null_key.load(Ordering::Relaxed) + } + + /// Marks the calling partition finished, returning `Some` only for + /// the last one, together with what all partitions saw. + /// + /// The `AcqRel` decrement publishes this partition's own stores + /// and acquires those of the partitions that finished earlier, so + /// the summary handed to the last caller is complete. + pub(super) fn report_completed(&self) -> Option { + // `fetch_sub` returns the value from *before* the decrement, + // so `1` means this call is the one that took it to zero. + let was_last = self.running.fetch_sub(1, Ordering::AcqRel) == 1; + was_last.then(|| ProbeSideSummary { + non_empty: self.saw_row.load(Ordering::Relaxed), + has_null: self.saw_null_key.load(Ordering::Relaxed), + }) + } + } + }; +} + +define_probe_completion!( + counter: std::sync::atomic::AtomicUsize, + flag: std::sync::atomic::AtomicBool +); + +#[cfg(test)] +mod tests { + use super::*; + + /// The sequential contract: only the last caller is handed the summary, + /// and it carries what the earlier partitions recorded. + /// + /// The concurrent invariant is covered by `loom_tests` instead. + #[test] + fn only_the_last_partition_receives_the_summary() { + let completion = ProbeCompletion::new(3); + + completion.record_batch(true, false); + assert_eq!(completion.report_completed(), None); + + completion.record_batch(true, true); + assert_eq!(completion.report_completed(), None); + + assert_eq!( + completion.report_completed(), + Some(ProbeSideSummary { + non_empty: true, + has_null: true, + }) + ); + } + + /// A probe side that never saw a row leaves both facts clear, which is how + /// `NULL NOT IN (empty)` stays TRUE. + #[test] + fn an_untouched_probe_side_reports_nothing_seen() { + let completion = ProbeCompletion::new(1); + + assert_eq!( + completion.report_completed(), + Some(ProbeSideSummary { + non_empty: false, + has_null: false, + }) + ); + } + + /// The hint may lag, but it must never report a NULL that was not + /// recorded. + #[test] + fn the_hint_reflects_recorded_nulls() { + let completion = ProbeCompletion::new(1); + assert!(!completion.saw_null_key_hint()); + + completion.record_batch(true, false); + assert!(!completion.saw_null_key_hint()); + + completion.record_batch(true, true); + assert!(completion.saw_null_key_hint()); + } +} + +/// Model-checked concurrency tests for the protocol. +/// +/// These instantiate the protocol over loom's atomics (see the +/// [module docs](self)) and let loom enumerate the interleavings and store +/// visibility the memory model allows. They fail if the decrement in +/// `report_completed` is weakened to `Relaxed`. +/// +/// Compiled only under `--cfg datafusion_loom`, which is also what pulls the `loom` +/// dev-dependency into the build (see `datafusion-physical-plan`'s +/// `Cargo.toml` for why it is kept out of the ordinary one). Run them with: +/// +/// ```text +/// RUSTFLAGS="--cfg datafusion_loom" cargo test -p datafusion-physical-plan --lib loom_tests +/// ``` +#[cfg(all(test, datafusion_loom))] +mod loom_tests { + use super::ProbeSideSummary; + use loom::sync::Arc; + use std::sync::atomic::Ordering; + + define_probe_completion!( + counter: loom::sync::atomic::AtomicUsize, + flag: loom::sync::atomic::AtomicBool + ); + + /// The invariant behind the fix: whichever partition observes itself to be + /// last sees the NULL its sibling recorded. + /// + /// This is the regression test for the wrong-results bug. One partition + /// records a NULL and finishes while the other finishes having recorded + /// nothing, and the last one must not conclude that no NULL was seen. + #[test] + fn the_last_partition_observes_a_sibling_null() { + loom::model(|| { + let completion = Arc::new(ProbeCompletion::new(2)); + + let recorder = { + let completion = Arc::clone(&completion); + loom::thread::spawn(move || { + completion.record_batch(true, true); + completion.report_completed() + }) + }; + + let observer = completion.report_completed(); + let recorded = recorder.join().unwrap(); + + let summary = match (recorded, observer) { + (Some(summary), None) | (None, Some(summary)) => summary, + (Some(_), Some(_)) => panic!("two partitions both finished last"), + (None, None) => panic!("no partition finished last"), + }; + + assert!( + summary.has_null, + "the last partition missed the NULL its sibling recorded" + ); + assert!( + summary.non_empty, + "the last partition missed the row its sibling recorded" + ); + }); + } + + /// With both partitions recording, the summary must be the union of what + /// they saw, whichever finishes last. + #[test] + fn the_summary_unions_what_every_partition_recorded() { + loom::model(|| { + let completion = Arc::new(ProbeCompletion::new(2)); + + let other = { + let completion = Arc::clone(&completion); + loom::thread::spawn(move || { + // Rows, but no NULL key. + completion.record_batch(true, false); + completion.report_completed() + }) + }; + + // A NULL key in an otherwise empty batch. + completion.record_batch(false, true); + let here = completion.report_completed(); + let there = other.join().unwrap(); + + let summary = here.or(there).expect("some partition must finish last"); + assert_eq!( + summary, + ProbeSideSummary { + non_empty: true, + has_null: true, + }, + "the summary lost a fact one of the partitions recorded" + ); + }); + } + + /// Exactly one partition may take the final stage. Two would emit the + /// build side twice; none would drop it. + #[test] + fn exactly_one_partition_finishes_last() { + loom::model(|| { + let completion = Arc::new(ProbeCompletion::new(2)); + + let other = { + let completion = Arc::clone(&completion); + loom::thread::spawn(move || completion.report_completed().is_some()) + }; + + let here = completion.report_completed().is_some(); + let there = other.join().unwrap(); + + assert!(here ^ there, "exactly one partition must finish last"); + }); + } + + /// The probe-phase hint never invents a NULL: if no partition recorded + /// one, no observation of the hint may report one. + #[test] + fn the_hint_never_reports_an_unrecorded_null() { + loom::model(|| { + let completion = Arc::new(ProbeCompletion::new(2)); + + let other = { + let completion = Arc::clone(&completion); + loom::thread::spawn(move || { + completion.record_batch(true, false); + completion.saw_null_key_hint() + }) + }; + + let here = completion.saw_null_key_hint(); + let there = other.join().unwrap(); + + assert!(!here && !there, "the hint reported an unrecorded NULL"); + }); + } +} diff --git a/datafusion/physical-plan/src/joins/hash_join/stream.rs b/datafusion/physical-plan/src/joins/hash_join/stream.rs index 3b5185909c7c7..9341b51a0e0f7 100644 --- a/datafusion/physical-plan/src/joins/hash_join/stream.rs +++ b/datafusion/physical-plan/src/joins/hash_join/stream.rs @@ -21,7 +21,6 @@ //! [`super::HashJoinExec`]. See comments in [`HashJoinStream`] for more details. use std::sync::Arc; -use std::sync::atomic::Ordering; use std::task::Poll; use crate::coalesce::{LimitedBatchCoalescer, PushBatchStatus}; @@ -29,6 +28,7 @@ use crate::joins::Map; use crate::joins::MapOffset; use crate::joins::PartitionMode; use crate::joins::hash_join::exec::JoinLeftData; +use crate::joins::hash_join::probe_completion::ProbeSideSummary; use crate::joins::hash_join::shared_bounds::{ PartitionBounds, PartitionBuildData, SharedBuildAccumulator, }; @@ -674,7 +674,7 @@ impl HashJoinStream { // Note: For null-aware anti join, we need to check the probe side (right) for NULLs, // not the build side (left). The probe-side NULL check happens during process_probe_batch. - // The probe_side_has_null flag will be set there if any probe batch contains NULL. + // The shared probe-completion state records it there if any probe batch contains NULL. self.state = self.transition_after_build_collected(&left_data); @@ -755,37 +755,45 @@ impl HashJoinStream { // For LeftAnti: output LEFT (build) rows where LEFT.key NOT IN RIGHT.key // 1. If RIGHT (probe) contains NULL in any batch, no LEFT rows should be output // 2. LEFT rows with NULL keys should not be output (handled in final stage) + // + // A correlated join (one with a join filter) narrows rule 1 to the build + // rows whose correlation scope and filter keep that NULL; see + // `mark_null_candidates_for_probe_batch`. if self.null_aware { - // Mark that we've seen a probe batch with actual rows (probe side is non-empty) - // Only set this if batch has rows - empty batches don't count - // Use shared atomic state so all partitions can see this global information - if state.batch.num_rows() > 0 { - build_side - .left_data - .probe_side_non_empty - .store(true, Ordering::Relaxed); - } - - // Check if probe side (RIGHT) contains NULL - // Since null_aware validation ensures single column join, we only check the first column - let probe_key_column = &state.values[0]; - if probe_key_column.null_count() > 0 { - // Found NULL in probe side - set shared flag to prevent any output - build_side - .left_data - .probe_side_has_null - .store(true, Ordering::Relaxed); - } - - // If probe side has NULL (detected in this or any other partition), return empty result - if build_side - .left_data - .probe_side_has_null - .load(Ordering::Relaxed) - { - timer.done(); - self.state = HashJoinStreamState::FetchProbeBatch; - return Ok(StatefulStreamResult::Continue); + // A correlated null-aware join (its join filter is the + // non-equality part of the subquery) decides UNKNOWN per build row + // instead of poisoning the whole join. + if self.filter.is_some() { + // Must precede the empty-build-map return below: an all-NULL-key + // build side has an empty full-key map but still needs UNKNOWN + // marks. + if state.offset == (0, None) { + mark_null_candidates_for_probe_batch( + build_side, + state, + self.filter.as_ref(), + self.join_type, + self.batch_size, + )?; + } + } else { + // Since null_aware validation ensures single column join, we only check the first column + let probe_key_column = &state.values[0]; + // Only batches with rows count: `NULL NOT IN (empty)` is TRUE. + build_side.left_data.record_probe_batch( + state.batch.num_rows() > 0, + probe_key_column.logical_null_count() > 0, + ); + + // If probe side has NULL (detected in this or any other + // partition so far), return empty result. This is a + // best-effort early exit: the final stage re-checks the flag + // through `report_probe_completed`. + if build_side.left_data.probe_side_has_null_hint() { + timer.done(); + self.state = HashJoinStreamState::FetchProbeBatch; + return Ok(StatefulStreamResult::Continue); + } } } @@ -970,19 +978,29 @@ impl HashJoinStream { let build_side = self.build_side.try_as_ready()?; - // For null-aware anti join, if probe side had NULL, no rows should be output - // Check shared atomic state to get global knowledge across all partitions - if self.null_aware - && build_side - .left_data - .probe_side_has_null - .load(Ordering::Relaxed) - { + // Only the last probe partition to finish emits the build-side rows, + // and only it receives the shared probe-side summary (see + // `JoinLeftData::report_probe_completed` for why the flags are not + // readable any other way here). + let Some(probe_summary) = build_side.left_data.report_probe_completed() else { timer.done(); self.state = HashJoinStreamState::Completed; return Ok(StatefulStreamResult::Continue); - } - if !build_side.left_data.report_probe_completed() { + }; + + // A NULL probe key seen by any partition makes `build.key NOT IN (probe)` + // UNKNOWN for every build row, so nothing is emitted. Decide that before + // materialising the unmatched indices: an uncorrelated join would build + // the whole index set only to have `null_aware_left_anti_final_indices` + // discard it, which costs an `O(build rows)` scan and two temporary arrays. + // Correlated joins keep going, because their UNKNOWN decision is per build + // row and already recorded by the probe phase. + if self.null_aware + && self.join_type == JoinType::LeftAnti + && self.filter.is_none() + && probe_summary.has_null + { + timer.done(); self.state = HashJoinStreamState::Completed; return Ok(StatefulStreamResult::Continue); } @@ -994,39 +1012,20 @@ impl HashJoinStream { true, ); - // For null-aware anti join, filter out LEFT rows with NULL in join keys - // BUT only if the probe side (RIGHT) was non-empty. If probe side is empty, - // NULL NOT IN (empty) = TRUE, so NULL rows should be returned. - // Use shared atomic state to get global knowledge across all partitions - if self.null_aware - && self.join_type == JoinType::LeftAnti - && build_side - .left_data - .probe_side_non_empty - .load(Ordering::Relaxed) - { - // Since null_aware validation ensures single column join, we only check the first column - let build_key_column = &build_side.left_data.values()[0]; - - // Filter out indices where the key is NULL - let filtered_indices: Vec = left_side - .iter() - .filter_map(|idx| { - let idx_usize = idx.unwrap() as usize; - if build_key_column.is_null(idx_usize) { - None // Skip rows with NULL keys - } else { - Some(idx.unwrap()) - } - }) - .collect(); - - left_side = UInt64Array::from(filtered_indices); - - // Update right_side to match the new length - let mut builder = arrow::array::UInt32Builder::with_capacity(left_side.len()); - builder.append_nulls(left_side.len()); - right_side = builder.finish(); + // Null-aware joins post-process the build rows under SQL three-valued + // logic; see the helper for the rules. + if self.null_aware && self.join_type == JoinType::LeftAnti { + let (null_aware_left, null_aware_right) = null_aware_left_anti_final_indices( + &build_side.left_data, + // A join filter makes the join correlated, so the UNKNOWN + // decision was recorded per build row by the probe phase. + self.filter.is_some(), + probe_summary, + left_side, + right_side, + ); + left_side = null_aware_left; + right_side = null_aware_right; } self.join_metrics.input_batches.add(1); @@ -1061,6 +1060,235 @@ impl HashJoinStream { } } +/// Records which build rows of a correlated null-aware join are UNKNOWN +/// candidates for this probe batch. +/// +/// An unmatched build row's `NOT IN` is UNKNOWN instead of TRUE (its row is +/// dropped instead of emitted) when either: +/// 1. its value key is NULL and some probe row passes the join filter, or +/// 2. some probe row with a NULL value key passes the join filter. +/// +/// The correlation scope of this fork's null-aware joins is carried by the +/// join filter alone (the null-aware validation allows a single join key), so +/// every pair on the relevant side is a candidate and the filter decides which +/// of them count. +/// +/// A build row stays UNKNOWN once it is marked, so candidates whose build row +/// is already marked are skipped, and the join filter is not evaluated for +/// them. This also ends the pairing as soon as no unmarked build row is left. +fn mark_null_candidates_for_probe_batch( + build_side: &BuildSideReadyState, + state: &ProcessProbeBatchState, + filter: Option<&JoinFilter>, + join_type: JoinType, + batch_size: usize, +) -> Result<()> { + let left_data = &build_side.left_data; + let probe_value_key = &state.values[0]; + let probe_has_null_values = probe_value_key.logical_null_count() > 0; + let build_value_key = &left_data.values()[0]; + let build_has_null_values = build_value_key.logical_null_count() > 0; + if !build_has_null_values && !probe_has_null_values { + return Ok(()); + } + + debug_assert_eq!( + left_data.values().len(), + state.values.len(), + "build/probe key counts must match" + ); + + // Keeps the candidate pairs that pass the join filter and marks their + // build rows as UNKNOWN. + let mut mark = |build_indices: UInt64Array, probe_indices: UInt32Array| { + let (build_indices, probe_indices) = + retain_unmarked(left_data, build_indices, probe_indices); + if build_indices.is_empty() { + return Ok(()); + } + let build_indices = match filter { + Some(filter) => { + apply_join_filter_to_indices( + left_data.batch(), + &state.batch, + build_indices, + probe_indices, + filter, + JoinSide::Left, + None, + join_type, + )? + .0 + } + None => build_indices, + }; + if !build_indices.is_empty() { + let mut null_bitmap = left_data.null_indices_bitmap().lock(); + for build_idx in build_indices.values() { + null_bitmap.set_bit(*build_idx as usize, true); + } + } + Ok(()) + }; + + // Case 1: build rows with a NULL value key are UNKNOWN as soon as any + // probe row passes the filter. + if build_has_null_values { + let null_mask = arrow::compute::is_null(build_value_key.as_ref())?; + let null_build_rows = UInt64Array::from_iter_values( + null_mask.values().set_indices().map(|idx| idx as u64), + ); + for_each_unmarked_cross_product( + left_data, + null_build_rows.values().iter().copied(), + 0..state.batch.num_rows() as u32, + batch_size, + &mut mark, + )?; + } + + // Case 2: NULL-valued probe rows make every build row that passes the + // filter an UNKNOWN candidate. + if probe_has_null_values { + let null_mask = arrow::compute::is_null(probe_value_key.as_ref())?; + let null_probe_rows = UInt32Array::from_iter_values( + null_mask.values().set_indices().map(|idx| idx as u32), + ); + for_each_unmarked_cross_product( + left_data, + 0..left_data.batch().num_rows() as u64, + null_probe_rows.values().iter().copied(), + batch_size, + &mut mark, + )?; + } + + Ok(()) +} + +/// Removes the candidate pairs whose build row is already marked UNKNOWN. +fn retain_unmarked( + left_data: &JoinLeftData, + build_indices: UInt64Array, + probe_indices: UInt32Array, +) -> (UInt64Array, UInt32Array) { + let bitmap = left_data.null_indices_bitmap().lock(); + let is_unmarked = |build_idx: &u64| !bitmap.get_bit(*build_idx as usize); + if build_indices.values().iter().all(is_unmarked) { + return (build_indices, probe_indices); + } + let (build, probe): (Vec, Vec) = build_indices + .values() + .iter() + .zip(probe_indices.values().iter()) + .filter(|(build_idx, _)| is_unmarked(build_idx)) + .unzip(); + (build.into(), probe.into()) +} + +/// Calls `f` with the pairs of `build_rows` x `probe_rows` whose build row is +/// not marked UNKNOWN, as chunks of at most `batch_size` pairs. +/// +/// `f` marks build rows, so the unmarked build rows are found again after each +/// chunk. The pairing stops when no unmarked build row is left. +fn for_each_unmarked_cross_product( + left_data: &JoinLeftData, + build_rows: impl Iterator, + probe_rows: impl Iterator, + batch_size: usize, + mut f: impl FnMut(UInt64Array, UInt32Array) -> Result<()>, +) -> Result<()> { + let retain_unmarked_rows = |rows: &mut Vec| { + let bitmap = left_data.null_indices_bitmap().lock(); + rows.retain(|idx| !bitmap.get_bit(*idx as usize)); + }; + + let mut build_rows: Vec = build_rows.collect(); + retain_unmarked_rows(&mut build_rows); + + let mut build_chunk = Vec::with_capacity(batch_size); + let mut probe_chunk = Vec::with_capacity(batch_size); + let mut marks_since_refresh = false; + for probe_row in probe_rows { + // Refresh only after a chunk was sent, so the cost of the refresh + // stays proportional to the pairs already evaluated. + if marks_since_refresh { + retain_unmarked_rows(&mut build_rows); + marks_since_refresh = false; + } + if build_rows.is_empty() { + break; + } + for build_row in &build_rows { + build_chunk.push(*build_row); + probe_chunk.push(probe_row); + if build_chunk.len() == batch_size { + f( + std::mem::replace(&mut build_chunk, Vec::with_capacity(batch_size)) + .into(), + std::mem::replace(&mut probe_chunk, Vec::with_capacity(batch_size)) + .into(), + )?; + marks_since_refresh = true; + } + } + } + if !build_chunk.is_empty() { + f(build_chunk.into(), probe_chunk.into())?; + } + Ok(()) +} + +/// Final-stage rules of a null-aware `LeftAnti` join, evaluated by the last +/// probe partition from what every partition together saw: +/// - a NULL probe key seen by any partition makes `build.key NOT IN (probe)` +/// UNKNOWN for every build row, so nothing is emitted; +/// - otherwise a NULL build key means `NULL NOT IN (probe)`, which is UNKNOWN +/// (row dropped) unless the probe side was empty, where it is TRUE (row +/// kept). +fn null_aware_left_anti_final_indices( + left_data: &JoinLeftData, + correlated: bool, + probe_summary: ProbeSideSummary, + left_side: UInt64Array, + right_side: UInt32Array, +) -> (UInt64Array, UInt32Array) { + if correlated { + // The probe phase recorded, per build row, whether the residual filter + // kept a NULL that makes its `NOT IN` UNKNOWN. Rows outside every + // correlation scope keep TRUE, which is what makes the empty-scope + // case (`NOT IN (empty)`) return them. + let null_indices_bitmap = left_data.null_indices_bitmap().lock(); + let left_side = UInt64Array::from_iter_values( + left_side + .values() + .iter() + .copied() + .filter(|idx| !null_indices_bitmap.get_bit(*idx as usize)), + ); + let right_side = UInt32Array::new_null(left_side.len()); + return (left_side, right_side); + } + if probe_summary.has_null { + return (UInt64Array::new_null(0), UInt32Array::new_null(0)); + } + if !probe_summary.non_empty { + return (left_side, right_side); + } + // Logical nulls, so that a dictionary key pointing at a NULL value counts + // as a NULL build key. Computed once with the build side. + let Some(build_key_nulls) = left_data.null_aware_build_key_nulls() else { + return (left_side, right_side); + }; + let left_side = UInt64Array::from_iter_values(left_side.iter().filter_map(|idx| { + let idx = + idx.expect("LeftAnti final indices should always contain build-side rows"); + (!build_key_nulls.is_null(idx as usize)).then_some(idx) + })); + let right_side = UInt32Array::new_null(left_side.len()); + (left_side, right_side) +} + impl Stream for HashJoinStream { type Item = Result; diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs index eb1df638c7dc5..c87760ebe6db2 100644 --- a/datafusion/physical-plan/src/joins/nested_loop_join.rs +++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs @@ -681,22 +681,23 @@ impl ExecutionPlan for NestedLoopJoinExec { // Determine if OOM fallback to memory-limited mode is possible. // Conditions: // 1. Disk manager supports temp files (needed for spilling). - // 2. FULL join with multiple right partitions is not yet supported - // in the fallback path. FULL join needs to track BOTH left-side - // matches (for unmatched left rows) AND right-side matches (for - // unmatched right rows). The fallback path builds a per-partition - // `JoinLeftData` with `probe_threads_counter == 1`, so each - // partition emits unmatched left rows based only on its own - // right-side matches, producing incorrect duplicate output for - // left rows that match in another partition. Other join types - // that need only one-sided final emission (LEFT, LEFT SEMI, - // LEFT ANTI, LEFT MARK) have a similar latent issue in the - // fallback path which predates this change; tracking is out of - // scope for this PR. - let full_join_multi_partition = - matches!(self.join_type, JoinType::Full) && right_partition_count > 1; + // 2. Join types whose final emission reads the visited-left bitmap + // (LEFT, LEFT SEMI, LEFT ANTI, LEFT MARK, FULL) need that bitmap + // complete across every probe partition. The memory-limited path + // builds a per-partition `JoinLeftData` with + // `probe_threads_counter == 1`, so with more than one right + // partition each partition emits from a bitmap that only saw its + // own right rows: unmatched rows are emitted once per partition, + // and rows matched only in another partition are emitted as + // unmatched. Refusing the fallback turns those wrong results into + // a plain ResourcesExhausted error. Right-side emission types are + // safe because each partition owns its right rows exclusively. + // Cross-partition coordination of the left bitmap is tracked in + // https://github.com/apache/datafusion/issues/22038. + let left_emission_multi_partition = + need_produce_result_in_final(self.join_type) && right_partition_count > 1; let spill_state = if context.runtime_env().disk_manager.tmp_files_enabled() - && !full_join_multi_partition + && !left_emission_multi_partition { SpillState::Pending { left_plan: Arc::clone(&self.left), @@ -3728,12 +3729,10 @@ pub(crate) mod tests { // Join types that support memory-limited fallback should succeed // even under tight memory limits (they spill to disk instead of OOM). + // Right-side emission types are safe with multiple right partitions + // because each partition owns its right rows exclusively. let fallback_join_types = vec![ JoinType::Inner, - JoinType::Left, - JoinType::LeftSemi, - JoinType::LeftAnti, - JoinType::LeftMark, JoinType::Right, JoinType::RightSemi, JoinType::RightAnti, @@ -3758,24 +3757,34 @@ pub(crate) mod tests { .await?; } - // FULL JOIN with multiple right partitions is intentionally not - // supported in the fallback path yet (cross-partition left-bitmap - // coordination is missing). It should still OOM under tight memory. - let runtime = RuntimeEnvBuilder::new() - .with_memory_limit(100, 1.0) - .build_arc()?; - let task_ctx = TaskContext::default().with_runtime(runtime); - let task_ctx = Arc::new(task_ctx); - let err = multi_partitioned_join_collect( - Arc::clone(&left), - Arc::clone(&right), - &JoinType::Full, - Some(filter.clone()), - task_ctx, - ) - .await - .unwrap_err(); - assert_contains!(err.to_string(), "Resources exhausted"); + // Join types whose final emission reads the visited-left bitmap are + // intentionally not supported in the fallback path with multiple right + // partitions (cross-partition left-bitmap coordination is missing; + // per-partition bitmaps emit wrong rows). They must OOM cleanly instead. + let gated_join_types = vec![ + JoinType::Left, + JoinType::LeftSemi, + JoinType::LeftAnti, + JoinType::LeftMark, + JoinType::Full, + ]; + for join_type in &gated_join_types { + let runtime = RuntimeEnvBuilder::new() + .with_memory_limit(100, 1.0) + .build_arc()?; + let task_ctx = TaskContext::default().with_runtime(runtime); + let task_ctx = Arc::new(task_ctx); + let err = multi_partitioned_join_collect( + Arc::clone(&left), + Arc::clone(&right), + join_type, + Some(filter.clone()), + task_ctx, + ) + .await + .unwrap_err(); + assert_contains!(err.to_string(), "Resources exhausted"); + } Ok(()) } @@ -3888,6 +3897,96 @@ pub(crate) mod tests { Ok(()) } + #[tokio::test] + async fn test_nlj_memory_limited_left_semi_join() -> Result<()> { + let task_ctx = task_ctx_with_memory_limit(50, 16)?; + let left = build_left_table(); + let right = build_right_table(); + let filter = prepare_join_filter(); + + let (columns, batches, metrics) = + join_collect(left, right, &JoinType::LeftSemi, Some(filter), task_ctx) + .await?; + + assert_eq!(columns, vec!["a1", "b1", "c1"]); + + assert!( + metrics.spill_count().unwrap_or(0) > 0, + "Expected spilling to occur under tight memory limit" + ); + + // Left semi: only left rows that matched at least one right row. + allow_duplicates!(assert_snapshot!(batches_to_sort_string(&batches), @r" + +----+----+----+ + | a1 | b1 | c1 | + +----+----+----+ + | 5 | 5 | 50 | + +----+----+----+ + ")); + Ok(()) + } + + #[tokio::test] + async fn test_nlj_memory_limited_left_anti_join() -> Result<()> { + let task_ctx = task_ctx_with_memory_limit(50, 16)?; + let left = build_left_table(); + let right = build_right_table(); + let filter = prepare_join_filter(); + + let (columns, batches, metrics) = + join_collect(left, right, &JoinType::LeftAnti, Some(filter), task_ctx) + .await?; + + assert_eq!(columns, vec!["a1", "b1", "c1"]); + + assert!( + metrics.spill_count().unwrap_or(0) > 0, + "Expected spilling to occur under tight memory limit" + ); + + // Left anti: left rows that did NOT match any right row. + allow_duplicates!(assert_snapshot!(batches_to_sort_string(&batches), @r" + +----+----+-----+ + | a1 | b1 | c1 | + +----+----+-----+ + | 11 | 8 | 110 | + | 9 | 8 | 90 | + +----+----+-----+ + ")); + Ok(()) + } + + #[tokio::test] + async fn test_nlj_memory_limited_left_mark_join() -> Result<()> { + let task_ctx = task_ctx_with_memory_limit(50, 16)?; + let left = build_left_table(); + let right = build_right_table(); + let filter = prepare_join_filter(); + + let (columns, batches, metrics) = + join_collect(left, right, &JoinType::LeftMark, Some(filter), task_ctx) + .await?; + + assert_eq!(columns, vec!["a1", "b1", "c1", "mark"]); + + assert!( + metrics.spill_count().unwrap_or(0) > 0, + "Expected spilling to occur under tight memory limit" + ); + + // Left mark: all left rows with a bool column indicating match. + allow_duplicates!(assert_snapshot!(batches_to_sort_string(&batches), @r" + +----+----+-----+-------+ + | a1 | b1 | c1 | mark | + +----+----+-----+-------+ + | 11 | 8 | 110 | false | + | 5 | 5 | 50 | true | + | 9 | 8 | 90 | false | + +----+----+-----+-------+ + ")); + Ok(()) + } + #[tokio::test] async fn test_nlj_fits_in_memory_no_spill() -> Result<()> { // Use a large memory limit — everything fits, no spilling needed. diff --git a/datafusion/physical-plan/src/spill/spill_pool.rs b/datafusion/physical-plan/src/spill/spill_pool.rs index 6e964d7a6497b..86e3e3b6cd8da 100644 --- a/datafusion/physical-plan/src/spill/spill_pool.rs +++ b/datafusion/physical-plan/src/spill/spill_pool.rs @@ -48,10 +48,11 @@ use super::spill_manager::SpillManager; /// **Lock ordering discipline**: Never hold both locks simultaneously to prevent deadlock. /// Always: acquire outer lock → release outer lock → acquire inner lock (if needed). struct SpillPoolShared { - /// Queue of ALL files (including the current write files if any exist). - /// Readers always read from the front of this queue (FIFO). - /// Each file has its own lock to enable concurrent reader/writer access. - files: VecDeque>>, + /// Files created by writers that the reader has not picked up yet, in creation + /// order. The reader moves them into its own list on every poll (see + /// [`SpillPoolReader`]), so this queue only ever holds files the reader has not + /// seen. Each file has its own lock to enable concurrent reader/writer access. + new_files: VecDeque>>, /// SpillManager for creating files and tracking metrics spill_manager: Arc, /// Pool-level waker to notify when new files are available (single reader) @@ -70,7 +71,7 @@ impl SpillPoolShared { /// Creates a new shared pool state fn new(spill_manager: Arc) -> Self { Self { - files: VecDeque::new(), + new_files: VecDeque::new(), spill_manager, waker: None, open_write_files: VecDeque::new(), @@ -237,7 +238,7 @@ impl SpillPoolSink { // Re-acquire lock and push to shared queue shared = self.shared.lock(); - shared.files.push_back(Arc::clone(&file_shared)); + shared.new_files.push_back(Arc::clone(&file_shared)); shared.wake(); // Wake readers waiting for new files file_shared }; @@ -570,17 +571,34 @@ struct SpillPoolFile { spill_manager: Arc, } -impl Stream for SpillPoolFile { - type Item = Result; +/// Outcome of polling a single file of the pool, see [`SpillPoolFile::poll_file`]. +enum FilePoll { + /// A batch was read from the file, or reading it failed. + Item(Result), + /// The file's stream ended: every batch written to it has been read and the + /// writer has finished it (the caller verifies the latter). + Done, + /// Every batch written so far has been read, but the writer has not finished + /// the file. The file's waker is registered, so the task is woken when the + /// writer appends another batch or finishes the file. + CaughtUp, + /// The file has unread batches, but the underlying I/O is not ready yet. The + /// I/O has registered the task's waker. + Pending, +} - fn poll_next( - mut self: std::pin::Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { +impl SpillPoolFile { + /// Polls the file for its next batch. + /// + /// Unlike a plain stream poll this tells the caller *why* no batch is + /// available, so [`SpillPoolReader`] can move on to a newer file when this + /// one is only waiting for its writer, but keeps waiting on it while its + /// data is still being read from disk (which keeps batches in file order). + fn poll_file(&mut self, cx: &mut std::task::Context<'_>) -> FilePoll { use std::task::Poll; // Step 1: Lock shared state and check coordination - let (should_read, file) = { + let file = { let mut shared = self.shared.lock(); // Determine if we can read @@ -588,24 +606,23 @@ impl Stream for SpillPoolFile { if batches_read < shared.batches_written { // More data available to read - take the file if we don't have a reader yet - let file = if self.reader.is_none() { + if self.reader.is_none() { shared.file.take() } else { None - }; - (true, file) + } } else if shared.writer_finished { // No more data and writer is done - EOF - return Poll::Ready(None); + return FilePoll::Done; } else { // Caught up to writer, but writer still active - register waker and wait shared.register_waker(cx.waker().clone()); - return Poll::Pending; + return FilePoll::CaughtUp; } }; // Lock released here // Step 2: Lazy-create reader stream if needed - if self.reader.is_none() && should_read { + if self.reader.is_none() { if let Some(file) = file { // we want this unbuffered because files are actively being written to match self @@ -618,36 +635,35 @@ impl Stream for SpillPoolFile { batches_read: 0, }); } - Err(e) => return Poll::Ready(Some(Err(e))), + Err(e) => return FilePoll::Item(Err(e)), } } else { // File not available yet (writer hasn't finished or already taken) // Register waker and wait for file to be ready let mut shared = self.shared.lock(); shared.register_waker(cx.waker().clone()); - return Poll::Pending; + return FilePoll::CaughtUp; } } // Step 3: Poll the reader stream (no lock held) - if let Some(reader) = &mut self.reader { - match reader.stream.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(batch))) => { - // Successfully read a batch - increment counter - reader.batches_read += 1; - Poll::Ready(Some(Ok(batch))) - } - Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))), - Poll::Ready(None) => { - // Stream exhausted unexpectedly - // This shouldn't happen if coordination is correct, but handle gracefully - Poll::Ready(None) - } - Poll::Pending => Poll::Pending, - } - } else { + let Some(reader) = &mut self.reader else { // Should not reach here, but handle gracefully - Poll::Ready(None) + return FilePoll::Done; + }; + match reader.stream.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(batch))) => { + // Successfully read a batch - increment counter + reader.batches_read += 1; + FilePoll::Item(Ok(batch)) + } + Poll::Ready(Some(Err(e))) => FilePoll::Item(Err(e)), + Poll::Ready(None) => { + // Stream exhausted unexpectedly + // This shouldn't happen if coordination is correct, but handle gracefully + FilePoll::Done + } + Poll::Pending => FilePoll::Pending, } } } @@ -669,11 +685,31 @@ impl Stream for SpillPoolFile { /// /// This makes it suitable for continuous streaming scenarios where the writer may /// produce data intermittently. +/// +/// # Reading from several open files +/// +/// With a single writer ([`spsc_channel`]) at most one file is open for writing at +/// a time and every earlier file is finished, so reading the files oldest first is +/// exactly FIFO. With several writers ([`mpsc_channel`]) concurrent pushes can +/// leave more than one file open, and a batch may land in a newer file while the +/// oldest file is drained but not finished. The reader therefore polls every file +/// it knows about, oldest first, and returns the first batch that is available. +/// A file that is merely caught up with its writer is skipped; the reader only +/// waits on a file while that file has unread batches whose bytes are still being +/// read from disk, which keeps batches in file order (strict FIFO for one writer). +/// +/// This matters for [`RepartitionExec`](crate::repartition::RepartitionExec), which +/// sends a "spilled" marker through its channel after every `push_batch` and then +/// blocks on this stream until it yields a batch. If the reader only ever drained the +/// oldest file, it could wait for a batch that had been written to a newer file; with +/// the channel gate closed the writers could not push anything to wake it, so the +/// query would deadlock (). pub struct SpillPoolReader { /// Shared reference to the spill pool shared: Arc>, - /// Current SpillPoolFile we're reading from - current_file: Option, + /// Files this reader has picked up from the pool and not fully consumed yet, in + /// creation order. Each carries its own lazily-created stream and read position. + files: VecDeque, /// Schema of the spilled data schema: SchemaRef, } @@ -689,7 +725,7 @@ impl SpillPoolReader { fn new(shared: Arc>, schema: SchemaRef) -> Self { Self { shared, - current_file: None, + files: VecDeque::new(), schema, } } @@ -704,75 +740,81 @@ impl Stream for SpillPoolReader { ) -> std::task::Poll> { use std::task::Poll; + // `Self: Unpin`; reborrow once so `files` and `shared` can be used together + let this = &mut *self; + loop { - // If we have a current file, try to read from it - if let Some(ref mut file) = self.current_file { - match file.poll_next_unpin(cx) { - Poll::Ready(Some(Ok(batch))) => { - // Got a batch, return it - return Poll::Ready(Some(Ok(batch))); - } - Poll::Ready(Some(Err(e))) => { - // Error reading batch - return Poll::Ready(Some(Err(e))); + // Pick up the files writers have created since the last poll, in + // creation order. + { + let mut shared = this.shared.lock(); + while let Some(file_shared) = shared.new_files.pop_front() { + let spill_manager = Arc::clone(&shared.spill_manager); + this.files.push_back(SpillPoolFile { + shared: file_shared, + reader: None, + spill_manager, + }); + } + } // Lock released here + + // Poll the files oldest first and return the first available batch. + // A file that is finished and fully read is dropped, which releases + // its disk space. A file that is only waiting for its writer has + // registered its own waker, so we move on to the next file instead + // of waiting on it. + let mut idx = 0; + while idx < this.files.len() { + match this.files[idx].poll_file(cx) { + FilePoll::Item(item) => { + // Got a batch (or an error), return it + return Poll::Ready(Some(item)); } - Poll::Ready(None) => { - // Current file stream exhausted + FilePoll::Done => { + // File stream exhausted // Check if this file is marked as writer_finished - let writer_finished = { file.shared.lock().writer_finished }; + let writer_finished = + { this.files[idx].shared.lock().writer_finished }; if writer_finished { - // File is complete, pop it from the queue and move to next - let mut shared = self.shared.lock(); - shared.files.pop_front(); - drop(shared); // Release lock - - // Clear current file and continue loop to get next file - self.current_file = None; - continue; + // File is complete, drop it and move on to the next + this.files.remove(idx); } else { // Stream exhausted but writer not finished - unexpected // This shouldn't happen with proper coordination return Poll::Ready(None); } } - Poll::Pending => { - // File not ready yet (waiting for writer) - // Register waker so we get notified when writer adds more batches - let mut shared = self.shared.lock(); - shared.register_waker(cx.waker().clone()); + FilePoll::Pending => { + // The oldest file with unread batches is still being read + // from disk. Wait for it rather than skipping ahead, so + // batches come back in file order (FIFO for one writer). return Poll::Pending; } + FilePoll::CaughtUp => { + // Nothing to read here until its writer appends more; a + // newer file may have unread batches, so try the next one + idx += 1; + } } } - // No current file, need to get the next one - let mut shared = self.shared.lock(); - - // Peek at the front of the queue (don't pop yet) - if let Some(file_shared) = shared.files.front() { - // Create a SpillPoolFile from the shared state - let spill_manager = Arc::clone(&shared.spill_manager); - let file_shared = Arc::clone(file_shared); - drop(shared); // Release lock before creating SpillPoolFile - - self.current_file = Some(SpillPoolFile { - shared: file_shared, - reader: None, - spill_manager, - }); + // No known file has an unread batch + let mut shared = this.shared.lock(); - // Continue loop to poll the new file + if !shared.new_files.is_empty() { + // A writer created a file while we were polling; pick it up continue; } - // No files in queue - check if writer is done - if shared.remaining_writer_count == 0 { + if this.files.is_empty() && shared.remaining_writer_count == 0 { // Writer is done and no more files will be added - EOF return Poll::Ready(None); } - // Writer still active, register waker that will get notified when new files are added + // Writer still active: register the pool-level waker so we are + // notified when a new file is created or the last writer is dropped. + // Each pending file has registered its own waker for new batches. shared.register_waker(cx.waker().clone()); return Poll::Pending; } @@ -791,8 +833,20 @@ mod tests { use crate::metrics::{ExecutionPlanMetricsSet, SpillMetrics}; use arrow::array::{ArrayRef, Int32Array}; use arrow::datatypes::{DataType, Field, Schema}; + use datafusion_common::instant::Instant; + use datafusion_common::{DataFusionError, exec_datafusion_err}; use datafusion_common_runtime::{JoinSet, SpawnedTask}; + use datafusion_execution::disk_manager::{ + DiskManager, DiskManagerBuilder, DiskManagerMode, + }; use datafusion_execution::runtime_env::RuntimeEnv; + use datafusion_execution::runtime_env::RuntimeEnvBuilder; + use datafusion_execution::{SpillFile, SpillWriter, TempFileFactory}; + use std::pin::Pin; + use std::sync::Barrier; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::mpsc::{self, Sender}; + use std::time::Duration; fn create_test_schema() -> SchemaRef { Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])) @@ -919,8 +973,7 @@ mod tests { // Reader should pend since no batches were written let mut reader = reader; let result = - tokio::time::timeout(std::time::Duration::from_millis(100), reader.next()) - .await; + tokio::time::timeout(Duration::from_millis(100), reader.next()).await; assert!(result.is_err(), "Reader should timeout on empty writer"); @@ -1230,7 +1283,7 @@ mod tests { let batch = create_test_batch(i * 10, 10); writer.push_batch(&batch).unwrap(); // Small delay to simulate real concurrent work - tokio::time::sleep(std::time::Duration::from_millis(5)).await; + tokio::time::sleep(Duration::from_millis(5)).await; } }); @@ -1523,10 +1576,9 @@ mod tests { proceed_tx.send(()).unwrap(); // The reader should wait (Pending) for writer2's data, not EOF. - let batch2 = - tokio::time::timeout(std::time::Duration::from_secs(5), reader.next()) - .await - .expect("Reader timed out — should not hang"); + let batch2 = tokio::time::timeout(Duration::from_secs(5), reader.next()) + .await + .expect("Reader timed out — should not hang"); assert!( batch2.is_some(), @@ -1645,4 +1697,576 @@ mod tests { Ok(()) } + + type WriteHook = Arc; + /// What a read of a spill file does. The two failure modes reach the two + /// different error paths of [`SpillPoolFile::poll_file`]: one fails while it + /// builds the stream, the other fails while it polls a stream that is + /// already built. + enum ReadBehavior { + /// Read the real bytes, after this delay. + Delay(Duration), + /// Fail to open the file, so the stream is never built. + FailOpen(DataFusionError), + /// Open the file, then give this error as the first item of the stream. + FailFirstItem(DataFusionError), + } + + type ReadHook = Arc ReadBehavior + Send + Sync>; + + /// Test double for a spill file that runs `write_hook` before every disk + /// write or flush, and consults `read_hook` before every read. Writers write + /// while holding their file's lock, so a hook that blocks pauses a + /// `push_batch` at exactly the point where it has a file checked out for + /// writing; a read delay makes an older file's bytes arrive after a newer + /// file's, and a read error fails the read. Data still goes to real + /// temporary files. + struct IoHookFactory { + inner: Arc, + write_hook: WriteHook, + read_hook: ReadHook, + } + + impl TempFileFactory for IoHookFactory { + fn create_temp_file(&self, description: &str) -> Result> { + Ok(Arc::new(IoHookFile { + inner: self.inner.create_tmp_file(description)?, + write_hook: Arc::clone(&self.write_hook), + read_hook: Arc::clone(&self.read_hook), + })) + } + } + + struct IoHookFile { + inner: Arc, + write_hook: WriteHook, + read_hook: ReadHook, + } + + impl SpillFile for IoHookFile { + fn path(&self) -> Option<&std::path::Path> { + self.inner.path() + } + + fn size(&self) -> Option { + self.inner.size() + } + + fn read_stream( + &self, + ) -> Result> + Send>>> { + let delay = match (self.read_hook)() { + ReadBehavior::Delay(delay) => delay, + ReadBehavior::FailOpen(e) => return Err(e), + ReadBehavior::FailFirstItem(e) => { + return Ok(Box::pin(futures::stream::once(async move { Err(e) }))); + } + }; + let mut inner = Some(self.inner.read_stream()?); + Ok(Box::pin( + futures::stream::once(tokio::time::sleep(delay)) + .flat_map(move |_| inner.take().expect("polled once")), + )) + } + + fn open_writer(&self) -> Result> { + Ok(Box::new(IoHookWriter { + inner: self.inner.open_writer()?, + hook: Arc::clone(&self.write_hook), + })) + } + } + + struct IoHookWriter { + inner: Box, + hook: WriteHook, + } + + impl std::io::Write for IoHookWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + (self.hook)(); + self.inner.write(buf) + } + + fn flush(&mut self) -> std::io::Result<()> { + (self.hook)(); + self.inner.flush() + } + } + + impl SpillWriter for IoHookWriter { + fn finish(&mut self) -> Result<()> { + self.inner.finish() + } + } + + /// A `SpillManager` whose files run `write_hook` before every disk write and + /// consult `read_hook` before every read, plus the `DiskManager` that owns + /// the files so tests can check disk usage. + fn spill_manager_with_io_hooks( + write_hook: WriteHook, + read_hook: ReadHook, + ) -> Result<(Arc, Arc)> { + let disk_manager = Arc::new(DiskManagerBuilder::default().build()?); + let runtime = RuntimeEnvBuilder::new() + .with_disk_manager_builder(DiskManagerBuilder::default().with_mode( + DiskManagerMode::Custom(Arc::new(IoHookFactory { + inner: Arc::clone(&disk_manager), + write_hook, + read_hook, + })), + )) + .build_arc()?; + let metrics = SpillMetrics::new(&ExecutionPlanMetricsSet::new(), 0); + let spill_manager = + Arc::new(SpillManager::new(runtime, metrics, create_test_schema())); + Ok((spill_manager, disk_manager)) + } + + /// Holds the first disk write on the pool until the test releases it, and + /// reports every other write so a test can wait for a second writer to + /// reach a file of its own instead of sleeping. + struct WriteGate { + taken: AtomicBool, + entered: Barrier, + release: Barrier, + other_writes: Sender<()>, + } + + impl WriteGate { + fn hold_if_first(&self) { + if self.taken.swap(true, Ordering::SeqCst) { + // A write only gets here once its writer holds a file of its + // own: a writer that had to wait for the held writer's file + // lock would not reach a write of its own at all. + let _ = self.other_writes.send(()); + } else { + self.entered.wait(); + self.release.wait(); + } + } + } + + /// Regression test for . + /// + /// Two writers push concurrently. The first writer is held inside its first + /// write to disk (holding its file's lock) while the second writer pushes. + /// On the unfixed pool that leaves two open files with one batch each, and + /// the reader, which only ever drained the oldest file, parked on that file + /// once it had read its single batch even though the second batch was on + /// disk. `RepartitionExec` blocks on this stream once per pushed batch and, + /// with its channel gate closed, cannot push anything else to wake it. + /// + /// The reader must yield both batches while both writers are still alive. + #[tokio::test] + async fn test_reader_does_not_wait_on_drained_file_while_another_has_data() + -> Result<()> { + let (other_writes_tx, other_writes) = mpsc::channel(); + let gate = Arc::new(WriteGate { + taken: AtomicBool::new(false), + entered: Barrier::new(2), + release: Barrier::new(2), + other_writes: other_writes_tx, + }); + let write_hook: WriteHook = { + let gate = Arc::clone(&gate); + Arc::new(move || gate.hold_if_first()) + }; + let (spill_manager, _disk_manager) = spill_manager_with_io_hooks( + write_hook, + Arc::new(|| ReadBehavior::Delay(Duration::ZERO)), + )?; + + let (writer1, mut reader) = mpsc_channel(1024 * 1024, Arc::clone(&spill_manager)); + let writer2 = writer1.clone(); + + // Writer 1 pushes and is held inside its first disk write, holding its + // file's lock. The writers run on plain threads because `push_batch` + // blocks. + let writer1 = std::thread::spawn(move || { + writer1.push_batch(&create_test_batch(0, 10)).unwrap(); + writer1 + }); + gate.entered.wait(); + + // Writer 2 pushes while writer 1 is held. + let writer2 = std::thread::spawn(move || { + writer2.push_batch(&create_test_batch(10, 10)).unwrap(); + writer2 + }); + // Wait for writer 2 to reach a write of its own, so that writer 1 is + // only released once the pool really holds two open files. Sleeping + // instead would let a loaded runner schedule writer 2 after writer 1 + // was released, and writer 2 would then reuse writer 1's returned file: + // a single-file run that says nothing about the case under test. + other_writes + .recv_timeout(Duration::from_secs(30)) + .expect("writer 2 must write to a file of its own while writer 1 is held"); + gate.release.wait(); + let writer1 = writer1.join().unwrap(); + let writer2 = writer2.join().unwrap(); + assert_eq!( + spill_manager.metrics.spill_file_count.value(), + 2, + "the writers must have written to two different files" + ); + + // Both batches are on disk and both writers are still alive, so no file + // is finished. The reader must yield both batches without waiting for a + // writer. + let mut values = vec![]; + for _ in 0..2 { + let batch = tokio::time::timeout(Duration::from_secs(5), reader.next()) + .await + .expect( + "reader waited on a drained file while another file had an unread batch", + ) + .expect("reader must not signal EOF while writers are alive")?; + assert_eq!(batch.num_rows(), 10); + values.push(id_of(&batch)); + } + // Multiple writers: the order between their batches is not guaranteed. + values.sort_unstable(); + assert_eq!(values, vec![0, 10]); + + // Only once every writer is gone does the reader signal EOF. + drop(writer1); + drop(writer2); + assert!(reader.next().await.is_none()); + + Ok(()) + } + + /// A spill file that cannot be read must fail the reader rather than stall + /// it. `RepartitionExec` blocks on this stream after every spilled marker, + /// so an error that is swallowed here would hang the query exactly like the + /// deadlock this module guards against. + /// + /// Both error paths of `SpillPoolFile::poll_file` are covered: the file that + /// does not open, so the failure comes from the construction of the stream, + /// and the file that opens and then gives an error, so the failure comes + /// from a poll of a stream that the reader already holds. + #[tokio::test] + async fn test_read_errors_are_reported_to_the_reader() -> Result<()> { + async fn assert_reader_reports(behavior: fn() -> ReadBehavior) -> Result<()> { + let (spill_manager, _disk_manager) = + spill_manager_with_io_hooks(Arc::new(|| {}), Arc::new(behavior))?; + let (writer, mut reader) = spsc_channel(1024 * 1024, spill_manager); + + // Nothing reads the file until the reader does, so every read that + // the pool makes is a read of the batch below. + writer.push_batch(&create_test_batch(0, 10))?; + + let item = tokio::time::timeout(Duration::from_secs(5), reader.next()) + .await + .expect("reader must report the read error instead of waiting") + .expect("reader must report the read error instead of signalling EOF"); + let err = item.expect_err("a failed read must not produce a batch"); + assert!( + err.to_string().contains("injected spill read failure"), + "unexpected error: {err}" + ); + Ok(()) + } + + assert_reader_reports(|| { + ReadBehavior::FailOpen(exec_datafusion_err!("injected spill read failure")) + }) + .await?; + assert_reader_reports(|| { + ReadBehavior::FailFirstItem(exec_datafusion_err!( + "injected spill read failure" + )) + }) + .await?; + + Ok(()) + } + + /// Pauses every writer of a "phase" inside its first disk write, so that + /// several `push_batch` calls overlap at the point where each holds a file + /// checked out for writing. A writer that blocks earlier (for example on + /// another writer's file lock) never arrives; `release_phase` stops waiting + /// for it after a short while. + #[derive(Default)] + struct PhasePauser { + state: Mutex, + changed: parking_lot::Condvar, + } + + #[derive(Default)] + struct PhaseState { + phase: u64, + expected: usize, + held: usize, + released: bool, + } + + thread_local! { + /// The phase in which the current writer thread has already been held. + static HELD_IN_PHASE: std::cell::Cell = const { std::cell::Cell::new(0) }; + } + + impl PhasePauser { + fn begin_phase(&self, expected: usize) { + let mut state = self.state.lock(); + state.phase += 1; + state.expected = expected; + state.held = 0; + state.released = false; + } + + /// Called by writer threads before every disk write. + fn on_io(&self) { + let mut state = self.state.lock(); + if state.released + || state.held >= state.expected + || HELD_IN_PHASE.get() == state.phase + { + return; + } + HELD_IN_PHASE.set(state.phase); + state.held += 1; + self.changed.notify_all(); + while !state.released { + self.changed.wait(&mut state); + } + } + + /// Waits until every writer of the phase is held (or 300 ms have + /// passed), then releases them all at once. + fn release_phase(&self) { + let mut state = self.state.lock(); + let deadline = Instant::now() + Duration::from_millis(300); + while state.held < state.expected { + if self.changed.wait_until(&mut state, deadline).timed_out() { + break; + } + } + state.released = true; + self.changed.notify_all(); + } + } + + fn batch_with_id(id: i32, rows: usize) -> RecordBatch { + let a: ArrayRef = Arc::new(Int32Array::from(vec![id; rows])); + RecordBatch::try_new(create_test_schema(), vec![a]).unwrap() + } + + fn id_of(batch: &RecordBatch) -> i32 { + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(0) + } + + /// One randomly generated scenario, see [`spill_pool_scenario_fuzz`]. + async fn run_spill_pool_scenario(seed: u64) -> Result<()> { + use rand::rngs::StdRng; + use rand::{Rng, SeedableRng}; + + const READ_TIMEOUT: Duration = Duration::from_secs(5); + + let mut rng = StdRng::seed_from_u64(seed); + let n_writers = rng.random_range(1..=4); + let batch_bytes = batch_with_id(0, 10).get_array_memory_size(); + // Rotate after every batch, after a few batches, or never. + let max_file_size = match rng.random_range(0..3) { + 0 => batch_bytes / 2, + 1 => batch_bytes * 3, + _ => usize::MAX / 2, + }; + let n_phases = rng.random_range(1..=6); + let context = format!( + "seed {seed}: {n_writers} writers, max_file_size {max_file_size}, {n_phases} phases" + ); + + let pauser = Arc::new(PhasePauser::default()); + let write_hook: WriteHook = { + let pauser = Arc::clone(&pauser); + Arc::new(move || pauser.on_io()) + }; + // Some files are slow to read back, so an older file's bytes can arrive + // after a newer file's. Batches must still come back in file order for a + // single writer. + let read_hook: ReadHook = { + let rng = Mutex::new(StdRng::seed_from_u64(seed ^ 0x5EED_4EAD)); + Arc::new(move || { + let mut rng = rng.lock(); + if rng.random_bool(0.5) { + ReadBehavior::Delay(Duration::ZERO) + } else { + ReadBehavior::Delay(Duration::from_millis(rng.random_range(1..=20))) + } + }) + }; + let (spill_manager, disk_manager) = + spill_manager_with_io_hooks(write_hook, read_hook)?; + let (writer, mut reader) = mpsc_channel(max_file_size, spill_manager); + let mut sinks: Vec> = + (0..n_writers).map(|_| Some(writer.new_sink())).collect(); + drop(writer); + + let mut pushed: Vec = vec![]; + let mut read: Vec = vec![]; + + for phase in 0..n_phases { + let alive: Vec = + (0..n_writers).filter(|w| sinks[*w].is_some()).collect(); + if alive.is_empty() { + break; + } + + // A random non-empty subset of the live writers each pushes a few + // batches, concurrently, held so that their pushes overlap. + let mut pushing: Vec = alive + .iter() + .copied() + .filter(|_| rng.random_bool(0.6)) + .collect(); + if pushing.is_empty() { + pushing.push(alive[rng.random_range(0..alive.len())]); + } + pauser.begin_phase(pushing.len()); + let mut threads = Vec::with_capacity(pushing.len()); + for w in pushing { + let sink = sinks[w].take().unwrap(); + let batches: Vec = (0..rng.random_range(1..=3)) + .map(|_| { + let id = pushed.len() as i32 + 1; + pushed.push(id); + batch_with_id(id, rng.random_range(1..=40)) + }) + .collect(); + threads.push(( + w, + std::thread::spawn(move || { + for batch in &batches { + sink.push_batch(batch).unwrap(); + } + sink + }), + )); + } + pauser.release_phase(); + for (w, thread) in threads { + sinks[w] = Some(thread.join().unwrap()); + } + + // Quiescent point: no push is in progress and none will happen until + // the reader is done. Every batch pushed so far must be readable now, + // in any amount. `RepartitionExec` relies on exactly this: it blocks + // on the reader once per pushed batch while its channel gate keeps + // the writers parked. Sometimes leave a backlog for later phases. + let read_now = rng.random_range(0..=pushed.len() - read.len()); + for _ in 0..read_now { + let next = tokio::time::timeout(READ_TIMEOUT, reader.next()) + .await + .unwrap_or_else(|_| { + panic!( + "{context}, phase {phase}: reader stalled with {} of {} \ + batches unread and {} writers alive", + pushed.len() - read.len(), + pushed.len(), + alive.len() + ) + }); + let batch = next.unwrap_or_else(|| { + panic!("{context}, phase {phase}: EOF while writers are alive") + })?; + read.push(id_of(&batch)); + } + + // With nothing left to read the reader must wait, not signal EOF. + if read.len() == pushed.len() && rng.random_bool(0.3) { + let probe = + tokio::time::timeout(Duration::from_millis(50), reader.next()).await; + assert!( + probe.is_err(), + "{context}, phase {phase}: reader yielded {probe:?} with nothing \ + pushed and writers alive" + ); + } + + // Dropping one of several writers must not disturb anything. + if alive.len() > 1 && rng.random_bool(0.3) { + sinks[alive[rng.random_range(0..alive.len())]] = None; + } + } + + // Once the last writer is gone the reader must hand out the backlog and + // then signal EOF. + sinks.clear(); + while read.len() < pushed.len() { + let next = tokio::time::timeout(READ_TIMEOUT, reader.next()) + .await + .unwrap_or_else(|_| { + panic!( + "{context}: reader stalled with {} of {} batches unread after \ + all writers were dropped", + pushed.len() - read.len(), + pushed.len() + ) + }); + let batch = next.unwrap_or_else(|| { + panic!( + "{context}: EOF with {} batches unread", + pushed.len() - read.len() + ) + })?; + read.push(id_of(&batch)); + } + let eof = tokio::time::timeout(READ_TIMEOUT, reader.next()) + .await + .unwrap_or_else(|_| { + panic!("{context}: reader stalled instead of signalling EOF after all writers were dropped") + }); + assert!(eof.is_none(), "{context}: batch after everything was read"); + + let mut expected = pushed.clone(); + expected.sort_unstable(); + let mut actual = read.clone(); + actual.sort_unstable(); + assert_eq!(actual, expected, "{context}: batches lost or duplicated"); + if n_writers == 1 { + assert_eq!(read, pushed, "{context}: single-writer pool must be FIFO"); + } + + drop(reader); + assert_eq!( + disk_manager.used_disk_space(), + 0, + "{context}: spill files not released" + ); + Ok(()) + } + + /// Randomized scenarios against the pool's contract: after any set of + /// overlapping pushes the reader can drain everything without further + /// writer activity (no lost wakeups, no parking on one file while another + /// has data), it never signals EOF while a writer is alive, it returns + /// exactly the pushed batches (in order for a single writer), and the + /// files are released when it is dropped. + /// + /// `DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS` and + /// `DATAFUSION_SPILL_POOL_FUZZ_SEED` select how many scenarios run and + /// from which seed; a failure names its seed so it can be replayed. + #[tokio::test] + async fn spill_pool_scenario_fuzz() -> Result<()> { + let env_u64 = |name: &str, default: u64| { + std::env::var(name) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) + }; + let first_seed = env_u64("DATAFUSION_SPILL_POOL_FUZZ_SEED", 0); + let iterations = env_u64("DATAFUSION_SPILL_POOL_FUZZ_ITERATIONS", 50); + // `saturating_add` so a replay seed near `u64::MAX` cannot overflow. + for seed in first_seed..first_seed.saturating_add(iterations) { + run_spill_pool_scenario(seed).await?; + } + Ok(()) + } } diff --git a/datafusion/pruning/src/pruning_predicate.rs b/datafusion/pruning/src/pruning_predicate.rs index 4735fa375ced8..8f176426fcc70 100644 --- a/datafusion/pruning/src/pruning_predicate.rs +++ b/datafusion/pruning/src/pruning_predicate.rs @@ -1418,6 +1418,16 @@ fn verify_support_type_for_prune(from_type: &DataType, to_type: &DataType) -> Re } _ => to_type, }; + // Casting a timestamp to a time-of-day wraps at midnight and does not + // preserve ordering across day boundaries. Casting timestamp min/max + // statistics could therefore produce invalid bounds and prune matches. + if matches!(from_type, DataType::Timestamp(_, _)) + && matches!(to_type, DataType::Time32(_) | DataType::Time64(_)) + { + return plan_err!( + "Try Cast/Cast with from type {from_type} to type {to_type} is not supported" + ); + } // If both types are strings or both are not strings (number, timestamp, etc) // then we can compare them. // PruningPredicate does not support casting of strings to numbers and such. @@ -5132,6 +5142,56 @@ mod tests { Ok(()) } + #[test] + fn row_group_predicate_cast_timestamp_to_time() { + // A timezone shifts where the day boundary falls but does not remove + // it, so a timezone-aware timestamp is just as unsafe to prune on. + for timestamp_type in [ + DataType::Timestamp(TimeUnit::Second, None), + DataType::Timestamp(TimeUnit::Second, Some("+05:00".into())), + DataType::Timestamp(TimeUnit::Nanosecond, Some("UTC".into())), + ] { + assert_timestamp_to_time_cast_is_not_prunable(timestamp_type); + } + } + + /// Asserts that no comparison of `CAST`/`TRY_CAST` from `timestamp_type` to + /// a time-of-day produces a pruning expression. + fn assert_timestamp_to_time_cast_is_not_prunable(timestamp_type: DataType) { + let schema = Schema::new(vec![Field::new("ts", timestamp_type, false)]); + + for (time_type, literal) in [ + ( + DataType::Time32(TimeUnit::Second), + ScalarValue::Time32Second(Some(0)), + ), + ( + DataType::Time32(TimeUnit::Millisecond), + ScalarValue::Time32Millisecond(Some(0)), + ), + ( + DataType::Time64(TimeUnit::Microsecond), + ScalarValue::Time64Microsecond(Some(0)), + ), + ( + DataType::Time64(TimeUnit::Nanosecond), + ScalarValue::Time64Nanosecond(Some(0)), + ), + ] { + for expr in [ + cast(col("ts"), time_type.clone()).eq(lit(literal.clone())), + try_cast(col("ts"), time_type).eq(lit(literal)), + ] { + let predicate_expr = test_build_predicate_expression( + &expr, + &schema, + &mut RequiredColumns::new(), + ); + assert_eq!(predicate_expr.to_string(), "true"); + } + } + } + #[test] fn row_group_predicate_cast_string_string() -> Result<()> { let schema = Schema::new(vec![Field::new("c1", DataType::Utf8View, false)]); diff --git a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt index bdb56cf22045a..7120458956b55 100644 --- a/datafusion/sqllogictest/test_files/null_aware_anti_join.slt +++ b/datafusion/sqllogictest/test_files/null_aware_anti_join.slt @@ -414,6 +414,62 @@ SELECT * FROM test_table WHERE (c1 NOT IN (SELECT c2 FROM test_table)) = true; # This is a known limitation - currently only supports single column null-aware anti joins. # This will be addressed in next Phase (multi-column support). +############# +## Correlated NOT IN with a non-equality (residual) correlation +############# +## The correlation `residual_outer.g > 0` cannot become an equi-join key, so it +## stays a join filter. A NULL in the subquery then only makes `NOT IN` +## UNKNOWN for the outer rows the filter keeps: the rows with `g = 0` or +## `g IS NULL` see an empty subquery, and `NOT IN (empty)` is TRUE for them. + +statement ok +CREATE TABLE residual_outer(id INT, g INT) AS VALUES (1,5),(2,5),(3,0),(4,NULL),(NULL,5),(NULL,0); + +statement ok +CREATE TABLE residual_inner(id INT) AS VALUES (1),(NULL); + +query II rowsort +SELECT id, g FROM residual_outer +WHERE id NOT IN (SELECT id FROM residual_inner WHERE residual_outer.g > 0); +---- +3 0 +4 NULL +NULL 0 + +# The shape the executor fix covers: a single equi key plus the residual +# `> 0` correlation as the join filter, on a null-aware `LeftAnti`. +query TT +EXPLAIN SELECT id, g FROM residual_outer +WHERE id NOT IN (SELECT id FROM residual_inner WHERE residual_outer.g > 0); +---- +logical_plan +01)LeftAnti Join: residual_outer.id = __correlated_sq_1.id Filter: residual_outer.g > Int32(0) null_aware +02)--TableScan: residual_outer projection=[id, g] +03)--SubqueryAlias: __correlated_sq_1 +04)----TableScan: residual_inner projection=[id] +physical_plan +01)HashJoinExec: mode=CollectLeft, join_type=LeftAnti, on=[(id@0, id@0)], filter=g@0 > 0, null_aware +02)--DataSourceExec: partitions=1, partition_sizes=[1] +03)--DataSourceExec: partitions=1, partition_sizes=[1] + +# Subquery left empty by the residual: NULL NOT IN (empty) is TRUE. +query II rowsort +SELECT id, g FROM residual_outer +WHERE id NOT IN (SELECT id FROM residual_inner WHERE residual_outer.g > 100); +---- +1 5 +2 5 +3 0 +4 NULL +NULL 0 +NULL 5 + +statement ok +DROP TABLE residual_outer; + +statement ok +DROP TABLE residual_inner; + ############# ## Cleanup ############# diff --git a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt index 72d034067663e..1dd691d9b8967 100644 --- a/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt +++ b/datafusion/sqllogictest/test_files/push_down_filter_parquet.slt @@ -1336,3 +1336,166 @@ RESET datafusion.execution.parquet.pushdown_filters; statement ok drop table t; + +########## +# Casting a timestamp to a time-of-day wraps at midnight +# +# https://github.com/apache/datafusion/issues/26451 +########## + +# Pruning must never change a query's answer. A cast whose output does not +# preserve the ordering of its input can invert the bounds derived from +# statistics, which makes pruning discard row groups that do contain matching +# rows. Every predicate below is therefore run twice: once against an +# in-memory table, which is never pruned and so gives the ground truth, and +# once against the same rows stored as Parquet. The two must agree. + +# Three timestamps at 23:00, 00:00 and 01:00, written as a single row group so +# that the row group's min/max bracket midnight. Casting those bounds to +# time-of-day turns them into 23:00:00 and 01:00:00, which excludes every value +# that actually lies between them. +statement ok +CREATE TABLE source_data(ts TIMESTAMP) AS VALUES + (TIMESTAMP '2024-03-10 23:00:00'), + (TIMESTAMP '2024-03-11 00:00:00'), + (TIMESTAMP '2024-03-11 01:00:00'); + +statement ok +COPY (SELECT ts FROM source_data) +TO 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps.parquet' +STORED AS PARQUET +OPTIONS ('format.max_row_group_size' '3'); + +statement ok +CREATE EXTERNAL TABLE timestamps +STORED AS PARQUET +LOCATION 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps.parquet'; + +# Equality against the row at midnight. +query P +SELECT ts FROM source_data WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00 + +query P +SELECT ts FROM timestamps WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00 + +# Range predicate spanning the wrap point. +query P +SELECT ts FROM source_data WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00 +2024-03-11T01:00:00 + +query P +SELECT ts FROM timestamps WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00 +2024-03-11T01:00:00 + +# The same predicates spelled with arrow_cast, which also reaches Time32. +query P +SELECT ts FROM source_data +WHERE arrow_cast(ts, 'Time32(Second)') = arrow_cast('00:00:00', 'Time32(Second)') +ORDER BY ts; +---- +2024-03-11T00:00:00 + +query P +SELECT ts FROM timestamps +WHERE arrow_cast(ts, 'Time32(Second)') = arrow_cast('00:00:00', 'Time32(Second)') +ORDER BY ts; +---- +2024-03-11T00:00:00 + +query P +SELECT ts FROM source_data +WHERE arrow_cast(ts, 'Time32(Second)') < arrow_cast('02:00:00', 'Time32(Second)') +ORDER BY ts; +---- +2024-03-11T00:00:00 +2024-03-11T01:00:00 + +query P +SELECT ts FROM timestamps +WHERE arrow_cast(ts, 'Time32(Second)') < arrow_cast('02:00:00', 'Time32(Second)') +ORDER BY ts; +---- +2024-03-11T00:00:00 +2024-03-11T01:00:00 + +statement ok +DROP TABLE timestamps; + +########## +# The same wrap happens at the local midnight of a timezone-aware column +########## + +# The same three local wall-clock times, tagged +05:00, so the instants +# actually stored are 18:00, 19:00 and 20:00 UTC on 2024-03-10. Those are +# monotonic and fall inside a single UTC day, so the min/max bounds only +# invert if the cast honours the column's timezone. +statement ok +CREATE TABLE source_data_tz AS +SELECT arrow_cast(ts, 'Timestamp(Nanosecond, Some("+05:00"))') AS ts +FROM source_data; + +statement ok +DROP TABLE source_data; + +statement ok +COPY (SELECT ts FROM source_data_tz) +TO 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps_tz.parquet' +STORED AS PARQUET +OPTIONS ('format.max_row_group_size' '3'); + +statement ok +CREATE EXTERNAL TABLE timestamps_tz +STORED AS PARQUET +LOCATION 'test_files/scratch/push_down_filter_parquet/timestamp_to_time/timestamps_tz.parquet'; + +# Pin the shape of the data this section depends on: the local times have to +# straddle a local midnight while the stored instants stay monotonic within a +# single UTC day. Without this, a change to how a timezone is attached would +# leave the queries below passing while no longer exercising the wrap at all. +query TIT +SELECT + ts::string AS local_time, + to_unixtime(ts) AS stored_instant, + CAST(ts AS TIME)::string AS time_of_day +FROM timestamps_tz ORDER BY ts; +---- +2024-03-10T23:00:00+05:00 1710093600 23:00:00 +2024-03-11T00:00:00+05:00 1710097200 00:00:00 +2024-03-11T01:00:00+05:00 1710100800 01:00:00 + +query P +SELECT ts FROM source_data_tz WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00+05:00 + +query P +SELECT ts FROM timestamps_tz WHERE CAST(ts AS TIME) = TIME '00:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00+05:00 + +query P +SELECT ts FROM source_data_tz WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00+05:00 +2024-03-11T01:00:00+05:00 + +query P +SELECT ts FROM timestamps_tz WHERE CAST(ts AS TIME) < TIME '02:00:00' ORDER BY ts; +---- +2024-03-11T00:00:00+05:00 +2024-03-11T01:00:00+05:00 + +statement ok +DROP TABLE timestamps_tz; + +statement ok +DROP TABLE source_data_tz; +