diff --git a/docs/source/contributor-guide/memory_management.md b/docs/source/contributor-guide/memory_management.md index c90b5b1e853..3953f55ec38 100644 --- a/docs/source/contributor-guide/memory_management.md +++ b/docs/source/contributor-guide/memory_management.md @@ -336,6 +336,19 @@ JNI, which goes through Spark's ordinary `TaskMemoryManager`. That means: granted. While overcommit is outstanding, `try_grow` asks Spark for it on top of the request and is refused unless Spark can cover both, so operators spill until the debt is repaid. Both pools' `Display` output and their `try_grow` errors report the current overcommit. +- When Spark throws while acquiring memory, for instance because a spill of another consumer + failed, every later `try_grow` of the task fails with that exception, without asking Spark and + before the fair pool checks its own limits. A later `grow` records the full amount as + overcommit, also without asking Spark. The exception is not `ResourcesExhausted`, so an operator + that checks for a refusal, such as the shuffle repartitioner, fails the task rather than + spilling, and one that takes any error for a refusal and asks again, such as DataFusion's sort, + fails when it asks again. The JNI boundary rethrows the exception as itself, unless an operator + has wrapped it in `Context` or `Shared`, which turns it into a `CometNativeException`. Nothing + checks for the failure when a plan's output ends, so the exception is still only logged if no + operator returns it: if no `try_grow` follows a failed `grow` in the task, or if every `try_grow` + that fails comes from DataFusion's sort-merge join, whose `materializing_stream.rs` and + `bitwise_stream.rs` take any error for a refusal, spill the batch and carry on + ([issue #7615](https://github.com/apache/datafusion-comet/issues/7615)). `CometFairMemoryPool` exists because Spark cannot ask a native operator to spill. Spark limits each task to an even share of its off-heap pool among the tasks that hold memory from it. Once one diff --git a/docs/source/contributor-guide/native_shuffle.md b/docs/source/contributor-guide/native_shuffle.md index 30e24b950f6..93829e3bad8 100644 --- a/docs/source/contributor-guide/native_shuffle.md +++ b/docs/source/contributor-guide/native_shuffle.md @@ -491,9 +491,11 @@ to handle sliced batches or an indeterminate stage's rollback. Native shuffle uses DataFusion's memory management with spilling support: - **Memory pool**: Tracks memory usage across the shuffle operation. -- **Spill triggers**: Partitions spill to disk when the memory pool denies an allocation, or - when the buffered bytes reach `spark.comet.shuffle.native.maxBufferBytes`. That config defaults to - 0, which disables the fixed limit and leaves memory pressure as the only trigger. +- **Spill triggers**: Partitions spill to disk when the memory pool refuses an allocation with + `ResourcesExhausted`, or when the buffered bytes reach + `spark.comet.shuffle.native.maxBufferBytes`. That config defaults to 0, which disables the fixed + limit and leaves memory pressure as the only trigger. Any other error from the pool, such as an + exception Spark threw while spilling another consumer, fails the task. - **One spill file per task**: Every output partition spills into the same file, not one file per partition. `PartitionedSpill` records the byte ranges each partition's blocks occupy, in write order, and the final output copies a partition's ranges back in that order. diff --git a/native/common/src/error.rs b/native/common/src/error.rs index c227c3f1b8e..35a7abb8c44 100644 --- a/native/common/src/error.rs +++ b/native/common/src/error.rs @@ -1067,6 +1067,18 @@ pub fn unwrap_arrow_external_error(error: ArrowError) -> DataFusionError { } } +/// Whether `e`, from growing or resizing a memory reservation, is the pool refusing the memory. +/// +/// Every pool reports a refusal as `ResourcesExhausted`, which [`DataFusionError::find_root`] +/// finds under `Context`, `Shared` and `External`. An operator that can give memory back, by +/// spilling or by closing files early, answers a refusal that way. A pool backed by Spark can also +/// fail with an exception Spark threw, such as a failed spill of another consumer or the +/// `InterruptedException` of a task killed while it waited for memory. That comes back as another +/// kind of error and has to fail the task: giving memory back and carrying on would hide it. +pub fn is_refusal(e: &DataFusionError) -> bool { + matches!(e.find_root(), DataFusionError::ResourcesExhausted(_)) +} + #[cfg(test)] mod tests { use super::*; diff --git a/native/common/src/lib.rs b/native/common/src/lib.rs index 57ac6b90ed4..f1bd54c1f1f 100644 --- a/native/common/src/lib.rs +++ b/native/common/src/lib.rs @@ -30,7 +30,7 @@ mod utils; pub use calendar::{civil_from_days, days_from_civil}; pub use error::{ - decimal_overflow_error, error_chain, unwrap_arrow_external_error, SparkError, + decimal_overflow_error, error_chain, is_refusal, unwrap_arrow_external_error, SparkError, SparkErrorWithContext, SparkResult, }; pub use ffi_offsets::zero_offsets; diff --git a/native/core/src/execution/delta_dv.rs b/native/core/src/execution/delta_dv.rs index a99409d2ed1..4b1902125a9 100644 --- a/native/core/src/execution/delta_dv.rs +++ b/native/core/src/execution/delta_dv.rs @@ -32,6 +32,7 @@ use std::mem::size_of; use std::sync::Arc; +use datafusion::common::DataFusionError; use datafusion::datasource::listing::PartitionedFile; use datafusion::datasource::physical_plan::parquet::metadata::DFParquetMetadata; use datafusion::datasource::physical_plan::parquet::{ParquetAccessPlan, RowGroupAccess}; @@ -45,8 +46,10 @@ use parquet::arrow::arrow_reader::{RowSelection, RowSelector}; use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData}; use roaring::{RoaringBitmap, RoaringTreemap}; +use crate::errors::CometError; use crate::execution::operators::ExecutionError; use crate::execution::operators::ExecutionError::GeneralError; +use datafusion_comet_common::is_refusal; use datafusion_comet_proto::spark_operator::DeltaSparkDvDescriptor; const NATIVE_MAGIC: i32 = 1681511376; @@ -710,10 +713,12 @@ async fn attach_access_plan( admission_bound_bytes(dv.cardinality, row_counts.len(), page_bound_selectors)?; let reservation = partition_reservation.new_empty(); reservation.try_grow(admission_bytes).map_err(|e| { - GeneralError(format!( - "Deletion vector access plan for {file_path} needs up to {admission_bytes} \ - bytes to construct, exceeding the execution memory pool: {e}" - )) + reservation_error(e, |e| { + format!( + "Deletion vector access plan for {file_path} needs up to {admission_bytes} \ + bytes to construct, exceeding the execution memory pool: {e}" + ) + }) })?; let plan = build_access_plan(&metadata, &deleted, file.range.as_ref()) @@ -763,11 +768,13 @@ async fn attach_access_plan( .min(total_rows); let retained_bytes_bound = reader_peak_bytes(reader_selector_bound, row_counts.len())?; reservation.try_resize(retained_bytes_bound).map_err(|e| { - GeneralError(format!( - "Deletion vector access plan for {file_path} retains {selector_count} row \ - selectors, needing up to {retained_bytes_bound} bytes at the reader's \ - normalization peak, exceeding the execution memory pool: {e}" - )) + reservation_error(e, |e| { + format!( + "Deletion vector access plan for {file_path} retains {selector_count} row \ + selectors, needing up to {retained_bytes_bound} bytes at the reader's \ + normalization peak, exceeding the execution memory pool: {e}" + ) + }) })?; // Keyed by concrete type: the parquet opener looks up @@ -784,13 +791,41 @@ async fn attach_access_plan( .with_extension(DvAccessPlanReservation(reservation))) } +/// The error for a reservation that the memory pool failed with `e`: `refused(e)` if the pool +/// refused the memory, and otherwise `e` itself (see [`is_refusal`]). An exception Spark threw +/// stays that exception, which the JNI boundary rethrows as itself. +fn reservation_error( + e: DataFusionError, + refused: impl FnOnce(&DataFusionError) -> String, +) -> ExecutionError { + if is_refusal(&e) { + return GeneralError(refused(&e)); + } + if let DataFusionError::External(inner) = e.find_root() { + if let Some(CometError::JavaException { + class, + msg, + throwable, + }) = inner.downcast_ref::() + { + return ExecutionError::JavaException { + class: class.clone(), + msg: msg.clone(), + throwable: Arc::clone(throwable), + }; + } + } + e.into() +} + #[cfg(test)] mod tests { use super::*; use datafusion::arrow::datatypes::Schema; use datafusion::arrow::record_batch::RecordBatch; - use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool}; + use datafusion::execution::memory_pool::{GreedyMemoryPool, MemoryPool, UnboundedMemoryPool}; use datafusion::execution::runtime_env::RuntimeEnvBuilder; + use jni::objects::{Global, JThrowable}; use parquet::arrow::ArrowWriter; use parquet::file::metadata::ParquetMetaDataReader; use parquet::file::properties::WriterProperties; @@ -1685,6 +1720,80 @@ mod tests { ); } + /// A pool with room to spare that fails every `try_grow` with an exception Spark threw, as a + /// pool backed by Spark does once a spill of another consumer has failed. + #[derive(Debug)] + struct ThrowingPool { + inner: UnboundedMemoryPool, + throwable: Arc>>, + } + + impl std::fmt::Display for ThrowingPool { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "ThrowingPool({})", self.inner) + } + } + + impl MemoryPool for ThrowingPool { + fn name(&self) -> &str { + "ThrowingPool" + } + + fn grow(&self, reservation: &MemoryReservation, additional: usize) { + self.inner.grow(reservation, additional) + } + + fn shrink(&self, reservation: &MemoryReservation, shrink: usize) { + self.inner.shrink(reservation, shrink) + } + + fn try_grow(&self, _: &MemoryReservation, _: usize) -> datafusion::common::Result<()> { + Err(CometError::JavaException { + class: "org.apache.spark.memory.SparkOutOfMemoryError".to_string(), + msg: "[SPILL_OUT_OF_MEMORY] Error while calling spill()".to_string(), + throwable: Arc::clone(&self.throwable), + } + .into()) + } + + fn reserved(&self) -> usize { + self.inner.reserved() + } + } + + /// An exception Spark threw while the access plan reserves its memory fails the scan with + /// that exception, not with a message that the plan exceeds the pool. + #[tokio::test] + async fn attach_access_plans_fails_with_the_exception_spark_threw() { + let tmp = tempfile::tempdir().unwrap(); + // A null reference stands in for the throwable, since no JVM runs here. + let throwable: Arc>> = Arc::new(Global::null()); + let pool: Arc = Arc::new(ThrowingPool { + inner: UnboundedMemoryPool::default(), + throwable: Arc::clone(&throwable), + }); + let runtime_env = Arc::new( + RuntimeEnvBuilder::new() + .with_memory_pool(Arc::clone(&pool)) + .build() + .unwrap(), + ); + let scan_file = alternating_dv_scan_file(&runtime_env, tmp.path(), 1024); + + let err = attach_access_plans(Arc::clone(&runtime_env), vec![scan_file]) + .await + .unwrap_err(); + assert!( + matches!( + &err, + ExecutionError::JavaException { throwable: thrown, .. } + if Arc::ptr_eq(thrown, &throwable) + ), + "{err:?}" + ); + assert_eq!(pool.reserved(), 0); + } + /// A pool with room for the plan succeeds, reserves exactly the reader-lifecycle peak /// bound (`reader_peak_bytes`, never a hardcoded constant) once construction's transient /// peak has passed, attaches the reservation alongside the access plan, and releases it diff --git a/native/core/src/execution/memory_pools/fair_pool.rs b/native/core/src/execution/memory_pools/fair_pool.rs index 93174ba10e9..fe09d756e34 100644 --- a/native/core/src/execution/memory_pools/fair_pool.rs +++ b/native/core/src/execution/memory_pools/fair_pool.rs @@ -242,6 +242,9 @@ impl MemoryPool for CometFairMemoryPool { ) -> Result<(), DataFusionError> { if additional > 0 { let mut state = self.state.lock(); + // Before the pool's own limits, whose refusal would make the operator spill and carry + // on, and under the lock, which a `grow` holds while Spark fails it. + self.spark.check_failure()?; let can_spill = reservation.consumer().can_spill(); let used = state.used; if limited_to_its_share(reservation.consumer()) { diff --git a/native/core/src/execution/memory_pools/mod.rs b/native/core/src/execution/memory_pools/mod.rs index 7e93c02f788..07a95586fbc 100644 --- a/native/core/src/execution/memory_pools/mod.rs +++ b/native/core/src/execution/memory_pools/mod.rs @@ -160,6 +160,7 @@ pub(crate) fn create_memory_pool_with_fake_spark( mod tests { use super::testing::spillable; use super::*; + use datafusion::common::DataFusionError; use datafusion::execution::memory_pool::MemoryConsumer; #[test] @@ -186,6 +187,50 @@ mod tests { } } + /// Spark refusing memory is a refusal, which makes an operator spill. Spark throwing in a + /// `grow` is not: every later `try_grow` of the task fails with the exception Spark threw, at + /// the top of the error, through the wrappers of each pool type. That holds also where the + /// pool's own limits would have refused the request, and for a final aggregate's replay, + /// which records a refused request with `grow` but must not record a failed one. + #[test] + fn a_failed_grow_fails_every_later_try_grow_of_the_task() { + for (name, task_attempt_id, pool_type) in [ + ("greedy_unified", -3009, MemoryPoolType::GreedyUnified), + ("fair_unified", -3010, MemoryPoolType::FairUnified), + ] { + let fake = spark_memory::fake::FakeSpark::with(1000); + let config = MemoryPoolConfig::new(pool_type, 1000); + let pool = create_pool(&config, task_attempt_id, &RunningTasks::default(), || { + fake.memory() + }); + let sort = spillable("ExternalSorter[0]", &pool); + let refusal = sort.try_grow(2000).unwrap_err(); + assert!( + matches!(refusal.find_root(), DataFusionError::ResourcesExhausted(_)), + "{name}: {refusal:?}" + ); + + // A final aggregate reading its spill files back holds the task's whole share, which + // Spark failed to grant. + let merge = spillable("FinalHashAggregateStream[0]", &pool); + let replay = merge.new_empty(); + fake.throw_next_acquire(); + merge.grow(1000); + assert_eq!(pool.reserved(), 1000, "{name}"); + // The sort asks again, as DataFusion's does after spilling on any error. + for reservation in [&sort, &replay, &sort] { + let error = reservation.try_grow(1).unwrap_err(); + assert!(fake.is_thrown(0, &error), "{name}: {error:?}"); + } + assert_eq!(pool.reserved(), 1000, "{name}"); + + drop(merge); + drop(replay); + assert_eq!(pool.reserved(), 0, "{name}"); + assert_eq!(fake.held(), 0, "{name}"); + } + } + /// The tasks of an executor share one count, which `create_memory_pool` takes from /// [`RunningTasks::of_executor`]. A pool created with a count of its own would see only its /// own task and give it the whole pool. No other test counts tasks in the executor's count. diff --git a/native/core/src/execution/memory_pools/spark_memory.rs b/native/core/src/execution/memory_pools/spark_memory.rs index 39a8030a572..d849c258cc3 100644 --- a/native/core/src/execution/memory_pools/spark_memory.rs +++ b/native/core/src/execution/memory_pools/spark_memory.rs @@ -17,13 +17,17 @@ use std::sync::{ atomic::{AtomicUsize, Ordering::Relaxed}, - Arc, + Arc, OnceLock, }; -use jni::objects::{Global, JObject}; +use datafusion::common::DataFusionError; +use jni::objects::{Global, JObject, JThrowable}; use log::warn; -use crate::{errors::CometResult, jvm_bridge::JVMClasses}; +use crate::{ + errors::{CometError, CometResult}, + jvm_bridge::JVMClasses, +}; /// Spark's side of a Comet pool: the calls that acquire and release off-heap execution memory. pub(super) trait SparkMemoryManager: Send + Sync { @@ -61,6 +65,15 @@ impl SparkMemoryManager for JniMemoryManager { /// While any is outstanding, [`Self::try_acquire`] also asks Spark for it, so a pool refuses /// `try_grow` until Spark can cover both the request and the debt, and the operator spills. /// +/// A call to Spark that fails in [`Self::acquire`] is carried the same way, as a zero grant. Spark +/// throws, for instance, when a spill of another consumer fails, or when the task is killed while +/// it waits for memory, and either way the task has to fail. So once a request for memory has +/// failed in Spark, in `acquire` or [`Self::try_acquire`], [`Self::check_failure`] fails, and with +/// it every later `try_acquire`, each time with the exception Spark threw; see [`Failure`]. Every +/// later `acquire` carries its whole size as overcommit, so neither asks Spark again. An +/// operator that takes any error for a refusal and asks again fails again with that exception, +/// which reaches the JVM as itself if the operator returns the error as it is. +/// /// # Invariant /// /// Every byte a pool records through this type is backed either by Spark's grant or by @@ -77,9 +90,60 @@ impl SparkMemoryManager for JniMemoryManager { pub(super) struct SparkMemory { manager: Box, overcommit: AtomicUsize, + /// The first request to Spark that failed. + failure: OnceLock, task_attempt_id: i64, } +/// A request to Spark that failed, which fails every later request of the task. +enum Failure { + /// Spark threw. Every later request fails with the same exception, which the JNI boundary + /// rethrows as it is, so the task fails with what Spark threw however many requests it + /// refuses first. The pool and each of those errors hold the throwable until they drop. + Thrown { + class: String, + msg: String, + throwable: Arc>>, + }, + /// The call failed without an exception from Spark, for instance in JNI. Every later request + /// fails with this message, which names the first request. + Other(String), +} + +impl Failure { + fn new(task_attempt_id: i64, size: usize, error: &CometError) -> Self { + match error { + CometError::JavaException { + class, + msg, + throwable, + } => Failure::Thrown { + class: class.clone(), + msg: msg.clone(), + throwable: Arc::clone(throwable), + }, + _ => Failure::Other(format!( + "Task {task_attempt_id} failed to acquire {size} bytes from Spark: {error}" + )), + } + } + + fn error(&self) -> CometError { + match self { + Failure::Thrown { + class, + msg, + throwable, + } => CometError::JavaException { + class: class.clone(), + msg: msg.clone(), + throwable: Arc::clone(throwable), + }, + Failure::Other(message) => DataFusionError::Execution(message.clone()).into(), + } + } +} + /// Why [`SparkMemory::try_acquire`] refused a request. #[derive(Debug, PartialEq, Eq)] pub(super) struct Refusal { @@ -98,6 +162,7 @@ impl SparkMemory { Self { manager, overcommit: AtomicUsize::new(0), + failure: OnceLock::new(), task_attempt_id, } } @@ -107,11 +172,16 @@ impl SparkMemory { } /// Acquires `size` bytes plus any outstanding overcommit, or nothing. A full grant repays the - /// overcommit; a partial one is handed back and reported as a [`Refusal`]. + /// overcommit; a partial one is handed back and reported as a [`Refusal`]. Fails without + /// asking Spark once a request to Spark has failed; see [`Self::check_failure`]. pub(super) fn try_acquire(&self, size: usize) -> CometResult> { + self.check_failure()?; let debt = self.overcommit.load(Relaxed); let request = size.saturating_add(debt); - let granted = granted(request, self.ask_spark(request)?); + let acquired = self + .ask_spark(request) + .inspect_err(|e| self.record_failure(request, e))?; + let granted = granted(request, acquired); if granted < request { if granted > 0 { self.manager.release(granted)?; @@ -133,16 +203,23 @@ impl SparkMemory { } /// Acquires what Spark will grant toward `size` bytes and carries the rest as overcommit. - /// Never fails; a failed call to Spark counts as a zero grant. + /// Never fails; a failed call to Spark counts as a zero grant, and fails every later + /// [`Self::try_acquire`]. Once a request to Spark has failed, it carries all of `size` without + /// asking Spark, which could otherwise spill again the consumer whose spill just failed. pub(super) fn acquire(&self, size: usize) { - let granted = match self.ask_spark(size) { - Ok(acquired) => granted(size, acquired), - Err(e) => { - warn!( - "Task {} failed to acquire {size} bytes from Spark: {e:?}", - self.task_attempt_id - ); - 0 + let granted = if self.failure.get().is_some() { + 0 + } else { + match self.ask_spark(size) { + Ok(acquired) => granted(size, acquired), + Err(e) => { + warn!( + "Task {} failed to acquire {size} bytes from Spark: {e:?}", + self.task_attempt_id + ); + self.record_failure(size, &e); + 0 + } } }; if granted < size { @@ -163,6 +240,20 @@ impl SparkMemory { self.overcommit.load(Relaxed) } + /// Fails once a request to Spark has failed, every time with that failure; see [`Failure`]. + pub(super) fn check_failure(&self) -> CometResult<()> { + match self.failure.get() { + None => Ok(()), + Some(failure) => Err(failure.error()), + } + } + + /// Keeps the first request to Spark that failed. + fn record_failure(&self, size: usize, error: &CometError) { + self.failure + .get_or_init(|| Failure::new(self.task_attempt_id, size, error)); + } + /// Asks Spark for `size` bytes and returns how many it granted. /// /// Spark can block the call until other tasks release memory, so it runs in `block_in_place`. @@ -191,7 +282,6 @@ fn granted(requested: usize, acquired: i64) -> usize { #[cfg(test)] pub(super) mod fake { use super::*; - use crate::errors::CometError; use parking_lot::Mutex; /// Grants up to `limit` bytes in total and records every release. Panics if it is handed @@ -202,6 +292,9 @@ pub(super) mod fake { held: Mutex, released: Mutex>, fail: bool, + throw_next: Mutex, + /// The exceptions acquires have thrown, in order. + thrown: Mutex>>>>, during_acquire: Mutex>>, } @@ -224,6 +317,28 @@ pub(super) mod fake { *self.limit.lock() = limit; } + /// Makes the next acquire throw, as Spark does when a spill of another consumer fails. + pub(in crate::execution::memory_pools) fn throw_next_acquire(&self) { + *self.throw_next.lock() = true; + } + + /// Whether `error`, as a pool returns it, is the `n`th exception this fake threw, as a + /// Java exception at the top, which the JNI boundary rethrows as itself. + pub(in crate::execution::memory_pools) fn is_thrown( + &self, + n: usize, + error: &DataFusionError, + ) -> bool { + let thrown = Arc::clone(&self.thrown.lock()[n]); + let DataFusionError::External(e) = error else { + return false; + }; + matches!( + e.downcast_ref::(), + Some(CometError::JavaException { throwable, .. }) if Arc::ptr_eq(throwable, &thrown) + ) + } + /// Runs `f` inside the next acquire, before Spark answers. That is where a call from /// another thread lands while this one waits on JNI. pub(in crate::execution::memory_pools) fn during_next_acquire( @@ -257,6 +372,17 @@ pub(super) mod fake { if self.fail { return Err(CometError::Internal("jni".to_string())); } + if std::mem::take(&mut *self.throw_next.lock()) { + // A null reference stands in for the throwable, since no JVM runs here. Each + // throw has its own, so a test can tell which one an error carries. + let throwable = Arc::new(Global::null()); + self.thrown.lock().push(Arc::clone(&throwable)); + return Err(CometError::JavaException { + class: "org.apache.spark.memory.SparkOutOfMemoryError".to_string(), + msg: "[SPILL_OUT_OF_MEMORY] Error while calling spill()".to_string(), + throwable, + }); + } let limit = *self.limit.lock(); let mut held = self.held.lock(); let granted = size.min(limit.saturating_sub(*held)); @@ -366,6 +492,56 @@ mod tests { assert!(fake.released().is_empty()); } + /// Once a call to Spark has thrown in `acquire`, nothing asks Spark again, even once Spark + /// could grant the request and the debt: a later `acquire` carries its whole size as + /// overcommit, and every later `try_acquire` fails with that exception. + #[test] + fn a_failed_acquire_fails_every_later_try_acquire() { + let fake = FakeSpark::with(200); + let spark = fake.memory(); + fake.throw_next_acquire(); + spark.acquire(64); + spark.acquire(32); + assert_eq!(spark.overcommit(), 96); + for _ in 0..3 { + let error = DataFusionError::from(spark.try_acquire(10).unwrap_err()); + assert!(fake.is_thrown(0, &error), "{error:?}"); + } + assert_eq!(fake.held(), 0); + } + + /// A `try_acquire` whose call to Spark fails returns that error, and every later one fails + /// too, without asking Spark: with the exception Spark threw, or with a message naming the + /// first request if the call failed without one. + #[test] + fn a_failed_try_acquire_fails_every_later_one() { + let fake = FakeSpark::with(100); + let spark = fake.memory(); + fake.throw_next_acquire(); + for _ in 0..3 { + let error = DataFusionError::from(spark.try_acquire(10).unwrap_err()); + assert!(fake.is_thrown(0, &error), "{error:?}"); + } + assert_eq!(fake.held(), 0); + + let fake = FakeSpark::failing(); + let spark = fake.memory(); + let first = spark.try_acquire(10).unwrap_err(); + assert!( + matches!(&first, CometError::Internal(m) if m == "jni"), + "{first:?}" + ); + let error = DataFusionError::from(spark.try_acquire(20).unwrap_err()); + assert!( + !matches!(error.find_root(), DataFusionError::ResourcesExhausted(_)), + "{error:?}" + ); + assert!( + error.to_string().contains("failed to acquire 10 bytes"), + "{error}" + ); + } + #[test] fn granted_is_clamped_to_the_request() { assert_eq!(granted(100, 40), 40); diff --git a/native/core/src/execution/memory_pools/spill_replay.rs b/native/core/src/execution/memory_pools/spill_replay.rs index c4805ce1a98..56bfbe3f3aa 100644 --- a/native/core/src/execution/memory_pools/spill_replay.rs +++ b/native/core/src/execution/memory_pools/spill_replay.rs @@ -321,31 +321,6 @@ mod tests { } } - #[test] - fn a_failed_spark_call_during_the_replay_is_not_recorded() { - for (name, pool_type, task_attempt_id) in [ - ("greedy_unified", MemoryPoolType::GreedyUnified, -3017), - ("fair_unified", MemoryPoolType::FairUnified, -3018), - ] { - let fake = FakeSpark::failing(); - let config = MemoryPoolConfig::new(pool_type, 1000); - let pool = create_pool(&config, task_attempt_id, &RunningTasks::default(), || { - fake.memory() - }); - let merge = spillable("FinalHashAggregateStream[0]", &pool); - let replay = merge.new_empty(); - merge.grow(90); - - // Only a refusal is recorded. A JNI error goes back to the aggregate. - let err = replay.try_grow(30).unwrap_err(); - assert!( - !matches!(err, DataFusionError::ResourcesExhausted(_)), - "{name}: {err}" - ); - assert_eq!(pool.reserved(), 90, "{name}"); - } - } - #[test] fn a_final_aggregate_is_tracked_across_its_reservations_until_it_unregisters() { let pool = Arc::new(SpillReplayPool::new( diff --git a/native/core/src/execution/operators/iceberg_write.rs b/native/core/src/execution/operators/iceberg_write.rs index c949b95cf66..41d0bef4eee 100644 --- a/native/core/src/execution/operators/iceberg_write.rs +++ b/native/core/src/execution/operators/iceberg_write.rs @@ -80,6 +80,7 @@ use parquet::basic::{BrotliLevel, Compression, GzipLevel, ZstdLevel}; use parquet::file::properties::{EnabledStatistics, WriterProperties}; use serde::{Deserialize, Serialize}; +use datafusion_comet_common::is_refusal; use datafusion_comet_proto::spark_operator::{ CompressionCodec as ProtoCompressionCodec, IcebergParquetWriteSettings, IcebergWrite, IcebergWriteCommon, IcebergWriterMode as ProtoIcebergWriterMode, @@ -1359,7 +1360,8 @@ impl InnerWriter { /// partitions, the one holding the most first, until what is left fits, and this returns how /// many it closed. The unpartitioned and clustered writers hold one file open and cannot close /// it early, so the pool's refusal stands. Any other error from a resize fails the task (see - /// [`is_refusal`]). + /// [`is_refusal`]): closing partitions would swallow it, because once the resize is down to a + /// shrink it no longer calls Spark. async fn reserve( &mut self, reservation: &MemoryReservation, @@ -1484,16 +1486,6 @@ fn write_consumer(partition: usize, writer_mode: ProtoIcebergWriterMode) -> Memo .with_can_spill(writer_mode == ProtoIcebergWriterMode::IcebergWriterFanout) } -/// Whether `e`, from resizing a reservation, is the pool refusing the memory, which every pool -/// reports as `ResourcesExhausted` and a fanout write answers by closing partitions early. A pool -/// backed by Spark can also fail a resize with an exception Spark threw: a spill of another -/// consumer that failed, or the `InterruptedException` of a task killed while it waited for -/// memory. That comes back as another kind of error and has to fail the task. Closing partitions -/// would swallow it, because once the resize is down to a shrink it no longer calls Spark. -fn is_refusal(e: &DataFusionError) -> bool { - matches!(e.find_root(), DataFusionError::ResourcesExhausted(_)) -} - fn parse_iceberg_schema(json: &str) -> DFResult { let schema: IcebergSchema = serde_json::from_str(json).map_err(|e| { DataFusionError::Internal(format!("Failed to parse iceberg schema JSON: {e}")) diff --git a/native/jni-bridge/src/errors.rs b/native/jni-bridge/src/errors.rs index 04c20611f3c..82b6a3293bf 100644 --- a/native/jni-bridge/src/errors.rs +++ b/native/jni-bridge/src/errors.rs @@ -23,7 +23,7 @@ use datafusion_comet_common::{error_chain, SparkError, SparkErrorWithContext}; use jni::errors::{Exception, ToException}; use regex::Regex; -use std::{any::Any, convert, panic::UnwindSafe, result, str, str::Utf8Error}; +use std::{any::Any, convert, panic::UnwindSafe, result, str, str::Utf8Error, sync::Arc}; // This is just a pointer. We'll be returning it from our function. We // can't return one of the objects with lifetime information because the @@ -59,7 +59,7 @@ pub enum ExecutionError { JavaException { class: String, msg: String, - throwable: Global>, + throwable: Arc>>, }, } @@ -175,11 +175,15 @@ pub enum CometError { #[error(transparent)] Spark(SparkError), + /// A Java exception raised during an upcall, which the JNI boundary rethrows as it is. The + /// throwable is behind an `Arc` because `Global` is not `Clone`: once Spark has thrown during + /// a request for memory, the memory pools fail every later request with the same exception, + /// without a JNI call each time. #[error("{class}: {msg}")] JavaException { class: String, msg: String, - throwable: Global>, + throwable: Arc>>, }, } @@ -1781,7 +1785,8 @@ mod tests { ); } - /// A throwable captured during an upcall is rethrown as the same object. + /// A throwable captured during an upcall is rethrown as the same object, by every error that + /// shares it. #[test] #[cfg_attr(miri, ignore)] // miri cannot create a JVM. fn java_exception_is_rethrown() { @@ -1794,23 +1799,25 @@ mod tests { &[jni::objects::JValue::Object(&message)], )?; let throwable = unsafe { JThrowable::from_raw(env, throwable.into_raw()) }; - let error = CometError::JavaException { - class: "java.lang.IllegalArgumentException".to_string(), - msg: "from jvm".to_string(), - throwable: env.new_global_ref(throwable)?, - }; - let native_error = NativeError::from_comet_error(&error); - assert_eq!(native_error.status, NativeStatus::JavaThrowable); - let payload: serde_json::Value = - serde_json::from_str(&native_error.to_payload()).unwrap(); - assert_eq!(payload["kind"], "rethrow"); - - unwrap_or_throw_default::<()>(env, Err(error)); - assert_pending_java_exception_detailed( - env, - Some("java/lang/IllegalArgumentException"), - Some("from jvm"), - ); + let throwable = Arc::new(env.new_global_ref(throwable)?); + for _ in 0..2 { + let error = CometError::JavaException { + class: "java.lang.IllegalArgumentException".to_string(), + msg: "from jvm".to_string(), + throwable: Arc::clone(&throwable), + }; + let native_error = NativeError::from_comet_error(&error); + assert_eq!(native_error.status, NativeStatus::JavaThrowable); + let payload: serde_json::Value = + serde_json::from_str(&native_error.to_payload()).unwrap(); + assert_eq!(payload["kind"], "rethrow"); + + unwrap_or_throw_default::<()>(env, Err(error)); + assert!(env.exception_check()); + let thrown = env.exception_occurred().expect("Unable to get exception"); + env.exception_clear(); + assert!(env.is_same_object(&thrown, &*throwable)?); + } Ok(()) }) .unwrap(); diff --git a/native/jni-bridge/src/lib.rs b/native/jni-bridge/src/lib.rs index e8601753705..fc9b0ca50b5 100644 --- a/native/jni-bridge/src/lib.rs +++ b/native/jni-bridge/src/lib.rs @@ -29,6 +29,7 @@ use jni::{ Env, JavaVM, DEFAULT_LOCAL_FRAME_CAPACITY, }; use once_cell::sync::OnceCell; +use std::sync::Arc; use errors::{CometError, CometResult}; @@ -482,7 +483,7 @@ pub fn convert_exception(env: &mut Env, throwable: &JThrowable) -> CometResult MultiPartitionShuffleRepartitioner { } /// Grows the spill reservation by `mem_growth`, spilling if that is refused or if the fixed - /// buffer limit has been reached. + /// buffer limit has been reached. Any other error from the pool fails the shuffle; see + /// [`is_refusal`]. fn reserve_and_may_spill(&mut self, mem_growth: usize) -> datafusion::common::Result<()> { // A rejected reservation does not include this batch's memory, even though the batch // and its partition indices have already been buffered and must be counted as spilled. - let reservation_failed = self.reservation.try_grow(mem_growth).is_err(); + let reservation_failed = match self.reservation.try_grow(mem_growth) { + Ok(()) => false, + Err(e) if is_refusal(&e) => true, + Err(e) => return Err(e), + }; // Checking after buffering lets the writer overshoot the limit by at most one batch, // which is how the memory-pressure trigger already behaves. if reservation_failed @@ -1150,6 +1156,129 @@ mod tests { } } + /// Only the pool refusing the spill reservation makes the repartitioner spill. + mod pool_errors { + use super::*; + use datafusion::execution::memory_pool::{MemoryPool, UnboundedMemoryPool}; + use datafusion::execution::runtime_env::RuntimeEnvBuilder; + + /// A pool with room to spare whose `try_grow` fails with `error()`, the way a pool backed + /// by Spark fails when Spark refuses the memory or throws. + #[derive(Debug)] + struct ErrorPool { + inner: UnboundedMemoryPool, + error: fn() -> DataFusionError, + } + + impl fmt::Display for ErrorPool { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + write!(f, "ErrorPool({})", self.inner) + } + } + + impl MemoryPool for ErrorPool { + fn name(&self) -> &str { + "ErrorPool" + } + + fn grow(&self, reservation: &MemoryReservation, additional: usize) { + self.inner.grow(reservation, additional) + } + + fn shrink(&self, reservation: &MemoryReservation, shrink: usize) { + self.inner.shrink(reservation, shrink) + } + + fn try_grow(&self, _: &MemoryReservation, _: usize) -> datafusion::common::Result<()> { + Err((self.error)()) + } + + fn reserved(&self) -> usize { + self.inner.reserved() + } + } + + /// Inserts one batch into a two-partition repartitioner whose pool fails every + /// `try_grow` with `error()`. + async fn insert_with( + error: fn() -> DataFusionError, + ) -> ( + datafusion::common::Result<()>, + MultiPartitionShuffleRepartitioner, + ) { + let pool = ErrorPool { + inner: UnboundedMemoryPool::default(), + error, + }; + let runtime = Arc::new( + RuntimeEnvBuilder::new() + .with_memory_pool(Arc::new(pool)) + .build() + .unwrap(), + ); + let mut repartitioner = MultiPartitionShuffleRepartitioner::try_new( + 0, + FailingPartitionWriter::default(), + CometPartitioning::RoundRobin(2, RoundRobinStrategy::default()), + ShufflePartitionerMetrics::new(&ExecutionPlanMetricsSet::new(), 0), + runtime, + 64, + false, + None, + ) + .unwrap(); + let result = repartitioner.insert_batch(int64_batch(0..64)).await; + (result, repartitioner) + } + + /// An error that is not a refusal, such as an exception Spark threw while spilling another + /// consumer, fails the shuffle with that error instead of making it spill and carry on. + #[tokio::test] + async fn pool_errors_other_than_refusal_fail_the_shuffle() { + let errors: [fn() -> DataFusionError; 3] = [ + // A pool backed by Spark passes on an exception Spark threw like this. + || DataFusionError::External(Box::new(std::io::Error::other("a spill failed"))), + // And a JNI call that failed like this. + || DataFusionError::Execution("a JNI call failed".to_string()), + // The same exception, handed to a later request. + || { + let error = DataFusionError::Execution("a JNI call failed".to_string()); + DataFusionError::Shared(Arc::new(error.context("an earlier request"))) + }, + ]; + for error in errors { + let (result, repartitioner) = insert_with(error).await; + let failure = result.expect_err("the shuffle went on after the pool failed"); + assert_eq!(failure.to_string(), error().to_string()); + assert_eq!(repartitioner.spill_count(), 0); + assert_eq!(repartitioner.partition_writer.write_calls, 0); + } + } + + /// A refusal makes the repartitioner spill, also when it reaches it wrapped, so the check + /// looks at the root of the error. + #[tokio::test] + async fn a_refused_reservation_spills() { + let refusals: [fn() -> DataFusionError; 3] = [ + || DataFusionError::ResourcesExhausted("refused".to_string()), + || DataFusionError::ResourcesExhausted("refused".to_string()).context("wrapped"), + || { + let refusal = DataFusionError::ResourcesExhausted("refused".to_string()); + DataFusionError::Shared(Arc::new(refusal)) + }, + ]; + for refusal in refusals { + let (result, repartitioner) = insert_with(refusal).await; + result.unwrap(); + assert_eq!(repartitioner.spill_count(), 1); + // The spill wrote both partitions and released the batch it had not reserved. + assert_eq!(repartitioner.partition_writer.write_calls, 2); + assert!(repartitioner.metrics.memory_spilled_bytes.value() > 0); + assert_eq!(repartitioner.reservation.size(), 0); + } + } + } + /// Collects every batch handed to it, per partition, so a whole write can be compared. #[derive(Default)] struct CollectingPartitionWriter { diff --git a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala index 1caaec887fa..bf32e66ceb4 100644 --- a/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala +++ b/spark/src/test/scala/org/apache/spark/CometExecIteratorLifecycleSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark -import java.io.ByteArrayInputStream +import java.io.{ByteArrayInputStream, IOException} import java.lang.ref.WeakReference import java.util.Properties import java.util.concurrent.TimeUnit @@ -33,13 +33,14 @@ import org.apache.arrow.vector.BigIntVector import org.apache.arrow.vector.ipc.ArrowReader import org.apache.arrow.vector.types.pojo.{ArrowType, Field, Schema} import org.apache.spark.executor.TaskMetrics -import org.apache.spark.memory.{TaskMemoryManager, TestMemoryManager} +import org.apache.spark.memory.{MemoryConsumer, MemoryMode, TaskMemoryManager, TestMemoryManager} import org.apache.spark.rdd.RDD import org.apache.spark.sql.CometTestBase import org.apache.spark.sql.catalyst.expressions.PrettyAttribute import org.apache.spark.sql.comet.{CometExec, CometExecUtils, CometMetricNode, CometNativeScanExec, CometProjectExec, CometSortExec} import org.apache.spark.sql.comet.execution.arrow.CometArrowStream import org.apache.spark.sql.functions.{col, udf} +import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.types.{LongType, StructField, StructType} import org.apache.spark.sql.vectorized.{ColumnarBatch, ColumnVector} import org.apache.spark.util.JsonProtocol @@ -455,6 +456,60 @@ class CometExecIteratorLifecycleSuite extends CometTestBase { } } + test("a native sort fails with the exception Spark threw while spilling another consumer") { + // When Spark cannot grant what the sort asks for, it spills the task's other consumers, and + // throws if a spill fails. DataFusion's sort takes any error from its pool for a refusal, so + // it spills and asks again, and the pool has to fail that request with the same exception + // even though Spark could now grant it. + withTempPath { path => + spark + .range(0, 1000000, 1, 1) + .selectExpr("id", "CAST(id AS STRING) AS s") + .write + .parquet(path.getAbsolutePath) + withParquetTable(path.getAbsolutePath, "tbl") { + withSQLConf( + // One task reads the whole file, however large. + SQLConf.FILES_MIN_PARTITION_NUM.key -> "1", + CometConf.COMET_RESPECT_DATAFUSION_CONFIGS.key -> "true", + "spark.comet.datafusion.execution.sort_spill_reservation_bytes" -> "65536") { + val sorted = sql("SELECT * FROM tbl SORT BY id DESC") + val plan = sorted.queryExecution.executedPlan + assert(plan.find(_.isInstanceOf[CometSortExec]).isDefined, s"Expected a sort:\n$plan") + val rdd = sorted.queryExecution.toRdd + assert(rdd.getNumPartitions == 1) + + val results = rdd + .mapPartitions { rows => + val context = TaskContext.get() + val other = new FailsToSpillOnce(context.taskMemoryManager()) + context.addTaskCompletionListener[Unit](_ => other.freeMemory(other.getUsed)) + // Takes all the off-heap memory the task can get, then leaves the sort room for its + // merge reservation and its first batches. + assert(other.acquireMemory(1L << 40) > (8L << 20)) + other.freeMemory(8L << 20) + val outcome = + try { + rows.foreach(_ => ()) + "finished" + } catch { + case e: Throwable => s"${e.getClass.getName}: ${e.getMessage}" + } + Iterator.single((outcome, other.spills)) + } + .collect() + + assert(results.length == 1) + val (outcome, spills) = results.head + assert(outcome.startsWith("org.apache.spark.memory.SparkOutOfMemoryError: "), outcome) + assert(outcome.contains("injected spill failure"), outcome) + // The first spill failed, and the task stopped there instead of making Spark spill again. + assert(spills == 1, outcome) + } + } + } + } + test("the last native plan in a task to close warns about the memory the task still holds") { withTaskContext(4600000L) { val first = planWithoutInput(4600001L) @@ -955,6 +1010,26 @@ class CometExecIteratorLifecycleSuite extends CometTestBase { } } +/** + * An off-heap consumer whose first spill fails, as a Spark consumer's does when it cannot write + * its spill file, and whose later spills free everything it holds. + */ +private class FailsToSpillOnce(taskMemoryManager: TaskMemoryManager) + extends MemoryConsumer(taskMemoryManager, 0L, MemoryMode.OFF_HEAP) { + + var spills = 0 + + override def spill(size: Long, trigger: MemoryConsumer): Long = { + spills += 1 + if (spills == 1) { + throw new IOException("injected spill failure") + } + val freed = getUsed + freeMemory(freed) + freed + } +} + /** * Adds `onTaskEnd` as a task completion listener before it computes `prev`. Spark runs completion * listeners in the reverse order they were added, so `onTaskEnd` runs after every listener that