From 3cbec8092d79357c262beb0a5676c0ee2310bf4f Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Tue, 22 Sep 2026 16:59:01 +0200 Subject: [PATCH 1/7] test(physical-plan): sort reservation charges a shared dictionary once per output chunk sort_batch_stream reserves the sum of get_record_batch_memory_size over the chunks from sort_batch_chunked. take() shares the dictionary values across every chunk, so the dictionary is counted once per chunk. A 2.5 MB batch with a 50k-value dictionary asks for 121.8 MB. With the disk manager disabled there is no spill fallback and the sort fails. --- datafusion/physical-plan/src/sorts/sort.rs | 61 ++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index 30a0e301823f7..accd4f30e6ad1 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -1459,6 +1459,7 @@ mod tests { use datafusion_common::{DataFusionError, Result, ScalarValue}; use datafusion_execution::RecordBatchStream; use datafusion_execution::config::SessionConfig; + use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; use datafusion_execution::runtime_env::RuntimeEnvBuilder; use datafusion_physical_expr::EquivalenceProperties; use datafusion_physical_expr::expressions::{Column, Literal}; @@ -1743,6 +1744,66 @@ mod tests { Ok(()) } + /// A single large batch with a dictionary column: `sort_batch_chunked` + /// shares the dictionary values across every output chunk, but the + /// reservation for the sorted output charges it once per chunk. + #[tokio::test] + async fn test_sort_single_batch_shared_dictionary_reservation() -> Result<()> { + const ROWS: usize = 100_000; + const CARDINALITY: usize = 50_000; + const BATCH_SIZE: usize = 1_000; + + let schema = Arc::new(Schema::new(vec![ + Field::new( + "tag", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + false, + ), + Field::new("time", DataType::Int64, false), + ])); + let mut tag = StringDictionaryBuilder::::new(); + for i in 0..ROWS { + tag.append_value(format!("truck_{:05}", (i * 7919) % CARDINALITY)); + } + let time = Int64Array::from_iter_values((0..ROWS).rev().map(|i| i as i64)); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(tag.finish()), Arc::new(time)], + )?; + let input_bytes = get_record_batch_memory_size(&batch); + + // Room for the input several times over, but not for the dictionary + // once per chunk. No disk manager, so the sort cannot spill. + let runtime = RuntimeEnvBuilder::new() + .with_memory_limit(input_bytes * 4, 1.0) + .with_disk_manager_builder( + DiskManagerBuilder::default().with_mode(DiskManagerMode::Disabled), + ) + .build_arc()?; + let task_ctx = Arc::new( + TaskContext::default() + .with_session_config(SessionConfig::new().with_batch_size(BATCH_SIZE)) + .with_runtime(runtime), + ); + + let plan = + TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?; + let sort_exec = Arc::new(SortExec::new( + [ + PhysicalSortExpr::new_default(col("tag", &schema)?), + PhysicalSortExpr::new_default(col("time", &schema)?), + ] + .into(), + plan, + )); + + let result = collect(sort_exec, task_ctx).await?; + let rows: usize = result.iter().map(|b| b.num_rows()).sum(); + assert_eq!(rows, ROWS); + assert_eq!(result.len(), ROWS.div_ceil(BATCH_SIZE)); + Ok(()) + } + #[tokio::test] async fn test_sort_spill_utf8_strings() -> Result<()> { let session_config = SessionConfig::new() From e18405a487fbc4d522787ad44fec44df7a0dfdf3 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Tue, 22 Sep 2026 17:44:35 +0200 Subject: [PATCH 2/7] fix(physical-plan): count buffers shared across sorted output chunks once sort_batch_stream reserved the sum of per-chunk sizes for the output of sort_batch_chunked. Dictionary values are shared by every chunk after take(), so a batch with a large dictionary reserved chunks x dictionary and failed with ResourcesExhausted or starved the pool. Size the sorted output with one pass over all chunks so a shared buffer is counted once. ReservationStream releases per batch, and per-batch sizes can now add up to more than was reserved, so it releases at most what remains; the rest is freed when the stream ends. --- datafusion/common/src/utils/memory.rs | 41 +++++++++++++++++++++- datafusion/physical-plan/src/sorts/sort.rs | 6 ++-- datafusion/physical-plan/src/stream.rs | 11 ++++-- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/datafusion/common/src/utils/memory.rs b/datafusion/common/src/utils/memory.rs index 78ec434d2b577..24808dac3aaf6 100644 --- a/datafusion/common/src/utils/memory.rs +++ b/datafusion/common/src/utils/memory.rs @@ -131,12 +131,22 @@ pub fn estimate_memory_size(num_elements: usize, fixed_size: usize) -> Result /// `Buffer`. This method provides temporary fix until the issue is resolved: /// pub fn get_record_batch_memory_size(batch: &RecordBatch) -> usize { + get_record_batches_memory_size(std::iter::once(batch)) +} + +/// Calculate total used memory of `batches`, counting a `Buffer` shared by +/// several batches (e.g. dictionary values across `take`n chunks) only once. +/// +/// See [`get_record_batch_memory_size`] for the per-batch semantics. +pub fn get_record_batches_memory_size<'a>( + batches: impl IntoIterator, +) -> usize { // Store pointers to `Buffer`'s start memory address (instead of actual // used data region's pointer represented by current `Array`) let mut counted_buffers: HashSet> = HashSet::new(); let mut total_size = 0; - for array in batch.columns() { + for array in batches.into_iter().flat_map(|batch| batch.columns()) { let array_data = array.to_data(); count_array_data_memory_size(&array_data, &mut counted_buffers, &mut total_size); } @@ -295,6 +305,35 @@ mod record_batch_tests { assert_eq!(size_origin, size_sliced); } + #[test] + fn test_get_record_batches_memory_size_shared_buffer() { + let original = Int32Array::from(vec![1, 2, 3, 4, 5]); + let slice1 = original.slice(0, 3); + let slice2 = original.slice(2, 3); + + let schema_origin = Arc::new(Schema::new(vec![Field::new( + "origin_col", + DataType::Int32, + false, + )])); + let batch_origin = + RecordBatch::try_new(Arc::clone(&schema_origin), vec![Arc::new(original)]) + .unwrap(); + let batch_slice1 = + RecordBatch::try_new(Arc::clone(&schema_origin), vec![Arc::new(slice1)]) + .unwrap(); + let batch_slice2 = + RecordBatch::try_new(schema_origin, vec![Arc::new(slice2)]).unwrap(); + + let size_origin = get_record_batch_memory_size(&batch_origin); + let size_slices = get_record_batches_memory_size([&batch_slice1, &batch_slice2]); + let size_summed = get_record_batch_memory_size(&batch_slice1) + + get_record_batch_memory_size(&batch_slice2); + + assert_eq!(size_origin, size_slices); + assert_eq!(size_summed, 2 * size_origin); + } + #[test] fn test_get_record_batch_memory_size_nested_array() { let schema = Arc::new(Schema::new(vec![ diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index accd4f30e6ad1..99a2c917ec076 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -50,6 +50,7 @@ use crate::{ ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream, Statistics, }; +use datafusion_common::utils::memory::get_record_batches_memory_size; use arrow::array::{Array, RecordBatch, RecordBatchOptions, StringViewArray}; use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays}; @@ -732,10 +733,7 @@ impl ExternalSorter { // Using try_resize avoids a release-then-reacquire cycle, which // matters for MemoryPool implementations where grow/shrink have // non-trivial cost (e.g. JNI calls in Comet). - let total_sorted_size: usize = sorted_batches - .iter() - .map(get_record_batch_memory_size) - .sum(); + let total_sorted_size = get_record_batches_memory_size(&sorted_batches); reservation .try_resize(total_sorted_size) .map_err(Self::err_with_oom_context)?; diff --git a/datafusion/physical-plan/src/stream.rs b/datafusion/physical-plan/src/stream.rs index 80c2233d05db6..5eeb45c930fb0 100644 --- a/datafusion/physical-plan/src/stream.rs +++ b/datafusion/physical-plan/src/stream.rs @@ -704,6 +704,7 @@ impl RecordBatchStream for BatchSplitStream { /// A stream that holds a memory reservation for its lifetime, /// shrinking the reservation as batches are consumed. /// The original reservation must have its batch sizes calculated using [`get_record_batch_memory_size`] +/// or [`get_record_batches_memory_size`](datafusion_common::utils::memory::get_record_batches_memory_size). /// On error, the reservation is *NOT* freed, until the stream is dropped. pub(crate) struct ReservationStream { schema: SchemaRef, @@ -738,8 +739,14 @@ impl Stream for ReservationStream { Poll::Ready(res) => { match res { Some(Ok(batch)) => { - self.reservation - .shrink(get_record_batch_memory_size(&batch)); + // Batches may share buffers (e.g. dictionary values + // after `take`), so their individual sizes can add up + // to more than was reserved for all of them together. + // Release at most what is left; the remainder is freed + // when the stream ends. + let size = get_record_batch_memory_size(&batch) + .min(self.reservation.size()); + self.reservation.shrink(size); Poll::Ready(Some(Ok(batch))) } Some(Err(err)) => Poll::Ready(Some(Err(err))), From 432da8de5d88abaa34f4fe128e20fa85f7f5a34b Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Wed, 23 Sep 2026 13:51:05 +0200 Subject: [PATCH 3/7] test(cli): pull the MinIO test image from quay.io MinIO withdrew minio/minio from Docker Hub on 2026-09-11, so the storage integration tests fail to start their container. Point the image pinned by testcontainers-modules at quay.io, which serves the same tags. Minimal backport of apache/datafusion#25216. --- datafusion-cli/tests/cli_integration.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/datafusion-cli/tests/cli_integration.rs b/datafusion-cli/tests/cli_integration.rs index 8b8b786d652e3..98a6284e54605 100644 --- a/datafusion-cli/tests/cli_integration.rs +++ b/datafusion-cli/tests/cli_integration.rs @@ -44,6 +44,12 @@ fn make_settings() -> Settings { settings } +/// Registry override for the image pinned by `testcontainers-modules`. +/// +/// MinIO withdrew `minio/minio` from Docker Hub on 2026-09-11. quay.io still +/// serves the same tags. Backport of apache/datafusion#25216. +const MINIO_IMAGE_NAME: &str = "quay.io/minio/minio"; + async fn setup_minio_container() -> ContainerAsync { const MINIO_ROOT_USER: &str = "TEST-DataFusionLogin"; const MINIO_ROOT_PASSWORD: &str = "TEST-DataFusionPassword"; @@ -56,6 +62,7 @@ async fn setup_minio_container() -> ContainerAsync { .expect("Failed to get absolute path for test data"); let container = minio::MinIO::default() + .with_name(MINIO_IMAGE_NAME) .with_env_var("MINIO_ROOT_USER", MINIO_ROOT_USER) .with_env_var("MINIO_ROOT_PASSWORD", MINIO_ROOT_PASSWORD) .with_mount(Mount::bind_mount( From 89dead2fbe4d12ad063f3cd464e82790fbc00292 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 24 Sep 2026 10:24:12 +0200 Subject: [PATCH 4/7] fix(physical-plan): release shared sort buffers with their last queued batch Clamping the per-batch release kept the reservation from underflowing but freed shared buffers as soon as the first batch was emitted, while the queued batches still referenced them. get_record_batches_release_sizes charges each buffer to the last batch that references it, so the sum is the shared-buffer total and the stream releases exactly what each batch stops holding. ReservationStream had no other user and is removed. --- datafusion/common/src/utils/memory.rs | 60 +++---- datafusion/physical-plan/src/sorts/sort.rs | 88 ++++++++-- datafusion/physical-plan/src/stream.rs | 192 --------------------- 3 files changed, 104 insertions(+), 236 deletions(-) diff --git a/datafusion/common/src/utils/memory.rs b/datafusion/common/src/utils/memory.rs index 24808dac3aaf6..72560c78f2ca1 100644 --- a/datafusion/common/src/utils/memory.rs +++ b/datafusion/common/src/utils/memory.rs @@ -131,27 +131,29 @@ pub fn estimate_memory_size(num_elements: usize, fixed_size: usize) -> Result /// `Buffer`. This method provides temporary fix until the issue is resolved: /// pub fn get_record_batch_memory_size(batch: &RecordBatch) -> usize { - get_record_batches_memory_size(std::iter::once(batch)) + get_record_batches_memory_size(std::slice::from_ref(batch)) } /// Calculate total used memory of `batches`, counting a `Buffer` shared by -/// several batches (e.g. dictionary values across `take`n chunks) only once. -/// -/// See [`get_record_batch_memory_size`] for the per-batch semantics. -pub fn get_record_batches_memory_size<'a>( - batches: impl IntoIterator, -) -> usize { - // Store pointers to `Buffer`'s start memory address (instead of actual - // used data region's pointer represented by current `Array`) +/// several batches only once. +pub fn get_record_batches_memory_size(batches: &[RecordBatch]) -> usize { + get_record_batches_release_sizes(batches).into_iter().sum() +} + +/// Memory each of `batches` releases when consumed in order: a `Buffer` +/// shared by several batches is charged to the last batch that references it. +pub fn get_record_batches_release_sizes(batches: &[RecordBatch]) -> Vec { let mut counted_buffers: HashSet> = HashSet::new(); - let mut total_size = 0; + let mut sizes = vec![0; batches.len()]; - for array in batches.into_iter().flat_map(|batch| batch.columns()) { - let array_data = array.to_data(); - count_array_data_memory_size(&array_data, &mut counted_buffers, &mut total_size); + for (size, batch) in sizes.iter_mut().zip(batches).rev() { + for array in batch.columns() { + let array_data = array.to_data(); + count_array_data_memory_size(&array_data, &mut counted_buffers, size); + } } - total_size + sizes } /// Count the memory usage of `array_data` and its children recursively. @@ -311,27 +313,21 @@ mod record_batch_tests { let slice1 = original.slice(0, 3); let slice2 = original.slice(2, 3); - let schema_origin = Arc::new(Schema::new(vec![Field::new( - "origin_col", - DataType::Int32, - false, - )])); + let schema = + Arc::new(Schema::new(vec![Field::new("col", DataType::Int32, false)])); let batch_origin = - RecordBatch::try_new(Arc::clone(&schema_origin), vec![Arc::new(original)]) - .unwrap(); - let batch_slice1 = - RecordBatch::try_new(Arc::clone(&schema_origin), vec![Arc::new(slice1)]) - .unwrap(); - let batch_slice2 = - RecordBatch::try_new(schema_origin, vec![Arc::new(slice2)]).unwrap(); + RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(original)]).unwrap(); + let slices = [ + RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(slice1)]).unwrap(), + RecordBatch::try_new(schema, vec![Arc::new(slice2)]).unwrap(), + ]; let size_origin = get_record_batch_memory_size(&batch_origin); - let size_slices = get_record_batches_memory_size([&batch_slice1, &batch_slice2]); - let size_summed = get_record_batch_memory_size(&batch_slice1) - + get_record_batch_memory_size(&batch_slice2); - - assert_eq!(size_origin, size_slices); - assert_eq!(size_summed, 2 * size_origin); + assert_eq!(get_record_batches_memory_size(&slices), size_origin); + assert_eq!( + get_record_batches_release_sizes(&slices), + vec![0, size_origin] + ); } #[test] diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index 99a2c917ec076..e522973c1663a 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -42,7 +42,6 @@ use crate::spill::get_record_batch_memory_size; use crate::spill::in_progress_spill_file::InProgressSpillFile; use crate::spill::spill_manager::{GetSlicedSize, SpillManager}; use crate::stream::RecordBatchStreamAdapter; -use crate::stream::ReservationStream; use crate::topk::TopK; use crate::topk::TopKDynamicFilters; use crate::{ @@ -50,7 +49,7 @@ use crate::{ ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream, Statistics, }; -use datafusion_common::utils::memory::get_record_batches_memory_size; +use datafusion_common::utils::memory::get_record_batches_release_sizes; use arrow::array::{Array, RecordBatch, RecordBatchOptions, StringViewArray}; use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays}; @@ -733,19 +732,20 @@ impl ExternalSorter { // Using try_resize avoids a release-then-reacquire cycle, which // matters for MemoryPool implementations where grow/shrink have // non-trivial cost (e.g. JNI calls in Comet). - let total_sorted_size = get_record_batches_memory_size(&sorted_batches); + let release_sizes = get_record_batches_release_sizes(&sorted_batches); reservation - .try_resize(total_sorted_size) + .try_resize(release_sizes.iter().sum()) .map_err(Self::err_with_oom_context)?; - // Wrap in ReservationStream to hold the reservation - Result::<_, DataFusionError>::Ok(Box::pin(ReservationStream::new( - Arc::clone(&schema), - Box::pin(RecordBatchStreamAdapter::new( - schema, - futures::stream::iter(sorted_batches.into_iter().map(Ok)), - )), - reservation, + let batches = sorted_batches.into_iter().zip(release_sizes).map( + move |(batch, size)| { + reservation.shrink(size); + Ok(batch) + }, + ); + Result::<_, DataFusionError>::Ok(Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::iter(batches), )) as SendableRecordBatchStream) }) .try_flatten() @@ -1454,6 +1454,7 @@ mod tests { use datafusion_common::cast::as_primitive_array; use datafusion_common::config::ConfigOptions; use datafusion_common::test_util::batches_to_string; + use datafusion_common::utils::memory::get_record_batches_memory_size; use datafusion_common::{DataFusionError, Result, ScalarValue}; use datafusion_execution::RecordBatchStream; use datafusion_execution::config::SessionConfig; @@ -1802,6 +1803,69 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_sort_shared_dictionary_release_per_batch() -> Result<()> { + const ROWS: usize = 10_000; + const BATCH_SIZE: usize = 1_000; + + let schema = Arc::new(Schema::new(vec![ + Field::new( + "tag", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + false, + ), + Field::new("time", DataType::Int64, false), + ])); + let mut tag = StringDictionaryBuilder::::new(); + for i in 0..ROWS { + tag.append_value(format!("truck_{:05}", (i * 7919) % ROWS)); + } + let time = Int64Array::from_iter_values((0..ROWS).rev().map(|i| i as i64)); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(tag.finish()), Arc::new(time)], + )?; + + let expressions: LexOrdering = [ + PhysicalSortExpr::new_default(col("tag", &schema)?), + PhysicalSortExpr::new_default(col("time", &schema)?), + ] + .into(); + let expected = sort_batch_chunked(&batch, &expressions, BATCH_SIZE)?; + + let runtime = RuntimeEnvBuilder::new() + .with_memory_limit(get_record_batch_memory_size(&batch) * 4, 1.0) + .with_disk_manager_builder( + DiskManagerBuilder::default().with_mode(DiskManagerMode::Disabled), + ) + .build_arc()?; + let task_ctx = Arc::new( + TaskContext::default() + .with_session_config(SessionConfig::new().with_batch_size(BATCH_SIZE)) + .with_runtime(runtime), + ); + + let plan = + TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?; + let sort_exec = SortExec::new(expressions, plan); + let mut stream = sort_exec.execute(0, Arc::clone(&task_ctx))?; + + // Emitted batches are dropped right away; the pool must still hold + // what the queued batches reference, shared dictionary included. + for emitted in 1..=expected.len() { + drop(stream.next().await.unwrap()?); + assert_eq!( + task_ctx.memory_pool().reserved(), + get_record_batches_memory_size(&expected[emitted..]), + "after {emitted} of {} batches", + expected.len() + ); + } + assert!(stream.next().await.is_none()); + assert_eq!(task_ctx.memory_pool().reserved(), 0); + Ok(()) + } + #[tokio::test] async fn test_sort_spill_utf8_strings() -> Result<()> { let session_config = SessionConfig::new() diff --git a/datafusion/physical-plan/src/stream.rs b/datafusion/physical-plan/src/stream.rs index 5eeb45c930fb0..e2e11b8fb23ac 100644 --- a/datafusion/physical-plan/src/stream.rs +++ b/datafusion/physical-plan/src/stream.rs @@ -27,13 +27,11 @@ use super::metrics::ExecutionPlanMetricsSet; use super::metrics::{BaselineMetrics, SplitMetrics}; use super::{ExecutionPlan, RecordBatchStream, SendableRecordBatchStream}; use crate::displayable; -use crate::spill::get_record_batch_memory_size; use arrow::{datatypes::SchemaRef, record_batch::RecordBatch}; use datafusion_common::{Result, exec_err}; use datafusion_common_runtime::JoinSet; use datafusion_execution::TaskContext; -use datafusion_execution::memory_pool::MemoryReservation; use futures::ready; use futures::stream::BoxStream; @@ -701,77 +699,6 @@ impl RecordBatchStream for BatchSplitStream { } } -/// A stream that holds a memory reservation for its lifetime, -/// shrinking the reservation as batches are consumed. -/// The original reservation must have its batch sizes calculated using [`get_record_batch_memory_size`] -/// or [`get_record_batches_memory_size`](datafusion_common::utils::memory::get_record_batches_memory_size). -/// On error, the reservation is *NOT* freed, until the stream is dropped. -pub(crate) struct ReservationStream { - schema: SchemaRef, - inner: SendableRecordBatchStream, - reservation: MemoryReservation, -} - -impl ReservationStream { - pub(crate) fn new( - schema: SchemaRef, - inner: SendableRecordBatchStream, - reservation: MemoryReservation, - ) -> Self { - Self { - schema, - inner, - reservation, - } - } -} - -impl Stream for ReservationStream { - type Item = Result; - - fn poll_next( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll> { - let res = self.inner.poll_next_unpin(cx); - - match res { - Poll::Ready(res) => { - match res { - Some(Ok(batch)) => { - // Batches may share buffers (e.g. dictionary values - // after `take`), so their individual sizes can add up - // to more than was reserved for all of them together. - // Release at most what is left; the remainder is freed - // when the stream ends. - let size = get_record_batch_memory_size(&batch) - .min(self.reservation.size()); - self.reservation.shrink(size); - Poll::Ready(Some(Ok(batch))) - } - Some(Err(err)) => Poll::Ready(Some(Err(err))), - None => { - // Stream is done so free the reservation completely - self.reservation.free(); - Poll::Ready(None) - } - } - } - Poll::Pending => Poll::Pending, - } - } - - fn size_hint(&self) -> (usize, Option) { - self.inner.size_hint() - } -} - -impl RecordBatchStream for ReservationStream { - fn schema(&self) -> SchemaRef { - Arc::clone(&self.schema) - } -} - #[cfg(test)] mod test { use super::*; @@ -1000,123 +927,4 @@ mod test { "Should have received exactly two empty batches" ); } - - #[tokio::test] - async fn test_reservation_stream_shrinks_on_poll() { - use arrow::array::Int32Array; - use datafusion_execution::memory_pool::MemoryConsumer; - use datafusion_execution::runtime_env::RuntimeEnvBuilder; - - let runtime = RuntimeEnvBuilder::new() - .with_memory_limit(10 * 1024 * 1024, 1.0) - .build_arc() - .unwrap(); - - let mut reservation = MemoryConsumer::new("test").register(&runtime.memory_pool); - - let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); - - // Create batches - let batch1 = RecordBatch::try_new( - Arc::clone(&schema), - vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))], - ) - .unwrap(); - let batch2 = RecordBatch::try_new( - Arc::clone(&schema), - vec![Arc::new(Int32Array::from(vec![6, 7, 8, 9, 10]))], - ) - .unwrap(); - - let batch1_size = get_record_batch_memory_size(&batch1); - let batch2_size = get_record_batch_memory_size(&batch2); - - // Reserve memory upfront - reservation.try_grow(batch1_size + batch2_size).unwrap(); - let initial_reserved = runtime.memory_pool.reserved(); - assert_eq!(initial_reserved, batch1_size + batch2_size); - - // Create stream with batches - let stream = futures::stream::iter(vec![Ok(batch1), Ok(batch2)]); - let inner = Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&schema), stream)) - as SendableRecordBatchStream; - - let mut res_stream = - ReservationStream::new(Arc::clone(&schema), inner, reservation); - - // Poll first batch - let result1 = res_stream.next().await; - assert!(result1.is_some()); - - // Memory should be reduced by batch1_size - let after_first = runtime.memory_pool.reserved(); - assert_eq!(after_first, batch2_size); - - // Poll second batch - let result2 = res_stream.next().await; - assert!(result2.is_some()); - - // Memory should be reduced by batch2_size - let after_second = runtime.memory_pool.reserved(); - assert_eq!(after_second, 0); - - // Poll None (end of stream) - let result3 = res_stream.next().await; - assert!(result3.is_none()); - - // Memory should still be 0 - assert_eq!(runtime.memory_pool.reserved(), 0); - } - - #[tokio::test] - async fn test_reservation_stream_error_handling() { - use datafusion_execution::memory_pool::MemoryConsumer; - use datafusion_execution::runtime_env::RuntimeEnvBuilder; - - let runtime = RuntimeEnvBuilder::new() - .with_memory_limit(10 * 1024 * 1024, 1.0) - .build_arc() - .unwrap(); - - let mut reservation = MemoryConsumer::new("test").register(&runtime.memory_pool); - - let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); - - reservation.try_grow(1000).unwrap(); - let initial = runtime.memory_pool.reserved(); - assert_eq!(initial, 1000); - - // Create a stream that errors - let stream = futures::stream::iter(vec![exec_err!("Test error")]); - let inner = Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&schema), stream)) - as SendableRecordBatchStream; - - let mut res_stream = - ReservationStream::new(Arc::clone(&schema), inner, reservation); - - // Get the error - let result = res_stream.next().await; - assert!(result.is_some()); - assert!(result.unwrap().is_err()); - - // Verify reservation is NOT automatically freed on error - // The reservation is only freed when poll_next returns Poll::Ready(None) - // After an error, the stream may continue to hold the reservation - // until it's explicitly dropped or polled to None - let after_error = runtime.memory_pool.reserved(); - assert_eq!( - after_error, 1000, - "Reservation should still be held after error" - ); - - // Drop the stream to free the reservation - drop(res_stream); - - // Now memory should be freed - assert_eq!( - runtime.memory_pool.reserved(), - 0, - "Memory should be freed when stream is dropped" - ); - } } From d70305eb9de92e3bad9c856f01f57416e59424d0 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 24 Sep 2026 10:35:37 +0200 Subject: [PATCH 5/7] fix(physical-plan): keep the scalar batch size path allocation-free get_record_batch_memory_size and get_record_batches_memory_size count into a scalar again; only sort_batch_stream builds the release-size vector. Add a test that drops a partially consumed sorted stream and checks the pool returns to zero, and share the test scaffolding. --- datafusion/common/src/utils/memory.rs | 27 +++++- datafusion/physical-plan/src/sorts/sort.rs | 108 +++++++++------------ 2 files changed, 69 insertions(+), 66 deletions(-) diff --git a/datafusion/common/src/utils/memory.rs b/datafusion/common/src/utils/memory.rs index 72560c78f2ca1..cee7e90a1f1e6 100644 --- a/datafusion/common/src/utils/memory.rs +++ b/datafusion/common/src/utils/memory.rs @@ -137,7 +137,16 @@ pub fn get_record_batch_memory_size(batch: &RecordBatch) -> usize { /// Calculate total used memory of `batches`, counting a `Buffer` shared by /// several batches only once. pub fn get_record_batches_memory_size(batches: &[RecordBatch]) -> usize { - get_record_batches_release_sizes(batches).into_iter().sum() + // Store pointers to `Buffer`'s start memory address (instead of actual + // used data region's pointer represented by current `Array`) + let mut counted_buffers: HashSet> = HashSet::new(); + let mut total_size = 0; + + for batch in batches { + count_record_batch_memory_size(batch, &mut counted_buffers, &mut total_size); + } + + total_size } /// Memory each of `batches` releases when consumed in order: a `Buffer` @@ -147,15 +156,23 @@ pub fn get_record_batches_release_sizes(batches: &[RecordBatch]) -> Vec { let mut sizes = vec![0; batches.len()]; for (size, batch) in sizes.iter_mut().zip(batches).rev() { - for array in batch.columns() { - let array_data = array.to_data(); - count_array_data_memory_size(&array_data, &mut counted_buffers, size); - } + count_record_batch_memory_size(batch, &mut counted_buffers, size); } sizes } +fn count_record_batch_memory_size( + batch: &RecordBatch, + counted_buffers: &mut HashSet>, + total_size: &mut usize, +) { + for array in batch.columns() { + let array_data = array.to_data(); + count_array_data_memory_size(&array_data, counted_buffers, total_size); + } +} + /// Count the memory usage of `array_data` and its children recursively. fn count_array_data_memory_size( array_data: &ArrayData, diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index e522973c1663a..7ed1f8da412c8 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -1743,15 +1743,9 @@ mod tests { Ok(()) } - /// A single large batch with a dictionary column: `sort_batch_chunked` - /// shares the dictionary values across every output chunk, but the - /// reservation for the sorted output charges it once per chunk. - #[tokio::test] - async fn test_sort_single_batch_shared_dictionary_reservation() -> Result<()> { - const ROWS: usize = 100_000; - const CARDINALITY: usize = 50_000; - const BATCH_SIZE: usize = 1_000; - + /// One batch whose `tag` dictionary is shared by every chunk of its sorted + /// output. + fn shared_dictionary_batch(rows: usize) -> Result { let schema = Arc::new(Schema::new(vec![ Field::new( "tag", @@ -1761,30 +1755,35 @@ mod tests { Field::new("time", DataType::Int64, false), ])); let mut tag = StringDictionaryBuilder::::new(); - for i in 0..ROWS { - tag.append_value(format!("truck_{:05}", (i * 7919) % CARDINALITY)); + for i in 0..rows { + tag.append_value(format!("truck_{:05}", (i * 7919) % (rows / 2))); } - let time = Int64Array::from_iter_values((0..ROWS).rev().map(|i| i as i64)); - let batch = RecordBatch::try_new( - Arc::clone(&schema), + let time = Int64Array::from_iter_values((0..rows).rev().map(|i| i as i64)); + Ok(RecordBatch::try_new( + schema, vec![Arc::new(tag.finish()), Arc::new(time)], - )?; - let input_bytes = get_record_batch_memory_size(&batch); + )?) + } - // Room for the input several times over, but not for the dictionary - // once per chunk. No disk manager, so the sort cannot spill. + /// `SortExec` on `batch` by `(tag, time)` with a `batch_size`-row output, + /// a pool of `pool_batches` times the input and no disk manager. + fn shared_dictionary_sort( + batch: RecordBatch, + batch_size: usize, + pool_batches: usize, + ) -> Result<(Arc, Arc)> { + let schema = batch.schema(); let runtime = RuntimeEnvBuilder::new() - .with_memory_limit(input_bytes * 4, 1.0) + .with_memory_limit(get_record_batch_memory_size(&batch) * pool_batches, 1.0) .with_disk_manager_builder( DiskManagerBuilder::default().with_mode(DiskManagerMode::Disabled), ) .build_arc()?; let task_ctx = Arc::new( TaskContext::default() - .with_session_config(SessionConfig::new().with_batch_size(BATCH_SIZE)) + .with_session_config(SessionConfig::new().with_batch_size(batch_size)) .with_runtime(runtime), ); - let plan = TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?; let sort_exec = Arc::new(SortExec::new( @@ -1795,6 +1794,16 @@ mod tests { .into(), plan, )); + Ok((sort_exec, task_ctx)) + } + + #[tokio::test] + async fn test_sort_single_batch_shared_dictionary_reservation() -> Result<()> { + const ROWS: usize = 100_000; + const BATCH_SIZE: usize = 1_000; + + let (sort_exec, task_ctx) = + shared_dictionary_sort(shared_dictionary_batch(ROWS)?, BATCH_SIZE, 4)?; let result = collect(sort_exec, task_ctx).await?; let rows: usize = result.iter().map(|b| b.num_rows()).sum(); @@ -1808,46 +1817,9 @@ mod tests { const ROWS: usize = 10_000; const BATCH_SIZE: usize = 1_000; - let schema = Arc::new(Schema::new(vec![ - Field::new( - "tag", - DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), - false, - ), - Field::new("time", DataType::Int64, false), - ])); - let mut tag = StringDictionaryBuilder::::new(); - for i in 0..ROWS { - tag.append_value(format!("truck_{:05}", (i * 7919) % ROWS)); - } - let time = Int64Array::from_iter_values((0..ROWS).rev().map(|i| i as i64)); - let batch = RecordBatch::try_new( - Arc::clone(&schema), - vec![Arc::new(tag.finish()), Arc::new(time)], - )?; - - let expressions: LexOrdering = [ - PhysicalSortExpr::new_default(col("tag", &schema)?), - PhysicalSortExpr::new_default(col("time", &schema)?), - ] - .into(); - let expected = sort_batch_chunked(&batch, &expressions, BATCH_SIZE)?; - - let runtime = RuntimeEnvBuilder::new() - .with_memory_limit(get_record_batch_memory_size(&batch) * 4, 1.0) - .with_disk_manager_builder( - DiskManagerBuilder::default().with_mode(DiskManagerMode::Disabled), - ) - .build_arc()?; - let task_ctx = Arc::new( - TaskContext::default() - .with_session_config(SessionConfig::new().with_batch_size(BATCH_SIZE)) - .with_runtime(runtime), - ); - - let plan = - TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?; - let sort_exec = SortExec::new(expressions, plan); + let batch = shared_dictionary_batch(ROWS)?; + let (sort_exec, task_ctx) = shared_dictionary_sort(batch.clone(), BATCH_SIZE, 4)?; + let expected = sort_batch_chunked(&batch, sort_exec.expr(), BATCH_SIZE)?; let mut stream = sort_exec.execute(0, Arc::clone(&task_ctx))?; // Emitted batches are dropped right away; the pool must still hold @@ -1866,6 +1838,20 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_sort_shared_dictionary_release_on_drop() -> Result<()> { + let (sort_exec, task_ctx) = + shared_dictionary_sort(shared_dictionary_batch(10_000)?, 1_000, 4)?; + let mut stream = sort_exec.execute(0, Arc::clone(&task_ctx))?; + + drop(stream.next().await.unwrap()?); + assert!(task_ctx.memory_pool().reserved() > 0); + + drop(stream); + assert_eq!(task_ctx.memory_pool().reserved(), 0); + Ok(()) + } + #[tokio::test] async fn test_sort_spill_utf8_strings() -> Result<()> { let session_config = SessionConfig::new() From f066031bbefba55cdc444c547c3eeff64210c1b2 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 24 Sep 2026 12:52:19 +0200 Subject: [PATCH 6/7] chore(common): port RecordBatchMemoryCounter from upstream datafusion/common/src/utils/memory.rs as of apache/datafusion main 9951f0b (2026-09-24): apache/datafusion#24319 introduced the counter and #24364, #24837, #25335 refined it. Counts each buffer once across a sequence of batches; replaces the fork-local helpers. --- datafusion/common/src/utils/memory.rs | 844 +++++++++++++++++++++++--- 1 file changed, 774 insertions(+), 70 deletions(-) diff --git a/datafusion/common/src/utils/memory.rs b/datafusion/common/src/utils/memory.rs index cee7e90a1f1e6..bb006295f8c1d 100644 --- a/datafusion/common/src/utils/memory.rs +++ b/datafusion/common/src/utils/memory.rs @@ -19,9 +19,27 @@ use crate::error::_exec_datafusion_err; use crate::{HashSet, Result}; -use arrow::array::ArrayData; +use arrow::array::types::{ + ByteArrayType, ByteViewType, Int8Type, Int16Type, Int32Type, Int64Type, + RunEndIndexType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, +}; +use arrow::array::{ + Array, ArrayRef, AsArray, GenericByteArray, GenericByteViewArray, GenericListArray, + GenericListViewArray, RunArray, +}; +use arrow::buffer::Buffer; +use arrow::datatypes::DataType; +use arrow::downcast_primitive_array; use arrow::record_batch::RecordBatch; -use std::{mem::size_of, ptr::NonNull}; +use std::mem::size_of; +use std::num::NonZero; +use std::sync::Arc; + +/// Maximum number of distinct buffer IDs retained inline before promotion to +/// a [`HashSet`]. Sixteen keeps small buffer sets allocation-free while +/// limiting linear lookup and inline storage to 16 pointer-sized entries. +/// This is a performance heuristic, not a semantic limit. +const INLINE_BUFFER_IDS: usize = 16; /// Estimates the memory size required for a hash table prior to allocation. /// @@ -117,84 +135,367 @@ pub fn estimate_memory_size(num_elements: usize, fixed_size: usize) -> Result /// For a `RecordBatch` with two columns: `col1` and `col2`, two columns are pointing /// to a sub-region of the same buffer. /// +/// ```text /// {xxxxxxxxxxxxxxxxxxx} <--- buffer /// ^ ^ ^ ^ /// | | | | /// col1->{ } | | /// col2--------->{ } +/// ``` /// /// In the above case, `get_record_batch_memory_size` will return the size of /// the buffer, instead of the sum of `col1` and `col2`'s actual memory size. /// -/// Note: Current `RecordBatch`.get_array_memory_size()` will double count the -/// buffer memory size if multiple arrays within the batch are sharing the same -/// `Buffer`. This method provides temporary fix until the issue is resolved: -/// +/// Note: [`RecordBatch::get_array_memory_size`] double counts the buffer +/// memory size if multiple arrays within the batch are sharing the same +/// `Buffer`, while this function counts each `Buffer` exactly once. pub fn get_record_batch_memory_size(batch: &RecordBatch) -> usize { - get_record_batches_memory_size(std::slice::from_ref(batch)) + RecordBatchMemoryCounter::new().count_batch(batch) } -/// Calculate total used memory of `batches`, counting a `Buffer` shared by -/// several batches only once. -pub fn get_record_batches_memory_size(batches: &[RecordBatch]) -> usize { - // Store pointers to `Buffer`'s start memory address (instead of actual - // used data region's pointer represented by current `Array`) - let mut counted_buffers: HashSet> = HashSet::new(); - let mut total_size = 0; +/// Tracks the memory used by a sequence of [`RecordBatch`]es that may share +/// underlying buffers, counting each buffer exactly once. +/// +/// Use this instead of [`get_record_batch_memory_size`] to account for the +/// total memory of a sequence of batches, e.g. when buffering the batches of +/// an input stream. Such batches can share buffers (for example, operators +/// like aggregates emit one large batch as multiple zero-copy slices), and +/// calling [`get_record_batch_memory_size`] per batch counts the shared +/// buffers once per batch, while this counter counts them exactly once. A +/// batch's buffers are kept alive by the batch even when only a sub-range is +/// referenced, so counting unique buffers in full reflects the memory the +/// batches actually retain. +#[derive(Debug, Default)] +pub struct RecordBatchMemoryCounter { + /// Start addresses of `Buffer`s that have already been counted (instead of + /// actual used data region's pointer represented by current `Array`) + counted_buffers: BufferIdSet, + /// Array objects already counted by [`Self::count_batch_with_array_overhead`] + counted_arrays: HashSet, + /// Total memory of all counted allocations + memory_usage: usize, +} - for batch in batches { - count_record_batch_memory_size(batch, &mut counted_buffers, &mut total_size); +impl RecordBatchMemoryCounter { + pub fn new() -> Self { + Self::default() } - total_size -} + /// Count `batch`, returning the memory used by its buffers that have not + /// been counted before. + pub fn count_batch(&mut self, batch: &RecordBatch) -> usize { + let previous_memory_usage = self.memory_usage; + + for array in batch.columns() { + self.count_array_memory_size(array.as_ref()); + } + + self.memory_usage - previous_memory_usage + } + + /// Counts unique buffers and Array objects retained by `batch`. + /// + /// This is useful for accounting a sequence of batches at an operator + /// boundary. It counts buffers once, including buffers shared by multiple + /// batches. Array-object overhead is deduplicated recursively when a shared + /// child has the same `ArrayRef` identity; Arrow constructors that rebuild + /// child array objects may conservatively count their object overhead again. + pub fn count_batch_with_array_overhead(&mut self, batch: &RecordBatch) -> usize { + let mut total_size = self.count_batch(batch); + let mut array_overhead = 0; -/// Memory each of `batches` releases when consumed in order: a `Buffer` -/// shared by several batches is charged to the last batch that references it. -pub fn get_record_batches_release_sizes(batches: &[RecordBatch]) -> Vec { - let mut counted_buffers: HashSet> = HashSet::new(); - let mut sizes = vec![0; batches.len()]; + for array in batch.columns() { + array_overhead += + count_unique_array_object_memory_size(array, &mut self.counted_arrays); + } + + total_size += array_overhead; + self.memory_usage += array_overhead; + total_size + } + + /// Total memory of all counted allocations. + pub fn memory_usage(&self) -> usize { + self.memory_usage + } + + fn count_buffer_memory_size(&mut self, buffer: &Buffer) { + if self.counted_buffers.insert(buffer.data_ptr().addr()) { + self.memory_usage += buffer.capacity(); + } + } + + /// Count the memory usage of `array` and its children recursively. + fn count_array_memory_size(&mut self, array: &dyn Array) { + if let Some(nulls) = array.nulls() { + self.count_buffer_memory_size(nulls.buffer()); + } + + downcast_primitive_array! { + array => self.count_buffer_memory_size(array.values().inner()), + DataType::Null => {} + DataType::Boolean => { + self.count_buffer_memory_size(array.as_boolean().values().inner()); + } + DataType::Binary => { + self.count_byte_array_memory_size(array.as_binary::()); + } + DataType::LargeBinary => { + self.count_byte_array_memory_size(array.as_binary::()); + } + DataType::Utf8 => { + self.count_byte_array_memory_size(array.as_string::()); + } + DataType::LargeUtf8 => { + self.count_byte_array_memory_size(array.as_string::()); + } + DataType::BinaryView => { + self.count_byte_view_array_memory_size(array.as_binary_view()); + } + DataType::Utf8View => { + self.count_byte_view_array_memory_size(array.as_string_view()); + } + DataType::FixedSizeBinary(_) => { + self.count_buffer_memory_size(array.as_fixed_size_binary().values()); + } + DataType::List(_) => { + self.count_list_array_memory_size(array.as_list::()); + } + DataType::LargeList(_) => { + self.count_list_array_memory_size(array.as_list::()); + } + DataType::ListView(_) => { + self.count_list_view_array_memory_size(array.as_list_view::()); + } + DataType::LargeListView(_) => { + self.count_list_view_array_memory_size(array.as_list_view::()); + } + DataType::FixedSizeList(_, _) => { + self.count_array_memory_size( + array.as_fixed_size_list().values().as_ref(), + ); + } + DataType::Struct(_) => { + for child in array.as_struct().columns() { + self.count_array_memory_size(child.as_ref()); + } + } + DataType::Union(_, _) => { + let array = array.as_union(); + self.count_buffer_memory_size(array.type_ids().inner()); + if let Some(offsets) = array.offsets() { + self.count_buffer_memory_size(offsets.inner()); + } + for (type_id, _) in array.fields().iter() { + self.count_array_memory_size(array.child(type_id).as_ref()); + } + } + DataType::Dictionary(_, _) => { + let array = array.as_any_dictionary(); + self.count_array_memory_size(array.keys()); + self.count_array_memory_size(array.values().as_ref()); + } + DataType::Map(_, _) => { + let array = array.as_map(); + self.count_buffer_memory_size(array.offsets().inner().inner()); + self.count_array_memory_size(array.entries()); + } + DataType::RunEndEncoded(run_ends, _) => match run_ends.data_type() { + DataType::Int16 => self.count_run_array_memory_size::(array), + DataType::Int32 => self.count_run_array_memory_size::(array), + DataType::Int64 => self.count_run_array_memory_size::(array), + // Arrow only permits Int16, Int32, and Int64 run-end indexes. A + // custom Array implementation may still expose malformed data; + // retain correct accounting for it without panicking. + _ => self.count_array_data_memory_size(&array.to_data()), + }, + // All currently supported non-primitive layouts are handled above. + // The Arrow macro requires a final arm for primitive variants that + // its nested dispatch has already consumed. Keep a safe generic + // fallback for custom or future Array implementations. + _ => self.count_array_data_memory_size(&array.to_data()), + } + } - for (size, batch) in sizes.iter_mut().zip(batches).rev() { - count_record_batch_memory_size(batch, &mut counted_buffers, size); + fn count_byte_array_memory_size( + &mut self, + array: &GenericByteArray, + ) { + self.count_buffer_memory_size(array.offsets().inner().inner()); + self.count_buffer_memory_size(array.values()); } - sizes + fn count_byte_view_array_memory_size( + &mut self, + array: &GenericByteViewArray, + ) { + self.count_buffer_memory_size(array.views().inner()); + for buffer in array.data_buffers().iter() { + self.count_buffer_memory_size(buffer); + } + } + + fn count_list_array_memory_size( + &mut self, + array: &GenericListArray, + ) { + self.count_buffer_memory_size(array.offsets().inner().inner()); + self.count_array_memory_size(array.values().as_ref()); + } + + fn count_list_view_array_memory_size( + &mut self, + array: &GenericListViewArray, + ) { + self.count_buffer_memory_size(array.offsets().inner()); + self.count_buffer_memory_size(array.sizes().inner()); + self.count_array_memory_size(array.values().as_ref()); + } + + fn count_run_array_memory_size(&mut self, array: &dyn Array) { + if let Some(array) = array.as_any().downcast_ref::>() { + self.count_buffer_memory_size(array.run_ends().inner().inner()); + self.count_array_memory_size(array.values().as_ref()); + } else { + // The DataType and concrete array implementation disagree. Use the + // generic representation rather than panic while accounting memory. + self.count_array_data_memory_size(&array.to_data()); + } + } + + fn count_array_data_memory_size(&mut self, array_data: &arrow::array::ArrayData) { + for buffer in array_data.buffers() { + self.count_buffer_memory_size(buffer); + } + if let Some(nulls) = array_data.nulls() { + self.count_buffer_memory_size(nulls.buffer()); + } + for child in array_data.child_data() { + self.count_array_data_memory_size(child); + } + } } -fn count_record_batch_memory_size( - batch: &RecordBatch, - counted_buffers: &mut HashSet>, - total_size: &mut usize, -) { - for array in batch.columns() { - let array_data = array.to_data(); - count_array_data_memory_size(&array_data, counted_buffers, total_size); +/// Counts the unique Array object memory retained by `array` and its children. +fn count_unique_array_object_memory_size( + array: &ArrayRef, + counted_arrays: &mut HashSet, +) -> usize { + let array_ptr = Arc::as_ptr(array).cast::<()>() as usize; + if !counted_arrays.insert(array_ptr) { + return 0; } + + let children = array_children(array); + let children_overhead: usize = children + .iter() + .map(|child| child.get_array_memory_size() - child.get_buffer_memory_size()) + .sum(); + let own_overhead = array.get_array_memory_size() + - array.get_buffer_memory_size() + - children_overhead; + + own_overhead + + children + .into_iter() + .map(|child| count_unique_array_object_memory_size(child, counted_arrays)) + .sum::() } -/// Count the memory usage of `array_data` and its children recursively. -fn count_array_data_memory_size( - array_data: &ArrayData, - counted_buffers: &mut HashSet>, - total_size: &mut usize, -) { - // Count memory usage for `array_data` - for buffer in array_data.buffers() { - if counted_buffers.insert(buffer.data_ptr()) { - *total_size += buffer.capacity(); - } // Otherwise the buffer's memory is already counted +/// Returns the `ArrayRef` children whose object allocations may be shared. +fn array_children(array: &ArrayRef) -> Vec<&ArrayRef> { + match array.data_type() { + DataType::Struct(_) => array.as_struct().columns().iter().collect(), + DataType::List(_) => vec![array.as_list::().values()], + DataType::LargeList(_) => vec![array.as_list::().values()], + DataType::ListView(_) => vec![array.as_list_view::().values()], + DataType::LargeListView(_) => vec![array.as_list_view::().values()], + DataType::FixedSizeList(_, _) => vec![array.as_fixed_size_list().values()], + DataType::Map(_, _) => { + let map = array.as_map(); + vec![map.keys(), map.values()] + } + DataType::Union(_, _) => { + let union = array.as_union(); + union + .fields() + .iter() + .map(|(type_id, _)| union.child(type_id)) + .collect() + } + DataType::Dictionary(key_type, _) => match key_type.as_ref() { + DataType::Int8 => vec![array.as_dictionary::().values()], + DataType::Int16 => vec![array.as_dictionary::().values()], + DataType::Int32 => vec![array.as_dictionary::().values()], + DataType::Int64 => vec![array.as_dictionary::().values()], + DataType::UInt8 => vec![array.as_dictionary::().values()], + DataType::UInt16 => vec![array.as_dictionary::().values()], + DataType::UInt32 => vec![array.as_dictionary::().values()], + DataType::UInt64 => vec![array.as_dictionary::().values()], + _ => unreachable!("invalid dictionary key type: {key_type}"), + }, + DataType::RunEndEncoded(run_ends, _) => match run_ends.data_type() { + DataType::Int16 => array + .as_any() + .downcast_ref::>() + .map(|array| vec![array.values()]) + .unwrap_or_default(), + DataType::Int32 => array + .as_any() + .downcast_ref::>() + .map(|array| vec![array.values()]) + .unwrap_or_default(), + DataType::Int64 => array + .as_any() + .downcast_ref::>() + .map(|array| vec![array.values()]) + .unwrap_or_default(), + _ => vec![], + }, + _ => vec![], } +} + +/// Tracks a small number of buffers inline, avoiding a heap allocation for +/// typical batches, and promotes to a hash set when more buffers are seen. +#[derive(Debug)] +struct BufferIdSet { + inline: [Option>; INLINE_BUFFER_IDS], + len: usize, + overflow: Option>>, +} - if let Some(null_buffer) = array_data.nulls() - && counted_buffers.insert(null_buffer.inner().inner().data_ptr()) - { - *total_size += null_buffer.inner().inner().capacity(); +impl Default for BufferIdSet { + fn default() -> Self { + Self { + inline: [None; INLINE_BUFFER_IDS], + len: 0, + overflow: None, + } } +} + +impl BufferIdSet { + fn insert(&mut self, buffer_id: NonZero) -> bool { + if let Some(overflow) = &mut self.overflow { + return overflow.insert(buffer_id); + } + + if self.inline[..self.len].contains(&Some(buffer_id)) { + return false; + } - // Count all children `ArrayData` recursively - for child in array_data.child_data() { - count_array_data_memory_size(child, counted_buffers, total_size); + if self.len < INLINE_BUFFER_IDS { + self.inline[self.len] = Some(buffer_id); + self.len += 1; + return true; + } + + let mut overflow = HashSet::with_capacity(INLINE_BUFFER_IDS + 1); + overflow.extend(self.inline.iter().flatten().copied()); + let inserted = overflow.insert(buffer_id); + self.overflow = Some(overflow); + inserted } } @@ -235,10 +536,51 @@ mod tests { #[cfg(test)] mod record_batch_tests { use super::*; - use arrow::array::{Float64Array, Int32Array, ListArray}; - use arrow::datatypes::{DataType, Field, Int32Type, Schema}; + use arrow::array::{ + ArrayData, ArrayRef, BinaryViewArray, DictionaryArray, Float64Array, Int16Array, + Int32Array, Int64Array, LargeListViewArray, ListArray, ListViewArray, MapArray, + RunArray, StringArray, StringViewArray, StructArray, UnionArray, new_null_array, + }; + use arrow::buffer::OffsetBuffer; + use arrow::datatypes::{ + DataType, Field, Fields, Int16Type, Int32Type, Int64Type, Schema, UnionFields, + UnionMode, + }; use std::sync::Arc; + fn array_data_memory_size(array: &dyn Array) -> usize { + fn count( + array_data: &ArrayData, + counted_buffers: &mut HashSet>, + total_size: &mut usize, + ) { + for buffer in array_data.buffers() { + if counted_buffers.insert(buffer.data_ptr().addr()) { + *total_size += buffer.capacity(); + } + } + if let Some(nulls) = array_data.nulls() { + let buffer = nulls.inner().inner(); + if counted_buffers.insert(buffer.data_ptr().addr()) { + *total_size += buffer.capacity(); + } + } + for child in array_data.child_data() { + count(child, counted_buffers, total_size); + } + } + + let mut total_size = 0; + count(&array.to_data(), &mut HashSet::default(), &mut total_size); + total_size + } + + fn assert_array_memory_size_matches(array: &dyn Array) { + let mut counter = RecordBatchMemoryCounter::new(); + counter.count_array_memory_size(array); + assert_eq!(counter.memory_usage(), array_data_memory_size(array)); + } + #[test] fn test_get_record_batch_memory_size() { let schema = Arc::new(Schema::new(vec![ @@ -325,28 +667,390 @@ mod record_batch_tests { } #[test] - fn test_get_record_batches_memory_size_shared_buffer() { - let original = Int32Array::from(vec![1, 2, 3, 4, 5]); - let slice1 = original.slice(0, 3); - let slice2 = original.slice(2, 3); + fn test_record_batch_memory_counter_array_overhead_shared_across_batches() { + let schema = Arc::new(Schema::new(vec![ + Field::new("ints", DataType::Int32, false), + Field::new("floats", DataType::Float64, false), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6])), + Arc::new(Float64Array::from(vec![1., 2., 3., 4., 5., 6.])), + ], + ) + .unwrap(); - let schema = - Arc::new(Schema::new(vec![Field::new("col", DataType::Int32, false)])); - let batch_origin = - RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(original)]).unwrap(); - let slices = [ - RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(slice1)]).unwrap(), - RecordBatch::try_new(schema, vec![Arc::new(slice2)]).unwrap(), - ]; + let mut counter = RecordBatchMemoryCounter::new(); + assert_eq!( + counter.count_batch_with_array_overhead(&batch), + batch.get_array_memory_size() + ); + assert_eq!(counter.count_batch_with_array_overhead(&batch), 0); + assert_eq!(counter.memory_usage(), batch.get_array_memory_size()); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_shared_nested_array_overhead() { + let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let fields = + Fields::from(vec![Arc::new(Field::new("value", DataType::Int32, false))]); + let first = Arc::new(StructArray::new( + fields.clone(), + vec![Arc::clone(&shared_child)], + None, + )) as _; + let second = Arc::new(StructArray::new( + fields, + vec![Arc::clone(&shared_child)], + None, + )) as _; + let batch = + RecordBatch::try_from_iter(vec![("first", first), ("second", second)]) + .unwrap(); + + let mut counter = RecordBatchMemoryCounter::new(); + counter.count_batch_with_array_overhead(&batch); - let size_origin = get_record_batch_memory_size(&batch_origin); - assert_eq!(get_record_batches_memory_size(&slices), size_origin); assert_eq!( - get_record_batches_release_sizes(&slices), - vec![0, size_origin] + counter.memory_usage(), + batch.get_array_memory_size() - shared_child.get_array_memory_size() ); } + fn assert_recursive_shared_child_memory( + name: &str, + first: ArrayRef, + second: ArrayRef, + shared_memory: usize, + ) { + let first_memory = first.get_array_memory_size(); + let second_memory = second.get_array_memory_size(); + let first_batch = RecordBatch::try_from_iter(vec![(name, first)]).unwrap(); + let second_batch = RecordBatch::try_from_iter(vec![(name, second)]).unwrap(); + let mut counter = RecordBatchMemoryCounter::new(); + + assert_eq!( + counter.count_batch_with_array_overhead(&first_batch), + first_memory, + "{name}: first batch" + ); + assert_eq!( + counter.count_batch_with_array_overhead(&second_batch), + second_memory - shared_memory, + "{name}: shared child" + ); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_shared_list_child() { + let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let list_field = Arc::new(Field::new_list_field(DataType::Int32, false)); + + let first = Arc::new(ListArray::new( + Arc::clone(&list_field), + OffsetBuffer::new(vec![0, 3].into()), + Arc::clone(&shared_child), + None, + )); + let second = Arc::new(ListArray::new( + list_field, + OffsetBuffer::new(vec![0, 3].into()), + Arc::clone(&shared_child), + None, + )); + + assert_recursive_shared_child_memory( + "list", + first, + second, + shared_child.get_array_memory_size(), + ); + } + + fn map_with_shared_children( + fields: &Fields, + shared_key: &ArrayRef, + shared_value: &ArrayRef, + ) -> ArrayRef { + Arc::new( + MapArray::try_new( + Arc::new(Field::new( + "entries", + DataType::Struct(fields.clone()), + false, + )), + OffsetBuffer::new(vec![0, 3].into()), + StructArray::new( + fields.clone(), + vec![Arc::clone(shared_key), Arc::clone(shared_value)], + None, + ), + None, + false, + ) + .unwrap(), + ) + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_shared_map_children() { + let shared_key: ArrayRef = Arc::new(Int32Array::from(vec![4, 5, 6])); + let shared_value: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let fields = Fields::from(vec![ + Arc::new(Field::new("key", DataType::Int32, false)), + Arc::new(Field::new("value", DataType::Int32, false)), + ]); + + let first = map_with_shared_children(&fields, &shared_key, &shared_value); + let second = map_with_shared_children(&fields, &shared_key, &shared_value); + + assert_recursive_shared_child_memory( + "map", + first, + second, + shared_key.get_array_memory_size() + shared_value.get_array_memory_size(), + ); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_union_slice_child_overhead() { + let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let fields: UnionFields = + std::iter::once((0, Arc::new(Field::new("value", DataType::Int32, false)))) + .collect(); + let first = Arc::new( + UnionArray::try_new( + fields, + vec![0, 0, 0].into(), + Some(vec![0, 1, 2].into()), + vec![Arc::clone(&shared_child)], + ) + .unwrap(), + ) as ArrayRef; + let second = Arc::new(first.as_union().slice(1, 2)) as ArrayRef; + let first_batch = + RecordBatch::try_from_iter(vec![("union", Arc::clone(&first))]).unwrap(); + let second_batch = + RecordBatch::try_from_iter(vec![("union", Arc::clone(&second))]).unwrap(); + let child_overhead = + shared_child.get_array_memory_size() - shared_child.get_buffer_memory_size(); + let second_parent_overhead = second.get_array_memory_size() + - second.get_buffer_memory_size() + - child_overhead; + let mut counter = RecordBatchMemoryCounter::new(); + + assert_eq!( + counter.count_batch_with_array_overhead(&first_batch), + first.get_array_memory_size() + ); + assert_eq!( + counter.count_batch_with_array_overhead(&second_batch), + second_parent_overhead + ); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_run_end_slice_child_overhead() { + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let first = Arc::new( + RunArray::::try_new( + &Int32Array::from(vec![1, 2, 3]), + values.as_ref(), + ) + .unwrap(), + ) as ArrayRef; + let second = Arc::new( + first + .as_any() + .downcast_ref::>() + .unwrap() + .slice(1, 2), + ) as ArrayRef; + let first_batch = + RecordBatch::try_from_iter(vec![("run", Arc::clone(&first))]).unwrap(); + let second_batch = + RecordBatch::try_from_iter(vec![("run", Arc::clone(&second))]).unwrap(); + let child_overhead = + values.get_array_memory_size() - values.get_buffer_memory_size(); + let second_parent_overhead = second.get_array_memory_size() + - second.get_buffer_memory_size() + - child_overhead; + let mut counter = RecordBatchMemoryCounter::new(); + + assert_eq!( + counter.count_batch_with_array_overhead(&first_batch), + first.get_array_memory_size() + ); + assert_eq!( + counter.count_batch_with_array_overhead(&second_batch), + second_parent_overhead + ); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_shared_dictionary_child() { + let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let make_dictionary = || { + Arc::new( + DictionaryArray::::try_new( + Int32Array::from(vec![0, 1, 2]), + Arc::clone(&shared_child), + ) + .unwrap(), + ) as ArrayRef + }; + + assert_recursive_shared_child_memory( + "dictionary", + make_dictionary(), + make_dictionary(), + shared_child.get_array_memory_size(), + ); + } + + #[test] + fn test_record_batch_memory_counter_buffer_shared_across_batches() { + let schema = Arc::new(Schema::new(vec![Field::new( + "ints", + DataType::Int32, + false, + )])); + + let int_array = Int32Array::from(vec![1, 2, 3, 4, 5, 6]); + let batch = RecordBatch::try_new(schema, vec![Arc::new(int_array)]).unwrap(); + let slices = [batch.slice(0, 2), batch.slice(2, 2), batch.slice(4, 2)]; + + // Counting each slice individually counts the shared buffer once per slice + let summed: usize = slices.iter().map(get_record_batch_memory_size).sum(); + assert_eq!(summed, 3 * get_record_batch_memory_size(&batch)); + + // A counter shared across the batches counts it exactly once + let mut counter = RecordBatchMemoryCounter::new(); + let deduped: usize = slices.iter().map(|slice| counter.count_batch(slice)).sum(); + assert_eq!(deduped, get_record_batch_memory_size(&batch)); + assert_eq!(counter.memory_usage(), get_record_batch_memory_size(&batch)); + } + + #[test] + fn test_record_batch_memory_counter_promotes_buffer_set() { + let fields = (0..=INLINE_BUFFER_IDS) + .map(|index| Field::new(format!("col_{index}"), DataType::Int32, false)) + .collect::>(); + let columns = (0..=INLINE_BUFFER_IDS) + .map(|value| Arc::new(Int32Array::from(vec![value as i32])) as _) + .collect::>(); + let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap(); + + let mut counter = RecordBatchMemoryCounter::new(); + assert_eq!( + counter.count_batch(&batch), + (INLINE_BUFFER_IDS + 1) * size_of::() + ); + assert!(counter.counted_buffers.overflow.is_some()); + assert_eq!(counter.count_batch(&batch), 0); + } + + #[test] + fn test_array_memory_size_matches_array_data_layouts() { + let list_field = Arc::new(Field::new_list_field(DataType::Int32, true)); + let struct_fields = vec![Field::new("value", DataType::Int32, true)].into(); + let union_fields = UnionFields::try_new( + vec![0], + vec![Field::new("value", DataType::Int32, true)], + ) + .unwrap(); + let map_entries = Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int32, true), + ] + .into(), + ), + false, + )); + let data_types = vec![ + DataType::Boolean, + DataType::Int32, + DataType::Binary, + DataType::LargeBinary, + DataType::FixedSizeBinary(4), + DataType::BinaryView, + DataType::Utf8, + DataType::LargeUtf8, + DataType::Utf8View, + DataType::List(Arc::clone(&list_field)), + DataType::LargeList(Arc::clone(&list_field)), + DataType::ListView(Arc::clone(&list_field)), + DataType::LargeListView(Arc::clone(&list_field)), + DataType::FixedSizeList(Arc::clone(&list_field), 2), + DataType::Struct(struct_fields), + DataType::Union(union_fields, UnionMode::Dense), + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + DataType::Map(map_entries, false), + ]; + + for data_type in data_types { + let array = new_null_array(&data_type, 3); + assert_array_memory_size_matches(array.as_ref()); + } + + // Exercise the view-specific buffers with concrete, non-empty values. + let view_arrays = [ + Arc::new(BinaryViewArray::from_iter_values([ + b"short".as_slice(), + b"a payload longer than twelve bytes".as_slice(), + ])) as ArrayRef, + Arc::new(StringViewArray::from_iter_values([ + "short", + "a payload longer than twelve bytes", + ])) as ArrayRef, + Arc::new(ListViewArray::from_iter_primitive::([ + Some(vec![Some(1), Some(2)]), + None, + Some(vec![Some(3)]), + ])) as ArrayRef, + Arc::new(LargeListViewArray::from_iter_primitive::( + [Some(vec![Some(1), Some(2)]), None, Some(vec![Some(3)])], + )) as ArrayRef, + ]; + + for array in view_arrays { + assert_array_memory_size_matches(array.as_ref()); + } + + let run_values = StringArray::from(vec!["alpha", "beta"]); + let run_arrays = [ + Arc::new( + RunArray::::try_new( + &Int16Array::from(vec![2_i16, 5]), + &run_values, + ) + .unwrap(), + ) as ArrayRef, + Arc::new( + RunArray::::try_new( + &Int32Array::from(vec![2_i32, 5]), + &run_values, + ) + .unwrap(), + ) as ArrayRef, + Arc::new( + RunArray::::try_new( + &Int64Array::from(vec![2_i64, 5]), + &run_values, + ) + .unwrap(), + ) as ArrayRef, + ]; + + for array in run_arrays { + assert_array_memory_size_matches(array.as_ref()); + } + } + #[test] fn test_get_record_batch_memory_size_nested_array() { let schema = Arc::new(Schema::new(vec![ From 3d201a2c1fe0f5ccb5e21ce54b50fab122af2f45 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 24 Sep 2026 12:52:19 +0200 Subject: [PATCH 7/7] fix(physical-plan): size chunked sort output with RecordBatchMemoryCounter Walk the chunks in reverse so each buffer is charged to the last chunk that references it; the counter total is the reservation and the per-chunk counts are what each emitted chunk releases. --- datafusion/physical-plan/src/sorts/sort.rs | 23 +++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index 7ed1f8da412c8..37a383aa3240a 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -49,7 +49,7 @@ use crate::{ ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream, Statistics, }; -use datafusion_common::utils::memory::get_record_batches_release_sizes; +use datafusion_common::utils::memory::RecordBatchMemoryCounter; use arrow::array::{Array, RecordBatch, RecordBatchOptions, StringViewArray}; use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays}; @@ -732,9 +732,18 @@ impl ExternalSorter { // Using try_resize avoids a release-then-reacquire cycle, which // matters for MemoryPool implementations where grow/shrink have // non-trivial cost (e.g. JNI calls in Comet). - let release_sizes = get_record_batches_release_sizes(&sorted_batches); + // Chunks share buffers (e.g. dictionary values after `take`), so + // count each buffer once and release it with the last chunk that + // references it. + let mut counter = RecordBatchMemoryCounter::new(); + let mut release_sizes: Vec = sorted_batches + .iter() + .rev() + .map(|batch| counter.count_batch(batch)) + .collect(); + release_sizes.reverse(); reservation - .try_resize(release_sizes.iter().sum()) + .try_resize(counter.memory_usage()) .map_err(Self::err_with_oom_context)?; let batches = sorted_batches.into_iter().zip(release_sizes).map( @@ -1454,7 +1463,6 @@ mod tests { use datafusion_common::cast::as_primitive_array; use datafusion_common::config::ConfigOptions; use datafusion_common::test_util::batches_to_string; - use datafusion_common::utils::memory::get_record_batches_memory_size; use datafusion_common::{DataFusionError, Result, ScalarValue}; use datafusion_execution::RecordBatchStream; use datafusion_execution::config::SessionConfig; @@ -1797,6 +1805,11 @@ mod tests { Ok((sort_exec, task_ctx)) } + fn batches_memory_size(batches: &[RecordBatch]) -> usize { + let mut counter = RecordBatchMemoryCounter::new(); + batches.iter().map(|batch| counter.count_batch(batch)).sum() + } + #[tokio::test] async fn test_sort_single_batch_shared_dictionary_reservation() -> Result<()> { const ROWS: usize = 100_000; @@ -1828,7 +1841,7 @@ mod tests { drop(stream.next().await.unwrap()?); assert_eq!( task_ctx.memory_pool().reserved(), - get_record_batches_memory_size(&expected[emitted..]), + batches_memory_size(&expected[emitted..]), "after {emitted} of {} batches", expected.len() );