Skip to content
Merged
79 changes: 39 additions & 40 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

20 changes: 10 additions & 10 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -90,31 +90,31 @@ version = "55.1.0"
#
# See for more details: https://github.com/rust-lang/cargo/issues/11329
apache-avro = { version = "0.21", default-features = false }
arrow = { version = "60.0.0", features = [
arrow = { version = "60.1.0", features = [
"prettyprint",
"chrono-tz",
] }
arrow-avro = { version = "60.0.0", default-features = false, features = [
arrow-avro = { version = "60.1.0", default-features = false, features = [
"deflate",
"snappy",
"zstd",
"bzip2",
"xz",
] }
arrow-buffer = { version = "60.0.0", default-features = false }
arrow-data = { version = "60.0.0", default-features = false }
arrow-flight = { version = "60.0.0", features = [
arrow-buffer = { version = "60.1.0", default-features = false }
arrow-data = { version = "60.1.0", default-features = false }
arrow-flight = { version = "60.1.0", features = [
"flight-sql",
] }
# Both codecs are required here to make sure that code paths like
# file-spilling have access to all compression codecs.
arrow-ipc = { version = "60.0.0", default-features = false, features = [
arrow-ipc = { version = "60.1.0", default-features = false, features = [
"lz4",
"zstd",
] }
arrow-ord = { version = "60.0.0", default-features = false }
arrow-schema = { version = "60.0.0", default-features = false }
arrow-select = { version = "60.0.0", default-features = false }
arrow-ord = { version = "60.1.0", default-features = false }
arrow-schema = { version = "60.1.0", default-features = false }
arrow-select = { version = "60.1.0", default-features = false }
async-trait = "0.1.89"
bigdecimal = "0.4.8"
bytes = "1.11"
Expand Down Expand Up @@ -181,7 +181,7 @@ memchr = "2.8.1"
num-traits = { version = "0.2" }
object_store = { version = "0.14.2", default-features = false }
parking_lot = "0.13"
parquet = { version = "60.0.0", default-features = false, features = [
parquet = { version = "60.1.0", default-features = false, features = [
"arrow",
"async",
"object_store",
Expand Down
25 changes: 11 additions & 14 deletions datafusion/physical-plan/src/spill/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -819,8 +819,8 @@ fn gc_array(array: &ArrayRef) -> Result<(ArrayRef, bool)> {
/// The IPC writer writes every data buffer of an array in full, once per
/// entry, and `gc()` copies the bytes of every view separately. Neither
/// handles views that share bytes, as the views of a dictionary-encoded
/// Parquet column do: after `concat` or `interleave` the dictionary is often
/// listed once per input batch, and `gc()` copies it once per row
/// Parquet column do: after `concat` the dictionary is listed once per input
/// batch, and `gc()` copies it once per row
/// (<https://github.com/apache/datafusion/issues/23564>).
///
/// Repeated buffer entries are merged first. `gc()` then runs only if it
Expand Down Expand Up @@ -1985,15 +1985,15 @@ mod tests {
// After appending each batch, spilled_rows and spilled_bytes should increase incrementally,
// while spill_file_count remains 1 (since we're writing to the same file)
in_progress_file.append_batch(&batch1)?;
verify_metrics(&in_progress_file, 1, 440, 3)?;
verify_metrics(&in_progress_file, 1, 424, 3)?;

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

size is smaller 🎉


in_progress_file.append_batch(&batch2)?;
verify_metrics(&in_progress_file, 1, 704, 6)?;
verify_metrics(&in_progress_file, 1, 672, 6)?;

let completed_file = in_progress_file.finish()?;
assert!(completed_file.is_some());
assert!(completed_file.unwrap().path().unwrap().exists());
verify_metrics(&in_progress_file, 1, 712, 6)?;
verify_metrics(&in_progress_file, 1, 680, 6)?;
// Double finish produce error
let result = in_progress_file.finish();
assert!(result.is_err());
Expand Down Expand Up @@ -2128,7 +2128,7 @@ mod tests {

// Metrics should be updated immediately (at least schema and first batch)
let bytes_after_batch1 = metrics.spilled_bytes.value();
assert_eq!(bytes_after_batch1, 440);
assert_eq!(bytes_after_batch1, 424);
assert_eq!(metrics.spill_file_count.value(), 1);

// Check global progress
Expand Down Expand Up @@ -2493,16 +2493,16 @@ mod tests {
}

/// Arrays that each hold views into one shared dictionary buffer, as the
/// Parquet reader produces for a dictionary-encoded column, combined the
/// way a sort combines its input batches. Each output row cycles through
/// Parquet reader produces for a dictionary-encoded column, combined with
/// `concat` as a sort combines its buffered batches. The rows cycle through

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

interleave no longer duplicates the buffers

/// the dictionary (<https://github.com/apache/datafusion/issues/23564>).
fn shared_dictionary_views(
dictionary_len: u32,
inputs: u32,
rows_per_input: u32,
) -> Result<(StringViewArray, StringViewArray)> {
use arrow::array::{AsArray, UInt32Array};
use arrow::compute::{interleave, take};
use arrow::compute::{concat, take};

let dictionary = dictionary_array(dictionary_len)?;
let arrays = (0..inputs)
Expand All @@ -2515,17 +2515,14 @@ mod tests {
})
.collect::<std::result::Result<Vec<_>, _>>()?;
let arrays: Vec<&dyn Array> = arrays.iter().map(|a| a.as_ref()).collect();
let indices: Vec<(usize, usize)> = (0..rows_per_input as usize)
.flat_map(|row| (0..inputs as usize).map(move |input| (input, row)))
.collect();
let merged = interleave(&arrays, &indices)?;
let merged = concat(&arrays)?;
Ok((dictionary, merged.as_string_view().clone()))
}

#[test]
fn test_gc_keeps_shared_dictionary_buffer() -> Result<()> {
let (dictionary, merged) = shared_dictionary_views(1000, 8, 1024)?;
// `interleave` lists the dictionary once per input
// `concat` lists the dictionary once per input
assert_eq!(merged.data_buffers().len(), 8);
// `gc()` would copy 8192 values of 63 bytes, 8x the dictionary
assert!(merged.total_buffer_bytes_used() > data_buffers_size(&dictionary));
Expand Down
4 changes: 2 additions & 2 deletions datafusion/physical-plan/src/spill/spill_pool.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1005,7 +1005,7 @@ mod tests {
);
assert_eq!(
metrics.spilled_bytes.value(),
320,
312,
"Spilled bytes should reflect data written (header + 1 batch)"
);
assert_eq!(
Expand Down Expand Up @@ -1480,7 +1480,7 @@ mod tests {
// Check metrics before drop - spilled_bytes already reflects written data
let spilled_bytes_before = metrics.spilled_bytes.value();
assert_eq!(
spilled_bytes_before, 1088,
spilled_bytes_before, 1048,
"Spilled bytes should reflect data written (header + 5 batches)"
);

Expand Down
Loading
Loading