From f7abe4ed8782fa5267c0df88b0cb55f1e2e05f34 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 24 Jun 2025 14:54:00 +0200 Subject: [PATCH 01/25] chore: skip order calculation / exponential planning --- .../src/equivalence/properties/union.rs | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/datafusion/physical-expr/src/equivalence/properties/union.rs b/datafusion/physical-expr/src/equivalence/properties/union.rs index d77129472a8ba..702688011668d 100644 --- a/datafusion/physical-expr/src/equivalence/properties/union.rs +++ b/datafusion/physical-expr/src/equivalence/properties/union.rs @@ -67,16 +67,43 @@ fn calculate_union_binary( }) .collect::>(); + // TEMP HACK WORKAROUND + // Revert code from https://github.com/apache/datafusion/pull/12562 + // Context: https://github.com/apache/datafusion/issues/13748 + // Context: https://github.com/influxdata/influxdb_iox/issues/13038 + // Next, calculate valid orderings for the union by searching for prefixes // in both sides. - let mut orderings = UnionEquivalentOrderingBuilder::new(); - orderings.add_satisfied_orderings(&lhs, &rhs)?; - orderings.add_satisfied_orderings(&rhs, &lhs)?; - let orderings = orderings.build(); + let mut orderings = vec![]; + for ordering in lhs.normalized_oeq_class().into_iter() { + let mut ordering: Vec = ordering.into(); + + // Progressively shorten the ordering to search for a satisfied prefix: + while !rhs.ordering_satisfy(ordering.clone())? { + ordering.pop(); + } + // There is a non-trivial satisfied prefix, add it as a valid ordering: + if !ordering.is_empty() { + orderings.push(ordering); + } + } + for ordering in rhs.normalized_oeq_class().into_iter() { + let mut ordering: Vec = ordering.into(); + + // Progressively shorten the ordering to search for a satisfied prefix: + while !lhs.ordering_satisfy(ordering.clone())? { + ordering.pop(); + } + // There is a non-trivial satisfied prefix, add it as a valid ordering: + if !ordering.is_empty() { + orderings.push(ordering); + } + } let mut eq_properties = EquivalenceProperties::new(lhs.schema); eq_properties.add_constants(constants)?; eq_properties.add_orderings(orderings); + Ok(eq_properties) } @@ -122,6 +149,7 @@ struct UnionEquivalentOrderingBuilder { orderings: Vec, } +#[expect(unused)] impl UnionEquivalentOrderingBuilder { fn new() -> Self { Self { orderings: vec![] } @@ -504,6 +532,7 @@ mod tests { } #[test] + #[ignore = "InfluxData patch: chore: skip order calculation / exponential planning"] fn test_union_equivalence_properties_constants_fill_gaps() -> Result<()> { let schema = create_test_schema().unwrap(); UnionEquivalenceTest::new(&schema) @@ -579,6 +608,7 @@ mod tests { } #[test] + #[ignore = "InfluxData patch: chore: skip order calculation / exponential planning"] fn test_union_equivalence_properties_constants_fill_gaps_non_symmetric() -> Result<()> { let schema = create_test_schema().unwrap(); @@ -607,6 +637,7 @@ mod tests { } #[test] + #[ignore = "InfluxData patch: chore: skip order calculation / exponential planning"] fn test_union_equivalence_properties_constants_gap_fill_symmetric() -> Result<()> { let schema = create_test_schema().unwrap(); UnionEquivalenceTest::new(&schema) @@ -658,6 +689,7 @@ mod tests { } #[test] + #[ignore = "InfluxData patch: chore: skip order calculation / exponential planning"] fn test_union_equivalence_properties_constants_middle_desc() -> Result<()> { let schema = create_test_schema().unwrap(); UnionEquivalenceTest::new(&schema) From 0febb6e2da8dae89e84731a6b363db29f9d40344 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Tue, 16 Jul 2024 12:14:19 -0400 Subject: [PATCH 02/25] (New) Test + workaround for SanityCheck plan --- datafusion/physical-optimizer/src/sanity_checker.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/datafusion/physical-optimizer/src/sanity_checker.rs b/datafusion/physical-optimizer/src/sanity_checker.rs index bff33a281556d..8eb00327143dc 100644 --- a/datafusion/physical-optimizer/src/sanity_checker.rs +++ b/datafusion/physical-optimizer/src/sanity_checker.rs @@ -32,6 +32,8 @@ use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion_physical_expr::intervals::utils::{check_support, is_datatype_supported}; use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion_physical_plan::joins::SymmetricHashJoinExec; +use datafusion_physical_plan::sorts::sort::SortExec; +use datafusion_physical_plan::union::UnionExec; use datafusion_physical_plan::{ExecutionPlanProperties, get_plan_string}; use crate::PhysicalOptimizerRule; @@ -136,6 +138,14 @@ pub fn check_plan_sanity( plan.required_input_ordering(), plan.required_input_distribution(), ) { + // TEMP HACK WORKAROUND https://github.com/apache/datafusion/issues/11492 + if child.as_any().downcast_ref::().is_some() { + continue; + } + if child.as_any().downcast_ref::().is_some() { + continue; + } + let child_eq_props = child.equivalence_properties(); if let Some(sort_req) = sort_req { let sort_req = sort_req.into_single(); From 3152337c316746b38c303ff2cab1ddf4192f5b62 Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Tue, 21 Oct 2025 13:33:16 -0400 Subject: [PATCH 03/25] chore: add debug logging and skip error on physical schema check --- datafusion/core/src/physical_planner.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index 12406b6c29dd5..9cd02fe5e6e79 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -970,10 +970,17 @@ impl DefaultPhysicalPlanner { )); } } - return internal_err!( + debug!( "Physical input schema should be the same as the one converted from logical input schema. Differences: {}", differences.iter().map(|s| format!("\n\t- {s}")).join("") ); + + //influx: temporarily remove error and only log so that we can find a + //reproducer in production + // return internal_err!("Physical input schema should be the same as the one converted from logical input schema. Differences: {}", differences + // .iter() + // .map(|s| format!("\n\t- {s}")) + // .join("")); } let groups = self.create_grouping_physical_expr( @@ -4501,6 +4508,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_metadata() { let logical_schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); @@ -4521,6 +4530,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_field_count() { let logical_schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); @@ -4541,6 +4552,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_field_name() { let logical_schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); @@ -4562,6 +4575,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_field_type() { let logical_schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); @@ -4580,6 +4595,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_field_nullability() { let logical_schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); @@ -4598,6 +4615,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_field_metadata() { let logical_schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); @@ -4618,6 +4637,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_multiple() { let logical_schema = Arc::new(Schema::new(vec![ Field::new("c1", DataType::Int32, false), From c9b795996b4dd32557e08faf43260cc759ecfa4c Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Tue, 16 Dec 2025 12:21:06 -0500 Subject: [PATCH 04/25] fix: wrap join operators with cooperative() for cancellation support --- datafusion/physical-plan/src/joins/cross_join.rs | 9 +++++---- datafusion/physical-plan/src/joins/hash_join/exec.rs | 5 +++-- datafusion/physical-plan/src/joins/nested_loop_join.rs | 5 +++-- .../physical-plan/src/joins/sort_merge_join/exec.rs | 5 +++-- .../physical-plan/src/joins/symmetric_hash_join.rs | 9 +++++---- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs index 342cb7e70a78b..b5ce14d7b7c06 100644 --- a/datafusion/physical-plan/src/joins/cross_join.rs +++ b/datafusion/physical-plan/src/joins/cross_join.rs @@ -25,6 +25,7 @@ use super::utils::{ OnceAsync, OnceFut, StatefulStreamResult, adjust_right_output_partitioning, reorder_output_after_swap, }; +use crate::coop::cooperative; use crate::execution_plan::{EmissionType, boundedness_from_children}; use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet}; use crate::projection::{ @@ -350,7 +351,7 @@ impl ExecutionPlan for CrossJoinExec { })?; if enforce_batch_size_in_joins { - Ok(Box::pin(CrossJoinStream { + Ok(Box::pin(cooperative(CrossJoinStream { schema: Arc::clone(&self.schema), left_fut, right: stream, @@ -359,9 +360,9 @@ impl ExecutionPlan for CrossJoinExec { state: CrossJoinStreamState::WaitBuildSide, left_data: RecordBatch::new_empty(self.left().schema()), batch_transformer: BatchSplitter::new(batch_size), - })) + }))) } else { - Ok(Box::pin(CrossJoinStream { + Ok(Box::pin(cooperative(CrossJoinStream { schema: Arc::clone(&self.schema), left_fut, right: stream, @@ -370,7 +371,7 @@ impl ExecutionPlan for CrossJoinExec { state: CrossJoinStreamState::WaitBuildSide, left_data: RecordBatch::new_empty(self.left().schema()), batch_transformer: NoopBatchTransformer::new(), - })) + }))) } } diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 25b320f985507..a10571315839e 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -23,6 +23,7 @@ use std::sync::{Arc, OnceLock}; use std::{any::Any, vec}; use crate::ExecutionPlanProperties; +use crate::coop::cooperative; use crate::execution_plan::{ EmissionType, boundedness_from_children, has_same_children_properties, stub_properties, @@ -1393,7 +1394,7 @@ impl ExecutionPlan for HashJoinExec { .map(|(_, right_expr)| Arc::clone(right_expr)) .collect::>(); - Ok(Box::pin(HashJoinStream::new( + Ok(Box::pin(cooperative(HashJoinStream::new( partition, self.schema(), on_right, @@ -1413,7 +1414,7 @@ impl ExecutionPlan for HashJoinExec { self.mode, self.null_aware, self.fetch, - ))) + )))) } fn metrics(&self) -> Option { diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs index 4fb7dabf673d4..8521649b2d96d 100644 --- a/datafusion/physical-plan/src/joins/nested_loop_join.rs +++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs @@ -29,6 +29,7 @@ use super::utils::{ reorder_output_after_swap, swap_join_projection, }; use crate::common::can_project; +use crate::coop::cooperative; use crate::execution_plan::{EmissionType, boundedness_from_children}; use crate::joins::SharedBitmapBuilder; use crate::joins::utils::{ @@ -618,7 +619,7 @@ impl ExecutionPlan for NestedLoopJoinExec { None => self.column_indices.clone(), }; - Ok(Box::pin(NestedLoopJoinStream::new( + Ok(Box::pin(cooperative(NestedLoopJoinStream::new( self.schema(), self.filter.clone(), self.join_type, @@ -627,7 +628,7 @@ impl ExecutionPlan for NestedLoopJoinExec { column_indices_after_projection, metrics, batch_size, - ))) + )))) } fn metrics(&self) -> Option { diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs index b34e811f91920..7e568d87f1494 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs @@ -23,6 +23,7 @@ use std::any::Any; use std::fmt::Formatter; use std::sync::Arc; +use crate::coop::cooperative; use crate::execution_plan::{EmissionType, boundedness_from_children}; use crate::expressions::PhysicalSortExpr; use crate::joins::sort_merge_join::metrics::SortMergeJoinMetrics; @@ -512,7 +513,7 @@ impl ExecutionPlan for SortMergeJoinExec { .register(context.memory_pool()); // create join stream - Ok(Box::pin(SortMergeJoinStream::try_new( + Ok(Box::pin(cooperative(SortMergeJoinStream::try_new( context.session_config().spill_compression(), Arc::clone(&self.schema), self.sort_options.clone(), @@ -527,7 +528,7 @@ impl ExecutionPlan for SortMergeJoinExec { SortMergeJoinMetrics::new(partition, &self.metrics), reservation, context.runtime_env(), - )?)) + )?))) } fn metrics(&self) -> Option { diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs index 7407b05ea5695..d726a4d140b2c 100644 --- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs +++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs @@ -34,6 +34,7 @@ use std::vec; use crate::check_if_same_properties; use crate::common::SharedMemoryReservation; +use crate::coop::cooperative; use crate::execution_plan::{boundedness_from_children, emission_type_from_children}; use crate::joins::stream_join_utils::{ PruningJoinHashMap, SortedFilterExpr, StreamJoinMetrics, @@ -545,7 +546,7 @@ impl ExecutionPlan for SymmetricHashJoinExec { } if enforce_batch_size_in_joins { - Ok(Box::pin(SymmetricHashJoinStream { + Ok(Box::pin(cooperative(SymmetricHashJoinStream { left_stream, right_stream, schema: self.schema(), @@ -563,9 +564,9 @@ impl ExecutionPlan for SymmetricHashJoinExec { state: SHJStreamState::PullRight, reservation, batch_transformer: BatchSplitter::new(batch_size), - })) + }))) } else { - Ok(Box::pin(SymmetricHashJoinStream { + Ok(Box::pin(cooperative(SymmetricHashJoinStream { left_stream, right_stream, schema: self.schema(), @@ -583,7 +584,7 @@ impl ExecutionPlan for SymmetricHashJoinExec { state: SHJStreamState::PullRight, reservation, batch_transformer: NoopBatchTransformer::new(), - })) + }))) } } From 9bca4652a590f48c9b3edba4d62a6d5cf045da1b Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Mon, 9 Mar 2026 16:34:43 -0400 Subject: [PATCH 05/25] Revert "Respect execution timezone in to_timestamp and related functions (#19078)" This reverts commit ada0923a3927d16dc5d810637cfbea4146c54f54. --- Cargo.lock | 1 - datafusion/functions/Cargo.toml | 3 +- datafusion/functions/benches/to_timestamp.rs | 24 +- datafusion/functions/src/datetime/common.rs | 153 +--- datafusion/functions/src/datetime/mod.rs | 46 +- datafusion/functions/src/datetime/to_date.rs | 6 +- .../functions/src/datetime/to_timestamp.rs | 855 ++++-------------- .../functions/src/datetime/to_unixtime.rs | 2 +- datafusion/functions/src/macros.rs | 29 - .../test_files/datetime/timestamps.slt | 12 +- .../test_files/to_timestamp_timezone.slt | 204 ----- .../source/user-guide/sql/scalar_functions.md | 84 +- 12 files changed, 236 insertions(+), 1183 deletions(-) delete mode 100644 datafusion/sqllogictest/test_files/to_timestamp_timezone.slt diff --git a/Cargo.lock b/Cargo.lock index 5a85136554ce2..9f9145db4f18f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2246,7 +2246,6 @@ dependencies = [ "blake2", "blake3", "chrono", - "chrono-tz", "criterion", "ctor", "datafusion-common", diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.toml index 1940f1378b635..56f3148396f47 100644 --- a/datafusion/functions/Cargo.toml +++ b/datafusion/functions/Cargo.toml @@ -40,7 +40,7 @@ workspace = true [features] crypto_expressions = ["md-5", "sha2", "blake2", "blake3"] # enable datetime functions -datetime_expressions = ["chrono-tz"] +datetime_expressions = [] # Enable encoding by default so the doctests work. In general don't automatically enable all packages. default = [ "datetime_expressions", @@ -71,7 +71,6 @@ base64 = { version = "0.22", optional = true } blake2 = { version = "^0.10.2", optional = true } blake3 = { version = "1.8", optional = true } chrono = { workspace = true } -chrono-tz = { version = "0.10.4", optional = true } datafusion-common = { workspace = true } datafusion-doc = { workspace = true } datafusion-execution = { workspace = true } diff --git a/datafusion/functions/benches/to_timestamp.rs b/datafusion/functions/benches/to_timestamp.rs index 90ea145d5d2c0..d4e9bccc25c12 100644 --- a/datafusion/functions/benches/to_timestamp.rs +++ b/datafusion/functions/benches/to_timestamp.rs @@ -112,21 +112,16 @@ fn criterion_benchmark(c: &mut Criterion) { Field::new("f", DataType::Timestamp(TimeUnit::Nanosecond, None), true).into(); let arg_field = Field::new("a", DataType::Utf8, false).into(); let arg_fields = vec![arg_field]; - let mut options = ConfigOptions::default(); - options.execution.time_zone = Some("UTC".into()); - let config_options = Arc::new(options); - - let to_timestamp_udf = to_timestamp(config_options.as_ref()); + let config_options = Arc::new(ConfigOptions::default()); c.bench_function("to_timestamp_no_formats_utf8", |b| { - let to_timestamp_udf = Arc::clone(&to_timestamp_udf); let arr_data = data(); let batch_len = arr_data.len(); let string_array = ColumnarValue::Array(Arc::new(arr_data) as ArrayRef); b.iter(|| { black_box( - to_timestamp_udf + to_timestamp() .invoke_with_args(ScalarFunctionArgs { args: vec![string_array.clone()], arg_fields: arg_fields.clone(), @@ -140,14 +135,13 @@ fn criterion_benchmark(c: &mut Criterion) { }); c.bench_function("to_timestamp_no_formats_largeutf8", |b| { - let to_timestamp_udf = Arc::clone(&to_timestamp_udf); let data = cast(&data(), &DataType::LargeUtf8).unwrap(); let batch_len = data.len(); let string_array = ColumnarValue::Array(Arc::new(data) as ArrayRef); b.iter(|| { black_box( - to_timestamp_udf + to_timestamp() .invoke_with_args(ScalarFunctionArgs { args: vec![string_array.clone()], arg_fields: arg_fields.clone(), @@ -161,14 +155,13 @@ fn criterion_benchmark(c: &mut Criterion) { }); c.bench_function("to_timestamp_no_formats_utf8view", |b| { - let to_timestamp_udf = Arc::clone(&to_timestamp_udf); let data = cast(&data(), &DataType::Utf8View).unwrap(); let batch_len = data.len(); let string_array = ColumnarValue::Array(Arc::new(data) as ArrayRef); b.iter(|| { black_box( - to_timestamp_udf + to_timestamp() .invoke_with_args(ScalarFunctionArgs { args: vec![string_array.clone()], arg_fields: arg_fields.clone(), @@ -182,7 +175,6 @@ fn criterion_benchmark(c: &mut Criterion) { }); c.bench_function("to_timestamp_with_formats_utf8", |b| { - let to_timestamp_udf = Arc::clone(&to_timestamp_udf); let (inputs, format1, format2, format3) = data_with_formats(); let batch_len = inputs.len(); @@ -202,7 +194,7 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| { black_box( - to_timestamp_udf + to_timestamp() .invoke_with_args(ScalarFunctionArgs { args: args.clone(), arg_fields: arg_fields.clone(), @@ -216,7 +208,6 @@ fn criterion_benchmark(c: &mut Criterion) { }); c.bench_function("to_timestamp_with_formats_largeutf8", |b| { - let to_timestamp_udf = Arc::clone(&to_timestamp_udf); let (inputs, format1, format2, format3) = data_with_formats(); let batch_len = inputs.len(); @@ -244,7 +235,7 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| { black_box( - to_timestamp_udf + to_timestamp() .invoke_with_args(ScalarFunctionArgs { args: args.clone(), arg_fields: arg_fields.clone(), @@ -258,7 +249,6 @@ fn criterion_benchmark(c: &mut Criterion) { }); c.bench_function("to_timestamp_with_formats_utf8view", |b| { - let to_timestamp_udf = Arc::clone(&to_timestamp_udf); let (inputs, format1, format2, format3) = data_with_formats(); let batch_len = inputs.len(); @@ -287,7 +277,7 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| { black_box( - to_timestamp_udf + to_timestamp() .invoke_with_args(ScalarFunctionArgs { args: args.clone(), arg_fields: arg_fields.clone(), diff --git a/datafusion/functions/src/datetime/common.rs b/datafusion/functions/src/datetime/common.rs index 2db64beafa9b7..5b152081b64ca 100644 --- a/datafusion/functions/src/datetime/common.rs +++ b/datafusion/functions/src/datetime/common.rs @@ -15,57 +15,31 @@ // specific language governing permissions and limitations // under the License. -use std::sync::{Arc, LazyLock}; +use std::sync::Arc; -use arrow::array::timezone::Tz; use arrow::array::{ Array, ArrowPrimitiveType, AsArray, GenericStringArray, PrimitiveArray, StringArrayType, StringViewArray, }; -use arrow::compute::DecimalCast; -use arrow::compute::kernels::cast_utils::string_to_datetime; -use arrow::datatypes::{DataType, TimeUnit}; -use arrow_buffer::ArrowNativeType; +use arrow::compute::kernels::cast_utils::string_to_timestamp_nanos; +use arrow::datatypes::DataType; use chrono::LocalResult::Single; use chrono::format::{Parsed, StrftimeItems, parse}; use chrono::{DateTime, TimeZone, Utc}; + use datafusion_common::cast::as_generic_string_array; use datafusion_common::{ - DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, - internal_datafusion_err, unwrap_or_internal_err, + DataFusionError, Result, ScalarType, ScalarValue, exec_datafusion_err, exec_err, + unwrap_or_internal_err, }; use datafusion_expr::ColumnarValue; /// Error message if nanosecond conversion request beyond supported interval const ERR_NANOSECONDS_NOT_SUPPORTED: &str = "The dates that can be represented as nanoseconds have to be between 1677-09-21T00:12:44.0 and 2262-04-11T23:47:16.854775804"; -static UTC: LazyLock = LazyLock::new(|| "UTC".parse().expect("UTC is always valid")); - -/// Converts a string representation of a date‑time into a timestamp expressed in -/// nanoseconds since the Unix epoch. -/// -/// This helper is a thin wrapper around the more general `string_to_datetime` -/// function. It accepts an optional `timezone` which, if `None`, defaults to -/// Coordinated Universal Time (UTC). The string `s` must contain a valid -/// date‑time format that can be parsed by the underlying chrono parser. -/// -/// # Return Value -/// -/// * `Ok(i64)` – The number of nanoseconds since `1970‑01‑01T00:00:00Z`. -/// * `Err(DataFusionError)` – If the string cannot be parsed, the parsed -/// value is out of range (between 1677-09-21T00:12:44.0 and 2262-04-11T23:47:16.854775804) -/// or the parsed value does not correspond to an unambiguous time. -pub(crate) fn string_to_timestamp_nanos_with_timezone( - timezone: &Option, - s: &str, -) -> Result { - let tz = timezone.as_ref().unwrap_or(&UTC); - let dt = string_to_datetime(tz, s)?; - let parsed = dt - .timestamp_nanos_opt() - .ok_or_else(|| exec_datafusion_err!("{ERR_NANOSECONDS_NOT_SUPPORTED}"))?; - - Ok(parsed) +/// Calls string_to_timestamp_nanos and converts the error type +pub(crate) fn string_to_timestamp_nanos_shim(s: &str) -> Result { + string_to_timestamp_nanos(s).map_err(|e| e.into()) } /// Checks that all the arguments from the second are of type [Utf8], [LargeUtf8] or [Utf8View] @@ -95,12 +69,13 @@ pub(crate) fn validate_data_types(args: &[ColumnarValue], name: &str) -> Result< /// Accepts a string and parses it using the [`chrono::format::strftime`] specifiers /// relative to the provided `timezone` /// +/// [IANA timezones] are only supported if the `arrow-array/chrono-tz` feature is enabled +/// +/// * `2023-01-01 040506 America/Los_Angeles` +/// /// If a timestamp is ambiguous, for example as a result of daylight-savings time, an error /// will be returned /// -/// Note that parsing [IANA timezones] is not supported yet in chrono - -/// and this implementation only supports named timezones at the end of the string preceded by a space. -/// /// [`chrono::format::strftime`]: https://docs.rs/chrono/latest/chrono/format/strftime/index.html /// [IANA timezones]: https://www.iana.org/time-zones pub(crate) fn string_to_datetime_formatted( @@ -114,55 +89,11 @@ pub(crate) fn string_to_datetime_formatted( ) }; - let mut datetime_str = s; - let mut format = format; - - // Manually handle the most common case of a named timezone at the end of the timestamp. - // Note that %+ handles 'Z' at the end of the string without a space. This code doesn't - // handle named timezones with no preceding space since that would require writing a - // custom parser (or switching to Jiff) - let tz: Option = if format.trim_end().ends_with(" %Z") { - // grab the string after the last space as the named timezone - if let Some((dt_str, timezone_name)) = datetime_str.trim_end().rsplit_once(' ') { - datetime_str = dt_str; - - // attempt to parse the timezone name - let result: Result = - timezone_name.parse(); - let Ok(tz) = result else { - return Err(err(&result.unwrap_err().to_string())); - }; - - // successfully parsed the timezone name, remove the ' %Z' from the format - format = &format[..format.len() - 3]; - - Some(tz) - } else { - None - } - } else if format.contains("%Z") { - return Err(err( - "'%Z' is only supported at the end of the format string preceded by a space", - )); - } else { - None - }; - let mut parsed = Parsed::new(); - parse(&mut parsed, datetime_str, StrftimeItems::new(format)) - .map_err(|e| err(&e.to_string()))?; + parse(&mut parsed, s, StrftimeItems::new(format)).map_err(|e| err(&e.to_string()))?; - let dt = match tz { - Some(tz) => { - // A timezone was manually parsed out, convert it to a fixed offset - match parsed.to_datetime_with_timezone(&tz) { - Ok(dt) => Ok(dt.fixed_offset()), - Err(e) => Err(e), - } - } - // default to parse the string assuming it has a timezone - None => parsed.to_datetime(), - }; + // attempt to parse the string assuming it has a timezone + let dt = parsed.to_datetime(); if let Err(e) = &dt { // no timezone or other failure, try without a timezone @@ -184,7 +115,7 @@ pub(crate) fn string_to_datetime_formatted( } /// Accepts a string with a `chrono` format and converts it to a -/// nanosecond precision timestamp relative to the provided `timezone`. +/// nanosecond precision timestamp. /// /// See [`chrono::format::strftime`] for the full set of supported formats. /// @@ -210,21 +141,19 @@ pub(crate) fn string_to_datetime_formatted( /// /// [`chrono::format::strftime`]: https://docs.rs/chrono/latest/chrono/format/strftime/index.html #[inline] -pub(crate) fn string_to_timestamp_nanos_formatted_with_timezone( - timezone: &Option, +pub(crate) fn string_to_timestamp_nanos_formatted( s: &str, format: &str, ) -> Result { - let dt = string_to_datetime_formatted(timezone.as_ref().unwrap_or(&UTC), s, format)?; - let parsed = dt + string_to_datetime_formatted(&Utc, s, format)? + .naive_utc() + .and_utc() .timestamp_nanos_opt() - .ok_or_else(|| exec_datafusion_err!("{ERR_NANOSECONDS_NOT_SUPPORTED}"))?; - - Ok(parsed) + .ok_or_else(|| exec_datafusion_err!("{ERR_NANOSECONDS_NOT_SUPPORTED}")) } /// Accepts a string with a `chrono` format and converts it to a -/// millisecond precision timestamp relative to the provided `timezone`. +/// millisecond precision timestamp. /// /// See [`chrono::format::strftime`] for the full set of supported formats. /// @@ -247,14 +176,14 @@ pub(crate) fn string_to_timestamp_millis_formatted(s: &str, format: &str) -> Res .timestamp_millis()) } -pub(crate) fn handle( +pub(crate) fn handle( args: &[ColumnarValue], op: F, name: &str, - dt: &DataType, ) -> Result where O: ArrowPrimitiveType, + S: ScalarType, F: Fn(&str) -> Result, { match &args[0] { @@ -281,13 +210,8 @@ where }, ColumnarValue::Scalar(scalar) => match scalar.try_as_str() { Some(a) => { - let result = a - .as_ref() - .map(|x| op(x)) - .transpose()? - .and_then(|v| v.to_i64()); - let s = scalar_value(dt, result)?; - Ok(ColumnarValue::Scalar(s)) + let result = a.as_ref().map(|x| op(x)).transpose()?; + Ok(ColumnarValue::Scalar(S::scalar(result))) } _ => exec_err!("Unsupported data type {scalar:?} for function {name}"), }, @@ -297,15 +221,15 @@ where // Given a function that maps a `&str`, `&str` to an arrow native type, // returns a `ColumnarValue` where the function is applied to either a `ArrayRef` or `ScalarValue` // depending on the `args`'s variant. -pub(crate) fn handle_multiple( +pub(crate) fn handle_multiple( args: &[ColumnarValue], op: F, op2: M, name: &str, - dt: &DataType, ) -> Result where O: ArrowPrimitiveType, + S: ScalarType, F: Fn(&str, &str) -> Result, M: Fn(O::Native) -> O::Native, { @@ -374,9 +298,9 @@ where if let Some(s) = x { match op(a, s.as_str()) { Ok(r) => { - let result = op2(r).to_i64(); - let s = scalar_value(dt, result)?; - ret = Some(Ok(ColumnarValue::Scalar(s))); + ret = Some(Ok(ColumnarValue::Scalar(S::scalar(Some( + op2(r), + ))))); break; } Err(e) => ret = Some(Err(e)), @@ -530,16 +454,3 @@ where // first map is the iterator, second is for the `Option<_>` array.iter().map(|x| x.map(&op).transpose()).collect() } - -fn scalar_value(dt: &DataType, r: Option) -> Result { - match dt { - DataType::Date32 => Ok(ScalarValue::Date32(r.and_then(|v| v.to_i32()))), - DataType::Timestamp(u, tz) => match u { - TimeUnit::Second => Ok(ScalarValue::TimestampSecond(r, tz.clone())), - TimeUnit::Millisecond => Ok(ScalarValue::TimestampMillisecond(r, tz.clone())), - TimeUnit::Microsecond => Ok(ScalarValue::TimestampMicrosecond(r, tz.clone())), - TimeUnit::Nanosecond => Ok(ScalarValue::TimestampNanosecond(r, tz.clone())), - }, - t => Err(internal_datafusion_err!("Unsupported data type: {t:?}")), - } -} diff --git a/datafusion/functions/src/datetime/mod.rs b/datafusion/functions/src/datetime/mod.rs index 39b9453295df6..9872db3faf556 100644 --- a/datafusion/functions/src/datetime/mod.rs +++ b/datafusion/functions/src/datetime/mod.rs @@ -53,14 +53,11 @@ make_udf_function!(to_date::ToDateFunc, to_date); make_udf_function!(to_local_time::ToLocalTimeFunc, to_local_time); make_udf_function!(to_time::ToTimeFunc, to_time); make_udf_function!(to_unixtime::ToUnixtimeFunc, to_unixtime); -make_udf_function_with_config!(to_timestamp::ToTimestampFunc, to_timestamp); -make_udf_function_with_config!( - to_timestamp::ToTimestampSecondsFunc, - to_timestamp_seconds -); -make_udf_function_with_config!(to_timestamp::ToTimestampMillisFunc, to_timestamp_millis); -make_udf_function_with_config!(to_timestamp::ToTimestampMicrosFunc, to_timestamp_micros); -make_udf_function_with_config!(to_timestamp::ToTimestampNanosFunc, to_timestamp_nanos); +make_udf_function!(to_timestamp::ToTimestampFunc, to_timestamp); +make_udf_function!(to_timestamp::ToTimestampSecondsFunc, to_timestamp_seconds); +make_udf_function!(to_timestamp::ToTimestampMillisFunc, to_timestamp_millis); +make_udf_function!(to_timestamp::ToTimestampMicrosFunc, to_timestamp_micros); +make_udf_function!(to_timestamp::ToTimestampNanosFunc, to_timestamp_nanos); // create UDF with config make_udf_function_with_config!(now::NowFunc, now); @@ -121,24 +118,24 @@ pub mod expr_fn { args, ),( to_timestamp, - "converts a string and optional formats to a `Timestamp(Nanoseconds, TimeZone)`", - @config args, + "converts a string and optional formats to a `Timestamp(Nanoseconds, None)`", + args, ),( to_timestamp_seconds, - "converts a string and optional formats to a `Timestamp(Seconds, TimeZone)`", - @config args, + "converts a string and optional formats to a `Timestamp(Seconds, None)`", + args, ),( to_timestamp_millis, - "converts a string and optional formats to a `Timestamp(Milliseconds, TimeZone)`", - @config args, + "converts a string and optional formats to a `Timestamp(Milliseconds, None)`", + args, ),( to_timestamp_micros, - "converts a string and optional formats to a `Timestamp(Microseconds, TimeZone)`", - @config args, + "converts a string and optional formats to a `Timestamp(Microseconds, None)`", + args, ),( to_timestamp_nanos, - "converts a string and optional formats to a `Timestamp(Nanoseconds, TimeZone)`", - @config args, + "converts a string and optional formats to a `Timestamp(Nanoseconds, None)`", + args, )); /// Returns a string representation of a date, time, timestamp or duration based @@ -274,7 +271,6 @@ pub mod expr_fn { /// Returns all DataFusion functions defined in this package pub fn functions() -> Vec> { use datafusion_common::config::ConfigOptions; - let config = ConfigOptions::default(); vec![ current_date(), current_time(), @@ -284,16 +280,16 @@ pub fn functions() -> Vec> { from_unixtime(), make_date(), make_time(), - now(&config), + now(&ConfigOptions::default()), to_char(), to_date(), to_local_time(), to_time(), to_unixtime(), - to_timestamp(&config), - to_timestamp_seconds(&config), - to_timestamp_millis(&config), - to_timestamp_micros(&config), - to_timestamp_nanos(&config), + to_timestamp(), + to_timestamp_seconds(), + to_timestamp_millis(), + to_timestamp_micros(), + to_timestamp_nanos(), ] } diff --git a/datafusion/functions/src/datetime/to_date.rs b/datafusion/functions/src/datetime/to_date.rs index 60c6fdf2df975..471995089ba3b 100644 --- a/datafusion/functions/src/datetime/to_date.rs +++ b/datafusion/functions/src/datetime/to_date.rs @@ -84,7 +84,7 @@ impl ToDateFunc { fn to_date(&self, args: &[ColumnarValue]) -> Result { match args.len() { - 1 => handle::( + 1 => handle::( args, |s| match Date32Type::parse(s) { Some(v) => Ok(v), @@ -94,9 +94,8 @@ impl ToDateFunc { )), }, "to_date", - &Date32, ), - 2.. => handle_multiple::( + 2.. => handle_multiple::( args, |s, format| { string_to_timestamp_millis_formatted(s, format) @@ -109,7 +108,6 @@ impl ToDateFunc { }, |n| n, "to_date", - &Date32, ), 0 => exec_err!("Unsupported 0 argument count for function to_date"), } diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rs index 6d40133bd29bf..ea63bda5bcfaf 100644 --- a/datafusion/functions/src/datetime/to_timestamp.rs +++ b/datafusion/functions/src/datetime/to_timestamp.rs @@ -19,7 +19,6 @@ use std::any::Any; use std::sync::Arc; use crate::datetime::common::*; -use arrow::array::timezone::Tz; use arrow::array::{ Array, Decimal128Array, Float16Array, Float32Array, Float64Array, TimestampNanosecondArray, @@ -27,33 +26,21 @@ use arrow::array::{ use arrow::datatypes::DataType::*; use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second}; use arrow::datatypes::{ - ArrowTimestampType, DataType, TimestampMicrosecondType, TimestampMillisecondType, - TimestampNanosecondType, TimestampSecondType, + ArrowTimestampType, DataType, TimeUnit, TimestampMicrosecondType, + TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, }; -use datafusion_common::config::ConfigOptions; use datafusion_common::{Result, ScalarType, ScalarValue, exec_err}; use datafusion_expr::{ - ColumnarValue, Documentation, ScalarUDF, ScalarUDFImpl, Signature, Volatility, + ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility, }; use datafusion_macros::user_doc; #[user_doc( doc_section(label = "Time and Date Functions"), description = r#" -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. -Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). - -Note: `to_timestamp` returns `Timestamp(ns, TimeZone)` where the time zone is the session time zone. The supported range -for integer input is between`-9223372037` and `9223372036`. Supported range for string input is between -`1677-09-21T00:12:44.0` and `2262-04-11T23:47:16.0`. Please use `to_timestamp_seconds` -for the input outside of supported bounds. - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. +Converts a value to a timestamp (`YYYY-MM-DDT00:00:00Z`). Supports strings, integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp. + +Note: `to_timestamp` returns `Timestamp(ns)`. The supported range for integer input is between `-9223372037` and `9223372036`. Supported range for string input is between `1677-09-21T00:12:44.0` and `2262-04-11T23:47:16.0`. Please use `to_timestamp_seconds` for the input outside of supported bounds. "#, syntax_example = "to_timestamp(expression[, ..., format_n])", sql_example = r#"```sql @@ -78,33 +65,17 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ), argument( name = "format_n", - description = r#" -Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. -Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is -only supported at the end of the string preceded by a space. -"# + description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned." ) )] #[derive(Debug, PartialEq, Eq, Hash)] pub struct ToTimestampFunc { signature: Signature, - timezone: Option>, } #[user_doc( doc_section(label = "Time and Date Functions"), - description = r#" -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. -Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. -"#, + description = "Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided. Integers and unsigned integers are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp.", syntax_example = "to_timestamp_seconds(expression[, ..., format_n])", sql_example = r#"```sql > select to_timestamp_seconds('2023-01-31T09:26:56.123456789-05:00'); @@ -128,33 +99,17 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ), argument( name = "format_n", - description = r#" -Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. -Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is -only supported at the end of the string preceded by a space. -"# + description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned." ) )] #[derive(Debug, PartialEq, Eq, Hash)] pub struct ToTimestampSecondsFunc { signature: Signature, - timezone: Option>, } #[user_doc( doc_section(label = "Time and Date Functions"), - description = r#" -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. -Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as milliseconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. -"#, + description = "Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. Integers and unsigned integers are interpreted as milliseconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp.", syntax_example = "to_timestamp_millis(expression[, ..., format_n])", sql_example = r#"```sql > select to_timestamp_millis('2023-01-31T09:26:56.123456789-05:00'); @@ -178,33 +133,17 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ), argument( name = "format_n", - description = r#" -Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. -Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is -only supported at the end of the string preceded by a space. -"# + description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned." ) )] #[derive(Debug, PartialEq, Eq, Hash)] pub struct ToTimestampMillisFunc { signature: Signature, - timezone: Option>, } #[user_doc( doc_section(label = "Time and Date Functions"), - description = r#" -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. -Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as microseconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. -"#, + description = "Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided. Integers and unsigned integers are interpreted as microseconds since the unix epoch (`1970-01-01T00:00:00Z`) Returns the corresponding timestamp.", syntax_example = "to_timestamp_micros(expression[, ..., format_n])", sql_example = r#"```sql > select to_timestamp_micros('2023-01-31T09:26:56.123456789-05:00'); @@ -228,32 +167,17 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ), argument( name = "format_n", - description = r#" -Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. -Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is -only supported at the end of the string preceded by a space. -"# + description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned." ) )] #[derive(Debug, PartialEq, Eq, Hash)] pub struct ToTimestampMicrosFunc { signature: Signature, - timezone: Option>, } #[user_doc( doc_section(label = "Time and Date Functions"), - description = r#" -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. -Strings that parse without a time zone are treated as if they are in the -session time zone. Integers, unsigned integers, and doubles are interpreted as nanoseconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. -"#, + description = "Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided. Integers and unsigned integers are interpreted as nanoseconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp.", syntax_example = "to_timestamp_nanos(expression[, ..., format_n])", sql_example = r#"```sql > select to_timestamp_nanos('2023-01-31T09:26:56.123456789-05:00'); @@ -277,60 +201,83 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ), argument( name = "format_n", - description = r#" -Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. -Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is -only supported at the end of the string preceded by a space. -"# + description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned." ) )] #[derive(Debug, PartialEq, Eq, Hash)] pub struct ToTimestampNanosFunc { signature: Signature, - timezone: Option>, } -/// Macro to generate boilerplate constructors and config methods for ToTimestamp* functions. -/// Generates: Default impl, deprecated new(), new_with_config(), and extracts timezone from ConfigOptions. -macro_rules! impl_to_timestamp_constructors { - ($func:ty) => { - impl Default for $func { - fn default() -> Self { - Self::new_with_config(&ConfigOptions::default()) - } +impl Default for ToTimestampFunc { + fn default() -> Self { + Self::new() + } +} + +impl ToTimestampFunc { + pub fn new() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), } + } +} - impl $func { - #[deprecated(since = "52.0.0", note = "use `new_with_config` instead")] - /// Deprecated constructor retained for backwards compatibility. - /// - /// Prefer `new_with_config` which allows specifying the - /// timezone via [`ConfigOptions`]. This helper now mirrors the - /// canonical default offset (None) provided by `ConfigOptions::default()`. - pub fn new() -> Self { - Self::new_with_config(&ConfigOptions::default()) - } +impl Default for ToTimestampSecondsFunc { + fn default() -> Self { + Self::new() + } +} - pub fn new_with_config(config: &ConfigOptions) -> Self { - Self { - signature: Signature::variadic_any(Volatility::Immutable), - timezone: config - .execution - .time_zone - .as_ref() - .map(|tz| Arc::from(tz.as_str())), - } - } +impl ToTimestampSecondsFunc { + pub fn new() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), } - }; + } } -impl_to_timestamp_constructors!(ToTimestampFunc); -impl_to_timestamp_constructors!(ToTimestampSecondsFunc); -impl_to_timestamp_constructors!(ToTimestampMillisFunc); -impl_to_timestamp_constructors!(ToTimestampMicrosFunc); -impl_to_timestamp_constructors!(ToTimestampNanosFunc); +impl Default for ToTimestampMillisFunc { + fn default() -> Self { + Self::new() + } +} + +impl ToTimestampMillisFunc { + pub fn new() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + } + } +} + +impl Default for ToTimestampMicrosFunc { + fn default() -> Self { + Self::new() + } +} + +impl ToTimestampMicrosFunc { + pub fn new() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + } + } +} + +impl Default for ToTimestampNanosFunc { + fn default() -> Self { + Self::new() + } +} + +impl ToTimestampNanosFunc { + pub fn new() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + } + } +} fn decimal_to_nanoseconds(value: i128, scale: i8) -> i64 { let nanos_exponent = 9_i16 - scale as i16; @@ -377,15 +324,6 @@ fn decimal128_to_timestamp_nanos( /// The supported range for integer input is between `-9223372037` and `9223372036`. /// Supported range for string input is between `1677-09-21T00:12:44.0` and `2262-04-11T23:47:16.0`. /// Please use `to_timestamp_seconds` for the input outside of supported bounds. -/// Macro to generate the with_updated_config method for ToTimestamp* functions. -macro_rules! impl_with_updated_config { - () => { - fn with_updated_config(&self, config: &ConfigOptions) -> Option { - Some(Self::new_with_config(config).into()) - } - }; -} - impl ScalarUDFImpl for ToTimestampFunc { fn as_any(&self) -> &dyn Any { self @@ -399,18 +337,15 @@ impl ScalarUDFImpl for ToTimestampFunc { &self.signature } - fn return_type(&self, _arg_types: &[DataType]) -> Result { - Ok(Timestamp(Nanosecond, self.timezone.clone())) + fn return_type(&self, arg_types: &[DataType]) -> Result { + Ok(return_type_for(&arg_types[0], Nanosecond)) } - impl_with_updated_config!(); - fn invoke_with_args( &self, args: datafusion_expr::ScalarFunctionArgs, ) -> Result { - let datafusion_expr::ScalarFunctionArgs { args, .. } = args; - + let args = args.args; if args.is_empty() { return exec_err!( "to_timestamp function requires 1 or more arguments, got {}", @@ -423,27 +358,26 @@ impl ScalarUDFImpl for ToTimestampFunc { validate_data_types(&args, "to_timestamp")?; } - let tz = self.timezone.clone(); - match args[0].data_type() { Int8 | Int16 | Int32 | Int64 | UInt8 | UInt16 | UInt32 | UInt64 => args[0] .cast_to(&Timestamp(Second, None), None)? - .cast_to(&Timestamp(Nanosecond, tz), None), - Null | Timestamp(_, _) => args[0].cast_to(&Timestamp(Nanosecond, tz), None), + .cast_to(&Timestamp(Nanosecond, None), None), + Null => args[0].cast_to(&Timestamp(Nanosecond, None), None), + Timestamp(_, tz) => args[0].cast_to(&Timestamp(Nanosecond, tz), None), Float16 => match &args[0] { ColumnarValue::Scalar(ScalarValue::Float16(value)) => { let timestamp_nanos = value.map(|v| (v.to_f64() * 1_000_000_000.0) as i64); Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( timestamp_nanos, - tz, + None, ))) } ColumnarValue::Array(arr) => { let f16_arr = downcast_arg!(arr, Float16Array); let result: TimestampNanosecondArray = f16_arr.unary(|x| (x.to_f64() * 1_000_000_000.0) as i64); - Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz)))) + Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(None::>)))) } _ => exec_err!("Invalid Float16 value for to_timestamp"), }, @@ -453,14 +387,14 @@ impl ScalarUDFImpl for ToTimestampFunc { value.map(|v| (v as f64 * 1_000_000_000.0) as i64); Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( timestamp_nanos, - tz, + None, ))) } ColumnarValue::Array(arr) => { let f32_arr = downcast_arg!(arr, Float32Array); let result: TimestampNanosecondArray = f32_arr.unary(|x| (x as f64 * 1_000_000_000.0) as i64); - Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz)))) + Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(None::>)))) } _ => exec_err!("Invalid Float32 value for to_timestamp"), }, @@ -469,31 +403,30 @@ impl ScalarUDFImpl for ToTimestampFunc { let timestamp_nanos = value.map(|v| (v * 1_000_000_000.0) as i64); Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( timestamp_nanos, - tz, + None, ))) } ColumnarValue::Array(arr) => { let f64_arr = downcast_arg!(arr, Float64Array); let result: TimestampNanosecondArray = f64_arr.unary(|x| (x * 1_000_000_000.0) as i64); - Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz)))) + Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(None::>)))) } _ => exec_err!("Invalid Float64 value for to_timestamp"), }, Decimal32(_, _) | Decimal64(_, _) | Decimal256(_, _) => { let arg = args[0].cast_to(&Decimal128(38, 9), None)?; - decimal128_to_timestamp_nanos(&arg, tz) + decimal128_to_timestamp_nanos(&arg, None) } - Decimal128(_, _) => decimal128_to_timestamp_nanos(&args[0], tz), + Decimal128(_, _) => decimal128_to_timestamp_nanos(&args[0], None), Utf8View | LargeUtf8 | Utf8 => { - to_timestamp_impl::(&args, "to_timestamp", &tz) + to_timestamp_impl::(&args, "to_timestamp") } other => { exec_err!("Unsupported data type {other} for function to_timestamp") } } } - fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -512,18 +445,15 @@ impl ScalarUDFImpl for ToTimestampSecondsFunc { &self.signature } - fn return_type(&self, _arg_types: &[DataType]) -> Result { - Ok(Timestamp(Second, self.timezone.clone())) + fn return_type(&self, arg_types: &[DataType]) -> Result { + Ok(return_type_for(&arg_types[0], Second)) } - impl_with_updated_config!(); - fn invoke_with_args( &self, args: datafusion_expr::ScalarFunctionArgs, ) -> Result { - let datafusion_expr::ScalarFunctionArgs { args, .. } = args; - + let args = args.args; if args.is_empty() { return exec_err!( "to_timestamp_seconds function requires 1 or more arguments, got {}", @@ -536,8 +466,6 @@ impl ScalarUDFImpl for ToTimestampSecondsFunc { validate_data_types(&args, "to_timestamp")?; } - let tz = self.timezone.clone(); - match args[0].data_type() { Null | Int8 @@ -548,19 +476,17 @@ impl ScalarUDFImpl for ToTimestampSecondsFunc { | UInt16 | UInt32 | UInt64 - | Timestamp(_, _) | Decimal32(_, _) | Decimal64(_, _) | Decimal128(_, _) - | Decimal256(_, _) => args[0].cast_to(&Timestamp(Second, tz), None), + | Decimal256(_, _) => args[0].cast_to(&Timestamp(Second, None), None), + Timestamp(_, tz) => args[0].cast_to(&Timestamp(Second, tz), None), Float16 | Float32 | Float64 => args[0] .cast_to(&Int64, None)? - .cast_to(&Timestamp(Second, tz), None), - Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( - &args, - "to_timestamp_seconds", - &self.timezone, - ), + .cast_to(&Timestamp(Second, None), None), + Utf8View | LargeUtf8 | Utf8 => { + to_timestamp_impl::(&args, "to_timestamp_seconds") + } other => { exec_err!( "Unsupported data type {} for function to_timestamp_seconds", @@ -569,7 +495,6 @@ impl ScalarUDFImpl for ToTimestampSecondsFunc { } } } - fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -588,18 +513,15 @@ impl ScalarUDFImpl for ToTimestampMillisFunc { &self.signature } - fn return_type(&self, _arg_types: &[DataType]) -> Result { - Ok(Timestamp(Millisecond, self.timezone.clone())) + fn return_type(&self, arg_types: &[DataType]) -> Result { + Ok(return_type_for(&arg_types[0], Millisecond)) } - impl_with_updated_config!(); - fn invoke_with_args( &self, args: datafusion_expr::ScalarFunctionArgs, ) -> Result { - let datafusion_expr::ScalarFunctionArgs { args, .. } = args; - + let args = args.args; if args.is_empty() { return exec_err!( "to_timestamp_millis function requires 1 or more arguments, got {}", @@ -622,20 +544,19 @@ impl ScalarUDFImpl for ToTimestampMillisFunc { | UInt16 | UInt32 | UInt64 - | Timestamp(_, _) | Decimal32(_, _) | Decimal64(_, _) | Decimal128(_, _) - | Decimal256(_, _) => { - args[0].cast_to(&Timestamp(Millisecond, self.timezone.clone()), None) + | Decimal256(_, _) => args[0].cast_to(&Timestamp(Millisecond, None), None), + Timestamp(_, tz) => { + args[0].cast_to(&Timestamp(Millisecond, tz), None) } Float16 | Float32 | Float64 => args[0] .cast_to(&Int64, None)? - .cast_to(&Timestamp(Millisecond, self.timezone.clone()), None), + .cast_to(&Timestamp(Millisecond, None), None), Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( &args, "to_timestamp_millis", - &self.timezone, ), other => { exec_err!( @@ -645,7 +566,6 @@ impl ScalarUDFImpl for ToTimestampMillisFunc { } } } - fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -664,18 +584,15 @@ impl ScalarUDFImpl for ToTimestampMicrosFunc { &self.signature } - fn return_type(&self, _arg_types: &[DataType]) -> Result { - Ok(Timestamp(Microsecond, self.timezone.clone())) + fn return_type(&self, arg_types: &[DataType]) -> Result { + Ok(return_type_for(&arg_types[0], Microsecond)) } - impl_with_updated_config!(); - fn invoke_with_args( &self, args: datafusion_expr::ScalarFunctionArgs, ) -> Result { - let datafusion_expr::ScalarFunctionArgs { args, .. } = args; - + let args = args.args; if args.is_empty() { return exec_err!( "to_timestamp_micros function requires 1 or more arguments, got {}", @@ -698,20 +615,19 @@ impl ScalarUDFImpl for ToTimestampMicrosFunc { | UInt16 | UInt32 | UInt64 - | Timestamp(_, _) | Decimal32(_, _) | Decimal64(_, _) | Decimal128(_, _) - | Decimal256(_, _) => { - args[0].cast_to(&Timestamp(Microsecond, self.timezone.clone()), None) + | Decimal256(_, _) => args[0].cast_to(&Timestamp(Microsecond, None), None), + Timestamp(_, tz) => { + args[0].cast_to(&Timestamp(Microsecond, tz), None) } Float16 | Float32 | Float64 => args[0] .cast_to(&Int64, None)? - .cast_to(&Timestamp(Microsecond, self.timezone.clone()), None), + .cast_to(&Timestamp(Microsecond, None), None), Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( &args, "to_timestamp_micros", - &self.timezone, ), other => { exec_err!( @@ -721,7 +637,6 @@ impl ScalarUDFImpl for ToTimestampMicrosFunc { } } } - fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -740,18 +655,15 @@ impl ScalarUDFImpl for ToTimestampNanosFunc { &self.signature } - fn return_type(&self, _arg_types: &[DataType]) -> Result { - Ok(Timestamp(Nanosecond, self.timezone.clone())) + fn return_type(&self, arg_types: &[DataType]) -> Result { + Ok(return_type_for(&arg_types[0], Nanosecond)) } - impl_with_updated_config!(); - fn invoke_with_args( &self, args: datafusion_expr::ScalarFunctionArgs, ) -> Result { - let datafusion_expr::ScalarFunctionArgs { args, .. } = args; - + let args = args.args; if args.is_empty() { return exec_err!( "to_timestamp_nanos function requires 1 or more arguments, got {}", @@ -774,21 +686,17 @@ impl ScalarUDFImpl for ToTimestampNanosFunc { | UInt16 | UInt32 | UInt64 - | Timestamp(_, _) | Decimal32(_, _) | Decimal64(_, _) | Decimal128(_, _) - | Decimal256(_, _) => { - args[0].cast_to(&Timestamp(Nanosecond, self.timezone.clone()), None) - } + | Decimal256(_, _) => args[0].cast_to(&Timestamp(Nanosecond, None), None), + Timestamp(_, tz) => args[0].cast_to(&Timestamp(Nanosecond, tz), None), Float16 | Float32 | Float64 => args[0] .cast_to(&Int64, None)? - .cast_to(&Timestamp(Nanosecond, self.timezone.clone()), None), - Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( - &args, - "to_timestamp_nanos", - &self.timezone, - ), + .cast_to(&Timestamp(Nanosecond, None), None), + Utf8View | LargeUtf8 | Utf8 => { + to_timestamp_impl::(&args, "to_timestamp_nanos") + } other => { exec_err!( "Unsupported data type {} for function to_timestamp_nanos", @@ -797,16 +705,23 @@ impl ScalarUDFImpl for ToTimestampNanosFunc { } } } - fn documentation(&self) -> Option<&Documentation> { self.doc() } } +/// Returns the return type for the to_timestamp_* function, preserving +/// the timezone if it exists. +fn return_type_for(arg: &DataType, unit: TimeUnit) -> DataType { + match arg { + Timestamp(_, Some(tz)) => Timestamp(unit, Some(Arc::clone(tz))), + _ => Timestamp(unit, None), + } +} + fn to_timestamp_impl>( args: &[ColumnarValue], name: &str, - timezone: &Option>, ) -> Result { let factor = match T::UNIT { Second => 1_000_000_000, @@ -815,26 +730,17 @@ fn to_timestamp_impl>( Nanosecond => 1, }; - let tz = match timezone.clone() { - Some(tz) => Some(tz.parse::()?), - None => None, - }; - match args.len() { - 1 => handle::( + 1 => handle::( args, - move |s| string_to_timestamp_nanos_with_timezone(&tz, s).map(|n| n / factor), + |s| string_to_timestamp_nanos_shim(s).map(|n| n / factor), name, - &Timestamp(T::UNIT, timezone.clone()), ), - n if n >= 2 => handle_multiple::( + n if n >= 2 => handle_multiple::( args, - move |s, format| { - string_to_timestamp_nanos_formatted_with_timezone(&tz, s, format) - }, + string_to_timestamp_nanos_formatted, |n| n / factor, name, - &Timestamp(T::UNIT, timezone.clone()), ), _ => exec_err!("Unsupported 0 argument count for function {name}"), } @@ -851,110 +757,35 @@ mod tests { }; use arrow::array::{ArrayRef, Int64Array, StringBuilder}; use arrow::datatypes::{Field, TimeUnit}; - use chrono::{DateTime, FixedOffset, Utc}; + use chrono::Utc; use datafusion_common::config::ConfigOptions; use datafusion_common::{DataFusionError, ScalarValue, assert_contains}; - use datafusion_expr::{ScalarFunctionArgs, ScalarFunctionImplementation}; + use datafusion_expr::ScalarFunctionImplementation; use super::*; fn to_timestamp(args: &[ColumnarValue]) -> Result { - let timezone: Option> = Some("UTC".into()); - to_timestamp_impl::(args, "to_timestamp", &timezone) + to_timestamp_impl::(args, "to_timestamp") } /// to_timestamp_millis SQL function fn to_timestamp_millis(args: &[ColumnarValue]) -> Result { - let timezone: Option> = Some("UTC".into()); - to_timestamp_impl::( - args, - "to_timestamp_millis", - &timezone, - ) + to_timestamp_impl::(args, "to_timestamp_millis") } /// to_timestamp_micros SQL function fn to_timestamp_micros(args: &[ColumnarValue]) -> Result { - let timezone: Option> = Some("UTC".into()); - to_timestamp_impl::( - args, - "to_timestamp_micros", - &timezone, - ) + to_timestamp_impl::(args, "to_timestamp_micros") } /// to_timestamp_nanos SQL function fn to_timestamp_nanos(args: &[ColumnarValue]) -> Result { - let timezone: Option> = Some("UTC".into()); - to_timestamp_impl::( - args, - "to_timestamp_nanos", - &timezone, - ) + to_timestamp_impl::(args, "to_timestamp_nanos") } /// to_timestamp_seconds SQL function fn to_timestamp_seconds(args: &[ColumnarValue]) -> Result { - let timezone: Option> = Some("UTC".into()); - to_timestamp_impl::(args, "to_timestamp_seconds", &timezone) - } - - fn udfs_and_timeunit() -> Vec<(Box, TimeUnit)> { - let udfs: Vec<(Box, TimeUnit)> = vec![ - ( - Box::new(ToTimestampFunc::new_with_config(&ConfigOptions::default())), - Nanosecond, - ), - ( - Box::new(ToTimestampSecondsFunc::new_with_config( - &ConfigOptions::default(), - )), - Second, - ), - ( - Box::new(ToTimestampMillisFunc::new_with_config( - &ConfigOptions::default(), - )), - Millisecond, - ), - ( - Box::new(ToTimestampMicrosFunc::new_with_config( - &ConfigOptions::default(), - )), - Microsecond, - ), - ( - Box::new(ToTimestampNanosFunc::new_with_config( - &ConfigOptions::default(), - )), - Nanosecond, - ), - ]; - udfs - } - - fn validate_expected_error( - options: &mut ConfigOptions, - args: ScalarFunctionArgs, - expected_err: &str, - ) { - let udfs = udfs_and_timeunit(); - - for (udf, _) in udfs { - match udf - .with_updated_config(options) - .unwrap() - .invoke_with_args(args.clone()) - { - Ok(_) => panic!("Expected error but got success"), - Err(e) => { - assert!( - e.to_string().contains(expected_err), - "Can not find expected error '{expected_err}'. Actual error '{e}'" - ); - } - } - } + to_timestamp_impl::(args, "to_timestamp_seconds") } #[test] @@ -1025,368 +856,6 @@ mod tests { Ok(()) } - #[test] - fn to_timestamp_respects_execution_timezone() -> Result<()> { - let udfs = udfs_and_timeunit(); - - let mut options = ConfigOptions::default(); - options.execution.time_zone = Some("-05:00".to_string()); - - let time_zone: Option> = options - .execution - .time_zone - .as_ref() - .map(|tz| Arc::from(tz.as_str())); - - for (udf, time_unit) in udfs { - let field = Field::new("arg", Utf8, true).into(); - - let args = ScalarFunctionArgs { - args: vec![ColumnarValue::Scalar(ScalarValue::Utf8(Some( - "2020-09-08T13:42:29".to_string(), - )))], - arg_fields: vec![field], - number_rows: 1, - return_field: Field::new( - "f", - Timestamp(time_unit, Some("-05:00".into())), - true, - ) - .into(), - config_options: Arc::new(options.clone()), - }; - - let result = udf - .with_updated_config(&options.clone()) - .unwrap() - .invoke_with_args(args)?; - let result = match time_unit { - Second => { - let ColumnarValue::Scalar(ScalarValue::TimestampSecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - Millisecond => { - let ColumnarValue::Scalar(ScalarValue::TimestampMillisecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - Microsecond => { - let ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - Nanosecond => { - let ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - }; - - let scale = match time_unit { - Second => 1_000_000_000, - Millisecond => 1_000_000, - Microsecond => 1_000, - Nanosecond => 1, - }; - - let offset = FixedOffset::west_opt(5 * 3600).unwrap(); - let result = Some( - DateTime::::from_timestamp_nanos(result * scale) - .with_timezone(&offset) - .to_string(), - ); - - assert_eq!(result, Some("2020-09-08 13:42:29 -05:00".to_string())); - } - - Ok(()) - } - - #[test] - fn to_timestamp_formats_respects_execution_timezone() -> Result<()> { - let udfs = udfs_and_timeunit(); - - let mut options = ConfigOptions::default(); - options.execution.time_zone = Some("-05:00".to_string()); - - let time_zone: Option> = options - .execution - .time_zone - .as_ref() - .map(|tz| Arc::from(tz.as_str())); - - let expr_field = Field::new("arg", Utf8, true).into(); - let format_field: Arc = Field::new("fmt", Utf8, true).into(); - - for (udf, time_unit) in udfs { - for (value, format, expected_str) in [ - ( - "2020-09-08 09:42:29 -05:00", - "%Y-%m-%d %H:%M:%S %z", - Some("2020-09-08 09:42:29 -05:00"), - ), - ( - "2020-09-08T13:42:29Z", - "%+", - Some("2020-09-08 08:42:29 -05:00"), - ), - ( - "2020-09-08 13:42:29 UTC", - "%Y-%m-%d %H:%M:%S %Z", - Some("2020-09-08 08:42:29 -05:00"), - ), - ( - "+0000 2024-01-01 12:00:00", - "%z %Y-%m-%d %H:%M:%S", - Some("2024-01-01 07:00:00 -05:00"), - ), - ( - "20200908134229+0100", - "%Y%m%d%H%M%S%z", - Some("2020-09-08 07:42:29 -05:00"), - ), - ( - "2020-09-08+0230 13:42", - "%Y-%m-%d%z %H:%M", - Some("2020-09-08 06:12:00 -05:00"), - ), - ] { - let args = ScalarFunctionArgs { - args: vec![ - ColumnarValue::Scalar(ScalarValue::Utf8(Some(value.to_string()))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some( - format.to_string(), - ))), - ], - arg_fields: vec![Arc::clone(&expr_field), Arc::clone(&format_field)], - number_rows: 1, - return_field: Field::new( - "f", - Timestamp(time_unit, Some("-05:00".into())), - true, - ) - .into(), - config_options: Arc::new(options.clone()), - }; - let result = udf - .with_updated_config(&options.clone()) - .unwrap() - .invoke_with_args(args)?; - let result = match time_unit { - Second => { - let ColumnarValue::Scalar(ScalarValue::TimestampSecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - Millisecond => { - let ColumnarValue::Scalar(ScalarValue::TimestampMillisecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - Microsecond => { - let ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - Nanosecond => { - let ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - }; - - let scale = match time_unit { - Second => 1_000_000_000, - Millisecond => 1_000_000, - Microsecond => 1_000, - Nanosecond => 1, - }; - let offset = FixedOffset::west_opt(5 * 3600).unwrap(); - let result = Some( - DateTime::::from_timestamp_nanos(result * scale) - .with_timezone(&offset) - .to_string(), - ); - - assert_eq!(result, expected_str.map(|s| s.to_string())); - } - } - - Ok(()) - } - - #[test] - fn to_timestamp_invalid_execution_timezone_behavior() -> Result<()> { - let field: Arc = Field::new("arg", Utf8, true).into(); - let return_field: Arc = - Field::new("f", Timestamp(Nanosecond, None), true).into(); - - let mut options = ConfigOptions::default(); - options.execution.time_zone = Some("Invalid/Timezone".to_string()); - - let args = ScalarFunctionArgs { - args: vec![ColumnarValue::Scalar(ScalarValue::Utf8(Some( - "2020-09-08T13:42:29Z".to_string(), - )))], - arg_fields: vec![Arc::clone(&field)], - number_rows: 1, - return_field: Arc::clone(&return_field), - config_options: Arc::new(options.clone()), - }; - - let expected_err = - "Invalid timezone \"Invalid/Timezone\": failed to parse timezone"; - - validate_expected_error(&mut options, args, expected_err); - - Ok(()) - } - - #[test] - fn to_timestamp_formats_invalid_execution_timezone_behavior() -> Result<()> { - let expr_field: Arc = Field::new("arg", Utf8, true).into(); - let format_field: Arc = Field::new("fmt", Utf8, true).into(); - let return_field: Arc = - Field::new("f", Timestamp(Nanosecond, None), true).into(); - - let mut options = ConfigOptions::default(); - options.execution.time_zone = Some("Invalid/Timezone".to_string()); - - let expected_err = - "Invalid timezone \"Invalid/Timezone\": failed to parse timezone"; - - let make_args = |value: &str, format: &str| ScalarFunctionArgs { - args: vec![ - ColumnarValue::Scalar(ScalarValue::Utf8(Some(value.to_string()))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some(format.to_string()))), - ], - arg_fields: vec![Arc::clone(&expr_field), Arc::clone(&format_field)], - number_rows: 1, - return_field: Arc::clone(&return_field), - config_options: Arc::new(options.clone()), - }; - - for (value, format, _expected_str) in [ - ( - "2020-09-08 09:42:29 -05:00", - "%Y-%m-%d %H:%M:%S %z", - Some("2020-09-08 09:42:29 -05:00"), - ), - ( - "2020-09-08T13:42:29Z", - "%+", - Some("2020-09-08 08:42:29 -05:00"), - ), - ( - "2020-09-08 13:42:29 +0000", - "%Y-%m-%d %H:%M:%S %z", - Some("2020-09-08 08:42:29 -05:00"), - ), - ( - "+0000 2024-01-01 12:00:00", - "%z %Y-%m-%d %H:%M:%S", - Some("2024-01-01 07:00:00 -05:00"), - ), - ( - "20200908134229+0100", - "%Y%m%d%H%M%S%z", - Some("2020-09-08 07:42:29 -05:00"), - ), - ( - "2020-09-08+0230 13:42", - "%Y-%m-%d%z %H:%M", - Some("2020-09-08 06:12:00 -05:00"), - ), - ] { - let args = make_args(value, format); - validate_expected_error(&mut options.clone(), args, expected_err); - } - - let args = ScalarFunctionArgs { - args: vec![ - ColumnarValue::Scalar(ScalarValue::Utf8(Some( - "2020-09-08T13:42:29".to_string(), - ))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some( - "%Y-%m-%dT%H:%M:%S".to_string(), - ))), - ], - arg_fields: vec![Arc::clone(&expr_field), Arc::clone(&format_field)], - number_rows: 1, - return_field: Arc::clone(&return_field), - config_options: Arc::new(options.clone()), - }; - - validate_expected_error(&mut options.clone(), args, expected_err); - - Ok(()) - } - #[test] fn to_timestamp_invalid_input_type() -> Result<()> { // pass the wrong type of input array to to_timestamp and test @@ -1556,11 +1025,7 @@ mod tests { } fn parse_timestamp_formatted(s: &str, format: &str) -> Result { - let result = string_to_timestamp_nanos_formatted_with_timezone( - &Some("UTC".parse()?), - s, - format, - ); + let result = string_to_timestamp_nanos_formatted(s, format); if let Err(e) = &result { eprintln!("Error parsing timestamp '{s}' using format '{format}': {e:?}"); } @@ -1628,21 +1093,13 @@ mod tests { } #[test] - fn test_no_tz() { + fn test_tz() { let udfs: Vec> = vec![ - Box::new(ToTimestampFunc::new_with_config(&ConfigOptions::default())), - Box::new(ToTimestampSecondsFunc::new_with_config( - &ConfigOptions::default(), - )), - Box::new(ToTimestampMillisFunc::new_with_config( - &ConfigOptions::default(), - )), - Box::new(ToTimestampNanosFunc::new_with_config( - &ConfigOptions::default(), - )), - Box::new(ToTimestampSecondsFunc::new_with_config( - &ConfigOptions::default(), - )), + Box::new(ToTimestampFunc::new()), + Box::new(ToTimestampSecondsFunc::new()), + Box::new(ToTimestampMillisFunc::new()), + Box::new(ToTimestampNanosFunc::new()), + Box::new(ToTimestampSecondsFunc::new()), ]; let mut nanos_builder = TimestampNanosecondArray::builder(2); @@ -1675,8 +1132,8 @@ mod tests { for array in arrays { let rt = udf.return_type(&[array.data_type()]).unwrap(); let arg_field = Field::new("arg", array.data_type().clone(), true).into(); - assert!(matches!(rt, Timestamp(_, None))); - let args = ScalarFunctionArgs { + assert!(matches!(rt, Timestamp(_, Some(_)))); + let args = datafusion_expr::ScalarFunctionArgs { args: vec![array.clone()], arg_fields: vec![arg_field], number_rows: 4, @@ -1691,7 +1148,7 @@ mod tests { _ => panic!("Expected a columnar array"), }; let ty = array.data_type(); - assert!(matches!(ty, Timestamp(_, None))); + assert!(matches!(ty, Timestamp(_, Some(_)))); } } @@ -1726,7 +1183,7 @@ mod tests { let rt = udf.return_type(&[array.data_type()]).unwrap(); assert!(matches!(rt, Timestamp(_, None))); let arg_field = Field::new("arg", array.data_type().clone(), true).into(); - let args = ScalarFunctionArgs { + let args = datafusion_expr::ScalarFunctionArgs { args: vec![array.clone()], arg_fields: vec![arg_field], number_rows: 5, diff --git a/datafusion/functions/src/datetime/to_unixtime.rs b/datafusion/functions/src/datetime/to_unixtime.rs index 2dd377282725a..9c7d743e11475 100644 --- a/datafusion/functions/src/datetime/to_unixtime.rs +++ b/datafusion/functions/src/datetime/to_unixtime.rs @@ -140,7 +140,7 @@ impl ScalarUDFImpl for ToUnixtimeFunc { .cast_to(&DataType::Timestamp(TimeUnit::Second, tz), None)? .cast_to(&DataType::Int64, None), DataType::Utf8View | DataType::LargeUtf8 | DataType::Utf8 => { - ToTimestampSecondsFunc::new_with_config(args.config_options.as_ref()) + ToTimestampSecondsFunc::new() .invoke_with_args(args)? .cast_to(&DataType::Int64, None) } diff --git a/datafusion/functions/src/macros.rs b/datafusion/functions/src/macros.rs index 380877b593643..b04db56200272 100644 --- a/datafusion/functions/src/macros.rs +++ b/datafusion/functions/src/macros.rs @@ -41,17 +41,6 @@ /// - `Vec` argument (single argument followed by a comma) /// - Variable number of `Expr` arguments (zero or more arguments, must be without commas) /// - Functions that require config (marked with `@config` prefix) -/// -/// Note on configuration construction paths: -/// - The convenience wrappers generated for `@config` functions call the inner -/// constructor with `ConfigOptions::default()`. These wrappers are intended -/// primarily for programmatic `Expr` construction and convenience usage. -/// - When functions are registered in a session, DataFusion will call -/// `with_updated_config()` to create a `ScalarUDF` instance using the session's -/// actual `ConfigOptions`. This also happens when configuration changes at runtime -/// (e.g., via `SET` statements). In short: the macro uses the default config for -/// convenience constructors; the session config is applied when functions are -/// registered or when configuration is updated. #[macro_export] macro_rules! export_functions { ($(($FUNC:ident, $DOC:expr, $($arg:tt)*)),*) => { @@ -70,24 +59,6 @@ macro_rules! export_functions { } }; - // function that requires config and takes a vector argument - (single $FUNC:ident, $DOC:expr, @config $arg:ident,) => { - #[doc = $DOC] - pub fn $FUNC($arg: Vec) -> datafusion_expr::Expr { - use datafusion_common::config::ConfigOptions; - super::$FUNC(&ConfigOptions::default()).call($arg) - } - }; - - // function that requires config and variadic arguments - (single $FUNC:ident, $DOC:expr, @config $($arg:ident)*) => { - #[doc = $DOC] - pub fn $FUNC($($arg: datafusion_expr::Expr),*) -> datafusion_expr::Expr { - use datafusion_common::config::ConfigOptions; - super::$FUNC(&ConfigOptions::default()).call(vec![$($arg),*]) - } - }; - // single vector argument (a single argument followed by a comma) (single $FUNC:ident, $DOC:expr, $arg:ident,) => { #[doc = $DOC] diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index c3d36b247b5a0..9f7c148a2c96e 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -193,8 +193,6 @@ SELECT TIMESTAMPTZ '2000-01-01T01:01:01' ---- 2000-01-01T01:01:01Z -statement ok -RESET datafusion.execution.time_zone ########## ## cast tests @@ -2345,7 +2343,7 @@ CREATE TABLE foo (time TIMESTAMPTZ) AS VALUES ('2020-01-01T03:00:00+05:00') statement ok -RESET datafusion.execution.time_zone +SET TIME ZONE = '+00' # verify column type query T @@ -2512,7 +2510,7 @@ true true true true true true # known issues. currently overflows (expects default precision to be microsecond instead of nanoseconds. Work pending) #verify extreme values #query PPPPPPPP -#SELECT to_timestamp(-62125747200), to_timestamp(1926632005177), -62125747200::timestamp as t1, 1926632005177::timestamp, cast(-62125747200 as timestamp), cast(1926632005177 as timestamp) as t2 +#SELECT to_timestamp(-62125747200), to_timestamp(1926632005177), -62125747200::timestamp, 1926632005177::timestamp, cast(-62125747200 as timestamp), cast(1926632005177 as timestamp) #---- #0001-04-25T00:00:00 +63022-07-16T12:59:37 0001-04-25T00:00:00 +63022-07-16T12:59:37 0001-04-25T00:00:00 +63022-07-16T12:59:37 @@ -2838,8 +2836,8 @@ SELECT t1.ts, t1.ts + INTERVAL '1' SECOND FROM t1; query PT SELECT t1.ts::timestamptz, arrow_typeof(t1.ts::timestamptz) FROM t1; ---- -2018-07-01T06:00:00 Timestamp(ns) -2018-07-01T07:00:00 Timestamp(ns) +2018-07-01T06:00:00Z Timestamp(ns, "+00") +2018-07-01T07:00:00Z Timestamp(ns, "+00") query D SELECT 0::TIME @@ -4421,7 +4419,7 @@ SELECT arrow_cast(a, 'LargeUtf8') FROM (SELECT CAST('2005-09-10 13:31:00 +02:00' AS timestamp with time zone) AS a) ---- -Timestamp(ns) 2005-09-10T11:31:00 2005-09-10T11:31:00 2005-09-10T11:31:00 2005-09-10T11:31:00 +Timestamp(ns, "+00") 2005-09-10T11:31:00Z 2005-09-10T11:31:00Z 2005-09-10T11:31:00Z 2005-09-10T11:31:00Z query P SELECT diff --git a/datafusion/sqllogictest/test_files/to_timestamp_timezone.slt b/datafusion/sqllogictest/test_files/to_timestamp_timezone.slt deleted file mode 100644 index d48e41d1204de..0000000000000 --- a/datafusion/sqllogictest/test_files/to_timestamp_timezone.slt +++ /dev/null @@ -1,204 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -########## -## to_timestamp timezone tests -########## - -## Reset timezone for other tests -statement ok -RESET datafusion.execution.time_zone - -## Test 1: Default timezone (None) with naive timestamp -## Naive timestamps (without explicit timezone) should be interpreted as UTC by default -query P -SELECT to_timestamp('2020-09-08T13:42:29'); ----- -2020-09-08T13:42:29 - -## Test 2: Explicit UTC timezone ('Z' suffix) -## Explicit timezone should be respected regardless of session timezone -query P -SELECT to_timestamp('2020-09-08T13:42:29Z'); ----- -2020-09-08T13:42:29 - -## Test 3: Explicit timezone offset (+05:00) -## Explicit offset should be respected - this is 13:42:29+05:00 which is 08:42:29 UTC -query P -SELECT to_timestamp('2020-09-08T13:42:29+05:00'); ----- -2020-09-08T08:42:29 - -## Test 4: Explicit timezone offset without colon (+0500) -## Should handle offset formats without colons -query P -SELECT to_timestamp('2020-09-08T13:42:29+0500'); ----- -2020-09-08T08:42:29 - -## Test 5: Negative timezone offset -query P -SELECT to_timestamp('2020-09-08T13:42:29-03:30'); ----- -2020-09-08T17:12:29 - -## Test 6: Configure session timezone to America/New_York -statement ok -SET datafusion.execution.time_zone = 'America/New_York'; - -## Test 7: Naive timestamp with configured timezone -## '2020-09-08T13:42:29' in America/New_York is EDT (UTC-4) -## So this should become '2020-09-08T13:42:29-04:00' -query P -SELECT to_timestamp('2020-09-08T13:42:29'); ----- -2020-09-08T13:42:29-04:00 - -## Test 8: Explicit UTC should be transformed to configured timezone -query P -SELECT to_timestamp('2020-09-08T13:42:29Z'); ----- -2020-09-08T09:42:29-04:00 - -## Test 9: Explicit offset should be transformed to configured timezone -query P -SELECT to_timestamp('2020-09-08T13:42:29+05:00'); ----- -2020-09-08T04:42:29-04:00 - -## Test 10: Check arrow_typeof returns timstamp in configured timezone -## Result should be Timestamp(Nanosecond, "America/New_York") regardless of input timezone -query T -SELECT arrow_typeof(to_timestamp('2020-09-08T13:42:29Z')); ----- -Timestamp(ns, "America/New_York") - -## Test 11: Configure to offset-based timezone -statement ok -SET datafusion.execution.time_zone = '+05:30'; - -## Test 12: Naive timestamp with offset-based configured timezone -query P -SELECT to_timestamp('2020-09-08T13:42:29'); ----- -2020-09-08T13:42:29+05:30 - -## Test 13: Reset to None -statement ok -RESET datafusion.execution.time_zone - -## Test 14: Naive timestamp -query P -SELECT to_timestamp('2020-09-08T13:42:29'); ----- -2020-09-08T13:42:29 - -query P -SELECT to_timestamp('2020-09-08T13:42:29Z'); ----- -2020-09-08T13:42:29 - -query P -SELECT to_timestamp('2020-09-08T13:42:29+05:00'); ----- -2020-09-08T08:42:29 - -statement ok -SET datafusion.execution.time_zone = 'America/New_York'; - -## Test 15: to_timestamp with format string - naive timestamp with session timezone - -query P -SELECT to_timestamp('2020-09-08 13:42:29', '%Y-%m-%d %H:%M:%S'); ----- -2020-09-08T13:42:29-04:00 - -## Test 16: to_timestamp with format string - explicit timezone should be respected -statement ok -SET datafusion.execution.time_zone = 'UTC'; - -query P -SELECT to_timestamp('2020-09-08 13:42:29 +0000', '%Y-%m-%d %H:%M:%S %z'); ----- -2020-09-08T13:42:29Z - -query P -SELECT to_timestamp('2020-09-08 13:42:29 America/Toronto', '%Y-%m-%d %H:%M:%S %Z'); ----- -2020-09-08T17:42:29Z - -query error Error parsing timestamp from '2020-09-08 13:42:29America/Toronto' using format '%Y-%m-%d %H:%M:%S%Z': '%Z' is only supported at the end of the format string preceded by a space -SELECT to_timestamp('2020-09-08 13:42:29America/Toronto', '%Y-%m-%d %H:%M:%S%Z'); - -## Test 17: Test all precision variants respect timezone -statement ok -SET datafusion.execution.time_zone = 'America/New_York'; - -## to_timestamp_seconds -query P -SELECT to_timestamp_seconds('2020-09-08T13:42:29'); ----- -2020-09-08T13:42:29-04:00 - -## to_timestamp_millis -query P -SELECT to_timestamp_millis('2020-09-08T13:42:29.123'); ----- -2020-09-08T13:42:29.123-04:00 - -## to_timestamp_micros -query P -SELECT to_timestamp_micros('2020-09-08T13:42:29.123456'); ----- -2020-09-08T13:42:29.123456-04:00 - -## to_timestamp_nanos -query P -SELECT to_timestamp_nanos('2020-09-08T13:42:29.123456789'); ----- -2020-09-08T13:42:29.123456789-04:00 - -## test integers -query T -select arrow_typeof(to_timestamp_seconds(61)) ----- -Timestamp(s, "America/New_York") - -query T -select arrow_typeof(to_timestamp_millis(61)) ----- -Timestamp(ms, "America/New_York") - -query T -select arrow_typeof(to_timestamp_micros(61)) ----- -Timestamp(µs, "America/New_York") - -query T -select arrow_typeof(to_timestamp_nanos(61)) ----- -Timestamp(ns, "America/New_York") - -query T -select arrow_typeof(to_timestamp(61)) ----- -Timestamp(ns, "America/New_York") - -## Reset timezone for other tests -statement ok -RESET datafusion.execution.time_zone diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index 254151c2c20eb..fa496dd70cba2 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -2878,20 +2878,9 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ### `to_timestamp` -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. -Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). - -Note: `to_timestamp` returns `Timestamp(ns, TimeZone)` where the time zone is the session time zone. The supported range -for integer input is between`-9223372037` and `9223372036`. Supported range for string input is between -`1677-09-21T00:12:44.0` and `2262-04-11T23:47:16.0`. Please use `to_timestamp_seconds` -for the input outside of supported bounds. +Converts a value to a timestamp (`YYYY-MM-DDT00:00:00Z`). Supports strings, integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp. -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. +Note: `to_timestamp` returns `Timestamp(ns)`. The supported range for integer input is between `-9223372037` and `9223372036`. Supported range for string input is between `1677-09-21T00:12:44.0` and `2262-04-11T23:47:16.0`. Please use `to_timestamp_seconds` for the input outside of supported bounds. ```sql to_timestamp(expression[, ..., format_n]) @@ -2900,11 +2889,7 @@ to_timestamp(expression[, ..., format_n]) #### Arguments - **expression**: Expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators. -- **format_n**: - Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. - Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is - only supported at the end of the string preceded by a space. +- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. #### Example @@ -2927,15 +2912,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ### `to_timestamp_micros` -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. -Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as microseconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. +Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided. Integers and unsigned integers are interpreted as microseconds since the unix epoch (`1970-01-01T00:00:00Z`) Returns the corresponding timestamp. ```sql to_timestamp_micros(expression[, ..., format_n]) @@ -2944,11 +2921,7 @@ to_timestamp_micros(expression[, ..., format_n]) #### Arguments - **expression**: Expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators. -- **format_n**: - Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. - Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is - only supported at the end of the string preceded by a space. +- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. #### Example @@ -2971,15 +2944,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ### `to_timestamp_millis` -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. -Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as milliseconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. +Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. Integers and unsigned integers are interpreted as milliseconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp. ```sql to_timestamp_millis(expression[, ..., format_n]) @@ -2988,11 +2953,7 @@ to_timestamp_millis(expression[, ..., format_n]) #### Arguments - **expression**: Expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators. -- **format_n**: - Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. - Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is - only supported at the end of the string preceded by a space. +- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. #### Example @@ -3015,14 +2976,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ### `to_timestamp_nanos` -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. -Strings that parse without a time zone are treated as if they are in the -session time zone. Integers, unsigned integers, and doubles are interpreted as nanoseconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. +Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided. Integers and unsigned integers are interpreted as nanoseconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp. ```sql to_timestamp_nanos(expression[, ..., format_n]) @@ -3031,11 +2985,7 @@ to_timestamp_nanos(expression[, ..., format_n]) #### Arguments - **expression**: Expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators. -- **format_n**: - Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. - Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is - only supported at the end of the string preceded by a space. +- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. #### Example @@ -3058,15 +3008,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ### `to_timestamp_seconds` -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. -Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. +Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided. Integers and unsigned integers are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp. ```sql to_timestamp_seconds(expression[, ..., format_n]) @@ -3075,11 +3017,7 @@ to_timestamp_seconds(expression[, ..., format_n]) #### Arguments - **expression**: Expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators. -- **format_n**: - Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. - Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is - only supported at the end of the string preceded by a space. +- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. #### Example From d91988f33c35b32c2e7cafab54644c43500486dd Mon Sep 17 00:00:00 2001 From: Adam Curtis Date: Tue, 19 May 2026 13:17:17 -0700 Subject: [PATCH 06/25] improve OneOf signature diagnostics Cherry-pick of apache/datafusion#21032. Adapted for 52.5.0: - Two unused imports in datafusion-expr's test module (`IntervalUnit`, `logical_int64`) compile-fail under -D warnings; dropped. - Two sql-integration tests and several sqllogictest expected-error patterns assumed the friendlier Display impl from apache/datafusion#20605 was already on 52.5.0 (it isn't); updated those patterns to the pre-#20605 verbose `TypeSignatureClass::Native(LogicalType(...))` shape and to the post-#21032 `failed with: ...` (no quoted inner message) shape. - Also includes the author's own follow-up CI fix (was a separate commit upstream, folded here). Signed-off-by: yaommen (cherry picked from commit f478cdcda838dfd8c7c41f71bcd8bdc63a4ce21e) --- .../expr/src/type_coercion/functions.rs | 569 ++++++++++++++++-- datafusion/expr/src/udaf.rs | 31 +- datafusion/functions-aggregate/src/average.rs | 16 +- datafusion/functions-aggregate/src/sum.rs | 17 +- datafusion/functions-window/src/nth_value.rs | 21 +- datafusion/sql/tests/sql_integration.rs | 4 +- datafusion/sqllogictest/test_files/array.slt | 10 +- datafusion/sqllogictest/test_files/errors.slt | 28 +- .../sqllogictest/test_files/functions.slt | 8 +- .../test_files/named_arguments.slt | 2 +- .../test_files/spark/math/hex.slt | 2 +- .../test_files/string/string_literal.slt | 4 +- 12 files changed, 641 insertions(+), 71 deletions(-) diff --git a/datafusion/expr/src/type_coercion/functions.rs b/datafusion/expr/src/type_coercion/functions.rs index fe259fb8c972a..b099c2d9d7455 100644 --- a/datafusion/expr/src/type_coercion/functions.rs +++ b/datafusion/expr/src/type_coercion/functions.rs @@ -16,20 +16,22 @@ // under the License. use super::binary::binary_numeric_coercion; -use crate::{AggregateUDF, ScalarUDF, Signature, TypeSignature, WindowUDF}; +use crate::{ + AggregateUDF, ScalarUDF, Signature, TypeSignature, TypeSignatureClass, WindowUDF, +}; use arrow::datatypes::{Field, FieldRef}; use arrow::{ compute::can_cast_types, datatypes::{DataType, TimeUnit}, }; -use datafusion_common::types::LogicalType; +use datafusion_common::types::{LogicalType, NativeType, logical_string}; use datafusion_common::utils::{ ListCoercion, base_type, coerced_fixed_size_list_to_list, }; use datafusion_common::{ - Result, exec_err, internal_err, plan_err, types::NativeType, utils::list_ndims, + DataFusionError, Result, internal_err, plan_err, utils::list_ndims, }; -use datafusion_expr_common::signature::ArrayFunctionArgument; +use datafusion_expr_common::signature::{Arity, ArrayFunctionArgument, Coercion}; use datafusion_expr_common::type_coercion::binary::type_union_resolution; use datafusion_expr_common::{ signature::{ArrayFunctionSignature, FIXED_SIZE_LIST_WILDCARD, TIMEZONE_WILDCARD}, @@ -37,6 +39,7 @@ use datafusion_expr_common::{ type_coercion::binary::string_coercion, }; use itertools::Itertools as _; +use std::collections::BTreeSet; use std::sync::Arc; /// Extension trait to unify common functionality between [`ScalarUDF`], [`AggregateUDF`] @@ -50,6 +53,15 @@ pub trait UDFCoercionExt { /// Should delegate to [`ScalarUDF::coerce_types`], [`AggregateUDF::coerce_types`] /// or [`WindowUDF::coerce_types`]. fn coerce_types(&self, arg_types: &[DataType]) -> Result>; + + /// Return a function-specific signature failure when generic `OneOf` + /// diagnostics are not descriptive enough. + fn diagnose_failed_signature( + &self, + _arg_types: &[DataType], + ) -> Option { + None + } } impl UDFCoercionExt for ScalarUDF { @@ -78,6 +90,13 @@ impl UDFCoercionExt for AggregateUDF { fn coerce_types(&self, arg_types: &[DataType]) -> Result> { self.coerce_types(arg_types) } + + fn diagnose_failed_signature( + &self, + arg_types: &[DataType], + ) -> Option { + self.diagnose_failed_signature(arg_types) + } } impl UDFCoercionExt for WindowUDF { @@ -307,51 +326,305 @@ fn try_coerce_types( ) } -fn get_valid_types_with_udf( +#[derive(Debug)] +struct PlanFailure { + err: DataFusionError, + matches_arity: bool, + has_hint: bool, + detail: Option, +} + +#[derive(Debug)] +enum SignatureFailure { + Plan(PlanFailure), + NonPlan(DataFusionError), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TypeMismatchDetail { + arg_index: usize, + required_type: String, + received_type: String, + data_type: String, +} + +fn signature_matches_arity(signature: &TypeSignature, actual: usize) -> bool { + match signature.arity() { + Arity::Fixed(expected) => expected == actual, + Arity::Variable => true, + } +} + +fn collect_fixed_arities(signature: &TypeSignature, arities: &mut BTreeSet) { + match signature { + TypeSignature::OneOf(signatures) => { + for signature in signatures { + collect_fixed_arities(signature, arities); + } + } + _ => { + if let Arity::Fixed(expected) = signature.arity() { + arities.insert(expected); + } + } + } +} + +fn build_one_of_arity_error( + function_name: &str, + actual: usize, + expected_arities: &[usize], +) -> Option { + if expected_arities.is_empty() { + return None; + } + + let expected = expected_arities.to_vec(); + let message = if expected.len() == 1 { + format!( + "Function '{function_name}' expects {} arguments but received {actual}", + expected[0] + ) + } else if expected.windows(2).all(|window| window[1] == window[0] + 1) { + format!( + "Function '{function_name}' expects {} to {} arguments but received {actual}", + expected.first().unwrap(), + expected.last().unwrap() + ) + } else { + format!( + "Function '{function_name}' expects one of {} arguments but received {actual}", + expected.iter().join(", ") + ) + }; + + Some(DataFusionError::Plan(message)) +} + +fn combine_type_mismatch_failures( + function_name: &str, + failures: &[PlanFailure], +) -> Option { + let details = failures + .iter() + .map(|failure| failure.detail.as_ref()) + .collect::>>()?; + + let first = details.first()?; + if !details.iter().all(|detail| { + detail.arg_index == first.arg_index + && detail.received_type == first.received_type + && detail.data_type == first.data_type + }) { + return None; + } + + let required_types = details + .iter() + .map(|detail| detail.required_type.as_str()) + .collect::>() + .into_iter() + .collect::>(); + + if required_types.is_empty() { + return None; + } + + let required = if required_types.len() == 1 { + required_types[0].to_string() + } else { + format!("one of {}", required_types.join(", ")) + }; + + Some(DataFusionError::Plan(format!( + "Function '{function_name}' requires {required}, but received {} (DataType: {}).", + first.received_type, first.data_type + ))) +} + +fn select_best_plan_failure( + function_name: &str, + failures: Vec, +) -> Option { + let matching_failures = failures + .into_iter() + .filter(|failure| failure.matches_arity) + .collect::>(); + + if matching_failures.is_empty() { + return None; + } + + if matching_failures.len() == 1 { + return Some(matching_failures.into_iter().next().unwrap().err); + } + + let hinted = matching_failures + .iter() + .enumerate() + .filter_map(|(idx, failure)| failure.has_hint.then_some(idx)) + .collect::>(); + if hinted.len() == 1 { + return Some(matching_failures.into_iter().nth(hinted[0]).unwrap().err); + } + + let messages = matching_failures + .iter() + .map(|failure| failure.err.strip_backtrace().to_string()) + .collect::>(); + if messages.windows(2).all(|pair| pair[0] == pair[1]) { + return Some(matching_failures.into_iter().next().unwrap().err); + } + + combine_type_mismatch_failures(function_name, &matching_failures) +} + +fn coercible_mismatch_detail( + coercions: &[Coercion], + current_types: &[DataType], +) -> Option { + if coercions.len() != current_types.len() { + return None; + } + + for (arg_index, (param, current_type)) in + coercions.iter().zip(current_types.iter()).enumerate() + { + let current_native_type = NativeType::from(current_type); + let desired_type = param.desired_type(); + let desired_matches = desired_type.matches_native_type(¤t_native_type); + let allowed_matches = param + .allowed_source_types() + .iter() + .any(|source_type| source_type.matches_native_type(¤t_native_type)); + + if !desired_matches && !allowed_matches { + return Some(TypeMismatchDetail { + arg_index, + required_type: desired_type.to_string(), + received_type: current_native_type.to_string(), + data_type: current_type.to_string(), + }); + } + } + + None +} + +fn to_signature_failure( signature: &TypeSignature, current_types: &[DataType], + err: DataFusionError, +) -> Box { + match err { + DataFusionError::Plan(message) => Box::new(SignatureFailure::Plan(PlanFailure { + matches_arity: signature_matches_arity(signature, current_types.len()), + has_hint: message.contains("Hint:"), + detail: match signature { + TypeSignature::Coercible(coercions) => { + coercible_mismatch_detail(coercions, current_types) + } + _ => None, + }, + err: DataFusionError::Plan(message), + })), + err => Box::new(SignatureFailure::NonPlan(err)), + } +} + +fn evaluate_one_of_with_udf( + signatures: &[TypeSignature], + current_types: &[DataType], func: &F, ) -> Result>> { - let valid_types = match signature { + let actual = current_types.len(); + let mut valid_types = vec![]; + let mut plan_failures = vec![]; + let mut deferred_non_plan = None; + let mut fixed_arities = BTreeSet::new(); + + for signature in signatures { + collect_fixed_arities(signature, &mut fixed_arities); + + match evaluate_signature_with_udf(signature, current_types, func) { + Ok(types) => valid_types.extend(types), + Err(failure) => match *failure { + SignatureFailure::Plan(failure) => plan_failures.push(failure), + SignatureFailure::NonPlan(err) => { + if deferred_non_plan.is_none() { + deferred_non_plan = Some(err); + } + } + }, + } + } + + if !valid_types.is_empty() { + return Ok(valid_types); + } + + if let Some(err) = func.diagnose_failed_signature(current_types) { + return Err(err); + } + + if let Some(err) = select_best_plan_failure(func.name(), plan_failures) { + return Err(err); + } + + let matching_arity_exists = signatures + .iter() + .any(|signature| signature_matches_arity(signature, actual)); + let expected_arities = fixed_arities.iter().copied().collect::>(); + if !matching_arity_exists + && let Some(err) = + build_one_of_arity_error(func.name(), actual, &expected_arities) + { + return Err(err); + } + + if let Some(err) = deferred_non_plan { + return Err(err); + } + + plan_err!("Function '{}' failed to match any signature", func.name()) +} + +fn evaluate_signature_with_udf( + signature: &TypeSignature, + current_types: &[DataType], + func: &F, +) -> std::result::Result>, Box> { + match signature { TypeSignature::UserDefined => match func.coerce_types(current_types) { - Ok(coerced_types) => vec![coerced_types], - Err(e) => { - return exec_err!( - "Function '{}' user-defined coercion failed with {:?}", + Ok(coerced_types) => Ok(vec![coerced_types]), + Err(e) => Err(Box::new(SignatureFailure::NonPlan( + DataFusionError::Execution(format!( + "Function '{}' user-defined coercion failed with: {}", func.name(), e.strip_backtrace() - ); - } + )), + ))), }, TypeSignature::OneOf(signatures) => { - let mut res = vec![]; - let mut errors = vec![]; - for sig in signatures { - match get_valid_types_with_udf(sig, current_types, func) { - Ok(valid_types) => { - res.extend(valid_types); - } - Err(e) => { - errors.push(e.to_string()); - } - } - } - - // Every signature failed, return the joined error - if res.is_empty() { - return internal_err!( - "Function '{}' failed to match any signature, errors: {}", - func.name(), - errors.join(",") - ); - } else { - res - } + evaluate_one_of_with_udf(signatures, current_types, func) + .map_err(|err| to_signature_failure(signature, current_types, err)) } - _ => get_valid_types(func.name(), signature, current_types)?, - }; + _ => get_valid_types(func.name(), signature, current_types) + .map_err(|err| to_signature_failure(signature, current_types, err)), + } +} - Ok(valid_types) +fn get_valid_types_with_udf( + signature: &TypeSignature, + current_types: &[DataType], + func: &F, +) -> Result>> { + evaluate_signature_with_udf(signature, current_types, func).map_err(|failure| { + match *failure { + SignatureFailure::Plan(failure) => failure.err, + SignatureFailure::NonPlan(err) => err, + } + }) } /// Returns a Vec of all possible valid argument types for the given signature. @@ -635,7 +908,10 @@ fn get_valid_types( default_casted_type.default_cast_for(current_type)?; new_types.push(casted_type); } else { - let hint = if matches!(current_native_type, NativeType::Binary) { + let hint = if matches!(current_native_type, NativeType::Binary) + && param.desired_type() + == &TypeSignatureClass::Native(logical_string()) + { "\n\nHint: Binary types are not automatically coerced to String. Use CAST(column AS VARCHAR) to convert Binary data to String." } else { "" @@ -939,7 +1215,7 @@ mod tests { use super::*; use arrow::datatypes::Field; use datafusion_common::{ - assert_contains, + assert_contains, exec_err, types::{logical_binary, logical_int64}, }; use datafusion_expr_common::signature::{Coercion, TypeSignatureClass}; @@ -1145,6 +1421,219 @@ mod tests { } } + struct AlwaysExecErrUdf(Signature); + + impl UDFCoercionExt for AlwaysExecErrUdf { + fn name(&self) -> &str { + "test" + } + + fn signature(&self) -> &Signature { + &self.0 + } + + fn coerce_types(&self, _arg_types: &[DataType]) -> Result> { + exec_err!("boom") + } + } + + struct DiagnoseErrUdf(Signature); + + impl UDFCoercionExt for DiagnoseErrUdf { + fn name(&self) -> &str { + "test" + } + + fn signature(&self) -> &Signature { + &self.0 + } + + fn coerce_types(&self, _arg_types: &[DataType]) -> Result> { + unimplemented!() + } + + fn diagnose_failed_signature( + &self, + _arg_types: &[DataType], + ) -> Option { + Some(DataFusionError::Plan( + "test semantic signature failure".to_string(), + )) + } + } + + #[test] + fn test_one_of_drops_internal_error_and_combines_type_errors() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Boolean, true))]; + let signature = Signature::one_of( + vec![ + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Decimal, + )]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Duration, + )]), + ], + Volatility::Immutable, + ); + + let err = fields_with_udf(¤t_fields, &MockUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Plan(_))); + let err = err.to_string(); + + assert_eq!( + err, + "Error during planning: Function 'test' requires one of TypeSignatureClass::Decimal, TypeSignatureClass::Duration, but received Boolean (DataType: Boolean)." + ); + assert!(!err.contains("Internal error")); + } + + #[test] + fn test_one_of_restores_arity_error() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Int32, true))]; + let signature = Signature::one_of( + vec![TypeSignature::Any(2), TypeSignature::Any(3)], + Volatility::Immutable, + ); + + let err = fields_with_udf(¤t_fields, &MockUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Plan(_))); + assert_eq!( + err.to_string(), + "Error during planning: Function 'test' expects 2 to 3 arguments but received 1" + ); + } + + #[test] + fn test_one_of_uses_specific_error_for_unique_matching_signature() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Int64, true))]; + let signature = Signature::one_of( + vec![ + TypeSignature::Any(3), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_string()), + )]), + ], + Volatility::Immutable, + ); + + let err = fields_with_udf(¤t_fields, &MockUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Plan(_))); + assert!( + err.to_string().starts_with( + "Error during planning: Function 'test' requires TypeSignatureClass::Native(LogicalType(Native(String), String)), but received Int64 (DataType: Int64)." + ), + "{}", + err + ); + } + + #[test] + fn test_one_of_prefers_hinted_matching_signature() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Binary, true))]; + let signature = Signature::one_of( + vec![ + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Decimal, + )]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_string()), + )]), + ], + Volatility::Immutable, + ); + + let err = fields_with_udf(¤t_fields, &MockUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Plan(_))); + let err = err.to_string(); + + assert!(err.contains( + "Function 'test' requires TypeSignatureClass::Native(LogicalType(Native(String), String)), but received Binary" + )); + assert!( + err.contains("Hint: Binary types are not automatically coerced to String.") + ); + } + + #[test] + fn test_one_of_combines_same_arity_type_mismatches() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Utf8, true))]; + let signature = Signature::one_of( + vec![ + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Decimal, + )]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Float, + )]), + ], + Volatility::Immutable, + ); + + let err = fields_with_udf(¤t_fields, &MockUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Plan(_))); + assert_eq!( + err.to_string(), + "Error during planning: Function 'test' requires one of TypeSignatureClass::Decimal, TypeSignatureClass::Float, but received String (DataType: Utf8)." + ); + } + + #[test] + fn test_one_of_propagates_non_plan_errors_when_no_plan_error_exists() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Int32, true))]; + let signature = + Signature::one_of(vec![TypeSignature::UserDefined], Volatility::Immutable); + + let err = + fields_with_udf(¤t_fields, &AlwaysExecErrUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Execution(_))); + assert!(err.to_string().starts_with( + "Execution error: Function 'test' user-defined coercion failed with: Execution error: boom" + )); + } + + #[test] + fn test_one_of_keeps_success_when_later_user_defined_branch_fails() -> Result<()> { + let current_fields = vec![Arc::new(Field::new("t", DataType::Int32, true))]; + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Int32]), + TypeSignature::UserDefined, + ], + Volatility::Immutable, + ); + + let fields = fields_with_udf(¤t_fields, &AlwaysExecErrUdf(signature))?; + + assert_eq!(fields.len(), 1); + assert_eq!(fields[0].data_type(), &DataType::Int32); + Ok(()) + } + + #[test] + fn test_one_of_allows_function_specific_diagnostic_override() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Boolean, true))]; + let signature = Signature::one_of( + vec![ + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Decimal, + )]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Duration, + )]), + ], + Volatility::Immutable, + ); + + let err = + fields_with_udf(¤t_fields, &DiagnoseErrUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Plan(_))); + assert_eq!( + err.to_string(), + "Error during planning: test semantic signature failure" + ); + } + #[test] fn test_fixed_list_wildcard_coerce() -> Result<()> { let inner = Arc::new(Field::new_list_field(DataType::Int32, false)); diff --git a/datafusion/expr/src/udaf.rs b/datafusion/expr/src/udaf.rs index ee38077dbf304..bb4501a560ef5 100644 --- a/datafusion/expr/src/udaf.rs +++ b/datafusion/expr/src/udaf.rs @@ -26,7 +26,9 @@ use std::vec; use arrow::datatypes::{DataType, Field, FieldRef}; -use datafusion_common::{Result, ScalarValue, Statistics, exec_err, not_impl_err}; +use datafusion_common::{ + DataFusionError, Result, ScalarValue, Statistics, exec_err, not_impl_err, +}; use datafusion_expr_common::dyn_eq::{DynEq, DynHash}; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; @@ -271,6 +273,16 @@ impl AggregateUDF { self.inner.coerce_types(arg_types) } + /// Returns a function-specific signature failure for argument combinations + /// that are better described semantically than by generic signature + /// diagnostics. + pub fn diagnose_failed_signature( + &self, + arg_types: &[DataType], + ) -> Option { + self.inner.diagnose_failed_signature(arg_types) + } + /// See [`AggregateUDFImpl::with_beneficial_ordering`] for more details. pub fn with_beneficial_ordering( self, @@ -711,6 +723,16 @@ pub trait AggregateUDFImpl: Debug + DynEq + DynHash + Send + Sync { not_impl_err!("Function {} does not implement coerce_types", self.name()) } + /// Returns a semantic signature failure for argument combinations that + /// should be surfaced directly instead of going through generic `OneOf` + /// diagnostics. + fn diagnose_failed_signature( + &self, + _arg_types: &[DataType], + ) -> Option { + None + } + /// If this function is max, return true /// If the function is min, return false /// Otherwise return None (the default) @@ -1243,6 +1265,13 @@ impl AggregateUDFImpl for AliasedAggregateUDFImpl { self.inner.coerce_types(arg_types) } + fn diagnose_failed_signature( + &self, + arg_types: &[DataType], + ) -> Option { + self.inner.diagnose_failed_signature(arg_types) + } + fn return_field(&self, arg_fields: &[FieldRef]) -> Result { self.inner.return_field(arg_fields) } diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rs index 543116db1ddb6..d8d7e5223cfcd 100644 --- a/datafusion/functions-aggregate/src/average.rs +++ b/datafusion/functions-aggregate/src/average.rs @@ -32,7 +32,9 @@ use arrow::datatypes::{ DurationSecondType, Field, FieldRef, Float64Type, TimeUnit, UInt64Type, i256, }; use datafusion_common::types::{NativeType, logical_float64}; -use datafusion_common::{Result, ScalarValue, exec_err, not_impl_err}; +use datafusion_common::{ + DataFusionError, Result, ScalarValue, exec_err, not_impl_err, plan_datafusion_err, +}; use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion_expr::utils::format_state_name; use datafusion_expr::{ @@ -139,6 +141,18 @@ impl AggregateUDFImpl for Avg { &self.signature } + fn diagnose_failed_signature( + &self, + arg_types: &[DataType], + ) -> Option { + match arg_types { + [DataType::Boolean] => { + Some(plan_datafusion_err!("Avg not supported for Boolean")) + } + _ => None, + } + } + fn return_type(&self, arg_types: &[DataType]) -> Result { match &arg_types[0] { DataType::Decimal32(precision, scale) => { diff --git a/datafusion/functions-aggregate/src/sum.rs b/datafusion/functions-aggregate/src/sum.rs index 198ba54adfa2a..a0a89d7538d27 100644 --- a/datafusion/functions-aggregate/src/sum.rs +++ b/datafusion/functions-aggregate/src/sum.rs @@ -31,7 +31,10 @@ use datafusion_common::types::{ NativeType, logical_float64, logical_int8, logical_int16, logical_int32, logical_int64, logical_uint8, logical_uint16, logical_uint32, logical_uint64, }; -use datafusion_common::{HashMap, Result, ScalarValue, exec_err, not_impl_err}; +use datafusion_common::{ + DataFusionError, HashMap, Result, ScalarValue, exec_err, not_impl_err, + plan_datafusion_err, +}; use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name}; use datafusion_expr::{ @@ -210,6 +213,18 @@ impl AggregateUDFImpl for Sum { &self.signature } + fn diagnose_failed_signature( + &self, + arg_types: &[DataType], + ) -> Option { + match arg_types { + [DataType::Boolean] => { + Some(plan_datafusion_err!("Sum not supported for Boolean")) + } + _ => None, + } + } + fn return_type(&self, arg_types: &[DataType]) -> Result { match &arg_types[0] { DataType::Int64 => Ok(DataType::Int64), diff --git a/datafusion/functions-window/src/nth_value.rs b/datafusion/functions-window/src/nth_value.rs index 8d37cf7e604a9..22883842358ff 100644 --- a/datafusion/functions-window/src/nth_value.rs +++ b/datafusion/functions-window/src/nth_value.rs @@ -28,7 +28,7 @@ use datafusion_doc::window_doc_sections::DOC_SECTION_ANALYTICAL; use datafusion_expr::window_state::WindowAggState; use datafusion_expr::{ Documentation, LimitEffect, Literal, PartitionEvaluator, ReversedUDWF, Signature, - TypeSignature, Volatility, WindowUDFImpl, + Volatility, WindowUDFImpl, }; use datafusion_functions_window_common::field; use datafusion_functions_window_common::partition::PartitionEvaluatorArgs; @@ -94,17 +94,14 @@ pub struct NthValue { impl NthValue { /// Create a new `nth_value` function pub fn new(kind: NthValueKind) -> Self { - Self { - signature: Signature::one_of( - vec![ - TypeSignature::Nullary, - TypeSignature::Any(1), - TypeSignature::Any(2), - ], - Volatility::Immutable, - ), - kind, - } + let signature = match kind { + NthValueKind::First | NthValueKind::Last => { + Signature::any(1, Volatility::Immutable) + } + NthValueKind::Nth => Signature::any(2, Volatility::Immutable), + }; + + Self { signature, kind } } pub fn first() -> Self { diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs index 444bdae73ac26..ddd2d9f033ed2 100644 --- a/datafusion/sql/tests/sql_integration.rs +++ b/datafusion/sql/tests/sql_integration.rs @@ -4756,7 +4756,7 @@ fn test_error_message_invalid_scalar_function_signature() { fn test_error_message_invalid_aggregate_function_signature() { error_message_test( "select sum()", - "Error during planning: Execution error: Function 'sum' user-defined coercion failed with \"Execution error: sum function requires 1 argument, got 0\"", + "Error during planning: Execution error: Function 'sum' user-defined coercion failed with: Execution error: sum function requires 1 argument, got 0", ); // We keep two different prefixes because they clarify each other. // It might be incorrect, and we should consider keeping only one. @@ -4778,7 +4778,7 @@ fn test_error_message_invalid_window_function_signature() { fn test_error_message_invalid_window_aggregate_function_signature() { error_message_test( "select sum() over()", - "Error during planning: Execution error: Function 'sum' user-defined coercion failed with \"Execution error: sum function requires 1 argument, got 0\"", + "Error during planning: Execution error: Function 'sum' user-defined coercion failed with: Execution error: sum function requires 1 argument, got 0", ); } diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index 45c6dd48996a2..e59e62aa4c990 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -3454,7 +3454,7 @@ select [1, 2, 3] List(Utf8View) # array_concat error -query error DataFusion error: Error during planning: Execution error: Function 'array_concat' user-defined coercion failed with "Error during planning: array_concat does not support type Int64" +query error DataFusion error: Error during planning: Execution error: Function 'array_concat' user-defined coercion failed with: Error during planning: array_concat does not support type Int64 select array_concat(1, 2); # array_concat scalar function #1 @@ -7874,17 +7874,17 @@ select generate_series(arrow_cast('2021-01-01T00:00:00', 'Timestamp(Nanosecond, [2021-01-01T00:00:00-05:00, 2021-01-01T01:29:54.500-05:00, 2021-01-01T02:59:49-05:00, 2021-01-01T04:29:43.500-05:00, 2021-01-01T05:59:38-05:00] ## mixing types for timestamps is not supported -query error DataFusion error: Error during planning: Internal error: Function 'generate_series' failed to match any signature +query error DataFusion error: Error during planning: Function 'generate_series' failed to match any signature select generate_series(arrow_cast('2021-01-01T00:00:00', 'Timestamp(Nanosecond, Some("-05:00"))'), DATE '2021-01-02', INTERVAL '1' HOUR); ## mixing types not allowed even if an argument is null -query error DataFusion error: Error during planning: Internal error: Function 'generate_series' failed to match any signature +query error DataFusion error: Error during planning: Function 'generate_series' failed to match any signature select generate_series(TIMESTAMP '1992-09-01', DATE '1993-03-01', NULL); -query error DataFusion error: Error during planning: Internal error: Function 'generate_series' failed to match any signature +query error DataFusion error: Error during planning: Function 'generate_series' failed to match any signature select generate_series(1, '2024-01-01', '2025-01-02'); -query error DataFusion error: Error during planning: Internal error: Function 'generate_series' failed to match any signature +query error DataFusion error: Error during planning: Function 'generate_series' failed to match any signature select generate_series('2024-01-01'::timestamp, '2025-01-02', interval '1 day'); ## should return NULL diff --git a/datafusion/sqllogictest/test_files/errors.slt b/datafusion/sqllogictest/test_files/errors.slt index 20c1db5cb1511..64db3740ad86d 100644 --- a/datafusion/sqllogictest/test_files/errors.slt +++ b/datafusion/sqllogictest/test_files/errors.slt @@ -125,13 +125,39 @@ from aggregate_test_100 order by c9 # WindowFunction wrong signature -statement error DataFusion error: Error during planning: Internal error: Function 'nth_value' failed to match any signature +statement error DataFusion error: Error during planning: The function 'nth_value' expected 2 arguments but received 3 select c9, nth_value(c5, 2, 3) over (order by c9) as nv1 from aggregate_test_100 order by c9 +statement error DataFusion error: Error during planning: The function 'nth_value' expected 2 arguments but received 1 +select +c9, +nth_value(c5) over (order by c9) as nv2 +from aggregate_test_100 +order by c9 + +statement error DataFusion error: Error during planning: The function 'first_value' expected 1 arguments but received 2 +select +c9, +first_value(c5, 2) over (order by c9) as fv1 +from aggregate_test_100 +order by c9 + +statement error DataFusion error: Error during planning: 'nth_value' does not support zero arguments +select nth_value() over (order by x) from (values (1), (2)) as t(x) + +statement error DataFusion error: Error during planning: 'first_value' does not support zero arguments +select first_value() over (order by x) from (values (1), (2)) as t(x) + +query error DataFusion error: Error during planning: Sum not supported for Boolean +select sum(bool_col) from (values (true), (false), (null)) as t(bool_col); + +query error DataFusion error: Error during planning: Avg not supported for Boolean +select avg(bool_col) from (values (true), (false), (null)) as t(bool_col); + # nth_value with wrong name statement error DataFusion error: Error during planning: Invalid function 'nth_vlue'.\nDid you mean 'nth_value'? diff --git a/datafusion/sqllogictest/test_files/functions.slt b/datafusion/sqllogictest/test_files/functions.slt index 5a43d18e23879..0cb5773520c00 100644 --- a/datafusion/sqllogictest/test_files/functions.slt +++ b/datafusion/sqllogictest/test_files/functions.slt @@ -208,10 +208,10 @@ SELECT substr('alphabet', NULL, 2) ---- NULL -statement error Function 'substr' failed to match any signature +statement error DataFusion error: Error during planning: Function 'substr' requires TypeSignatureClass::Native\(LogicalType\(Native\(String\), String\)\), but received Int64 \(DataType: Int64\) SELECT substr(1, 3) -statement error Function 'substr' failed to match any signature +statement error DataFusion error: Error during planning: Function 'substr' requires TypeSignatureClass::Native\(LogicalType\(Native\(String\), String\)\), but received Int64 \(DataType: Int64\) SELECT substr(1, 3, 4) query T @@ -887,7 +887,7 @@ SELECT greatest(-1, 1, 2.3, 123456789, 3 + 5, -(-4), abs(-9.0)) 123456789 -query error Function 'greatest' user-defined coercion failed with "Error during planning: greatest was called without any arguments. It requires at least 1." +query error Function 'greatest' user-defined coercion failed with: Error during planning: greatest was called without any arguments. It requires at least 1. SELECT greatest() query I @@ -1085,7 +1085,7 @@ SELECT least(-1, 1, 2.3, 123456789, 3 + 5, -(-4), abs(-9.0)) -1 -query error Function 'least' user-defined coercion failed with "Error during planning: least was called without any arguments. It requires at least 1." +query error Function 'least' user-defined coercion failed with: Error during planning: least was called without any arguments. It requires at least 1. SELECT least() query I diff --git a/datafusion/sqllogictest/test_files/named_arguments.slt b/datafusion/sqllogictest/test_files/named_arguments.slt index 07b6cc6a79a0c..f43246fc991fb 100644 --- a/datafusion/sqllogictest/test_files/named_arguments.slt +++ b/datafusion/sqllogictest/test_files/named_arguments.slt @@ -86,7 +86,7 @@ SELECT substr("STR" => 'hello world', "start_pos" => 7); # Error: wrong number of arguments # This query provides only 1 argument but substr requires 2 or 3 -query error Function 'substr' failed to match any signature +query error DataFusion error: Error during planning: Function 'substr' expects 2 to 3 arguments but received 1 SELECT substr(str => 'hello world'); ############# diff --git a/datafusion/sqllogictest/test_files/spark/math/hex.slt b/datafusion/sqllogictest/test_files/spark/math/hex.slt index 17e9ff432890d..442ede4e1ee68 100644 --- a/datafusion/sqllogictest/test_files/spark/math/hex.slt +++ b/datafusion/sqllogictest/test_files/spark/math/hex.slt @@ -56,7 +56,7 @@ SELECT hex(column1) FROM VALUES (arrow_cast('hello', 'LargeBinary')), (NULL), (a NULL 776F726C64 -statement error Function 'hex' expects 1 arguments but received 2 +statement error DataFusion error: Error during planning: Function 'hex' expects 1 arguments but received 2 SELECT hex(1, 2); query T diff --git a/datafusion/sqllogictest/test_files/string/string_literal.slt b/datafusion/sqllogictest/test_files/string/string_literal.slt index 6cf02218872df..1338cbe4328c5 100644 --- a/datafusion/sqllogictest/test_files/string/string_literal.slt +++ b/datafusion/sqllogictest/test_files/string/string_literal.slt @@ -132,10 +132,10 @@ SELECT substr('Hello🌏世界', 5, 3) ---- o🌏世 -statement error Function 'substr' failed to match any signature +statement error DataFusion error: Error during planning: Function 'substr' requires TypeSignatureClass::Native\(LogicalType\(Native\(String\), String\)\), but received Int64 \(DataType: Int64\) SELECT substr(1, 3) -statement error Function 'substr' failed to match any signature +statement error DataFusion error: Error during planning: Function 'substr' requires TypeSignatureClass::Native\(LogicalType\(Native\(String\), String\)\), but received Int64 \(DataType: Int64\) SELECT substr(1, 3, 4) statement error Execution error: negative substring length not allowed From 6d8e44677c2ecd32351dcc052d49fafbeaccf65f Mon Sep 17 00:00:00 2001 From: "xudong.w" Date: Mon, 20 Jul 2026 12:09:26 +0800 Subject: [PATCH 07/25] fix: prevent LEAD/LAG IGNORE NULLS panic without null bitmap (#23706) ## Which issue does this PR close? - Closes #23705. ## Rationale for this change Arrow arrays containing no null values commonly omit the null bitmap. The whole-partition evaluation path for `LEAD` and `LAG` with `IGNORE NULLS` unconditionally unwrapped that optional bitmap, causing a panic for valid non-null input arrays. ## What changes are included in this PR? - Fall back to the existing regular shift implementation when the input has no null bitmap. - Add a regression test covering both `LEAD` and `LAG` with an array that explicitly has no null bitmap. ## Are these changes tested? Yes. ## Are there any user-facing changes? `LEAD` and `LAG` with `IGNORE NULLS` no longer panic when the input Arrow array has no null bitmap. There are no public API changes. (cherry picked from commit a8d1af6b8a82ded56eb547ad698b224c1f8bf7be) --- datafusion/functions-window/src/lead_lag.rs | 29 +++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/datafusion/functions-window/src/lead_lag.rs b/datafusion/functions-window/src/lead_lag.rs index 7569dac9ac106..5564ac04122e9 100644 --- a/datafusion/functions-window/src/lead_lag.rs +++ b/datafusion/functions-window/src/lead_lag.rs @@ -428,8 +428,12 @@ fn evaluate_all_with_ignore_null( default_value: &ScalarValue, is_lag: bool, ) -> Result { - let valid_indices: Vec = - array.nulls().unwrap().valid_indices().collect::>(); + // Arrays without NULLs do not necessarily have a null bitmap. + let Some(nulls) = array.nulls() else { + return shift_with_default_value(array, offset, default_value); + }; + + let valid_indices: Vec = nulls.valid_indices().collect::>(); let direction = !is_lag; let new_array_results: Result, DataFusionError> = (0..array.len()) .map(|id| { @@ -827,4 +831,25 @@ mod tests { .collect::(), ) } + + #[test] + fn test_ignore_nulls_without_null_bitmap() -> Result<()> { + let input = Int32Array::from(vec![1, 2, 3]); + assert!(input.nulls().is_none()); + let input: ArrayRef = Arc::new(input); + + for (offset, expected) in [ + (1, Int32Array::from(vec![None, Some(1), Some(2)])), + (-1, Int32Array::from(vec![Some(2), Some(3), None])), + ] { + let actual = evaluate_all_with_ignore_null( + &input, + offset, + &ScalarValue::Int32(None), + offset > 0, + )?; + assert_eq!(expected, *as_int32_array(&actual)?); + } + Ok(()) + } } From 64fbe9dc4a7fe746c4892bd455f90d140a64f5c3 Mon Sep 17 00:00:00 2001 From: Reid Kaufmann Date: Wed, 12 Aug 2026 11:36:58 -0500 Subject: [PATCH 08/25] fix: fast paths for UnionExec schema recomputation UnionExec::try_new recomputes the union schema via union_schema() on every construction, including with_new_children rebuilds performed by optimizer passes. For wide unions whose children share one schema (generated UNION ALL, unions of per-partition scans) this makes physical planning O(n^2) in child count. Two fast paths: - union_schema(): if every input schema is pointer- or content-equal to the first, return the first schema (merging N identical schemas is the identity operation). - UnionExec::with_new_children(): when child count and per-position child schemas are unchanged, reuse the existing schema. Plan properties are always recomputed, since they can legitimately change when schemas do not. (DataFusion 53 has an analogous properties optimization upstream, apache/datafusion#19792; this variant targets the 51 line.) Adds a union_schema benchmark covering shared-Arc, content-equal, and adversarial (last child differs) shapes. All existing union unit tests pass. Co-Authored-By: Claude Fable 5 --- datafusion/physical-plan/Cargo.toml | 4 + .../physical-plan/benches/union_schema.rs | 180 ++++++++++++++++++ datafusion/physical-plan/src/union.rs | 31 +++ 3 files changed, 215 insertions(+) create mode 100644 datafusion/physical-plan/benches/union_schema.rs diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml index 6a28486cca5dc..b25e0f7453228 100644 --- a/datafusion/physical-plan/Cargo.toml +++ b/datafusion/physical-plan/Cargo.toml @@ -90,6 +90,10 @@ tokio = { workspace = true, features = [ harness = false name = "partial_ordering" +[[bench]] +harness = false +name = "union_schema" + [[bench]] harness = false name = "spill_io" diff --git a/datafusion/physical-plan/benches/union_schema.rs b/datafusion/physical-plan/benches/union_schema.rs new file mode 100644 index 0000000000000..5dc0e2b39b207 --- /dev/null +++ b/datafusion/physical-plan/benches/union_schema.rs @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Benchmark for `UnionExec` construction cost as a function of child count. +//! +//! Scenarios (run against a flat and a nested/struct schema): +//! - `shared_arc`: every child returns the same `Arc` +//! - `content_equal`: pointer-distinct but identical schemas per child +//! - `last_differs`: identical except the last child's deepest field (worst +//! case for any equality fast path: the scan is wasted, then the full +//! merge runs) + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use datafusion_physical_plan::ExecutionPlan; +use datafusion_physical_plan::empty::EmptyExec; +use datafusion_physical_plan::union::UnionExec; + +const NUM_FIELDS: usize = 10; +const NESTED_CHILDREN: usize = 5; +const METADATA_PER_FIELD: usize = 2; + +fn metadata(tag: &str, i: usize) -> HashMap { + (0..METADATA_PER_FIELD) + .map(|m| (format!("key_{m}"), format!("value_{tag}_{i}_{m}"))) + .collect() +} + +fn flat_schema() -> Schema { + let fields: Vec = (0..NUM_FIELDS) + .map(|i| { + Field::new(format!("col_{i}"), DataType::Int64, true) + .with_metadata(metadata("f", i)) + }) + .collect(); + Schema::new(fields) +} + +fn nested_schema() -> Schema { + let fields: Vec = (0..NUM_FIELDS) + .map(|i| { + let children: Vec = (0..NESTED_CHILDREN) + .map(|c| { + Field::new(format!("sub_{i}_{c}"), DataType::Int64, true) + .with_metadata(metadata("n", i * NESTED_CHILDREN + c)) + }) + .collect(); + Field::new(format!("col_{i}"), DataType::Struct(children.into()), true) + .with_metadata(metadata("s", i)) + }) + .collect(); + Schema::new(fields) +} + +/// Clone `schema` with extra metadata on its last field. Divergence must stay +/// at the top level: differing nested fields change the field `DataType` +/// itself, which `UnionExec::try_new` rejects ("Schemas have to be aligned"). +fn divergent(schema: &Schema) -> Schema { + let mut fields: Vec = + schema.fields().iter().map(|f| f.as_ref().clone()).collect(); + let last = fields.pop().unwrap(); + let mut md = last.metadata().clone(); + md.insert("divergent".to_string(), "true".to_string()); + fields.push(last.with_metadata(md)); + Schema::new(fields) +} + +fn child(schema: SchemaRef) -> Arc { + Arc::new(EmptyExec::new(schema)) +} + +fn children_shared_arc(schema: &Schema, n: usize) -> Vec> { + let schema: SchemaRef = Arc::new(schema.clone()); + (0..n).map(|_| child(Arc::clone(&schema))).collect() +} + +fn children_content_equal(schema: &Schema, n: usize) -> Vec> { + (0..n).map(|_| child(Arc::new(schema.clone()))).collect() +} + +/// First child keeps `schema`; the rest rename the first field — the common +/// real-world unequal union (`SELECT a .. UNION ALL SELECT b ..`), where any +/// equality check fails immediately. +fn children_names_differ(schema: &Schema, n: usize) -> Vec> { + let mut fields: Vec = + schema.fields().iter().map(|f| f.as_ref().clone()).collect(); + let first = fields.remove(0); + let renamed = first.clone().with_name(format!("renamed_{}", first.name())); + fields.insert(0, renamed); + let alt = Schema::new(fields); + let mut children = vec![child(Arc::new(schema.clone()))]; + children.extend((1..n).map(|_| child(Arc::new(alt.clone())))); + children +} + +fn children_last_differs(schema: &Schema, n: usize) -> Vec> { + let mut children = children_content_equal(schema, n - 1); + children.push(child(Arc::new(divergent(schema)))); + children +} + +fn bench_union_construction(c: &mut Criterion) { + for (suffix, schema, sizes) in [ + ("", flat_schema(), &[100usize, 1000, 4000][..]), + ("_nested", nested_schema(), &[1000, 4000][..]), + ] { + let mut group = c.benchmark_group(format!("union_exec_try_new{suffix}")); + for &n in sizes { + let shared = children_shared_arc(&schema, n); + let content = children_content_equal(&schema, n); + let differs = children_last_differs(&schema, n); + + group.bench_with_input( + BenchmarkId::new("shared_arc", n), + &shared, + |b, ch| b.iter(|| UnionExec::try_new(ch.clone()).unwrap()), + ); + group.bench_with_input( + BenchmarkId::new("content_equal", n), + &content, + |b, ch| b.iter(|| UnionExec::try_new(ch.clone()).unwrap()), + ); + group.bench_with_input( + BenchmarkId::new("last_differs", n), + &differs, + |b, ch| b.iter(|| UnionExec::try_new(ch.clone()).unwrap()), + ); + let names = children_names_differ(&schema, n); + group.bench_with_input( + BenchmarkId::new("names_differ", n), + &names, + |b, ch| b.iter(|| UnionExec::try_new(ch.clone()).unwrap()), + ); + } + group.finish(); + } + + // Reconstruct the union once per child, as optimizer rewrites do. + let mut group = c.benchmark_group("union_exec_rebuild_per_child"); + let schema = flat_schema(); + for n in [100, 1000] { + for (label, children) in [ + ("content_equal", children_content_equal(&schema, n)), + ("last_differs", children_last_differs(&schema, n)), + ] { + let union: Arc = + UnionExec::try_new(children.clone()).unwrap(); + group.bench_with_input(BenchmarkId::new(label, n), &n, |b, _| { + b.iter(|| { + let mut plan = Arc::clone(&union); + for _ in 0..n { + plan = plan.with_new_children(children.clone()).unwrap(); + } + plan + }) + }); + } + } + group.finish(); +} + +criterion_group!(benches, bench_union_construction); +criterion_main!(benches); diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs index 9fc02e730d022..e161ec96f9cb9 100644 --- a/datafusion/physical-plan/src/union.rs +++ b/datafusion/physical-plan/src/union.rs @@ -278,6 +278,26 @@ impl ExecutionPlan for UnionExec { children: Vec>, ) -> Result> { check_if_same_properties!(self, children); + // Fast path: if the children's schemas are unchanged the union schema + // cannot change, so skip the O(children x fields) `union_schema` + // recomputation. Plan properties can change even when schemas do not, + // so they are always recomputed. + if children.len() == self.inputs.len() + && children.len() >= 2 + && children.iter().zip(self.inputs.iter()).all(|(new, old)| { + let new_schema = new.schema(); + let old_schema = old.schema(); + Arc::ptr_eq(&new_schema, &old_schema) || new_schema == old_schema + }) + { + let schema = self.schema(); + let cache = Self::compute_properties(&children, schema)?; + return Ok(Arc::new(UnionExec { + inputs: children, + metrics: ExecutionPlanMetricsSet::new(), + cache: Arc::new(cache), + })); + } UnionExec::try_new(children) } @@ -706,6 +726,17 @@ fn union_schema(inputs: &[Arc]) -> Result { } } + // Merging N identical schemas is the identity operation: if every input + // schema is pointer- or content-equal to the first (`Schema` equality + // covers every property the merge below reads), return it directly + // instead of paying the per-field metadata merge. + if inputs.iter().all(|input| { + let schema = input.schema(); + Arc::ptr_eq(&schema, &first_schema) || schema == first_schema + }) { + return Ok(first_schema); + } + let fields = (0..first_field_count) .map(|i| { // We take the name from the left side of the union to match how names are coerced during logical planning, From 3cf8991f90e5087623e12e1bdd99327297025edc Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Tue, 22 Sep 2026 16:59:01 +0200 Subject: [PATCH 09/25] test(physical-plan): sort reservation charges a shared dictionary once per output chunk sort_batch_stream reserves the sum of get_record_batch_memory_size over the chunks from sort_batch_chunked. take() shares the dictionary values across every chunk, so the dictionary is counted once per chunk. A 2.5 MB batch with a 50k-value dictionary asks for 121.8 MB. With the disk manager disabled there is no spill fallback and the sort fails. --- datafusion/physical-plan/src/sorts/sort.rs | 61 ++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index ae881dcd4b791..570f61c246709 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -1428,6 +1428,7 @@ mod tests { use datafusion_common::{DataFusionError, Result, ScalarValue}; use datafusion_execution::RecordBatchStream; use datafusion_execution::config::SessionConfig; + use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; use datafusion_execution::runtime_env::RuntimeEnvBuilder; use datafusion_physical_expr::EquivalenceProperties; use datafusion_physical_expr::expressions::{Column, Literal}; @@ -1727,6 +1728,66 @@ mod tests { Ok(()) } + /// A single large batch with a dictionary column: `sort_batch_chunked` + /// shares the dictionary values across every output chunk, but the + /// reservation for the sorted output charges it once per chunk. + #[tokio::test] + async fn test_sort_single_batch_shared_dictionary_reservation() -> Result<()> { + const ROWS: usize = 100_000; + const CARDINALITY: usize = 50_000; + const BATCH_SIZE: usize = 1_000; + + let schema = Arc::new(Schema::new(vec![ + Field::new( + "tag", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + false, + ), + Field::new("time", DataType::Int64, false), + ])); + let mut tag = StringDictionaryBuilder::::new(); + for i in 0..ROWS { + tag.append_value(format!("truck_{:05}", (i * 7919) % CARDINALITY)); + } + let time = Int64Array::from_iter_values((0..ROWS).rev().map(|i| i as i64)); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(tag.finish()), Arc::new(time)], + )?; + let input_bytes = get_record_batch_memory_size(&batch); + + // Room for the input several times over, but not for the dictionary + // once per chunk. No disk manager, so the sort cannot spill. + let runtime = RuntimeEnvBuilder::new() + .with_memory_limit(input_bytes * 4, 1.0) + .with_disk_manager_builder( + DiskManagerBuilder::default().with_mode(DiskManagerMode::Disabled), + ) + .build_arc()?; + let task_ctx = Arc::new( + TaskContext::default() + .with_session_config(SessionConfig::new().with_batch_size(BATCH_SIZE)) + .with_runtime(runtime), + ); + + let plan = + TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?; + let sort_exec = Arc::new(SortExec::new( + [ + PhysicalSortExpr::new_default(col("tag", &schema)?), + PhysicalSortExpr::new_default(col("time", &schema)?), + ] + .into(), + plan, + )); + + let result = collect(sort_exec, task_ctx).await?; + let rows: usize = result.iter().map(|b| b.num_rows()).sum(); + assert_eq!(rows, ROWS); + assert_eq!(result.len(), ROWS.div_ceil(BATCH_SIZE)); + Ok(()) + } + #[tokio::test] async fn test_sort_spill_utf8_strings() -> Result<()> { let session_config = SessionConfig::new() From 3466f3d7c257ecfa3c97df2afb436b667f61be9f Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Tue, 22 Sep 2026 17:44:35 +0200 Subject: [PATCH 10/25] fix(physical-plan): count buffers shared across sorted output chunks once sort_batch_stream reserved the sum of per-chunk sizes for the output of sort_batch_chunked. Dictionary values are shared by every chunk after take(), so a batch with a large dictionary reserved chunks x dictionary and failed with ResourcesExhausted or starved the pool. Size the sorted output with one pass over all chunks so a shared buffer is counted once. ReservationStream releases per batch, and per-batch sizes can now add up to more than was reserved, so it releases at most what remains; the rest is freed when the stream ends. --- datafusion/common/src/utils/memory.rs | 41 +++++++++++++++++++++- datafusion/physical-plan/src/sorts/sort.rs | 6 ++-- datafusion/physical-plan/src/stream.rs | 11 ++++-- 3 files changed, 51 insertions(+), 7 deletions(-) diff --git a/datafusion/common/src/utils/memory.rs b/datafusion/common/src/utils/memory.rs index 78ec434d2b577..24808dac3aaf6 100644 --- a/datafusion/common/src/utils/memory.rs +++ b/datafusion/common/src/utils/memory.rs @@ -131,12 +131,22 @@ pub fn estimate_memory_size(num_elements: usize, fixed_size: usize) -> Result /// `Buffer`. This method provides temporary fix until the issue is resolved: /// pub fn get_record_batch_memory_size(batch: &RecordBatch) -> usize { + get_record_batches_memory_size(std::iter::once(batch)) +} + +/// Calculate total used memory of `batches`, counting a `Buffer` shared by +/// several batches (e.g. dictionary values across `take`n chunks) only once. +/// +/// See [`get_record_batch_memory_size`] for the per-batch semantics. +pub fn get_record_batches_memory_size<'a>( + batches: impl IntoIterator, +) -> usize { // Store pointers to `Buffer`'s start memory address (instead of actual // used data region's pointer represented by current `Array`) let mut counted_buffers: HashSet> = HashSet::new(); let mut total_size = 0; - for array in batch.columns() { + for array in batches.into_iter().flat_map(|batch| batch.columns()) { let array_data = array.to_data(); count_array_data_memory_size(&array_data, &mut counted_buffers, &mut total_size); } @@ -295,6 +305,35 @@ mod record_batch_tests { assert_eq!(size_origin, size_sliced); } + #[test] + fn test_get_record_batches_memory_size_shared_buffer() { + let original = Int32Array::from(vec![1, 2, 3, 4, 5]); + let slice1 = original.slice(0, 3); + let slice2 = original.slice(2, 3); + + let schema_origin = Arc::new(Schema::new(vec![Field::new( + "origin_col", + DataType::Int32, + false, + )])); + let batch_origin = + RecordBatch::try_new(Arc::clone(&schema_origin), vec![Arc::new(original)]) + .unwrap(); + let batch_slice1 = + RecordBatch::try_new(Arc::clone(&schema_origin), vec![Arc::new(slice1)]) + .unwrap(); + let batch_slice2 = + RecordBatch::try_new(schema_origin, vec![Arc::new(slice2)]).unwrap(); + + let size_origin = get_record_batch_memory_size(&batch_origin); + let size_slices = get_record_batches_memory_size([&batch_slice1, &batch_slice2]); + let size_summed = get_record_batch_memory_size(&batch_slice1) + + get_record_batch_memory_size(&batch_slice2); + + assert_eq!(size_origin, size_slices); + assert_eq!(size_summed, 2 * size_origin); + } + #[test] fn test_get_record_batch_memory_size_nested_array() { let schema = Arc::new(Schema::new(vec![ diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index 570f61c246709..7a865dbe82ae5 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -53,6 +53,7 @@ use crate::{ ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream, Statistics, }; +use datafusion_common::utils::memory::get_record_batches_memory_size; use arrow::array::{Array, RecordBatch, RecordBatchOptions, StringViewArray}; use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays}; @@ -734,10 +735,7 @@ impl ExternalSorter { // Using try_resize avoids a release-then-reacquire cycle, which // matters for MemoryPool implementations where grow/shrink have // non-trivial cost (e.g. JNI calls in Comet). - let total_sorted_size: usize = sorted_batches - .iter() - .map(get_record_batch_memory_size) - .sum(); + let total_sorted_size = get_record_batches_memory_size(&sorted_batches); reservation .try_resize(total_sorted_size) .map_err(Self::err_with_oom_context)?; diff --git a/datafusion/physical-plan/src/stream.rs b/datafusion/physical-plan/src/stream.rs index 4b7e707fccedd..50f5cd8f994fa 100644 --- a/datafusion/physical-plan/src/stream.rs +++ b/datafusion/physical-plan/src/stream.rs @@ -704,6 +704,7 @@ impl RecordBatchStream for BatchSplitStream { /// A stream that holds a memory reservation for its lifetime, /// shrinking the reservation as batches are consumed. /// The original reservation must have its batch sizes calculated using [`get_record_batch_memory_size`] +/// or [`get_record_batches_memory_size`](datafusion_common::utils::memory::get_record_batches_memory_size). /// On error, the reservation is *NOT* freed, until the stream is dropped. pub(crate) struct ReservationStream { schema: SchemaRef, @@ -738,8 +739,14 @@ impl Stream for ReservationStream { Poll::Ready(res) => { match res { Some(Ok(batch)) => { - self.reservation - .shrink(get_record_batch_memory_size(&batch)); + // Batches may share buffers (e.g. dictionary values + // after `take`), so their individual sizes can add up + // to more than was reserved for all of them together. + // Release at most what is left; the remainder is freed + // when the stream ends. + let size = get_record_batch_memory_size(&batch) + .min(self.reservation.size()); + self.reservation.shrink(size); Poll::Ready(Some(Ok(batch))) } Some(Err(err)) => Poll::Ready(Some(Err(err))), From 65c303bc44562cfb89e86c0120e5587c4a6aa5df Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Wed, 23 Sep 2026 13:51:05 +0200 Subject: [PATCH 11/25] test(cli): pull the MinIO test image from quay.io MinIO withdrew minio/minio from Docker Hub on 2026-09-11, so the storage integration tests fail to start their container. Point the image pinned by testcontainers-modules at quay.io, which serves the same tags. Minimal backport of apache/datafusion#25216. --- datafusion-cli/tests/cli_integration.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/datafusion-cli/tests/cli_integration.rs b/datafusion-cli/tests/cli_integration.rs index 99fc2d527eea5..8cd7a7cd7b825 100644 --- a/datafusion-cli/tests/cli_integration.rs +++ b/datafusion-cli/tests/cli_integration.rs @@ -44,6 +44,12 @@ fn make_settings() -> Settings { settings } +/// Registry override for the image pinned by `testcontainers-modules`. +/// +/// MinIO withdrew `minio/minio` from Docker Hub on 2026-09-11. quay.io still +/// serves the same tags. Backport of apache/datafusion#25216. +const MINIO_IMAGE_NAME: &str = "quay.io/minio/minio"; + async fn setup_minio_container() -> Result, String> { const MINIO_ROOT_USER: &str = "TEST-DataFusionLogin"; const MINIO_ROOT_PASSWORD: &str = "TEST-DataFusionPassword"; @@ -56,6 +62,7 @@ async fn setup_minio_container() -> Result, String> .expect("Failed to get absolute path for test data"); let container = minio::MinIO::default() + .with_name(MINIO_IMAGE_NAME) .with_env_var("MINIO_ROOT_USER", MINIO_ROOT_USER) .with_env_var("MINIO_ROOT_PASSWORD", MINIO_ROOT_PASSWORD) .with_mount(Mount::bind_mount( From 1b37c8fe5b2f6072c1fd7949b3db88ac6c0f753f Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 24 Sep 2026 10:24:12 +0200 Subject: [PATCH 12/25] fix(physical-plan): release shared sort buffers with their last queued batch Clamping the per-batch release kept the reservation from underflowing but freed shared buffers as soon as the first batch was emitted, while the queued batches still referenced them. get_record_batches_release_sizes charges each buffer to the last batch that references it, so the sum is the shared-buffer total and the stream releases exactly what each batch stops holding. ReservationStream had no other user and is removed. --- datafusion/common/src/utils/memory.rs | 60 +++---- datafusion/physical-plan/src/sorts/sort.rs | 88 ++++++++-- datafusion/physical-plan/src/stream.rs | 192 --------------------- 3 files changed, 104 insertions(+), 236 deletions(-) diff --git a/datafusion/common/src/utils/memory.rs b/datafusion/common/src/utils/memory.rs index 24808dac3aaf6..72560c78f2ca1 100644 --- a/datafusion/common/src/utils/memory.rs +++ b/datafusion/common/src/utils/memory.rs @@ -131,27 +131,29 @@ pub fn estimate_memory_size(num_elements: usize, fixed_size: usize) -> Result /// `Buffer`. This method provides temporary fix until the issue is resolved: /// pub fn get_record_batch_memory_size(batch: &RecordBatch) -> usize { - get_record_batches_memory_size(std::iter::once(batch)) + get_record_batches_memory_size(std::slice::from_ref(batch)) } /// Calculate total used memory of `batches`, counting a `Buffer` shared by -/// several batches (e.g. dictionary values across `take`n chunks) only once. -/// -/// See [`get_record_batch_memory_size`] for the per-batch semantics. -pub fn get_record_batches_memory_size<'a>( - batches: impl IntoIterator, -) -> usize { - // Store pointers to `Buffer`'s start memory address (instead of actual - // used data region's pointer represented by current `Array`) +/// several batches only once. +pub fn get_record_batches_memory_size(batches: &[RecordBatch]) -> usize { + get_record_batches_release_sizes(batches).into_iter().sum() +} + +/// Memory each of `batches` releases when consumed in order: a `Buffer` +/// shared by several batches is charged to the last batch that references it. +pub fn get_record_batches_release_sizes(batches: &[RecordBatch]) -> Vec { let mut counted_buffers: HashSet> = HashSet::new(); - let mut total_size = 0; + let mut sizes = vec![0; batches.len()]; - for array in batches.into_iter().flat_map(|batch| batch.columns()) { - let array_data = array.to_data(); - count_array_data_memory_size(&array_data, &mut counted_buffers, &mut total_size); + for (size, batch) in sizes.iter_mut().zip(batches).rev() { + for array in batch.columns() { + let array_data = array.to_data(); + count_array_data_memory_size(&array_data, &mut counted_buffers, size); + } } - total_size + sizes } /// Count the memory usage of `array_data` and its children recursively. @@ -311,27 +313,21 @@ mod record_batch_tests { let slice1 = original.slice(0, 3); let slice2 = original.slice(2, 3); - let schema_origin = Arc::new(Schema::new(vec![Field::new( - "origin_col", - DataType::Int32, - false, - )])); + let schema = + Arc::new(Schema::new(vec![Field::new("col", DataType::Int32, false)])); let batch_origin = - RecordBatch::try_new(Arc::clone(&schema_origin), vec![Arc::new(original)]) - .unwrap(); - let batch_slice1 = - RecordBatch::try_new(Arc::clone(&schema_origin), vec![Arc::new(slice1)]) - .unwrap(); - let batch_slice2 = - RecordBatch::try_new(schema_origin, vec![Arc::new(slice2)]).unwrap(); + RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(original)]).unwrap(); + let slices = [ + RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(slice1)]).unwrap(), + RecordBatch::try_new(schema, vec![Arc::new(slice2)]).unwrap(), + ]; let size_origin = get_record_batch_memory_size(&batch_origin); - let size_slices = get_record_batches_memory_size([&batch_slice1, &batch_slice2]); - let size_summed = get_record_batch_memory_size(&batch_slice1) - + get_record_batch_memory_size(&batch_slice2); - - assert_eq!(size_origin, size_slices); - assert_eq!(size_summed, 2 * size_origin); + assert_eq!(get_record_batches_memory_size(&slices), size_origin); + assert_eq!( + get_record_batches_release_sizes(&slices), + vec![0, size_origin] + ); } #[test] diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index 7a865dbe82ae5..b7db79b968d43 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -45,7 +45,6 @@ use crate::spill::get_record_batch_memory_size; use crate::spill::in_progress_spill_file::InProgressSpillFile; use crate::spill::spill_manager::{GetSlicedSize, SpillManager}; use crate::stream::RecordBatchStreamAdapter; -use crate::stream::ReservationStream; use crate::topk::TopK; use crate::topk::TopKDynamicFilters; use crate::{ @@ -53,7 +52,7 @@ use crate::{ ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream, Statistics, }; -use datafusion_common::utils::memory::get_record_batches_memory_size; +use datafusion_common::utils::memory::get_record_batches_release_sizes; use arrow::array::{Array, RecordBatch, RecordBatchOptions, StringViewArray}; use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays}; @@ -735,19 +734,20 @@ impl ExternalSorter { // Using try_resize avoids a release-then-reacquire cycle, which // matters for MemoryPool implementations where grow/shrink have // non-trivial cost (e.g. JNI calls in Comet). - let total_sorted_size = get_record_batches_memory_size(&sorted_batches); + let release_sizes = get_record_batches_release_sizes(&sorted_batches); reservation - .try_resize(total_sorted_size) + .try_resize(release_sizes.iter().sum()) .map_err(Self::err_with_oom_context)?; - // Wrap in ReservationStream to hold the reservation - Result::<_, DataFusionError>::Ok(Box::pin(ReservationStream::new( - Arc::clone(&schema), - Box::pin(RecordBatchStreamAdapter::new( - Arc::clone(&schema), - futures::stream::iter(sorted_batches.into_iter().map(Ok)), - )), - reservation, + let batches = sorted_batches.into_iter().zip(release_sizes).map( + move |(batch, size)| { + reservation.shrink(size); + Ok(batch) + }, + ); + Result::<_, DataFusionError>::Ok(Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::iter(batches), )) as SendableRecordBatchStream) }) .try_flatten() @@ -1423,6 +1423,7 @@ mod tests { use datafusion_common::cast::as_primitive_array; use datafusion_common::config::ConfigOptions; use datafusion_common::test_util::batches_to_string; + use datafusion_common::utils::memory::get_record_batches_memory_size; use datafusion_common::{DataFusionError, Result, ScalarValue}; use datafusion_execution::RecordBatchStream; use datafusion_execution::config::SessionConfig; @@ -1786,6 +1787,69 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_sort_shared_dictionary_release_per_batch() -> Result<()> { + const ROWS: usize = 10_000; + const BATCH_SIZE: usize = 1_000; + + let schema = Arc::new(Schema::new(vec![ + Field::new( + "tag", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + false, + ), + Field::new("time", DataType::Int64, false), + ])); + let mut tag = StringDictionaryBuilder::::new(); + for i in 0..ROWS { + tag.append_value(format!("truck_{:05}", (i * 7919) % ROWS)); + } + let time = Int64Array::from_iter_values((0..ROWS).rev().map(|i| i as i64)); + let batch = RecordBatch::try_new( + Arc::clone(&schema), + vec![Arc::new(tag.finish()), Arc::new(time)], + )?; + + let expressions: LexOrdering = [ + PhysicalSortExpr::new_default(col("tag", &schema)?), + PhysicalSortExpr::new_default(col("time", &schema)?), + ] + .into(); + let expected = sort_batch_chunked(&batch, &expressions, BATCH_SIZE)?; + + let runtime = RuntimeEnvBuilder::new() + .with_memory_limit(get_record_batch_memory_size(&batch) * 4, 1.0) + .with_disk_manager_builder( + DiskManagerBuilder::default().with_mode(DiskManagerMode::Disabled), + ) + .build_arc()?; + let task_ctx = Arc::new( + TaskContext::default() + .with_session_config(SessionConfig::new().with_batch_size(BATCH_SIZE)) + .with_runtime(runtime), + ); + + let plan = + TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?; + let sort_exec = SortExec::new(expressions, plan); + let mut stream = sort_exec.execute(0, Arc::clone(&task_ctx))?; + + // Emitted batches are dropped right away; the pool must still hold + // what the queued batches reference, shared dictionary included. + for emitted in 1..=expected.len() { + drop(stream.next().await.unwrap()?); + assert_eq!( + task_ctx.memory_pool().reserved(), + get_record_batches_memory_size(&expected[emitted..]), + "after {emitted} of {} batches", + expected.len() + ); + } + assert!(stream.next().await.is_none()); + assert_eq!(task_ctx.memory_pool().reserved(), 0); + Ok(()) + } + #[tokio::test] async fn test_sort_spill_utf8_strings() -> Result<()> { let session_config = SessionConfig::new() diff --git a/datafusion/physical-plan/src/stream.rs b/datafusion/physical-plan/src/stream.rs index 50f5cd8f994fa..e2e11b8fb23ac 100644 --- a/datafusion/physical-plan/src/stream.rs +++ b/datafusion/physical-plan/src/stream.rs @@ -27,13 +27,11 @@ use super::metrics::ExecutionPlanMetricsSet; use super::metrics::{BaselineMetrics, SplitMetrics}; use super::{ExecutionPlan, RecordBatchStream, SendableRecordBatchStream}; use crate::displayable; -use crate::spill::get_record_batch_memory_size; use arrow::{datatypes::SchemaRef, record_batch::RecordBatch}; use datafusion_common::{Result, exec_err}; use datafusion_common_runtime::JoinSet; use datafusion_execution::TaskContext; -use datafusion_execution::memory_pool::MemoryReservation; use futures::ready; use futures::stream::BoxStream; @@ -701,77 +699,6 @@ impl RecordBatchStream for BatchSplitStream { } } -/// A stream that holds a memory reservation for its lifetime, -/// shrinking the reservation as batches are consumed. -/// The original reservation must have its batch sizes calculated using [`get_record_batch_memory_size`] -/// or [`get_record_batches_memory_size`](datafusion_common::utils::memory::get_record_batches_memory_size). -/// On error, the reservation is *NOT* freed, until the stream is dropped. -pub(crate) struct ReservationStream { - schema: SchemaRef, - inner: SendableRecordBatchStream, - reservation: MemoryReservation, -} - -impl ReservationStream { - pub(crate) fn new( - schema: SchemaRef, - inner: SendableRecordBatchStream, - reservation: MemoryReservation, - ) -> Self { - Self { - schema, - inner, - reservation, - } - } -} - -impl Stream for ReservationStream { - type Item = Result; - - fn poll_next( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll> { - let res = self.inner.poll_next_unpin(cx); - - match res { - Poll::Ready(res) => { - match res { - Some(Ok(batch)) => { - // Batches may share buffers (e.g. dictionary values - // after `take`), so their individual sizes can add up - // to more than was reserved for all of them together. - // Release at most what is left; the remainder is freed - // when the stream ends. - let size = get_record_batch_memory_size(&batch) - .min(self.reservation.size()); - self.reservation.shrink(size); - Poll::Ready(Some(Ok(batch))) - } - Some(Err(err)) => Poll::Ready(Some(Err(err))), - None => { - // Stream is done so free the reservation completely - self.reservation.free(); - Poll::Ready(None) - } - } - } - Poll::Pending => Poll::Pending, - } - } - - fn size_hint(&self) -> (usize, Option) { - self.inner.size_hint() - } -} - -impl RecordBatchStream for ReservationStream { - fn schema(&self) -> SchemaRef { - Arc::clone(&self.schema) - } -} - #[cfg(test)] mod test { use super::*; @@ -1000,123 +927,4 @@ mod test { "Should have received exactly two empty batches" ); } - - #[tokio::test] - async fn test_reservation_stream_shrinks_on_poll() { - use arrow::array::Int32Array; - use datafusion_execution::memory_pool::MemoryConsumer; - use datafusion_execution::runtime_env::RuntimeEnvBuilder; - - let runtime = RuntimeEnvBuilder::new() - .with_memory_limit(10 * 1024 * 1024, 1.0) - .build_arc() - .unwrap(); - - let reservation = MemoryConsumer::new("test").register(&runtime.memory_pool); - - let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); - - // Create batches - let batch1 = RecordBatch::try_new( - Arc::clone(&schema), - vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))], - ) - .unwrap(); - let batch2 = RecordBatch::try_new( - Arc::clone(&schema), - vec![Arc::new(Int32Array::from(vec![6, 7, 8, 9, 10]))], - ) - .unwrap(); - - let batch1_size = get_record_batch_memory_size(&batch1); - let batch2_size = get_record_batch_memory_size(&batch2); - - // Reserve memory upfront - reservation.try_grow(batch1_size + batch2_size).unwrap(); - let initial_reserved = runtime.memory_pool.reserved(); - assert_eq!(initial_reserved, batch1_size + batch2_size); - - // Create stream with batches - let stream = futures::stream::iter(vec![Ok(batch1), Ok(batch2)]); - let inner = Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&schema), stream)) - as SendableRecordBatchStream; - - let mut res_stream = - ReservationStream::new(Arc::clone(&schema), inner, reservation); - - // Poll first batch - let result1 = res_stream.next().await; - assert!(result1.is_some()); - - // Memory should be reduced by batch1_size - let after_first = runtime.memory_pool.reserved(); - assert_eq!(after_first, batch2_size); - - // Poll second batch - let result2 = res_stream.next().await; - assert!(result2.is_some()); - - // Memory should be reduced by batch2_size - let after_second = runtime.memory_pool.reserved(); - assert_eq!(after_second, 0); - - // Poll None (end of stream) - let result3 = res_stream.next().await; - assert!(result3.is_none()); - - // Memory should still be 0 - assert_eq!(runtime.memory_pool.reserved(), 0); - } - - #[tokio::test] - async fn test_reservation_stream_error_handling() { - use datafusion_execution::memory_pool::MemoryConsumer; - use datafusion_execution::runtime_env::RuntimeEnvBuilder; - - let runtime = RuntimeEnvBuilder::new() - .with_memory_limit(10 * 1024 * 1024, 1.0) - .build_arc() - .unwrap(); - - let reservation = MemoryConsumer::new("test").register(&runtime.memory_pool); - - let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); - - reservation.try_grow(1000).unwrap(); - let initial = runtime.memory_pool.reserved(); - assert_eq!(initial, 1000); - - // Create a stream that errors - let stream = futures::stream::iter(vec![exec_err!("Test error")]); - let inner = Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&schema), stream)) - as SendableRecordBatchStream; - - let mut res_stream = - ReservationStream::new(Arc::clone(&schema), inner, reservation); - - // Get the error - let result = res_stream.next().await; - assert!(result.is_some()); - assert!(result.unwrap().is_err()); - - // Verify reservation is NOT automatically freed on error - // The reservation is only freed when poll_next returns Poll::Ready(None) - // After an error, the stream may continue to hold the reservation - // until it's explicitly dropped or polled to None - let after_error = runtime.memory_pool.reserved(); - assert_eq!( - after_error, 1000, - "Reservation should still be held after error" - ); - - // Drop the stream to free the reservation - drop(res_stream); - - // Now memory should be freed - assert_eq!( - runtime.memory_pool.reserved(), - 0, - "Memory should be freed when stream is dropped" - ); - } } From 65f4c9187c8ba49c612101ae3405c52e78486941 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 24 Sep 2026 10:35:37 +0200 Subject: [PATCH 13/25] fix(physical-plan): keep the scalar batch size path allocation-free get_record_batch_memory_size and get_record_batches_memory_size count into a scalar again; only sort_batch_stream builds the release-size vector. Add a test that drops a partially consumed sorted stream and checks the pool returns to zero, and share the test scaffolding. --- datafusion/common/src/utils/memory.rs | 27 +++++- datafusion/physical-plan/src/sorts/sort.rs | 108 +++++++++------------ 2 files changed, 69 insertions(+), 66 deletions(-) diff --git a/datafusion/common/src/utils/memory.rs b/datafusion/common/src/utils/memory.rs index 72560c78f2ca1..cee7e90a1f1e6 100644 --- a/datafusion/common/src/utils/memory.rs +++ b/datafusion/common/src/utils/memory.rs @@ -137,7 +137,16 @@ pub fn get_record_batch_memory_size(batch: &RecordBatch) -> usize { /// Calculate total used memory of `batches`, counting a `Buffer` shared by /// several batches only once. pub fn get_record_batches_memory_size(batches: &[RecordBatch]) -> usize { - get_record_batches_release_sizes(batches).into_iter().sum() + // Store pointers to `Buffer`'s start memory address (instead of actual + // used data region's pointer represented by current `Array`) + let mut counted_buffers: HashSet> = HashSet::new(); + let mut total_size = 0; + + for batch in batches { + count_record_batch_memory_size(batch, &mut counted_buffers, &mut total_size); + } + + total_size } /// Memory each of `batches` releases when consumed in order: a `Buffer` @@ -147,15 +156,23 @@ pub fn get_record_batches_release_sizes(batches: &[RecordBatch]) -> Vec { let mut sizes = vec![0; batches.len()]; for (size, batch) in sizes.iter_mut().zip(batches).rev() { - for array in batch.columns() { - let array_data = array.to_data(); - count_array_data_memory_size(&array_data, &mut counted_buffers, size); - } + count_record_batch_memory_size(batch, &mut counted_buffers, size); } sizes } +fn count_record_batch_memory_size( + batch: &RecordBatch, + counted_buffers: &mut HashSet>, + total_size: &mut usize, +) { + for array in batch.columns() { + let array_data = array.to_data(); + count_array_data_memory_size(&array_data, counted_buffers, total_size); + } +} + /// Count the memory usage of `array_data` and its children recursively. fn count_array_data_memory_size( array_data: &ArrayData, diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index b7db79b968d43..cad4b54fb9856 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -1727,15 +1727,9 @@ mod tests { Ok(()) } - /// A single large batch with a dictionary column: `sort_batch_chunked` - /// shares the dictionary values across every output chunk, but the - /// reservation for the sorted output charges it once per chunk. - #[tokio::test] - async fn test_sort_single_batch_shared_dictionary_reservation() -> Result<()> { - const ROWS: usize = 100_000; - const CARDINALITY: usize = 50_000; - const BATCH_SIZE: usize = 1_000; - + /// One batch whose `tag` dictionary is shared by every chunk of its sorted + /// output. + fn shared_dictionary_batch(rows: usize) -> Result { let schema = Arc::new(Schema::new(vec![ Field::new( "tag", @@ -1745,30 +1739,35 @@ mod tests { Field::new("time", DataType::Int64, false), ])); let mut tag = StringDictionaryBuilder::::new(); - for i in 0..ROWS { - tag.append_value(format!("truck_{:05}", (i * 7919) % CARDINALITY)); + for i in 0..rows { + tag.append_value(format!("truck_{:05}", (i * 7919) % (rows / 2))); } - let time = Int64Array::from_iter_values((0..ROWS).rev().map(|i| i as i64)); - let batch = RecordBatch::try_new( - Arc::clone(&schema), + let time = Int64Array::from_iter_values((0..rows).rev().map(|i| i as i64)); + Ok(RecordBatch::try_new( + schema, vec![Arc::new(tag.finish()), Arc::new(time)], - )?; - let input_bytes = get_record_batch_memory_size(&batch); + )?) + } - // Room for the input several times over, but not for the dictionary - // once per chunk. No disk manager, so the sort cannot spill. + /// `SortExec` on `batch` by `(tag, time)` with a `batch_size`-row output, + /// a pool of `pool_batches` times the input and no disk manager. + fn shared_dictionary_sort( + batch: RecordBatch, + batch_size: usize, + pool_batches: usize, + ) -> Result<(Arc, Arc)> { + let schema = batch.schema(); let runtime = RuntimeEnvBuilder::new() - .with_memory_limit(input_bytes * 4, 1.0) + .with_memory_limit(get_record_batch_memory_size(&batch) * pool_batches, 1.0) .with_disk_manager_builder( DiskManagerBuilder::default().with_mode(DiskManagerMode::Disabled), ) .build_arc()?; let task_ctx = Arc::new( TaskContext::default() - .with_session_config(SessionConfig::new().with_batch_size(BATCH_SIZE)) + .with_session_config(SessionConfig::new().with_batch_size(batch_size)) .with_runtime(runtime), ); - let plan = TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?; let sort_exec = Arc::new(SortExec::new( @@ -1779,6 +1778,16 @@ mod tests { .into(), plan, )); + Ok((sort_exec, task_ctx)) + } + + #[tokio::test] + async fn test_sort_single_batch_shared_dictionary_reservation() -> Result<()> { + const ROWS: usize = 100_000; + const BATCH_SIZE: usize = 1_000; + + let (sort_exec, task_ctx) = + shared_dictionary_sort(shared_dictionary_batch(ROWS)?, BATCH_SIZE, 4)?; let result = collect(sort_exec, task_ctx).await?; let rows: usize = result.iter().map(|b| b.num_rows()).sum(); @@ -1792,46 +1801,9 @@ mod tests { const ROWS: usize = 10_000; const BATCH_SIZE: usize = 1_000; - let schema = Arc::new(Schema::new(vec![ - Field::new( - "tag", - DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), - false, - ), - Field::new("time", DataType::Int64, false), - ])); - let mut tag = StringDictionaryBuilder::::new(); - for i in 0..ROWS { - tag.append_value(format!("truck_{:05}", (i * 7919) % ROWS)); - } - let time = Int64Array::from_iter_values((0..ROWS).rev().map(|i| i as i64)); - let batch = RecordBatch::try_new( - Arc::clone(&schema), - vec![Arc::new(tag.finish()), Arc::new(time)], - )?; - - let expressions: LexOrdering = [ - PhysicalSortExpr::new_default(col("tag", &schema)?), - PhysicalSortExpr::new_default(col("time", &schema)?), - ] - .into(); - let expected = sort_batch_chunked(&batch, &expressions, BATCH_SIZE)?; - - let runtime = RuntimeEnvBuilder::new() - .with_memory_limit(get_record_batch_memory_size(&batch) * 4, 1.0) - .with_disk_manager_builder( - DiskManagerBuilder::default().with_mode(DiskManagerMode::Disabled), - ) - .build_arc()?; - let task_ctx = Arc::new( - TaskContext::default() - .with_session_config(SessionConfig::new().with_batch_size(BATCH_SIZE)) - .with_runtime(runtime), - ); - - let plan = - TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?; - let sort_exec = SortExec::new(expressions, plan); + let batch = shared_dictionary_batch(ROWS)?; + let (sort_exec, task_ctx) = shared_dictionary_sort(batch.clone(), BATCH_SIZE, 4)?; + let expected = sort_batch_chunked(&batch, sort_exec.expr(), BATCH_SIZE)?; let mut stream = sort_exec.execute(0, Arc::clone(&task_ctx))?; // Emitted batches are dropped right away; the pool must still hold @@ -1850,6 +1822,20 @@ mod tests { Ok(()) } + #[tokio::test] + async fn test_sort_shared_dictionary_release_on_drop() -> Result<()> { + let (sort_exec, task_ctx) = + shared_dictionary_sort(shared_dictionary_batch(10_000)?, 1_000, 4)?; + let mut stream = sort_exec.execute(0, Arc::clone(&task_ctx))?; + + drop(stream.next().await.unwrap()?); + assert!(task_ctx.memory_pool().reserved() > 0); + + drop(stream); + assert_eq!(task_ctx.memory_pool().reserved(), 0); + Ok(()) + } + #[tokio::test] async fn test_sort_spill_utf8_strings() -> Result<()> { let session_config = SessionConfig::new() From 8e20b774272bbd1eee5a1e6ad7727a4d5147b34c Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 24 Sep 2026 12:52:19 +0200 Subject: [PATCH 14/25] chore(common): port RecordBatchMemoryCounter from upstream datafusion/common/src/utils/memory.rs as of apache/datafusion main 9951f0b (2026-09-24): apache/datafusion#24319 introduced the counter and #24364, #24837, #25335 refined it. Counts each buffer once across a sequence of batches; replaces the fork-local helpers. --- datafusion/common/src/utils/memory.rs | 844 +++++++++++++++++++++++--- 1 file changed, 774 insertions(+), 70 deletions(-) diff --git a/datafusion/common/src/utils/memory.rs b/datafusion/common/src/utils/memory.rs index cee7e90a1f1e6..bb006295f8c1d 100644 --- a/datafusion/common/src/utils/memory.rs +++ b/datafusion/common/src/utils/memory.rs @@ -19,9 +19,27 @@ use crate::error::_exec_datafusion_err; use crate::{HashSet, Result}; -use arrow::array::ArrayData; +use arrow::array::types::{ + ByteArrayType, ByteViewType, Int8Type, Int16Type, Int32Type, Int64Type, + RunEndIndexType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, +}; +use arrow::array::{ + Array, ArrayRef, AsArray, GenericByteArray, GenericByteViewArray, GenericListArray, + GenericListViewArray, RunArray, +}; +use arrow::buffer::Buffer; +use arrow::datatypes::DataType; +use arrow::downcast_primitive_array; use arrow::record_batch::RecordBatch; -use std::{mem::size_of, ptr::NonNull}; +use std::mem::size_of; +use std::num::NonZero; +use std::sync::Arc; + +/// Maximum number of distinct buffer IDs retained inline before promotion to +/// a [`HashSet`]. Sixteen keeps small buffer sets allocation-free while +/// limiting linear lookup and inline storage to 16 pointer-sized entries. +/// This is a performance heuristic, not a semantic limit. +const INLINE_BUFFER_IDS: usize = 16; /// Estimates the memory size required for a hash table prior to allocation. /// @@ -117,84 +135,367 @@ pub fn estimate_memory_size(num_elements: usize, fixed_size: usize) -> Result /// For a `RecordBatch` with two columns: `col1` and `col2`, two columns are pointing /// to a sub-region of the same buffer. /// +/// ```text /// {xxxxxxxxxxxxxxxxxxx} <--- buffer /// ^ ^ ^ ^ /// | | | | /// col1->{ } | | /// col2--------->{ } +/// ``` /// /// In the above case, `get_record_batch_memory_size` will return the size of /// the buffer, instead of the sum of `col1` and `col2`'s actual memory size. /// -/// Note: Current `RecordBatch`.get_array_memory_size()` will double count the -/// buffer memory size if multiple arrays within the batch are sharing the same -/// `Buffer`. This method provides temporary fix until the issue is resolved: -/// +/// Note: [`RecordBatch::get_array_memory_size`] double counts the buffer +/// memory size if multiple arrays within the batch are sharing the same +/// `Buffer`, while this function counts each `Buffer` exactly once. pub fn get_record_batch_memory_size(batch: &RecordBatch) -> usize { - get_record_batches_memory_size(std::slice::from_ref(batch)) + RecordBatchMemoryCounter::new().count_batch(batch) } -/// Calculate total used memory of `batches`, counting a `Buffer` shared by -/// several batches only once. -pub fn get_record_batches_memory_size(batches: &[RecordBatch]) -> usize { - // Store pointers to `Buffer`'s start memory address (instead of actual - // used data region's pointer represented by current `Array`) - let mut counted_buffers: HashSet> = HashSet::new(); - let mut total_size = 0; +/// Tracks the memory used by a sequence of [`RecordBatch`]es that may share +/// underlying buffers, counting each buffer exactly once. +/// +/// Use this instead of [`get_record_batch_memory_size`] to account for the +/// total memory of a sequence of batches, e.g. when buffering the batches of +/// an input stream. Such batches can share buffers (for example, operators +/// like aggregates emit one large batch as multiple zero-copy slices), and +/// calling [`get_record_batch_memory_size`] per batch counts the shared +/// buffers once per batch, while this counter counts them exactly once. A +/// batch's buffers are kept alive by the batch even when only a sub-range is +/// referenced, so counting unique buffers in full reflects the memory the +/// batches actually retain. +#[derive(Debug, Default)] +pub struct RecordBatchMemoryCounter { + /// Start addresses of `Buffer`s that have already been counted (instead of + /// actual used data region's pointer represented by current `Array`) + counted_buffers: BufferIdSet, + /// Array objects already counted by [`Self::count_batch_with_array_overhead`] + counted_arrays: HashSet, + /// Total memory of all counted allocations + memory_usage: usize, +} - for batch in batches { - count_record_batch_memory_size(batch, &mut counted_buffers, &mut total_size); +impl RecordBatchMemoryCounter { + pub fn new() -> Self { + Self::default() } - total_size -} + /// Count `batch`, returning the memory used by its buffers that have not + /// been counted before. + pub fn count_batch(&mut self, batch: &RecordBatch) -> usize { + let previous_memory_usage = self.memory_usage; + + for array in batch.columns() { + self.count_array_memory_size(array.as_ref()); + } + + self.memory_usage - previous_memory_usage + } + + /// Counts unique buffers and Array objects retained by `batch`. + /// + /// This is useful for accounting a sequence of batches at an operator + /// boundary. It counts buffers once, including buffers shared by multiple + /// batches. Array-object overhead is deduplicated recursively when a shared + /// child has the same `ArrayRef` identity; Arrow constructors that rebuild + /// child array objects may conservatively count their object overhead again. + pub fn count_batch_with_array_overhead(&mut self, batch: &RecordBatch) -> usize { + let mut total_size = self.count_batch(batch); + let mut array_overhead = 0; -/// Memory each of `batches` releases when consumed in order: a `Buffer` -/// shared by several batches is charged to the last batch that references it. -pub fn get_record_batches_release_sizes(batches: &[RecordBatch]) -> Vec { - let mut counted_buffers: HashSet> = HashSet::new(); - let mut sizes = vec![0; batches.len()]; + for array in batch.columns() { + array_overhead += + count_unique_array_object_memory_size(array, &mut self.counted_arrays); + } + + total_size += array_overhead; + self.memory_usage += array_overhead; + total_size + } + + /// Total memory of all counted allocations. + pub fn memory_usage(&self) -> usize { + self.memory_usage + } + + fn count_buffer_memory_size(&mut self, buffer: &Buffer) { + if self.counted_buffers.insert(buffer.data_ptr().addr()) { + self.memory_usage += buffer.capacity(); + } + } + + /// Count the memory usage of `array` and its children recursively. + fn count_array_memory_size(&mut self, array: &dyn Array) { + if let Some(nulls) = array.nulls() { + self.count_buffer_memory_size(nulls.buffer()); + } + + downcast_primitive_array! { + array => self.count_buffer_memory_size(array.values().inner()), + DataType::Null => {} + DataType::Boolean => { + self.count_buffer_memory_size(array.as_boolean().values().inner()); + } + DataType::Binary => { + self.count_byte_array_memory_size(array.as_binary::()); + } + DataType::LargeBinary => { + self.count_byte_array_memory_size(array.as_binary::()); + } + DataType::Utf8 => { + self.count_byte_array_memory_size(array.as_string::()); + } + DataType::LargeUtf8 => { + self.count_byte_array_memory_size(array.as_string::()); + } + DataType::BinaryView => { + self.count_byte_view_array_memory_size(array.as_binary_view()); + } + DataType::Utf8View => { + self.count_byte_view_array_memory_size(array.as_string_view()); + } + DataType::FixedSizeBinary(_) => { + self.count_buffer_memory_size(array.as_fixed_size_binary().values()); + } + DataType::List(_) => { + self.count_list_array_memory_size(array.as_list::()); + } + DataType::LargeList(_) => { + self.count_list_array_memory_size(array.as_list::()); + } + DataType::ListView(_) => { + self.count_list_view_array_memory_size(array.as_list_view::()); + } + DataType::LargeListView(_) => { + self.count_list_view_array_memory_size(array.as_list_view::()); + } + DataType::FixedSizeList(_, _) => { + self.count_array_memory_size( + array.as_fixed_size_list().values().as_ref(), + ); + } + DataType::Struct(_) => { + for child in array.as_struct().columns() { + self.count_array_memory_size(child.as_ref()); + } + } + DataType::Union(_, _) => { + let array = array.as_union(); + self.count_buffer_memory_size(array.type_ids().inner()); + if let Some(offsets) = array.offsets() { + self.count_buffer_memory_size(offsets.inner()); + } + for (type_id, _) in array.fields().iter() { + self.count_array_memory_size(array.child(type_id).as_ref()); + } + } + DataType::Dictionary(_, _) => { + let array = array.as_any_dictionary(); + self.count_array_memory_size(array.keys()); + self.count_array_memory_size(array.values().as_ref()); + } + DataType::Map(_, _) => { + let array = array.as_map(); + self.count_buffer_memory_size(array.offsets().inner().inner()); + self.count_array_memory_size(array.entries()); + } + DataType::RunEndEncoded(run_ends, _) => match run_ends.data_type() { + DataType::Int16 => self.count_run_array_memory_size::(array), + DataType::Int32 => self.count_run_array_memory_size::(array), + DataType::Int64 => self.count_run_array_memory_size::(array), + // Arrow only permits Int16, Int32, and Int64 run-end indexes. A + // custom Array implementation may still expose malformed data; + // retain correct accounting for it without panicking. + _ => self.count_array_data_memory_size(&array.to_data()), + }, + // All currently supported non-primitive layouts are handled above. + // The Arrow macro requires a final arm for primitive variants that + // its nested dispatch has already consumed. Keep a safe generic + // fallback for custom or future Array implementations. + _ => self.count_array_data_memory_size(&array.to_data()), + } + } - for (size, batch) in sizes.iter_mut().zip(batches).rev() { - count_record_batch_memory_size(batch, &mut counted_buffers, size); + fn count_byte_array_memory_size( + &mut self, + array: &GenericByteArray, + ) { + self.count_buffer_memory_size(array.offsets().inner().inner()); + self.count_buffer_memory_size(array.values()); } - sizes + fn count_byte_view_array_memory_size( + &mut self, + array: &GenericByteViewArray, + ) { + self.count_buffer_memory_size(array.views().inner()); + for buffer in array.data_buffers().iter() { + self.count_buffer_memory_size(buffer); + } + } + + fn count_list_array_memory_size( + &mut self, + array: &GenericListArray, + ) { + self.count_buffer_memory_size(array.offsets().inner().inner()); + self.count_array_memory_size(array.values().as_ref()); + } + + fn count_list_view_array_memory_size( + &mut self, + array: &GenericListViewArray, + ) { + self.count_buffer_memory_size(array.offsets().inner()); + self.count_buffer_memory_size(array.sizes().inner()); + self.count_array_memory_size(array.values().as_ref()); + } + + fn count_run_array_memory_size(&mut self, array: &dyn Array) { + if let Some(array) = array.as_any().downcast_ref::>() { + self.count_buffer_memory_size(array.run_ends().inner().inner()); + self.count_array_memory_size(array.values().as_ref()); + } else { + // The DataType and concrete array implementation disagree. Use the + // generic representation rather than panic while accounting memory. + self.count_array_data_memory_size(&array.to_data()); + } + } + + fn count_array_data_memory_size(&mut self, array_data: &arrow::array::ArrayData) { + for buffer in array_data.buffers() { + self.count_buffer_memory_size(buffer); + } + if let Some(nulls) = array_data.nulls() { + self.count_buffer_memory_size(nulls.buffer()); + } + for child in array_data.child_data() { + self.count_array_data_memory_size(child); + } + } } -fn count_record_batch_memory_size( - batch: &RecordBatch, - counted_buffers: &mut HashSet>, - total_size: &mut usize, -) { - for array in batch.columns() { - let array_data = array.to_data(); - count_array_data_memory_size(&array_data, counted_buffers, total_size); +/// Counts the unique Array object memory retained by `array` and its children. +fn count_unique_array_object_memory_size( + array: &ArrayRef, + counted_arrays: &mut HashSet, +) -> usize { + let array_ptr = Arc::as_ptr(array).cast::<()>() as usize; + if !counted_arrays.insert(array_ptr) { + return 0; } + + let children = array_children(array); + let children_overhead: usize = children + .iter() + .map(|child| child.get_array_memory_size() - child.get_buffer_memory_size()) + .sum(); + let own_overhead = array.get_array_memory_size() + - array.get_buffer_memory_size() + - children_overhead; + + own_overhead + + children + .into_iter() + .map(|child| count_unique_array_object_memory_size(child, counted_arrays)) + .sum::() } -/// Count the memory usage of `array_data` and its children recursively. -fn count_array_data_memory_size( - array_data: &ArrayData, - counted_buffers: &mut HashSet>, - total_size: &mut usize, -) { - // Count memory usage for `array_data` - for buffer in array_data.buffers() { - if counted_buffers.insert(buffer.data_ptr()) { - *total_size += buffer.capacity(); - } // Otherwise the buffer's memory is already counted +/// Returns the `ArrayRef` children whose object allocations may be shared. +fn array_children(array: &ArrayRef) -> Vec<&ArrayRef> { + match array.data_type() { + DataType::Struct(_) => array.as_struct().columns().iter().collect(), + DataType::List(_) => vec![array.as_list::().values()], + DataType::LargeList(_) => vec![array.as_list::().values()], + DataType::ListView(_) => vec![array.as_list_view::().values()], + DataType::LargeListView(_) => vec![array.as_list_view::().values()], + DataType::FixedSizeList(_, _) => vec![array.as_fixed_size_list().values()], + DataType::Map(_, _) => { + let map = array.as_map(); + vec![map.keys(), map.values()] + } + DataType::Union(_, _) => { + let union = array.as_union(); + union + .fields() + .iter() + .map(|(type_id, _)| union.child(type_id)) + .collect() + } + DataType::Dictionary(key_type, _) => match key_type.as_ref() { + DataType::Int8 => vec![array.as_dictionary::().values()], + DataType::Int16 => vec![array.as_dictionary::().values()], + DataType::Int32 => vec![array.as_dictionary::().values()], + DataType::Int64 => vec![array.as_dictionary::().values()], + DataType::UInt8 => vec![array.as_dictionary::().values()], + DataType::UInt16 => vec![array.as_dictionary::().values()], + DataType::UInt32 => vec![array.as_dictionary::().values()], + DataType::UInt64 => vec![array.as_dictionary::().values()], + _ => unreachable!("invalid dictionary key type: {key_type}"), + }, + DataType::RunEndEncoded(run_ends, _) => match run_ends.data_type() { + DataType::Int16 => array + .as_any() + .downcast_ref::>() + .map(|array| vec![array.values()]) + .unwrap_or_default(), + DataType::Int32 => array + .as_any() + .downcast_ref::>() + .map(|array| vec![array.values()]) + .unwrap_or_default(), + DataType::Int64 => array + .as_any() + .downcast_ref::>() + .map(|array| vec![array.values()]) + .unwrap_or_default(), + _ => vec![], + }, + _ => vec![], } +} + +/// Tracks a small number of buffers inline, avoiding a heap allocation for +/// typical batches, and promotes to a hash set when more buffers are seen. +#[derive(Debug)] +struct BufferIdSet { + inline: [Option>; INLINE_BUFFER_IDS], + len: usize, + overflow: Option>>, +} - if let Some(null_buffer) = array_data.nulls() - && counted_buffers.insert(null_buffer.inner().inner().data_ptr()) - { - *total_size += null_buffer.inner().inner().capacity(); +impl Default for BufferIdSet { + fn default() -> Self { + Self { + inline: [None; INLINE_BUFFER_IDS], + len: 0, + overflow: None, + } } +} + +impl BufferIdSet { + fn insert(&mut self, buffer_id: NonZero) -> bool { + if let Some(overflow) = &mut self.overflow { + return overflow.insert(buffer_id); + } + + if self.inline[..self.len].contains(&Some(buffer_id)) { + return false; + } - // Count all children `ArrayData` recursively - for child in array_data.child_data() { - count_array_data_memory_size(child, counted_buffers, total_size); + if self.len < INLINE_BUFFER_IDS { + self.inline[self.len] = Some(buffer_id); + self.len += 1; + return true; + } + + let mut overflow = HashSet::with_capacity(INLINE_BUFFER_IDS + 1); + overflow.extend(self.inline.iter().flatten().copied()); + let inserted = overflow.insert(buffer_id); + self.overflow = Some(overflow); + inserted } } @@ -235,10 +536,51 @@ mod tests { #[cfg(test)] mod record_batch_tests { use super::*; - use arrow::array::{Float64Array, Int32Array, ListArray}; - use arrow::datatypes::{DataType, Field, Int32Type, Schema}; + use arrow::array::{ + ArrayData, ArrayRef, BinaryViewArray, DictionaryArray, Float64Array, Int16Array, + Int32Array, Int64Array, LargeListViewArray, ListArray, ListViewArray, MapArray, + RunArray, StringArray, StringViewArray, StructArray, UnionArray, new_null_array, + }; + use arrow::buffer::OffsetBuffer; + use arrow::datatypes::{ + DataType, Field, Fields, Int16Type, Int32Type, Int64Type, Schema, UnionFields, + UnionMode, + }; use std::sync::Arc; + fn array_data_memory_size(array: &dyn Array) -> usize { + fn count( + array_data: &ArrayData, + counted_buffers: &mut HashSet>, + total_size: &mut usize, + ) { + for buffer in array_data.buffers() { + if counted_buffers.insert(buffer.data_ptr().addr()) { + *total_size += buffer.capacity(); + } + } + if let Some(nulls) = array_data.nulls() { + let buffer = nulls.inner().inner(); + if counted_buffers.insert(buffer.data_ptr().addr()) { + *total_size += buffer.capacity(); + } + } + for child in array_data.child_data() { + count(child, counted_buffers, total_size); + } + } + + let mut total_size = 0; + count(&array.to_data(), &mut HashSet::default(), &mut total_size); + total_size + } + + fn assert_array_memory_size_matches(array: &dyn Array) { + let mut counter = RecordBatchMemoryCounter::new(); + counter.count_array_memory_size(array); + assert_eq!(counter.memory_usage(), array_data_memory_size(array)); + } + #[test] fn test_get_record_batch_memory_size() { let schema = Arc::new(Schema::new(vec![ @@ -325,28 +667,390 @@ mod record_batch_tests { } #[test] - fn test_get_record_batches_memory_size_shared_buffer() { - let original = Int32Array::from(vec![1, 2, 3, 4, 5]); - let slice1 = original.slice(0, 3); - let slice2 = original.slice(2, 3); + fn test_record_batch_memory_counter_array_overhead_shared_across_batches() { + let schema = Arc::new(Schema::new(vec![ + Field::new("ints", DataType::Int32, false), + Field::new("floats", DataType::Float64, false), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6])), + Arc::new(Float64Array::from(vec![1., 2., 3., 4., 5., 6.])), + ], + ) + .unwrap(); - let schema = - Arc::new(Schema::new(vec![Field::new("col", DataType::Int32, false)])); - let batch_origin = - RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(original)]).unwrap(); - let slices = [ - RecordBatch::try_new(Arc::clone(&schema), vec![Arc::new(slice1)]).unwrap(), - RecordBatch::try_new(schema, vec![Arc::new(slice2)]).unwrap(), - ]; + let mut counter = RecordBatchMemoryCounter::new(); + assert_eq!( + counter.count_batch_with_array_overhead(&batch), + batch.get_array_memory_size() + ); + assert_eq!(counter.count_batch_with_array_overhead(&batch), 0); + assert_eq!(counter.memory_usage(), batch.get_array_memory_size()); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_shared_nested_array_overhead() { + let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let fields = + Fields::from(vec![Arc::new(Field::new("value", DataType::Int32, false))]); + let first = Arc::new(StructArray::new( + fields.clone(), + vec![Arc::clone(&shared_child)], + None, + )) as _; + let second = Arc::new(StructArray::new( + fields, + vec![Arc::clone(&shared_child)], + None, + )) as _; + let batch = + RecordBatch::try_from_iter(vec![("first", first), ("second", second)]) + .unwrap(); + + let mut counter = RecordBatchMemoryCounter::new(); + counter.count_batch_with_array_overhead(&batch); - let size_origin = get_record_batch_memory_size(&batch_origin); - assert_eq!(get_record_batches_memory_size(&slices), size_origin); assert_eq!( - get_record_batches_release_sizes(&slices), - vec![0, size_origin] + counter.memory_usage(), + batch.get_array_memory_size() - shared_child.get_array_memory_size() ); } + fn assert_recursive_shared_child_memory( + name: &str, + first: ArrayRef, + second: ArrayRef, + shared_memory: usize, + ) { + let first_memory = first.get_array_memory_size(); + let second_memory = second.get_array_memory_size(); + let first_batch = RecordBatch::try_from_iter(vec![(name, first)]).unwrap(); + let second_batch = RecordBatch::try_from_iter(vec![(name, second)]).unwrap(); + let mut counter = RecordBatchMemoryCounter::new(); + + assert_eq!( + counter.count_batch_with_array_overhead(&first_batch), + first_memory, + "{name}: first batch" + ); + assert_eq!( + counter.count_batch_with_array_overhead(&second_batch), + second_memory - shared_memory, + "{name}: shared child" + ); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_shared_list_child() { + let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let list_field = Arc::new(Field::new_list_field(DataType::Int32, false)); + + let first = Arc::new(ListArray::new( + Arc::clone(&list_field), + OffsetBuffer::new(vec![0, 3].into()), + Arc::clone(&shared_child), + None, + )); + let second = Arc::new(ListArray::new( + list_field, + OffsetBuffer::new(vec![0, 3].into()), + Arc::clone(&shared_child), + None, + )); + + assert_recursive_shared_child_memory( + "list", + first, + second, + shared_child.get_array_memory_size(), + ); + } + + fn map_with_shared_children( + fields: &Fields, + shared_key: &ArrayRef, + shared_value: &ArrayRef, + ) -> ArrayRef { + Arc::new( + MapArray::try_new( + Arc::new(Field::new( + "entries", + DataType::Struct(fields.clone()), + false, + )), + OffsetBuffer::new(vec![0, 3].into()), + StructArray::new( + fields.clone(), + vec![Arc::clone(shared_key), Arc::clone(shared_value)], + None, + ), + None, + false, + ) + .unwrap(), + ) + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_shared_map_children() { + let shared_key: ArrayRef = Arc::new(Int32Array::from(vec![4, 5, 6])); + let shared_value: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let fields = Fields::from(vec![ + Arc::new(Field::new("key", DataType::Int32, false)), + Arc::new(Field::new("value", DataType::Int32, false)), + ]); + + let first = map_with_shared_children(&fields, &shared_key, &shared_value); + let second = map_with_shared_children(&fields, &shared_key, &shared_value); + + assert_recursive_shared_child_memory( + "map", + first, + second, + shared_key.get_array_memory_size() + shared_value.get_array_memory_size(), + ); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_union_slice_child_overhead() { + let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let fields: UnionFields = + std::iter::once((0, Arc::new(Field::new("value", DataType::Int32, false)))) + .collect(); + let first = Arc::new( + UnionArray::try_new( + fields, + vec![0, 0, 0].into(), + Some(vec![0, 1, 2].into()), + vec![Arc::clone(&shared_child)], + ) + .unwrap(), + ) as ArrayRef; + let second = Arc::new(first.as_union().slice(1, 2)) as ArrayRef; + let first_batch = + RecordBatch::try_from_iter(vec![("union", Arc::clone(&first))]).unwrap(); + let second_batch = + RecordBatch::try_from_iter(vec![("union", Arc::clone(&second))]).unwrap(); + let child_overhead = + shared_child.get_array_memory_size() - shared_child.get_buffer_memory_size(); + let second_parent_overhead = second.get_array_memory_size() + - second.get_buffer_memory_size() + - child_overhead; + let mut counter = RecordBatchMemoryCounter::new(); + + assert_eq!( + counter.count_batch_with_array_overhead(&first_batch), + first.get_array_memory_size() + ); + assert_eq!( + counter.count_batch_with_array_overhead(&second_batch), + second_parent_overhead + ); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_run_end_slice_child_overhead() { + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let first = Arc::new( + RunArray::::try_new( + &Int32Array::from(vec![1, 2, 3]), + values.as_ref(), + ) + .unwrap(), + ) as ArrayRef; + let second = Arc::new( + first + .as_any() + .downcast_ref::>() + .unwrap() + .slice(1, 2), + ) as ArrayRef; + let first_batch = + RecordBatch::try_from_iter(vec![("run", Arc::clone(&first))]).unwrap(); + let second_batch = + RecordBatch::try_from_iter(vec![("run", Arc::clone(&second))]).unwrap(); + let child_overhead = + values.get_array_memory_size() - values.get_buffer_memory_size(); + let second_parent_overhead = second.get_array_memory_size() + - second.get_buffer_memory_size() + - child_overhead; + let mut counter = RecordBatchMemoryCounter::new(); + + assert_eq!( + counter.count_batch_with_array_overhead(&first_batch), + first.get_array_memory_size() + ); + assert_eq!( + counter.count_batch_with_array_overhead(&second_batch), + second_parent_overhead + ); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_shared_dictionary_child() { + let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let make_dictionary = || { + Arc::new( + DictionaryArray::::try_new( + Int32Array::from(vec![0, 1, 2]), + Arc::clone(&shared_child), + ) + .unwrap(), + ) as ArrayRef + }; + + assert_recursive_shared_child_memory( + "dictionary", + make_dictionary(), + make_dictionary(), + shared_child.get_array_memory_size(), + ); + } + + #[test] + fn test_record_batch_memory_counter_buffer_shared_across_batches() { + let schema = Arc::new(Schema::new(vec![Field::new( + "ints", + DataType::Int32, + false, + )])); + + let int_array = Int32Array::from(vec![1, 2, 3, 4, 5, 6]); + let batch = RecordBatch::try_new(schema, vec![Arc::new(int_array)]).unwrap(); + let slices = [batch.slice(0, 2), batch.slice(2, 2), batch.slice(4, 2)]; + + // Counting each slice individually counts the shared buffer once per slice + let summed: usize = slices.iter().map(get_record_batch_memory_size).sum(); + assert_eq!(summed, 3 * get_record_batch_memory_size(&batch)); + + // A counter shared across the batches counts it exactly once + let mut counter = RecordBatchMemoryCounter::new(); + let deduped: usize = slices.iter().map(|slice| counter.count_batch(slice)).sum(); + assert_eq!(deduped, get_record_batch_memory_size(&batch)); + assert_eq!(counter.memory_usage(), get_record_batch_memory_size(&batch)); + } + + #[test] + fn test_record_batch_memory_counter_promotes_buffer_set() { + let fields = (0..=INLINE_BUFFER_IDS) + .map(|index| Field::new(format!("col_{index}"), DataType::Int32, false)) + .collect::>(); + let columns = (0..=INLINE_BUFFER_IDS) + .map(|value| Arc::new(Int32Array::from(vec![value as i32])) as _) + .collect::>(); + let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap(); + + let mut counter = RecordBatchMemoryCounter::new(); + assert_eq!( + counter.count_batch(&batch), + (INLINE_BUFFER_IDS + 1) * size_of::() + ); + assert!(counter.counted_buffers.overflow.is_some()); + assert_eq!(counter.count_batch(&batch), 0); + } + + #[test] + fn test_array_memory_size_matches_array_data_layouts() { + let list_field = Arc::new(Field::new_list_field(DataType::Int32, true)); + let struct_fields = vec![Field::new("value", DataType::Int32, true)].into(); + let union_fields = UnionFields::try_new( + vec![0], + vec![Field::new("value", DataType::Int32, true)], + ) + .unwrap(); + let map_entries = Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int32, true), + ] + .into(), + ), + false, + )); + let data_types = vec![ + DataType::Boolean, + DataType::Int32, + DataType::Binary, + DataType::LargeBinary, + DataType::FixedSizeBinary(4), + DataType::BinaryView, + DataType::Utf8, + DataType::LargeUtf8, + DataType::Utf8View, + DataType::List(Arc::clone(&list_field)), + DataType::LargeList(Arc::clone(&list_field)), + DataType::ListView(Arc::clone(&list_field)), + DataType::LargeListView(Arc::clone(&list_field)), + DataType::FixedSizeList(Arc::clone(&list_field), 2), + DataType::Struct(struct_fields), + DataType::Union(union_fields, UnionMode::Dense), + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + DataType::Map(map_entries, false), + ]; + + for data_type in data_types { + let array = new_null_array(&data_type, 3); + assert_array_memory_size_matches(array.as_ref()); + } + + // Exercise the view-specific buffers with concrete, non-empty values. + let view_arrays = [ + Arc::new(BinaryViewArray::from_iter_values([ + b"short".as_slice(), + b"a payload longer than twelve bytes".as_slice(), + ])) as ArrayRef, + Arc::new(StringViewArray::from_iter_values([ + "short", + "a payload longer than twelve bytes", + ])) as ArrayRef, + Arc::new(ListViewArray::from_iter_primitive::([ + Some(vec![Some(1), Some(2)]), + None, + Some(vec![Some(3)]), + ])) as ArrayRef, + Arc::new(LargeListViewArray::from_iter_primitive::( + [Some(vec![Some(1), Some(2)]), None, Some(vec![Some(3)])], + )) as ArrayRef, + ]; + + for array in view_arrays { + assert_array_memory_size_matches(array.as_ref()); + } + + let run_values = StringArray::from(vec!["alpha", "beta"]); + let run_arrays = [ + Arc::new( + RunArray::::try_new( + &Int16Array::from(vec![2_i16, 5]), + &run_values, + ) + .unwrap(), + ) as ArrayRef, + Arc::new( + RunArray::::try_new( + &Int32Array::from(vec![2_i32, 5]), + &run_values, + ) + .unwrap(), + ) as ArrayRef, + Arc::new( + RunArray::::try_new( + &Int64Array::from(vec![2_i64, 5]), + &run_values, + ) + .unwrap(), + ) as ArrayRef, + ]; + + for array in run_arrays { + assert_array_memory_size_matches(array.as_ref()); + } + } + #[test] fn test_get_record_batch_memory_size_nested_array() { let schema = Arc::new(Schema::new(vec![ From 3b30756a0dd7780be76f97b5a0f05df7999494f7 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Thu, 24 Sep 2026 12:52:19 +0200 Subject: [PATCH 15/25] fix(physical-plan): size chunked sort output with RecordBatchMemoryCounter Walk the chunks in reverse so each buffer is charged to the last chunk that references it; the counter total is the reservation and the per-chunk counts are what each emitted chunk releases. --- datafusion/physical-plan/src/sorts/sort.rs | 23 +++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index cad4b54fb9856..649dd1e29ff4d 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -52,7 +52,7 @@ use crate::{ ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream, Statistics, }; -use datafusion_common::utils::memory::get_record_batches_release_sizes; +use datafusion_common::utils::memory::RecordBatchMemoryCounter; use arrow::array::{Array, RecordBatch, RecordBatchOptions, StringViewArray}; use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays}; @@ -734,9 +734,18 @@ impl ExternalSorter { // Using try_resize avoids a release-then-reacquire cycle, which // matters for MemoryPool implementations where grow/shrink have // non-trivial cost (e.g. JNI calls in Comet). - let release_sizes = get_record_batches_release_sizes(&sorted_batches); + // Chunks share buffers (e.g. dictionary values after `take`), so + // count each buffer once and release it with the last chunk that + // references it. + let mut counter = RecordBatchMemoryCounter::new(); + let mut release_sizes: Vec = sorted_batches + .iter() + .rev() + .map(|batch| counter.count_batch(batch)) + .collect(); + release_sizes.reverse(); reservation - .try_resize(release_sizes.iter().sum()) + .try_resize(counter.memory_usage()) .map_err(Self::err_with_oom_context)?; let batches = sorted_batches.into_iter().zip(release_sizes).map( @@ -1423,7 +1432,6 @@ mod tests { use datafusion_common::cast::as_primitive_array; use datafusion_common::config::ConfigOptions; use datafusion_common::test_util::batches_to_string; - use datafusion_common::utils::memory::get_record_batches_memory_size; use datafusion_common::{DataFusionError, Result, ScalarValue}; use datafusion_execution::RecordBatchStream; use datafusion_execution::config::SessionConfig; @@ -1781,6 +1789,11 @@ mod tests { Ok((sort_exec, task_ctx)) } + fn batches_memory_size(batches: &[RecordBatch]) -> usize { + let mut counter = RecordBatchMemoryCounter::new(); + batches.iter().map(|batch| counter.count_batch(batch)).sum() + } + #[tokio::test] async fn test_sort_single_batch_shared_dictionary_reservation() -> Result<()> { const ROWS: usize = 100_000; @@ -1812,7 +1825,7 @@ mod tests { drop(stream.next().await.unwrap()?); assert_eq!( task_ctx.memory_pool().reserved(), - get_record_batches_memory_size(&expected[emitted..]), + batches_memory_size(&expected[emitted..]), "after {emitted} of {} batches", expected.len() ); From f9de0f19a225c2800885eec3a0bea905f2be52a0 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 5 Oct 2026 13:29:53 +0200 Subject: [PATCH 16/25] chore: bump arrow/parquet to 58.4.0 --- Cargo.lock | 153 ++++++++++++++++++++++++++++++++++------------------- Cargo.toml | 14 ++--- 2 files changed, 107 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9f9145db4f18f..43a4c9c052465 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,9 +256,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602268ce9f569f282cedb9a9f6bac569b680af47b9b077d515900c03c5d190da" +checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" dependencies = [ "arrow-arith", "arrow-array", @@ -279,9 +279,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd53c6bf277dea91f136ae8e3a5d7041b44b5e489e244e637d00ae302051f56f" +checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" dependencies = [ "arrow-array", "arrow-buffer", @@ -293,9 +293,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e53796e07a6525edaf7dc28b540d477a934aff14af97967ad1d5550878969b9e" +checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" dependencies = [ "ahash", "arrow-buffer", @@ -304,7 +304,7 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "num-complex", "num-integer", "num-traits", @@ -312,9 +312,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c1a85bb2e94ee10b76531d8bc3ce9b7b4c0d508cabfb17d477f63f2617bd20" +checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" dependencies = [ "bytes", "half", @@ -324,9 +324,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89fb245db6b0e234ed8e15b644edb8664673fefe630575e94e62cd9d489a8a26" +checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" dependencies = [ "arrow-array", "arrow-buffer", @@ -346,9 +346,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d374882fb465a194462527c0c15a93aa19a554cf690a6b77a26b2a02539937a7" +checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" dependencies = [ "arrow-array", "arrow-cast", @@ -361,9 +361,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "189d210bc4244c715fa3ed9e6e22864673cccb73d5da28c2723fb2e527329b33" +checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -374,9 +374,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4f5cdf00ee0003ba0768d3575d0afc47d736b29673b14c3c228fdffa9a3fb29" +checksum = "b2dbe34824c639e43136af8f106992792ab456540d54b880bc320a3192502d2e" dependencies = [ "arrow-arith", "arrow-array", @@ -402,9 +402,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7968c2e5210c41f4909b2ef76f6e05e172b99021c2def5edf3cc48fdd39d1d6c" +checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" dependencies = [ "arrow-array", "arrow-buffer", @@ -418,15 +418,16 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92111dba5bf900f443488e01f00d8c4ddc2f47f5c50039d18120287b580baa22" +checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", - "arrow-data", + "arrow-ord", "arrow-schema", + "arrow-select", "chrono", "half", "indexmap 2.13.0", @@ -442,9 +443,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "211136cb253577ee1a6665f741a13136d4e563f64f5093ffd6fb837af90b9495" +checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" dependencies = [ "arrow-array", "arrow-buffer", @@ -455,9 +456,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e0f20145f9f5ea3fe383e2ba7a7487bf19be36aa9dbf5dd6a1f92f657179663" +checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" dependencies = [ "arrow-array", "arrow-buffer", @@ -468,9 +469,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b47e0ca91cc438d2c7879fe95e0bca5329fff28649e30a88c6f760b1faeddcb" +checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" dependencies = [ "bitflags", "serde", @@ -480,9 +481,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "750a7d1dda177735f5e82a314485b6915c7cccdbb278262ac44090f4aba4a325" +checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" dependencies = [ "ahash", "arrow-array", @@ -494,9 +495,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1eab1208bc4fe55d768cdc9b9f3d9df5a794cdb3ee2586bf89f9b30dc31ad8c" +checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" dependencies = [ "arrow-array", "arrow-buffer", @@ -1084,7 +1085,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.2.17", ] [[package]] @@ -1293,6 +1294,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "rand_core 0.10.1", +] + [[package]] name = "chrono" version = "0.4.44" @@ -1548,6 +1560,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -2768,7 +2789,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2912,7 +2933,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3252,6 +3273,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -3340,6 +3362,12 @@ dependencies = [ "foldhash 0.2.0", ] +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" @@ -4033,9 +4061,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.12.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c23545df7ecf1b16c303910a69b079e8e251d60f7dd2cc9b4177f2afaf1746" +checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" dependencies = [ "twox-hash", ] @@ -4171,7 +4199,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4294,16 +4322,18 @@ dependencies = [ [[package]] name = "object_store" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2858065e55c148d294a9f3aae3b0fa9458edadb41a108397094566f4e3c0dfb" +checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" dependencies = [ "async-trait", "base64 0.22.1", "bytes", "chrono", "form_urlencoded", - "futures", + "futures-channel", + "futures-core", + "futures-util", "http 1.4.0", "http-body-util", "humantime", @@ -4313,7 +4343,7 @@ dependencies = [ "parking_lot", "percent-encoding", "quick-xml", - "rand 0.9.2", + "rand 0.10.3", "reqwest", "ring", "rustls-pki-types", @@ -4415,9 +4445,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.0.0" +version = "58.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f491d0ef1b510194426ee67ddc18a9b747ef3c42050c19322a2cd2e1666c29b" +checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" dependencies = [ "ahash", "arrow-array", @@ -4433,7 +4463,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "lz4_flex", "num-bigint", "num-integer", @@ -4865,9 +4895,9 @@ checksum = "5a651516ddc9168ebd67b24afd085a718be02f8858fe406591b013d101ce2f40" [[package]] name = "quick-xml" -version = "0.38.4" +version = "0.39.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66c2058c55a409d601666cffe35f04333cf1013010882cec174a7467cd4e21c" +checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" dependencies = [ "memchr", "serde", @@ -4925,7 +4955,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -4980,6 +5010,17 @@ dependencies = [ "rand_core 0.9.5", ] +[[package]] +name = "rand" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" +dependencies = [ + "chacha20", + "getrandom 0.4.2", + "rand_core 0.10.1", +] + [[package]] name = "rand_chacha" version = "0.3.1" @@ -5018,6 +5059,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + [[package]] name = "rand_distr" version = "0.5.1" @@ -5298,7 +5345,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5646,7 +5693,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -5657,7 +5704,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.2.17", "digest", ] @@ -6006,7 +6053,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6956,7 +7003,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index d8a7e424873c2..02a3c975dd17b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,19 +91,19 @@ ahash = { version = "0.8", default-features = false, features = [ "runtime-rng", ] } apache-avro = { version = "0.21", default-features = false } -arrow = { version = "58.0.0", features = [ +arrow = { version = "58.4.0", features = [ "prettyprint", "chrono-tz", ] } -arrow-buffer = { version = "58.0.0", default-features = false } -arrow-flight = { version = "58.0.0", features = [ +arrow-buffer = { version = "58.4.0", default-features = false } +arrow-flight = { version = "58.4.0", features = [ "flight-sql-experimental", ] } -arrow-ipc = { version = "58.0.0", default-features = false, features = [ +arrow-ipc = { version = "58.4.0", default-features = false, features = [ "lz4", ] } -arrow-ord = { version = "58.0.0", default-features = false } -arrow-schema = { version = "58.0.0", default-features = false } +arrow-ord = { version = "58.4.0", default-features = false } +arrow-schema = { version = "58.4.0", default-features = false } async-trait = "0.1.89" bigdecimal = "0.4.8" bytes = "1.11" @@ -168,7 +168,7 @@ memchr = "2.8.0" num-traits = { version = "0.2" } object_store = { version = "0.13.1", default-features = false } parking_lot = "0.12" -parquet = { version = "58.0.0", default-features = false, features = [ +parquet = { version = "58.4.0", default-features = false, features = [ "arrow", "async", "object_store", From 2fb636f85736c6ecb349b21f4141bc8ad2dde2ee Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 5 Oct 2026 13:30:01 +0200 Subject: [PATCH 17/25] chore(deps): bump rustls-webpki and astral-tokio-tar for cargo audit --- Cargo.lock | 68 ++++++++++++++++-------------------------------------- 1 file changed, 20 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 43a4c9c052465..f53ef8efee194 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -184,7 +184,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -195,7 +195,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -524,15 +524,15 @@ dependencies = [ [[package]] name = "astral-tokio-tar" -version = "0.6.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c23f3af104b40a3430ccb90ed5f7bd877a8dc5c26fc92fde51a22b40890dcf9" +checksum = "b18457efd137254e016bbde5e1d88df61c4e1a5ae2223746e56123bac6af2463" dependencies = [ - "filetime", "futures-core", "libc", "portable-atomic", "rustc-hash", + "rustix", "tokio", "tokio-stream", "xattr", @@ -668,9 +668,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.2" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "zeroize", @@ -678,14 +678,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.1" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -2789,7 +2790,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2933,7 +2934,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3022,17 +3023,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "filetime" -version = "0.2.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" -dependencies = [ - "cfg-if", - "libc", - "libredox", -] - [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -4008,10 +3998,7 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" dependencies = [ - "bitflags", "libc", - "plain", - "redox_syscall 0.7.3", ] [[package]] @@ -4199,7 +4186,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4438,7 +4425,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.5.18", + "redox_syscall", "smallvec", "windows-link", ] @@ -4663,12 +4650,6 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" -[[package]] -name = "plain" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" - [[package]] name = "plotters" version = "0.3.7" @@ -5124,15 +5105,6 @@ dependencies = [ "bitflags", ] -[[package]] -name = "redox_syscall" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" -dependencies = [ - "bitflags", -] - [[package]] name = "redox_users" version = "0.5.2" @@ -5345,7 +5317,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5388,9 +5360,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.11" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20a6af516fea4b20eccceaf166e8aa666ac996208e8a644ce3ef5aa783bc7cd4" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -5800,7 +5772,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -6053,7 +6025,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7003,7 +6975,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] From 70eebafd4823a1ebd7009e701426e2bea9754342 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 5 Oct 2026 13:41:55 +0200 Subject: [PATCH 18/25] style: cargo fmt to_timestamp after timezone revert --- .../functions/src/datetime/to_timestamp.rs | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rs index ea63bda5bcfaf..059f353141386 100644 --- a/datafusion/functions/src/datetime/to_timestamp.rs +++ b/datafusion/functions/src/datetime/to_timestamp.rs @@ -377,7 +377,9 @@ impl ScalarUDFImpl for ToTimestampFunc { let f16_arr = downcast_arg!(arr, Float16Array); let result: TimestampNanosecondArray = f16_arr.unary(|x| (x.to_f64() * 1_000_000_000.0) as i64); - Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(None::>)))) + Ok(ColumnarValue::Array(Arc::new( + result.with_timezone_opt(None::>), + ))) } _ => exec_err!("Invalid Float16 value for to_timestamp"), }, @@ -394,7 +396,9 @@ impl ScalarUDFImpl for ToTimestampFunc { let f32_arr = downcast_arg!(arr, Float32Array); let result: TimestampNanosecondArray = f32_arr.unary(|x| (x as f64 * 1_000_000_000.0) as i64); - Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(None::>)))) + Ok(ColumnarValue::Array(Arc::new( + result.with_timezone_opt(None::>), + ))) } _ => exec_err!("Invalid Float32 value for to_timestamp"), }, @@ -410,7 +414,9 @@ impl ScalarUDFImpl for ToTimestampFunc { let f64_arr = downcast_arg!(arr, Float64Array); let result: TimestampNanosecondArray = f64_arr.unary(|x| (x * 1_000_000_000.0) as i64); - Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(None::>)))) + Ok(ColumnarValue::Array(Arc::new( + result.with_timezone_opt(None::>), + ))) } _ => exec_err!("Invalid Float64 value for to_timestamp"), }, @@ -548,9 +554,7 @@ impl ScalarUDFImpl for ToTimestampMillisFunc { | Decimal64(_, _) | Decimal128(_, _) | Decimal256(_, _) => args[0].cast_to(&Timestamp(Millisecond, None), None), - Timestamp(_, tz) => { - args[0].cast_to(&Timestamp(Millisecond, tz), None) - } + Timestamp(_, tz) => args[0].cast_to(&Timestamp(Millisecond, tz), None), Float16 | Float32 | Float64 => args[0] .cast_to(&Int64, None)? .cast_to(&Timestamp(Millisecond, None), None), @@ -619,9 +623,7 @@ impl ScalarUDFImpl for ToTimestampMicrosFunc { | Decimal64(_, _) | Decimal128(_, _) | Decimal256(_, _) => args[0].cast_to(&Timestamp(Microsecond, None), None), - Timestamp(_, tz) => { - args[0].cast_to(&Timestamp(Microsecond, tz), None) - } + Timestamp(_, tz) => args[0].cast_to(&Timestamp(Microsecond, tz), None), Float16 | Float32 | Float64 => args[0] .cast_to(&Int64, None)? .cast_to(&Timestamp(Microsecond, None), None), From fef177f6118d71d07ed2905f0949d9b50dab65a3 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 5 Oct 2026 13:42:18 +0200 Subject: [PATCH 19/25] ci: bump taiki-e/install-action to v2.87.21 for wasm-pack install --- .github/workflows/rust.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6194262e40f35..61fbad9dadfe1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -431,7 +431,7 @@ jobs: sudo apt-get update -qq sudo apt-get install -y -qq clang - name: Setup wasm-pack - uses: taiki-e/install-action@cfdb446e391c69574ebc316dfb7d7849ec12b940 # v2.68.8 + uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 with: tool: wasm-pack - name: Run tests with headless mode @@ -771,7 +771,7 @@ jobs: - name: Setup Rust toolchain uses: ./.github/actions/setup-builder - name: Install cargo-msrv - uses: taiki-e/install-action@cfdb446e391c69574ebc316dfb7d7849ec12b940 # v2.68.8 + uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 with: tool: cargo-msrv From ba3e7a1004b739ebca7bb27d21dc309cb2d31d2e Mon Sep 17 00:00:00 2001 From: Paul Dix Date: Wed, 27 May 2026 17:03:54 +0300 Subject: [PATCH 20/25] chore: replace deprecated object_store Path::child with join object_store 0.13 deprecates Path::child in favor of .join()/.clone().join() (exact behavioral equivalent: single percent-encoded segment; join takes self by value). Fixes 8 call sites that would fail CI under -D warnings. Also adds UPGRADE-NOTES capturing the to_timestamp upstream-kept decision and the encrypted_parquet feature-flag finding. --- datafusion/core/src/datasource/physical_plan/parquet.rs | 2 +- datafusion/core/tests/sql/path_partition.rs | 2 +- datafusion/datasource/src/url.rs | 2 +- datafusion/datasource/src/write/demux.rs | 7 ++++--- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/datafusion/core/src/datasource/physical_plan/parquet.rs b/datafusion/core/src/datasource/physical_plan/parquet.rs index 4c6d915d5bcaa..a684ceb9b53af 100644 --- a/datafusion/core/src/datasource/physical_plan/parquet.rs +++ b/datafusion/core/src/datasource/physical_plan/parquet.rs @@ -1703,7 +1703,7 @@ mod tests { let state = session_ctx.state(); let location = Path::from_filesystem_path(".") .unwrap() - .child("invalid.parquet"); + .join("invalid.parquet"); let partitioned_file = PartitionedFile::new_from_meta(ObjectMeta { location, diff --git a/datafusion/core/tests/sql/path_partition.rs b/datafusion/core/tests/sql/path_partition.rs index a9061849795c7..97c627ad95f2c 100644 --- a/datafusion/core/tests/sql/path_partition.rs +++ b/datafusion/core/tests/sql/path_partition.rs @@ -766,7 +766,7 @@ impl ObjectStore for MirroringObjectStore { }; if parts.next().is_some() { - common_prefixes.insert(prefix.child(common_prefix)); + common_prefixes.insert(prefix.clone().join(common_prefix)); } else { let object = ObjectMeta { location: k.clone(), diff --git a/datafusion/datasource/src/url.rs b/datafusion/datasource/src/url.rs index 39d1047984ff6..409eeb7b5b106 100644 --- a/datafusion/datasource/src/url.rs +++ b/datafusion/datasource/src/url.rs @@ -536,7 +536,7 @@ mod tests { let root = root.to_string_lossy(); let url = ListingTableUrl::parse(root).unwrap(); - let child = url.prefix.child("partition").child("file"); + let child = url.prefix.clone().join("partition").join("file"); let prefix: Vec<_> = url.strip_prefix(&child).unwrap().collect(); assert_eq!(prefix, vec!["partition", "file"]); diff --git a/datafusion/datasource/src/write/demux.rs b/datafusion/datasource/src/write/demux.rs index 1648624747af2..acc6435acf371 100644 --- a/datafusion/datasource/src/write/demux.rs +++ b/datafusion/datasource/src/write/demux.rs @@ -260,7 +260,8 @@ fn generate_file_path( if !single_file_output { base_output_path .prefix() - .child(format!("{write_id}_{part_idx}.{file_extension}")) + .clone() + .join(format!("{write_id}_{part_idx}.{file_extension}")) } else { base_output_path.prefix().to_owned() } @@ -588,8 +589,8 @@ fn compute_hive_style_file_path( ) -> Path { let mut file_path = base_output_path.prefix().clone(); for j in 0..part_key.len() { - file_path = file_path.child(format!("{}={}", partition_by[j].0, part_key[j])); + file_path = file_path.join(format!("{}={}", partition_by[j].0, part_key[j])); } - file_path.child(format!("{write_id}.{file_extension}")) + file_path.join(format!("{write_id}.{file_extension}")) } From 395a6dfd7a24db336150d772855bb8dad3390d51 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 5 Oct 2026 15:31:15 +0200 Subject: [PATCH 21/25] chore(deps): bump crates flagged by cargo audit --- Cargo.lock | 186 +++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 144 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f53ef8efee194..d1c895a20b9cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -214,7 +214,7 @@ dependencies = [ "bon", "bzip2", "crc32fast", - "digest", + "digest 0.10.7", "liblzma", "log", "miniz_oxide", @@ -800,11 +800,11 @@ dependencies = [ "bytes", "form_urlencoded", "hex", - "hmac", + "hmac 0.12.1", "http 0.2.12", "http 1.4.0", "percent-encoding", - "sha2", + "sha2 0.10.9", "time", "tracing", ] @@ -1072,7 +1072,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -1098,6 +1098,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + [[package]] name = "bollard" version = "0.20.2" @@ -1415,6 +1424,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colorchoice" version = "1.0.5" @@ -1486,6 +1501,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1637,9 +1658,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] @@ -1666,6 +1687,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -1703,6 +1733,15 @@ version = "0.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "cty" version = "0.2.2" @@ -2280,12 +2319,12 @@ dependencies = [ "hex", "itertools 0.14.0", "log", - "md-5", + "md-5 0.10.6", "memchr", "num-traits", "rand 0.9.2", "regex", - "sha2", + "sha2 0.10.9", "tokio", "unicode-segmentation", "uuid", @@ -2640,7 +2679,7 @@ dependencies = [ "rand 0.9.2", "serde_json", "sha1", - "sha2", + "sha2 0.10.9", "url", ] @@ -2767,11 +2806,23 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.10.4", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "const-oid", + "crypto-common 0.2.2", + "ctutils", +] + [[package]] name = "dirs" version = "6.0.0" @@ -2934,7 +2985,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3261,11 +3312,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi 6.0.0", "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -3289,9 +3342,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.13" +version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" dependencies = [ "atomic-waker", "bytes", @@ -3376,7 +3429,16 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", ] [[package]] @@ -3461,6 +3523,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.8.1" @@ -4068,7 +4139,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", ] [[package]] @@ -4326,7 +4407,7 @@ dependencies = [ "humantime", "hyper", "itertools 0.14.0", - "md-5", + "md-5 0.10.6", "parking_lot", "percent-encoding", "quick-xml", @@ -4695,9 +4776,9 @@ dependencies = [ [[package]] name = "postgres-derive" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56df96f5394370d1b20e49de146f9e6c25aa9ae750f449c9d665eafecb3ccae6" +checksum = "4d9d9089bb0ce62f4b5d52a0be0f4acfb35738b979380670d3dea85fe38d6ddd" dependencies = [ "heck", "proc-macro2", @@ -4707,27 +4788,27 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.10" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ee9dd5fe15055d2b6806f4736aa0c9637217074e224bbec46d4041b91bb9491" +checksum = "08808e3c483c46e999108051c78334f473d5adb59d78bb80a1268c7e6aa6c514" dependencies = [ "base64 0.22.1", "byteorder", "bytes", "fallible-iterator", - "hmac", - "md-5", + "hmac 0.13.0", + "md-5 0.11.0", "memchr", - "rand 0.9.2", - "sha2", + "rand 0.10.3", + "sha2 0.11.0", "stringprep", ] [[package]] name = "postgres-types" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54b858f82211e84682fecd373f68e1ceae642d8d751a1ebd13f33de6257b3e20" +checksum = "851ca9db4932932d69f3ea811b1abe63087a0f740a47692619dd40d4899b68be" dependencies = [ "bytes", "chrono", @@ -4906,14 +4987,15 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.14" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" +checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "bytes", - "getrandom 0.3.4", + "getrandom 0.4.2", "lru-slab", - "rand 0.9.2", + "rand 0.10.3", + "rand_pcg", "ring", "rustc-hash", "rustls", @@ -5056,6 +5138,15 @@ dependencies = [ "rand 0.9.2", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rayon" version = "1.11.0" @@ -5317,14 +5408,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "log", @@ -5666,7 +5757,7 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -5677,7 +5768,18 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", + "digest 0.11.3", ] [[package]] @@ -5789,7 +5891,7 @@ dependencies = [ "humantime", "itertools 0.13.0", "libtest-mimic", - "md-5", + "md-5 0.10.6", "owo-colors", "rand 0.8.5", "regex", @@ -6025,7 +6127,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6224,9 +6326,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.16" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcea47c8f71744367793f16c2db1f11cb859d28f436bdb4ca9193eb1f787ee42" +checksum = "a528f7d280f6d5b9cd149635c8705b0dd049754bc67d81d31fa25169a93809d3" dependencies = [ "async-trait", "byteorder", @@ -6241,7 +6343,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.2", + "rand 0.10.3", "socket2", "tokio", "tokio-util", @@ -6975,7 +7077,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] From 5ddce0465612175bb1e3fb0684078e7f14033e2b Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Mon, 5 Oct 2026 15:31:15 +0200 Subject: [PATCH 22/25] ci: ignore quick-xml advisories pinned by object_store --- .github/workflows/audit.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index 691fd4f685e13..06666b4ecd1e5 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -48,4 +48,8 @@ jobs: - name: Run audit check # Note: you can ignore specific RUSTSEC issues using the `--ignore` flag ,for example: # run: cargo audit --ignore RUSTSEC-2026-0001 - run: cargo audit + # + # RUSTSEC-2026-0194/0195: quick-xml < 0.41 DoS. object_store 0.13 pins + # quick-xml ^0.39 and only parses its own store's API responses. Remove + # once object_store moves to quick-xml >= 0.41. + run: cargo audit --ignore RUSTSEC-2026-0194 --ignore RUSTSEC-2026-0195 From e0d502945b756d2495ede4725293454eaa313bef Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Tue, 6 Oct 2026 10:15:26 +0200 Subject: [PATCH 23/25] test(cli): run the storage integration tests against RustFS MinIO pulled its images from quay.io on 2026-09-24, so the container no longer starts. Minimal backport of apache/datafusion#25706 (without its retry loop and CI pre-pull). --- datafusion-cli/Cargo.toml | 2 +- datafusion-cli/tests/cli_integration.rs | 216 ++++++++++++++---------- 2 files changed, 127 insertions(+), 91 deletions(-) diff --git a/datafusion-cli/Cargo.toml b/datafusion-cli/Cargo.toml index 3fe6be964c3f6..0a32012147a24 100644 --- a/datafusion-cli/Cargo.toml +++ b/datafusion-cli/Cargo.toml @@ -77,4 +77,4 @@ ctor = { workspace = true } insta = { workspace = true } insta-cmd = "0.6.0" rstest = { workspace = true } -testcontainers-modules = { workspace = true, features = ["minio"] } +testcontainers-modules = { workspace = true } diff --git a/datafusion-cli/tests/cli_integration.rs b/datafusion-cli/tests/cli_integration.rs index 8cd7a7cd7b825..96887ed7c3257 100644 --- a/datafusion-cli/tests/cli_integration.rs +++ b/datafusion-cli/tests/cli_integration.rs @@ -20,15 +20,20 @@ use std::process::Command; use rstest::rstest; use async_trait::async_trait; +use futures::TryStreamExt; use insta::{Settings, glob}; use insta_cmd::{assert_cmd_snapshot, get_cargo_bin}; +use object_store::{ + ObjectStore, ObjectStoreExt, aws::AmazonS3Builder, local::LocalFileSystem, +}; use std::path::PathBuf; use std::{env, fs}; -use testcontainers_modules::minio; -use testcontainers_modules::testcontainers::core::{CmdWaitFor, ExecCommand, Mount}; +use testcontainers_modules::testcontainers::core::{ + CmdWaitFor, ExecCommand, IntoContainerPort, +}; use testcontainers_modules::testcontainers::runners::AsyncRunner; use testcontainers_modules::testcontainers::{ - ContainerAsync, ImageExt, TestcontainersError, + ContainerAsync, GenericImage, ImageExt, TestcontainersError, }; fn cli() -> Command { @@ -44,86 +49,115 @@ fn make_settings() -> Settings { settings } -/// Registry override for the image pinned by `testcontainers-modules`. -/// -/// MinIO withdrew `minio/minio` from Docker Hub on 2026-09-11. quay.io still -/// serves the same tags. Backport of apache/datafusion#25216. -const MINIO_IMAGE_NAME: &str = "quay.io/minio/minio"; - -async fn setup_minio_container() -> Result, String> { - const MINIO_ROOT_USER: &str = "TEST-DataFusionLogin"; - const MINIO_ROOT_PASSWORD: &str = "TEST-DataFusionPassword"; +const RUSTFS_ACCESS_KEY: &str = "TEST-DataFusionLogin"; +const RUSTFS_SECRET_KEY: &str = "TEST-DataFusionPassword"; - let data_path = - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../datafusion/core/tests/data"); +/// RustFS replaces MinIO, whose images are no longer publicly pullable. +/// Backport of apache/datafusion#25706. +const RUSTFS_IMAGE_NAME: &str = "docker.io/rustfs/rustfs"; +const RUSTFS_IMAGE_TAG: &str = "1.0.0"; - let absolute_data_path = data_path - .canonicalize() - .expect("Failed to get absolute path for test data"); - - let container = minio::MinIO::default() - .with_name(MINIO_IMAGE_NAME) - .with_env_var("MINIO_ROOT_USER", MINIO_ROOT_USER) - .with_env_var("MINIO_ROOT_PASSWORD", MINIO_ROOT_PASSWORD) - .with_mount(Mount::bind_mount( - absolute_data_path.to_str().unwrap(), - "/source", - )) +async fn setup_rustfs_container() -> Result, String> { + let container = GenericImage::new(RUSTFS_IMAGE_NAME, RUSTFS_IMAGE_TAG) + .with_exposed_port(9000.tcp()) + .with_env_var("RUSTFS_ACCESS_KEY", RUSTFS_ACCESS_KEY) + .with_env_var("RUSTFS_SECRET_KEY", RUSTFS_SECRET_KEY) + .with_env_var("RUSTFS_CONSOLE_ENABLE", "false") .start() - .await; - - match container { - Ok(container) => { - // We wait for MinIO to be healthy and prepare test files. We do it via CLI to avoid s3 dependency - let commands = [ - ExecCommand::new(["/usr/bin/mc", "ready", "local"]), - ExecCommand::new([ - "/usr/bin/mc", - "alias", - "set", - "localminio", - "http://localhost:9000", - MINIO_ROOT_USER, - MINIO_ROOT_PASSWORD, - ]), - ExecCommand::new(["/usr/bin/mc", "mb", "localminio/data"]), - ExecCommand::new([ - "/usr/bin/mc", - "cp", - "-r", - "/source/", - "localminio/data/", - ]), - ]; - - for command in commands { - let command = - command.with_cmd_ready_condition(CmdWaitFor::Exit { code: Some(0) }); - - let cmd_ref = format!("{command:?}"); - - if let Err(e) = container.exec(command).await { - let stdout = container.stdout_to_vec().await.unwrap_or_default(); - let stderr = container.stderr_to_vec().await.unwrap_or_default(); - - return Err(format!( - "Failed to execute command: {}\nError: {}\nStdout: {:?}\nStderr: {:?}", - cmd_ref, - e, - String::from_utf8_lossy(&stdout), - String::from_utf8_lossy(&stderr) - )); - } - } - - Ok(container) + .await + .map_err(|e| match e { + TestcontainersError::Client(e) => format!( + "Failed to start RustFS container. Ensure Docker is running and accessible: {e}" + ), + e => format!("Failed to start RustFS container: {e}"), + })?; + + // Wait for RustFS to be healthy and create the bucket with the image's `curl`, + // then upload the test files through `object_store`. + let credentials = format!("{RUSTFS_ACCESS_KEY}:{RUSTFS_SECRET_KEY}"); + let commands = [ + ExecCommand::new([ + "curl", + "--fail", + "--silent", + "--show-error", + "--retry", + "60", + "--retry-delay", + "1", + "--retry-all-errors", + "--max-time", + "5", + "http://localhost:9000/health/ready", + ]), + ExecCommand::new([ + "curl", + "--fail", + "--silent", + "--show-error", + "--aws-sigv4", + "aws:amz:us-east-1:s3", + "--user", + &credentials, + "--request", + "PUT", + "http://localhost:9000/data", + ]), + ]; + + for command in commands { + let command = + command.with_cmd_ready_condition(CmdWaitFor::Exit { code: Some(0) }); + + let cmd_ref = format!("{command:?}"); + + if let Err(e) = container.exec(command).await { + let stdout = container.stdout_to_vec().await.unwrap_or_default(); + let stderr = container.stderr_to_vec().await.unwrap_or_default(); + + return Err(format!( + "Failed to execute command: {}\nError: {}\nStdout: {:?}\nStderr: {:?}", + cmd_ref, + e, + String::from_utf8_lossy(&stdout), + String::from_utf8_lossy(&stderr) + )); } + } - Err(TestcontainersError::Client(e)) => Err(format!( - "Failed to start MinIO container. Ensure Docker is running and accessible: {e}" - )), - Err(e) => Err(format!("Failed to start MinIO container: {e}")), + let port = container + .get_host_port_ipv4(9000) + .await + .map_err(|e| e.to_string())?; + let store = AmazonS3Builder::new() + .with_bucket_name("data") + .with_region("us-east-1") + .with_access_key_id(RUSTFS_ACCESS_KEY) + .with_secret_access_key(RUSTFS_SECRET_KEY) + .with_endpoint(format!("http://localhost:{port}")) + .with_allow_http(true) + .build() + .map_err(|e| e.to_string())?; + let data_path = + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../datafusion/core/tests/data"); + let source = + LocalFileSystem::new_with_prefix(data_path).map_err(|e| e.to_string())?; + let mut files = source.list(None); + while let Some(file) = files.try_next().await.map_err(|e| e.to_string())? { + let data = source + .get(&file.location) + .await + .map_err(|e| e.to_string())? + .bytes() + .await + .map_err(|e| e.to_string())?; + store + .put(&file.location, data.into()) + .await + .map_err(|e| e.to_string())?; } + + Ok(container) } #[cfg(test)] @@ -256,7 +290,7 @@ async fn test_cli() { return; } - let container = match setup_minio_container().await { + let container = match setup_rustfs_container().await { Ok(c) => c, Err(e) if e.contains("toomanyrequests") => { eprintln!("Skipping test: Docker pull rate limit reached: {e}"); @@ -296,7 +330,7 @@ async fn test_aws_options() { let settings = make_settings(); let _bound = settings.bind_to_scope(); - let container = match setup_minio_container().await { + let container = match setup_rustfs_container().await { Ok(c) => c, Err(e) if e.contains("toomanyrequests") => { eprintln!("Skipping test: Docker pull rate limit reached: {e}"); @@ -394,7 +428,7 @@ async fn test_s3_url_fallback() { return; } - let container = match setup_minio_container().await { + let container = match setup_rustfs_container().await { Ok(c) => c, Err(e) if e.contains("toomanyrequests") => { eprintln!("Skipping test: Docker pull rate limit reached: {e}"); @@ -420,7 +454,7 @@ OPTIONS ( SELECT * FROM partitioned_data ORDER BY column_1, column_2 LIMIT 5; "#; - assert_cmd_snapshot!(cli().with_minio(&container).await.pass_stdin(input)); + assert_cmd_snapshot!(cli().with_rustfs(&container).await.pass_stdin(input)); } /// Validate object store profiling output @@ -431,7 +465,7 @@ async fn test_object_store_profiling() { return; } - let container = match setup_minio_container().await { + let container = match setup_rustfs_container().await { Ok(c) => c, Err(e) if e.contains("toomanyrequests") => { eprintln!("Skipping test: Docker pull rate limit reached: {e}"); @@ -481,21 +515,23 @@ SELECT * from CARS LIMIT 1; SELECT * from CARS LIMIT 1; "#; - assert_cmd_snapshot!(cli().with_minio(&container).await.pass_stdin(input)); + assert_cmd_snapshot!(cli().with_rustfs(&container).await.pass_stdin(input)); } -/// Extension trait to Add the minio connection information to a Command +/// Extension trait to Add the RustFS connection information to a Command #[async_trait] -trait MinioCommandExt { - async fn with_minio(&mut self, container: &ContainerAsync) - -> &mut Self; +trait RustfsCommandExt { + async fn with_rustfs( + &mut self, + container: &ContainerAsync, + ) -> &mut Self; } #[async_trait] -impl MinioCommandExt for Command { - async fn with_minio( +impl RustfsCommandExt for Command { + async fn with_rustfs( &mut self, - container: &ContainerAsync, + container: &ContainerAsync, ) -> &mut Self { let port = container.get_host_port_ipv4(9000).await.unwrap(); From 47f888e7af4d16e6b5ea61dc37b12880897b2539 Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Tue, 6 Oct 2026 13:32:37 +0200 Subject: [PATCH 24/25] chore: pin arrow/parquet to 58.0.0, as shipped with 53.1.0 With arrow >= 58.1, 53.1.0 fails the spilling_fuzz high-cardinality aggregate test with ResourcesExhausted; the fix (apache/datafusion#21633) only landed in 54. Keeps object_store 0.13.2 (Path::join) and the cargo audit bumps. --- Cargo.lock | 99 +++++++++++++++++++++++++----------------------------- Cargo.toml | 14 ++++---- 2 files changed, 53 insertions(+), 60 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d1c895a20b9cc..13e88356ff1f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -256,9 +256,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3" +checksum = "602268ce9f569f282cedb9a9f6bac569b680af47b9b077d515900c03c5d190da" dependencies = [ "arrow-arith", "arrow-array", @@ -279,9 +279,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a" +checksum = "cd53c6bf277dea91f136ae8e3a5d7041b44b5e489e244e637d00ae302051f56f" dependencies = [ "arrow-array", "arrow-buffer", @@ -293,9 +293,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69" +checksum = "e53796e07a6525edaf7dc28b540d477a934aff14af97967ad1d5550878969b9e" dependencies = [ "ahash", "arrow-buffer", @@ -304,7 +304,7 @@ dependencies = [ "chrono", "chrono-tz", "half", - "hashbrown 0.17.1", + "hashbrown 0.16.1", "num-complex", "num-integer", "num-traits", @@ -312,9 +312,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5" +checksum = "f2c1a85bb2e94ee10b76531d8bc3ce9b7b4c0d508cabfb17d477f63f2617bd20" dependencies = [ "bytes", "half", @@ -324,9 +324,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34" +checksum = "89fb245db6b0e234ed8e15b644edb8664673fefe630575e94e62cd9d489a8a26" dependencies = [ "arrow-array", "arrow-buffer", @@ -346,9 +346,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26" +checksum = "d374882fb465a194462527c0c15a93aa19a554cf690a6b77a26b2a02539937a7" dependencies = [ "arrow-array", "arrow-cast", @@ -361,9 +361,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b" +checksum = "189d210bc4244c715fa3ed9e6e22864673cccb73d5da28c2723fb2e527329b33" dependencies = [ "arrow-buffer", "arrow-schema", @@ -374,9 +374,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2dbe34824c639e43136af8f106992792ab456540d54b880bc320a3192502d2e" +checksum = "b4f5cdf00ee0003ba0768d3575d0afc47d736b29673b14c3c228fdffa9a3fb29" dependencies = [ "arrow-arith", "arrow-array", @@ -402,9 +402,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b" +checksum = "7968c2e5210c41f4909b2ef76f6e05e172b99021c2def5edf3cc48fdd39d1d6c" dependencies = [ "arrow-array", "arrow-buffer", @@ -418,16 +418,15 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14" +checksum = "92111dba5bf900f443488e01f00d8c4ddc2f47f5c50039d18120287b580baa22" dependencies = [ "arrow-array", "arrow-buffer", "arrow-cast", - "arrow-ord", + "arrow-data", "arrow-schema", - "arrow-select", "chrono", "half", "indexmap 2.13.0", @@ -443,9 +442,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0" +checksum = "211136cb253577ee1a6665f741a13136d4e563f64f5093ffd6fb837af90b9495" dependencies = [ "arrow-array", "arrow-buffer", @@ -456,9 +455,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c" +checksum = "8e0f20145f9f5ea3fe383e2ba7a7487bf19be36aa9dbf5dd6a1f92f657179663" dependencies = [ "arrow-array", "arrow-buffer", @@ -469,9 +468,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e" +checksum = "1b47e0ca91cc438d2c7879fe95e0bca5329fff28649e30a88c6f760b1faeddcb" dependencies = [ "bitflags", "serde", @@ -481,9 +480,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad" +checksum = "750a7d1dda177735f5e82a314485b6915c7cccdbb278262ac44090f4aba4a325" dependencies = [ "ahash", "arrow-array", @@ -495,9 +494,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c" +checksum = "e1eab1208bc4fe55d768cdc9b9f3d9df5a794cdb3ee2586bf89f9b30dc31ad8c" dependencies = [ "arrow-array", "arrow-buffer", @@ -1735,9 +1734,9 @@ checksum = "52560adf09603e58c9a7ee1fe1dcb95a16927b17c127f0ac02d6e768a0e25bc1" [[package]] name = "ctutils" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" dependencies = [ "cmov", ] @@ -2985,7 +2984,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3405,12 +3404,6 @@ dependencies = [ "foldhash 0.2.0", ] -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - [[package]] name = "heck" version = "0.5.0" @@ -4119,9 +4112,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.13.1" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +checksum = "98c23545df7ecf1b16c303910a69b079e8e251d60f7dd2cc9b4177f2afaf1746" dependencies = [ "twox-hash", ] @@ -4513,9 +4506,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.4.0" +version = "58.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704" +checksum = "3f491d0ef1b510194426ee67ddc18a9b747ef3c42050c19322a2cd2e1666c29b" dependencies = [ "ahash", "arrow-array", @@ -4531,7 +4524,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.17.1", + "hashbrown 0.16.1", "lz4_flex", "num-bigint", "num-integer", @@ -4896,7 +4889,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -4915,7 +4908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.117", @@ -5018,7 +5011,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -5408,7 +5401,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6127,7 +6120,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -7077,7 +7070,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 02a3c975dd17b..d8a7e424873c2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,19 +91,19 @@ ahash = { version = "0.8", default-features = false, features = [ "runtime-rng", ] } apache-avro = { version = "0.21", default-features = false } -arrow = { version = "58.4.0", features = [ +arrow = { version = "58.0.0", features = [ "prettyprint", "chrono-tz", ] } -arrow-buffer = { version = "58.4.0", default-features = false } -arrow-flight = { version = "58.4.0", features = [ +arrow-buffer = { version = "58.0.0", default-features = false } +arrow-flight = { version = "58.0.0", features = [ "flight-sql-experimental", ] } -arrow-ipc = { version = "58.4.0", default-features = false, features = [ +arrow-ipc = { version = "58.0.0", default-features = false, features = [ "lz4", ] } -arrow-ord = { version = "58.4.0", default-features = false } -arrow-schema = { version = "58.4.0", default-features = false } +arrow-ord = { version = "58.0.0", default-features = false } +arrow-schema = { version = "58.0.0", default-features = false } async-trait = "0.1.89" bigdecimal = "0.4.8" bytes = "1.11" @@ -168,7 +168,7 @@ memchr = "2.8.0" num-traits = { version = "0.2" } object_store = { version = "0.13.1", default-features = false } parking_lot = "0.12" -parquet = { version = "58.4.0", default-features = false, features = [ +parquet = { version = "58.0.0", default-features = false, features = [ "arrow", "async", "object_store", From f55327ab63bd3b5619c2ee098cbc61dd2c7d51bc Mon Sep 17 00:00:00 2001 From: Lili Cosic Date: Wed, 7 Oct 2026 16:48:07 +0200 Subject: [PATCH 25/25] test(sqllogictest): expect null bitmaps in output_bytes on arrow 58.0 Parquet 58.0 attaches a 64-byte null bitmap to every nullable column even without nulls, and the ported RecordBatchMemoryCounter counts it (the previous counter went through to_data(), which drops it). From arrow 58.1 (apache/arrow-rs#9411) the reader omits empty bitmaps and this goes back to 80 B. --- .../sqllogictest/test_files/dynamic_filter_pushdown_config.slt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt index cbf9f81e425fa..42aebbf56527a 100644 --- a/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt +++ b/datafusion/sqllogictest/test_files/dynamic_filter_pushdown_config.slt @@ -104,7 +104,7 @@ Plan with Metrics 03)----ProjectionExec: expr=[id@0 as id, value@1 as v, value@1 + id@0 as name], metrics=[output_rows=10, ] 04)------FilterExec: value@1 > 3, metrics=[output_rows=10, , selectivity=100% (10/10)] 05)--------RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1, metrics=[output_rows=10, ] -06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value], file_type=parquet, predicate=value@1 > 3 AND DynamicFilter [ value@1 IS NULL OR value@1 > 800 ], pruning_predicate=value_null_count@1 != row_count@2 AND value_max@0 > 3 AND (value_null_count@1 > 0 OR value_null_count@1 != row_count@2 AND value_max@0 > 800), required_guarantees=[], metrics=[output_rows=10, elapsed_compute=1ns, output_bytes=80.0 B, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched -> 1 fully matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=210, metadata_load_time=, scan_efficiency_ratio=18% (210/1.15 K)] +06)----------DataSourceExec: file_groups={1 group: [[WORKSPACE_ROOT/datafusion/sqllogictest/test_files/scratch/dynamic_filter_pushdown_config/test_data.parquet]]}, projection=[id, value], file_type=parquet, predicate=value@1 > 3 AND DynamicFilter [ value@1 IS NULL OR value@1 > 800 ], pruning_predicate=value_null_count@1 != row_count@2 AND value_max@0 > 3 AND (value_null_count@1 > 0 OR value_null_count@1 != row_count@2 AND value_max@0 > 800), required_guarantees=[], metrics=[output_rows=10, elapsed_compute=1ns, output_bytes=208.0 B, files_ranges_pruned_statistics=1 total → 1 matched, row_groups_pruned_statistics=1 total → 1 matched -> 1 fully matched, row_groups_pruned_bloom_filter=1 total → 1 matched, page_index_pages_pruned=1 total → 1 matched, limit_pruned_row_groups=0 total → 0 matched, bytes_scanned=210, metadata_load_time=, scan_efficiency_ratio=18% (210/1.15 K)] statement ok set datafusion.explain.analyze_level = dev;