diff --git a/Cargo.lock b/Cargo.lock index dad2e6a185458..49cef2a7d47ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -223,9 +223,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb372a7cbcac02a35d3fb7b3fc1f969ec078e871f9bb899bf00a2e1809bec8a3" +checksum = "e4754a624e5ae42081f464514be454b39711daae0458906dacde5f4c632f33a8" dependencies = [ "arrow-arith", "arrow-array", @@ -246,9 +246,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f377dcd19e440174596d83deb49cd724886d91060c07fec4f67014ef9d54049" +checksum = "f7b3141e0ec5145a22d8694ea8b6d6f69305971c4fa1c1a13ef0195aef2d678b" dependencies = [ "arrow-array", "arrow-buffer", @@ -260,9 +260,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a23eaff85a44e9fa914660fb0d0bb00b79c4a3d888b5334adb3ea4330c84f002" +checksum = "4c8955af33b25f3b175ee10af580577280b4bd01f7e823d94c7cdef7cf8c9aef" dependencies = [ "ahash", "arrow-buffer", @@ -279,9 +279,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2819d893750cb3380ab31ebdc8c68874dd4429f90fd09180f3c93538bd21626" +checksum = "c697ddca96183182f35b3a18e50b9110b11e916d7b7799cbfd4d34662f2c56c2" dependencies = [ "bytes", "half", @@ -291,9 +291,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d131abb183f80c450d4591dc784f8d7750c50c6e2bc3fcaad148afc8361271" +checksum = "646bbb821e86fd57189c10b4fcdaa941deaf4181924917b0daa92735baa6ada5" dependencies = [ "arrow-array", "arrow-buffer", @@ -313,9 +313,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2275877a0e5e7e7c76954669366c2aa1a829e340ab1f612e647507860906fb6b" +checksum = "8da746f4180004e3ce7b83c977daf6394d768332349d3d913998b10a120b790a" dependencies = [ "arrow-array", "arrow-cast", @@ -328,9 +328,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05738f3d42cb922b9096f7786f606fcb8669260c2640df8490533bb2fa38c9d3" +checksum = "1fdd994a9d28e6365aa78e15da3f3950c0fdcea6b963a12fa1c391afb637b304" dependencies = [ "arrow-buffer", "arrow-schema", @@ -341,9 +341,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b5f57c3d39d1b1b7c1376a772ea86a131e7da310aed54ebea9363124bb885e3" +checksum = "58c5b083668e6230eae3eab2fc4b5fb989974c845d0aa538dde61a4327c78675" dependencies = [ "arrow-arith", "arrow-array", @@ -369,9 +369,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d09446e8076c4b3f235603d9ea7c5494e73d441b01cd61fb33d7254c11964b3" +checksum = "abf7df950701ab528bf7c0cf7eeadc0445d03ef5d6ffc151eaae6b38a58feff1" dependencies = [ "arrow-array", "arrow-buffer", @@ -385,9 +385,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "371ffd66fa77f71d7628c63f209c9ca5341081051aa32f9c8020feb0def787c0" +checksum = "0ff8357658bedc49792b13e2e862b80df908171275f8e6e075c460da5ee4bf86" dependencies = [ "arrow-array", "arrow-buffer", @@ -409,9 +409,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc94fc7adec5d1ba9e8cd1b1e8d6f72423b33fe978bf1f46d970fafab787521" +checksum = "f7d8f1870e03d4cbed632959498bcc84083b5a24bded52905ae1695bd29da45b" dependencies = [ "arrow-array", "arrow-buffer", @@ -422,9 +422,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169676f317157dc079cc5def6354d16db63d8861d61046d2f3883268ced6f99f" +checksum = "18228633bad92bff92a95746bbeb16e5fc318e8382b75619dec26db79e4de4c0" dependencies = [ "arrow-array", "arrow-buffer", @@ -435,9 +435,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27609cd7dd45f006abae27995c2729ef6f4b9361cde1ddd019dc31a5aa017e0" +checksum = "8c872d36b7bf2a6a6a2b40de9156265f0242910791db366a2c17476ba8330d68" dependencies = [ "bitflags", "serde", @@ -447,9 +447,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae980d021879ea119dd6e2a13912d81e64abed372d53163e804dfe84639d8010" +checksum = "68bf3e3efbd1278f770d67e5dc410257300b161b93baedb3aae836144edcaf4b" dependencies = [ "ahash", "arrow-array", @@ -461,9 +461,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf35e8ef49dcf0c5f6d175edee6b8af7b45611805333129c541a8b89a0fc0534" +checksum = "85e968097061b3c0e9fe3079cf2e703e487890700546b5b0647f60fca1b5a8d8" dependencies = [ "arrow-array", "arrow-buffer", @@ -490,9 +490,9 @@ dependencies = [ [[package]] name = "astral-tokio-tar" -version = "0.6.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c23f3af104b40a3430ccb90ed5f7bd877a8dc5c26fc92fde51a22b40890dcf9" +checksum = "cb50a7aae84a03bf55b067832bc376f4961b790c97e64d3eacee97d389b90277" dependencies = [ "filetime", "futures-core", @@ -2206,7 +2206,6 @@ dependencies = [ "blake2", "blake3", "chrono", - "chrono-tz", "criterion", "ctor", "datafusion-common", @@ -2860,7 +2859,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -4317,9 +4316,9 @@ dependencies = [ [[package]] name = "parquet" -version = "57.1.0" +version = "57.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be3e4f6d320dd92bfa7d612e265d7d08bba0a240bab86af3425e1d255a511d89" +checksum = "6ee96b29972a257b855ff2341b37e61af5f12d6af1158b6dcdb5b31ea07bb3cb" dependencies = [ "ahash", "arrow-array", @@ -4810,7 +4809,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5168,7 +5167,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -5220,9 +5219,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "aws-lc-rs", "ring", @@ -5886,7 +5885,7 @@ dependencies = [ "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6774,7 +6773,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index dcb6f2f447cdc..0ade4e524041f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -91,19 +91,19 @@ ahash = { version = "0.8", default-features = false, features = [ "runtime-rng", ] } apache-avro = { version = "0.21", default-features = false } -arrow = { version = "57.1.0", features = [ +arrow = { version = "57.2.0", features = [ "prettyprint", "chrono-tz", ] } -arrow-buffer = { version = "57.1.0", default-features = false } -arrow-flight = { version = "57.1.0", features = [ +arrow-buffer = { version = "57.2.0", default-features = false } +arrow-flight = { version = "57.2.0", features = [ "flight-sql-experimental", ] } -arrow-ipc = { version = "57.1.0", default-features = false, features = [ +arrow-ipc = { version = "57.2.0", default-features = false, features = [ "lz4", ] } -arrow-ord = { version = "57.1.0", default-features = false } -arrow-schema = { version = "57.1.0", default-features = false } +arrow-ord = { version = "57.2.0", default-features = false } +arrow-schema = { version = "57.2.0", default-features = false } async-trait = "0.1.89" bigdecimal = "0.4.8" bytes = "1.11" @@ -161,12 +161,12 @@ hex = { version = "0.4.3" } indexmap = "2.12.1" insta = { version = "1.46.0", features = ["glob", "filters"] } itertools = "0.14" -liblzma = { version = "0.4.4", features = ["static"] } +liblzma = { version = "0.4.5", features = ["static"] } log = "^0.4" num-traits = { version = "0.2" } object_store = { version = "0.12.4", default-features = false } parking_lot = "0.12" -parquet = { version = "57.1.0", default-features = false, features = [ +parquet = { version = "57.2.0", default-features = false, features = [ "arrow", "async", "object_store", diff --git a/datafusion-cli/tests/cli_integration.rs b/datafusion-cli/tests/cli_integration.rs index 8b8b786d652e3..98a6284e54605 100644 --- a/datafusion-cli/tests/cli_integration.rs +++ b/datafusion-cli/tests/cli_integration.rs @@ -44,6 +44,12 @@ fn make_settings() -> Settings { settings } +/// Registry override for the image pinned by `testcontainers-modules`. +/// +/// MinIO withdrew `minio/minio` from Docker Hub on 2026-09-11. quay.io still +/// serves the same tags. Backport of apache/datafusion#25216. +const MINIO_IMAGE_NAME: &str = "quay.io/minio/minio"; + async fn setup_minio_container() -> ContainerAsync { const MINIO_ROOT_USER: &str = "TEST-DataFusionLogin"; const MINIO_ROOT_PASSWORD: &str = "TEST-DataFusionPassword"; @@ -56,6 +62,7 @@ async fn setup_minio_container() -> ContainerAsync { .expect("Failed to get absolute path for test data"); let container = minio::MinIO::default() + .with_name(MINIO_IMAGE_NAME) .with_env_var("MINIO_ROOT_USER", MINIO_ROOT_USER) .with_env_var("MINIO_ROOT_PASSWORD", MINIO_ROOT_PASSWORD) .with_mount(Mount::bind_mount( diff --git a/datafusion/common/src/scalar/mod.rs b/datafusion/common/src/scalar/mod.rs index e4e048ad3c0d8..eda4952cf590b 100644 --- a/datafusion/common/src/scalar/mod.rs +++ b/datafusion/common/src/scalar/mod.rs @@ -8868,7 +8868,7 @@ mod tests { .unwrap(), ScalarValue::try_new_null(&DataType::Map(map_field_ref, false)).unwrap(), ScalarValue::try_new_null(&DataType::Union( - UnionFields::new(vec![42], vec![field_ref]), + UnionFields::try_new(vec![42], vec![field_ref]).unwrap(), UnionMode::Dense, )) .unwrap(), @@ -8971,13 +8971,14 @@ mod tests { } // Test union type - let union_fields = UnionFields::new( + let union_fields = UnionFields::try_new( vec![0, 1], vec![ Field::new("i32", DataType::Int32, false), Field::new("f64", DataType::Float64, false), ], - ); + ) + .unwrap(); let union_result = ScalarValue::new_default(&DataType::Union( union_fields.clone(), UnionMode::Sparse, diff --git a/datafusion/common/src/utils/memory.rs b/datafusion/common/src/utils/memory.rs index 78ec434d2b577..bb006295f8c1d 100644 --- a/datafusion/common/src/utils/memory.rs +++ b/datafusion/common/src/utils/memory.rs @@ -19,9 +19,27 @@ use crate::error::_exec_datafusion_err; use crate::{HashSet, Result}; -use arrow::array::ArrayData; +use arrow::array::types::{ + ByteArrayType, ByteViewType, Int8Type, Int16Type, Int32Type, Int64Type, + RunEndIndexType, UInt8Type, UInt16Type, UInt32Type, UInt64Type, +}; +use arrow::array::{ + Array, ArrayRef, AsArray, GenericByteArray, GenericByteViewArray, GenericListArray, + GenericListViewArray, RunArray, +}; +use arrow::buffer::Buffer; +use arrow::datatypes::DataType; +use arrow::downcast_primitive_array; use arrow::record_batch::RecordBatch; -use std::{mem::size_of, ptr::NonNull}; +use std::mem::size_of; +use std::num::NonZero; +use std::sync::Arc; + +/// Maximum number of distinct buffer IDs retained inline before promotion to +/// a [`HashSet`]. Sixteen keeps small buffer sets allocation-free while +/// limiting linear lookup and inline storage to 16 pointer-sized entries. +/// This is a performance heuristic, not a semantic limit. +const INLINE_BUFFER_IDS: usize = 16; /// Estimates the memory size required for a hash table prior to allocation. /// @@ -117,55 +135,367 @@ pub fn estimate_memory_size(num_elements: usize, fixed_size: usize) -> Result /// For a `RecordBatch` with two columns: `col1` and `col2`, two columns are pointing /// to a sub-region of the same buffer. /// +/// ```text /// {xxxxxxxxxxxxxxxxxxx} <--- buffer /// ^ ^ ^ ^ /// | | | | /// col1->{ } | | /// col2--------->{ } +/// ``` /// /// In the above case, `get_record_batch_memory_size` will return the size of /// the buffer, instead of the sum of `col1` and `col2`'s actual memory size. /// -/// Note: Current `RecordBatch`.get_array_memory_size()` will double count the -/// buffer memory size if multiple arrays within the batch are sharing the same -/// `Buffer`. This method provides temporary fix until the issue is resolved: -/// +/// Note: [`RecordBatch::get_array_memory_size`] double counts the buffer +/// memory size if multiple arrays within the batch are sharing the same +/// `Buffer`, while this function counts each `Buffer` exactly once. pub fn get_record_batch_memory_size(batch: &RecordBatch) -> usize { - // Store pointers to `Buffer`'s start memory address (instead of actual - // used data region's pointer represented by current `Array`) - let mut counted_buffers: HashSet> = HashSet::new(); - let mut total_size = 0; + RecordBatchMemoryCounter::new().count_batch(batch) +} + +/// Tracks the memory used by a sequence of [`RecordBatch`]es that may share +/// underlying buffers, counting each buffer exactly once. +/// +/// Use this instead of [`get_record_batch_memory_size`] to account for the +/// total memory of a sequence of batches, e.g. when buffering the batches of +/// an input stream. Such batches can share buffers (for example, operators +/// like aggregates emit one large batch as multiple zero-copy slices), and +/// calling [`get_record_batch_memory_size`] per batch counts the shared +/// buffers once per batch, while this counter counts them exactly once. A +/// batch's buffers are kept alive by the batch even when only a sub-range is +/// referenced, so counting unique buffers in full reflects the memory the +/// batches actually retain. +#[derive(Debug, Default)] +pub struct RecordBatchMemoryCounter { + /// Start addresses of `Buffer`s that have already been counted (instead of + /// actual used data region's pointer represented by current `Array`) + counted_buffers: BufferIdSet, + /// Array objects already counted by [`Self::count_batch_with_array_overhead`] + counted_arrays: HashSet, + /// Total memory of all counted allocations + memory_usage: usize, +} + +impl RecordBatchMemoryCounter { + pub fn new() -> Self { + Self::default() + } + + /// Count `batch`, returning the memory used by its buffers that have not + /// been counted before. + pub fn count_batch(&mut self, batch: &RecordBatch) -> usize { + let previous_memory_usage = self.memory_usage; + + for array in batch.columns() { + self.count_array_memory_size(array.as_ref()); + } + + self.memory_usage - previous_memory_usage + } + + /// Counts unique buffers and Array objects retained by `batch`. + /// + /// This is useful for accounting a sequence of batches at an operator + /// boundary. It counts buffers once, including buffers shared by multiple + /// batches. Array-object overhead is deduplicated recursively when a shared + /// child has the same `ArrayRef` identity; Arrow constructors that rebuild + /// child array objects may conservatively count their object overhead again. + pub fn count_batch_with_array_overhead(&mut self, batch: &RecordBatch) -> usize { + let mut total_size = self.count_batch(batch); + let mut array_overhead = 0; + + for array in batch.columns() { + array_overhead += + count_unique_array_object_memory_size(array, &mut self.counted_arrays); + } + + total_size += array_overhead; + self.memory_usage += array_overhead; + total_size + } + + /// Total memory of all counted allocations. + pub fn memory_usage(&self) -> usize { + self.memory_usage + } + + fn count_buffer_memory_size(&mut self, buffer: &Buffer) { + if self.counted_buffers.insert(buffer.data_ptr().addr()) { + self.memory_usage += buffer.capacity(); + } + } + + /// Count the memory usage of `array` and its children recursively. + fn count_array_memory_size(&mut self, array: &dyn Array) { + if let Some(nulls) = array.nulls() { + self.count_buffer_memory_size(nulls.buffer()); + } + + downcast_primitive_array! { + array => self.count_buffer_memory_size(array.values().inner()), + DataType::Null => {} + DataType::Boolean => { + self.count_buffer_memory_size(array.as_boolean().values().inner()); + } + DataType::Binary => { + self.count_byte_array_memory_size(array.as_binary::()); + } + DataType::LargeBinary => { + self.count_byte_array_memory_size(array.as_binary::()); + } + DataType::Utf8 => { + self.count_byte_array_memory_size(array.as_string::()); + } + DataType::LargeUtf8 => { + self.count_byte_array_memory_size(array.as_string::()); + } + DataType::BinaryView => { + self.count_byte_view_array_memory_size(array.as_binary_view()); + } + DataType::Utf8View => { + self.count_byte_view_array_memory_size(array.as_string_view()); + } + DataType::FixedSizeBinary(_) => { + self.count_buffer_memory_size(array.as_fixed_size_binary().values()); + } + DataType::List(_) => { + self.count_list_array_memory_size(array.as_list::()); + } + DataType::LargeList(_) => { + self.count_list_array_memory_size(array.as_list::()); + } + DataType::ListView(_) => { + self.count_list_view_array_memory_size(array.as_list_view::()); + } + DataType::LargeListView(_) => { + self.count_list_view_array_memory_size(array.as_list_view::()); + } + DataType::FixedSizeList(_, _) => { + self.count_array_memory_size( + array.as_fixed_size_list().values().as_ref(), + ); + } + DataType::Struct(_) => { + for child in array.as_struct().columns() { + self.count_array_memory_size(child.as_ref()); + } + } + DataType::Union(_, _) => { + let array = array.as_union(); + self.count_buffer_memory_size(array.type_ids().inner()); + if let Some(offsets) = array.offsets() { + self.count_buffer_memory_size(offsets.inner()); + } + for (type_id, _) in array.fields().iter() { + self.count_array_memory_size(array.child(type_id).as_ref()); + } + } + DataType::Dictionary(_, _) => { + let array = array.as_any_dictionary(); + self.count_array_memory_size(array.keys()); + self.count_array_memory_size(array.values().as_ref()); + } + DataType::Map(_, _) => { + let array = array.as_map(); + self.count_buffer_memory_size(array.offsets().inner().inner()); + self.count_array_memory_size(array.entries()); + } + DataType::RunEndEncoded(run_ends, _) => match run_ends.data_type() { + DataType::Int16 => self.count_run_array_memory_size::(array), + DataType::Int32 => self.count_run_array_memory_size::(array), + DataType::Int64 => self.count_run_array_memory_size::(array), + // Arrow only permits Int16, Int32, and Int64 run-end indexes. A + // custom Array implementation may still expose malformed data; + // retain correct accounting for it without panicking. + _ => self.count_array_data_memory_size(&array.to_data()), + }, + // All currently supported non-primitive layouts are handled above. + // The Arrow macro requires a final arm for primitive variants that + // its nested dispatch has already consumed. Keep a safe generic + // fallback for custom or future Array implementations. + _ => self.count_array_data_memory_size(&array.to_data()), + } + } + + fn count_byte_array_memory_size( + &mut self, + array: &GenericByteArray, + ) { + self.count_buffer_memory_size(array.offsets().inner().inner()); + self.count_buffer_memory_size(array.values()); + } + + fn count_byte_view_array_memory_size( + &mut self, + array: &GenericByteViewArray, + ) { + self.count_buffer_memory_size(array.views().inner()); + for buffer in array.data_buffers().iter() { + self.count_buffer_memory_size(buffer); + } + } + + fn count_list_array_memory_size( + &mut self, + array: &GenericListArray, + ) { + self.count_buffer_memory_size(array.offsets().inner().inner()); + self.count_array_memory_size(array.values().as_ref()); + } + + fn count_list_view_array_memory_size( + &mut self, + array: &GenericListViewArray, + ) { + self.count_buffer_memory_size(array.offsets().inner()); + self.count_buffer_memory_size(array.sizes().inner()); + self.count_array_memory_size(array.values().as_ref()); + } - for array in batch.columns() { - let array_data = array.to_data(); - count_array_data_memory_size(&array_data, &mut counted_buffers, &mut total_size); + fn count_run_array_memory_size(&mut self, array: &dyn Array) { + if let Some(array) = array.as_any().downcast_ref::>() { + self.count_buffer_memory_size(array.run_ends().inner().inner()); + self.count_array_memory_size(array.values().as_ref()); + } else { + // The DataType and concrete array implementation disagree. Use the + // generic representation rather than panic while accounting memory. + self.count_array_data_memory_size(&array.to_data()); + } } - total_size + fn count_array_data_memory_size(&mut self, array_data: &arrow::array::ArrayData) { + for buffer in array_data.buffers() { + self.count_buffer_memory_size(buffer); + } + if let Some(nulls) = array_data.nulls() { + self.count_buffer_memory_size(nulls.buffer()); + } + for child in array_data.child_data() { + self.count_array_data_memory_size(child); + } + } +} + +/// Counts the unique Array object memory retained by `array` and its children. +fn count_unique_array_object_memory_size( + array: &ArrayRef, + counted_arrays: &mut HashSet, +) -> usize { + let array_ptr = Arc::as_ptr(array).cast::<()>() as usize; + if !counted_arrays.insert(array_ptr) { + return 0; + } + + let children = array_children(array); + let children_overhead: usize = children + .iter() + .map(|child| child.get_array_memory_size() - child.get_buffer_memory_size()) + .sum(); + let own_overhead = array.get_array_memory_size() + - array.get_buffer_memory_size() + - children_overhead; + + own_overhead + + children + .into_iter() + .map(|child| count_unique_array_object_memory_size(child, counted_arrays)) + .sum::() } -/// Count the memory usage of `array_data` and its children recursively. -fn count_array_data_memory_size( - array_data: &ArrayData, - counted_buffers: &mut HashSet>, - total_size: &mut usize, -) { - // Count memory usage for `array_data` - for buffer in array_data.buffers() { - if counted_buffers.insert(buffer.data_ptr()) { - *total_size += buffer.capacity(); - } // Otherwise the buffer's memory is already counted +/// Returns the `ArrayRef` children whose object allocations may be shared. +fn array_children(array: &ArrayRef) -> Vec<&ArrayRef> { + match array.data_type() { + DataType::Struct(_) => array.as_struct().columns().iter().collect(), + DataType::List(_) => vec![array.as_list::().values()], + DataType::LargeList(_) => vec![array.as_list::().values()], + DataType::ListView(_) => vec![array.as_list_view::().values()], + DataType::LargeListView(_) => vec![array.as_list_view::().values()], + DataType::FixedSizeList(_, _) => vec![array.as_fixed_size_list().values()], + DataType::Map(_, _) => { + let map = array.as_map(); + vec![map.keys(), map.values()] + } + DataType::Union(_, _) => { + let union = array.as_union(); + union + .fields() + .iter() + .map(|(type_id, _)| union.child(type_id)) + .collect() + } + DataType::Dictionary(key_type, _) => match key_type.as_ref() { + DataType::Int8 => vec![array.as_dictionary::().values()], + DataType::Int16 => vec![array.as_dictionary::().values()], + DataType::Int32 => vec![array.as_dictionary::().values()], + DataType::Int64 => vec![array.as_dictionary::().values()], + DataType::UInt8 => vec![array.as_dictionary::().values()], + DataType::UInt16 => vec![array.as_dictionary::().values()], + DataType::UInt32 => vec![array.as_dictionary::().values()], + DataType::UInt64 => vec![array.as_dictionary::().values()], + _ => unreachable!("invalid dictionary key type: {key_type}"), + }, + DataType::RunEndEncoded(run_ends, _) => match run_ends.data_type() { + DataType::Int16 => array + .as_any() + .downcast_ref::>() + .map(|array| vec![array.values()]) + .unwrap_or_default(), + DataType::Int32 => array + .as_any() + .downcast_ref::>() + .map(|array| vec![array.values()]) + .unwrap_or_default(), + DataType::Int64 => array + .as_any() + .downcast_ref::>() + .map(|array| vec![array.values()]) + .unwrap_or_default(), + _ => vec![], + }, + _ => vec![], } +} + +/// Tracks a small number of buffers inline, avoiding a heap allocation for +/// typical batches, and promotes to a hash set when more buffers are seen. +#[derive(Debug)] +struct BufferIdSet { + inline: [Option>; INLINE_BUFFER_IDS], + len: usize, + overflow: Option>>, +} - if let Some(null_buffer) = array_data.nulls() - && counted_buffers.insert(null_buffer.inner().inner().data_ptr()) - { - *total_size += null_buffer.inner().inner().capacity(); +impl Default for BufferIdSet { + fn default() -> Self { + Self { + inline: [None; INLINE_BUFFER_IDS], + len: 0, + overflow: None, + } } +} + +impl BufferIdSet { + fn insert(&mut self, buffer_id: NonZero) -> bool { + if let Some(overflow) = &mut self.overflow { + return overflow.insert(buffer_id); + } + + if self.inline[..self.len].contains(&Some(buffer_id)) { + return false; + } - // Count all children `ArrayData` recursively - for child in array_data.child_data() { - count_array_data_memory_size(child, counted_buffers, total_size); + if self.len < INLINE_BUFFER_IDS { + self.inline[self.len] = Some(buffer_id); + self.len += 1; + return true; + } + + let mut overflow = HashSet::with_capacity(INLINE_BUFFER_IDS + 1); + overflow.extend(self.inline.iter().flatten().copied()); + let inserted = overflow.insert(buffer_id); + self.overflow = Some(overflow); + inserted } } @@ -206,10 +536,51 @@ mod tests { #[cfg(test)] mod record_batch_tests { use super::*; - use arrow::array::{Float64Array, Int32Array, ListArray}; - use arrow::datatypes::{DataType, Field, Int32Type, Schema}; + use arrow::array::{ + ArrayData, ArrayRef, BinaryViewArray, DictionaryArray, Float64Array, Int16Array, + Int32Array, Int64Array, LargeListViewArray, ListArray, ListViewArray, MapArray, + RunArray, StringArray, StringViewArray, StructArray, UnionArray, new_null_array, + }; + use arrow::buffer::OffsetBuffer; + use arrow::datatypes::{ + DataType, Field, Fields, Int16Type, Int32Type, Int64Type, Schema, UnionFields, + UnionMode, + }; use std::sync::Arc; + fn array_data_memory_size(array: &dyn Array) -> usize { + fn count( + array_data: &ArrayData, + counted_buffers: &mut HashSet>, + total_size: &mut usize, + ) { + for buffer in array_data.buffers() { + if counted_buffers.insert(buffer.data_ptr().addr()) { + *total_size += buffer.capacity(); + } + } + if let Some(nulls) = array_data.nulls() { + let buffer = nulls.inner().inner(); + if counted_buffers.insert(buffer.data_ptr().addr()) { + *total_size += buffer.capacity(); + } + } + for child in array_data.child_data() { + count(child, counted_buffers, total_size); + } + } + + let mut total_size = 0; + count(&array.to_data(), &mut HashSet::default(), &mut total_size); + total_size + } + + fn assert_array_memory_size_matches(array: &dyn Array) { + let mut counter = RecordBatchMemoryCounter::new(); + counter.count_array_memory_size(array); + assert_eq!(counter.memory_usage(), array_data_memory_size(array)); + } + #[test] fn test_get_record_batch_memory_size() { let schema = Arc::new(Schema::new(vec![ @@ -295,6 +666,391 @@ mod record_batch_tests { assert_eq!(size_origin, size_sliced); } + #[test] + fn test_record_batch_memory_counter_array_overhead_shared_across_batches() { + let schema = Arc::new(Schema::new(vec![ + Field::new("ints", DataType::Int32, false), + Field::new("floats", DataType::Float64, false), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5, 6])), + Arc::new(Float64Array::from(vec![1., 2., 3., 4., 5., 6.])), + ], + ) + .unwrap(); + + let mut counter = RecordBatchMemoryCounter::new(); + assert_eq!( + counter.count_batch_with_array_overhead(&batch), + batch.get_array_memory_size() + ); + assert_eq!(counter.count_batch_with_array_overhead(&batch), 0); + assert_eq!(counter.memory_usage(), batch.get_array_memory_size()); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_shared_nested_array_overhead() { + let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let fields = + Fields::from(vec![Arc::new(Field::new("value", DataType::Int32, false))]); + let first = Arc::new(StructArray::new( + fields.clone(), + vec![Arc::clone(&shared_child)], + None, + )) as _; + let second = Arc::new(StructArray::new( + fields, + vec![Arc::clone(&shared_child)], + None, + )) as _; + let batch = + RecordBatch::try_from_iter(vec![("first", first), ("second", second)]) + .unwrap(); + + let mut counter = RecordBatchMemoryCounter::new(); + counter.count_batch_with_array_overhead(&batch); + + assert_eq!( + counter.memory_usage(), + batch.get_array_memory_size() - shared_child.get_array_memory_size() + ); + } + + fn assert_recursive_shared_child_memory( + name: &str, + first: ArrayRef, + second: ArrayRef, + shared_memory: usize, + ) { + let first_memory = first.get_array_memory_size(); + let second_memory = second.get_array_memory_size(); + let first_batch = RecordBatch::try_from_iter(vec![(name, first)]).unwrap(); + let second_batch = RecordBatch::try_from_iter(vec![(name, second)]).unwrap(); + let mut counter = RecordBatchMemoryCounter::new(); + + assert_eq!( + counter.count_batch_with_array_overhead(&first_batch), + first_memory, + "{name}: first batch" + ); + assert_eq!( + counter.count_batch_with_array_overhead(&second_batch), + second_memory - shared_memory, + "{name}: shared child" + ); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_shared_list_child() { + let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let list_field = Arc::new(Field::new_list_field(DataType::Int32, false)); + + let first = Arc::new(ListArray::new( + Arc::clone(&list_field), + OffsetBuffer::new(vec![0, 3].into()), + Arc::clone(&shared_child), + None, + )); + let second = Arc::new(ListArray::new( + list_field, + OffsetBuffer::new(vec![0, 3].into()), + Arc::clone(&shared_child), + None, + )); + + assert_recursive_shared_child_memory( + "list", + first, + second, + shared_child.get_array_memory_size(), + ); + } + + fn map_with_shared_children( + fields: &Fields, + shared_key: &ArrayRef, + shared_value: &ArrayRef, + ) -> ArrayRef { + Arc::new( + MapArray::try_new( + Arc::new(Field::new( + "entries", + DataType::Struct(fields.clone()), + false, + )), + OffsetBuffer::new(vec![0, 3].into()), + StructArray::new( + fields.clone(), + vec![Arc::clone(shared_key), Arc::clone(shared_value)], + None, + ), + None, + false, + ) + .unwrap(), + ) + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_shared_map_children() { + let shared_key: ArrayRef = Arc::new(Int32Array::from(vec![4, 5, 6])); + let shared_value: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let fields = Fields::from(vec![ + Arc::new(Field::new("key", DataType::Int32, false)), + Arc::new(Field::new("value", DataType::Int32, false)), + ]); + + let first = map_with_shared_children(&fields, &shared_key, &shared_value); + let second = map_with_shared_children(&fields, &shared_key, &shared_value); + + assert_recursive_shared_child_memory( + "map", + first, + second, + shared_key.get_array_memory_size() + shared_value.get_array_memory_size(), + ); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_union_slice_child_overhead() { + let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let fields: UnionFields = + std::iter::once((0, Arc::new(Field::new("value", DataType::Int32, false)))) + .collect(); + let first = Arc::new( + UnionArray::try_new( + fields, + vec![0, 0, 0].into(), + Some(vec![0, 1, 2].into()), + vec![Arc::clone(&shared_child)], + ) + .unwrap(), + ) as ArrayRef; + let second = Arc::new(first.as_union().slice(1, 2)) as ArrayRef; + let first_batch = + RecordBatch::try_from_iter(vec![("union", Arc::clone(&first))]).unwrap(); + let second_batch = + RecordBatch::try_from_iter(vec![("union", Arc::clone(&second))]).unwrap(); + let child_overhead = + shared_child.get_array_memory_size() - shared_child.get_buffer_memory_size(); + let second_parent_overhead = second.get_array_memory_size() + - second.get_buffer_memory_size() + - child_overhead; + let mut counter = RecordBatchMemoryCounter::new(); + + assert_eq!( + counter.count_batch_with_array_overhead(&first_batch), + first.get_array_memory_size() + ); + assert_eq!( + counter.count_batch_with_array_overhead(&second_batch), + second_parent_overhead + ); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_run_end_slice_child_overhead() { + let values: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let first = Arc::new( + RunArray::::try_new( + &Int32Array::from(vec![1, 2, 3]), + values.as_ref(), + ) + .unwrap(), + ) as ArrayRef; + let second = Arc::new( + first + .as_any() + .downcast_ref::>() + .unwrap() + .slice(1, 2), + ) as ArrayRef; + let first_batch = + RecordBatch::try_from_iter(vec![("run", Arc::clone(&first))]).unwrap(); + let second_batch = + RecordBatch::try_from_iter(vec![("run", Arc::clone(&second))]).unwrap(); + let child_overhead = + values.get_array_memory_size() - values.get_buffer_memory_size(); + let second_parent_overhead = second.get_array_memory_size() + - second.get_buffer_memory_size() + - child_overhead; + let mut counter = RecordBatchMemoryCounter::new(); + + assert_eq!( + counter.count_batch_with_array_overhead(&first_batch), + first.get_array_memory_size() + ); + assert_eq!( + counter.count_batch_with_array_overhead(&second_batch), + second_parent_overhead + ); + } + + #[test] + fn test_record_batch_memory_counter_deduplicates_shared_dictionary_child() { + let shared_child: ArrayRef = Arc::new(Int32Array::from(vec![1, 2, 3])); + let make_dictionary = || { + Arc::new( + DictionaryArray::::try_new( + Int32Array::from(vec![0, 1, 2]), + Arc::clone(&shared_child), + ) + .unwrap(), + ) as ArrayRef + }; + + assert_recursive_shared_child_memory( + "dictionary", + make_dictionary(), + make_dictionary(), + shared_child.get_array_memory_size(), + ); + } + + #[test] + fn test_record_batch_memory_counter_buffer_shared_across_batches() { + let schema = Arc::new(Schema::new(vec![Field::new( + "ints", + DataType::Int32, + false, + )])); + + let int_array = Int32Array::from(vec![1, 2, 3, 4, 5, 6]); + let batch = RecordBatch::try_new(schema, vec![Arc::new(int_array)]).unwrap(); + let slices = [batch.slice(0, 2), batch.slice(2, 2), batch.slice(4, 2)]; + + // Counting each slice individually counts the shared buffer once per slice + let summed: usize = slices.iter().map(get_record_batch_memory_size).sum(); + assert_eq!(summed, 3 * get_record_batch_memory_size(&batch)); + + // A counter shared across the batches counts it exactly once + let mut counter = RecordBatchMemoryCounter::new(); + let deduped: usize = slices.iter().map(|slice| counter.count_batch(slice)).sum(); + assert_eq!(deduped, get_record_batch_memory_size(&batch)); + assert_eq!(counter.memory_usage(), get_record_batch_memory_size(&batch)); + } + + #[test] + fn test_record_batch_memory_counter_promotes_buffer_set() { + let fields = (0..=INLINE_BUFFER_IDS) + .map(|index| Field::new(format!("col_{index}"), DataType::Int32, false)) + .collect::>(); + let columns = (0..=INLINE_BUFFER_IDS) + .map(|value| Arc::new(Int32Array::from(vec![value as i32])) as _) + .collect::>(); + let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap(); + + let mut counter = RecordBatchMemoryCounter::new(); + assert_eq!( + counter.count_batch(&batch), + (INLINE_BUFFER_IDS + 1) * size_of::() + ); + assert!(counter.counted_buffers.overflow.is_some()); + assert_eq!(counter.count_batch(&batch), 0); + } + + #[test] + fn test_array_memory_size_matches_array_data_layouts() { + let list_field = Arc::new(Field::new_list_field(DataType::Int32, true)); + let struct_fields = vec![Field::new("value", DataType::Int32, true)].into(); + let union_fields = UnionFields::try_new( + vec![0], + vec![Field::new("value", DataType::Int32, true)], + ) + .unwrap(); + let map_entries = Arc::new(Field::new( + "entries", + DataType::Struct( + vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", DataType::Int32, true), + ] + .into(), + ), + false, + )); + let data_types = vec![ + DataType::Boolean, + DataType::Int32, + DataType::Binary, + DataType::LargeBinary, + DataType::FixedSizeBinary(4), + DataType::BinaryView, + DataType::Utf8, + DataType::LargeUtf8, + DataType::Utf8View, + DataType::List(Arc::clone(&list_field)), + DataType::LargeList(Arc::clone(&list_field)), + DataType::ListView(Arc::clone(&list_field)), + DataType::LargeListView(Arc::clone(&list_field)), + DataType::FixedSizeList(Arc::clone(&list_field), 2), + DataType::Struct(struct_fields), + DataType::Union(union_fields, UnionMode::Dense), + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + DataType::Map(map_entries, false), + ]; + + for data_type in data_types { + let array = new_null_array(&data_type, 3); + assert_array_memory_size_matches(array.as_ref()); + } + + // Exercise the view-specific buffers with concrete, non-empty values. + let view_arrays = [ + Arc::new(BinaryViewArray::from_iter_values([ + b"short".as_slice(), + b"a payload longer than twelve bytes".as_slice(), + ])) as ArrayRef, + Arc::new(StringViewArray::from_iter_values([ + "short", + "a payload longer than twelve bytes", + ])) as ArrayRef, + Arc::new(ListViewArray::from_iter_primitive::([ + Some(vec![Some(1), Some(2)]), + None, + Some(vec![Some(3)]), + ])) as ArrayRef, + Arc::new(LargeListViewArray::from_iter_primitive::( + [Some(vec![Some(1), Some(2)]), None, Some(vec![Some(3)])], + )) as ArrayRef, + ]; + + for array in view_arrays { + assert_array_memory_size_matches(array.as_ref()); + } + + let run_values = StringArray::from(vec!["alpha", "beta"]); + let run_arrays = [ + Arc::new( + RunArray::::try_new( + &Int16Array::from(vec![2_i16, 5]), + &run_values, + ) + .unwrap(), + ) as ArrayRef, + Arc::new( + RunArray::::try_new( + &Int32Array::from(vec![2_i32, 5]), + &run_values, + ) + .unwrap(), + ) as ArrayRef, + Arc::new( + RunArray::::try_new( + &Int64Array::from(vec![2_i64, 5]), + &run_values, + ) + .unwrap(), + ) as ArrayRef, + ]; + + for array in run_arrays { + assert_array_memory_size_matches(array.as_ref()); + } + } + #[test] fn test_get_record_batch_memory_size_nested_array() { let schema = Arc::new(Schema::new(vec![ diff --git a/datafusion/core/src/physical_planner.rs b/datafusion/core/src/physical_planner.rs index cc7d534776d7e..b006d8ae39a9c 100644 --- a/datafusion/core/src/physical_planner.rs +++ b/datafusion/core/src/physical_planner.rs @@ -802,10 +802,17 @@ impl DefaultPhysicalPlanner { )); } } - return internal_err!( + debug!( "Physical input schema should be the same as the one converted from logical input schema. Differences: {}", differences.iter().map(|s| format!("\n\t- {s}")).join("") ); + + //influx: temporarily remove error and only log so that we can find a + //reproducer in production + // return internal_err!("Physical input schema should be the same as the one converted from logical input schema. Differences: {}", differences + // .iter() + // .map(|s| format!("\n\t- {s}")) + // .join("")); } let groups = self.create_grouping_physical_expr( @@ -4207,6 +4214,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_metadata() { let logical_schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); @@ -4227,6 +4236,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_field_count() { let logical_schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); @@ -4247,6 +4258,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_field_name() { let logical_schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); @@ -4268,6 +4281,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_field_type() { let logical_schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); @@ -4286,6 +4301,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_field_nullability() { let logical_schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); @@ -4304,6 +4321,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_field_metadata() { let logical_schema = Arc::new(Schema::new(vec![Field::new("c1", DataType::Int32, false)])); @@ -4324,6 +4343,8 @@ digraph { } #[tokio::test] + // Ignored due to disabling the physical schema check skip. + #[ignore] async fn test_aggregate_schema_mismatch_multiple() { let logical_schema = Arc::new(Schema::new(vec![ Field::new("c1", DataType::Int32, false), diff --git a/datafusion/datasource-avro/src/avro_to_arrow/schema.rs b/datafusion/datasource-avro/src/avro_to_arrow/schema.rs index 0e8f2a4d56088..053be3c9aff94 100644 --- a/datafusion/datasource-avro/src/avro_to_arrow/schema.rs +++ b/datafusion/datasource-avro/src/avro_to_arrow/schema.rs @@ -117,8 +117,8 @@ fn schema_to_field_with_props( .iter() .map(|s| schema_to_field_with_props(s, None, has_nullable, None)) .collect::>>()?; - let type_ids = 0_i8..fields.len() as i8; - DataType::Union(UnionFields::new(type_ids, fields), UnionMode::Dense) + // Assign type_ids based on the order in which they appear + DataType::Union(UnionFields::from_fields(fields), UnionMode::Dense) } } AvroSchema::Record(RecordSchema { fields, .. }) => { diff --git a/datafusion/expr/src/type_coercion/functions.rs b/datafusion/expr/src/type_coercion/functions.rs index e1f2a19672825..4f43bec21329d 100644 --- a/datafusion/expr/src/type_coercion/functions.rs +++ b/datafusion/expr/src/type_coercion/functions.rs @@ -16,20 +16,22 @@ // under the License. use super::binary::binary_numeric_coercion; -use crate::{AggregateUDF, ScalarUDF, Signature, TypeSignature, WindowUDF}; +use crate::{ + AggregateUDF, ScalarUDF, Signature, TypeSignature, TypeSignatureClass, WindowUDF, +}; use arrow::datatypes::{Field, FieldRef}; use arrow::{ compute::can_cast_types, datatypes::{DataType, TimeUnit}, }; -use datafusion_common::types::LogicalType; +use datafusion_common::types::{LogicalType, NativeType, logical_string}; use datafusion_common::utils::{ ListCoercion, base_type, coerced_fixed_size_list_to_list, }; use datafusion_common::{ - Result, exec_err, internal_err, plan_err, types::NativeType, utils::list_ndims, + DataFusionError, Result, internal_err, plan_err, utils::list_ndims, }; -use datafusion_expr_common::signature::ArrayFunctionArgument; +use datafusion_expr_common::signature::{Arity, ArrayFunctionArgument, Coercion}; use datafusion_expr_common::type_coercion::binary::type_union_resolution; use datafusion_expr_common::{ signature::{ArrayFunctionSignature, FIXED_SIZE_LIST_WILDCARD, TIMEZONE_WILDCARD}, @@ -37,6 +39,7 @@ use datafusion_expr_common::{ type_coercion::binary::string_coercion, }; use itertools::Itertools as _; +use std::collections::BTreeSet; use std::sync::Arc; /// Extension trait to unify common functionality between [`ScalarUDF`], [`AggregateUDF`] @@ -50,6 +53,15 @@ pub trait UDFCoercionExt { /// Should delegate to [`ScalarUDF::coerce_types`], [`AggregateUDF::coerce_types`] /// or [`WindowUDF::coerce_types`]. fn coerce_types(&self, arg_types: &[DataType]) -> Result>; + + /// Return a function-specific signature failure when generic `OneOf` + /// diagnostics are not descriptive enough. + fn diagnose_failed_signature( + &self, + _arg_types: &[DataType], + ) -> Option { + None + } } impl UDFCoercionExt for ScalarUDF { @@ -78,6 +90,13 @@ impl UDFCoercionExt for AggregateUDF { fn coerce_types(&self, arg_types: &[DataType]) -> Result> { self.coerce_types(arg_types) } + + fn diagnose_failed_signature( + &self, + arg_types: &[DataType], + ) -> Option { + self.diagnose_failed_signature(arg_types) + } } impl UDFCoercionExt for WindowUDF { @@ -307,51 +326,305 @@ fn try_coerce_types( ) } -fn get_valid_types_with_udf( +#[derive(Debug)] +struct PlanFailure { + err: DataFusionError, + matches_arity: bool, + has_hint: bool, + detail: Option, +} + +#[derive(Debug)] +enum SignatureFailure { + Plan(PlanFailure), + NonPlan(DataFusionError), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TypeMismatchDetail { + arg_index: usize, + required_type: String, + received_type: String, + data_type: String, +} + +fn signature_matches_arity(signature: &TypeSignature, actual: usize) -> bool { + match signature.arity() { + Arity::Fixed(expected) => expected == actual, + Arity::Variable => true, + } +} + +fn collect_fixed_arities(signature: &TypeSignature, arities: &mut BTreeSet) { + match signature { + TypeSignature::OneOf(signatures) => { + for signature in signatures { + collect_fixed_arities(signature, arities); + } + } + _ => { + if let Arity::Fixed(expected) = signature.arity() { + arities.insert(expected); + } + } + } +} + +fn build_one_of_arity_error( + function_name: &str, + actual: usize, + expected_arities: &[usize], +) -> Option { + if expected_arities.is_empty() { + return None; + } + + let expected = expected_arities.to_vec(); + let message = if expected.len() == 1 { + format!( + "Function '{function_name}' expects {} arguments but received {actual}", + expected[0] + ) + } else if expected.windows(2).all(|window| window[1] == window[0] + 1) { + format!( + "Function '{function_name}' expects {} to {} arguments but received {actual}", + expected.first().unwrap(), + expected.last().unwrap() + ) + } else { + format!( + "Function '{function_name}' expects one of {} arguments but received {actual}", + expected.iter().join(", ") + ) + }; + + Some(DataFusionError::Plan(message)) +} + +fn combine_type_mismatch_failures( + function_name: &str, + failures: &[PlanFailure], +) -> Option { + let details = failures + .iter() + .map(|failure| failure.detail.as_ref()) + .collect::>>()?; + + let first = details.first()?; + if !details.iter().all(|detail| { + detail.arg_index == first.arg_index + && detail.received_type == first.received_type + && detail.data_type == first.data_type + }) { + return None; + } + + let required_types = details + .iter() + .map(|detail| detail.required_type.as_str()) + .collect::>() + .into_iter() + .collect::>(); + + if required_types.is_empty() { + return None; + } + + let required = if required_types.len() == 1 { + required_types[0].to_string() + } else { + format!("one of {}", required_types.join(", ")) + }; + + Some(DataFusionError::Plan(format!( + "Function '{function_name}' requires {required}, but received {} (DataType: {}).", + first.received_type, first.data_type + ))) +} + +fn select_best_plan_failure( + function_name: &str, + failures: Vec, +) -> Option { + let matching_failures = failures + .into_iter() + .filter(|failure| failure.matches_arity) + .collect::>(); + + if matching_failures.is_empty() { + return None; + } + + if matching_failures.len() == 1 { + return Some(matching_failures.into_iter().next().unwrap().err); + } + + let hinted = matching_failures + .iter() + .enumerate() + .filter_map(|(idx, failure)| failure.has_hint.then_some(idx)) + .collect::>(); + if hinted.len() == 1 { + return Some(matching_failures.into_iter().nth(hinted[0]).unwrap().err); + } + + let messages = matching_failures + .iter() + .map(|failure| failure.err.strip_backtrace().to_string()) + .collect::>(); + if messages.windows(2).all(|pair| pair[0] == pair[1]) { + return Some(matching_failures.into_iter().next().unwrap().err); + } + + combine_type_mismatch_failures(function_name, &matching_failures) +} + +fn coercible_mismatch_detail( + coercions: &[Coercion], + current_types: &[DataType], +) -> Option { + if coercions.len() != current_types.len() { + return None; + } + + for (arg_index, (param, current_type)) in + coercions.iter().zip(current_types.iter()).enumerate() + { + let current_native_type = NativeType::from(current_type); + let desired_type = param.desired_type(); + let desired_matches = desired_type.matches_native_type(¤t_native_type); + let allowed_matches = param + .allowed_source_types() + .iter() + .any(|source_type| source_type.matches_native_type(¤t_native_type)); + + if !desired_matches && !allowed_matches { + return Some(TypeMismatchDetail { + arg_index, + required_type: desired_type.to_string(), + received_type: current_native_type.to_string(), + data_type: current_type.to_string(), + }); + } + } + + None +} + +fn to_signature_failure( signature: &TypeSignature, current_types: &[DataType], + err: DataFusionError, +) -> Box { + match err { + DataFusionError::Plan(message) => Box::new(SignatureFailure::Plan(PlanFailure { + matches_arity: signature_matches_arity(signature, current_types.len()), + has_hint: message.contains("Hint:"), + detail: match signature { + TypeSignature::Coercible(coercions) => { + coercible_mismatch_detail(coercions, current_types) + } + _ => None, + }, + err: DataFusionError::Plan(message), + })), + err => Box::new(SignatureFailure::NonPlan(err)), + } +} + +fn evaluate_one_of_with_udf( + signatures: &[TypeSignature], + current_types: &[DataType], func: &F, ) -> Result>> { - let valid_types = match signature { + let actual = current_types.len(); + let mut valid_types = vec![]; + let mut plan_failures = vec![]; + let mut deferred_non_plan = None; + let mut fixed_arities = BTreeSet::new(); + + for signature in signatures { + collect_fixed_arities(signature, &mut fixed_arities); + + match evaluate_signature_with_udf(signature, current_types, func) { + Ok(types) => valid_types.extend(types), + Err(failure) => match *failure { + SignatureFailure::Plan(failure) => plan_failures.push(failure), + SignatureFailure::NonPlan(err) => { + if deferred_non_plan.is_none() { + deferred_non_plan = Some(err); + } + } + }, + } + } + + if !valid_types.is_empty() { + return Ok(valid_types); + } + + if let Some(err) = func.diagnose_failed_signature(current_types) { + return Err(err); + } + + if let Some(err) = select_best_plan_failure(func.name(), plan_failures) { + return Err(err); + } + + let matching_arity_exists = signatures + .iter() + .any(|signature| signature_matches_arity(signature, actual)); + let expected_arities = fixed_arities.iter().copied().collect::>(); + if !matching_arity_exists + && let Some(err) = + build_one_of_arity_error(func.name(), actual, &expected_arities) + { + return Err(err); + } + + if let Some(err) = deferred_non_plan { + return Err(err); + } + + plan_err!("Function '{}' failed to match any signature", func.name()) +} + +fn evaluate_signature_with_udf( + signature: &TypeSignature, + current_types: &[DataType], + func: &F, +) -> std::result::Result>, Box> { + match signature { TypeSignature::UserDefined => match func.coerce_types(current_types) { - Ok(coerced_types) => vec![coerced_types], - Err(e) => { - return exec_err!( - "Function '{}' user-defined coercion failed with {:?}", + Ok(coerced_types) => Ok(vec![coerced_types]), + Err(e) => Err(Box::new(SignatureFailure::NonPlan( + DataFusionError::Execution(format!( + "Function '{}' user-defined coercion failed with: {}", func.name(), e.strip_backtrace() - ); - } + )), + ))), }, TypeSignature::OneOf(signatures) => { - let mut res = vec![]; - let mut errors = vec![]; - for sig in signatures { - match get_valid_types_with_udf(sig, current_types, func) { - Ok(valid_types) => { - res.extend(valid_types); - } - Err(e) => { - errors.push(e.to_string()); - } - } - } - - // Every signature failed, return the joined error - if res.is_empty() { - return internal_err!( - "Function '{}' failed to match any signature, errors: {}", - func.name(), - errors.join(",") - ); - } else { - res - } + evaluate_one_of_with_udf(signatures, current_types, func) + .map_err(|err| to_signature_failure(signature, current_types, err)) } - _ => get_valid_types(func.name(), signature, current_types)?, - }; + _ => get_valid_types(func.name(), signature, current_types) + .map_err(|err| to_signature_failure(signature, current_types, err)), + } +} - Ok(valid_types) +fn get_valid_types_with_udf( + signature: &TypeSignature, + current_types: &[DataType], + func: &F, +) -> Result>> { + evaluate_signature_with_udf(signature, current_types, func).map_err(|failure| { + match *failure { + SignatureFailure::Plan(failure) => failure.err, + SignatureFailure::NonPlan(err) => err, + } + }) } /// Returns a Vec of all possible valid argument types for the given signature. @@ -635,8 +908,16 @@ fn get_valid_types( default_casted_type.default_cast_for(current_type)?; new_types.push(casted_type); } else { - return internal_err!( - "Expect {} but received NativeType::{}, DataType: {}", + let hint = if matches!(current_native_type, NativeType::Binary) + && param.desired_type() + == &TypeSignatureClass::Native(logical_string()) + { + "\n\nHint: Binary types are not automatically coerced to String. Use CAST(column AS VARCHAR) to convert Binary data to String." + } else { + "" + }; + return plan_err!( + "Function '{function_name}' requires {}, but received {} (DataType: {}).{hint}", param.desired_type(), current_native_type, current_type @@ -928,7 +1209,7 @@ mod tests { use super::*; use arrow::datatypes::Field; - use datafusion_common::{assert_contains, types::logical_binary}; + use datafusion_common::{assert_contains, exec_err, types::logical_binary}; use datafusion_expr_common::signature::{Coercion, TypeSignatureClass}; #[test] @@ -1118,6 +1399,233 @@ mod tests { Ok(()) } + struct MockUdf(Signature); + + impl UDFCoercionExt for MockUdf { + fn name(&self) -> &str { + "test" + } + fn signature(&self) -> &Signature { + &self.0 + } + fn coerce_types(&self, _arg_types: &[DataType]) -> Result> { + unimplemented!() + } + } + + struct AlwaysExecErrUdf(Signature); + + impl UDFCoercionExt for AlwaysExecErrUdf { + fn name(&self) -> &str { + "test" + } + + fn signature(&self) -> &Signature { + &self.0 + } + + fn coerce_types(&self, _arg_types: &[DataType]) -> Result> { + exec_err!("boom") + } + } + + struct DiagnoseErrUdf(Signature); + + impl UDFCoercionExt for DiagnoseErrUdf { + fn name(&self) -> &str { + "test" + } + + fn signature(&self) -> &Signature { + &self.0 + } + + fn coerce_types(&self, _arg_types: &[DataType]) -> Result> { + unimplemented!() + } + + fn diagnose_failed_signature( + &self, + _arg_types: &[DataType], + ) -> Option { + Some(DataFusionError::Plan( + "test semantic signature failure".to_string(), + )) + } + } + + #[test] + fn test_one_of_drops_internal_error_and_combines_type_errors() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Boolean, true))]; + let signature = Signature::one_of( + vec![ + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Decimal, + )]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Duration, + )]), + ], + Volatility::Immutable, + ); + + let err = fields_with_udf(¤t_fields, &MockUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Plan(_))); + let err = err.to_string(); + + assert_eq!( + err, + "Error during planning: Function 'test' requires one of TypeSignatureClass::Decimal, TypeSignatureClass::Duration, but received Boolean (DataType: Boolean)." + ); + assert!(!err.contains("Internal error")); + } + + #[test] + fn test_one_of_restores_arity_error() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Int32, true))]; + let signature = Signature::one_of( + vec![TypeSignature::Any(2), TypeSignature::Any(3)], + Volatility::Immutable, + ); + + let err = fields_with_udf(¤t_fields, &MockUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Plan(_))); + assert_eq!( + err.to_string(), + "Error during planning: Function 'test' expects 2 to 3 arguments but received 1" + ); + } + + #[test] + fn test_one_of_uses_specific_error_for_unique_matching_signature() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Int64, true))]; + let signature = Signature::one_of( + vec![ + TypeSignature::Any(3), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_string()), + )]), + ], + Volatility::Immutable, + ); + + let err = fields_with_udf(¤t_fields, &MockUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Plan(_))); + assert!( + err.to_string().starts_with( + "Error during planning: Function 'test' requires TypeSignatureClass::Native(LogicalType(Native(String), String)), but received Int64 (DataType: Int64)." + ), + "{}", + err + ); + } + + #[test] + fn test_one_of_prefers_hinted_matching_signature() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Binary, true))]; + let signature = Signature::one_of( + vec![ + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Decimal, + )]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Native(logical_string()), + )]), + ], + Volatility::Immutable, + ); + + let err = fields_with_udf(¤t_fields, &MockUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Plan(_))); + let err = err.to_string(); + + assert!(err.contains( + "Function 'test' requires TypeSignatureClass::Native(LogicalType(Native(String), String)), but received Binary" + )); + assert!( + err.contains("Hint: Binary types are not automatically coerced to String.") + ); + } + + #[test] + fn test_one_of_combines_same_arity_type_mismatches() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Utf8, true))]; + let signature = Signature::one_of( + vec![ + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Decimal, + )]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Float, + )]), + ], + Volatility::Immutable, + ); + + let err = fields_with_udf(¤t_fields, &MockUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Plan(_))); + assert_eq!( + err.to_string(), + "Error during planning: Function 'test' requires one of TypeSignatureClass::Decimal, TypeSignatureClass::Float, but received String (DataType: Utf8)." + ); + } + + #[test] + fn test_one_of_propagates_non_plan_errors_when_no_plan_error_exists() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Int32, true))]; + let signature = + Signature::one_of(vec![TypeSignature::UserDefined], Volatility::Immutable); + + let err = + fields_with_udf(¤t_fields, &AlwaysExecErrUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Execution(_))); + assert!(err.to_string().starts_with( + "Execution error: Function 'test' user-defined coercion failed with: Execution error: boom" + )); + } + + #[test] + fn test_one_of_keeps_success_when_later_user_defined_branch_fails() -> Result<()> { + let current_fields = vec![Arc::new(Field::new("t", DataType::Int32, true))]; + let signature = Signature::one_of( + vec![ + TypeSignature::Exact(vec![DataType::Int32]), + TypeSignature::UserDefined, + ], + Volatility::Immutable, + ); + + let fields = fields_with_udf(¤t_fields, &AlwaysExecErrUdf(signature))?; + + assert_eq!(fields.len(), 1); + assert_eq!(fields[0].data_type(), &DataType::Int32); + Ok(()) + } + + #[test] + fn test_one_of_allows_function_specific_diagnostic_override() { + let current_fields = vec![Arc::new(Field::new("t", DataType::Boolean, true))]; + let signature = Signature::one_of( + vec![ + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Decimal, + )]), + TypeSignature::Coercible(vec![Coercion::new_exact( + TypeSignatureClass::Duration, + )]), + ], + Volatility::Immutable, + ); + + let err = + fields_with_udf(¤t_fields, &DiagnoseErrUdf(signature)).unwrap_err(); + assert!(matches!(err, DataFusionError::Plan(_))); + assert_eq!( + err.to_string(), + "Error during planning: test semantic signature failure" + ); + } + #[test] fn test_fixed_list_wildcard_coerce() -> Result<()> { struct MockUdf(Signature); diff --git a/datafusion/expr/src/udaf.rs b/datafusion/expr/src/udaf.rs index a69176e1173a5..d4365e83c472c 100644 --- a/datafusion/expr/src/udaf.rs +++ b/datafusion/expr/src/udaf.rs @@ -26,7 +26,9 @@ use std::vec; use arrow::datatypes::{DataType, Field, FieldRef}; -use datafusion_common::{Result, ScalarValue, Statistics, exec_err, not_impl_err}; +use datafusion_common::{ + DataFusionError, Result, ScalarValue, Statistics, exec_err, not_impl_err, +}; use datafusion_expr_common::dyn_eq::{DynEq, DynHash}; use datafusion_physical_expr_common::physical_expr::PhysicalExpr; @@ -271,6 +273,16 @@ impl AggregateUDF { self.inner.coerce_types(arg_types) } + /// Returns a function-specific signature failure for argument combinations + /// that are better described semantically than by generic signature + /// diagnostics. + pub fn diagnose_failed_signature( + &self, + arg_types: &[DataType], + ) -> Option { + self.inner.diagnose_failed_signature(arg_types) + } + /// See [`AggregateUDFImpl::with_beneficial_ordering`] for more details. pub fn with_beneficial_ordering( self, @@ -711,6 +723,16 @@ pub trait AggregateUDFImpl: Debug + DynEq + DynHash + Send + Sync { not_impl_err!("Function {} does not implement coerce_types", self.name()) } + /// Returns a semantic signature failure for argument combinations that + /// should be surfaced directly instead of going through generic `OneOf` + /// diagnostics. + fn diagnose_failed_signature( + &self, + _arg_types: &[DataType], + ) -> Option { + None + } + /// If this function is max, return true /// If the function is min, return false /// Otherwise return None (the default) @@ -1243,6 +1265,13 @@ impl AggregateUDFImpl for AliasedAggregateUDFImpl { self.inner.coerce_types(arg_types) } + fn diagnose_failed_signature( + &self, + arg_types: &[DataType], + ) -> Option { + self.inner.diagnose_failed_signature(arg_types) + } + fn return_field(&self, arg_fields: &[FieldRef]) -> Result { self.inner.return_field(arg_fields) } diff --git a/datafusion/functions-aggregate/src/average.rs b/datafusion/functions-aggregate/src/average.rs index 46a8dbf9540b6..2de2b7632244b 100644 --- a/datafusion/functions-aggregate/src/average.rs +++ b/datafusion/functions-aggregate/src/average.rs @@ -32,7 +32,9 @@ use arrow::datatypes::{ DurationSecondType, Field, FieldRef, Float64Type, TimeUnit, UInt64Type, i256, }; use datafusion_common::types::{NativeType, logical_float64}; -use datafusion_common::{Result, ScalarValue, exec_err, not_impl_err}; +use datafusion_common::{ + DataFusionError, Result, ScalarValue, exec_err, not_impl_err, plan_datafusion_err, +}; use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion_expr::utils::format_state_name; use datafusion_expr::{ @@ -139,6 +141,18 @@ impl AggregateUDFImpl for Avg { &self.signature } + fn diagnose_failed_signature( + &self, + arg_types: &[DataType], + ) -> Option { + match arg_types { + [DataType::Boolean] => { + Some(plan_datafusion_err!("Avg not supported for Boolean")) + } + _ => None, + } + } + fn return_type(&self, arg_types: &[DataType]) -> Result { match &arg_types[0] { DataType::Decimal32(precision, scale) => { diff --git a/datafusion/functions-aggregate/src/sum.rs b/datafusion/functions-aggregate/src/sum.rs index 198ba54adfa2a..a0a89d7538d27 100644 --- a/datafusion/functions-aggregate/src/sum.rs +++ b/datafusion/functions-aggregate/src/sum.rs @@ -31,7 +31,10 @@ use datafusion_common::types::{ NativeType, logical_float64, logical_int8, logical_int16, logical_int32, logical_int64, logical_uint8, logical_uint16, logical_uint32, logical_uint64, }; -use datafusion_common::{HashMap, Result, ScalarValue, exec_err, not_impl_err}; +use datafusion_common::{ + DataFusionError, HashMap, Result, ScalarValue, exec_err, not_impl_err, + plan_datafusion_err, +}; use datafusion_expr::function::{AccumulatorArgs, StateFieldsArgs}; use datafusion_expr::utils::{AggregateOrderSensitivity, format_state_name}; use datafusion_expr::{ @@ -210,6 +213,18 @@ impl AggregateUDFImpl for Sum { &self.signature } + fn diagnose_failed_signature( + &self, + arg_types: &[DataType], + ) -> Option { + match arg_types { + [DataType::Boolean] => { + Some(plan_datafusion_err!("Sum not supported for Boolean")) + } + _ => None, + } + } + fn return_type(&self, arg_types: &[DataType]) -> Result { match &arg_types[0] { DataType::Int64 => Ok(DataType::Int64), diff --git a/datafusion/functions-window/src/lead_lag.rs b/datafusion/functions-window/src/lead_lag.rs index 7569dac9ac106..5564ac04122e9 100644 --- a/datafusion/functions-window/src/lead_lag.rs +++ b/datafusion/functions-window/src/lead_lag.rs @@ -428,8 +428,12 @@ fn evaluate_all_with_ignore_null( default_value: &ScalarValue, is_lag: bool, ) -> Result { - let valid_indices: Vec = - array.nulls().unwrap().valid_indices().collect::>(); + // Arrays without NULLs do not necessarily have a null bitmap. + let Some(nulls) = array.nulls() else { + return shift_with_default_value(array, offset, default_value); + }; + + let valid_indices: Vec = nulls.valid_indices().collect::>(); let direction = !is_lag; let new_array_results: Result, DataFusionError> = (0..array.len()) .map(|id| { @@ -827,4 +831,25 @@ mod tests { .collect::(), ) } + + #[test] + fn test_ignore_nulls_without_null_bitmap() -> Result<()> { + let input = Int32Array::from(vec![1, 2, 3]); + assert!(input.nulls().is_none()); + let input: ArrayRef = Arc::new(input); + + for (offset, expected) in [ + (1, Int32Array::from(vec![None, Some(1), Some(2)])), + (-1, Int32Array::from(vec![Some(2), Some(3), None])), + ] { + let actual = evaluate_all_with_ignore_null( + &input, + offset, + &ScalarValue::Int32(None), + offset > 0, + )?; + assert_eq!(expected, *as_int32_array(&actual)?); + } + Ok(()) + } } diff --git a/datafusion/functions-window/src/nth_value.rs b/datafusion/functions-window/src/nth_value.rs index be08f25ec404b..6c029b662accf 100644 --- a/datafusion/functions-window/src/nth_value.rs +++ b/datafusion/functions-window/src/nth_value.rs @@ -27,7 +27,7 @@ use datafusion_doc::window_doc_sections::DOC_SECTION_ANALYTICAL; use datafusion_expr::window_state::WindowAggState; use datafusion_expr::{ Documentation, LimitEffect, Literal, PartitionEvaluator, ReversedUDWF, Signature, - TypeSignature, Volatility, WindowUDFImpl, + Volatility, WindowUDFImpl, }; use datafusion_functions_window_common::field; use datafusion_functions_window_common::partition::PartitionEvaluatorArgs; @@ -93,17 +93,14 @@ pub struct NthValue { impl NthValue { /// Create a new `nth_value` function pub fn new(kind: NthValueKind) -> Self { - Self { - signature: Signature::one_of( - vec![ - TypeSignature::Any(0), - TypeSignature::Any(1), - TypeSignature::Any(2), - ], - Volatility::Immutable, - ), - kind, - } + let signature = match kind { + NthValueKind::First | NthValueKind::Last => { + Signature::any(1, Volatility::Immutable) + } + NthValueKind::Nth => Signature::any(2, Volatility::Immutable), + }; + + Self { signature, kind } } pub fn first() -> Self { diff --git a/datafusion/functions/Cargo.toml b/datafusion/functions/Cargo.toml index 2bdc05abe3806..a4f9211fe017c 100644 --- a/datafusion/functions/Cargo.toml +++ b/datafusion/functions/Cargo.toml @@ -40,7 +40,7 @@ workspace = true [features] crypto_expressions = ["md-5", "sha2", "blake2", "blake3"] # enable datetime functions -datetime_expressions = ["chrono-tz"] +datetime_expressions = [] # Enable encoding by default so the doctests work. In general don't automatically enable all packages. default = [ "datetime_expressions", @@ -71,7 +71,6 @@ base64 = { version = "0.22", optional = true } blake2 = { version = "^0.10.2", optional = true } blake3 = { version = "1.8", optional = true } chrono = { workspace = true } -chrono-tz = { version = "0.10.4", optional = true } datafusion-common = { workspace = true } datafusion-doc = { workspace = true } datafusion-execution = { workspace = true } diff --git a/datafusion/functions/benches/to_timestamp.rs b/datafusion/functions/benches/to_timestamp.rs index ed865fa6e8d50..d95a197872eb3 100644 --- a/datafusion/functions/benches/to_timestamp.rs +++ b/datafusion/functions/benches/to_timestamp.rs @@ -114,21 +114,16 @@ fn criterion_benchmark(c: &mut Criterion) { Field::new("f", DataType::Timestamp(TimeUnit::Nanosecond, None), true).into(); let arg_field = Field::new("a", DataType::Utf8, false).into(); let arg_fields = vec![arg_field]; - let mut options = ConfigOptions::default(); - options.execution.time_zone = Some("UTC".into()); - let config_options = Arc::new(options); - - let to_timestamp_udf = to_timestamp(config_options.as_ref()); + let config_options = Arc::new(ConfigOptions::default()); c.bench_function("to_timestamp_no_formats_utf8", |b| { - let to_timestamp_udf = Arc::clone(&to_timestamp_udf); let arr_data = data(); let batch_len = arr_data.len(); let string_array = ColumnarValue::Array(Arc::new(arr_data) as ArrayRef); b.iter(|| { black_box( - to_timestamp_udf + to_timestamp() .invoke_with_args(ScalarFunctionArgs { args: vec![string_array.clone()], arg_fields: arg_fields.clone(), @@ -142,14 +137,13 @@ fn criterion_benchmark(c: &mut Criterion) { }); c.bench_function("to_timestamp_no_formats_largeutf8", |b| { - let to_timestamp_udf = Arc::clone(&to_timestamp_udf); let data = cast(&data(), &DataType::LargeUtf8).unwrap(); let batch_len = data.len(); let string_array = ColumnarValue::Array(Arc::new(data) as ArrayRef); b.iter(|| { black_box( - to_timestamp_udf + to_timestamp() .invoke_with_args(ScalarFunctionArgs { args: vec![string_array.clone()], arg_fields: arg_fields.clone(), @@ -163,14 +157,13 @@ fn criterion_benchmark(c: &mut Criterion) { }); c.bench_function("to_timestamp_no_formats_utf8view", |b| { - let to_timestamp_udf = Arc::clone(&to_timestamp_udf); let data = cast(&data(), &DataType::Utf8View).unwrap(); let batch_len = data.len(); let string_array = ColumnarValue::Array(Arc::new(data) as ArrayRef); b.iter(|| { black_box( - to_timestamp_udf + to_timestamp() .invoke_with_args(ScalarFunctionArgs { args: vec![string_array.clone()], arg_fields: arg_fields.clone(), @@ -184,7 +177,6 @@ fn criterion_benchmark(c: &mut Criterion) { }); c.bench_function("to_timestamp_with_formats_utf8", |b| { - let to_timestamp_udf = Arc::clone(&to_timestamp_udf); let (inputs, format1, format2, format3) = data_with_formats(); let batch_len = inputs.len(); @@ -204,7 +196,7 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| { black_box( - to_timestamp_udf + to_timestamp() .invoke_with_args(ScalarFunctionArgs { args: args.clone(), arg_fields: arg_fields.clone(), @@ -218,7 +210,6 @@ fn criterion_benchmark(c: &mut Criterion) { }); c.bench_function("to_timestamp_with_formats_largeutf8", |b| { - let to_timestamp_udf = Arc::clone(&to_timestamp_udf); let (inputs, format1, format2, format3) = data_with_formats(); let batch_len = inputs.len(); @@ -246,7 +237,7 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| { black_box( - to_timestamp_udf + to_timestamp() .invoke_with_args(ScalarFunctionArgs { args: args.clone(), arg_fields: arg_fields.clone(), @@ -260,7 +251,6 @@ fn criterion_benchmark(c: &mut Criterion) { }); c.bench_function("to_timestamp_with_formats_utf8view", |b| { - let to_timestamp_udf = Arc::clone(&to_timestamp_udf); let (inputs, format1, format2, format3) = data_with_formats(); let batch_len = inputs.len(); @@ -289,7 +279,7 @@ fn criterion_benchmark(c: &mut Criterion) { b.iter(|| { black_box( - to_timestamp_udf + to_timestamp() .invoke_with_args(ScalarFunctionArgs { args: args.clone(), arg_fields: arg_fields.clone(), diff --git a/datafusion/functions/src/core/union_extract.rs b/datafusion/functions/src/core/union_extract.rs index 56d4f23cc4e2e..8d915fb2e2c07 100644 --- a/datafusion/functions/src/core/union_extract.rs +++ b/datafusion/functions/src/core/union_extract.rs @@ -189,13 +189,14 @@ mod tests { fn test_scalar_value() -> Result<()> { let fun = UnionExtractFun::new(); - let fields = UnionFields::new( + let fields = UnionFields::try_new( vec![1, 3], vec![ Field::new("str", DataType::Utf8, false), Field::new("int", DataType::Int32, false), ], - ); + ) + .unwrap(); let args = vec![ ColumnarValue::Scalar(ScalarValue::Union( diff --git a/datafusion/functions/src/datetime/common.rs b/datafusion/functions/src/datetime/common.rs index 2db64beafa9b7..5b152081b64ca 100644 --- a/datafusion/functions/src/datetime/common.rs +++ b/datafusion/functions/src/datetime/common.rs @@ -15,57 +15,31 @@ // specific language governing permissions and limitations // under the License. -use std::sync::{Arc, LazyLock}; +use std::sync::Arc; -use arrow::array::timezone::Tz; use arrow::array::{ Array, ArrowPrimitiveType, AsArray, GenericStringArray, PrimitiveArray, StringArrayType, StringViewArray, }; -use arrow::compute::DecimalCast; -use arrow::compute::kernels::cast_utils::string_to_datetime; -use arrow::datatypes::{DataType, TimeUnit}; -use arrow_buffer::ArrowNativeType; +use arrow::compute::kernels::cast_utils::string_to_timestamp_nanos; +use arrow::datatypes::DataType; use chrono::LocalResult::Single; use chrono::format::{Parsed, StrftimeItems, parse}; use chrono::{DateTime, TimeZone, Utc}; + use datafusion_common::cast::as_generic_string_array; use datafusion_common::{ - DataFusionError, Result, ScalarValue, exec_datafusion_err, exec_err, - internal_datafusion_err, unwrap_or_internal_err, + DataFusionError, Result, ScalarType, ScalarValue, exec_datafusion_err, exec_err, + unwrap_or_internal_err, }; use datafusion_expr::ColumnarValue; /// Error message if nanosecond conversion request beyond supported interval const ERR_NANOSECONDS_NOT_SUPPORTED: &str = "The dates that can be represented as nanoseconds have to be between 1677-09-21T00:12:44.0 and 2262-04-11T23:47:16.854775804"; -static UTC: LazyLock = LazyLock::new(|| "UTC".parse().expect("UTC is always valid")); - -/// Converts a string representation of a date‑time into a timestamp expressed in -/// nanoseconds since the Unix epoch. -/// -/// This helper is a thin wrapper around the more general `string_to_datetime` -/// function. It accepts an optional `timezone` which, if `None`, defaults to -/// Coordinated Universal Time (UTC). The string `s` must contain a valid -/// date‑time format that can be parsed by the underlying chrono parser. -/// -/// # Return Value -/// -/// * `Ok(i64)` – The number of nanoseconds since `1970‑01‑01T00:00:00Z`. -/// * `Err(DataFusionError)` – If the string cannot be parsed, the parsed -/// value is out of range (between 1677-09-21T00:12:44.0 and 2262-04-11T23:47:16.854775804) -/// or the parsed value does not correspond to an unambiguous time. -pub(crate) fn string_to_timestamp_nanos_with_timezone( - timezone: &Option, - s: &str, -) -> Result { - let tz = timezone.as_ref().unwrap_or(&UTC); - let dt = string_to_datetime(tz, s)?; - let parsed = dt - .timestamp_nanos_opt() - .ok_or_else(|| exec_datafusion_err!("{ERR_NANOSECONDS_NOT_SUPPORTED}"))?; - - Ok(parsed) +/// Calls string_to_timestamp_nanos and converts the error type +pub(crate) fn string_to_timestamp_nanos_shim(s: &str) -> Result { + string_to_timestamp_nanos(s).map_err(|e| e.into()) } /// Checks that all the arguments from the second are of type [Utf8], [LargeUtf8] or [Utf8View] @@ -95,12 +69,13 @@ pub(crate) fn validate_data_types(args: &[ColumnarValue], name: &str) -> Result< /// Accepts a string and parses it using the [`chrono::format::strftime`] specifiers /// relative to the provided `timezone` /// +/// [IANA timezones] are only supported if the `arrow-array/chrono-tz` feature is enabled +/// +/// * `2023-01-01 040506 America/Los_Angeles` +/// /// If a timestamp is ambiguous, for example as a result of daylight-savings time, an error /// will be returned /// -/// Note that parsing [IANA timezones] is not supported yet in chrono - -/// and this implementation only supports named timezones at the end of the string preceded by a space. -/// /// [`chrono::format::strftime`]: https://docs.rs/chrono/latest/chrono/format/strftime/index.html /// [IANA timezones]: https://www.iana.org/time-zones pub(crate) fn string_to_datetime_formatted( @@ -114,55 +89,11 @@ pub(crate) fn string_to_datetime_formatted( ) }; - let mut datetime_str = s; - let mut format = format; - - // Manually handle the most common case of a named timezone at the end of the timestamp. - // Note that %+ handles 'Z' at the end of the string without a space. This code doesn't - // handle named timezones with no preceding space since that would require writing a - // custom parser (or switching to Jiff) - let tz: Option = if format.trim_end().ends_with(" %Z") { - // grab the string after the last space as the named timezone - if let Some((dt_str, timezone_name)) = datetime_str.trim_end().rsplit_once(' ') { - datetime_str = dt_str; - - // attempt to parse the timezone name - let result: Result = - timezone_name.parse(); - let Ok(tz) = result else { - return Err(err(&result.unwrap_err().to_string())); - }; - - // successfully parsed the timezone name, remove the ' %Z' from the format - format = &format[..format.len() - 3]; - - Some(tz) - } else { - None - } - } else if format.contains("%Z") { - return Err(err( - "'%Z' is only supported at the end of the format string preceded by a space", - )); - } else { - None - }; - let mut parsed = Parsed::new(); - parse(&mut parsed, datetime_str, StrftimeItems::new(format)) - .map_err(|e| err(&e.to_string()))?; + parse(&mut parsed, s, StrftimeItems::new(format)).map_err(|e| err(&e.to_string()))?; - let dt = match tz { - Some(tz) => { - // A timezone was manually parsed out, convert it to a fixed offset - match parsed.to_datetime_with_timezone(&tz) { - Ok(dt) => Ok(dt.fixed_offset()), - Err(e) => Err(e), - } - } - // default to parse the string assuming it has a timezone - None => parsed.to_datetime(), - }; + // attempt to parse the string assuming it has a timezone + let dt = parsed.to_datetime(); if let Err(e) = &dt { // no timezone or other failure, try without a timezone @@ -184,7 +115,7 @@ pub(crate) fn string_to_datetime_formatted( } /// Accepts a string with a `chrono` format and converts it to a -/// nanosecond precision timestamp relative to the provided `timezone`. +/// nanosecond precision timestamp. /// /// See [`chrono::format::strftime`] for the full set of supported formats. /// @@ -210,21 +141,19 @@ pub(crate) fn string_to_datetime_formatted( /// /// [`chrono::format::strftime`]: https://docs.rs/chrono/latest/chrono/format/strftime/index.html #[inline] -pub(crate) fn string_to_timestamp_nanos_formatted_with_timezone( - timezone: &Option, +pub(crate) fn string_to_timestamp_nanos_formatted( s: &str, format: &str, ) -> Result { - let dt = string_to_datetime_formatted(timezone.as_ref().unwrap_or(&UTC), s, format)?; - let parsed = dt + string_to_datetime_formatted(&Utc, s, format)? + .naive_utc() + .and_utc() .timestamp_nanos_opt() - .ok_or_else(|| exec_datafusion_err!("{ERR_NANOSECONDS_NOT_SUPPORTED}"))?; - - Ok(parsed) + .ok_or_else(|| exec_datafusion_err!("{ERR_NANOSECONDS_NOT_SUPPORTED}")) } /// Accepts a string with a `chrono` format and converts it to a -/// millisecond precision timestamp relative to the provided `timezone`. +/// millisecond precision timestamp. /// /// See [`chrono::format::strftime`] for the full set of supported formats. /// @@ -247,14 +176,14 @@ pub(crate) fn string_to_timestamp_millis_formatted(s: &str, format: &str) -> Res .timestamp_millis()) } -pub(crate) fn handle( +pub(crate) fn handle( args: &[ColumnarValue], op: F, name: &str, - dt: &DataType, ) -> Result where O: ArrowPrimitiveType, + S: ScalarType, F: Fn(&str) -> Result, { match &args[0] { @@ -281,13 +210,8 @@ where }, ColumnarValue::Scalar(scalar) => match scalar.try_as_str() { Some(a) => { - let result = a - .as_ref() - .map(|x| op(x)) - .transpose()? - .and_then(|v| v.to_i64()); - let s = scalar_value(dt, result)?; - Ok(ColumnarValue::Scalar(s)) + let result = a.as_ref().map(|x| op(x)).transpose()?; + Ok(ColumnarValue::Scalar(S::scalar(result))) } _ => exec_err!("Unsupported data type {scalar:?} for function {name}"), }, @@ -297,15 +221,15 @@ where // Given a function that maps a `&str`, `&str` to an arrow native type, // returns a `ColumnarValue` where the function is applied to either a `ArrayRef` or `ScalarValue` // depending on the `args`'s variant. -pub(crate) fn handle_multiple( +pub(crate) fn handle_multiple( args: &[ColumnarValue], op: F, op2: M, name: &str, - dt: &DataType, ) -> Result where O: ArrowPrimitiveType, + S: ScalarType, F: Fn(&str, &str) -> Result, M: Fn(O::Native) -> O::Native, { @@ -374,9 +298,9 @@ where if let Some(s) = x { match op(a, s.as_str()) { Ok(r) => { - let result = op2(r).to_i64(); - let s = scalar_value(dt, result)?; - ret = Some(Ok(ColumnarValue::Scalar(s))); + ret = Some(Ok(ColumnarValue::Scalar(S::scalar(Some( + op2(r), + ))))); break; } Err(e) => ret = Some(Err(e)), @@ -530,16 +454,3 @@ where // first map is the iterator, second is for the `Option<_>` array.iter().map(|x| x.map(&op).transpose()).collect() } - -fn scalar_value(dt: &DataType, r: Option) -> Result { - match dt { - DataType::Date32 => Ok(ScalarValue::Date32(r.and_then(|v| v.to_i32()))), - DataType::Timestamp(u, tz) => match u { - TimeUnit::Second => Ok(ScalarValue::TimestampSecond(r, tz.clone())), - TimeUnit::Millisecond => Ok(ScalarValue::TimestampMillisecond(r, tz.clone())), - TimeUnit::Microsecond => Ok(ScalarValue::TimestampMicrosecond(r, tz.clone())), - TimeUnit::Nanosecond => Ok(ScalarValue::TimestampNanosecond(r, tz.clone())), - }, - t => Err(internal_datafusion_err!("Unsupported data type: {t:?}")), - } -} diff --git a/datafusion/functions/src/datetime/mod.rs b/datafusion/functions/src/datetime/mod.rs index 39b9453295df6..9872db3faf556 100644 --- a/datafusion/functions/src/datetime/mod.rs +++ b/datafusion/functions/src/datetime/mod.rs @@ -53,14 +53,11 @@ make_udf_function!(to_date::ToDateFunc, to_date); make_udf_function!(to_local_time::ToLocalTimeFunc, to_local_time); make_udf_function!(to_time::ToTimeFunc, to_time); make_udf_function!(to_unixtime::ToUnixtimeFunc, to_unixtime); -make_udf_function_with_config!(to_timestamp::ToTimestampFunc, to_timestamp); -make_udf_function_with_config!( - to_timestamp::ToTimestampSecondsFunc, - to_timestamp_seconds -); -make_udf_function_with_config!(to_timestamp::ToTimestampMillisFunc, to_timestamp_millis); -make_udf_function_with_config!(to_timestamp::ToTimestampMicrosFunc, to_timestamp_micros); -make_udf_function_with_config!(to_timestamp::ToTimestampNanosFunc, to_timestamp_nanos); +make_udf_function!(to_timestamp::ToTimestampFunc, to_timestamp); +make_udf_function!(to_timestamp::ToTimestampSecondsFunc, to_timestamp_seconds); +make_udf_function!(to_timestamp::ToTimestampMillisFunc, to_timestamp_millis); +make_udf_function!(to_timestamp::ToTimestampMicrosFunc, to_timestamp_micros); +make_udf_function!(to_timestamp::ToTimestampNanosFunc, to_timestamp_nanos); // create UDF with config make_udf_function_with_config!(now::NowFunc, now); @@ -121,24 +118,24 @@ pub mod expr_fn { args, ),( to_timestamp, - "converts a string and optional formats to a `Timestamp(Nanoseconds, TimeZone)`", - @config args, + "converts a string and optional formats to a `Timestamp(Nanoseconds, None)`", + args, ),( to_timestamp_seconds, - "converts a string and optional formats to a `Timestamp(Seconds, TimeZone)`", - @config args, + "converts a string and optional formats to a `Timestamp(Seconds, None)`", + args, ),( to_timestamp_millis, - "converts a string and optional formats to a `Timestamp(Milliseconds, TimeZone)`", - @config args, + "converts a string and optional formats to a `Timestamp(Milliseconds, None)`", + args, ),( to_timestamp_micros, - "converts a string and optional formats to a `Timestamp(Microseconds, TimeZone)`", - @config args, + "converts a string and optional formats to a `Timestamp(Microseconds, None)`", + args, ),( to_timestamp_nanos, - "converts a string and optional formats to a `Timestamp(Nanoseconds, TimeZone)`", - @config args, + "converts a string and optional formats to a `Timestamp(Nanoseconds, None)`", + args, )); /// Returns a string representation of a date, time, timestamp or duration based @@ -274,7 +271,6 @@ pub mod expr_fn { /// Returns all DataFusion functions defined in this package pub fn functions() -> Vec> { use datafusion_common::config::ConfigOptions; - let config = ConfigOptions::default(); vec![ current_date(), current_time(), @@ -284,16 +280,16 @@ pub fn functions() -> Vec> { from_unixtime(), make_date(), make_time(), - now(&config), + now(&ConfigOptions::default()), to_char(), to_date(), to_local_time(), to_time(), to_unixtime(), - to_timestamp(&config), - to_timestamp_seconds(&config), - to_timestamp_millis(&config), - to_timestamp_micros(&config), - to_timestamp_nanos(&config), + to_timestamp(), + to_timestamp_seconds(), + to_timestamp_millis(), + to_timestamp_micros(), + to_timestamp_nanos(), ] } diff --git a/datafusion/functions/src/datetime/to_date.rs b/datafusion/functions/src/datetime/to_date.rs index 60c6fdf2df975..471995089ba3b 100644 --- a/datafusion/functions/src/datetime/to_date.rs +++ b/datafusion/functions/src/datetime/to_date.rs @@ -84,7 +84,7 @@ impl ToDateFunc { fn to_date(&self, args: &[ColumnarValue]) -> Result { match args.len() { - 1 => handle::( + 1 => handle::( args, |s| match Date32Type::parse(s) { Some(v) => Ok(v), @@ -94,9 +94,8 @@ impl ToDateFunc { )), }, "to_date", - &Date32, ), - 2.. => handle_multiple::( + 2.. => handle_multiple::( args, |s, format| { string_to_timestamp_millis_formatted(s, format) @@ -109,7 +108,6 @@ impl ToDateFunc { }, |n| n, "to_date", - &Date32, ), 0 => exec_err!("Unsupported 0 argument count for function to_date"), } diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rs index 58077694b07a0..fdaf8a764ad8a 100644 --- a/datafusion/functions/src/datetime/to_timestamp.rs +++ b/datafusion/functions/src/datetime/to_timestamp.rs @@ -20,37 +20,25 @@ use std::sync::Arc; use crate::datetime::common::*; use arrow::array::Float64Array; -use arrow::array::timezone::Tz; use arrow::datatypes::DataType::*; use arrow::datatypes::TimeUnit::{Microsecond, Millisecond, Nanosecond, Second}; use arrow::datatypes::{ - ArrowTimestampType, DataType, TimestampMicrosecondType, TimestampMillisecondType, - TimestampNanosecondType, TimestampSecondType, + ArrowTimestampType, DataType, TimeUnit, TimestampMicrosecondType, + TimestampMillisecondType, TimestampNanosecondType, TimestampSecondType, }; -use datafusion_common::config::ConfigOptions; use datafusion_common::format::DEFAULT_CAST_OPTIONS; use datafusion_common::{Result, ScalarType, ScalarValue, exec_err}; use datafusion_expr::{ - ColumnarValue, Documentation, ScalarUDF, ScalarUDFImpl, Signature, Volatility, + ColumnarValue, Documentation, ScalarUDFImpl, Signature, Volatility, }; use datafusion_macros::user_doc; #[user_doc( doc_section(label = "Time and Date Functions"), description = r#" -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats] are provided. Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). - -Note: `to_timestamp` returns `Timestamp(ns, TimeZone)` where the time zone is the session time zone. The supported range -for integer input is between`-9223372037` and `9223372036`. Supported range for string input is between -`1677-09-21T00:12:44.0` and `2262-04-11T23:47:16.0`. Please use `to_timestamp_seconds` -for the input outside of supported bounds. - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. +Converts a value to a timestamp (`YYYY-MM-DDT00:00:00Z`). Supports strings, integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats] are provided. Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp. + +Note: `to_timestamp` returns `Timestamp(ns)`. The supported range for integer input is between `-9223372037` and `9223372036`. Supported range for string input is between `1677-09-21T00:12:44.0` and `2262-04-11T23:47:16.0`. Please use `to_timestamp_seconds` for the input outside of supported bounds. "#, syntax_example = "to_timestamp(expression[, ..., format_n])", sql_example = r#"```sql @@ -75,32 +63,17 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ), argument( name = "format_n", - description = r#" -Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. -Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is -only supported at the end of the string preceded by a space. -"# + description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned." ) )] #[derive(Debug, PartialEq, Eq, Hash)] pub struct ToTimestampFunc { signature: Signature, - timezone: Option>, } #[user_doc( doc_section(label = "Time and Date Functions"), - description = r#" -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats] are provided. Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. -"#, + description = "Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided. Integers and unsigned integers are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp.", syntax_example = "to_timestamp_seconds(expression[, ..., format_n])", sql_example = r#"```sql > select to_timestamp_seconds('2023-01-31T09:26:56.123456789-05:00'); @@ -124,32 +97,17 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ), argument( name = "format_n", - description = r#" -Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. -Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is -only supported at the end of the string preceded by a space. -"# + description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned." ) )] #[derive(Debug, PartialEq, Eq, Hash)] pub struct ToTimestampSecondsFunc { signature: Signature, - timezone: Option>, } #[user_doc( doc_section(label = "Time and Date Functions"), - description = r#" -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats] are provided. Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as milliseconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. -"#, + description = "Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. Integers and unsigned integers are interpreted as milliseconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp.", syntax_example = "to_timestamp_millis(expression[, ..., format_n])", sql_example = r#"```sql > select to_timestamp_millis('2023-01-31T09:26:56.123456789-05:00'); @@ -173,32 +131,17 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ), argument( name = "format_n", - description = r#" -Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. -Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is -only supported at the end of the string preceded by a space. -"# + description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned." ) )] #[derive(Debug, PartialEq, Eq, Hash)] pub struct ToTimestampMillisFunc { signature: Signature, - timezone: Option>, } #[user_doc( doc_section(label = "Time and Date Functions"), - description = r#" -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats] are provided. Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as microseconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. -"#, + description = "Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided. Integers and unsigned integers are interpreted as microseconds since the unix epoch (`1970-01-01T00:00:00Z`) Returns the corresponding timestamp.", syntax_example = "to_timestamp_micros(expression[, ..., format_n])", sql_example = r#"```sql > select to_timestamp_micros('2023-01-31T09:26:56.123456789-05:00'); @@ -222,31 +165,17 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ), argument( name = "format_n", - description = r#" -Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. -Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is -only supported at the end of the string preceded by a space. -"# + description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned." ) )] #[derive(Debug, PartialEq, Eq, Hash)] pub struct ToTimestampMicrosFunc { signature: Signature, - timezone: Option>, } #[user_doc( doc_section(label = "Time and Date Functions"), - description = r#" -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats] are provided. Strings that parse without a time zone are treated as if they are in the -session time zone. Integers, unsigned integers, and doubles are interpreted as nanoseconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. -"#, + description = "Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided. Integers and unsigned integers are interpreted as nanoseconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp.", syntax_example = "to_timestamp_nanos(expression[, ..., format_n])", sql_example = r#"```sql > select to_timestamp_nanos('2023-01-31T09:26:56.123456789-05:00'); @@ -270,60 +199,83 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ), argument( name = "format_n", - description = r#" -Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. -Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is -only supported at the end of the string preceded by a space. -"# + description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned." ) )] #[derive(Debug, PartialEq, Eq, Hash)] pub struct ToTimestampNanosFunc { signature: Signature, - timezone: Option>, } -/// Macro to generate boilerplate constructors and config methods for ToTimestamp* functions. -/// Generates: Default impl, deprecated new(), new_with_config(), and extracts timezone from ConfigOptions. -macro_rules! impl_to_timestamp_constructors { - ($func:ty) => { - impl Default for $func { - fn default() -> Self { - Self::new_with_config(&ConfigOptions::default()) - } +impl Default for ToTimestampFunc { + fn default() -> Self { + Self::new() + } +} + +impl ToTimestampFunc { + pub fn new() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), } + } +} - impl $func { - #[deprecated(since = "52.0.0", note = "use `new_with_config` instead")] - /// Deprecated constructor retained for backwards compatibility. - /// - /// Prefer `new_with_config` which allows specifying the - /// timezone via [`ConfigOptions`]. This helper now mirrors the - /// canonical default offset (None) provided by `ConfigOptions::default()`. - pub fn new() -> Self { - Self::new_with_config(&ConfigOptions::default()) - } +impl Default for ToTimestampSecondsFunc { + fn default() -> Self { + Self::new() + } +} - pub fn new_with_config(config: &ConfigOptions) -> Self { - Self { - signature: Signature::variadic_any(Volatility::Immutable), - timezone: config - .execution - .time_zone - .as_ref() - .map(|tz| Arc::from(tz.as_str())), - } - } +impl ToTimestampSecondsFunc { + pub fn new() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), } - }; + } } -impl_to_timestamp_constructors!(ToTimestampFunc); -impl_to_timestamp_constructors!(ToTimestampSecondsFunc); -impl_to_timestamp_constructors!(ToTimestampMillisFunc); -impl_to_timestamp_constructors!(ToTimestampMicrosFunc); -impl_to_timestamp_constructors!(ToTimestampNanosFunc); +impl Default for ToTimestampMillisFunc { + fn default() -> Self { + Self::new() + } +} + +impl ToTimestampMillisFunc { + pub fn new() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + } + } +} + +impl Default for ToTimestampMicrosFunc { + fn default() -> Self { + Self::new() + } +} + +impl ToTimestampMicrosFunc { + pub fn new() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + } + } +} + +impl Default for ToTimestampNanosFunc { + fn default() -> Self { + Self::new() + } +} + +impl ToTimestampNanosFunc { + pub fn new() -> Self { + Self { + signature: Signature::variadic_any(Volatility::Immutable), + } + } +} /// to_timestamp SQL function /// @@ -331,15 +283,6 @@ impl_to_timestamp_constructors!(ToTimestampNanosFunc); /// The supported range for integer input is between `-9223372037` and `9223372036`. /// Supported range for string input is between `1677-09-21T00:12:44.0` and `2262-04-11T23:47:16.0`. /// Please use `to_timestamp_seconds` for the input outside of supported bounds. -/// Macro to generate the with_updated_config method for ToTimestamp* functions. -macro_rules! impl_with_updated_config { - () => { - fn with_updated_config(&self, config: &ConfigOptions) -> Option { - Some(Self::new_with_config(config).into()) - } - }; -} - impl ScalarUDFImpl for ToTimestampFunc { fn as_any(&self) -> &dyn Any { self @@ -353,18 +296,15 @@ impl ScalarUDFImpl for ToTimestampFunc { &self.signature } - fn return_type(&self, _arg_types: &[DataType]) -> Result { - Ok(Timestamp(Nanosecond, self.timezone.clone())) + fn return_type(&self, arg_types: &[DataType]) -> Result { + Ok(return_type_for(&arg_types[0], Nanosecond)) } - impl_with_updated_config!(); - fn invoke_with_args( &self, args: datafusion_expr::ScalarFunctionArgs, ) -> Result { - let datafusion_expr::ScalarFunctionArgs { args, .. } = args; - + let args = args.args; if args.is_empty() { return exec_err!( "to_timestamp function requires 1 or more arguments, got {}", @@ -377,13 +317,13 @@ impl ScalarUDFImpl for ToTimestampFunc { validate_data_types(&args, "to_timestamp")?; } - let tz = self.timezone.clone(); - match args[0].data_type() { Int32 | Int64 => args[0] .cast_to(&Timestamp(Second, None), None)? - .cast_to(&Timestamp(Nanosecond, tz), None), - Null | Timestamp(_, _) => args[0].cast_to(&Timestamp(Nanosecond, tz), None), + .cast_to(&Timestamp(Nanosecond, None), None), + Null | Timestamp(_, None) => { + args[0].cast_to(&Timestamp(Nanosecond, None), None) + } Float64 => { let rescaled = arrow::compute::kernels::numeric::mul( &args[0].to_array(1)?, @@ -393,12 +333,15 @@ impl ScalarUDFImpl for ToTimestampFunc { )?; Ok(ColumnarValue::Array(arrow::compute::cast_with_options( &rescaled, - &Timestamp(Nanosecond, tz), + &Timestamp(Nanosecond, None), &DEFAULT_CAST_OPTIONS, )?)) } + Timestamp(_, Some(tz)) => { + args[0].cast_to(&Timestamp(Nanosecond, Some(tz)), None) + } Utf8View | LargeUtf8 | Utf8 => { - to_timestamp_impl::(&args, "to_timestamp", &tz) + to_timestamp_impl::(&args, "to_timestamp") } Decimal128(_, _) => { match &args[0] { @@ -411,12 +354,14 @@ impl ScalarUDFImpl for ToTimestampFunc { let scale_factor = 10_i128.pow(*scale as u32); let seconds = value / scale_factor; let fraction = value % scale_factor; + let nanos = (fraction * 1_000_000_000) / scale_factor; + let timestamp_nanos = seconds * 1_000_000_000 + nanos; Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( Some(timestamp_nanos as i64), - tz, + None, ))) } _ => exec_err!("Invalid decimal value"), @@ -427,7 +372,6 @@ impl ScalarUDFImpl for ToTimestampFunc { } } } - fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -446,18 +390,15 @@ impl ScalarUDFImpl for ToTimestampSecondsFunc { &self.signature } - fn return_type(&self, _arg_types: &[DataType]) -> Result { - Ok(Timestamp(Second, self.timezone.clone())) + fn return_type(&self, arg_types: &[DataType]) -> Result { + Ok(return_type_for(&arg_types[0], Second)) } - impl_with_updated_config!(); - fn invoke_with_args( &self, args: datafusion_expr::ScalarFunctionArgs, ) -> Result { - let datafusion_expr::ScalarFunctionArgs { args, .. } = args; - + let args = args.args; if args.is_empty() { return exec_err!( "to_timestamp_seconds function requires 1 or more arguments, got {}", @@ -470,17 +411,14 @@ impl ScalarUDFImpl for ToTimestampSecondsFunc { validate_data_types(&args, "to_timestamp")?; } - let tz = self.timezone.clone(); - match args[0].data_type() { - Null | Int32 | Int64 | Timestamp(_, _) | Decimal128(_, _) => { - args[0].cast_to(&Timestamp(Second, tz), None) + Null | Int32 | Int64 | Timestamp(_, None) | Decimal128(_, _) => { + args[0].cast_to(&Timestamp(Second, None), None) + } + Timestamp(_, Some(tz)) => args[0].cast_to(&Timestamp(Second, Some(tz)), None), + Utf8View | LargeUtf8 | Utf8 => { + to_timestamp_impl::(&args, "to_timestamp_seconds") } - Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( - &args, - "to_timestamp_seconds", - &self.timezone, - ), other => { exec_err!( "Unsupported data type {} for function to_timestamp_seconds", @@ -489,7 +427,6 @@ impl ScalarUDFImpl for ToTimestampSecondsFunc { } } } - fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -508,18 +445,15 @@ impl ScalarUDFImpl for ToTimestampMillisFunc { &self.signature } - fn return_type(&self, _arg_types: &[DataType]) -> Result { - Ok(Timestamp(Millisecond, self.timezone.clone())) + fn return_type(&self, arg_types: &[DataType]) -> Result { + Ok(return_type_for(&arg_types[0], Millisecond)) } - impl_with_updated_config!(); - fn invoke_with_args( &self, args: datafusion_expr::ScalarFunctionArgs, ) -> Result { - let datafusion_expr::ScalarFunctionArgs { args, .. } = args; - + let args = args.args; if args.is_empty() { return exec_err!( "to_timestamp_millis function requires 1 or more arguments, got {}", @@ -533,13 +467,15 @@ impl ScalarUDFImpl for ToTimestampMillisFunc { } match args[0].data_type() { - Null | Int32 | Int64 | Timestamp(_, _) => { - args[0].cast_to(&Timestamp(Millisecond, self.timezone.clone()), None) + Null | Int32 | Int64 | Timestamp(_, None) => { + args[0].cast_to(&Timestamp(Millisecond, None), None) + } + Timestamp(_, Some(tz)) => { + args[0].cast_to(&Timestamp(Millisecond, Some(tz)), None) } Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( &args, "to_timestamp_millis", - &self.timezone, ), other => { exec_err!( @@ -549,7 +485,6 @@ impl ScalarUDFImpl for ToTimestampMillisFunc { } } } - fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -568,18 +503,15 @@ impl ScalarUDFImpl for ToTimestampMicrosFunc { &self.signature } - fn return_type(&self, _arg_types: &[DataType]) -> Result { - Ok(Timestamp(Microsecond, self.timezone.clone())) + fn return_type(&self, arg_types: &[DataType]) -> Result { + Ok(return_type_for(&arg_types[0], Microsecond)) } - impl_with_updated_config!(); - fn invoke_with_args( &self, args: datafusion_expr::ScalarFunctionArgs, ) -> Result { - let datafusion_expr::ScalarFunctionArgs { args, .. } = args; - + let args = args.args; if args.is_empty() { return exec_err!( "to_timestamp_micros function requires 1 or more arguments, got {}", @@ -593,13 +525,15 @@ impl ScalarUDFImpl for ToTimestampMicrosFunc { } match args[0].data_type() { - Null | Int32 | Int64 | Timestamp(_, _) => { - args[0].cast_to(&Timestamp(Microsecond, self.timezone.clone()), None) + Null | Int32 | Int64 | Timestamp(_, None) => { + args[0].cast_to(&Timestamp(Microsecond, None), None) + } + Timestamp(_, Some(tz)) => { + args[0].cast_to(&Timestamp(Microsecond, Some(tz)), None) } Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( &args, "to_timestamp_micros", - &self.timezone, ), other => { exec_err!( @@ -609,7 +543,6 @@ impl ScalarUDFImpl for ToTimestampMicrosFunc { } } } - fn documentation(&self) -> Option<&Documentation> { self.doc() } @@ -628,18 +561,15 @@ impl ScalarUDFImpl for ToTimestampNanosFunc { &self.signature } - fn return_type(&self, _arg_types: &[DataType]) -> Result { - Ok(Timestamp(Nanosecond, self.timezone.clone())) + fn return_type(&self, arg_types: &[DataType]) -> Result { + Ok(return_type_for(&arg_types[0], Nanosecond)) } - impl_with_updated_config!(); - fn invoke_with_args( &self, args: datafusion_expr::ScalarFunctionArgs, ) -> Result { - let datafusion_expr::ScalarFunctionArgs { args, .. } = args; - + let args = args.args; if args.is_empty() { return exec_err!( "to_timestamp_nanos function requires 1 or more arguments, got {}", @@ -653,14 +583,15 @@ impl ScalarUDFImpl for ToTimestampNanosFunc { } match args[0].data_type() { - Null | Int32 | Int64 | Timestamp(_, _) => { - args[0].cast_to(&Timestamp(Nanosecond, self.timezone.clone()), None) + Null | Int32 | Int64 | Timestamp(_, None) => { + args[0].cast_to(&Timestamp(Nanosecond, None), None) + } + Timestamp(_, Some(tz)) => { + args[0].cast_to(&Timestamp(Nanosecond, Some(tz)), None) + } + Utf8View | LargeUtf8 | Utf8 => { + to_timestamp_impl::(&args, "to_timestamp_nanos") } - Utf8View | LargeUtf8 | Utf8 => to_timestamp_impl::( - &args, - "to_timestamp_nanos", - &self.timezone, - ), other => { exec_err!( "Unsupported data type {} for function to_timestamp_nanos", @@ -669,16 +600,23 @@ impl ScalarUDFImpl for ToTimestampNanosFunc { } } } - fn documentation(&self) -> Option<&Documentation> { self.doc() } } +/// Returns the return type for the to_timestamp_* function, preserving +/// the timezone if it exists. +fn return_type_for(arg: &DataType, unit: TimeUnit) -> DataType { + match arg { + Timestamp(_, Some(tz)) => Timestamp(unit, Some(Arc::clone(tz))), + _ => Timestamp(unit, None), + } +} + fn to_timestamp_impl>( args: &[ColumnarValue], name: &str, - timezone: &Option>, ) -> Result { let factor = match T::UNIT { Second => 1_000_000_000, @@ -687,26 +625,17 @@ fn to_timestamp_impl>( Nanosecond => 1, }; - let tz = match timezone.clone() { - Some(tz) => Some(tz.parse::()?), - None => None, - }; - match args.len() { - 1 => handle::( + 1 => handle::( args, - move |s| string_to_timestamp_nanos_with_timezone(&tz, s).map(|n| n / factor), + |s| string_to_timestamp_nanos_shim(s).map(|n| n / factor), name, - &Timestamp(T::UNIT, timezone.clone()), ), - n if n >= 2 => handle_multiple::( + n if n >= 2 => handle_multiple::( args, - move |s, format| { - string_to_timestamp_nanos_formatted_with_timezone(&tz, s, format) - }, + string_to_timestamp_nanos_formatted, |n| n / factor, name, - &Timestamp(T::UNIT, timezone.clone()), ), _ => exec_err!("Unsupported 0 argument count for function {name}"), } @@ -723,110 +652,35 @@ mod tests { }; use arrow::array::{ArrayRef, Int64Array, StringBuilder}; use arrow::datatypes::{Field, TimeUnit}; - use chrono::{DateTime, FixedOffset, Utc}; + use chrono::Utc; use datafusion_common::config::ConfigOptions; use datafusion_common::{DataFusionError, ScalarValue, assert_contains}; - use datafusion_expr::{ScalarFunctionArgs, ScalarFunctionImplementation}; + use datafusion_expr::ScalarFunctionImplementation; use super::*; fn to_timestamp(args: &[ColumnarValue]) -> Result { - let timezone: Option> = Some("UTC".into()); - to_timestamp_impl::(args, "to_timestamp", &timezone) + to_timestamp_impl::(args, "to_timestamp") } /// to_timestamp_millis SQL function fn to_timestamp_millis(args: &[ColumnarValue]) -> Result { - let timezone: Option> = Some("UTC".into()); - to_timestamp_impl::( - args, - "to_timestamp_millis", - &timezone, - ) + to_timestamp_impl::(args, "to_timestamp_millis") } /// to_timestamp_micros SQL function fn to_timestamp_micros(args: &[ColumnarValue]) -> Result { - let timezone: Option> = Some("UTC".into()); - to_timestamp_impl::( - args, - "to_timestamp_micros", - &timezone, - ) + to_timestamp_impl::(args, "to_timestamp_micros") } /// to_timestamp_nanos SQL function fn to_timestamp_nanos(args: &[ColumnarValue]) -> Result { - let timezone: Option> = Some("UTC".into()); - to_timestamp_impl::( - args, - "to_timestamp_nanos", - &timezone, - ) + to_timestamp_impl::(args, "to_timestamp_nanos") } /// to_timestamp_seconds SQL function fn to_timestamp_seconds(args: &[ColumnarValue]) -> Result { - let timezone: Option> = Some("UTC".into()); - to_timestamp_impl::(args, "to_timestamp_seconds", &timezone) - } - - fn udfs_and_timeunit() -> Vec<(Box, TimeUnit)> { - let udfs: Vec<(Box, TimeUnit)> = vec![ - ( - Box::new(ToTimestampFunc::new_with_config(&ConfigOptions::default())), - Nanosecond, - ), - ( - Box::new(ToTimestampSecondsFunc::new_with_config( - &ConfigOptions::default(), - )), - Second, - ), - ( - Box::new(ToTimestampMillisFunc::new_with_config( - &ConfigOptions::default(), - )), - Millisecond, - ), - ( - Box::new(ToTimestampMicrosFunc::new_with_config( - &ConfigOptions::default(), - )), - Microsecond, - ), - ( - Box::new(ToTimestampNanosFunc::new_with_config( - &ConfigOptions::default(), - )), - Nanosecond, - ), - ]; - udfs - } - - fn validate_expected_error( - options: &mut ConfigOptions, - args: ScalarFunctionArgs, - expected_err: &str, - ) { - let udfs = udfs_and_timeunit(); - - for (udf, _) in udfs { - match udf - .with_updated_config(options) - .unwrap() - .invoke_with_args(args.clone()) - { - Ok(_) => panic!("Expected error but got success"), - Err(e) => { - assert!( - e.to_string().contains(expected_err), - "Can not find expected error '{expected_err}'. Actual error '{e}'" - ); - } - } - } + to_timestamp_impl::(args, "to_timestamp_seconds") } #[test] @@ -897,368 +751,6 @@ mod tests { Ok(()) } - #[test] - fn to_timestamp_respects_execution_timezone() -> Result<()> { - let udfs = udfs_and_timeunit(); - - let mut options = ConfigOptions::default(); - options.execution.time_zone = Some("-05:00".to_string()); - - let time_zone: Option> = options - .execution - .time_zone - .as_ref() - .map(|tz| Arc::from(tz.as_str())); - - for (udf, time_unit) in udfs { - let field = Field::new("arg", Utf8, true).into(); - - let args = ScalarFunctionArgs { - args: vec![ColumnarValue::Scalar(ScalarValue::Utf8(Some( - "2020-09-08T13:42:29".to_string(), - )))], - arg_fields: vec![field], - number_rows: 1, - return_field: Field::new( - "f", - Timestamp(time_unit, Some("-05:00".into())), - true, - ) - .into(), - config_options: Arc::new(options.clone()), - }; - - let result = udf - .with_updated_config(&options.clone()) - .unwrap() - .invoke_with_args(args)?; - let result = match time_unit { - Second => { - let ColumnarValue::Scalar(ScalarValue::TimestampSecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - Millisecond => { - let ColumnarValue::Scalar(ScalarValue::TimestampMillisecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - Microsecond => { - let ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - Nanosecond => { - let ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - }; - - let scale = match time_unit { - Second => 1_000_000_000, - Millisecond => 1_000_000, - Microsecond => 1_000, - Nanosecond => 1, - }; - - let offset = FixedOffset::west_opt(5 * 3600).unwrap(); - let result = Some( - DateTime::::from_timestamp_nanos(result * scale) - .with_timezone(&offset) - .to_string(), - ); - - assert_eq!(result, Some("2020-09-08 13:42:29 -05:00".to_string())); - } - - Ok(()) - } - - #[test] - fn to_timestamp_formats_respects_execution_timezone() -> Result<()> { - let udfs = udfs_and_timeunit(); - - let mut options = ConfigOptions::default(); - options.execution.time_zone = Some("-05:00".to_string()); - - let time_zone: Option> = options - .execution - .time_zone - .as_ref() - .map(|tz| Arc::from(tz.as_str())); - - let expr_field = Field::new("arg", Utf8, true).into(); - let format_field: Arc = Field::new("fmt", Utf8, true).into(); - - for (udf, time_unit) in udfs { - for (value, format, expected_str) in [ - ( - "2020-09-08 09:42:29 -05:00", - "%Y-%m-%d %H:%M:%S %z", - Some("2020-09-08 09:42:29 -05:00"), - ), - ( - "2020-09-08T13:42:29Z", - "%+", - Some("2020-09-08 08:42:29 -05:00"), - ), - ( - "2020-09-08 13:42:29 UTC", - "%Y-%m-%d %H:%M:%S %Z", - Some("2020-09-08 08:42:29 -05:00"), - ), - ( - "+0000 2024-01-01 12:00:00", - "%z %Y-%m-%d %H:%M:%S", - Some("2024-01-01 07:00:00 -05:00"), - ), - ( - "20200908134229+0100", - "%Y%m%d%H%M%S%z", - Some("2020-09-08 07:42:29 -05:00"), - ), - ( - "2020-09-08+0230 13:42", - "%Y-%m-%d%z %H:%M", - Some("2020-09-08 06:12:00 -05:00"), - ), - ] { - let args = ScalarFunctionArgs { - args: vec![ - ColumnarValue::Scalar(ScalarValue::Utf8(Some(value.to_string()))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some( - format.to_string(), - ))), - ], - arg_fields: vec![Arc::clone(&expr_field), Arc::clone(&format_field)], - number_rows: 1, - return_field: Field::new( - "f", - Timestamp(time_unit, Some("-05:00".into())), - true, - ) - .into(), - config_options: Arc::new(options.clone()), - }; - let result = udf - .with_updated_config(&options.clone()) - .unwrap() - .invoke_with_args(args)?; - let result = match time_unit { - Second => { - let ColumnarValue::Scalar(ScalarValue::TimestampSecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - Millisecond => { - let ColumnarValue::Scalar(ScalarValue::TimestampMillisecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - Microsecond => { - let ColumnarValue::Scalar(ScalarValue::TimestampMicrosecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - Nanosecond => { - let ColumnarValue::Scalar(ScalarValue::TimestampNanosecond( - Some(value), - tz, - )) = result - else { - panic!("expected scalar timestamp"); - }; - - assert_eq!(tz, time_zone); - - value - } - }; - - let scale = match time_unit { - Second => 1_000_000_000, - Millisecond => 1_000_000, - Microsecond => 1_000, - Nanosecond => 1, - }; - let offset = FixedOffset::west_opt(5 * 3600).unwrap(); - let result = Some( - DateTime::::from_timestamp_nanos(result * scale) - .with_timezone(&offset) - .to_string(), - ); - - assert_eq!(result, expected_str.map(|s| s.to_string())); - } - } - - Ok(()) - } - - #[test] - fn to_timestamp_invalid_execution_timezone_behavior() -> Result<()> { - let field: Arc = Field::new("arg", Utf8, true).into(); - let return_field: Arc = - Field::new("f", Timestamp(Nanosecond, None), true).into(); - - let mut options = ConfigOptions::default(); - options.execution.time_zone = Some("Invalid/Timezone".to_string()); - - let args = ScalarFunctionArgs { - args: vec![ColumnarValue::Scalar(ScalarValue::Utf8(Some( - "2020-09-08T13:42:29Z".to_string(), - )))], - arg_fields: vec![Arc::clone(&field)], - number_rows: 1, - return_field: Arc::clone(&return_field), - config_options: Arc::new(options.clone()), - }; - - let expected_err = - "Invalid timezone \"Invalid/Timezone\": failed to parse timezone"; - - validate_expected_error(&mut options, args, expected_err); - - Ok(()) - } - - #[test] - fn to_timestamp_formats_invalid_execution_timezone_behavior() -> Result<()> { - let expr_field: Arc = Field::new("arg", Utf8, true).into(); - let format_field: Arc = Field::new("fmt", Utf8, true).into(); - let return_field: Arc = - Field::new("f", Timestamp(Nanosecond, None), true).into(); - - let mut options = ConfigOptions::default(); - options.execution.time_zone = Some("Invalid/Timezone".to_string()); - - let expected_err = - "Invalid timezone \"Invalid/Timezone\": failed to parse timezone"; - - let make_args = |value: &str, format: &str| ScalarFunctionArgs { - args: vec![ - ColumnarValue::Scalar(ScalarValue::Utf8(Some(value.to_string()))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some(format.to_string()))), - ], - arg_fields: vec![Arc::clone(&expr_field), Arc::clone(&format_field)], - number_rows: 1, - return_field: Arc::clone(&return_field), - config_options: Arc::new(options.clone()), - }; - - for (value, format, _expected_str) in [ - ( - "2020-09-08 09:42:29 -05:00", - "%Y-%m-%d %H:%M:%S %z", - Some("2020-09-08 09:42:29 -05:00"), - ), - ( - "2020-09-08T13:42:29Z", - "%+", - Some("2020-09-08 08:42:29 -05:00"), - ), - ( - "2020-09-08 13:42:29 +0000", - "%Y-%m-%d %H:%M:%S %z", - Some("2020-09-08 08:42:29 -05:00"), - ), - ( - "+0000 2024-01-01 12:00:00", - "%z %Y-%m-%d %H:%M:%S", - Some("2024-01-01 07:00:00 -05:00"), - ), - ( - "20200908134229+0100", - "%Y%m%d%H%M%S%z", - Some("2020-09-08 07:42:29 -05:00"), - ), - ( - "2020-09-08+0230 13:42", - "%Y-%m-%d%z %H:%M", - Some("2020-09-08 06:12:00 -05:00"), - ), - ] { - let args = make_args(value, format); - validate_expected_error(&mut options.clone(), args, expected_err); - } - - let args = ScalarFunctionArgs { - args: vec![ - ColumnarValue::Scalar(ScalarValue::Utf8(Some( - "2020-09-08T13:42:29".to_string(), - ))), - ColumnarValue::Scalar(ScalarValue::Utf8(Some( - "%Y-%m-%dT%H:%M:%S".to_string(), - ))), - ], - arg_fields: vec![Arc::clone(&expr_field), Arc::clone(&format_field)], - number_rows: 1, - return_field: Arc::clone(&return_field), - config_options: Arc::new(options.clone()), - }; - - validate_expected_error(&mut options.clone(), args, expected_err); - - Ok(()) - } - #[test] fn to_timestamp_invalid_input_type() -> Result<()> { // pass the wrong type of input array to to_timestamp and test @@ -1428,11 +920,7 @@ mod tests { } fn parse_timestamp_formatted(s: &str, format: &str) -> Result { - let result = string_to_timestamp_nanos_formatted_with_timezone( - &Some("UTC".parse()?), - s, - format, - ); + let result = string_to_timestamp_nanos_formatted(s, format); if let Err(e) = &result { eprintln!("Error parsing timestamp '{s}' using format '{format}': {e:?}"); } @@ -1500,21 +988,13 @@ mod tests { } #[test] - fn test_no_tz() { + fn test_tz() { let udfs: Vec> = vec![ - Box::new(ToTimestampFunc::new_with_config(&ConfigOptions::default())), - Box::new(ToTimestampSecondsFunc::new_with_config( - &ConfigOptions::default(), - )), - Box::new(ToTimestampMillisFunc::new_with_config( - &ConfigOptions::default(), - )), - Box::new(ToTimestampNanosFunc::new_with_config( - &ConfigOptions::default(), - )), - Box::new(ToTimestampSecondsFunc::new_with_config( - &ConfigOptions::default(), - )), + Box::new(ToTimestampFunc::new()), + Box::new(ToTimestampSecondsFunc::new()), + Box::new(ToTimestampMillisFunc::new()), + Box::new(ToTimestampNanosFunc::new()), + Box::new(ToTimestampSecondsFunc::new()), ]; let mut nanos_builder = TimestampNanosecondArray::builder(2); @@ -1547,8 +1027,8 @@ mod tests { for array in arrays { let rt = udf.return_type(&[array.data_type()]).unwrap(); let arg_field = Field::new("arg", array.data_type().clone(), true).into(); - assert!(matches!(rt, Timestamp(_, None))); - let args = ScalarFunctionArgs { + assert!(matches!(rt, Timestamp(_, Some(_)))); + let args = datafusion_expr::ScalarFunctionArgs { args: vec![array.clone()], arg_fields: vec![arg_field], number_rows: 4, @@ -1563,7 +1043,7 @@ mod tests { _ => panic!("Expected a columnar array"), }; let ty = array.data_type(); - assert!(matches!(ty, Timestamp(_, None))); + assert!(matches!(ty, Timestamp(_, Some(_)))); } } @@ -1598,7 +1078,7 @@ mod tests { let rt = udf.return_type(&[array.data_type()]).unwrap(); assert!(matches!(rt, Timestamp(_, None))); let arg_field = Field::new("arg", array.data_type().clone(), true).into(); - let args = ScalarFunctionArgs { + let args = datafusion_expr::ScalarFunctionArgs { args: vec![array.clone()], arg_fields: vec![arg_field], number_rows: 5, diff --git a/datafusion/functions/src/datetime/to_unixtime.rs b/datafusion/functions/src/datetime/to_unixtime.rs index 5ebcce0a7cfc2..fb6315d8f0e06 100644 --- a/datafusion/functions/src/datetime/to_unixtime.rs +++ b/datafusion/functions/src/datetime/to_unixtime.rs @@ -135,7 +135,7 @@ impl ScalarUDFImpl for ToUnixtimeFunc { .cast_to(&DataType::Timestamp(TimeUnit::Second, tz), None)? .cast_to(&DataType::Int64, None), DataType::Utf8View | DataType::LargeUtf8 | DataType::Utf8 => { - ToTimestampSecondsFunc::new_with_config(args.config_options.as_ref()) + ToTimestampSecondsFunc::new() .invoke_with_args(args)? .cast_to(&DataType::Int64, None) } diff --git a/datafusion/functions/src/macros.rs b/datafusion/functions/src/macros.rs index 4adc331fef669..3d728a178497a 100644 --- a/datafusion/functions/src/macros.rs +++ b/datafusion/functions/src/macros.rs @@ -41,17 +41,6 @@ /// - `Vec` argument (single argument followed by a comma) /// - Variable number of `Expr` arguments (zero or more arguments, must be without commas) /// - Functions that require config (marked with `@config` prefix) -/// -/// Note on configuration construction paths: -/// - The convenience wrappers generated for `@config` functions call the inner -/// constructor with `ConfigOptions::default()`. These wrappers are intended -/// primarily for programmatic `Expr` construction and convenience usage. -/// - When functions are registered in a session, DataFusion will call -/// `with_updated_config()` to create a `ScalarUDF` instance using the session's -/// actual `ConfigOptions`. This also happens when configuration changes at runtime -/// (e.g., via `SET` statements). In short: the macro uses the default config for -/// convenience constructors; the session config is applied when functions are -/// registered or when configuration is updated. #[macro_export] macro_rules! export_functions { ($(($FUNC:ident, $DOC:expr, $($arg:tt)*)),*) => { @@ -70,24 +59,6 @@ macro_rules! export_functions { } }; - // function that requires config and takes a vector argument - (single $FUNC:ident, $DOC:expr, @config $arg:ident,) => { - #[doc = $DOC] - pub fn $FUNC($arg: Vec) -> datafusion_expr::Expr { - use datafusion_common::config::ConfigOptions; - super::$FUNC(&ConfigOptions::default()).call($arg) - } - }; - - // function that requires config and variadic arguments - (single $FUNC:ident, $DOC:expr, @config $($arg:ident)*) => { - #[doc = $DOC] - pub fn $FUNC($($arg: datafusion_expr::Expr),*) -> datafusion_expr::Expr { - use datafusion_common::config::ConfigOptions; - super::$FUNC(&ConfigOptions::default()).call(vec![$($arg),*]) - } - }; - // single vector argument (a single argument followed by a comma) (single $FUNC:ident, $DOC:expr, $arg:ident,) => { #[doc = $DOC] diff --git a/datafusion/physical-expr/src/equivalence/properties/union.rs b/datafusion/physical-expr/src/equivalence/properties/union.rs index d77129472a8ba..702688011668d 100644 --- a/datafusion/physical-expr/src/equivalence/properties/union.rs +++ b/datafusion/physical-expr/src/equivalence/properties/union.rs @@ -67,16 +67,43 @@ fn calculate_union_binary( }) .collect::>(); + // TEMP HACK WORKAROUND + // Revert code from https://github.com/apache/datafusion/pull/12562 + // Context: https://github.com/apache/datafusion/issues/13748 + // Context: https://github.com/influxdata/influxdb_iox/issues/13038 + // Next, calculate valid orderings for the union by searching for prefixes // in both sides. - let mut orderings = UnionEquivalentOrderingBuilder::new(); - orderings.add_satisfied_orderings(&lhs, &rhs)?; - orderings.add_satisfied_orderings(&rhs, &lhs)?; - let orderings = orderings.build(); + let mut orderings = vec![]; + for ordering in lhs.normalized_oeq_class().into_iter() { + let mut ordering: Vec = ordering.into(); + + // Progressively shorten the ordering to search for a satisfied prefix: + while !rhs.ordering_satisfy(ordering.clone())? { + ordering.pop(); + } + // There is a non-trivial satisfied prefix, add it as a valid ordering: + if !ordering.is_empty() { + orderings.push(ordering); + } + } + for ordering in rhs.normalized_oeq_class().into_iter() { + let mut ordering: Vec = ordering.into(); + + // Progressively shorten the ordering to search for a satisfied prefix: + while !lhs.ordering_satisfy(ordering.clone())? { + ordering.pop(); + } + // There is a non-trivial satisfied prefix, add it as a valid ordering: + if !ordering.is_empty() { + orderings.push(ordering); + } + } let mut eq_properties = EquivalenceProperties::new(lhs.schema); eq_properties.add_constants(constants)?; eq_properties.add_orderings(orderings); + Ok(eq_properties) } @@ -122,6 +149,7 @@ struct UnionEquivalentOrderingBuilder { orderings: Vec, } +#[expect(unused)] impl UnionEquivalentOrderingBuilder { fn new() -> Self { Self { orderings: vec![] } @@ -504,6 +532,7 @@ mod tests { } #[test] + #[ignore = "InfluxData patch: chore: skip order calculation / exponential planning"] fn test_union_equivalence_properties_constants_fill_gaps() -> Result<()> { let schema = create_test_schema().unwrap(); UnionEquivalenceTest::new(&schema) @@ -579,6 +608,7 @@ mod tests { } #[test] + #[ignore = "InfluxData patch: chore: skip order calculation / exponential planning"] fn test_union_equivalence_properties_constants_fill_gaps_non_symmetric() -> Result<()> { let schema = create_test_schema().unwrap(); @@ -607,6 +637,7 @@ mod tests { } #[test] + #[ignore = "InfluxData patch: chore: skip order calculation / exponential planning"] fn test_union_equivalence_properties_constants_gap_fill_symmetric() -> Result<()> { let schema = create_test_schema().unwrap(); UnionEquivalenceTest::new(&schema) @@ -658,6 +689,7 @@ mod tests { } #[test] + #[ignore = "InfluxData patch: chore: skip order calculation / exponential planning"] fn test_union_equivalence_properties_constants_middle_desc() -> Result<()> { let schema = create_test_schema().unwrap(); UnionEquivalenceTest::new(&schema) diff --git a/datafusion/physical-optimizer/src/sanity_checker.rs b/datafusion/physical-optimizer/src/sanity_checker.rs index bff33a281556d..8eb00327143dc 100644 --- a/datafusion/physical-optimizer/src/sanity_checker.rs +++ b/datafusion/physical-optimizer/src/sanity_checker.rs @@ -32,6 +32,8 @@ use datafusion_common::tree_node::{Transformed, TransformedResult, TreeNode}; use datafusion_physical_expr::intervals::utils::{check_support, is_datatype_supported}; use datafusion_physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion_physical_plan::joins::SymmetricHashJoinExec; +use datafusion_physical_plan::sorts::sort::SortExec; +use datafusion_physical_plan::union::UnionExec; use datafusion_physical_plan::{ExecutionPlanProperties, get_plan_string}; use crate::PhysicalOptimizerRule; @@ -136,6 +138,14 @@ pub fn check_plan_sanity( plan.required_input_ordering(), plan.required_input_distribution(), ) { + // TEMP HACK WORKAROUND https://github.com/apache/datafusion/issues/11492 + if child.as_any().downcast_ref::().is_some() { + continue; + } + if child.as_any().downcast_ref::().is_some() { + continue; + } + let child_eq_props = child.equivalence_properties(); if let Some(sort_req) = sort_req { let sort_req = sort_req.into_single(); diff --git a/datafusion/physical-plan/Cargo.toml b/datafusion/physical-plan/Cargo.toml index 68e67fa018f08..55184df5dc385 100644 --- a/datafusion/physical-plan/Cargo.toml +++ b/datafusion/physical-plan/Cargo.toml @@ -89,6 +89,10 @@ tokio = { workspace = true, features = [ harness = false name = "partial_ordering" +[[bench]] +harness = false +name = "union_schema" + [[bench]] harness = false name = "spill_io" diff --git a/datafusion/physical-plan/benches/union_schema.rs b/datafusion/physical-plan/benches/union_schema.rs new file mode 100644 index 0000000000000..5dc0e2b39b207 --- /dev/null +++ b/datafusion/physical-plan/benches/union_schema.rs @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Benchmark for `UnionExec` construction cost as a function of child count. +//! +//! Scenarios (run against a flat and a nested/struct schema): +//! - `shared_arc`: every child returns the same `Arc` +//! - `content_equal`: pointer-distinct but identical schemas per child +//! - `last_differs`: identical except the last child's deepest field (worst +//! case for any equality fast path: the scan is wasted, then the full +//! merge runs) + +use std::collections::HashMap; +use std::sync::Arc; + +use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; +use datafusion_physical_plan::ExecutionPlan; +use datafusion_physical_plan::empty::EmptyExec; +use datafusion_physical_plan::union::UnionExec; + +const NUM_FIELDS: usize = 10; +const NESTED_CHILDREN: usize = 5; +const METADATA_PER_FIELD: usize = 2; + +fn metadata(tag: &str, i: usize) -> HashMap { + (0..METADATA_PER_FIELD) + .map(|m| (format!("key_{m}"), format!("value_{tag}_{i}_{m}"))) + .collect() +} + +fn flat_schema() -> Schema { + let fields: Vec = (0..NUM_FIELDS) + .map(|i| { + Field::new(format!("col_{i}"), DataType::Int64, true) + .with_metadata(metadata("f", i)) + }) + .collect(); + Schema::new(fields) +} + +fn nested_schema() -> Schema { + let fields: Vec = (0..NUM_FIELDS) + .map(|i| { + let children: Vec = (0..NESTED_CHILDREN) + .map(|c| { + Field::new(format!("sub_{i}_{c}"), DataType::Int64, true) + .with_metadata(metadata("n", i * NESTED_CHILDREN + c)) + }) + .collect(); + Field::new(format!("col_{i}"), DataType::Struct(children.into()), true) + .with_metadata(metadata("s", i)) + }) + .collect(); + Schema::new(fields) +} + +/// Clone `schema` with extra metadata on its last field. Divergence must stay +/// at the top level: differing nested fields change the field `DataType` +/// itself, which `UnionExec::try_new` rejects ("Schemas have to be aligned"). +fn divergent(schema: &Schema) -> Schema { + let mut fields: Vec = + schema.fields().iter().map(|f| f.as_ref().clone()).collect(); + let last = fields.pop().unwrap(); + let mut md = last.metadata().clone(); + md.insert("divergent".to_string(), "true".to_string()); + fields.push(last.with_metadata(md)); + Schema::new(fields) +} + +fn child(schema: SchemaRef) -> Arc { + Arc::new(EmptyExec::new(schema)) +} + +fn children_shared_arc(schema: &Schema, n: usize) -> Vec> { + let schema: SchemaRef = Arc::new(schema.clone()); + (0..n).map(|_| child(Arc::clone(&schema))).collect() +} + +fn children_content_equal(schema: &Schema, n: usize) -> Vec> { + (0..n).map(|_| child(Arc::new(schema.clone()))).collect() +} + +/// First child keeps `schema`; the rest rename the first field — the common +/// real-world unequal union (`SELECT a .. UNION ALL SELECT b ..`), where any +/// equality check fails immediately. +fn children_names_differ(schema: &Schema, n: usize) -> Vec> { + let mut fields: Vec = + schema.fields().iter().map(|f| f.as_ref().clone()).collect(); + let first = fields.remove(0); + let renamed = first.clone().with_name(format!("renamed_{}", first.name())); + fields.insert(0, renamed); + let alt = Schema::new(fields); + let mut children = vec![child(Arc::new(schema.clone()))]; + children.extend((1..n).map(|_| child(Arc::new(alt.clone())))); + children +} + +fn children_last_differs(schema: &Schema, n: usize) -> Vec> { + let mut children = children_content_equal(schema, n - 1); + children.push(child(Arc::new(divergent(schema)))); + children +} + +fn bench_union_construction(c: &mut Criterion) { + for (suffix, schema, sizes) in [ + ("", flat_schema(), &[100usize, 1000, 4000][..]), + ("_nested", nested_schema(), &[1000, 4000][..]), + ] { + let mut group = c.benchmark_group(format!("union_exec_try_new{suffix}")); + for &n in sizes { + let shared = children_shared_arc(&schema, n); + let content = children_content_equal(&schema, n); + let differs = children_last_differs(&schema, n); + + group.bench_with_input( + BenchmarkId::new("shared_arc", n), + &shared, + |b, ch| b.iter(|| UnionExec::try_new(ch.clone()).unwrap()), + ); + group.bench_with_input( + BenchmarkId::new("content_equal", n), + &content, + |b, ch| b.iter(|| UnionExec::try_new(ch.clone()).unwrap()), + ); + group.bench_with_input( + BenchmarkId::new("last_differs", n), + &differs, + |b, ch| b.iter(|| UnionExec::try_new(ch.clone()).unwrap()), + ); + let names = children_names_differ(&schema, n); + group.bench_with_input( + BenchmarkId::new("names_differ", n), + &names, + |b, ch| b.iter(|| UnionExec::try_new(ch.clone()).unwrap()), + ); + } + group.finish(); + } + + // Reconstruct the union once per child, as optimizer rewrites do. + let mut group = c.benchmark_group("union_exec_rebuild_per_child"); + let schema = flat_schema(); + for n in [100, 1000] { + for (label, children) in [ + ("content_equal", children_content_equal(&schema, n)), + ("last_differs", children_last_differs(&schema, n)), + ] { + let union: Arc = + UnionExec::try_new(children.clone()).unwrap(); + group.bench_with_input(BenchmarkId::new(label, n), &n, |b, _| { + b.iter(|| { + let mut plan = Arc::clone(&union); + for _ in 0..n { + plan = plan.with_new_children(children.clone()).unwrap(); + } + plan + }) + }); + } + } + group.finish(); +} + +criterion_group!(benches, bench_union_construction); +criterion_main!(benches); diff --git a/datafusion/physical-plan/src/filter.rs b/datafusion/physical-plan/src/filter.rs index 9e849759ca653..695b1b6e20374 100644 --- a/datafusion/physical-plan/src/filter.rs +++ b/datafusion/physical-plan/src/filter.rs @@ -1654,13 +1654,14 @@ mod tests { #[test] fn test_equivalence_properties_union_type() -> Result<()> { let union_type = DataType::Union( - UnionFields::new( + UnionFields::try_new( vec![0, 1], vec![ Field::new("f1", DataType::Int32, true), Field::new("f2", DataType::Utf8, true), ], - ), + ) + .unwrap(), UnionMode::Sparse, ); diff --git a/datafusion/physical-plan/src/joins/cross_join.rs b/datafusion/physical-plan/src/joins/cross_join.rs index 4f32b6176ec39..1774a8ea95e53 100644 --- a/datafusion/physical-plan/src/joins/cross_join.rs +++ b/datafusion/physical-plan/src/joins/cross_join.rs @@ -25,6 +25,7 @@ use super::utils::{ OnceAsync, OnceFut, StatefulStreamResult, adjust_right_output_partitioning, reorder_output_after_swap, }; +use crate::coop::cooperative; use crate::execution_plan::{EmissionType, boundedness_from_children}; use crate::metrics::{ExecutionPlanMetricsSet, MetricsSet}; use crate::projection::{ @@ -332,7 +333,7 @@ impl ExecutionPlan for CrossJoinExec { })?; if enforce_batch_size_in_joins { - Ok(Box::pin(CrossJoinStream { + Ok(Box::pin(cooperative(CrossJoinStream { schema: Arc::clone(&self.schema), left_fut, right: stream, @@ -341,9 +342,9 @@ impl ExecutionPlan for CrossJoinExec { state: CrossJoinStreamState::WaitBuildSide, left_data: RecordBatch::new_empty(self.left().schema()), batch_transformer: BatchSplitter::new(batch_size), - })) + }))) } else { - Ok(Box::pin(CrossJoinStream { + Ok(Box::pin(cooperative(CrossJoinStream { schema: Arc::clone(&self.schema), left_fut, right: stream, @@ -352,7 +353,7 @@ impl ExecutionPlan for CrossJoinExec { state: CrossJoinStreamState::WaitBuildSide, left_data: RecordBatch::new_empty(self.left().schema()), batch_transformer: NoopBatchTransformer::new(), - })) + }))) } } diff --git a/datafusion/physical-plan/src/joins/hash_join/exec.rs b/datafusion/physical-plan/src/joins/hash_join/exec.rs index 91fc1ee4436ee..0a45887eff3e4 100644 --- a/datafusion/physical-plan/src/joins/hash_join/exec.rs +++ b/datafusion/physical-plan/src/joins/hash_join/exec.rs @@ -22,6 +22,7 @@ use std::sync::{Arc, OnceLock}; use std::{any::Any, vec}; use crate::ExecutionPlanProperties; +use crate::coop::cooperative; use crate::execution_plan::{EmissionType, boundedness_from_children}; use crate::filter_pushdown::{ ChildPushdownResult, FilterDescription, FilterPushdownPhase, @@ -1061,7 +1062,7 @@ impl ExecutionPlan for HashJoinExec { .map(|(_, right_expr)| Arc::clone(right_expr)) .collect::>(); - Ok(Box::pin(HashJoinStream::new( + Ok(Box::pin(cooperative(HashJoinStream::new( partition, self.schema(), on_right, @@ -1079,7 +1080,7 @@ impl ExecutionPlan for HashJoinExec { self.right.output_ordering().is_some(), build_accumulator, self.mode, - ))) + )))) } fn metrics(&self) -> Option { diff --git a/datafusion/physical-plan/src/joins/nested_loop_join.rs b/datafusion/physical-plan/src/joins/nested_loop_join.rs index 44637321a7e35..fa435c0d815b1 100644 --- a/datafusion/physical-plan/src/joins/nested_loop_join.rs +++ b/datafusion/physical-plan/src/joins/nested_loop_join.rs @@ -29,6 +29,7 @@ use super::utils::{ reorder_output_after_swap, swap_join_projection, }; use crate::common::can_project; +use crate::coop::cooperative; use crate::execution_plan::{EmissionType, boundedness_from_children}; use crate::joins::SharedBitmapBuilder; use crate::joins::utils::{ @@ -529,7 +530,7 @@ impl ExecutionPlan for NestedLoopJoinExec { None => self.column_indices.clone(), }; - Ok(Box::pin(NestedLoopJoinStream::new( + Ok(Box::pin(cooperative(NestedLoopJoinStream::new( self.schema(), self.filter.clone(), self.join_type, @@ -538,7 +539,7 @@ impl ExecutionPlan for NestedLoopJoinExec { column_indices_after_projection, metrics, batch_size, - ))) + )))) } fn metrics(&self) -> Option { diff --git a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs index ae7a5fa764bcc..82929107806de 100644 --- a/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs +++ b/datafusion/physical-plan/src/joins/sort_merge_join/exec.rs @@ -23,6 +23,7 @@ use std::any::Any; use std::fmt::Formatter; use std::sync::Arc; +use crate::coop::cooperative; use crate::execution_plan::{EmissionType, boundedness_from_children}; use crate::expressions::PhysicalSortExpr; use crate::joins::sort_merge_join::metrics::SortMergeJoinMetrics; @@ -497,7 +498,7 @@ impl ExecutionPlan for SortMergeJoinExec { .register(context.memory_pool()); // create join stream - Ok(Box::pin(SortMergeJoinStream::try_new( + Ok(Box::pin(cooperative(SortMergeJoinStream::try_new( context.session_config().spill_compression(), Arc::clone(&self.schema), self.sort_options.clone(), @@ -512,7 +513,7 @@ impl ExecutionPlan for SortMergeJoinExec { SortMergeJoinMetrics::new(partition, &self.metrics), reservation, context.runtime_env(), - )?)) + )?))) } fn metrics(&self) -> Option { diff --git a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs index a75a9893e9f1a..86723b382764d 100644 --- a/datafusion/physical-plan/src/joins/symmetric_hash_join.rs +++ b/datafusion/physical-plan/src/joins/symmetric_hash_join.rs @@ -33,6 +33,7 @@ use std::task::{Context, Poll}; use std::vec; use crate::common::SharedMemoryReservation; +use crate::coop::cooperative; use crate::execution_plan::{boundedness_from_children, emission_type_from_children}; use crate::joins::stream_join_utils::{ PruningJoinHashMap, SortedFilterExpr, StreamJoinMetrics, @@ -534,7 +535,7 @@ impl ExecutionPlan for SymmetricHashJoinExec { } if enforce_batch_size_in_joins { - Ok(Box::pin(SymmetricHashJoinStream { + Ok(Box::pin(cooperative(SymmetricHashJoinStream { left_stream, right_stream, schema: self.schema(), @@ -552,9 +553,9 @@ impl ExecutionPlan for SymmetricHashJoinExec { state: SHJStreamState::PullRight, reservation, batch_transformer: BatchSplitter::new(batch_size), - })) + }))) } else { - Ok(Box::pin(SymmetricHashJoinStream { + Ok(Box::pin(cooperative(SymmetricHashJoinStream { left_stream, right_stream, schema: self.schema(), @@ -572,7 +573,7 @@ impl ExecutionPlan for SymmetricHashJoinExec { state: SHJStreamState::PullRight, reservation, batch_transformer: NoopBatchTransformer::new(), - })) + }))) } } diff --git a/datafusion/physical-plan/src/sorts/sort.rs b/datafusion/physical-plan/src/sorts/sort.rs index 30a0e301823f7..37a383aa3240a 100644 --- a/datafusion/physical-plan/src/sorts/sort.rs +++ b/datafusion/physical-plan/src/sorts/sort.rs @@ -42,7 +42,6 @@ use crate::spill::get_record_batch_memory_size; use crate::spill::in_progress_spill_file::InProgressSpillFile; use crate::spill::spill_manager::{GetSlicedSize, SpillManager}; use crate::stream::RecordBatchStreamAdapter; -use crate::stream::ReservationStream; use crate::topk::TopK; use crate::topk::TopKDynamicFilters; use crate::{ @@ -50,6 +49,7 @@ use crate::{ ExecutionPlanProperties, Partitioning, PlanProperties, SendableRecordBatchStream, Statistics, }; +use datafusion_common::utils::memory::RecordBatchMemoryCounter; use arrow::array::{Array, RecordBatch, RecordBatchOptions, StringViewArray}; use arrow::compute::{concat_batches, lexsort_to_indices, take_arrays}; @@ -732,22 +732,29 @@ impl ExternalSorter { // Using try_resize avoids a release-then-reacquire cycle, which // matters for MemoryPool implementations where grow/shrink have // non-trivial cost (e.g. JNI calls in Comet). - let total_sorted_size: usize = sorted_batches + // Chunks share buffers (e.g. dictionary values after `take`), so + // count each buffer once and release it with the last chunk that + // references it. + let mut counter = RecordBatchMemoryCounter::new(); + let mut release_sizes: Vec = sorted_batches .iter() - .map(get_record_batch_memory_size) - .sum(); + .rev() + .map(|batch| counter.count_batch(batch)) + .collect(); + release_sizes.reverse(); reservation - .try_resize(total_sorted_size) + .try_resize(counter.memory_usage()) .map_err(Self::err_with_oom_context)?; - // Wrap in ReservationStream to hold the reservation - Result::<_, DataFusionError>::Ok(Box::pin(ReservationStream::new( - Arc::clone(&schema), - Box::pin(RecordBatchStreamAdapter::new( - schema, - futures::stream::iter(sorted_batches.into_iter().map(Ok)), - )), - reservation, + let batches = sorted_batches.into_iter().zip(release_sizes).map( + move |(batch, size)| { + reservation.shrink(size); + Ok(batch) + }, + ); + Result::<_, DataFusionError>::Ok(Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::iter(batches), )) as SendableRecordBatchStream) }) .try_flatten() @@ -1459,6 +1466,7 @@ mod tests { use datafusion_common::{DataFusionError, Result, ScalarValue}; use datafusion_execution::RecordBatchStream; use datafusion_execution::config::SessionConfig; + use datafusion_execution::disk_manager::{DiskManagerBuilder, DiskManagerMode}; use datafusion_execution::runtime_env::RuntimeEnvBuilder; use datafusion_physical_expr::EquivalenceProperties; use datafusion_physical_expr::expressions::{Column, Literal}; @@ -1743,6 +1751,120 @@ mod tests { Ok(()) } + /// One batch whose `tag` dictionary is shared by every chunk of its sorted + /// output. + fn shared_dictionary_batch(rows: usize) -> Result { + let schema = Arc::new(Schema::new(vec![ + Field::new( + "tag", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + false, + ), + Field::new("time", DataType::Int64, false), + ])); + let mut tag = StringDictionaryBuilder::::new(); + for i in 0..rows { + tag.append_value(format!("truck_{:05}", (i * 7919) % (rows / 2))); + } + let time = Int64Array::from_iter_values((0..rows).rev().map(|i| i as i64)); + Ok(RecordBatch::try_new( + schema, + vec![Arc::new(tag.finish()), Arc::new(time)], + )?) + } + + /// `SortExec` on `batch` by `(tag, time)` with a `batch_size`-row output, + /// a pool of `pool_batches` times the input and no disk manager. + fn shared_dictionary_sort( + batch: RecordBatch, + batch_size: usize, + pool_batches: usize, + ) -> Result<(Arc, Arc)> { + let schema = batch.schema(); + let runtime = RuntimeEnvBuilder::new() + .with_memory_limit(get_record_batch_memory_size(&batch) * pool_batches, 1.0) + .with_disk_manager_builder( + DiskManagerBuilder::default().with_mode(DiskManagerMode::Disabled), + ) + .build_arc()?; + let task_ctx = Arc::new( + TaskContext::default() + .with_session_config(SessionConfig::new().with_batch_size(batch_size)) + .with_runtime(runtime), + ); + let plan = + TestMemoryExec::try_new_exec(&[vec![batch]], Arc::clone(&schema), None)?; + let sort_exec = Arc::new(SortExec::new( + [ + PhysicalSortExpr::new_default(col("tag", &schema)?), + PhysicalSortExpr::new_default(col("time", &schema)?), + ] + .into(), + plan, + )); + Ok((sort_exec, task_ctx)) + } + + fn batches_memory_size(batches: &[RecordBatch]) -> usize { + let mut counter = RecordBatchMemoryCounter::new(); + batches.iter().map(|batch| counter.count_batch(batch)).sum() + } + + #[tokio::test] + async fn test_sort_single_batch_shared_dictionary_reservation() -> Result<()> { + const ROWS: usize = 100_000; + const BATCH_SIZE: usize = 1_000; + + let (sort_exec, task_ctx) = + shared_dictionary_sort(shared_dictionary_batch(ROWS)?, BATCH_SIZE, 4)?; + + let result = collect(sort_exec, task_ctx).await?; + let rows: usize = result.iter().map(|b| b.num_rows()).sum(); + assert_eq!(rows, ROWS); + assert_eq!(result.len(), ROWS.div_ceil(BATCH_SIZE)); + Ok(()) + } + + #[tokio::test] + async fn test_sort_shared_dictionary_release_per_batch() -> Result<()> { + const ROWS: usize = 10_000; + const BATCH_SIZE: usize = 1_000; + + let batch = shared_dictionary_batch(ROWS)?; + let (sort_exec, task_ctx) = shared_dictionary_sort(batch.clone(), BATCH_SIZE, 4)?; + let expected = sort_batch_chunked(&batch, sort_exec.expr(), BATCH_SIZE)?; + let mut stream = sort_exec.execute(0, Arc::clone(&task_ctx))?; + + // Emitted batches are dropped right away; the pool must still hold + // what the queued batches reference, shared dictionary included. + for emitted in 1..=expected.len() { + drop(stream.next().await.unwrap()?); + assert_eq!( + task_ctx.memory_pool().reserved(), + batches_memory_size(&expected[emitted..]), + "after {emitted} of {} batches", + expected.len() + ); + } + assert!(stream.next().await.is_none()); + assert_eq!(task_ctx.memory_pool().reserved(), 0); + Ok(()) + } + + #[tokio::test] + async fn test_sort_shared_dictionary_release_on_drop() -> Result<()> { + let (sort_exec, task_ctx) = + shared_dictionary_sort(shared_dictionary_batch(10_000)?, 1_000, 4)?; + let mut stream = sort_exec.execute(0, Arc::clone(&task_ctx))?; + + drop(stream.next().await.unwrap()?); + assert!(task_ctx.memory_pool().reserved() > 0); + + drop(stream); + assert_eq!(task_ctx.memory_pool().reserved(), 0); + Ok(()) + } + #[tokio::test] async fn test_sort_spill_utf8_strings() -> Result<()> { let session_config = SessionConfig::new() diff --git a/datafusion/physical-plan/src/stream.rs b/datafusion/physical-plan/src/stream.rs index 80c2233d05db6..e2e11b8fb23ac 100644 --- a/datafusion/physical-plan/src/stream.rs +++ b/datafusion/physical-plan/src/stream.rs @@ -27,13 +27,11 @@ use super::metrics::ExecutionPlanMetricsSet; use super::metrics::{BaselineMetrics, SplitMetrics}; use super::{ExecutionPlan, RecordBatchStream, SendableRecordBatchStream}; use crate::displayable; -use crate::spill::get_record_batch_memory_size; use arrow::{datatypes::SchemaRef, record_batch::RecordBatch}; use datafusion_common::{Result, exec_err}; use datafusion_common_runtime::JoinSet; use datafusion_execution::TaskContext; -use datafusion_execution::memory_pool::MemoryReservation; use futures::ready; use futures::stream::BoxStream; @@ -701,70 +699,6 @@ impl RecordBatchStream for BatchSplitStream { } } -/// A stream that holds a memory reservation for its lifetime, -/// shrinking the reservation as batches are consumed. -/// The original reservation must have its batch sizes calculated using [`get_record_batch_memory_size`] -/// On error, the reservation is *NOT* freed, until the stream is dropped. -pub(crate) struct ReservationStream { - schema: SchemaRef, - inner: SendableRecordBatchStream, - reservation: MemoryReservation, -} - -impl ReservationStream { - pub(crate) fn new( - schema: SchemaRef, - inner: SendableRecordBatchStream, - reservation: MemoryReservation, - ) -> Self { - Self { - schema, - inner, - reservation, - } - } -} - -impl Stream for ReservationStream { - type Item = Result; - - fn poll_next( - mut self: Pin<&mut Self>, - cx: &mut Context<'_>, - ) -> Poll> { - let res = self.inner.poll_next_unpin(cx); - - match res { - Poll::Ready(res) => { - match res { - Some(Ok(batch)) => { - self.reservation - .shrink(get_record_batch_memory_size(&batch)); - Poll::Ready(Some(Ok(batch))) - } - Some(Err(err)) => Poll::Ready(Some(Err(err))), - None => { - // Stream is done so free the reservation completely - self.reservation.free(); - Poll::Ready(None) - } - } - } - Poll::Pending => Poll::Pending, - } - } - - fn size_hint(&self) -> (usize, Option) { - self.inner.size_hint() - } -} - -impl RecordBatchStream for ReservationStream { - fn schema(&self) -> SchemaRef { - Arc::clone(&self.schema) - } -} - #[cfg(test)] mod test { use super::*; @@ -993,123 +927,4 @@ mod test { "Should have received exactly two empty batches" ); } - - #[tokio::test] - async fn test_reservation_stream_shrinks_on_poll() { - use arrow::array::Int32Array; - use datafusion_execution::memory_pool::MemoryConsumer; - use datafusion_execution::runtime_env::RuntimeEnvBuilder; - - let runtime = RuntimeEnvBuilder::new() - .with_memory_limit(10 * 1024 * 1024, 1.0) - .build_arc() - .unwrap(); - - let mut reservation = MemoryConsumer::new("test").register(&runtime.memory_pool); - - let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); - - // Create batches - let batch1 = RecordBatch::try_new( - Arc::clone(&schema), - vec![Arc::new(Int32Array::from(vec![1, 2, 3, 4, 5]))], - ) - .unwrap(); - let batch2 = RecordBatch::try_new( - Arc::clone(&schema), - vec![Arc::new(Int32Array::from(vec![6, 7, 8, 9, 10]))], - ) - .unwrap(); - - let batch1_size = get_record_batch_memory_size(&batch1); - let batch2_size = get_record_batch_memory_size(&batch2); - - // Reserve memory upfront - reservation.try_grow(batch1_size + batch2_size).unwrap(); - let initial_reserved = runtime.memory_pool.reserved(); - assert_eq!(initial_reserved, batch1_size + batch2_size); - - // Create stream with batches - let stream = futures::stream::iter(vec![Ok(batch1), Ok(batch2)]); - let inner = Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&schema), stream)) - as SendableRecordBatchStream; - - let mut res_stream = - ReservationStream::new(Arc::clone(&schema), inner, reservation); - - // Poll first batch - let result1 = res_stream.next().await; - assert!(result1.is_some()); - - // Memory should be reduced by batch1_size - let after_first = runtime.memory_pool.reserved(); - assert_eq!(after_first, batch2_size); - - // Poll second batch - let result2 = res_stream.next().await; - assert!(result2.is_some()); - - // Memory should be reduced by batch2_size - let after_second = runtime.memory_pool.reserved(); - assert_eq!(after_second, 0); - - // Poll None (end of stream) - let result3 = res_stream.next().await; - assert!(result3.is_none()); - - // Memory should still be 0 - assert_eq!(runtime.memory_pool.reserved(), 0); - } - - #[tokio::test] - async fn test_reservation_stream_error_handling() { - use datafusion_execution::memory_pool::MemoryConsumer; - use datafusion_execution::runtime_env::RuntimeEnvBuilder; - - let runtime = RuntimeEnvBuilder::new() - .with_memory_limit(10 * 1024 * 1024, 1.0) - .build_arc() - .unwrap(); - - let mut reservation = MemoryConsumer::new("test").register(&runtime.memory_pool); - - let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, false)])); - - reservation.try_grow(1000).unwrap(); - let initial = runtime.memory_pool.reserved(); - assert_eq!(initial, 1000); - - // Create a stream that errors - let stream = futures::stream::iter(vec![exec_err!("Test error")]); - let inner = Box::pin(RecordBatchStreamAdapter::new(Arc::clone(&schema), stream)) - as SendableRecordBatchStream; - - let mut res_stream = - ReservationStream::new(Arc::clone(&schema), inner, reservation); - - // Get the error - let result = res_stream.next().await; - assert!(result.is_some()); - assert!(result.unwrap().is_err()); - - // Verify reservation is NOT automatically freed on error - // The reservation is only freed when poll_next returns Poll::Ready(None) - // After an error, the stream may continue to hold the reservation - // until it's explicitly dropped or polled to None - let after_error = runtime.memory_pool.reserved(); - assert_eq!( - after_error, 1000, - "Reservation should still be held after error" - ); - - // Drop the stream to free the reservation - drop(res_stream); - - // Now memory should be freed - assert_eq!( - runtime.memory_pool.reserved(), - 0, - "Memory should be freed when stream is dropped" - ); - } } diff --git a/datafusion/physical-plan/src/union.rs b/datafusion/physical-plan/src/union.rs index d27c81b968490..fd269367a9125 100644 --- a/datafusion/physical-plan/src/union.rs +++ b/datafusion/physical-plan/src/union.rs @@ -259,6 +259,26 @@ impl ExecutionPlan for UnionExec { self: Arc, children: Vec>, ) -> Result> { + // Fast path: if the children's schemas are unchanged the union schema + // cannot change, so skip the O(children x fields) `union_schema` + // recomputation. Plan properties can change even when schemas do not, + // so they are always recomputed. + if children.len() == self.inputs.len() + && children.len() >= 2 + && children.iter().zip(self.inputs.iter()).all(|(new, old)| { + let new_schema = new.schema(); + let old_schema = old.schema(); + Arc::ptr_eq(&new_schema, &old_schema) || new_schema == old_schema + }) + { + let schema = self.schema(); + let cache = Self::compute_properties(&children, schema)?; + return Ok(Arc::new(UnionExec { + inputs: children, + metrics: ExecutionPlanMetricsSet::new(), + cache, + })); + } UnionExec::try_new(children) } @@ -594,6 +614,17 @@ fn union_schema(inputs: &[Arc]) -> Result { let first_schema = inputs[0].schema(); + // Merging N identical schemas is the identity operation: if every input + // schema is pointer- or content-equal to the first (`Schema` equality + // covers every property the merge below reads), return it directly + // instead of paying the per-field metadata merge. + if inputs.iter().all(|input| { + let schema = input.schema(); + Arc::ptr_eq(&schema, &first_schema) || schema == first_schema + }) { + return Ok(first_schema); + } + let fields = (0..first_schema.fields().len()) .map(|i| { // We take the name from the left side of the union to match how names are coerced during logical planning, diff --git a/datafusion/proto-common/src/from_proto/mod.rs b/datafusion/proto-common/src/from_proto/mod.rs index e8e71c3884586..3c41b8cad9ed1 100644 --- a/datafusion/proto-common/src/from_proto/mod.rs +++ b/datafusion/proto-common/src/from_proto/mod.rs @@ -304,13 +304,16 @@ impl TryFrom<&protobuf::arrow_type::ArrowTypeEnum> for DataType { }; let union_fields = parse_proto_fields_to_fields(&union.union_types)?; - // Default to index based type ids if not provided - let type_ids: Vec<_> = match union.type_ids.is_empty() { - true => (0..union_fields.len() as i8).collect(), - false => union.type_ids.iter().map(|i| *i as i8).collect(), + // Default to index based type ids if not explicitly provided + let union_fields = if union.type_ids.is_empty() { + UnionFields::from_fields(union_fields) + } else { + let type_ids = union.type_ids.iter().map(|i| *i as i8); + UnionFields::try_new(type_ids, union_fields).map_err(|e| { + DataFusionError::from(e).context("Deserializing Union DataType") + })? }; - - DataType::Union(UnionFields::new(type_ids, union_fields), union_mode) + DataType::Union(union_fields, union_mode) } arrow_type::ArrowTypeEnum::Dictionary(dict) => { let key_datatype = dict.as_ref().key.as_deref().required("key")?; @@ -602,7 +605,9 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue { .collect::>>(); let fields = fields.ok_or_else(|| Error::required("UnionField"))?; let fields = parse_proto_fields_to_fields(&fields)?; - let fields = UnionFields::new(ids, fields); + let union_fields = UnionFields::try_new(ids, fields).map_err(|e| { + DataFusionError::from(e).context("Deserializing Union ScalarValue") + })?; let v_id = val.value_id as i8; let val = match &val.value { None => None, @@ -614,7 +619,7 @@ impl TryFrom<&protobuf::ScalarValue> for ScalarValue { Some((v_id, Box::new(val))) } }; - Self::Union(val, fields, mode) + Self::Union(val, union_fields, mode) } Value::FixedSizeBinaryValue(v) => { Self::FixedSizeBinary(v.length, Some(v.clone().values)) diff --git a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs index bcfda648b53e5..b9af9fc9352b2 100644 --- a/datafusion/proto/tests/cases/roundtrip_logical_plan.rs +++ b/datafusion/proto/tests/cases/roundtrip_logical_plan.rs @@ -1780,19 +1780,20 @@ fn round_trip_datatype() { ), ])), DataType::Union( - UnionFields::new( + UnionFields::try_new( vec![7, 5, 3], vec![ Field::new("nullable", DataType::Boolean, false), Field::new("name", DataType::Utf8, false), Field::new("datatype", DataType::Binary, false), ], - ), + ) + .unwrap(), UnionMode::Sparse, ), DataType::Union( - UnionFields::new( - vec![5, 8, 1], + UnionFields::try_new( + vec![5, 8, 1, 100], vec![ Field::new("nullable", DataType::Boolean, false), Field::new("name", DataType::Utf8, false), @@ -1807,7 +1808,8 @@ fn round_trip_datatype() { true, ), ], - ), + ) + .unwrap(), UnionMode::Dense, ), DataType::Dictionary( diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs index 969d56afdae0a..3ab454d745ba4 100644 --- a/datafusion/sql/tests/sql_integration.rs +++ b/datafusion/sql/tests/sql_integration.rs @@ -4719,7 +4719,7 @@ fn test_error_message_invalid_scalar_function_signature() { fn test_error_message_invalid_aggregate_function_signature() { error_message_test( "select sum()", - "Error during planning: Execution error: Function 'sum' user-defined coercion failed with \"Execution error: sum function requires 1 argument, got 0\"", + "Error during planning: Execution error: Function 'sum' user-defined coercion failed with: Execution error: sum function requires 1 argument, got 0", ); // We keep two different prefixes because they clarify each other. // It might be incorrect, and we should consider keeping only one. @@ -4741,7 +4741,7 @@ fn test_error_message_invalid_window_function_signature() { fn test_error_message_invalid_window_aggregate_function_signature() { error_message_test( "select sum() over()", - "Error during planning: Execution error: Function 'sum' user-defined coercion failed with \"Execution error: sum function requires 1 argument, got 0\"", + "Error during planning: Execution error: Function 'sum' user-defined coercion failed with: Execution error: sum function requires 1 argument, got 0", ); } diff --git a/datafusion/sqllogictest/src/test_context.rs b/datafusion/sqllogictest/src/test_context.rs index 9ec085b41eec0..a69bdac663002 100644 --- a/datafusion/sqllogictest/src/test_context.rs +++ b/datafusion/sqllogictest/src/test_context.rs @@ -436,14 +436,15 @@ fn create_example_udf() -> ScalarUDF { fn register_union_table(ctx: &SessionContext) { let union = UnionArray::try_new( - UnionFields::new( + UnionFields::try_new( // typeids: 3 for int, 1 for string vec![3, 1], vec![ Field::new("int", DataType::Int32, false), Field::new("string", DataType::Utf8, false), ], - ), + ) + .unwrap(), ScalarBuffer::from(vec![3, 1, 3]), None, vec![ diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index 65df1e50b8981..6ca50e563a03c 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -3361,7 +3361,7 @@ select [1, 2, 3] List(Utf8View) # array_concat error -query error DataFusion error: Error during planning: Execution error: Function 'array_concat' user-defined coercion failed with "Error during planning: array_concat does not support type Int64" +query error DataFusion error: Error during planning: Execution error: Function 'array_concat' user-defined coercion failed with: Error during planning: array_concat does not support type Int64 select array_concat(1, 2); # array_concat scalar function #1 @@ -4772,10 +4772,11 @@ select array_union(arrow_cast([], 'LargeList(Int64)'), arrow_cast([], 'LargeList [] # array_union scalar function #7 -query ? -select array_union([[null]], []); ----- -[[]] +# re-enable when https://github.com/apache/arrow-rs/issues/9227 is fixed +# query ? +# select array_union([[null]], []); +# ---- +# [[]] query error DataFusion error: Error during planning: Failed to coerce arguments to satisfy a call to 'array_union' function: select array_union(arrow_cast([[null]], 'LargeList(List(Int64))'), arrow_cast([], 'LargeList(Int64)')); @@ -7240,17 +7241,17 @@ select generate_series(arrow_cast('2021-01-01T00:00:00', 'Timestamp(Nanosecond, [2021-01-01T00:00:00-05:00, 2021-01-01T01:29:54.500-05:00, 2021-01-01T02:59:49-05:00, 2021-01-01T04:29:43.500-05:00, 2021-01-01T05:59:38-05:00] ## mixing types for timestamps is not supported -query error DataFusion error: Error during planning: Internal error: Function 'generate_series' failed to match any signature +query error DataFusion error: Error during planning: Function 'generate_series' failed to match any signature select generate_series(arrow_cast('2021-01-01T00:00:00', 'Timestamp(Nanosecond, Some("-05:00"))'), DATE '2021-01-02', INTERVAL '1' HOUR); ## mixing types not allowed even if an argument is null -query error DataFusion error: Error during planning: Internal error: Function 'generate_series' failed to match any signature +query error DataFusion error: Error during planning: Function 'generate_series' failed to match any signature select generate_series(TIMESTAMP '1992-09-01', DATE '1993-03-01', NULL); -query error DataFusion error: Error during planning: Internal error: Function 'generate_series' failed to match any signature +query error DataFusion error: Error during planning: Function 'generate_series' failed to match any signature select generate_series(1, '2024-01-01', '2025-01-02'); -query error DataFusion error: Error during planning: Internal error: Function 'generate_series' failed to match any signature +query error DataFusion error: Error during planning: Function 'generate_series' failed to match any signature select generate_series('2024-01-01'::timestamp, '2025-01-02', interval '1 day'); ## should return NULL diff --git a/datafusion/sqllogictest/test_files/arrow_typeof.slt b/datafusion/sqllogictest/test_files/arrow_typeof.slt index ee1f204664a14..99fc54e2cc1c3 100644 --- a/datafusion/sqllogictest/test_files/arrow_typeof.slt +++ b/datafusion/sqllogictest/test_files/arrow_typeof.slt @@ -95,7 +95,7 @@ SELECT arrow_cast('1', 'Int16') query error SELECT arrow_cast('1') -query error Expect TypeSignatureClass::Native\(LogicalType\(Native\(String\), String\)\) but received NativeType::Int64, DataType: Int64 +query error Function 'arrow_cast' requires TypeSignatureClass::Native\(LogicalType\(Native\(String\), String\)\), but received Int64 \(DataType: Int64\) SELECT arrow_cast('1', 43) query error DataFusion error: Execution error: arrow_cast requires its second argument to be a non\-empty constant string diff --git a/datafusion/sqllogictest/test_files/case.slt b/datafusion/sqllogictest/test_files/case.slt index 074d216ac7524..481dde5be9f5c 100644 --- a/datafusion/sqllogictest/test_files/case.slt +++ b/datafusion/sqllogictest/test_files/case.slt @@ -384,8 +384,7 @@ SELECT column2, column3, column4 FROM t; {foo: a, xxx: b} {xxx: c, foo: d} {xxx: e} # coerce structs with different field orders, -# (note the *value*s are from column2 but the field name is 'xxx', as the coerced -# type takes the field name from the last argument (column3) +# should keep the same field values query ? SELECT case @@ -394,7 +393,7 @@ SELECT end FROM t; ---- -{xxx: a, foo: b} +{xxx: b, foo: a} # coerce structs with different field orders query ? diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt b/datafusion/sqllogictest/test_files/datetime/timestamps.slt index dbb924ef7aa63..a09d4bae7a0ec 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt @@ -193,8 +193,6 @@ SELECT TIMESTAMPTZ '2000-01-01T01:01:01' ---- 2000-01-01T01:01:01Z -statement ok -RESET datafusion.execution.time_zone ########## ## cast tests @@ -2333,7 +2331,7 @@ CREATE TABLE foo (time TIMESTAMPTZ) AS VALUES ('2020-01-01T03:00:00+05:00') statement ok -RESET datafusion.execution.time_zone +SET TIME ZONE = '+00' # verify column type query T @@ -2447,7 +2445,7 @@ true true true true true true # known issues. currently overflows (expects default precision to be microsecond instead of nanoseconds. Work pending) #verify extreme values #query PPPPPPPP -#SELECT to_timestamp(-62125747200), to_timestamp(1926632005177), -62125747200::timestamp as t1, 1926632005177::timestamp, cast(-62125747200 as timestamp), cast(1926632005177 as timestamp) as t2 +#SELECT to_timestamp(-62125747200), to_timestamp(1926632005177), -62125747200::timestamp, 1926632005177::timestamp, cast(-62125747200 as timestamp), cast(1926632005177 as timestamp) #---- #0001-04-25T00:00:00 +63022-07-16T12:59:37 0001-04-25T00:00:00 +63022-07-16T12:59:37 0001-04-25T00:00:00 +63022-07-16T12:59:37 @@ -2773,8 +2771,8 @@ SELECT t1.ts, t1.ts + INTERVAL '1' SECOND FROM t1; query PT SELECT t1.ts::timestamptz, arrow_typeof(t1.ts::timestamptz) FROM t1; ---- -2018-07-01T06:00:00 Timestamp(ns) -2018-07-01T07:00:00 Timestamp(ns) +2018-07-01T06:00:00Z Timestamp(ns, "+00") +2018-07-01T07:00:00Z Timestamp(ns, "+00") query D SELECT 0::TIME @@ -3011,7 +3009,7 @@ NULL query error DataFusion error: Error during planning: Function 'make_date' expects 3 arguments but received 1 select make_date(1); -query error Expect TypeSignatureClass::Native\(LogicalType\(Native\(Int32\), Int32\)\) but received NativeType::Interval\(MonthDayNano\), DataType: Interval\(MonthDayNano\) +query error Function 'make_date' requires TypeSignatureClass::Native\(LogicalType\(Native\(Int32\), Int32\)\), but received Interval\(MonthDayNano\) \(DataType: Interval\(MonthDayNano\)\) select make_date(interval '1 day', '2001-05-21'::timestamp, '2001-05-21'::timestamp); ########## @@ -3284,7 +3282,7 @@ select make_time(22, '', 27); query error Cannot cast string '' to value of Int32 type select make_time(22, 1, ''); -query error Expect TypeSignatureClass::Native\(LogicalType\(Native\(Int32\), Int32\)\) but received NativeType::Float64, DataType: Float64 +query error Function 'make_time' requires TypeSignatureClass::Native\(LogicalType\(Native\(Int32\), Int32\)\), but received Float64 \(DataType: Float64\) select make_time(arrow_cast(22, 'Float64'), 1, ''); ########## @@ -3899,7 +3897,7 @@ statement error select to_local_time('2024-04-01T00:00:20Z'::timestamp, 'some string'); # invalid argument data type -statement error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Timestamp but received NativeType::String, DataType: Utf8 +statement error DataFusion error: Error during planning: Function 'to_local_time' requires TypeSignatureClass::Timestamp, but received String \(DataType: Utf8\) select to_local_time('2024-04-01T00:00:20Z'); # invalid timezone @@ -4341,7 +4339,7 @@ SELECT arrow_cast(a, 'LargeUtf8') FROM (SELECT CAST('2005-09-10 13:31:00 +02:00' AS timestamp with time zone) AS a) ---- -Timestamp(ns) 2005-09-10T11:31:00 2005-09-10T11:31:00 2005-09-10T11:31:00 2005-09-10T11:31:00 +Timestamp(ns, "+00") 2005-09-10T11:31:00Z 2005-09-10T11:31:00Z 2005-09-10T11:31:00Z 2005-09-10T11:31:00Z query P SELECT diff --git a/datafusion/sqllogictest/test_files/encoding.slt b/datafusion/sqllogictest/test_files/encoding.slt index ef91eade01e5b..1b1acbc385348 100644 --- a/datafusion/sqllogictest/test_files/encoding.slt +++ b/datafusion/sqllogictest/test_files/encoding.slt @@ -55,10 +55,10 @@ CREATE TABLE test( ; # errors -query error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Binary but received NativeType::Int64, DataType: Int64 +query error DataFusion error: Error during planning: Function 'encode' requires TypeSignatureClass::Binary, but received Int64 \(DataType: Int64\) select encode(12, 'hex'); -query error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Binary but received NativeType::Int64, DataType: Int64 +query error DataFusion error: Error during planning: Function 'decode' requires TypeSignatureClass::Binary, but received Int64 \(DataType: Int64\) select decode(12, 'hex'); query error DataFusion error: Error during planning: There is no built\-in encoding named 'non_encoding', currently supported encodings are: base64, hex @@ -73,7 +73,7 @@ select decode('', null) from test; query error DataFusion error: This feature is not implemented: Encoding must be a scalar; array specified encoding is not yet supported select decode('', hex_field) from test; -query error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Integer but received NativeType::String, DataType: Utf8View +query error DataFusion error: Error during planning: Function 'to_hex' requires TypeSignatureClass::Integer, but received String \(DataType: Utf8View\) select to_hex(hex_field) from test; query error DataFusion error: Execution error: Failed to decode value using base64 diff --git a/datafusion/sqllogictest/test_files/errors.slt b/datafusion/sqllogictest/test_files/errors.slt index 20c1db5cb1511..64db3740ad86d 100644 --- a/datafusion/sqllogictest/test_files/errors.slt +++ b/datafusion/sqllogictest/test_files/errors.slt @@ -125,13 +125,39 @@ from aggregate_test_100 order by c9 # WindowFunction wrong signature -statement error DataFusion error: Error during planning: Internal error: Function 'nth_value' failed to match any signature +statement error DataFusion error: Error during planning: The function 'nth_value' expected 2 arguments but received 3 select c9, nth_value(c5, 2, 3) over (order by c9) as nv1 from aggregate_test_100 order by c9 +statement error DataFusion error: Error during planning: The function 'nth_value' expected 2 arguments but received 1 +select +c9, +nth_value(c5) over (order by c9) as nv2 +from aggregate_test_100 +order by c9 + +statement error DataFusion error: Error during planning: The function 'first_value' expected 1 arguments but received 2 +select +c9, +first_value(c5, 2) over (order by c9) as fv1 +from aggregate_test_100 +order by c9 + +statement error DataFusion error: Error during planning: 'nth_value' does not support zero arguments +select nth_value() over (order by x) from (values (1), (2)) as t(x) + +statement error DataFusion error: Error during planning: 'first_value' does not support zero arguments +select first_value() over (order by x) from (values (1), (2)) as t(x) + +query error DataFusion error: Error during planning: Sum not supported for Boolean +select sum(bool_col) from (values (true), (false), (null)) as t(bool_col); + +query error DataFusion error: Error during planning: Avg not supported for Boolean +select avg(bool_col) from (values (true), (false), (null)) as t(bool_col); + # nth_value with wrong name statement error DataFusion error: Error during planning: Invalid function 'nth_vlue'.\nDid you mean 'nth_value'? diff --git a/datafusion/sqllogictest/test_files/expr.slt b/datafusion/sqllogictest/test_files/expr.slt index cec9b63675a66..00b581632748c 100644 --- a/datafusion/sqllogictest/test_files/expr.slt +++ b/datafusion/sqllogictest/test_files/expr.slt @@ -589,7 +589,7 @@ select repeat('-1.2', arrow_cast(3, 'Int32')); ---- -1.2-1.2-1.2 -query error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Native\(LogicalType\(Native\(Int64\), Int64\)\) but received NativeType::Float64, DataType: Float64 +query error DataFusion error: Error during planning: Function 'repeat' requires TypeSignatureClass::Native\(LogicalType\(Native\(Int64\), Int64\)\), but received Float64 \(DataType: Float64\) select repeat('-1.2', 3.2); query T diff --git a/datafusion/sqllogictest/test_files/functions.slt b/datafusion/sqllogictest/test_files/functions.slt index 6c87d618c7278..697232a63f090 100644 --- a/datafusion/sqllogictest/test_files/functions.slt +++ b/datafusion/sqllogictest/test_files/functions.slt @@ -208,10 +208,10 @@ SELECT substr('alphabet', NULL, 2) ---- NULL -statement error Function 'substr' failed to match any signature +statement error DataFusion error: Error during planning: Function 'substr' requires TypeSignatureClass::Native\(LogicalType\(Native\(String\), String\)\), but received Int64 \(DataType: Int64\) SELECT substr(1, 3) -statement error Function 'substr' failed to match any signature +statement error DataFusion error: Error during planning: Function 'substr' requires TypeSignatureClass::Native\(LogicalType\(Native\(String\), String\)\), but received Int64 \(DataType: Int64\) SELECT substr(1, 3, 4) query T @@ -873,7 +873,7 @@ SELECT greatest(-1, 1, 2.3, 123456789, 3 + 5, -(-4), abs(-9.0)) 123456789 -query error Function 'greatest' user-defined coercion failed with "Error during planning: greatest was called without any arguments. It requires at least 1." +query error Function 'greatest' user-defined coercion failed with: Error during planning: greatest was called without any arguments. It requires at least 1. SELECT greatest() query I @@ -1071,7 +1071,7 @@ SELECT least(-1, 1, 2.3, 123456789, 3 + 5, -(-4), abs(-9.0)) -1 -query error Function 'least' user-defined coercion failed with "Error during planning: least was called without any arguments. It requires at least 1." +query error Function 'least' user-defined coercion failed with: Error during planning: least was called without any arguments. It requires at least 1. SELECT least() query I diff --git a/datafusion/sqllogictest/test_files/named_arguments.slt b/datafusion/sqllogictest/test_files/named_arguments.slt index 07b6cc6a79a0c..f43246fc991fb 100644 --- a/datafusion/sqllogictest/test_files/named_arguments.slt +++ b/datafusion/sqllogictest/test_files/named_arguments.slt @@ -86,7 +86,7 @@ SELECT substr("STR" => 'hello world', "start_pos" => 7); # Error: wrong number of arguments # This query provides only 1 argument but substr requires 2 or 3 -query error Function 'substr' failed to match any signature +query error DataFusion error: Error during planning: Function 'substr' expects 2 to 3 arguments but received 1 SELECT substr(str => 'hello world'); ############# diff --git a/datafusion/sqllogictest/test_files/scalar.slt b/datafusion/sqllogictest/test_files/scalar.slt index 7be7de5a4def8..b01fe227a965b 100644 --- a/datafusion/sqllogictest/test_files/scalar.slt +++ b/datafusion/sqllogictest/test_files/scalar.slt @@ -2069,7 +2069,7 @@ select position('' in '') ---- 1 -query error DataFusion error: Error during planning: Internal error: Expect TypeSignatureClass::Native\(LogicalType\(Native\(String\), String\)\) but received NativeType::Int64, DataType: Int64 +query error DataFusion error: Error during planning: Function 'strpos' requires TypeSignatureClass::Native\(LogicalType\(Native\(String\), String\)\), but received Int64 \(DataType: Int64\) select position(1 in 1) query I diff --git a/datafusion/sqllogictest/test_files/spark/hash/crc32.slt b/datafusion/sqllogictest/test_files/spark/hash/crc32.slt index 6fbeb11fb9a36..df5588c75837d 100644 --- a/datafusion/sqllogictest/test_files/spark/hash/crc32.slt +++ b/datafusion/sqllogictest/test_files/spark/hash/crc32.slt @@ -81,7 +81,7 @@ SELECT crc32(arrow_cast('Spark', 'BinaryView')); ---- 1557323817 -# Upstream arrow-rs issue: https://github.com/apache/arrow-rs/issues/8841 -# This should succeed after we receive the fix -query error Arrow error: Compute error: Internal Error: Cannot cast BinaryView to BinaryArray of expected type +query I select crc32(arrow_cast(null, 'Dictionary(Int32, Utf8)')) +---- +NULL diff --git a/datafusion/sqllogictest/test_files/spark/math/hex.slt b/datafusion/sqllogictest/test_files/spark/math/hex.slt index 05c9fb3f31b28..0b5cbb61094c2 100644 --- a/datafusion/sqllogictest/test_files/spark/math/hex.slt +++ b/datafusion/sqllogictest/test_files/spark/math/hex.slt @@ -56,7 +56,7 @@ SELECT hex(column1) FROM VALUES (arrow_cast('hello', 'LargeBinary')), (NULL), (a NULL 776F726C64 -statement error Function 'hex' expects 1 arguments but received 2 +statement error DataFusion error: Error during planning: Function 'hex' expects 1 arguments but received 2 SELECT hex(1, 2); query T diff --git a/datafusion/sqllogictest/test_files/string/string_literal.slt b/datafusion/sqllogictest/test_files/string/string_literal.slt index 6cf02218872df..1338cbe4328c5 100644 --- a/datafusion/sqllogictest/test_files/string/string_literal.slt +++ b/datafusion/sqllogictest/test_files/string/string_literal.slt @@ -132,10 +132,10 @@ SELECT substr('Hello🌏世界', 5, 3) ---- o🌏世 -statement error Function 'substr' failed to match any signature +statement error DataFusion error: Error during planning: Function 'substr' requires TypeSignatureClass::Native\(LogicalType\(Native\(String\), String\)\), but received Int64 \(DataType: Int64\) SELECT substr(1, 3) -statement error Function 'substr' failed to match any signature +statement error DataFusion error: Error during planning: Function 'substr' requires TypeSignatureClass::Native\(LogicalType\(Native\(String\), String\)\), but received Int64 \(DataType: Int64\) SELECT substr(1, 3, 4) statement error Execution error: negative substring length not allowed diff --git a/datafusion/sqllogictest/test_files/struct.slt b/datafusion/sqllogictest/test_files/struct.slt index d985af1104da3..a91a5e7f870a9 100644 --- a/datafusion/sqllogictest/test_files/struct.slt +++ b/datafusion/sqllogictest/test_files/struct.slt @@ -492,9 +492,18 @@ Struct("r": Utf8, "c": Float64) statement ok drop table t; -query error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'a' to value of Float64 type +statement ok create table t as values({r: 'a', c: 1}), ({c: 2.3, r: 'b'}); +query ? +select * from t; +---- +{c: 1.0, r: a} +{c: 2.3, r: b} + +statement ok +drop table t; + ################################## ## Test Coalesce with Struct ################################## @@ -560,10 +569,18 @@ create table t(a struct(r varchar, c int), b struct(r varchar, c float)) as valu (row('purple', 1), row('green', 2.3)); # out of order struct literal -# TODO: This query should not fail -statement error DataFusion error: Optimizer rule 'simplify_expressions' failed[\s\S]*Arrow error: Cast error: Cannot cast string 'b' to value of Int32 type +statement ok create table t(a struct(r varchar, c int)) as values ({r: 'a', c: 1}), ({c: 2, r: 'b'}); +query ? +select * from t; +---- +{r: a, c: 1} +{r: b, c: 2} + +statement ok +drop table t; + ################################## ## Test Array of Struct ################################## @@ -573,9 +590,11 @@ select [{r: 'a', c: 1}, {r: 'b', c: 2}]; ---- [{r: a, c: 1}, {r: b, c: 2}] -# Can't create a list of struct with different field types -query error +# Create a list of struct with different field types +query ? select [{r: 'a', c: 1}, {c: 2, r: 'b'}]; +---- +[{c: 1, r: a}, {c: 2, r: b}] statement ok create table t(a struct(r varchar, c int), b struct(r varchar, c float)) as values (row('a', 1), row('b', 2.3)); @@ -592,9 +611,11 @@ drop table t; statement ok create table t(a struct(r varchar, c int), b struct(c float, r varchar)) as values (row('a', 1), row(2.3, 'b')); -# create array with different struct type is not valid -query error +# create array with different struct type should be cast +query T select arrow_typeof([a, b]) from t; +---- +List(Struct("c": Float32, "r": Utf8View)) statement ok drop table t; @@ -602,13 +623,13 @@ drop table t; statement ok create table t(a struct(r varchar, c int, g float), b struct(r varchar, c float, g int)) as values (row('a', 1, 2.3), row('b', 2.3, 2)); -# type of each column should not coerced but perserve as it is +# type of each column should not coerced but preserve as it is query T select arrow_typeof(a) from t; ---- Struct("r": Utf8View, "c": Int32, "g": Float32) -# type of each column should not coerced but perserve as it is +# type of each column should not coerced but preserve as it is query T select arrow_typeof(b) from t; ---- diff --git a/datafusion/sqllogictest/test_files/to_timestamp_timezone.slt b/datafusion/sqllogictest/test_files/to_timestamp_timezone.slt deleted file mode 100644 index d48e41d1204de..0000000000000 --- a/datafusion/sqllogictest/test_files/to_timestamp_timezone.slt +++ /dev/null @@ -1,204 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -########## -## to_timestamp timezone tests -########## - -## Reset timezone for other tests -statement ok -RESET datafusion.execution.time_zone - -## Test 1: Default timezone (None) with naive timestamp -## Naive timestamps (without explicit timezone) should be interpreted as UTC by default -query P -SELECT to_timestamp('2020-09-08T13:42:29'); ----- -2020-09-08T13:42:29 - -## Test 2: Explicit UTC timezone ('Z' suffix) -## Explicit timezone should be respected regardless of session timezone -query P -SELECT to_timestamp('2020-09-08T13:42:29Z'); ----- -2020-09-08T13:42:29 - -## Test 3: Explicit timezone offset (+05:00) -## Explicit offset should be respected - this is 13:42:29+05:00 which is 08:42:29 UTC -query P -SELECT to_timestamp('2020-09-08T13:42:29+05:00'); ----- -2020-09-08T08:42:29 - -## Test 4: Explicit timezone offset without colon (+0500) -## Should handle offset formats without colons -query P -SELECT to_timestamp('2020-09-08T13:42:29+0500'); ----- -2020-09-08T08:42:29 - -## Test 5: Negative timezone offset -query P -SELECT to_timestamp('2020-09-08T13:42:29-03:30'); ----- -2020-09-08T17:12:29 - -## Test 6: Configure session timezone to America/New_York -statement ok -SET datafusion.execution.time_zone = 'America/New_York'; - -## Test 7: Naive timestamp with configured timezone -## '2020-09-08T13:42:29' in America/New_York is EDT (UTC-4) -## So this should become '2020-09-08T13:42:29-04:00' -query P -SELECT to_timestamp('2020-09-08T13:42:29'); ----- -2020-09-08T13:42:29-04:00 - -## Test 8: Explicit UTC should be transformed to configured timezone -query P -SELECT to_timestamp('2020-09-08T13:42:29Z'); ----- -2020-09-08T09:42:29-04:00 - -## Test 9: Explicit offset should be transformed to configured timezone -query P -SELECT to_timestamp('2020-09-08T13:42:29+05:00'); ----- -2020-09-08T04:42:29-04:00 - -## Test 10: Check arrow_typeof returns timstamp in configured timezone -## Result should be Timestamp(Nanosecond, "America/New_York") regardless of input timezone -query T -SELECT arrow_typeof(to_timestamp('2020-09-08T13:42:29Z')); ----- -Timestamp(ns, "America/New_York") - -## Test 11: Configure to offset-based timezone -statement ok -SET datafusion.execution.time_zone = '+05:30'; - -## Test 12: Naive timestamp with offset-based configured timezone -query P -SELECT to_timestamp('2020-09-08T13:42:29'); ----- -2020-09-08T13:42:29+05:30 - -## Test 13: Reset to None -statement ok -RESET datafusion.execution.time_zone - -## Test 14: Naive timestamp -query P -SELECT to_timestamp('2020-09-08T13:42:29'); ----- -2020-09-08T13:42:29 - -query P -SELECT to_timestamp('2020-09-08T13:42:29Z'); ----- -2020-09-08T13:42:29 - -query P -SELECT to_timestamp('2020-09-08T13:42:29+05:00'); ----- -2020-09-08T08:42:29 - -statement ok -SET datafusion.execution.time_zone = 'America/New_York'; - -## Test 15: to_timestamp with format string - naive timestamp with session timezone - -query P -SELECT to_timestamp('2020-09-08 13:42:29', '%Y-%m-%d %H:%M:%S'); ----- -2020-09-08T13:42:29-04:00 - -## Test 16: to_timestamp with format string - explicit timezone should be respected -statement ok -SET datafusion.execution.time_zone = 'UTC'; - -query P -SELECT to_timestamp('2020-09-08 13:42:29 +0000', '%Y-%m-%d %H:%M:%S %z'); ----- -2020-09-08T13:42:29Z - -query P -SELECT to_timestamp('2020-09-08 13:42:29 America/Toronto', '%Y-%m-%d %H:%M:%S %Z'); ----- -2020-09-08T17:42:29Z - -query error Error parsing timestamp from '2020-09-08 13:42:29America/Toronto' using format '%Y-%m-%d %H:%M:%S%Z': '%Z' is only supported at the end of the format string preceded by a space -SELECT to_timestamp('2020-09-08 13:42:29America/Toronto', '%Y-%m-%d %H:%M:%S%Z'); - -## Test 17: Test all precision variants respect timezone -statement ok -SET datafusion.execution.time_zone = 'America/New_York'; - -## to_timestamp_seconds -query P -SELECT to_timestamp_seconds('2020-09-08T13:42:29'); ----- -2020-09-08T13:42:29-04:00 - -## to_timestamp_millis -query P -SELECT to_timestamp_millis('2020-09-08T13:42:29.123'); ----- -2020-09-08T13:42:29.123-04:00 - -## to_timestamp_micros -query P -SELECT to_timestamp_micros('2020-09-08T13:42:29.123456'); ----- -2020-09-08T13:42:29.123456-04:00 - -## to_timestamp_nanos -query P -SELECT to_timestamp_nanos('2020-09-08T13:42:29.123456789'); ----- -2020-09-08T13:42:29.123456789-04:00 - -## test integers -query T -select arrow_typeof(to_timestamp_seconds(61)) ----- -Timestamp(s, "America/New_York") - -query T -select arrow_typeof(to_timestamp_millis(61)) ----- -Timestamp(ms, "America/New_York") - -query T -select arrow_typeof(to_timestamp_micros(61)) ----- -Timestamp(µs, "America/New_York") - -query T -select arrow_typeof(to_timestamp_nanos(61)) ----- -Timestamp(ns, "America/New_York") - -query T -select arrow_typeof(to_timestamp(61)) ----- -Timestamp(ns, "America/New_York") - -## Reset timezone for other tests -statement ok -RESET datafusion.execution.time_zone diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index fe1ed1cab6bd7..360311552f025 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -2867,19 +2867,9 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ### `to_timestamp` -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats] are provided. Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). +Converts a value to a timestamp (`YYYY-MM-DDT00:00:00Z`). Supports strings, integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats] are provided. Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp. -Note: `to_timestamp` returns `Timestamp(ns, TimeZone)` where the time zone is the session time zone. The supported range -for integer input is between`-9223372037` and `9223372036`. Supported range for string input is between -`1677-09-21T00:12:44.0` and `2262-04-11T23:47:16.0`. Please use `to_timestamp_seconds` -for the input outside of supported bounds. - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. +Note: `to_timestamp` returns `Timestamp(ns)`. The supported range for integer input is between `-9223372037` and `9223372036`. Supported range for string input is between `1677-09-21T00:12:44.0` and `2262-04-11T23:47:16.0`. Please use `to_timestamp_seconds` for the input outside of supported bounds. ```sql to_timestamp(expression[, ..., format_n]) @@ -2888,11 +2878,7 @@ to_timestamp(expression[, ..., format_n]) #### Arguments - **expression**: Expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators. -- **format_n**: - Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. - Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is - only supported at the end of the string preceded by a space. +- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. #### Example @@ -2915,14 +2901,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ### `to_timestamp_micros` -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats] are provided. Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as microseconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. +Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided. Integers and unsigned integers are interpreted as microseconds since the unix epoch (`1970-01-01T00:00:00Z`) Returns the corresponding timestamp. ```sql to_timestamp_micros(expression[, ..., format_n]) @@ -2931,11 +2910,7 @@ to_timestamp_micros(expression[, ..., format_n]) #### Arguments - **expression**: Expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators. -- **format_n**: - Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. - Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is - only supported at the end of the string preceded by a space. +- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. #### Example @@ -2958,14 +2933,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ### `to_timestamp_millis` -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats] are provided. Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as milliseconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. +Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono formats](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) are provided. Integers and unsigned integers are interpreted as milliseconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp. ```sql to_timestamp_millis(expression[, ..., format_n]) @@ -2974,11 +2942,7 @@ to_timestamp_millis(expression[, ..., format_n]) #### Arguments - **expression**: Expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators. -- **format_n**: - Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. - Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is - only supported at the end of the string preceded by a space. +- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. #### Example @@ -3001,13 +2965,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ### `to_timestamp_nanos` -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000000`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats] are provided. Strings that parse without a time zone are treated as if they are in the -session time zone. Integers, unsigned integers, and doubles are interpreted as nanoseconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. +Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000000000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided. Integers and unsigned integers are interpreted as nanoseconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp. ```sql to_timestamp_nanos(expression[, ..., format_n]) @@ -3016,11 +2974,7 @@ to_timestamp_nanos(expression[, ..., format_n]) #### Arguments - **expression**: Expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators. -- **format_n**: - Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. - Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is - only supported at the end of the string preceded by a space. +- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. #### Example @@ -3043,14 +2997,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo ### `to_timestamp_seconds` -Converts a value to a timestamp (`YYYY-MM-DDT00:00:00`) in the session time zone. Supports strings, -integer, unsigned integer, and double types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') -if no [Chrono formats] are provided. Strings that parse without a time zone are treated as if they are in the -session time zone, or UTC if no session time zone is set. -Integers, unsigned integers, and doubles are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). - -The session time zone can be set using the statement `SET TIMEZONE = 'desired time zone'`. -The time zone can be a value like +00:00, 'Europe/London' etc. +Converts a value to a timestamp (`YYYY-MM-DDT00:00:00.000Z`). Supports strings, integer, and unsigned integer types as input. Strings are parsed as RFC3339 (e.g. '2023-07-20T05:44:00') if no [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)s are provided. Integers and unsigned integers are interpreted as seconds since the unix epoch (`1970-01-01T00:00:00Z`). Returns the corresponding timestamp. ```sql to_timestamp_seconds(expression[, ..., format_n]) @@ -3059,11 +3006,7 @@ to_timestamp_seconds(expression[, ..., format_n]) #### Arguments - **expression**: Expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators. -- **format_n**: - Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. - Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is - only supported at the end of the string preceded by a space. +- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. #### Example