From 5a12062ee4ac9eb636b51db2ca03054225e90f8e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 29 Oct 2026 12:42:20 -0400 Subject: [PATCH 1/8] chore: Update to arrow 60.1.0 --- Cargo.lock | 79 +++++++++---------- Cargo.toml | 20 ++--- datafusion/physical-plan/src/spill/mod.rs | 47 +++++++---- .../physical-plan/src/spill/spill_pool.rs | 4 +- datafusion/physical-plan/src/unnest.rs | 2 +- .../datetime/timestamps_timezone.slt | 76 ++++++++++-------- 6 files changed, 128 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbf3c06465bef..b673ca422a406 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,9 +158,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234ce58e76c04e8162c2e4bc3fe53fa7c2418ef5fba7bb2838b8531b7d4fbf14" +checksum = "d965156f2220e94dcf676e04322b7b0b41c97e85cb3e1c365b8e56d0f70be999" dependencies = [ "arrow-arith", "arrow-array", @@ -181,9 +181,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2303ebedb1071f4f72d71171e383859a9a285c94d99e4892685a96fca9890720" +checksum = "ca16fdd1daeb1eccefb42df52ec65448ef33eec72488598675d46dfa9ce58923" dependencies = [ "arrow-array", "arrow-buffer", @@ -195,9 +195,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51dff6e4b9f158864a0aeb6a131ad53858ecbae7a2fc8989307b9b3c6b2f122e" +checksum = "0f3d22a41d3df78b71df0b685af219ae3a94188ab98d6fb29c958eb55c3247f4" dependencies = [ "ahash", "arrow-buffer", @@ -215,9 +215,9 @@ dependencies = [ [[package]] name = "arrow-avro" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edad137a3609e2c6a4af0b5ff1c5c049b7af532649d235c7e22e9a9cad933b35" +checksum = "cfd40a71a38e4366b28742cc46912b9289d89e0c858895c8276ac0fc83dacdc6" dependencies = [ "arrow-array", "arrow-buffer", @@ -239,9 +239,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7235b863533e9db3ab86905b4521251de11226275b436ff3cb2d0010558d1c5" +checksum = "7bd46a530bc1a8e763102fc36c1dc0f51f3ffa150c78e7f5658ccb54b1458ade" dependencies = [ "bytes", "half", @@ -251,9 +251,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52cf840f0c34aeeb7e8b05aba72d66878e6c001827f423331f0b260209e7b7e" +checksum = "baa83a0752814a1922d5ee75032524ca9bbfb7f60ee35a67ed4ab8b275260aaf" dependencies = [ "arrow-array", "arrow-buffer", @@ -273,9 +273,9 @@ dependencies = [ [[package]] name = "arrow-cmp" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73a0e9adb28d95739fabcdae637261a2c14e618caf58ba1b88857120f9ee2687" +checksum = "1913e57a64fbf2c4502fa6d54b547d6a1ae59e3146cbc15321d47468c937abd1" dependencies = [ "arrow-array", "arrow-buffer", @@ -284,9 +284,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ad83176cc677f996096d694a3cccd7310069faeee4b3df9e2dabf92fdcf8b4b" +checksum = "e4572af05fadde76a063139805c089d0915b7eb64ed5bef14456c6e458a80b28" dependencies = [ "arrow-array", "arrow-cast", @@ -299,9 +299,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aa06d5e22786c4966ffc4d2f171d0cb65d95f6bdea108a14e8c335e9faa437f" +checksum = "415b7fbc4cf56b080bb5677a4ae74211b5c79e571ed37a1d547633ec07f3b43b" dependencies = [ "arrow-buffer", "arrow-schema", @@ -312,9 +312,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbc77d8656483656cbf966e04b62f193f74111ff7d9cc15f58a549314a3b061e" +checksum = "8709a798bd0825cb6207861aad09093133d64314c25569132dbc7f9f083879a5" dependencies = [ "arrow-arith", "arrow-array", @@ -330,7 +330,6 @@ dependencies = [ "base64 0.23.1", "bytes", "futures", - "once_cell", "prost", "prost-types", "tonic", @@ -339,9 +338,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86c523472d22e31f984df7a44fc36e6323d3c9b05ffe53d267ee4c3aa7c128d2" +checksum = "45f3e766bccf5a0cb57a5de52a4cba450dea5171547c2cb5939b486d08887cba" dependencies = [ "arrow-array", "arrow-buffer", @@ -355,9 +354,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "342fc6600a93a48038d853431803c941436b7b51f756763e76cad9784b7e3718" +checksum = "d843cca4b5361d7667564083b151a30d816d2b7e7ce2baa53948e2188f087f06" dependencies = [ "arrow-array", "arrow-buffer", @@ -380,9 +379,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a7ac2c6a9cdc782fa9548d3763461eaec8c93347e297fddbaa44807e70a3698" +checksum = "4c58f2ae6dd1480fc413f7320a547e39dcc499c23d4f36ca19572b685811e6e4" dependencies = [ "arrow-array", "arrow-buffer", @@ -394,9 +393,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758b87e9648d80001e0916c083334ed93074c677a81a96144d5c84bd2d8eab2a" +checksum = "68f9db8345b0638416d3b56d117d262d13070fdef788c6d75b53a180215c477e" dependencies = [ "arrow-array", "arrow-buffer", @@ -407,9 +406,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f85a9a58cd03560526b0066487574aacb9008e413504fa0ee23d61141e96c338" +checksum = "de2c8e714b466404cdc1189e89f892426efea614d938e16c15518c340cd43ba3" dependencies = [ "bitflags", "serde", @@ -419,9 +418,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba3482b7d03bf4a3bf0b195f08a7f2b071595ccd54897e35b975ac63d3104977" +checksum = "a5af3f6656c5f2c6944f93d9a357a7f204c93d5f349385a44cf46c4a26fbcf30" dependencies = [ "ahash", "arrow-array", @@ -434,9 +433,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f300b4400b1e7cd58f7cbbd5dd2a0072cc59e73bfaee2a152b3ac660e46d9dd1" +checksum = "ab1be9306b9a413b5b004e2a5a19caac728870d76e8891b351f040d554c93862" dependencies = [ "arrow-array", "arrow-buffer", @@ -529,9 +528,9 @@ dependencies = [ [[package]] name = "atoi" -version = "3.1.0" +version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a8bbe9949e43a1edaa043038c68703b04774156afdfb62ba2cef5bf93d67be" +checksum = "d12c37257553c19e9aee6cba77814fdc9e07844abfc8b3a2c377cd920ee5a993" dependencies = [ "num-traits", ] @@ -4665,9 +4664,9 @@ dependencies = [ [[package]] name = "parquet" -version = "60.0.0" +version = "60.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8af83d2940bc0510f9aef86d865f56fdc6095f87ab115ac885a80b7c5226d3ba" +checksum = "806eaa0f1c0436dcb43bee8e6a82bf2efff90521a60172f8281cb9ef577b43a4" dependencies = [ "ahash", "arrow-array", @@ -6815,7 +6814,7 @@ version = "2.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" dependencies = [ - "rand 0.10.1", + "rand 0.9.4", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 3070ed33f3231..007ca5401e0f9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -90,31 +90,31 @@ version = "55.1.0" # # See for more details: https://github.com/rust-lang/cargo/issues/11329 apache-avro = { version = "0.21", default-features = false } -arrow = { version = "60.0.0", features = [ +arrow = { version = "60.1.0", features = [ "prettyprint", "chrono-tz", ] } -arrow-avro = { version = "60.0.0", default-features = false, features = [ +arrow-avro = { version = "60.1.0", default-features = false, features = [ "deflate", "snappy", "zstd", "bzip2", "xz", ] } -arrow-buffer = { version = "60.0.0", default-features = false } -arrow-data = { version = "60.0.0", default-features = false } -arrow-flight = { version = "60.0.0", features = [ +arrow-buffer = { version = "60.1.0", default-features = false } +arrow-data = { version = "60.1.0", default-features = false } +arrow-flight = { version = "60.1.0", features = [ "flight-sql", ] } # Both codecs are required here to make sure that code paths like # file-spilling have access to all compression codecs. -arrow-ipc = { version = "60.0.0", default-features = false, features = [ +arrow-ipc = { version = "60.1.0", default-features = false, features = [ "lz4", "zstd", ] } -arrow-ord = { version = "60.0.0", default-features = false } -arrow-schema = { version = "60.0.0", default-features = false } -arrow-select = { version = "60.0.0", default-features = false } +arrow-ord = { version = "60.1.0", default-features = false } +arrow-schema = { version = "60.1.0", default-features = false } +arrow-select = { version = "60.1.0", default-features = false } async-trait = "0.1.89" bigdecimal = "0.4.8" bytes = "1.11" @@ -181,7 +181,7 @@ memchr = "2.8.1" num-traits = { version = "0.2" } object_store = { version = "0.14.2", default-features = false } parking_lot = "0.13" -parquet = { version = "60.0.0", default-features = false, features = [ +parquet = { version = "60.1.0", default-features = false, features = [ "arrow", "async", "object_store", diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rs index ab57d87dceecb..b31ef5de69ff0 100644 --- a/datafusion/physical-plan/src/spill/mod.rs +++ b/datafusion/physical-plan/src/spill/mod.rs @@ -1985,15 +1985,15 @@ mod tests { // After appending each batch, spilled_rows and spilled_bytes should increase incrementally, // while spill_file_count remains 1 (since we're writing to the same file) in_progress_file.append_batch(&batch1)?; - verify_metrics(&in_progress_file, 1, 440, 3)?; + verify_metrics(&in_progress_file, 1, 424, 3)?; in_progress_file.append_batch(&batch2)?; - verify_metrics(&in_progress_file, 1, 704, 6)?; + verify_metrics(&in_progress_file, 1, 672, 6)?; let completed_file = in_progress_file.finish()?; assert!(completed_file.is_some()); assert!(completed_file.unwrap().path().unwrap().exists()); - verify_metrics(&in_progress_file, 1, 712, 6)?; + verify_metrics(&in_progress_file, 1, 680, 6)?; // Double finish produce error let result = in_progress_file.finish(); assert!(result.is_err()); @@ -2128,7 +2128,7 @@ mod tests { // Metrics should be updated immediately (at least schema and first batch) let bytes_after_batch1 = metrics.spilled_bytes.value(); - assert_eq!(bytes_after_batch1, 440); + assert_eq!(bytes_after_batch1, 424); assert_eq!(metrics.spill_file_count.value(), 1); // Check global progress @@ -2494,7 +2494,8 @@ mod tests { /// Arrays that each hold views into one shared dictionary buffer, as the /// Parquet reader produces for a dictionary-encoded column, combined the - /// way a sort combines its input batches. Each output row cycles through + /// way a sort combines its input batches: the combined array lists the + /// dictionary buffer once per input and each output row cycles through /// the dictionary (). fn shared_dictionary_views( dictionary_len: u32, @@ -2502,7 +2503,7 @@ mod tests { rows_per_input: u32, ) -> Result<(StringViewArray, StringViewArray)> { use arrow::array::{AsArray, UInt32Array}; - use arrow::compute::{interleave, take}; + use arrow::compute::take; let dictionary = dictionary_array(dictionary_len)?; let arrays = (0..inputs) @@ -2511,21 +2512,39 @@ mod tests { (0..rows_per_input) .map(|row| (input * rows_per_input + row) % dictionary_len), ); - take(&dictionary, &indices, None) + let taken = take(&dictionary, &indices, None)?; + let taken = taken.as_string_view().clone(); + assert_eq!(taken.data_buffers().len(), 1); + Ok(taken) + }) + .collect::>>()?; + // Output row `r` holds row `r` of each input in turn, and the views of + // input `i` point at buffer entry `i` + let views: Vec = (0..rows_per_input as usize) + .flat_map(|row| { + arrays.iter().enumerate().map(move |(input, array)| { + let raw = array.views()[row]; + if (raw as u32) <= MAX_INLINE_VIEW_LEN { + raw + } else { + ByteView { + buffer_index: input as u32, + ..ByteView::from(raw) + } + .as_u128() + } + }) }) - .collect::, _>>()?; - let arrays: Vec<&dyn Array> = arrays.iter().map(|a| a.as_ref()).collect(); - let indices: Vec<(usize, usize)> = (0..rows_per_input as usize) - .flat_map(|row| (0..inputs as usize).map(move |input| (input, row))) .collect(); - let merged = interleave(&arrays, &indices)?; - Ok((dictionary, merged.as_string_view().clone())) + let buffers = vec![dictionary.data_buffers()[0].clone(); inputs as usize]; + let merged = StringViewArray::try_new(ScalarBuffer::from(views), buffers, None)?; + Ok((dictionary, merged)) } #[test] fn test_gc_keeps_shared_dictionary_buffer() -> Result<()> { let (dictionary, merged) = shared_dictionary_views(1000, 8, 1024)?; - // `interleave` lists the dictionary once per input + // the combined array lists the dictionary once per input assert_eq!(merged.data_buffers().len(), 8); // `gc()` would copy 8192 values of 63 bytes, 8x the dictionary assert!(merged.total_buffer_bytes_used() > data_buffers_size(&dictionary)); diff --git a/datafusion/physical-plan/src/spill/spill_pool.rs b/datafusion/physical-plan/src/spill/spill_pool.rs index 6d50245ea60af..48e67e72d2e75 100644 --- a/datafusion/physical-plan/src/spill/spill_pool.rs +++ b/datafusion/physical-plan/src/spill/spill_pool.rs @@ -1005,7 +1005,7 @@ mod tests { ); assert_eq!( metrics.spilled_bytes.value(), - 320, + 312, "Spilled bytes should reflect data written (header + 1 batch)" ); assert_eq!( @@ -1480,7 +1480,7 @@ mod tests { // Check metrics before drop - spilled_bytes already reflects written data let spilled_bytes_before = metrics.spilled_bytes.value(); assert_eq!( - spilled_bytes_before, 1088, + spilled_bytes_before, 1048, "Spilled bytes should reflect data written (header + 5 batches)" ); diff --git a/datafusion/physical-plan/src/unnest.rs b/datafusion/physical-plan/src/unnest.rs index a5e57c745a6ec..8dbbed5558de9 100644 --- a/datafusion/physical-plan/src/unnest.rs +++ b/datafusion/physical-plan/src/unnest.rs @@ -1214,7 +1214,7 @@ impl ListArrayType for FixedSizeListArray { } fn value_offsets(&self, row: usize) -> (i64, i64) { - let start = self.value_offset(row) as i64; + let start = self.value_offset_at(row) as i64; (start, start + self.value_length() as i64) } } diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt index 747433b4d68f3..8da1b57cdc8ad 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt @@ -1377,13 +1377,9 @@ Timestamp(ns, "+00:00") 2024-07-01T06:00:00Z ########## ## SECTION 16: DST boundaries ## -## KNOWN WRONG, and the single largest divergence in this file: DataFusion -## cannot represent an ambiguous or non-existent local time in a named zone at -## all. Every route into a named zone -- a column cast, `AT TIME ZONE`, a -## string literal, or `::timestamptz` under a session zone -- fails with an -## error, where PostgreSQL resolves the value. -## -## PostgreSQL's answers, for reference: +## Casting a naive timestamp into a named zone (a column cast or +## `AT TIME ZONE`) resolves an ambiguous or non-existent local time the way +## PostgreSQL does, since https://github.com/apache/arrow-rs/pull/11038: ## ambiguous 2024-11-03 01:30:00 America/Denver -> 2024-11-03 01:30:00-07 ## (the second, standard-time occurrence) ## nonexistent 2024-03-10 02:30:00 America/Denver -> 2024-03-10 03:30:00-06 @@ -1391,19 +1387,17 @@ Timestamp(ns, "+00:00") 2024-07-01T06:00:00Z ## ambiguous 2024-10-27 02:30:00 Europe/Brussels -> 2024-10-27 02:30:00+01 ## nonexistent 2024-03-31 02:30:00 Europe/Brussels -> 2024-03-31 03:30:00+02 ## -## Note that the column path and the literal path fail with *different* errors -## even though the value is identical: the column path reports -## "Cannot cast timezone to different timezone" from the Arrow cast kernel, -## while the literal path is folded by `simplify_expressions` and reports -## "error computing timezone offset" from the Arrow timestamp parser. +## KNOWN WRONG: the *parsing* routes into a named zone -- a string literal cast +## to a timestamp with a named zone, or `::timestamptz` under a session zone -- +## still fail with "error computing timezone offset" from the Arrow timestamp +## parser, where PostgreSQL resolves the value. ## -## See https://github.com/apache/datafusion/issues/25084 and the arrow-rs fix -## https://github.com/apache/arrow-rs/pull/11038 +## See https://github.com/apache/datafusion/issues/25084 ########## # --- US fall back: 2024-11-03 01:30:00 America/Denver occurs twice --- -# The naive value itself is fine; only the move into a named zone fails. +# The naive value itself is fine query P SELECT '2024-11-03 01:30:00'::timestamp ---- @@ -1412,25 +1406,29 @@ SELECT '2024-11-03 01:30:00'::timestamp statement ok CREATE TABLE dst_ambiguous AS VALUES ('2024-11-03 01:30:00'::timestamp) -# These expectations match only the stable `Arrow error: ...` tail. The wrapper -# chain above it (optimizer rule, field context) is not what these cases pin, -# and it changed in https://github.com/apache/datafusion/pull/24920. -query error Arrow error: Cast error: Cannot cast timezone to different timezone +# The ambiguous wall clock resolves to the second (standard time) occurrence +query P SELECT column1 AT TIME ZONE 'America/Denver' FROM dst_ambiguous +---- +2024-11-03T01:30:00-07:00 -query error Arrow error: Cast error: Cannot cast timezone to different timezone +query P SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("America/Denver"))') FROM dst_ambiguous +---- +2024-11-03T01:30:00-07:00 -# The same wall clock arriving as a string literal instead of a column +# The same wall clock arriving as a string literal instead of a column still +# fails, as it goes through the timestamp parser rather than the cast kernel query error Arrow\ error:\ Parser\ error:\ Error\ parsing\ timestamp\ from\ '2024\-11\-03T01:30:00':\ error\ computing\ timezone\ offset SELECT arrow_cast('2024-11-03T01:30:00', 'Timestamp(Nanosecond, Some("America/Denver"))') -query error Arrow\ error:\ Cast\ error:\ Cannot\ cast\ timezone\ to\ different\ timezone +query P SELECT TIMESTAMP '2024-11-03 01:30:00' AT TIME ZONE 'America/Denver' +---- +2024-11-03T01:30:00-07:00 -# Both instants are representable when the offset is spelled out explicitly -- -# it is only the *resolution* of a bare wall clock that fails. Note the two -# rows differ solely in the rendered offset. +# Both instants are representable when the offset is spelled out explicitly. +# Note the two rows differ solely in the rendered offset. query PP SELECT arrow_cast('2024-11-03T07:30:00Z', 'Timestamp(Nanosecond, Some("America/Denver"))'), arrow_cast('2024-11-03T08:30:00Z', 'Timestamp(Nanosecond, Some("America/Denver"))') @@ -1442,17 +1440,23 @@ SELECT arrow_cast('2024-11-03T07:30:00Z', 'Timestamp(Nanosecond, Some("America/D statement ok CREATE TABLE dst_nonexistent AS VALUES ('2024-03-10 02:30:00'::timestamp) -query error Arrow error: Cast error: Cannot cast timezone to different timezone +query P SELECT column1 AT TIME ZONE 'America/Denver' FROM dst_nonexistent +---- +2024-03-10T03:30:00-06:00 -query error Arrow error: Cast error: Cannot cast timezone to different timezone +query P SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("America/Denver"))') FROM dst_nonexistent +---- +2024-03-10T03:30:00-06:00 query error Arrow\ error:\ Parser\ error:\ Error\ parsing\ timestamp\ from\ '2024\-03\-10T02:30:00':\ error\ computing\ timezone\ offset SELECT arrow_cast('2024-03-10T02:30:00', 'Timestamp(Nanosecond, Some("America/Denver"))') -query error Arrow\ error:\ Cast\ error:\ Cannot\ cast\ timezone\ to\ different\ timezone +query P SELECT TIMESTAMP '2024-03-10 02:30:00' AT TIME ZONE 'America/Denver' +---- +2024-03-10T03:30:00-06:00 statement ok SET datafusion.execution.time_zone = 'America/Denver' @@ -1471,11 +1475,17 @@ RESET datafusion.execution.time_zone statement ok CREATE TABLE dst_eu AS VALUES ('2024-10-27 02:30:00'::timestamp), ('2024-03-31 02:30:00'::timestamp) -query error Arrow error: Cast error: Cannot cast timezone to different timezone +query P SELECT column1 AT TIME ZONE 'Europe/Brussels' FROM dst_eu +---- +2024-10-27T02:30:00+01:00 +2024-03-31T03:30:00+02:00 -query error Arrow error: Cast error: Cannot cast timezone to different timezone +query P SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("Europe/Brussels"))') FROM dst_eu +---- +2024-10-27T02:30:00+01:00 +2024-03-31T03:30:00+02:00 query error Arrow\ error:\ Parser\ error:\ Error\ parsing\ timestamp\ from\ '2024\-10\-27T02:30:00':\ error\ computing\ timezone\ offset SELECT arrow_cast('2024-10-27T02:30:00', 'Timestamp(Nanosecond, Some("Europe/Brussels"))') @@ -1496,9 +1506,9 @@ statement ok RESET datafusion.execution.time_zone # A tz-aware column crossing the fall-back boundary. Constructing the values -# from explicit UTC instants sidesteps the failure above, so the behaviour of -# date_trunc / date_bin / date_part / interval arithmetic across a repeated -# hour can still be pinned down. +# from explicit UTC instants covers both occurrences of the repeated hour, so +# the behaviour of date_trunc / date_bin / date_part / interval arithmetic +# across it can be pinned down. statement ok CREATE TABLE dst_cross AS SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("America/Denver"))') AS ts From 19e54579391cf0e4c043b14dc384f15fe6e28e46 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 29 Oct 2026 12:56:55 -0400 Subject: [PATCH 2/8] trim comments --- .../datetime/timestamps_timezone.slt | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt index 8da1b57cdc8ad..c4e3286791eb1 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt @@ -1377,9 +1377,9 @@ Timestamp(ns, "+00:00") 2024-07-01T06:00:00Z ########## ## SECTION 16: DST boundaries ## -## Casting a naive timestamp into a named zone (a column cast or -## `AT TIME ZONE`) resolves an ambiguous or non-existent local time the way -## PostgreSQL does, since https://github.com/apache/arrow-rs/pull/11038: +## Casting a naive timestamp into a named zone resolves an ambiguous or +## non-existent local time like PostgreSQL. +## ## ambiguous 2024-11-03 01:30:00 America/Denver -> 2024-11-03 01:30:00-07 ## (the second, standard-time occurrence) ## nonexistent 2024-03-10 02:30:00 America/Denver -> 2024-03-10 03:30:00-06 @@ -1387,11 +1387,8 @@ Timestamp(ns, "+00:00") 2024-07-01T06:00:00Z ## ambiguous 2024-10-27 02:30:00 Europe/Brussels -> 2024-10-27 02:30:00+01 ## nonexistent 2024-03-31 02:30:00 Europe/Brussels -> 2024-03-31 03:30:00+02 ## -## KNOWN WRONG: the *parsing* routes into a named zone -- a string literal cast -## to a timestamp with a named zone, or `::timestamptz` under a session zone -- -## still fail with "error computing timezone offset" from the Arrow timestamp -## parser, where PostgreSQL resolves the value. -## +## KNOWN WRONG: the *parsing* routes into a named zone with a session zone +## still fail with "error computing timezone offset". See ## See https://github.com/apache/datafusion/issues/25084 ########## @@ -1417,8 +1414,7 @@ SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("America/Denver"))') FROM ---- 2024-11-03T01:30:00-07:00 -# The same wall clock arriving as a string literal instead of a column still -# fails, as it goes through the timestamp parser rather than the cast kernel +# String literal parsing fails. query error Arrow\ error:\ Parser\ error:\ Error\ parsing\ timestamp\ from\ '2024\-11\-03T01:30:00':\ error\ computing\ timezone\ offset SELECT arrow_cast('2024-11-03T01:30:00', 'Timestamp(Nanosecond, Some("America/Denver"))') @@ -1506,9 +1502,7 @@ statement ok RESET datafusion.execution.time_zone # A tz-aware column crossing the fall-back boundary. Constructing the values -# from explicit UTC instants covers both occurrences of the repeated hour, so -# the behaviour of date_trunc / date_bin / date_part / interval arithmetic -# across it can be pinned down. +# from explicit UTC instants covers a repeated hour statement ok CREATE TABLE dst_cross AS SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("America/Denver"))') AS ts From e17a2c42846a95b91dc69fb7f186f959c2314126 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 29 Oct 2026 12:57:51 -0400 Subject: [PATCH 3/8] cleaning up --- .../sqllogictest/test_files/datetime/timestamps_timezone.slt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt index c4e3286791eb1..ce525005ded0d 100644 --- a/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt +++ b/datafusion/sqllogictest/test_files/datetime/timestamps_timezone.slt @@ -1387,7 +1387,7 @@ Timestamp(ns, "+00:00") 2024-07-01T06:00:00Z ## ambiguous 2024-10-27 02:30:00 Europe/Brussels -> 2024-10-27 02:30:00+01 ## nonexistent 2024-03-31 02:30:00 Europe/Brussels -> 2024-03-31 03:30:00+02 ## -## KNOWN WRONG: the *parsing* routes into a named zone with a session zone +## The *parsing* routes into a named zone with a session zone ## still fail with "error computing timezone offset". See ## See https://github.com/apache/datafusion/issues/25084 ########## @@ -1415,6 +1415,7 @@ SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("America/Denver"))') FROM 2024-11-03T01:30:00-07:00 # String literal parsing fails. +# https://github.com/apache/arrow-rs/issues/11039 query error Arrow\ error:\ Parser\ error:\ Error\ parsing\ timestamp\ from\ '2024\-11\-03T01:30:00':\ error\ computing\ timezone\ offset SELECT arrow_cast('2024-11-03T01:30:00', 'Timestamp(Nanosecond, Some("America/Denver"))') From 2d6fe153b8eac7ba09511a78d6b6ec51e09ee570 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 29 Oct 2026 13:03:53 -0400 Subject: [PATCH 4/8] Update sqllogictest expectations for arrow-rs 60.1.0 behavior changes Co-Authored-By: Claude Fable 5.1 --- datafusion/sqllogictest/test_files/aggregate.slt | 4 +++- .../sqllogictest/test_files/datetime/date_part.slt | 8 +++++--- .../test_files/nested_loop_join_spill.slt | 13 ++++++++----- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index f9feb6230ec84..b05eb96c705b2 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -1729,10 +1729,12 @@ SELECT approx_distinct(c9) AS a, approx_distinct(c9) AS b FROM aggregate_test_10 99 99 # csv_query_approx_count_date_timestamp +# Casting a Timestamp to Date64 truncates to whole days (arrow-rs 60.1.0), so +# only 7 distinct dates remain. query IIIII SELECT approx_distinct(c14) AS a, approx_distinct(c15) AS b, approx_distinct(arrow_cast(c15, 'Date64')), approx_distinct(arrow_cast(c15, 'Time32(Second)')) as c, approx_distinct(arrow_cast(c15, 'Time64(Nanosecond)')) AS d FROM aggregate_test_100 ---- -18 60 60 60 60 +18 60 7 60 60 # approx_distinct over Boolean: exact count via flag-pair accumulator (0..=2). statement ok diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.slt index 8ff029712cbb0..83b5aa80a9430 100644 --- a/datafusion/sqllogictest/test_files/datetime/date_part.slt +++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt @@ -1069,12 +1069,14 @@ FROM (SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("Africa/Monrovia")) 28 28500 28500000 28500000000 NULL NULL NULL NULL -# timestamps outside the range arrow represents are NULL, like the other parts +# The sub-second parts of a timestamp outside the range arrow represents are +# NULL, like the other parts. The second is computed directly from the stored +# value since arrow-rs 60.1.0, so it is i64::MAX % 60 = 7 instead. query IIII SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)')), (NULL)) AS t(ts) ---- -NULL NULL NULL NULL +7 NULL NULL NULL NULL NULL NULL NULL # test_extract_interval @@ -1603,7 +1605,7 @@ NULL # Non-null inputs can give NULL results, so the result is nullable. # Reproducers from https://github.com/apache/datafusion/issues/26685 query I -SELECT extract(second from ts) FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))) AS t(ts) +SELECT extract(millisecond from ts) FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)'))) AS t(ts) ---- NULL diff --git a/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt b/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt index 5dfdfff6d57cb..16c324d07d983 100644 --- a/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt +++ b/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt @@ -294,6 +294,9 @@ RIGHT ANTI JOIN existence_spill_right r ON l.threshold > r.k ---- 1064 NULL payload-64 +# The right side is replayed once per left chunk. Spilled batches are read back +# without validity buffers for null-free columns (arrow-rs 60.1.0), so they take +# less memory and fit in three chunks instead of four. query TT EXPLAIN ANALYZE SELECT count(*) FROM existence_spill_left l @@ -302,7 +305,7 @@ RIGHT SEMI JOIN existence_spill_right r ON l.threshold > r.k Plan with Metrics 01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[] 02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))], metrics=[] -03)----NestedLoopJoinExec: join_type=RightSemi, filter=threshold@0 > k@1, projection=[], metrics=[output_rows=128, spill_count=2, input_batches=8, input_rows=516, ] +03)----NestedLoopJoinExec: join_type=RightSemi, filter=threshold@0 > k@1, projection=[], metrics=[output_rows=128, spill_count=2, input_batches=6, input_rows=387, ] 04)------ProjectionExec: expr=[CASE WHEN value@0 = 4097 THEN 130 WHEN value@0 % 65 = 0 THEN 65 WHEN value@0 % 65 = 1 THEN 97 ELSE 114 END as threshold], metrics=[] 05)--------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=5000, batch_size=65], metrics=[] 06)------ProjectionExec: expr=[CASE WHEN __common_expr_1@0 = 128 THEN NULL ELSE __common_expr_1@0 END as k], metrics=[] @@ -317,7 +320,7 @@ RIGHT ANTI JOIN existence_spill_right r ON l.threshold > r.k Plan with Metrics 01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[] 02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))], metrics=[] -03)----NestedLoopJoinExec: join_type=RightAnti, filter=threshold@0 > k@1, projection=[], metrics=[output_rows=1, spill_count=2, input_batches=8, input_rows=516, ] +03)----NestedLoopJoinExec: join_type=RightAnti, filter=threshold@0 > k@1, projection=[], metrics=[output_rows=1, spill_count=2, input_batches=6, input_rows=387, ] 04)------ProjectionExec: expr=[CASE WHEN value@0 = 4097 THEN 130 WHEN value@0 % 65 = 0 THEN 65 WHEN value@0 % 65 = 1 THEN 97 ELSE 114 END as threshold], metrics=[] 05)--------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=5000, batch_size=65], metrics=[] 06)------ProjectionExec: expr=[CASE WHEN __common_expr_1@0 = 128 THEN NULL ELSE __common_expr_1@0 END as k], metrics=[] @@ -339,7 +342,7 @@ RIGHT SEMI JOIN range(129) r ON l.threshold > r.value ---- 129 8256 0 128 -# Both joins must spill and replay the two right batches across four left chunks. +# Both joins must spill and replay the two right batches across three left chunks. query TT EXPLAIN ANALYZE SELECT count(*) FROM existence_spill_early_left l @@ -348,7 +351,7 @@ RIGHT SEMI JOIN range(129) r ON l.threshold > r.value Plan with Metrics 01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[] 02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))], metrics=[] -03)----NestedLoopJoinExec: join_type=RightSemi, filter=threshold@0 > value@1, projection=[], metrics=[output_rows=129, spill_count=2, input_batches=8, input_rows=516, ] +03)----NestedLoopJoinExec: join_type=RightSemi, filter=threshold@0 > value@1, projection=[], metrics=[output_rows=129, spill_count=2, input_batches=6, input_rows=387, ] 04)------ProjectionExec: expr=[CASE WHEN value@0 = 0 THEN 129 ELSE 0 END as threshold], metrics=[] 05)--------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=5000, batch_size=65], metrics=[] 06)------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=129, batch_size=65], metrics=[] @@ -361,7 +364,7 @@ RIGHT ANTI JOIN range(129) r ON l.threshold > r.value Plan with Metrics 01)ProjectionExec: expr=[count(Int64(1))@0 as count(*)], metrics=[] 02)--AggregateExec: mode=Single, gby=[], aggr=[count(Int64(1))], metrics=[] -03)----NestedLoopJoinExec: join_type=RightAnti, filter=threshold@0 > value@1, projection=[], metrics=[output_rows=0, spill_count=2, input_batches=8, input_rows=516, ] +03)----NestedLoopJoinExec: join_type=RightAnti, filter=threshold@0 > value@1, projection=[], metrics=[output_rows=0, spill_count=2, input_batches=6, input_rows=387, ] 04)------ProjectionExec: expr=[CASE WHEN value@0 = 0 THEN 129 ELSE 0 END as threshold], metrics=[] 05)--------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=5000, batch_size=65], metrics=[] 06)------LazyMemoryExec: partitions=1, batch_generators=[range: start=0, end=129, batch_size=65], metrics=[] From 50516041b1ac6b88584d222c7d609ea541e624bb Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 29 Oct 2026 13:28:22 -0400 Subject: [PATCH 5/8] Move behavior-change explanations out of slt comments Co-Authored-By: Claude Fable 5.1 --- datafusion/sqllogictest/test_files/aggregate.slt | 2 -- datafusion/sqllogictest/test_files/datetime/date_part.slt | 6 +++--- .../sqllogictest/test_files/nested_loop_join_spill.slt | 3 --- 3 files changed, 3 insertions(+), 8 deletions(-) diff --git a/datafusion/sqllogictest/test_files/aggregate.slt b/datafusion/sqllogictest/test_files/aggregate.slt index b05eb96c705b2..3dd6ec1a07c32 100644 --- a/datafusion/sqllogictest/test_files/aggregate.slt +++ b/datafusion/sqllogictest/test_files/aggregate.slt @@ -1729,8 +1729,6 @@ SELECT approx_distinct(c9) AS a, approx_distinct(c9) AS b FROM aggregate_test_10 99 99 # csv_query_approx_count_date_timestamp -# Casting a Timestamp to Date64 truncates to whole days (arrow-rs 60.1.0), so -# only 7 distinct dates remain. query IIIII SELECT approx_distinct(c14) AS a, approx_distinct(c15) AS b, approx_distinct(arrow_cast(c15, 'Date64')), approx_distinct(arrow_cast(c15, 'Time32(Second)')) as c, approx_distinct(arrow_cast(c15, 'Time64(Nanosecond)')) AS d FROM aggregate_test_100 ---- diff --git a/datafusion/sqllogictest/test_files/datetime/date_part.slt b/datafusion/sqllogictest/test_files/datetime/date_part.slt index 83b5aa80a9430..b8ebe007175f0 100644 --- a/datafusion/sqllogictest/test_files/datetime/date_part.slt +++ b/datafusion/sqllogictest/test_files/datetime/date_part.slt @@ -1069,9 +1069,9 @@ FROM (SELECT arrow_cast(column1, 'Timestamp(Nanosecond, Some("Africa/Monrovia")) 28 28500 28500000 28500000000 NULL NULL NULL NULL -# The sub-second parts of a timestamp outside the range arrow represents are -# NULL, like the other parts. The second is computed directly from the stored -# value since arrow-rs 60.1.0, so it is i64::MAX % 60 = 7 instead. +# The second is computed directly from the stored value (i64::MAX % 60 = 7). The +# sub-second parts of a timestamp outside the range arrow represents are NULL, +# like the other parts. query IIII SELECT extract(second from ts), extract(millisecond from ts), extract(microsecond from ts), extract(nanosecond from ts) FROM (VALUES (arrow_cast(9223372036854775807, 'Timestamp(Second, None)')), (NULL)) AS t(ts) diff --git a/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt b/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt index 16c324d07d983..adcd7b949e1e3 100644 --- a/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt +++ b/datafusion/sqllogictest/test_files/nested_loop_join_spill.slt @@ -294,9 +294,6 @@ RIGHT ANTI JOIN existence_spill_right r ON l.threshold > r.k ---- 1064 NULL payload-64 -# The right side is replayed once per left chunk. Spilled batches are read back -# without validity buffers for null-free columns (arrow-rs 60.1.0), so they take -# less memory and fit in three chunks instead of four. query TT EXPLAIN ANALYZE SELECT count(*) FROM existence_spill_left l From 601a21700337d62410f315634d48d5cd4ce1715e Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Fri, 30 Oct 2026 15:43:22 -0400 Subject: [PATCH 6/8] simplify test --- datafusion-testing | 2 +- datafusion/physical-plan/src/spill/mod.rs | 44 ++++++----------------- 2 files changed, 12 insertions(+), 34 deletions(-) diff --git a/datafusion-testing b/datafusion-testing index 64e2529880aee..81c7440e08b9f 160000 --- a/datafusion-testing +++ b/datafusion-testing @@ -1 +1 @@ -Subproject commit 64e2529880aeefb81844ea0de26548332086379d +Subproject commit 81c7440e08b9fe81a982e6b0b7e0fe3a6e82fb78 diff --git a/datafusion/physical-plan/src/spill/mod.rs b/datafusion/physical-plan/src/spill/mod.rs index b31ef5de69ff0..842105c99da6b 100644 --- a/datafusion/physical-plan/src/spill/mod.rs +++ b/datafusion/physical-plan/src/spill/mod.rs @@ -819,8 +819,8 @@ fn gc_array(array: &ArrayRef) -> Result<(ArrayRef, bool)> { /// The IPC writer writes every data buffer of an array in full, once per /// entry, and `gc()` copies the bytes of every view separately. Neither /// handles views that share bytes, as the views of a dictionary-encoded -/// Parquet column do: after `concat` or `interleave` the dictionary is often -/// listed once per input batch, and `gc()` copies it once per row +/// Parquet column do: after `concat` the dictionary is listed once per input +/// batch, and `gc()` copies it once per row /// (). /// /// Repeated buffer entries are merged first. `gc()` then runs only if it @@ -2493,9 +2493,8 @@ mod tests { } /// Arrays that each hold views into one shared dictionary buffer, as the - /// Parquet reader produces for a dictionary-encoded column, combined the - /// way a sort combines its input batches: the combined array lists the - /// dictionary buffer once per input and each output row cycles through + /// Parquet reader produces for a dictionary-encoded column, combined with + /// `concat` as a sort combines its buffered batches. The rows cycle through /// the dictionary (). fn shared_dictionary_views( dictionary_len: u32, @@ -2503,7 +2502,7 @@ mod tests { rows_per_input: u32, ) -> Result<(StringViewArray, StringViewArray)> { use arrow::array::{AsArray, UInt32Array}; - use arrow::compute::take; + use arrow::compute::{concat, take}; let dictionary = dictionary_array(dictionary_len)?; let arrays = (0..inputs) @@ -2512,39 +2511,18 @@ mod tests { (0..rows_per_input) .map(|row| (input * rows_per_input + row) % dictionary_len), ); - let taken = take(&dictionary, &indices, None)?; - let taken = taken.as_string_view().clone(); - assert_eq!(taken.data_buffers().len(), 1); - Ok(taken) - }) - .collect::>>()?; - // Output row `r` holds row `r` of each input in turn, and the views of - // input `i` point at buffer entry `i` - let views: Vec = (0..rows_per_input as usize) - .flat_map(|row| { - arrays.iter().enumerate().map(move |(input, array)| { - let raw = array.views()[row]; - if (raw as u32) <= MAX_INLINE_VIEW_LEN { - raw - } else { - ByteView { - buffer_index: input as u32, - ..ByteView::from(raw) - } - .as_u128() - } - }) + take(&dictionary, &indices, None) }) - .collect(); - let buffers = vec![dictionary.data_buffers()[0].clone(); inputs as usize]; - let merged = StringViewArray::try_new(ScalarBuffer::from(views), buffers, None)?; - Ok((dictionary, merged)) + .collect::, _>>()?; + let arrays: Vec<&dyn Array> = arrays.iter().map(|a| a.as_ref()).collect(); + let merged = concat(&arrays)?; + Ok((dictionary, merged.as_string_view().clone())) } #[test] fn test_gc_keeps_shared_dictionary_buffer() -> Result<()> { let (dictionary, merged) = shared_dictionary_views(1000, 8, 1024)?; - // the combined array lists the dictionary once per input + // `concat` lists the dictionary once per input assert_eq!(merged.data_buffers().len(), 8); // `gc()` would copy 8192 values of 63 bytes, 8x the dictionary assert!(merged.total_buffer_bytes_used() > data_buffers_size(&dictionary)); From aaa4740420e9b0ce789d8c99e9c2910be03cc540 Mon Sep 17 00:00:00 2001 From: Jefffrey Date: Sun, 1 Nov 2026 14:34:24 +0900 Subject: [PATCH 7/8] fix datafusion-testing submodule --- datafusion-testing | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datafusion-testing b/datafusion-testing index 81c7440e08b9f..64e2529880aee 160000 --- a/datafusion-testing +++ b/datafusion-testing @@ -1 +1 @@ -Subproject commit 81c7440e08b9fe81a982e6b0b7e0fe3a6e82fb78 +Subproject commit 64e2529880aeefb81844ea0de26548332086379d From 4e776715e8ef88c076e7dc71fd3bb7566ff59915 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Sun, 1 Nov 2026 07:46:53 -0500 Subject: [PATCH 8/8] clarify versions --- docs/source/library-user-guide/upgrading/56.0.0.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/library-user-guide/upgrading/56.0.0.md b/docs/source/library-user-guide/upgrading/56.0.0.md index 6b33b5ededd17..4665bfa52e1f7 100644 --- a/docs/source/library-user-guide/upgrading/56.0.0.md +++ b/docs/source/library-user-guide/upgrading/56.0.0.md @@ -96,16 +96,16 @@ Aggregate plans now display `group_clustering_mode=Full` or `group_clustering_mode=Partial(indices)` instead of `ordering_mode=Sorted` or `ordering_mode=PartiallySorted(indices)`. -### Upgrade arrow/parquet to 60.0.0 and object_store to 0.14.2 +### Upgrade arrow/parquet to 60 and object_store to 0.14 -DataFusion 56.0.0 uses `arrow` and `parquet` 60.0.0, and `object_store` 0.14.2. +DataFusion 56.0.0 uses `arrow` and `parquet` 60, and `object_store` 0.14. This may require updates to your Cargo.toml if you have direct dependencies on these crates. -See the [Arrow 60.0.0 release notes] and the [object_store 0.14.2 upgrade guide] for details on breaking changes in those versions. +See the [Arrow 60.0.0 release notes] and the [object_store 0.14.0 changelog] for details on breaking changes in those versions. [arrow 60.0.0 release notes]: https://github.com/apache/arrow-rs/releases/tag/60.0.0 -[object_store 0.14.2 upgrade guide]: https://github.com/apache/arrow-rs-object-store/blob/v0.14.2/CHANGELOG.md +[object_store 0.14.0 changelog]: https://github.com/apache/arrow-rs-object-store/blob/v0.14.2/CHANGELOG-old.md#v0140-2026-06-18 #### Field and schema metadata now use arrow's `Metadata` type