From 1d9516e753a69a89fc13da8f3adba98cb7815c9f Mon Sep 17 00:00:00 2001 From: Yuang Gao Date: Sat, 7 Nov 2026 10:43:28 -0800 Subject: [PATCH] chore(storage-opendal): address review nits from #3179 --- crates/storage/opendal/src/lib.rs | 1 + crates/storage/opendal/src/s3.rs | 3 +++ .../storage/opendal/tests/file_io_s3_test.rs | 22 +++++++++---------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index b1279b34b4..2fb571f548 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -560,6 +560,7 @@ impl OpenDalStorage { multipart_part_size, .. } => WriteOptions { + // The parser caps this at `i32::MAX`, so the cast can't fail for parsed values. chunk: Some(usize::try_from(*multipart_part_size).unwrap_or(usize::MAX)), ..WriteOptions::default() }, diff --git a/crates/storage/opendal/src/s3.rs b/crates/storage/opendal/src/s3.rs index 8f1828dc46..2d40f73a92 100644 --- a/crates/storage/opendal/src/s3.rs +++ b/crates/storage/opendal/src/s3.rs @@ -45,6 +45,9 @@ const MULTIPART_PART_SIZE_MAX: u64 = i32::MAX as u64; /// Matches Java `S3FileIOProperties.MULTIPART_SIZE_DEFAULT`. const MULTIPART_PART_SIZE_DEFAULT: u64 = 32 * 1024 * 1024; +/// Default part size, also the serde default for `OpenDalStorage::S3::multipart_part_size`. +/// A bare `#[serde(default)]` would read payloads without the field as 0, which OpenDAL +/// raises to 5 MiB instead of 32 MiB. pub(crate) fn default_multipart_part_size() -> u64 { MULTIPART_PART_SIZE_DEFAULT } diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs index 14e54dcf59..7e81a3ee12 100644 --- a/crates/storage/opendal/tests/file_io_s3_test.rs +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -69,12 +69,13 @@ mod tests { .build() } - /// Number of upload parts S3 recorded for `key`. A multipart upload reports - /// a `-` ETag; a single-request upload reports a bare MD5. + /// Number of upload parts S3 recorded for `key`, or `None` for a single-request + /// upload. A multipart upload reports a `-` ETag; a single-request + /// upload reports a bare MD5. /// /// A server-side encrypted object can report an ETag without the suffix, so /// this helper holds only for the object store fixture these tests run against. - async fn upload_part_count(key: &str) -> usize { + async fn upload_part_count(key: &str) -> Option { let mut config = opendal::services::S3Config::default(); config.endpoint = Some(get_object_store_endpoint()); config.access_key_id = Some(OBJECT_STORE_ACCESS_KEY_ID.to_string()); @@ -92,12 +93,11 @@ mod tests { .etag() .expect("object store reports an ETag") .trim_matches('"'); - match etag.rsplit_once('-') { - Some((_, parts)) => parts + etag.rsplit_once('-').map(|(_, parts)| { + parts .parse() - .unwrap_or_else(|e| panic!("unexpected ETag format {etag}: {e}")), - None => 1, - } + .unwrap_or_else(|e| panic!("unexpected ETag format {etag}: {e}")) + }) } fn roundtrip_file_io(file_io: &FileIO) -> FileIO { @@ -181,7 +181,7 @@ mod tests { .unwrap(); writer.close().await.unwrap(); - assert_eq!(upload_part_count(&key).await, PARTS); + assert_eq!(upload_part_count(&key).await, Some(PARTS)); file_io.delete(&path).await.unwrap(); } @@ -206,7 +206,7 @@ mod tests { .await .unwrap(); - assert_eq!(upload_part_count(&key).await, PARTS); + assert_eq!(upload_part_count(&key).await, Some(PARTS)); file_io.delete(&path).await.unwrap(); } @@ -230,7 +230,7 @@ mod tests { .await .unwrap(); - assert_eq!(upload_part_count(&key).await, 1); + assert_eq!(upload_part_count(&key).await, None); file_io.delete(&path).await.unwrap(); }