diff --git a/Cargo.lock b/Cargo.lock index 667120ff16..bbfe017ad6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3163,6 +3163,7 @@ dependencies = [ "reqsign-aws-v4", "reqsign-core", "reqwest", + "rstest", "serde", "serde_json", "tempfile", diff --git a/crates/catalog/rest/public-api.txt b/crates/catalog/rest/public-api.txt index a0c6fd03ce..5f89dadfce 100644 --- a/crates/catalog/rest/public-api.txt +++ b/crates/catalog/rest/public-api.txt @@ -196,7 +196,7 @@ pub struct iceberg_catalog_rest::LoadTableResult pub iceberg_catalog_rest::LoadTableResult::config: std::collections::hash::map::HashMap pub iceberg_catalog_rest::LoadTableResult::metadata: iceberg::spec::table_metadata::TableMetadata pub iceberg_catalog_rest::LoadTableResult::metadata_location: core::option::Option -pub iceberg_catalog_rest::LoadTableResult::storage_credentials: core::option::Option> +pub iceberg_catalog_rest::LoadTableResult::storage_credentials: core::option::Option> impl core::clone::Clone for iceberg_catalog_rest::LoadTableResult pub fn iceberg_catalog_rest::LoadTableResult::clone(&self) -> iceberg_catalog_rest::LoadTableResult impl core::cmp::Eq for iceberg_catalog_rest::LoadTableResult @@ -355,21 +355,6 @@ impl core::fmt::Debug for iceberg_catalog_rest::SigV4Signer pub fn iceberg_catalog_rest::SigV4Signer::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl iceberg_catalog_rest::SigV4Signer pub fn iceberg_catalog_rest::SigV4Signer::builder() -> SigV4SignerBuilder<((), (), ())> -pub struct iceberg_catalog_rest::StorageCredential -pub iceberg_catalog_rest::StorageCredential::config: std::collections::hash::map::HashMap -pub iceberg_catalog_rest::StorageCredential::prefix: alloc::string::String -impl core::clone::Clone for iceberg_catalog_rest::StorageCredential -pub fn iceberg_catalog_rest::StorageCredential::clone(&self) -> iceberg_catalog_rest::StorageCredential -impl core::cmp::Eq for iceberg_catalog_rest::StorageCredential -impl core::cmp::PartialEq for iceberg_catalog_rest::StorageCredential -pub fn iceberg_catalog_rest::StorageCredential::eq(&self, other: &iceberg_catalog_rest::StorageCredential) -> bool -impl core::fmt::Debug for iceberg_catalog_rest::StorageCredential -pub fn iceberg_catalog_rest::StorageCredential::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result -impl core::marker::StructuralPartialEq for iceberg_catalog_rest::StorageCredential -impl serde_core::ser::Serialize for iceberg_catalog_rest::StorageCredential -pub fn iceberg_catalog_rest::StorageCredential::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer -impl<'de> serde_core::de::Deserialize<'de> for iceberg_catalog_rest::StorageCredential -pub fn iceberg_catalog_rest::StorageCredential::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> pub struct iceberg_catalog_rest::UpdateNamespacePropertiesRequest pub iceberg_catalog_rest::UpdateNamespacePropertiesRequest::removals: core::option::Option> pub iceberg_catalog_rest::UpdateNamespacePropertiesRequest::updates: std::collections::hash::map::HashMap diff --git a/crates/catalog/rest/src/catalog.rs b/crates/catalog/rest/src/catalog.rs index f8e7c4179b..7f25e59799 100644 --- a/crates/catalog/rest/src/catalog.rs +++ b/crates/catalog/rest/src/catalog.rs @@ -28,7 +28,8 @@ use std::time::Duration; use async_trait::async_trait; use iceberg::encryption::kms::{KeyManagementClient, KmsClientFactory}; use iceberg::io::{ - ADLS_SAS_TOKEN, FileIO, FileIOBuilder, GCS_TOKEN, S3_SESSION_TOKEN, StorageFactory, + ADLS_SAS_TOKEN, FileIO, FileIOBuilder, GCS_TOKEN, S3_SESSION_TOKEN, StorageConfig, + StorageCredential, StorageFactory, }; use iceberg::table::Table; use iceberg::{ @@ -987,6 +988,7 @@ impl RestSessionCatalog { &self, metadata_location: Option<&str>, extra_config: Option>, + storage_credentials: Option>, ) -> Result { let mut props = self.client().await?.config.props.clone(); if let Some(config) = extra_config { @@ -1019,7 +1021,11 @@ impl RestSessionCatalog { ) })?; - let file_io = FileIOBuilder::new(factory).with_props(props).build(); + let config = StorageConfig::builder() + .props(props) + .storage_credentials(storage_credentials.unwrap_or_default()) + .build(); + let file_io = FileIOBuilder::new(factory).with_config(config).build(); Ok(file_io) } @@ -1342,7 +1348,11 @@ impl SessionCatalog for RestSessionCatalog { .collect(); let file_io = self - .load_file_io(Some(metadata_location), Some(config)) + .load_file_io( + Some(metadata_location), + Some(config), + response.storage_credentials, + ) .await?; let mut table_builder = Table::builder() @@ -1439,7 +1449,11 @@ impl SessionCatalog for RestSessionCatalog { .collect(); let file_io = self - .load_file_io(response.metadata_location.as_deref(), Some(config)) + .load_file_io( + response.metadata_location.as_deref(), + Some(config), + response.storage_credentials, + ) .await?; let mut table_builder = Table::builder() @@ -1590,7 +1604,19 @@ impl SessionCatalog for RestSessionCatalog { "Metadata location missing in `register_table` response!", ))?; - let file_io = self.load_file_io(Some(metadata_location), None).await?; + let config = response + .config + .into_iter() + .chain(self.user_config.props.clone()) + .collect(); + + let file_io = self + .load_file_io( + Some(metadata_location), + Some(config), + response.storage_credentials, + ) + .await?; let mut table_builder = Table::builder() .identifier(table_ident.clone()) @@ -1670,7 +1696,7 @@ impl SessionCatalog for RestSessionCatalog { }; let file_io = self - .load_file_io(Some(&response.metadata_location), None) + .load_file_io(Some(&response.metadata_location), None, None) .await?; let mut table_builder = Table::builder() @@ -4761,6 +4787,50 @@ mod tests { load.assert_async().await; } + /// A `loadTable` body vending a storage credential, and that credential. + fn load_table_body_with_credential() -> (serde_json::Value, StorageCredential) { + let credential = StorageCredential::new( + "s3://warehouse/database/table", + HashMap::from([("s3.access-key-id".to_string(), "vended".to_string())]), + ) + .unwrap(); + let mut body = load_table_body("v1"); + body["storage-credentials"] = + json!([{ "prefix": credential.prefix(), "config": credential.config() }]); + (body, credential) + } + + #[tokio::test] + async fn test_load_table_passes_vended_credentials_to_file_io() { + let mut server = Server::new_async().await; + let _config = create_config_mock(&mut server).await; + let (body, credential) = load_table_body_with_credential(); + let load = server + .mock("GET", "/v1/namespaces/ns1/tables/test1") + .match_header("x-iceberg-access-delegation", "vended-credentials") + .with_status(200) + .with_body(body.to_string()) + .create_async() + .await; + let catalog = session_catalog( + RestCatalogConfig::builder() + .uri(server.url()) + .props(HashMap::from([( + "header.X-Iceberg-Access-Delegation".to_string(), + "vended-credentials".to_string(), + )])) + .build(), + ); + + let table = catalog + .load_table(&SessionContext::empty(), &test_table_ident()) + .await + .unwrap(); + + assert_eq!(table.file_io().config().storage_credentials(), [credential]); + load.assert_async().await; + } + #[test] fn test_table_cache_rejects_out_of_range_expiry() { for value in ["0", "31536000000001"] { @@ -5334,6 +5404,34 @@ mod tests { register_table_mock.assert_async().await; } + #[tokio::test] + async fn test_register_table_passes_vended_config_and_credentials_to_file_io() { + let mut server = Server::new_async().await; + let _config = create_config_mock(&mut server).await; + let (body, credential) = load_table_body_with_credential(); + let register = server + .mock("POST", "/v1/namespaces/ns1/register") + .with_status(200) + .with_body(body.to_string()) + .create_async() + .await; + let catalog = session_catalog(RestCatalogConfig::builder().uri(server.url()).build()); + + let table = catalog + .register_table( + &SessionContext::empty(), + &test_table_ident(), + "s3://warehouse/v1.metadata.json".to_string(), + ) + .await + .unwrap(); + + let config = table.file_io().config(); + assert_eq!(config.get("region").map(String::as_str), Some("us-west-2")); + assert_eq!(config.storage_credentials(), [credential]); + register.assert_async().await; + } + #[tokio::test] async fn test_register_table_404() { let mut server = Server::new_async().await; diff --git a/crates/catalog/rest/src/types.rs b/crates/catalog/rest/src/types.rs index 390521229e..d3e2e4fda3 100644 --- a/crates/catalog/rest/src/types.rs +++ b/crates/catalog/rest/src/types.rs @@ -19,6 +19,7 @@ use std::collections::HashMap; +use iceberg::io::StorageCredential; use iceberg::spec::{Schema, SortOrder, TableMetadata, UnboundPartitionSpec}; use iceberg::{ Error, ErrorKind, Namespace, NamespaceIdent, TableIdent, TableRequirement, TableUpdate, @@ -231,19 +232,6 @@ pub struct LoadTableResult { pub storage_credentials: Option>, } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -/// Storage credential for a specific location prefix. -/// -/// Indicates a storage location prefix where the credential is relevant. Clients should -/// choose the most specific prefix (by selecting the longest prefix) if several credentials -/// of the same type are available. -pub struct StorageCredential { - /// Storage location prefix where this credential is relevant - pub prefix: String, - /// Configuration map containing credential information - pub config: HashMap, -} - #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] /// Request to create a new table in a namespace. diff --git a/crates/iceberg/public-api.txt b/crates/iceberg/public-api.txt index 6dd6bd5f37..18157216cd 100644 --- a/crates/iceberg/public-api.txt +++ b/crates/iceberg/public-api.txt @@ -878,6 +878,7 @@ impl iceberg::io::FileIOBuilder pub fn iceberg::io::FileIOBuilder::build(self) -> iceberg::io::FileIO pub fn iceberg::io::FileIOBuilder::config(&self) -> &iceberg::io::StorageConfig pub fn iceberg::io::FileIOBuilder::new(factory: alloc::sync::Arc) -> Self +pub fn iceberg::io::FileIOBuilder::with_config(self, config: iceberg::io::StorageConfig) -> Self pub fn iceberg::io::FileIOBuilder::with_prop(self, key: impl alloc::string::ToString, value: impl alloc::string::ToString) -> Self pub fn iceberg::io::FileIOBuilder::with_props(self, args: impl core::iter::traits::collect::IntoIterator) -> Self impl core::clone::Clone for iceberg::io::FileIOBuilder @@ -1100,6 +1101,7 @@ pub fn iceberg::io::StorageConfig::from_props(props: std::collections::hash::map pub fn iceberg::io::StorageConfig::get(&self, key: &str) -> core::option::Option<&alloc::string::String> pub fn iceberg::io::StorageConfig::new() -> Self pub fn iceberg::io::StorageConfig::props(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::io::StorageConfig::storage_credentials(&self) -> &[iceberg::io::StorageCredential] pub fn iceberg::io::StorageConfig::with_prop(self, key: impl core::convert::Into, value: impl core::convert::Into) -> Self pub fn iceberg::io::StorageConfig::with_props(self, props: impl core::iter::traits::collect::IntoIterator, impl core::convert::Into)>) -> Self impl core::clone::Clone for iceberg::io::StorageConfig @@ -1127,10 +1129,29 @@ pub fn iceberg::io::StorageConfig::default() -> iceberg::io::StorageConfig impl core::fmt::Debug for iceberg::io::StorageConfig pub fn iceberg::io::StorageConfig::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result impl core::marker::StructuralPartialEq for iceberg::io::StorageConfig +impl iceberg::io::StorageConfig +pub fn iceberg::io::StorageConfig::builder() -> StorageConfigBuilder<((), ())> impl serde_core::ser::Serialize for iceberg::io::StorageConfig pub fn iceberg::io::StorageConfig::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::StorageConfig pub fn iceberg::io::StorageConfig::deserialize<__D>(__deserializer: __D) -> core::result::Result::Error> where __D: serde_core::de::Deserializer<'de> +pub struct iceberg::io::StorageCredential +impl iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::config(&self) -> &std::collections::hash::map::HashMap +pub fn iceberg::io::StorageCredential::new(prefix: impl core::convert::Into, config: std::collections::hash::map::HashMap) -> iceberg::Result +pub fn iceberg::io::StorageCredential::prefix(&self) -> &str +impl core::clone::Clone for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::clone(&self) -> iceberg::io::StorageCredential +impl core::cmp::Eq for iceberg::io::StorageCredential +impl core::cmp::PartialEq for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::eq(&self, other: &iceberg::io::StorageCredential) -> bool +impl core::fmt::Debug for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result +impl core::marker::StructuralPartialEq for iceberg::io::StorageCredential +impl serde_core::ser::Serialize for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::serialize<__S>(&self, __serializer: __S) -> core::result::Result<<__S as serde_core::ser::Serializer>::Ok, <__S as serde_core::ser::Serializer>::Error> where __S: serde_core::ser::Serializer +impl<'de> serde_core::de::Deserialize<'de> for iceberg::io::StorageCredential +pub fn iceberg::io::StorageCredential::deserialize>(deserializer: D) -> core::result::Result::Error> pub const iceberg::io::ADLS_ACCOUNT_KEY: &str pub const iceberg::io::ADLS_ACCOUNT_NAME: &str pub const iceberg::io::ADLS_AUTHORITY_HOST: &str diff --git a/crates/iceberg/src/io/file_io.rs b/crates/iceberg/src/io/file_io.rs index 42eec4db56..7029ed4074 100644 --- a/crates/iceberg/src/io/file_io.rs +++ b/crates/iceberg/src/io/file_io.rs @@ -275,6 +275,12 @@ impl FileIOBuilder { self } + /// Replace the storage configuration set so far. + pub fn with_config(mut self, config: StorageConfig) -> Self { + self.config = config; + self + } + /// Get the storage configuration. pub fn config(&self) -> &StorageConfig { &self.config @@ -453,7 +459,7 @@ mod tests { use tempfile::TempDir; use super::{FileIO, FileIOBuilder}; - use crate::io::{LocalFsStorageFactory, MemoryStorageFactory}; + use crate::io::{LocalFsStorageFactory, MemoryStorageFactory, StorageConfig}; fn create_local_file_io() -> FileIO { FileIO::new_with_fs() @@ -601,6 +607,17 @@ mod tests { assert_eq!(file_io.config().get("key2"), Some(&"value2".to_string())); } + #[test] + fn test_file_io_builder_with_config_replaces_config() { + let config = StorageConfig::new().with_prop("key2", "value2"); + let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) + .with_prop("key1", "value1") + .with_config(config.clone()) + .build(); + + assert_eq!(file_io.config(), &config); + } + #[tokio::test] async fn test_memory_file_io_serialization_roundtrip() { let file_io = FileIOBuilder::new(Arc::new(MemoryStorageFactory)) diff --git a/crates/iceberg/src/io/storage/config/mod.rs b/crates/iceberg/src/io/storage/config/mod.rs index 8ea4af080e..242e6f5a67 100644 --- a/crates/iceberg/src/io/storage/config/mod.rs +++ b/crates/iceberg/src/io/storage/config/mod.rs @@ -37,7 +37,7 @@ mod hf; mod oss; mod s3; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; pub use azdls::*; pub use gcs::*; @@ -45,17 +45,52 @@ pub use hdfs_native::*; pub use hf::*; pub use oss::*; pub use s3::*; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; +use typed_builder::TypedBuilder; + +use crate::{Error, ErrorKind, Result}; /// Configuration properties for storage backends. /// /// This struct contains only configuration properties without specifying /// which storage backend to use. The storage type is determined by the /// explicit factory selection. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, Default, TypedBuilder)] pub struct StorageConfig { /// Configuration properties for the storage backend + #[builder(default)] props: HashMap, + /// Credentials that apply to locations under their prefixes + #[builder( + default, + setter(transform = |storage_credentials: Vec| { + last_credential_per_prefix(storage_credentials) + }) + )] + #[serde( + default, + skip_serializing_if = "Vec::is_empty", + deserialize_with = "deserialize_storage_credentials" + )] + storage_credentials: Vec, +} + +/// Keeps the last credential for each prefix, as Java's `S3FileIO.clientByPrefix` does. +fn last_credential_per_prefix(credentials: Vec) -> Vec { + let mut prefixes = HashSet::new(); + let mut credentials: Vec<_> = credentials + .into_iter() + .rev() + .filter(|credential| prefixes.insert(credential.prefix.clone())) + .collect(); + credentials.reverse(); + credentials +} + +fn deserialize_storage_credentials<'de, D: Deserializer<'de>>( + deserializer: D, +) -> std::result::Result, D::Error> { + Vec::deserialize(deserializer).map(last_credential_per_prefix) } impl StorageConfig { @@ -63,6 +98,7 @@ impl StorageConfig { pub fn new() -> Self { Self { props: HashMap::new(), + storage_credentials: Vec::new(), } } @@ -72,7 +108,10 @@ impl StorageConfig { /// /// * `props` - Configuration properties for the storage backend pub fn from_props(props: HashMap) -> Self { - Self { props } + Self { + props, + storage_credentials: Vec::new(), + } } /// Get all configuration properties. @@ -121,6 +160,76 @@ impl StorageConfig { .extend(props.into_iter().map(|(k, v)| (k.into(), v.into()))); self } + + /// Get the credentials that apply to locations under their prefixes. + /// + /// There is one per prefix, the last one given for it. A factory that + /// supports them overlays the props with the config of a location's + /// longest matching credential. + pub fn storage_credentials(&self) -> &[StorageCredential] { + &self.storage_credentials + } +} + +/// Storage properties, such as a vended credential, that apply to locations +/// under a prefix, like Java's `StorageCredential`. +#[derive(Clone, PartialEq, Eq, Serialize)] +pub struct StorageCredential { + /// Location prefix, such as `s3://bucket/table` + prefix: String, + /// Storage properties holding the credential + config: HashMap, +} + +impl StorageCredential { + /// Create a credential for locations under `prefix`. + /// + /// # Errors + /// + /// Returns an error if `prefix` is empty. + pub fn new(prefix: impl Into, config: HashMap) -> Result { + let prefix = prefix.into(); + if prefix.is_empty() { + return Err(Error::new( + ErrorKind::DataInvalid, + "Invalid storage credential: prefix must be non-empty", + )); + } + Ok(Self { prefix, config }) + } + + /// Return the location prefix this credential applies to. + pub fn prefix(&self) -> &str { + &self.prefix + } + + /// Return the storage properties holding the credential. + pub fn config(&self) -> &HashMap { + &self.config + } +} + +impl<'de> Deserialize<'de> for StorageCredential { + fn deserialize>(deserializer: D) -> std::result::Result { + #[derive(Deserialize)] + struct Fields { + prefix: String, + config: HashMap, + } + + let Fields { prefix, config } = Fields::deserialize(deserializer)?; + Self::new(prefix, config).map_err(serde::de::Error::custom) + } +} + +// The config values are secrets. +impl std::fmt::Debug for StorageCredential { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StorageCredential") + .field("prefix", &self.prefix) + .field("config_keys", &self.config.keys()) + .finish() + } } #[cfg(test)] @@ -200,6 +309,87 @@ mod tests { assert_eq!(config, deserialized); } + #[test] + fn test_storage_config_serialization_keeps_storage_credentials() { + let config = StorageConfig::builder() + .props(HashMap::from([( + "region".to_string(), + "us-east-1".to_string(), + )])) + .storage_credentials(vec![ + StorageCredential::new( + "s3://bucket/table", + HashMap::from([("s3.access-key-id".to_string(), "vended".to_string())]), + ) + .unwrap(), + ]) + .build(); + + let serialized = serde_json::to_string(&config).unwrap(); + let deserialized: StorageConfig = serde_json::from_str(&serialized).unwrap(); + + assert_eq!(config, deserialized); + } + + #[test] + fn test_storage_config_keeps_the_last_credential_for_each_prefix() { + let credential = |prefix: &str, access_key_id: &str| { + StorageCredential::new( + prefix, + HashMap::from([("s3.access-key-id".to_string(), access_key_id.to_string())]), + ) + .unwrap() + }; + + let credentials = vec![ + credential("s3://bucket/a", "1"), + credential("s3://bucket/b", "2"), + credential("s3://bucket/a", "3"), + ]; + let expected = [ + credential("s3://bucket/b", "2"), + credential("s3://bucket/a", "3"), + ]; + + let built = StorageConfig::builder() + .storage_credentials(credentials.clone()) + .build(); + let deserialized: StorageConfig = serde_json::from_value(serde_json::json!({ + "props": {}, + "storage_credentials": credentials, + })) + .unwrap(); + + assert_eq!(built.storage_credentials(), expected); + assert_eq!(deserialized.storage_credentials(), expected); + } + + #[test] + fn test_storage_credential_debug_omits_config_values() { + let credential = StorageCredential::new( + "s3://bucket/table", + HashMap::from([("s3.secret-access-key".to_string(), "secret".to_string())]), + ) + .unwrap(); + + assert_eq!( + format!("{credential:?}"), + r#"StorageCredential { prefix: "s3://bucket/table", config_keys: ["s3.secret-access-key"] }"# + ); + } + + #[test] + fn test_storage_credential_rejects_an_empty_prefix() { + let config = HashMap::from([("s3.access-key-id".to_string(), "vended".to_string())]); + + let err = StorageCredential::new("", config).unwrap_err(); + let deserialized = + serde_json::from_str::(r#"{"prefix":"","config":{}}"#); + + assert_eq!(err.kind(), ErrorKind::DataInvalid); + assert!(deserialized.is_err()); + } + #[test] fn test_storage_config_clone_independence() { let original = StorageConfig::new().with_prop("region", "us-east-1"); diff --git a/crates/storage/opendal/Cargo.toml b/crates/storage/opendal/Cargo.toml index 83541e89c5..3203d7a03b 100644 --- a/crates/storage/opendal/Cargo.toml +++ b/crates/storage/opendal/Cargo.toml @@ -73,6 +73,7 @@ url = { workspace = true } async-trait = { workspace = true } iceberg_test_utils = { path = "../../test_utils", features = ["tests"] } reqwest = { workspace = true } +rstest = { workspace = true } serde_json = { workspace = true } tempfile = { workspace = true } tokio = { workspace = true, features = ["macros", "test-util"] } diff --git a/crates/storage/opendal/src/lib.rs b/crates/storage/opendal/src/lib.rs index 55b651ae1f..f24ae681e5 100644 --- a/crates/storage/opendal/src/lib.rs +++ b/crates/storage/opendal/src/lib.rs @@ -44,6 +44,7 @@ use iceberg_property_macro::Properties; use opendal::Operator; use opendal::layers::{RetryLayer, TimeoutLayer}; use opendal::options::WriteOptions; +use prefixed::PrefixedStorage; use serde::{Deserialize, Serialize}; use utils::from_opendal_error; @@ -109,6 +110,7 @@ cfg_if! { } } +mod prefixed; mod resolving; pub use resolving::{OpenDalResolvingStorage, OpenDalResolvingStorageFactory}; @@ -218,6 +220,8 @@ fn parse_positive_ms(value: &str) -> Result { /// Maps scheme to the corresponding OpenDalStorage storage variant. /// Use this factory with `FileIOBuilder::new(factory)` to create FileIO instances. /// +/// Supports [storage credentials](iceberg::io::StorageConfig::storage_credentials). +/// /// # Serialization /// /// The receiving binary must enable the feature corresponding to the serialized backend variant. @@ -277,53 +281,59 @@ where #[typetag::serde(name = "OpenDalStorageFactory")] impl StorageFactory for OpenDalStorageFactory { - #[allow(unused_variables)] fn build(&self, config: &StorageConfig) -> Result> { + PrefixedStorage::build(config, |config| self.build_storage(config)) + } +} + +impl OpenDalStorageFactory { + #[allow(unused_variables)] + fn build_storage(&self, config: &StorageConfig) -> Result { let client_config = OpenDalClientConfig::from_properties(config.props())?; match self { #[cfg(feature = "opendal-memory")] - OpenDalStorageFactory::Memory => Ok(Arc::new(OpenDalStorage::Memory { + OpenDalStorageFactory::Memory => Ok(OpenDalStorage::Memory { operator: memory_config_build()?, client_config, - })), + }), #[cfg(feature = "opendal-fs")] - OpenDalStorageFactory::Fs => Ok(Arc::new(OpenDalStorage::LocalFs { client_config })), + OpenDalStorageFactory::Fs => Ok(OpenDalStorage::LocalFs { client_config }), #[cfg(feature = "opendal-s3")] OpenDalStorageFactory::S3 { customized_credential_load, - } => Ok(Arc::new(OpenDalStorage::S3 { + } => Ok(OpenDalStorage::S3 { config: s3_config_parse(config.props().clone())?.into(), multipart_part_size: s3_multipart_part_size_parse(config.props())?, customized_credential_load: customized_credential_load.clone(), client_config: s3_client_config(config.props(), client_config)?, - })), + }), #[cfg(feature = "opendal-gcs")] - OpenDalStorageFactory::Gcs => Ok(Arc::new(OpenDalStorage::Gcs { + OpenDalStorageFactory::Gcs => Ok(OpenDalStorage::Gcs { config: gcs_config_parse(config.props().clone())?.into(), client_config, - })), + }), #[cfg(feature = "opendal-hdfs-native")] - OpenDalStorageFactory::HdfsNative => Ok(Arc::new(OpenDalStorage::HdfsNative( - HdfsNativeStorage::new( + OpenDalStorageFactory::HdfsNative => { + Ok(OpenDalStorage::HdfsNative(HdfsNativeStorage::new( hdfs_native_config_parse(config.props().clone())?, client_config, - ), - ))), + ))) + } #[cfg(feature = "opendal-oss")] - OpenDalStorageFactory::Oss => Ok(Arc::new(OpenDalStorage::Oss { + OpenDalStorageFactory::Oss => Ok(OpenDalStorage::Oss { config: oss_config_parse(config.props().clone())?.into(), client_config, - })), + }), #[cfg(feature = "opendal-azdls")] - OpenDalStorageFactory::Azdls => Ok(Arc::new(OpenDalStorage::Azdls { + OpenDalStorageFactory::Azdls => Ok(OpenDalStorage::Azdls { config: azdls_config_parse(config.props().clone())?.into(), client_config, - })), + }), #[cfg(feature = "opendal-hf")] - OpenDalStorageFactory::Hf => Ok(Arc::new(OpenDalStorage::Hf { + OpenDalStorageFactory::Hf => Ok(OpenDalStorage::Hf { config: hf_config_parse(config.props().clone())?.into(), client_config, - })), + }), #[cfg(all( not(feature = "opendal-memory"), not(feature = "opendal-fs"), diff --git a/crates/storage/opendal/src/prefixed.rs b/crates/storage/opendal/src/prefixed.rs new file mode 100644 index 0000000000..20d90c107f --- /dev/null +++ b/crates/storage/opendal/src/prefixed.rs @@ -0,0 +1,287 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//! Storage that serves each location with its prefix's credential. + +use std::collections::HashMap; +use std::fmt::{self, Debug}; +use std::sync::Arc; + +use async_trait::async_trait; +use bytes::Bytes; +use futures::StreamExt; +use futures::stream::{self, BoxStream}; +use iceberg::Result; +use iceberg::io::{ + FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, +}; +use serde::{Deserialize, Serialize}; + +use crate::OpenDalStorage; + +/// Batch size of `delete_stream`, as in Java's `S3FileIO.deleteFiles`. +pub(crate) const DELETE_BATCH_SIZE: usize = 1000; + +/// Serves each path with the credential of its longest matching prefix, as +/// Java's `S3FileIO.clientForStoragePath` does, else without credentials. +#[derive(Serialize, Deserialize)] +pub(crate) struct PrefixedStorage { + default: OpenDalStorage, + /// Longest prefix first. + prefixed: Vec<(String, OpenDalStorage)>, +} + +impl PrefixedStorage { + /// Builds the storage for `config` with `build_fn`, which is passed configs + /// without storage credentials. + pub(crate) fn build( + config: &StorageConfig, + build_fn: impl Fn(&StorageConfig) -> Result, + ) -> Result> { + if config.storage_credentials().is_empty() { + return Ok(Arc::new(build_fn(config)?)); + } + Ok(Arc::new(Self::new(config, build_fn)?)) + } + + /// Builds one storage per credential of `config` and one for its props + /// alone, with `build_fn` as in [`Self::build`]. + pub(crate) fn new( + config: &StorageConfig, + build_fn: impl Fn(&StorageConfig) -> Result, + ) -> Result { + let base = StorageConfig::from_props(config.props().clone()); + let mut prefixed = config + .storage_credentials() + .iter() + .map(|credential| { + let config = base.clone().with_props(credential.config().clone()); + Ok((credential.prefix().to_string(), build_fn(&config)?)) + }) + .collect::>>()?; + prefixed.sort_by_key(|(prefix, _)| std::cmp::Reverse(prefix.len())); + Ok(Self { + default: build_fn(&base)?, + prefixed, + }) + } + + /// Matches raw strings, as Java does: `s3://b/data` covers `s3://b/database`. + fn prefix_index(&self, path: &str) -> Option { + self.prefixed + .iter() + .position(|(prefix, _)| path.starts_with(prefix.as_str())) + } + + fn storage_at(&self, index: Option) -> &OpenDalStorage { + match index { + Some(index) => &self.prefixed[index].1, + None => &self.default, + } + } + + pub(crate) fn storage_for(&self, path: &str) -> &OpenDalStorage { + self.storage_at(self.prefix_index(path)) + } +} + +// A backend's Debug may print the credentials it was built with. +impl Debug for PrefixedStorage { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let prefixes = fmt::from_fn(|f| { + f.debug_list() + .entries(self.prefixed.iter().map(|(prefix, _)| prefix)) + .finish() + }); + f.debug_struct("PrefixedStorage") + .field("default", &self.default) + .field("prefixes", &prefixes) + .finish_non_exhaustive() + } +} + +#[async_trait] +#[typetag::serde] +impl Storage for PrefixedStorage { + async fn exists(&self, path: &str) -> Result { + self.storage_for(path).exists(path).await + } + + async fn metadata(&self, path: &str) -> Result { + self.storage_for(path).metadata(path).await + } + + async fn read(&self, path: &str) -> Result { + self.storage_for(path).read(path).await + } + + async fn reader(&self, path: &str) -> Result> { + self.storage_for(path).reader(path).await + } + + async fn write(&self, path: &str, bs: Bytes) -> Result<()> { + self.storage_for(path).write(path, bs).await + } + + async fn writer(&self, path: &str) -> Result> { + self.storage_for(path).writer(path).await + } + + async fn delete(&self, path: &str) -> Result<()> { + self.storage_for(path).delete(path).await + } + + async fn delete_prefix(&self, path: &str) -> Result<()> { + // Route as a directory, so an `s3://b/t/` prefix covers `s3://b/t`. + let dir = format!("{}/", path.strip_suffix('/').unwrap_or(path)); + self.storage_for(&dir).delete_prefix(path).await + } + + async fn delete_stream(&self, mut paths: BoxStream<'static, String>) -> Result<()> { + let mut batches: HashMap, Vec> = HashMap::new(); + while let Some(path) = paths.next().await { + let index = self.prefix_index(&path); + let batch = batches.entry(index).or_default(); + batch.push(path); + if batch.len() >= DELETE_BATCH_SIZE { + let batch = std::mem::take(batch); + self.storage_at(index) + .delete_stream(stream::iter(batch).boxed()) + .await?; + } + } + for (index, batch) in batches { + if !batch.is_empty() { + self.storage_at(index) + .delete_stream(stream::iter(batch).boxed()) + .await?; + } + } + Ok(()) + } + + fn new_input(&self, path: &str) -> Result { + self.storage_for(path).new_input(path) + } + + fn new_output(&self, path: &str) -> Result { + self.storage_for(path).new_output(path) + } +} + +#[cfg(all(test, feature = "opendal-memory"))] +mod tests { + use std::time::Duration; + + use iceberg::io::StorageCredential; + use rstest::rstest; + + use super::*; + use crate::{OPENDAL_IO_TIMEOUT_MS, OpenDalStorageFactory}; + + fn timeout_ms(ms: u64) -> HashMap { + HashMap::from([(OPENDAL_IO_TIMEOUT_MS.to_string(), ms.to_string())]) + } + + /// Memory storages with an IO timeout of 1s by default and of `ms` under each `prefix`. + fn memory_storage(credentials: &[(&str, u64)]) -> PrefixedStorage { + let config = StorageConfig::builder() + .props(timeout_ms(1000)) + .storage_credentials( + credentials + .iter() + .map(|(prefix, ms)| StorageCredential::new(*prefix, timeout_ms(*ms)).unwrap()) + .collect(), + ) + .build(); + PrefixedStorage::new(&config, |config| { + OpenDalStorageFactory::Memory.build_storage(config) + }) + .unwrap() + } + + fn timeout_for(storage: &PrefixedStorage, path: &str) -> Duration { + storage.storage_for(path).client_config().io_timeout() + } + + #[rstest] + #[case::longest_prefix("memory:/data/warehouse/f", 3000)] + #[case::shorter_prefix("memory:/data/other/f", 2000)] + #[case::prefix_of_a_longer_name("memory:/database/f", 2000)] + #[case::no_prefix("memory:/elsewhere/f", 1000)] + fn test_routes_a_path_to_its_longest_matching_prefix( + #[case] path: &str, + #[case] expected_ms: u64, + ) { + let storage = memory_storage(&[("memory:/data", 2000), ("memory:/data/warehouse", 3000)]); + + assert_eq!( + timeout_for(&storage, path), + Duration::from_millis(expected_ms) + ); + } + + #[rstest] + #[case::without_slash("memory:/table")] + #[case::with_slash("memory:/table/")] + #[tokio::test] + async fn test_delete_prefix_routes_a_directory_with_or_without_its_slash(#[case] path: &str) { + let storage = memory_storage(&[("memory:/table/", 2000)]); + storage + .write("memory:/table/f", Bytes::from("data")) + .await + .unwrap(); + + storage.delete_prefix(path).await.unwrap(); + + assert!(!storage.exists("memory:/table/f").await.unwrap()); + } + + #[tokio::test] + async fn test_delete_stream_deletes_each_path_with_its_prefix_storage() { + let storage = memory_storage(&[("memory:/table/", 2000)]); + let paths: Vec<_> = (0..DELETE_BATCH_SIZE + 1) + .map(|i| format!("memory:/table/{i}")) + .chain(["memory:/other/f".to_string()]) + .collect(); + for path in &paths { + storage.write(path, Bytes::from("data")).await.unwrap(); + } + + storage + .delete_stream(stream::iter(paths.clone()).boxed()) + .await + .unwrap(); + + for path in &paths { + assert!(!storage.exists(path).await.unwrap(), "{path}"); + } + } + + #[test] + fn test_debug_omits_prefixed_storages() { + let storage = memory_storage(&[("memory:/table/", 2000)]); + + assert_eq!( + format!("{storage:?}"), + format!( + r#"PrefixedStorage {{ default: {:?}, prefixes: ["memory:/table/"], .. }}"#, + storage.default + ) + ); + } +} diff --git a/crates/storage/opendal/src/resolving.rs b/crates/storage/opendal/src/resolving.rs index 47459c42c8..6ec59078fe 100644 --- a/crates/storage/opendal/src/resolving.rs +++ b/crates/storage/opendal/src/resolving.rs @@ -27,12 +27,13 @@ use futures::StreamExt; use futures::stream::BoxStream; use iceberg::io::{ FileMetadata, FileRead, FileWrite, InputFile, OutputFile, Storage, StorageConfig, - StorageFactory, + StorageCredential, StorageFactory, }; use iceberg::{Error, ErrorKind, Result}; use serde::{Deserialize, Serialize}; use url::Url; +use crate::prefixed::{DELETE_BATCH_SIZE, PrefixedStorage}; #[cfg(feature = "opendal-s3")] use crate::s3::CustomAwsCredentialLoader; use crate::{OpenDalClientConfig, OpenDalStorage}; @@ -160,6 +161,8 @@ fn build_storage_for_scheme( /// delegates operations to the appropriate [`OpenDalStorage`] variant based on /// the path scheme. /// +/// Supports [storage credentials](iceberg::io::StorageConfig::storage_credentials). +/// /// # Serialization /// /// Serialization fails when a custom S3 credential loader is configured because the loader holds @@ -218,6 +221,7 @@ impl StorageFactory for OpenDalResolvingStorageFactory { fn build(&self, config: &StorageConfig) -> Result> { Ok(Arc::new(OpenDalResolvingStorage { props: config.props().clone(), + storage_credentials: config.storage_credentials().to_vec(), storages: RwLock::new(HashMap::new()), #[cfg(feature = "opendal-s3")] customized_credential_load: self.customized_credential_load.clone(), @@ -235,9 +239,12 @@ impl StorageFactory for OpenDalResolvingStorageFactory { pub struct OpenDalResolvingStorage { /// Configuration properties shared across all backends. props: HashMap, + /// Credentials that apply to locations under their prefixes. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + storage_credentials: Vec, /// Cache of canonical scheme to storage mappings. #[serde(skip, default)] - storages: RwLock>>, + storages: RwLock>>, /// Custom AWS credential loader for S3 storage. #[cfg(feature = "opendal-s3")] #[serde(skip)] @@ -246,8 +253,8 @@ pub struct OpenDalResolvingStorage { impl OpenDalResolvingStorage { /// Resolve the storage for the given path by extracting the canonical scheme and - /// returning the cached or newly-created [`OpenDalStorage`]. - fn resolve(&self, path: &str) -> Result> { + /// returning the cached or newly-created storage for it. + fn resolve(&self, path: &str) -> Result> { let scheme = extract_scheme(path)?; // Fast path: check read lock first. @@ -272,12 +279,30 @@ impl OpenDalResolvingStorage { return Ok(storage.clone()); } - let storage = build_storage_for_scheme( - scheme, - &self.props, - #[cfg(feature = "opendal-s3")] - &self.customized_credential_load, - )?; + // A prefix such as `s3://bucket/` or `s3` names its scheme, and never + // matches the paths of another. + let storage_credentials = self + .storage_credentials + .iter() + .filter(|credential| { + let prefix = credential.prefix(); + let name = prefix.split_once(':').map_or(prefix, |(name, _)| name); + parse_scheme(&name.to_ascii_lowercase()).map_or(true, |s| s == scheme) + }) + .cloned() + .collect(); + let config = StorageConfig::builder() + .props(self.props.clone()) + .storage_credentials(storage_credentials) + .build(); + let storage = PrefixedStorage::new(&config, |config| { + build_storage_for_scheme( + scheme, + config.props(), + #[cfg(feature = "opendal-s3")] + &self.customized_credential_load, + ) + })?; let storage = Arc::new(storage); cache.insert(scheme, storage.clone()); Ok(storage) @@ -325,10 +350,21 @@ impl Storage for OpenDalResolvingStorage { let mut grouped: HashMap<&'static str, Vec> = HashMap::new(); while let Some(path) = paths.next().await { let scheme = extract_scheme(&path)?; - grouped.entry(scheme).or_default().push(path); + let batch = grouped.entry(scheme).or_default(); + batch.push(path); + if batch.len() >= DELETE_BATCH_SIZE { + let batch = std::mem::take(batch); + let storage = self.resolve(&batch[0])?; + storage + .delete_stream(futures::stream::iter(batch).boxed()) + .await?; + } } for (_, paths) in grouped { + if paths.is_empty() { + continue; + } let storage = self.resolve(&paths[0])?; storage .delete_stream(futures::stream::iter(paths).boxed()) @@ -338,17 +374,11 @@ impl Storage for OpenDalResolvingStorage { } fn new_input(&self, path: &str) -> Result { - Ok(InputFile::new( - Arc::new(self.resolve(path)?.as_ref().clone()), - path.to_string(), - )) + self.resolve(path)?.new_input(path) } fn new_output(&self, path: &str) -> Result { - Ok(OutputFile::new( - Arc::new(self.resolve(path)?.as_ref().clone()), - path.to_string(), - )) + self.resolve(path)?.new_output(path) } } @@ -394,6 +424,7 @@ mod tests { fn empty_resolving_storage() -> OpenDalResolvingStorage { OpenDalResolvingStorage { props: HashMap::new(), + storage_credentials: Vec::new(), storages: RwLock::new(HashMap::new()), #[cfg(feature = "opendal-s3")] customized_credential_load: None, @@ -426,20 +457,106 @@ mod tests { for path in paths { let resolved = storage.resolve(path).unwrap(); assert_eq!( - resolved.client_config().io_timeout(), + resolved.storage_for(path).client_config().io_timeout(), Duration::from_secs(45), "{path}" ); } } + #[cfg(feature = "opendal-memory")] + #[test] + fn test_resolve_ignores_credentials_for_other_schemes() { + let mut storage = empty_resolving_storage(); + storage.storage_credentials = ["s3://bucket/", "s3"] + .map(|prefix| { + StorageCredential::new( + prefix, + HashMap::from([(OPENDAL_IO_TIMEOUT_MS.to_string(), "invalid".to_string())]), + ) + .unwrap() + }) + .to_vec(); + + assert!(storage.resolve("memory:/key").is_ok()); + } + + #[cfg(feature = "opendal-s3")] + #[test] + fn test_resolve_applies_a_scheme_name_prefix_to_its_aliases() { + let mut storage = empty_resolving_storage(); + storage.storage_credentials = vec![ + StorageCredential::new( + "s3", + HashMap::from([(OPENDAL_IO_TIMEOUT_MS.to_string(), "2000".to_string())]), + ) + .unwrap(), + ]; + + let timeouts = ["s3://bucket/key", "s3a://bucket/key"].map(|path| { + storage + .resolve(path) + .unwrap() + .storage_for(path) + .client_config() + .io_timeout() + }); + + assert_eq!(timeouts, [Duration::from_secs(2); 2]); + } + + /// A resolving storage holding one delete batch of memory files, and their paths. + #[cfg(feature = "opendal-memory")] + async fn storage_with_a_full_batch() -> (OpenDalResolvingStorage, Vec) { + let storage = empty_resolving_storage(); + let paths: Vec<_> = (0..DELETE_BATCH_SIZE) + .map(|i| format!("memory:/table/{i}")) + .collect(); + for path in &paths { + storage.write(path, Bytes::from("data")).await.unwrap(); + } + (storage, paths) + } + + #[cfg(feature = "opendal-memory")] + #[tokio::test(start_paused = true)] + async fn test_delete_stream_deletes_a_full_batch_before_the_stream_ends() { + let (storage, paths) = storage_with_a_full_batch().await; + let stream = futures::stream::iter(paths.clone()) + .chain(futures::stream::pending()) + .boxed(); + + tokio::time::timeout(Duration::from_secs(1), storage.delete_stream(stream)) + .await + .unwrap_err(); + + for path in &paths { + assert!(!storage.exists(path).await.unwrap(), "{path}"); + } + } + + #[cfg(feature = "opendal-memory")] + #[tokio::test] + async fn test_delete_stream_deletes_a_stream_of_exactly_one_batch() { + let (storage, paths) = storage_with_a_full_batch().await; + + storage + .delete_stream(futures::stream::iter(paths.clone()).boxed()) + .await + .unwrap(); + + for path in &paths { + assert!(!storage.exists(path).await.unwrap(), "{path}"); + } + } + #[cfg(feature = "opendal-s3")] #[test] fn test_resolve_s3_aliases_share_instance() { let storage = empty_resolving_storage(); // All three S3-family schemes must collapse to a single cached - // `Arc` so that catalogs handing the resolver a mix + // storage so that catalogs handing the resolver a mix // of `s3://`, `s3a://`, `s3n://` paths don't rebuild operators. let a = storage.resolve("s3://bucket/key").unwrap(); let b = storage.resolve("s3a://bucket/key").unwrap(); @@ -454,10 +571,11 @@ mod tests { fn test_resolve_hdfs_native_returns_hdfs_variant() { let storage = empty_resolving_storage(); - let resolved = storage.resolve("hdfs://nameservice1/a/b").unwrap(); + let path = "hdfs://nameservice1/a/b"; + let resolved = storage.resolve(path).unwrap(); assert!( - matches!(&*resolved, OpenDalStorage::HdfsNative(_)), + matches!(resolved.storage_for(path), OpenDalStorage::HdfsNative(_)), "expected HdfsNative variant, got {resolved:?}" ); } diff --git a/crates/storage/opendal/tests/file_io_s3_test.rs b/crates/storage/opendal/tests/file_io_s3_test.rs index 14e54dcf59..f178ece6d3 100644 --- a/crates/storage/opendal/tests/file_io_s3_test.rs +++ b/crates/storage/opendal/tests/file_io_s3_test.rs @@ -21,13 +21,14 @@ //! Each test uses unique file paths based on module path to avoid conflicts. #[cfg(feature = "opendal-s3")] mod tests { + use std::collections::HashMap; use std::sync::Arc; use bytes::Bytes; use futures::StreamExt; use iceberg::io::{ FileIO, FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_MULTIPART_PART_SIZE_BYTES, - S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, + S3_PATH_STYLE_ACCESS, S3_REGION, S3_SECRET_ACCESS_KEY, StorageConfig, StorageCredential, }; use iceberg_storage_opendal::{ AwsCredential, CustomAwsCredentialLoader, OpenDalStorageFactory, ProvideCredential, @@ -415,4 +416,36 @@ mod tests { // Should succeed with no-op file_io.delete_stream(stream).await.unwrap(); } + + #[tokio::test] + async fn test_file_io_s3_storage_credential_authorizes_its_prefix() { + set_up(); + let config = StorageConfig::builder() + .props(HashMap::from([ + (S3_ENDPOINT.to_string(), get_object_store_endpoint()), + (S3_ACCESS_KEY_ID.to_string(), "invalid".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "invalid".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS.to_string(), "true".to_string()), + ])) + .storage_credentials(vec![ + StorageCredential::new( + "s3://bucket1/vended/", + HashMap::from([ + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + ]), + ) + .unwrap(), + ]) + .build(); + let file_io = FileIOBuilder::new(Arc::new(OpenDalStorageFactory::S3 { + customized_credential_load: None, + })) + .with_config(config) + .build(); + + assert!(!file_io.exists("s3://bucket1/vended/f").await.unwrap()); + assert!(file_io.exists("s3://bucket1/other/f").await.is_err()); + } } diff --git a/crates/storage/opendal/tests/resolving_storage_test.rs b/crates/storage/opendal/tests/resolving_storage_test.rs index bcc3d51842..c11a948c6b 100644 --- a/crates/storage/opendal/tests/resolving_storage_test.rs +++ b/crates/storage/opendal/tests/resolving_storage_test.rs @@ -26,11 +26,12 @@ feature = "opendal-memory" ))] mod tests { + use std::collections::HashMap; use std::sync::Arc; use iceberg::io::{ FileIOBuilder, S3_ACCESS_KEY_ID, S3_ENDPOINT, S3_PATH_STYLE_ACCESS, S3_REGION, - S3_SECRET_ACCESS_KEY, + S3_SECRET_ACCESS_KEY, StorageConfig, StorageCredential, }; use iceberg_storage_opendal::OpenDalResolvingStorageFactory; use iceberg_test_utils::{get_object_store_endpoint, normalize_test_name_with_parts, set_up}; @@ -239,6 +240,37 @@ mod tests { ); } + #[tokio::test] + async fn test_storage_credential_authorizes_its_prefix_after_serialization() { + set_up(); + let config = StorageConfig::builder() + .props(HashMap::from([ + (S3_ENDPOINT.to_string(), get_object_store_endpoint()), + (S3_ACCESS_KEY_ID.to_string(), "invalid".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "invalid".to_string()), + (S3_REGION.to_string(), "us-east-1".to_string()), + (S3_PATH_STYLE_ACCESS.to_string(), "true".to_string()), + ])) + .storage_credentials(vec![ + StorageCredential::new( + "s3://bucket1/vended/", + HashMap::from([ + (S3_ACCESS_KEY_ID.to_string(), "admin".to_string()), + (S3_SECRET_ACCESS_KEY.to_string(), "password".to_string()), + ]), + ) + .unwrap(), + ]) + .build(); + let file_io = FileIOBuilder::new(Arc::new(OpenDalResolvingStorageFactory::new())) + .with_config(config) + .build(); + let file_io = roundtrip_file_io(&file_io); + + assert!(!file_io.exists("s3://bucket1/vended/f").await.unwrap()); + assert!(file_io.exists("s3://bucket1/other/f").await.is_err()); + } + #[tokio::test] async fn test_invalid_scheme() { let file_io = get_resolving_file_io();